Merge branch 'develop' into 'fix/extended-nickname-format-default-config'

# Conflicts:
#   CHANGELOG.md

14 files changed, 129 insertions, 262 deletions

diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex
index 462940e7e..11e4fde43 100644
--- a/lib/mix/tasks/pleroma/config.ex
+++ b/lib/mix/tasks/pleroma/config.ex
@@ -8,18 +8,7 @@ defmodule Mix.Tasks.Pleroma.Config do
   alias Pleroma.Repo
   alias Pleroma.Web.AdminAPI.Config
   @shortdoc "Manages the location of the config"
-  @moduledoc """
-  Manages the location of the config.
-
-  ## Transfers config from file to DB.
-
-      mix pleroma.config migrate_to_db
-
-  ## Transfers config from DB to file `config/env.exported_from_db.secret.exs`
-
-      mix pleroma.config migrate_from_db ENV
-  """
-
+  @moduledoc File.read!("docs/administration/CLI_tasks/config.md")
   def run(["migrate_to_db"]) do
     start_pleroma()
 
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index 890a383df..cfd9eeada 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -13,34 +13,8 @@ defmodule Mix.Tasks.Pleroma.Database do
   use Mix.Task
 
   @shortdoc "A collection of database related tasks"
-  @moduledoc """
-   A collection of database related tasks
+  @moduledoc File.read!("docs/administration/CLI_tasks/database.md")
 
-   ## Replace embedded objects with their references
-
-   Replaces embedded objects with references to them in the `objects` table. Only needs to be ran once. The reason why this is not a migration is because it could significantly increase the database size after being ran, however after this `VACUUM FULL` will be able to reclaim about 20% (really depends on what is in the database, your mileage may vary) of the db size before the migration.
-
-       mix pleroma.database remove_embedded_objects
-
-    Options:
-    - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
-
-  ## Prune old objects from the database
-
-      mix pleroma.database prune_objects
-
-  ## Create a conversation for all existing DMs. Can be safely re-run.
-
-      mix pleroma.database bump_all_conversations
-
-  ## Remove duplicated items from following and update followers count for all users
-
-      mix pleroma.database update_users_following_followers_counts
-
-  ## Fix the pre-existing "likes" collections for all objects
-
-      mix pleroma.database fix_likes_collections
-  """
   def run(["remove_embedded_objects" | args]) do
     {options, [], []} =
       OptionParser.parse(
diff --git a/lib/mix/tasks/pleroma/digest.ex b/lib/mix/tasks/pleroma/digest.ex
index 430116a50..7d09e70c5 100644
--- a/lib/mix/tasks/pleroma/digest.ex
+++ b/lib/mix/tasks/pleroma/digest.ex
@@ -2,16 +2,8 @@ defmodule Mix.Tasks.Pleroma.Digest do
   use Mix.Task
 
   @shortdoc "Manages digest emails"
-  @moduledoc """
-  Manages digest emails
+  @moduledoc File.read!("docs/administration/CLI_tasks/digest.md")
 
-  ## Send digest email since given date (user registration date by default)
-  ignoring user activity status.
-
-  ``mix pleroma.digest test <nickname> <since_date>``
-
-  Example: ``mix pleroma.digest test donaldtheduck 2019-05-20``
-  """
   def run(["test", nickname | opts]) do
     Mix.Pleroma.start_pleroma()
 
diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex
index 881a6f725..6ef0a635d 100644
--- a/lib/mix/tasks/pleroma/emoji.ex
+++ b/lib/mix/tasks/pleroma/emoji.ex
@@ -6,54 +6,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
   use Mix.Task
 
   @shortdoc "Manages emoji packs"
-  @moduledoc """
-  Manages emoji packs
-
-  ## ls-packs
-
-      mix pleroma.emoji ls-packs [OPTION...]
-
-  Lists the emoji packs and metadata specified in the manifest.
-
-  ### Options
-
-  - `-m, --manifest PATH/URL` - path to a custom manifest, it can
-    either be an URL starting with `http`, in that case the
-    manifest will be fetched from that address, or a local path
-
-  ## get-packs
-
-      mix pleroma.emoji get-packs [OPTION...] PACKS
-
-  Fetches, verifies and installs the specified PACKS from the
-  manifest into the `STATIC-DIR/emoji/PACK-NAME`
-
-  ### Options
-
-  - `-m, --manifest PATH/URL` - same as ls-packs
-
-  ## gen-pack
-
-      mix pleroma.emoji gen-pack PACK-URL
-
-  Creates a new manifest entry and a file list from the specified
-  remote pack file. Currently, only .zip archives are recognized
-  as remote pack files and packs are therefore assumed to be zip
-  archives. This command is intended to run interactively and will
-  first ask you some basic questions about the pack, then download
-  the remote file and generate an SHA256 checksum for it, then
-  generate an emoji file list for you.
-
-  The manifest entry will either be written to a newly created
-  `index.json` file or appended to the existing one, *replacing*
-  the old pack with the same name if it was in the file previously.
-
-  The file list will be written to the file specified previously,
-  *replacing* that file. You _should_ check that the file list doesn't
-  contain anything you don't need in the pack, that is, anything that is
-  not an emoji (the whole pack is downloaded, but only emoji files
-  are extracted).
-  """
+  @moduledoc File.read!("docs/administration/CLI_tasks/emoji.md")
 
   def run(["ls-packs" | args]) do
     Application.ensure_all_started(:hackney)
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index 1a1634fe9..9af6cda30 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -7,36 +7,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
   import Mix.Pleroma
 
   @shortdoc "Manages Pleroma instance"
-  @moduledoc """
-  Manages Pleroma instance.
-
-  ## Generate a new instance config.
-
-    mix pleroma.instance gen [OPTION...]
-
-  If any options are left unspecified, you will be prompted interactively
-
-  ## Options
-
-  - `-f`, `--force` - overwrite any output files
-  - `-o PATH`, `--output PATH` - the output file for the generated configuration
-  - `--output-psql PATH` - the output file for the generated PostgreSQL setup
-  - `--domain DOMAIN` - the domain of your instance
-  - `--instance-name INSTANCE_NAME` - the name of your instance
-  - `--admin-email ADMIN_EMAIL` - the email address of the instance admin
-  - `--notify-email NOTIFY_EMAIL` - email address for notifications
-  - `--dbhost HOSTNAME` - the hostname of the PostgreSQL database to use
-  - `--dbname DBNAME` - the name of the database to use
-  - `--dbuser DBUSER` - the user (aka role) to use for the database connection
-  - `--dbpass DBPASS` - the password to use for the database connection
-  - `--rum Y/N` - Whether to enable RUM indexes
-  - `--indexable Y/N` - Allow/disallow indexing site by search engines
-  - `--db-configurable Y/N` - Allow/disallow configuring instance from admin part
-  - `--uploads-dir` - the directory uploads go in when using a local uploader
-  - `--static-dir` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
-  - `--listen-ip` - the ip the app should listen to, defaults to 127.0.0.1
-  - `--listen-port` - the port the app should listen to, defaults to 4000
-  """
+  @moduledoc File.read!("docs/administration/CLI_tasks/instance.md")
 
   def run(["gen" | rest]) do
     {options, [], []} =
diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex
index 200721163..d7a7b599f 100644
--- a/lib/mix/tasks/pleroma/relay.ex
+++ b/lib/mix/tasks/pleroma/relay.ex
@@ -9,25 +9,8 @@ defmodule Mix.Tasks.Pleroma.Relay do
   alias Pleroma.Web.ActivityPub.Relay
 
   @shortdoc "Manages remote relays"
-  @moduledoc """
-  Manages remote relays
+  @moduledoc File.read!("docs/administration/CLI_tasks/relay.md")
 
-  ## Follow a remote relay
-
-  ``mix pleroma.relay follow <relay_url>``
-
-  Example: ``mix pleroma.relay follow  https://example.org/relay``
-
-  ## Unfollow a remote relay
-
-  ``mix pleroma.relay unfollow <relay_url>``
-
-  Example: ``mix pleroma.relay unfollow https://example.org/relay``
-
-  ## List relay subscriptions
-
-  ``mix pleroma.relay list``
-  """
   def run(["follow", target]) do
     start_pleroma()
 
diff --git a/lib/mix/tasks/pleroma/uploads.ex b/lib/mix/tasks/pleroma/uploads.ex
index 95392d81b..3e6fc7ee0 100644
--- a/lib/mix/tasks/pleroma/uploads.ex
+++ b/lib/mix/tasks/pleroma/uploads.ex
@@ -12,16 +12,8 @@ defmodule Mix.Tasks.Pleroma.Uploads do
   @log_every 50
 
   @shortdoc "Migrates uploads from local to remote storage"
-  @moduledoc """
-   Manages uploads
+  @moduledoc File.read!("docs/administration/CLI_tasks/uploads.md")
 
-   ## Migrate uploads from local to remote storage
-       mix pleroma.uploads migrate_local TARGET_UPLOADER [OPTIONS...]
-   Options:
-   - `--delete` - delete local uploads after migrating them to the target uploader
-
-   A list of available uploaders can be seen in config.exs
-  """
   def run(["migrate_local", target_uploader | args]) do
     delete? = Enum.member?(args, "--delete")
     start_pleroma()
diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex
index d93ba8dee..134b5bccc 100644
--- a/lib/mix/tasks/pleroma/user.ex
+++ b/lib/mix/tasks/pleroma/user.ex
@@ -10,86 +10,8 @@ defmodule Mix.Tasks.Pleroma.User do
   alias Pleroma.Web.OAuth
 
   @shortdoc "Manages Pleroma users"
-  @moduledoc """
-  Manages Pleroma users.
+  @moduledoc File.read!("docs/administration/CLI_tasks/user.md")
 
-  ## Create a new user.
-
-      mix pleroma.user new NICKNAME EMAIL [OPTION...]
-
-  Options:
-  - `--name NAME` - the user's name (i.e., "Lain Iwakura")
-  - `--bio BIO` - the user's bio
-  - `--password PASSWORD` - the user's password
-  - `--moderator`/`--no-moderator` - whether the user is a moderator
-  - `--admin`/`--no-admin` - whether the user is an admin
-  - `-y`, `--assume-yes`/`--no-assume-yes` - whether to assume yes to all questions
-
-  ## Generate an invite link.
-
-      mix pleroma.user invite [OPTION...]
-
-    Options:
-    - `--expires-at DATE` - last day on which token is active (e.g. "2019-04-05")
-    - `--max-use NUMBER` - maximum numbers of token uses
-
-  ## List generated invites
-
-      mix pleroma.user invites
-
-  ## Revoke invite
-
-      mix pleroma.user revoke_invite TOKEN OR TOKEN_ID
-
-  ## Delete the user's account.
-
-      mix pleroma.user rm NICKNAME
-
-  ## Delete the user's activities.
-
-      mix pleroma.user delete_activities NICKNAME
-
-  ## Sign user out from all applications (delete user's OAuth tokens and authorizations).
-
-      mix pleroma.user sign_out NICKNAME
-
-  ## Deactivate or activate the user's account.
-
-      mix pleroma.user toggle_activated NICKNAME
-
-  ## Unsubscribe local users from user's account and deactivate it
-
-      mix pleroma.user unsubscribe NICKNAME
-
-  ## Unsubscribe local users from an entire instance and deactivate all accounts
-
-      mix pleroma.user unsubscribe_all_from_instance INSTANCE
-
-  ## Create a password reset link.
-
-      mix pleroma.user reset_password NICKNAME
-
-  ## Set the value of the given user's settings.
-
-      mix pleroma.user set NICKNAME [OPTION...]
-
-  Options:
-  - `--locked`/`--no-locked` - whether the user's account is locked
-  - `--moderator`/`--no-moderator` - whether the user is a moderator
-  - `--admin`/`--no-admin` - whether the user is an admin
-
-  ## Add tags to a user.
-
-      mix pleroma.user tag NICKNAME TAGS
-
-  ## Delete tags from a user.
-
-      mix pleroma.user untag NICKNAME TAGS
-
-  ## Toggle confirmation of the user's account.
-
-      mix pleroma.user toggle_confirmed NICKNAME
-  """
   def run(["new", nickname, email | rest]) do
     {options, [], []} =
       OptionParser.parse(
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 9e35b02c0..0bf218bc7 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -42,6 +42,7 @@ defmodule Pleroma.Application do
         hackney_pool_children() ++
         [
           Pleroma.Stats,
+          Pleroma.JobQueueMonitor,
           {Oban, Pleroma.Config.get(Oban)}
         ] ++
         task_children(@env) ++
diff --git a/lib/pleroma/healthcheck.ex b/lib/pleroma/healthcheck.ex
index 977b78c26..fc2129815 100644
--- a/lib/pleroma/healthcheck.ex
+++ b/lib/pleroma/healthcheck.ex
@@ -14,6 +14,7 @@ defmodule Pleroma.Healthcheck do
             active: 0,
             idle: 0,
             memory_used: 0,
+            job_queue_stats: nil,
             healthy: true
 
   @type t :: %__MODULE__{
@@ -21,6 +22,7 @@ defmodule Pleroma.Healthcheck do
           active: non_neg_integer(),
           idle: non_neg_integer(),
           memory_used: number(),
+          job_queue_stats: map(),
           healthy: boolean()
         }
 
@@ -30,6 +32,7 @@ defmodule Pleroma.Healthcheck do
       memory_used: Float.round(:erlang.memory(:total) / 1024 / 1024, 2)
     }
     |> assign_db_info()
+    |> assign_job_queue_stats()
     |> check_health()
   end
 
@@ -55,6 +58,11 @@ defmodule Pleroma.Healthcheck do
     Map.merge(healthcheck, db_info)
   end
 
+  defp assign_job_queue_stats(healthcheck) do
+    stats = Pleroma.JobQueueMonitor.stats()
+    Map.put(healthcheck, :job_queue_stats, stats)
+  end
+
   @spec check_health(Healthcheck.t()) :: Healthcheck.t()
   def check_health(%{pool_size: pool_size, active: active} = check)
       when active >= pool_size do
diff --git a/lib/pleroma/job_queue_monitor.ex b/lib/pleroma/job_queue_monitor.ex
new file mode 100644
index 000000000..3feea8381
--- /dev/null
+++ b/lib/pleroma/job_queue_monitor.ex
@@ -0,0 +1,78 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.JobQueueMonitor do
+  use GenServer
+
+  @initial_state %{workers: %{}, queues: %{}, processed_jobs: 0}
+  @queue %{processed_jobs: 0, success: 0, failure: 0}
+  @operation %{processed_jobs: 0, success: 0, failure: 0}
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, @initial_state, name: __MODULE__)
+  end
+
+  @impl true
+  def init(state) do
+    :telemetry.attach("oban-monitor-failure", [:oban, :failure], &handle_event/4, nil)
+    :telemetry.attach("oban-monitor-success", [:oban, :success], &handle_event/4, nil)
+
+    {:ok, state}
+  end
+
+  def stats do
+    GenServer.call(__MODULE__, :stats)
+  end
+
+  def handle_event([:oban, status], %{duration: duration}, meta, _) do
+    GenServer.cast(__MODULE__, {:process_event, status, duration, meta})
+  end
+
+  @impl true
+  def handle_call(:stats, _from, state) do
+    {:reply, state, state}
+  end
+
+  @impl true
+  def handle_cast({:process_event, status, duration, meta}, state) do
+    state =
+      state
+      |> Map.update!(:workers, fn workers ->
+        workers
+        |> Map.put_new(meta.worker, %{})
+        |> Map.update!(meta.worker, &update_worker(&1, status, meta, duration))
+      end)
+      |> Map.update!(:queues, fn workers ->
+        workers
+        |> Map.put_new(meta.queue, @queue)
+        |> Map.update!(meta.queue, &update_queue(&1, status, meta, duration))
+      end)
+      |> Map.update!(:processed_jobs, &(&1 + 1))
+
+    {:noreply, state}
+  end
+
+  defp update_worker(worker, status, meta, duration) do
+    worker
+    |> Map.put_new(meta.args["op"], @operation)
+    |> Map.update!(meta.args["op"], &update_op(&1, status, meta, duration))
+  end
+
+  defp update_op(op, :enqueue, _meta, _duration) do
+    op
+    |> Map.update!(:enqueued, &(&1 + 1))
+  end
+
+  defp update_op(op, status, _meta, _duration) do
+    op
+    |> Map.update!(:processed_jobs, &(&1 + 1))
+    |> Map.update!(status, &(&1 + 1))
+  end
+
+  defp update_queue(queue, status, _meta, _duration) do
+    queue
+    |> Map.update!(:processed_jobs, &(&1 + 1))
+    |> Map.update!(status, &(&1 + 1))
+  end
+end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 4c1cdd042..c2f8fa0d7 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -583,7 +583,7 @@ defmodule Pleroma.User do
       is_integer(nickname_or_id) or FlakeId.flake_id?(nickname_or_id) ->
         get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id)
 
-      restrict_to_local == false ->
+      restrict_to_local == false or not String.contains?(nickname_or_id, "@") ->
         get_cached_by_nickname(nickname_or_id)
 
       restrict_to_local == :unauthenticated and match?(%User{}, opts[:for]) ->
diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex
index 60b58dc90..5e3dbe728 100644
--- a/lib/pleroma/web/mastodon_api/views/notification_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex
@@ -25,40 +25,44 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
     parent_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
     mastodon_type = Activity.mastodon_notification_type(activity)
 
-    response = %{
-      id: to_string(notification.id),
-      type: mastodon_type,
-      created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at),
-      account: AccountView.render("show.json", %{user: actor, for: user}),
-      pleroma: %{
-        is_seen: notification.seen
+    with %{id: _} = account <- AccountView.render("show.json", %{user: actor, for: user}) do
+      response = %{
+        id: to_string(notification.id),
+        type: mastodon_type,
+        created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at),
+        account: account,
+        pleroma: %{
+          is_seen: notification.seen
+        }
       }
-    }
 
-    case mastodon_type do
-      "mention" ->
-        response
-        |> Map.merge(%{
-          status: StatusView.render("show.json", %{activity: activity, for: user})
-        })
+      case mastodon_type do
+        "mention" ->
+          response
+          |> Map.merge(%{
+            status: StatusView.render("show.json", %{activity: activity, for: user})
+          })
 
-      "favourite" ->
-        response
-        |> Map.merge(%{
-          status: StatusView.render("show.json", %{activity: parent_activity, for: user})
-        })
+        "favourite" ->
+          response
+          |> Map.merge(%{
+            status: StatusView.render("show.json", %{activity: parent_activity, for: user})
+          })
 
-      "reblog" ->
-        response
-        |> Map.merge(%{
-          status: StatusView.render("show.json", %{activity: parent_activity, for: user})
-        })
+        "reblog" ->
+          response
+          |> Map.merge(%{
+            status: StatusView.render("show.json", %{activity: parent_activity, for: user})
+          })
 
-      "follow" ->
-        response
+        "follow" ->
+          response
 
-      _ ->
-        nil
+        _ ->
+          nil
+      end
+    else
+      _ -> nil
     end
   end
 end
diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index e418dc70d..1cd7294e7 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -460,7 +460,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
   end
 
   # Special case: Local MastodonFE
-  defp redirect_uri(%Plug.Conn{} = conn, "."), do: mastodon_api_url(conn, :login)
+  defp redirect_uri(%Plug.Conn{} = conn, "."), do: auth_url(conn, :login)
 
   defp redirect_uri(%Plug.Conn{}, redirect_uri), do: redirect_uri