Merge remote-tracking branch 'remotes/upstream/develop' into features/mastoapi/2.6.0-conversations

47 files changed, 919 insertions, 501 deletions

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 79dc26b01..bc3f8caba 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Activity do
 
   alias Pleroma.Activity
   alias Pleroma.Notification
+  alias Pleroma.Object
   alias Pleroma.Repo
 
   import Ecto.Query
@@ -22,6 +23,10 @@ defmodule Pleroma.Activity do
     "Like" => "favourite"
   }
 
+  @mastodon_to_ap_notification_types for {k, v} <- @mastodon_notification_types,
+                                         into: %{},
+                                         do: {v, k}
+
   schema "activities" do
     field(:data, :map)
     field(:local, :boolean, default: true)
@@ -29,9 +34,42 @@ defmodule Pleroma.Activity do
     field(:recipients, {:array, :string})
     has_many(:notifications, Notification, on_delete: :delete_all)
 
+    # Attention: this is a fake relation, don't try to preload it blindly and expect it to work!
+    # The foreign key is embedded in a jsonb field.
+    #
+    # To use it, you probably want to do an inner join and a preload:
+    #
+    # ```
+    # |> join(:inner, [activity], o in Object,
+    #      on: fragment("(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
+    #        o.data, activity.data, activity.data))
+    # |> preload([activity, object], [object: object])
+    # ```
+    #
+    # As a convenience, Activity.with_preloaded_object() sets up an inner join and preload for the
+    # typical case.
+    has_one(:object, Object, on_delete: :nothing, foreign_key: :id)
+
     timestamps()
   end
 
+  def with_preloaded_object(query) do
+    query
+    |> join(
+      :inner,
+      [activity],
+      o in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+          o.data,
+          activity.data,
+          activity.data
+        )
+    )
+    |> preload([activity, object], object: object)
+  end
+
   def get_by_ap_id(ap_id) do
     Repo.one(
       from(
@@ -41,10 +79,44 @@ defmodule Pleroma.Activity do
     )
   end
 
+  def get_by_ap_id_with_object(ap_id) do
+    Repo.one(
+      from(
+        activity in Activity,
+        where: fragment("(?)->>'id' = ?", activity.data, ^to_string(ap_id)),
+        left_join: o in Object,
+        on:
+          fragment(
+            "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+            o.data,
+            activity.data,
+            activity.data
+          ),
+        preload: [object: o]
+      )
+    )
+  end
+
   def get_by_id(id) do
     Repo.get(Activity, id)
   end
 
+  def get_by_id_with_object(id) do
+    from(activity in Activity,
+      where: activity.id == ^id,
+      inner_join: o in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+          o.data,
+          activity.data,
+          activity.data
+        ),
+      preload: [object: o]
+    )
+    |> Repo.one()
+  end
+
   def by_object_ap_id(ap_id) do
     from(
       activity in Activity,
@@ -72,7 +144,7 @@ defmodule Pleroma.Activity do
     )
   end
 
-  def create_by_object_ap_id(ap_id) do
+  def create_by_object_ap_id(ap_id) when is_binary(ap_id) do
     from(
       activity in Activity,
       where:
@@ -86,6 +158,8 @@ defmodule Pleroma.Activity do
     )
   end
 
+  def create_by_object_ap_id(_), do: nil
+
   def get_all_create_by_object_ap_id(ap_id) do
     Repo.all(create_by_object_ap_id(ap_id))
   end
@@ -97,8 +171,39 @@ defmodule Pleroma.Activity do
 
   def get_create_by_object_ap_id(_), do: nil
 
-  def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"])
-  def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id)
+  def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
+    from(
+      activity in Activity,
+      where:
+        fragment(
+          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
+          activity.data,
+          activity.data,
+          ^to_string(ap_id)
+        ),
+      where: fragment("(?)->>'type' = 'Create'", activity.data),
+      inner_join: o in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+          o.data,
+          activity.data,
+          activity.data
+        ),
+      preload: [object: o]
+    )
+  end
+
+  def create_by_object_ap_id_with_object(_), do: nil
+
+  def get_create_by_object_ap_id_with_object(ap_id) do
+    ap_id
+    |> create_by_object_ap_id_with_object()
+    |> Repo.one()
+  end
+
+  def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"])
+  def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id)
   def normalize(_), do: nil
 
   def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do
@@ -109,7 +214,8 @@ defmodule Pleroma.Activity do
 
   def delete_by_ap_id(id) when is_binary(id) do
     by_object_ap_id(id)
-    |> Repo.delete_all(returning: true)
+    |> select([u], u)
+    |> Repo.delete_all()
     |> elem(1)
     |> Enum.find(fn
       %{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id
@@ -126,6 +232,10 @@ defmodule Pleroma.Activity do
 
   def mastodon_notification_type(%Activity{}), do: nil
 
+  def from_mastodon_notification_type(type) do
+    Map.get(@mastodon_to_ap_notification_types, type)
+  end
+
   def all_by_actor_and_id(actor, status_ids \\ [])
   def all_by_actor_and_id(_actor, []), do: []
 
@@ -135,4 +245,50 @@ defmodule Pleroma.Activity do
     |> where([s], s.actor == ^actor)
     |> Repo.all()
   end
+
+  def increase_replies_count(id) do
+    Activity
+    |> where(id: ^id)
+    |> update([a],
+      set: [
+        data:
+          fragment(
+            """
+            jsonb_set(?, '{object, repliesCount}',
+              (coalesce((?->'object'->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true)
+            """,
+            a.data,
+            a.data
+          )
+      ]
+    )
+    |> Repo.update_all([])
+    |> case do
+      {1, [activity]} -> activity
+      _ -> {:error, "Not found"}
+    end
+  end
+
+  def decrease_replies_count(id) do
+    Activity
+    |> where(id: ^id)
+    |> update([a],
+      set: [
+        data:
+          fragment(
+            """
+            jsonb_set(?, '{object, repliesCount}',
+              (greatest(0, (?->'object'->>'repliesCount')::int - 1))::varchar::jsonb, true)
+            """,
+            a.data,
+            a.data
+          )
+      ]
+    )
+    |> Repo.update_all([])
+    |> case do
+      {1, [activity]} -> activity
+      _ -> {:error, "Not found"}
+    end
+  end
 end
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index cc81e1805..782d1d589 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -110,7 +110,6 @@ defmodule Pleroma.Application do
           worker(Pleroma.Web.Federator.RetryQueue, []),
           worker(Pleroma.Stats, []),
           worker(Pleroma.Web.Push, []),
-          worker(Pleroma.Jobs, []),
           worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary)
         ] ++
         streamer_child() ++
diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex
index 9b20c7e08..afefccec5 100644
--- a/lib/pleroma/emails/admin_email.ex
+++ b/lib/pleroma/emails/admin_email.ex
@@ -29,9 +29,13 @@ defmodule Pleroma.AdminEmail do
       if length(statuses) > 0 do
         statuses_list_html =
           statuses
-          |> Enum.map(fn %{id: id} ->
-            status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, id)
-            "<li><a href=\"#{status_url}\">#{status_url}</li>"
+          |> Enum.map(fn
+            %{id: id} ->
+              status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, id)
+              "<li><a href=\"#{status_url}\">#{status_url}</li>"
+
+            id when is_binary(id) ->
+              "<li><a href=\"#{id}\">#{id}</li>"
           end)
           |> Enum.join("\n")
 
diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex
index f7e3aa78b..b384e6fec 100644
--- a/lib/pleroma/emails/mailer.ex
+++ b/lib/pleroma/emails/mailer.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Mailer do
   use Swoosh.Mailer, otp_app: :pleroma
 
   def deliver_async(email, config \\ []) do
-    Pleroma.Jobs.enqueue(:mailer, __MODULE__, [:deliver_async, email, config])
+    PleromaJobQueue.enqueue(:mailer, __MODULE__, [:deliver_async, email, config])
   end
 
   def perform(:deliver_async, email, config), do: deliver(email, config)
diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index 1e4ede3f2..e3625383b 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Formatter do
   alias Pleroma.User
   alias Pleroma.Web.MediaProxy
 
+  @safe_mention_regex ~r/^(\s*(?<mentions>@.+?\s+)+)(?<rest>.*)/
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
   @link_regex ~r{((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+}ui
   # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
@@ -45,15 +46,28 @@ defmodule Pleroma.Formatter do
 
   @doc """
   Parses a text and replace plain text links with HTML. Returns a tuple with a result text, mentions, and hashtags.
+
+  If the 'safe_mention' option is given, only consecutive mentions at the start the post are actually mentioned.
   """
   @spec linkify(String.t(), keyword()) ::
           {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]}
   def linkify(text, options \\ []) do
     options = options ++ @auto_linker_config
-    acc = %{mentions: MapSet.new(), tags: MapSet.new()}
-    {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options)
 
-    {text, MapSet.to_list(mentions), MapSet.to_list(tags)}
+    if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do
+      %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text)
+      acc = %{mentions: MapSet.new(), tags: MapSet.new()}
+
+      {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options)
+      {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options)
+
+      {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)}
+    else
+      acc = %{mentions: MapSet.new(), tags: MapSet.new()}
+      {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options)
+
+      {text, MapSet.to_list(mentions), MapSet.to_list(tags)}
+    end
   end
 
   def emojify(text) do
diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex
index 6baacc566..3b9629d77 100644
--- a/lib/pleroma/gopher/server.ex
+++ b/lib/pleroma/gopher/server.ex
@@ -66,7 +66,8 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do
   def link(name, selector, type \\ 1) do
     address = Pleroma.Web.Endpoint.host()
     port = Pleroma.Config.get([:gopher, :port], 1234)
-    "#{type}#{name}\t#{selector}\t#{address}\t#{port}\r\n"
+    dstport = Pleroma.Config.get([:gopher, :dstport], port)
+    "#{type}#{name}\t#{selector}\t#{address}\t#{dstport}\r\n"
   end
 
   def render_activities(activities) do
diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex
index 05253157e..5b152d926 100644
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@@ -95,6 +95,13 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
   Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
   Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"])
 
+  Meta.allow_tag_with_this_attribute_values("a", "rel", [
+    "tag",
+    "nofollow",
+    "noopener",
+    "noreferrer"
+  ])
+
   # paragraphs and linebreaks
   Meta.allow_tag_with_these_attributes("br", [])
   Meta.allow_tag_with_these_attributes("p", [])
@@ -137,6 +144,13 @@ defmodule Pleroma.HTML.Scrubber.Default do
   Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
   Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"])
 
+  Meta.allow_tag_with_this_attribute_values("a", "rel", [
+    "tag",
+    "nofollow",
+    "noopener",
+    "noreferrer"
+  ])
+
   Meta.allow_tag_with_these_attributes("abbr", ["title"])
 
   Meta.allow_tag_with_these_attributes("b", [])
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index e92006151..420803a8f 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -12,7 +12,7 @@ defmodule Pleroma.Instances.Instance do
 
   schema "instances" do
     field(:host, :string)
-    field(:unreachable_since, :naive_datetime)
+    field(:unreachable_since, :naive_datetime_usec)
 
     timestamps()
   end
diff --git a/lib/pleroma/jobs.ex b/lib/pleroma/jobs.ex
deleted file mode 100644
index 24b7e5e46..000000000
--- a/lib/pleroma/jobs.ex
+++ /dev/null
@@ -1,152 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Jobs do
-  @moduledoc """
-  A basic job queue
-  """
-  use GenServer
-
-  require Logger
-
-  def init(args) do
-    {:ok, args}
-  end
-
-  def start_link do
-    queues =
-      Pleroma.Config.get(Pleroma.Jobs)
-      |> Enum.map(fn {name, _} -> create_queue(name) end)
-      |> Enum.into(%{})
-
-    state = %{
-      queues: queues,
-      refs: %{}
-    }
-
-    GenServer.start_link(__MODULE__, state, name: __MODULE__)
-  end
-
-  def create_queue(name) do
-    {name, {:sets.new(), []}}
-  end
-
-  @doc """
-  Enqueues a job.
-
-  Returns `:ok`.
-
-  ## Arguments
-
-  - `queue_name` - a queue name(must be specified in the config).
-  - `mod` - a worker module (must have `perform` function).
-  - `args` - a list of arguments for the `perform` function of the worker module.
-  - `priority` - a job priority (`0` by default).
-
-  ## Examples
-
-  Enqueue `Module.perform/0` with `priority=1`:
-
-      iex> Pleroma.Jobs.enqueue(:example_queue, Module, [])
-      :ok
-
-  Enqueue `Module.perform(:job_name)` with `priority=5`:
-
-      iex> Pleroma.Jobs.enqueue(:example_queue, Module, [:job_name], 5)
-      :ok
-
-  Enqueue `Module.perform(:another_job, data)` with `priority=1`:
-
-      iex> data = "foobar"
-      iex> Pleroma.Jobs.enqueue(:example_queue, Module, [:another_job, data])
-      :ok
-
-  Enqueue `Module.perform(:foobar_job, :foo, :bar, 42)` with `priority=1`:
-
-      iex> Pleroma.Jobs.enqueue(:example_queue, Module, [:foobar_job, :foo, :bar, 42])
-      :ok
-
-  """
-
-  def enqueue(queue_name, mod, args, priority \\ 1)
-
-  if Mix.env() == :test do
-    def enqueue(_queue_name, mod, args, _priority) do
-      apply(mod, :perform, args)
-    end
-  else
-    @spec enqueue(atom(), atom(), [any()], integer()) :: :ok
-    def enqueue(queue_name, mod, args, priority) do
-      GenServer.cast(__MODULE__, {:enqueue, queue_name, mod, args, priority})
-    end
-  end
-
-  def handle_cast({:enqueue, queue_name, mod, args, priority}, state) do
-    {running_jobs, queue} = state[:queues][queue_name]
-
-    queue = enqueue_sorted(queue, {mod, args}, priority)
-
-    state =
-      state
-      |> update_queue(queue_name, {running_jobs, queue})
-      |> maybe_start_job(queue_name, running_jobs, queue)
-
-    {:noreply, state}
-  end
-
-  def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
-    queue_name = state.refs[ref]
-
-    {running_jobs, queue} = state[:queues][queue_name]
-
-    running_jobs = :sets.del_element(ref, running_jobs)
-
-    state =
-      state
-      |> remove_ref(ref)
-      |> update_queue(queue_name, {running_jobs, queue})
-      |> maybe_start_job(queue_name, running_jobs, queue)
-
-    {:noreply, state}
-  end
-
-  def maybe_start_job(state, queue_name, running_jobs, queue) do
-    if :sets.size(running_jobs) < Pleroma.Config.get([__MODULE__, queue_name, :max_jobs]) &&
-         queue != [] do
-      {{mod, args}, queue} = queue_pop(queue)
-      {:ok, pid} = Task.start(fn -> apply(mod, :perform, args) end)
-      mref = Process.monitor(pid)
-
-      state
-      |> add_ref(queue_name, mref)
-      |> update_queue(queue_name, {:sets.add_element(mref, running_jobs), queue})
-    else
-      state
-    end
-  end
-
-  def enqueue_sorted(queue, element, priority) do
-    [%{item: element, priority: priority} | queue]
-    |> Enum.sort_by(fn %{priority: priority} -> priority end)
-  end
-
-  def queue_pop([%{item: element} | queue]) do
-    {element, queue}
-  end
-
-  defp add_ref(state, queue_name, ref) do
-    refs = Map.put(state[:refs], ref, queue_name)
-    Map.put(state, :refs, refs)
-  end
-
-  defp remove_ref(state, ref) do
-    refs = Map.delete(state[:refs], ref)
-    Map.put(state, :refs, refs)
-  end
-
-  defp update_queue(state, queue_name, data) do
-    queues = Map.put(state[:queues], queue_name, data)
-    Map.put(state, :queues, queues)
-  end
-end
diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 765191275..cac10f24a 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -7,6 +7,8 @@ defmodule Pleroma.Notification do
 
   alias Pleroma.Activity
   alias Pleroma.Notification
+  alias Pleroma.Object
+  alias Pleroma.Pagination
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
@@ -28,36 +30,25 @@ defmodule Pleroma.Notification do
     |> cast(attrs, [:seen])
   end
 
-  # TODO: Make generic and unify (see activity_pub.ex)
-  defp restrict_max(query, %{"max_id" => max_id}) do
-    from(activity in query, where: activity.id < ^max_id)
+  def for_user_query(user) do
+    Notification
+    |> where(user_id: ^user.id)
+    |> join(:inner, [n], activity in assoc(n, :activity))
+    |> join(:left, [n, a], object in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
+          object.data,
+          a.data
+        )
+    )
+    |> preload([n, a, o], activity: {a, object: o})
   end
 
-  defp restrict_max(query, _), do: query
-
-  defp restrict_since(query, %{"since_id" => since_id}) do
-    from(activity in query, where: activity.id > ^since_id)
-  end
-
-  defp restrict_since(query, _), do: query
-
   def for_user(user, opts \\ %{}) do
-    query =
-      from(
-        n in Notification,
-        where: n.user_id == ^user.id,
-        order_by: [desc: n.id],
-        join: activity in assoc(n, :activity),
-        preload: [activity: activity],
-        limit: 20
-      )
-
-    query =
-      query
-      |> restrict_since(opts)
-      |> restrict_max(opts)
-
-    Repo.all(query)
+    user
+    |> for_user_query()
+    |> Pagination.fetch_paginated(opts)
   end
 
   def set_read_up_to(%{id: user_id} = _user, id) do
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index 58e46ef1d..8a670645d 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -14,6 +14,8 @@ defmodule Pleroma.Object do
   import Ecto.Query
   import Ecto.Changeset
 
+  require Logger
+
   schema "objects" do
     field(:data, :map)
 
@@ -38,6 +40,33 @@ defmodule Pleroma.Object do
     Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
   end
 
+  # If we pass an Activity to Object.normalize(), we can try to use the preloaded object.
+  # Use this whenever possible, especially when walking graphs in an O(N) loop!
+  def normalize(%Activity{object: %Object{} = object}), do: object
+
+  # Catch and log Object.normalize() calls where the Activity's child object is not
+  # preloaded.
+  def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}) do
+    Logger.debug(
+      "Object.normalize() called without preloaded object (#{ap_id}).  Consider preloading the object!"
+    )
+
+    Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
+
+    normalize(ap_id)
+  end
+
+  def normalize(%Activity{data: %{"object" => ap_id}}) do
+    Logger.debug(
+      "Object.normalize() called without preloaded object (#{ap_id}).  Consider preloading the object!"
+    )
+
+    Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
+
+    normalize(ap_id)
+  end
+
+  # Old way, try fetching the object through cache.
   def normalize(%{"id" => ap_id}), do: normalize(ap_id)
   def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
   def normalize(_), do: nil
@@ -104,4 +133,50 @@ defmodule Pleroma.Object do
       e -> e
     end
   end
+
+  def increase_replies_count(ap_id) do
+    Object
+    |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id)))
+    |> update([o],
+      set: [
+        data:
+          fragment(
+            """
+            jsonb_set(?, '{repliesCount}',
+              (coalesce((?->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true)
+            """,
+            o.data,
+            o.data
+          )
+      ]
+    )
+    |> Repo.update_all([])
+    |> case do
+      {1, [object]} -> set_cache(object)
+      _ -> {:error, "Not found"}
+    end
+  end
+
+  def decrease_replies_count(ap_id) do
+    Object
+    |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id)))
+    |> update([o],
+      set: [
+        data:
+          fragment(
+            """
+            jsonb_set(?, '{repliesCount}',
+              (greatest(0, (?->>'repliesCount')::int - 1))::varchar::jsonb, true)
+            """,
+            o.data,
+            o.data
+          )
+      ]
+    )
+    |> Repo.update_all([])
+    |> case do
+      {1, [object]} -> set_cache(object)
+      _ -> {:error, "Not found"}
+    end
+  end
 end
diff --git a/lib/pleroma/pagination.ex b/lib/pleroma/pagination.ex
new file mode 100644
index 000000000..7c864deef
--- /dev/null
+++ b/lib/pleroma/pagination.ex
@@ -0,0 +1,78 @@
+defmodule Pleroma.Pagination do
+  @moduledoc """
+  Implements Mastodon-compatible pagination.
+  """
+
+  import Ecto.Query
+  import Ecto.Changeset
+
+  alias Pleroma.Repo
+
+  @default_limit 20
+
+  def fetch_paginated(query, params) do
+    options = cast_params(params)
+
+    query
+    |> paginate(options)
+    |> Repo.all()
+    |> enforce_order(options)
+  end
+
+  def paginate(query, options) do
+    query
+    |> restrict(:min_id, options)
+    |> restrict(:since_id, options)
+    |> restrict(:max_id, options)
+    |> restrict(:order, options)
+    |> restrict(:limit, options)
+  end
+
+  defp cast_params(params) do
+    param_types = %{
+      min_id: :string,
+      since_id: :string,
+      max_id: :string,
+      limit: :integer
+    }
+
+    changeset = cast({%{}, param_types}, params, Map.keys(param_types))
+    changeset.changes
+  end
+
+  defp restrict(query, :min_id, %{min_id: min_id}) do
+    where(query, [q], q.id > ^min_id)
+  end
+
+  defp restrict(query, :since_id, %{since_id: since_id}) do
+    where(query, [q], q.id > ^since_id)
+  end
+
+  defp restrict(query, :max_id, %{max_id: max_id}) do
+    where(query, [q], q.id < ^max_id)
+  end
+
+  defp restrict(query, :order, %{min_id: _}) do
+    order_by(query, [u], fragment("? asc nulls last", u.id))
+  end
+
+  defp restrict(query, :order, _options) do
+    order_by(query, [u], fragment("? desc nulls last", u.id))
+  end
+
+  defp restrict(query, :limit, options) do
+    limit = Map.get(options, :limit, @default_limit)
+
+    query
+    |> limit(^limit)
+  end
+
+  defp restrict(query, _, _), do: query
+
+  defp enforce_order(result, %{min_id: _}) do
+    result
+    |> Enum.reverse()
+  end
+
+  defp enforce_order(result, _), do: result
+end
diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex
index e6a51b19e..4af1bde56 100644
--- a/lib/pleroma/repo.ex
+++ b/lib/pleroma/repo.ex
@@ -3,7 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Repo do
-  use Ecto.Repo, otp_app: :pleroma
+  use Ecto.Repo,
+    otp_app: :pleroma,
+    adapter: Ecto.Adapters.Postgres,
+    migration_timestamps: [type: :naive_datetime_usec]
 
   @doc """
   Dynamically loads the repository url from the
diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex
index e7de3f3e0..521daa93b 100644
--- a/lib/pleroma/uploaders/s3.ex
+++ b/lib/pleroma/uploaders/s3.ex
@@ -13,10 +13,15 @@ defmodule Pleroma.Uploaders.S3 do
     bucket = Keyword.fetch!(config, :bucket)
 
     bucket_with_namespace =
-      if namespace = Keyword.get(config, :bucket_namespace) do
-        namespace <> ":" <> bucket
-      else
-        bucket
+      cond do
+        truncated_namespace = Keyword.get(config, :truncated_namespace) ->
+          truncated_namespace
+
+        namespace = Keyword.get(config, :bucket_namespace) ->
+          namespace <> ":" <> bucket
+
+        true ->
+          bucket
       end
 
     {:ok,
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 692ae836c..728b00a56 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -50,23 +50,20 @@ defmodule Pleroma.User do
     field(:local, :boolean, default: true)
     field(:follower_address, :string)
     field(:search_rank, :float, virtual: true)
+    field(:search_type, :integer, virtual: true)
     field(:tags, {:array, :string}, default: [])
     field(:bookmarks, {:array, :string}, default: [])
-    field(:last_refreshed_at, :naive_datetime)
+    field(:last_refreshed_at, :naive_datetime_usec)
     has_many(:notifications, Notification)
     embeds_one(:info, Pleroma.User.Info)
 
     timestamps()
   end
 
-  def auth_active?(%User{local: false}), do: true
-
-  def auth_active?(%User{info: %User.Info{confirmation_pending: false}}), do: true
-
   def auth_active?(%User{info: %User.Info{confirmation_pending: true}}),
     do: !Pleroma.Config.get([:instance, :account_activation_required])
 
-  def auth_active?(_), do: false
+  def auth_active?(%User{}), do: true
 
   def visible_for?(user, for_user \\ nil)
 
@@ -82,17 +79,17 @@ defmodule Pleroma.User do
   def superuser?(%User{local: true, info: %User.Info{is_moderator: true}}), do: true
   def superuser?(_), do: false
 
-  def avatar_url(user) do
+  def avatar_url(user, options \\ []) do
     case user.avatar do
       %{"url" => [%{"href" => href} | _]} -> href
-      _ -> "#{Web.base_url()}/images/avi.png"
+      _ -> !options[:no_default] && "#{Web.base_url()}/images/avi.png"
     end
   end
 
-  def banner_url(user) do
+  def banner_url(user, options \\ []) do
     case user.info.banner do
       %{"url" => [%{"href" => href} | _]} -> href
-      _ -> "#{Web.base_url()}/images/banner.png"
+      _ -> !options[:no_default] && "#{Web.base_url()}/images/banner.png"
     end
   end
 
@@ -104,9 +101,8 @@ defmodule Pleroma.User do
     "#{Web.base_url()}/users/#{nickname}"
   end
 
-  def ap_followers(%User{} = user) do
-    "#{ap_id(user)}/followers"
-  end
+  def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa
+  def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers"
 
   def user_info(%User{} = user) do
     oneself = if user.local, do: 1, else: 0
@@ -335,10 +331,11 @@ defmodule Pleroma.User do
                 ^followed_addresses
               )
           ]
-        ]
+        ],
+        select: u
       )
 
-    {1, [follower]} = Repo.update_all(q, [], returning: true)
+    {1, [follower]} = Repo.update_all(q, [])
 
     Enum.each(followeds, fn followed ->
       update_follower_count(followed)
@@ -368,10 +365,11 @@ defmodule Pleroma.User do
         q =
           from(u in User,
             where: u.id == ^follower.id,
-            update: [push: [following: ^ap_followers]]
+            update: [push: [following: ^ap_followers]],
+            select: u
           )
 
-        {1, [follower]} = Repo.update_all(q, [], returning: true)
+        {1, [follower]} = Repo.update_all(q, [])
 
         {:ok, _} = update_follower_count(followed)
 
@@ -386,10 +384,11 @@ defmodule Pleroma.User do
       q =
         from(u in User,
           where: u.id == ^follower.id,
-          update: [pull: [following: ^ap_followers]]
+          update: [pull: [following: ^ap_followers]],
+          select: u
         )
 
-      {1, [follower]} = Repo.update_all(q, [], returning: true)
+      {1, [follower]} = Repo.update_all(q, [])
 
       {:ok, followed} = update_follower_count(followed)
 
@@ -637,7 +636,7 @@ defmodule Pleroma.User do
     users =
       user
       |> User.get_follow_requests_query()
-      |> join(:inner, [a], u in User, a.actor == u.ap_id)
+      |> join(:inner, [a], u in User, on: a.actor == u.ap_id)
       |> where([a, u], not fragment("? @> ?", u.following, ^[user.follower_address]))
       |> group_by([a, u], u.id)
       |> select([a, u], u)
@@ -659,7 +658,8 @@ defmodule Pleroma.User do
           )
       ]
     )
-    |> Repo.update_all([], returning: true)
+    |> select([u], u)
+    |> Repo.update_all([])
     |> case do
       {1, [user]} -> set_cache(user)
       _ -> {:error, user}
@@ -679,7 +679,8 @@ defmodule Pleroma.User do
           )
       ]
     )
-    |> Repo.update_all([], returning: true)
+    |> select([u], u)
+    |> Repo.update_all([])
     |> case do
       {1, [user]} -> set_cache(user)
       _ -> {:error, user}
@@ -725,7 +726,8 @@ defmodule Pleroma.User do
           )
       ]
     )
-    |> Repo.update_all([], returning: true)
+    |> select([u], u)
+    |> Repo.update_all([])
     |> case do
       {1, [user]} -> set_cache(user)
       _ -> {:error, user}
@@ -766,90 +768,59 @@ defmodule Pleroma.User do
     Repo.all(query)
   end
 
-  @spec search_for_admin(%{
-          local: boolean(),
-          page: number(),
-          page_size: number()
-        }) :: {:ok, [Pleroma.User.t()], number()}
-  def search_for_admin(%{query: nil, local: local, page: page, page_size: page_size}) do
-    query =
-      from(u in User, order_by: u.id)
-      |> maybe_local_user_query(local)
-
-    paginated_query =
-      query
-      |> paginate(page, page_size)
-
-    count =
-      query
-      |> Repo.aggregate(:count, :id)
-
-    {:ok, Repo.all(paginated_query), count}
-  end
-
-  @spec search_for_admin(%{
-          query: binary(),
-          admin: Pleroma.User.t(),
-          local: boolean(),
-          page: number(),
-          page_size: number()
-        }) :: {:ok, [Pleroma.User.t()], number()}
-  def search_for_admin(%{
-        query: term,
-        admin: admin,
-        local: local,
-        page: page,
-        page_size: page_size
-      }) do
-    term = String.trim_leading(term, "@")
-
-    local_paginated_query =
-      User
-      |> maybe_local_user_query(local)
-      |> paginate(page, page_size)
-
-    search_query = fts_search_subquery(term, local_paginated_query)
-
-    count =
-      term
-      |> fts_search_subquery()
-      |> maybe_local_user_query(local)
-      |> Repo.aggregate(:count, :id)
-
-    {:ok, do_search(search_query, admin), count}
-  end
-
   def search(query, resolve \\ false, for_user \\ nil) do
     # Strip the beginning @ off if there is a query
     query = String.trim_leading(query, "@")
 
     if resolve, do: get_or_fetch(query)
 
-    fts_results = do_search(fts_search_subquery(query), for_user)
-
-    {:ok, trigram_results} =
+    {:ok, results} =
       Repo.transaction(fn ->
         Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
-        do_search(trigram_search_subquery(query), for_user)
+        Repo.all(search_query(query, for_user))
       end)
 
-    Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
+    results
   end
 
-  defp do_search(subquery, for_user, options \\ []) do
-    q =
-      from(
-        s in subquery(subquery),
-        order_by: [desc: s.search_rank],
-        limit: ^(options[:limit] || 20)
-      )
+  def search_query(query, for_user) do
+    fts_subquery = fts_search_subquery(query)
+    trigram_subquery = trigram_search_subquery(query)
+    union_query = from(s in trigram_subquery, union_all: ^fts_subquery)
+    distinct_query = from(s in subquery(union_query), order_by: s.search_type, distinct: s.id)
 
-    results =
-      q
-      |> Repo.all()
-      |> Enum.filter(&(&1.search_rank > 0))
+    from(s in subquery(boost_search_rank_query(distinct_query, for_user)),
+      order_by: [desc: s.search_rank],
+      limit: 20
+    )
+  end
 
-    boost_search_results(results, for_user)
+  defp boost_search_rank_query(query, nil), do: query
+
+  defp boost_search_rank_query(query, for_user) do
+    friends_ids = get_friends_ids(for_user)
+    followers_ids = get_followers_ids(for_user)
+
+    from(u in subquery(query),
+      select_merge: %{
+        search_rank:
+          fragment(
+            """
+             CASE WHEN (?) THEN (?) * 1.3
+             WHEN (?) THEN (?) * 1.2
+             WHEN (?) THEN (?) * 1.1
+             ELSE (?) END
+            """,
+            u.id in ^friends_ids and u.id in ^followers_ids,
+            u.search_rank,
+            u.id in ^friends_ids,
+            u.search_rank,
+            u.id in ^followers_ids,
+            u.search_rank,
+            u.search_rank
+          )
+      }
+    )
   end
 
   defp fts_search_subquery(term, query \\ User) do
@@ -864,6 +835,7 @@ defmodule Pleroma.User do
     from(
       u in query,
       select_merge: %{
+        search_type: ^0,
         search_rank:
           fragment(
             """
@@ -896,6 +868,8 @@ defmodule Pleroma.User do
     from(
       u in User,
       select_merge: %{
+        # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason
+        search_type: fragment("?", 1),
         search_rank:
           fragment(
             "similarity(?, trim(? || ' ' || coalesce(?, '')))",
@@ -908,33 +882,6 @@ defmodule Pleroma.User do
     )
   end
 
-  defp boost_search_results(results, nil), do: results
-
-  defp boost_search_results(results, for_user) do
-    friends_ids = get_friends_ids(for_user)
-    followers_ids = get_followers_ids(for_user)
-
-    Enum.map(
-      results,
-      fn u ->
-        search_rank_coef =
-          cond do
-            u.id in friends_ids ->
-              1.2
-
-            u.id in followers_ids ->
-              1.1
-
-            true ->
-              1
-          end
-
-        Map.put(u, :search_rank, u.search_rank * search_rank_coef)
-      end
-    )
-    |> Enum.sort_by(&(-&1.search_rank))
-  end
-
   def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do
     Enum.map(
       blocked_identifiers,
@@ -1074,6 +1021,42 @@ defmodule Pleroma.User do
     )
   end
 
+  def maybe_external_user_query(query, external) do
+    if external, do: external_user_query(query), else: query
+  end
+
+  def external_user_query(query \\ User) do
+    from(
+      u in query,
+      where: u.local == false,
+      where: not is_nil(u.nickname)
+    )
+  end
+
+  def maybe_active_user_query(query, active) do
+    if active, do: active_user_query(query), else: query
+  end
+
+  def active_user_query(query \\ User) do
+    from(
+      u in query,
+      where: fragment("not (?->'deactivated' @> 'true')", u.info),
+      where: not is_nil(u.nickname)
+    )
+  end
+
+  def maybe_deactivated_user_query(query, deactivated) do
+    if deactivated, do: deactivated_user_query(query), else: query
+  end
+
+  def deactivated_user_query(query \\ User) do
+    from(
+      u in query,
+      where: fragment("(?->'deactivated' @> 'true')", u.info),
+      where: not is_nil(u.nickname)
+    )
+  end
+
   def active_local_user_query do
     from(
       u in local_user_query(),
@@ -1113,13 +1096,15 @@ defmodule Pleroma.User do
     friends
     |> Enum.each(fn followed -> User.unfollow(user, followed) end)
 
-    query = from(a in Activity, where: a.actor == ^user.ap_id)
+    query =
+      from(a in Activity, where: a.actor == ^user.ap_id)
+      |> Activity.with_preloaded_object()
 
     Repo.all(query)
     |> Enum.each(fn activity ->
       case activity.data["type"] do
         "Create" ->
-          ActivityPub.delete(Object.normalize(activity.data["object"]))
+          ActivityPub.delete(Object.normalize(activity))
 
         # TODO: Do something with likes, follows, repeats.
         _ ->
@@ -1159,9 +1144,12 @@ defmodule Pleroma.User do
     if !is_nil(user) and !User.needs_update?(user) do
       user
     else
+      # Whether to fetch initial posts for the user (if it's a new user & the fetching is enabled)
+      should_fetch_initial = is_nil(user) and Pleroma.Config.get([:fetch_initial_posts, :enabled])
+
       user = fetch_by_ap_id(ap_id)
 
-      if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do
+      if should_fetch_initial do
         with %User{} = user do
           {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
         end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 2470b4a71..6e1ed7ec9 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -89,13 +89,37 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     if is_public?(object), do: User.decrease_note_count(actor), else: {:ok, actor}
   end
 
+  def increase_replies_count_if_reply(%{
+        "object" =>
+          %{"inReplyTo" => reply_ap_id, "inReplyToStatusId" => reply_status_id} = object,
+        "type" => "Create"
+      }) do
+    if is_public?(object) do
+      Activity.increase_replies_count(reply_status_id)
+      Object.increase_replies_count(reply_ap_id)
+    end
+  end
+
+  def increase_replies_count_if_reply(_create_data), do: :noop
+
+  def decrease_replies_count_if_reply(%Object{
+        data: %{"inReplyTo" => reply_ap_id, "inReplyToStatusId" => reply_status_id} = object
+      }) do
+    if is_public?(object) do
+      Activity.decrease_replies_count(reply_status_id)
+      Object.decrease_replies_count(reply_ap_id)
+    end
+  end
+
+  def decrease_replies_count_if_reply(_object), do: :noop
+
   def insert(map, local \\ true) when is_map(map) do
     with nil <- Activity.normalize(map),
          map <- lazy_put_activity_defaults(map),
          :ok <- check_actor_is_active(map["actor"]),
          {_, true} <- {:remote_limit_error, check_remote_limit(map)},
          {:ok, map} <- MRF.filter(map),
-         :ok <- insert_full_object(map) do
+         {:ok, object} <- insert_full_object(map) do
       {recipients, _, _} = get_recipients(map)
 
       {:ok, activity} =
@@ -106,6 +130,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           recipients: recipients
         })
 
+      # Splice in the child object if we have one.
+      activity =
+        if !is_nil(object) do
+          Map.put(activity, :object, object)
+        else
+          activity
+        end
+
       Task.start(fn ->
         Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
       end)
@@ -170,6 +202,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
              additional
            ),
          {:ok, activity} <- insert(create_data, local),
+         _ <- increase_replies_count_if_reply(create_data),
          # Changing note count prior to enqueuing federation task in order to avoid
          # race conditions on updating user.info
          {:ok, _actor} <- increase_note_count_if_public(actor, activity),
@@ -321,6 +354,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
            "deleted_activity_id" => activity && activity.id
          },
          {:ok, activity} <- insert(data, local),
+         _ <- decrease_replies_count_if_reply(object),
          # Changing note count prior to enqueuing federation task in order to avoid
          # race conditions on updating user.info
          {:ok, _actor} <- decrease_note_count_if_public(user, object),
@@ -430,6 +464,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           ),
         order_by: [desc: :id]
       )
+      |> Activity.with_preloaded_object()
 
     Repo.all(query)
   end
@@ -702,13 +737,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
     from(
       activity in query,
-      where: fragment("not ?->>'type' = 'Announce'", activity.data),
-      where: fragment("not ? = ANY(?)", activity.actor, ^muted_reblogs)
+      where:
+        fragment(
+          "not ( ?->>'type' = 'Announce' and ? = ANY(?))",
+          activity.data,
+          activity.actor,
+          ^muted_reblogs
+        )
     )
   end
 
   defp restrict_muted_reblogs(query, _), do: query
 
+  defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query
+
+  defp maybe_preload_objects(query, _) do
+    query
+    |> Activity.with_preloaded_object()
+  end
+
   def fetch_activities_query(recipients, opts \\ %{}) do
     base_query =
       from(
@@ -718,6 +765,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       )
 
     base_query
+    |> maybe_preload_objects(opts)
     |> restrict_recipients(recipients, opts["user"])
     |> restrict_tag(opts)
     |> restrict_tag_reject(opts)
@@ -940,7 +988,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
            },
            :ok <- Transmogrifier.contain_origin(id, params),
            {:ok, activity} <- Transmogrifier.handle_incoming(params) do
-        {:ok, Object.normalize(activity.data["object"])}
+        {:ok, Object.normalize(activity)}
       else
         {:error, {:reject, nil}} ->
           {:reject, nil}
@@ -952,7 +1000,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
 
           case OStatus.fetch_activity_from_url(id) do
-            {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])}
+            {:ok, [activity | _]} -> {:ok, Object.normalize(activity)}
             e -> e
           end
       end
diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
index 25d5f9cd3..e8dfba672 100644
--- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
@@ -4,6 +4,10 @@
 
 defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
   @behaviour Pleroma.Web.ActivityPub.MRF
+  defp string_matches?(string, _) when not is_binary(string) do
+    false
+  end
+
   defp string_matches?(string, pattern) when is_binary(pattern) do
     String.contains?(string, pattern)
   end
@@ -44,6 +48,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
   end
 
   defp check_replace(%{"object" => %{"content" => content, "summary" => summary}} = message) do
+    content =
+      if is_binary(content) do
+        content
+      else
+        ""
+      end
+
+    summary =
+      if is_binary(summary) do
+        summary
+      else
+        ""
+      end
+
     {content, summary} =
       Enum.reduce(
         Pleroma.Config.get([:mrf_keyword, :replace]),
@@ -61,11 +79,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
   end
 
   @impl true
-  def filter(%{"object" => %{"content" => nil}} = message) do
-    {:ok, message}
-  end
-
-  @impl true
   def filter(%{"type" => "Create", "object" => %{"content" => _content}} = message) do
     with {:ok, message} <- check_reject(message),
          {:ok, message} <- check_ftl_removal(message),
diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex
index 01fef71b9..a7a20ca37 100644
--- a/lib/pleroma/web/activity_pub/relay.ex
+++ b/lib/pleroma/web/activity_pub/relay.ex
@@ -41,7 +41,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do
 
   def publish(%Activity{data: %{"type" => "Create"}} = activity) do
     with %User{} = user <- get_actor(),
-         %Object{} = object <- Object.normalize(activity.data["object"]["id"]) do
+         %Object{} = object <- Object.normalize(activity) do
       ActivityPub.announce(user, object, nil, true, false)
     else
       e -> Logger.error("error: #{inspect(e)}")
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 8e4bf7b47..f733ae7e1 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -86,11 +86,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def fix_addressing_list(map, field) do
-    if is_binary(map[field]) do
-      map
-      |> Map.put(field, [map[field]])
-    else
-      map
+    cond do
+      is_binary(map[field]) ->
+        Map.put(map, field, [map[field]])
+
+      is_nil(map[field]) ->
+        Map.put(map, field, [])
+
+      true ->
+        map
     end
   end
 
@@ -128,13 +132,42 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> fix_explicit_addressing(explicit_mentions)
   end
 
+  # if as:Public is addressed, then make sure the followers collection is also addressed
+  # so that the activities will be delivered to local users.
+  def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collection) do
+    recipients = to ++ cc
+
+    if followers_collection not in recipients do
+      cond do
+        "https://www.w3.org/ns/activitystreams#Public" in cc ->
+          to = to ++ [followers_collection]
+          Map.put(object, "to", to)
+
+        "https://www.w3.org/ns/activitystreams#Public" in to ->
+          cc = cc ++ [followers_collection]
+          Map.put(object, "cc", cc)
+
+        true ->
+          object
+      end
+    else
+      object
+    end
+  end
+
+  def fix_implicit_addressing(object, _), do: object
+
   def fix_addressing(object) do
+    %User{} = user = User.get_or_fetch_by_ap_id(object["actor"])
+    followers_collection = User.ap_followers(user)
+
     object
     |> fix_addressing_list("to")
     |> fix_addressing_list("cc")
     |> fix_addressing_list("bto")
     |> fix_addressing_list("bcc")
     |> fix_explicit_addressing
+    |> fix_implicit_addressing(followers_collection)
   end
 
   def fix_actor(%{"attributedTo" => actor} = object) do
@@ -922,7 +955,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   defp strip_internal_tags(object), do: object
 
   defp user_upgrade_task(user) do
-    old_follower_address = User.ap_followers(user)
+    # we pass a fake user so that the followers collection is stripped away
+    old_follower_address = User.ap_followers(%User{nickname: user.nickname})
 
     q =
       from(
diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
index af317245f..2e9ffe41c 100644
--- a/lib/pleroma/web/activity_pub/utils.ex
+++ b/lib/pleroma/web/activity_pub/utils.ex
@@ -209,12 +209,12 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   """
   def insert_full_object(%{"object" => %{"type" => type} = object_data})
       when is_map(object_data) and type in @supported_object_types do
-    with {:ok, _} <- Object.create(object_data) do
-      :ok
+    with {:ok, object} <- Object.create(object_data) do
+      {:ok, object}
     end
   end
 
-  def insert_full_object(_), do: :ok
+  def insert_full_object(_), do: {:ok, nil}
 
   def update_object_in_activities(%{data: %{"id" => id}} = object) do
     # TODO
diff --git a/lib/pleroma/web/activity_pub/views/object_view.ex b/lib/pleroma/web/activity_pub/views/object_view.ex
index 84fa94e32..6028b773c 100644
--- a/lib/pleroma/web/activity_pub/views/object_view.ex
+++ b/lib/pleroma/web/activity_pub/views/object_view.ex
@@ -17,7 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
 
   def render("object.json", %{object: %Activity{data: %{"type" => "Create"}} = activity}) do
     base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
-    object = Object.normalize(activity.data["object"])
+    object = Object.normalize(activity)
 
     additional =
       Transmogrifier.prepare_object(activity.data)
@@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
 
   def render("object.json", %{object: %Activity{} = activity}) do
     base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
-    object = Object.normalize(activity.data["object"])
+    object = Object.normalize(activity)
 
     additional =
       Transmogrifier.prepare_object(activity.data)
diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex
index 3d00dcbf2..5926a3294 100644
--- a/lib/pleroma/web/activity_pub/views/user_view.ex
+++ b/lib/pleroma/web/activity_pub/views/user_view.ex
@@ -87,16 +87,10 @@ defmodule Pleroma.Web.ActivityPub.UserView do
         "publicKeyPem" => public_key
       },
       "endpoints" => endpoints,
-      "icon" => %{
-        "type" => "Image",
-        "url" => User.avatar_url(user)
-      },
-      "image" => %{
-        "type" => "Image",
-        "url" => User.banner_url(user)
-      },
       "tag" => user.info.source_data["tag"] || []
     }
+    |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
+    |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user))
     |> Map.merge(Utils.make_json_ld_header())
   end
 
@@ -294,4 +288,17 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       map
     end
   end
+
+  defp maybe_make_image(func, key, user) do
+    if image = func.(user, no_default: true) do
+      %{
+        key => %{
+          "type" => "Image",
+          "url" => image
+        }
+      }
+    else
+      %{}
+    end
+  end
 end
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index 6d9bf2895..b3a09e49e 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -3,17 +3,18 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.AdminAPI.AdminAPIController do
-  @users_page_size 50
-
   use Pleroma.Web, :controller
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.AdminAPI.AccountView
+  alias Pleroma.Web.AdminAPI.Search
 
   import Pleroma.Web.ControllerHelper, only: [json_response: 3]
 
   require Logger
 
+  @users_page_size 50
+
   action_fallback(:errors)
 
   def user_delete(conn, %{"nickname" => nickname}) do
@@ -44,6 +45,15 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     |> json(user.nickname)
   end
 
+  def user_show(conn, %{"nickname" => nickname}) do
+    with %User{} = user <- User.get_by_nickname(nickname) do
+      conn
+      |> json(AccountView.render("show.json", %{user: user}))
+    else
+      _ -> {:error, :not_found}
+    end
+  end
+
   def user_toggle_activation(conn, %{"nickname" => nickname}) do
     user = User.get_by_nickname(nickname)
 
@@ -63,17 +73,17 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
          do: json_response(conn, :no_content, "")
   end
 
-  def list_users(%{assigns: %{user: admin}} = conn, params) do
+  def list_users(conn, params) do
     {page, page_size} = page_params(params)
+    filters = maybe_parse_filters(params["filters"])
 
-    with {:ok, users, count} <-
-           User.search_for_admin(%{
-             query: params["query"],
-             admin: admin,
-             local: params["local_only"] == "true",
-             page: page,
-             page_size: page_size
-           }),
+    search_params = %{
+      query: params["query"],
+      page: page,
+      page_size: page_size
+    }
+
+    with {:ok, users, count} <- Search.user(Map.merge(search_params, filters)),
          do:
            conn
            |> json(
@@ -85,6 +95,19 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
            )
   end
 
+  @filters ~w(local external active deactivated)
+
+  defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
+
+  @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{}
+  defp maybe_parse_filters(filters) do
+    filters
+    |> String.split(",")
+    |> Enum.filter(&Enum.member?(@filters, &1))
+    |> Enum.map(&String.to_atom(&1))
+    |> Enum.into(%{}, &{&1, true})
+  end
+
   def right_add(conn, %{"permission_group" => permission_group, "nickname" => nickname})
       when permission_group in ["moderator", "admin"] do
     user = User.get_by_nickname(nickname)
@@ -217,6 +240,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     |> json(token.token)
   end
 
+  def errors(conn, {:error, :not_found}) do
+    conn
+    |> put_status(404)
+    |> json("Not found")
+  end
+
   def errors(conn, {:param_cast, _}) do
     conn
     |> put_status(400)
diff --git a/lib/pleroma/web/admin_api/search.ex b/lib/pleroma/web/admin_api/search.ex
new file mode 100644
index 000000000..9a8e41c2a
--- /dev/null
+++ b/lib/pleroma/web/admin_api/search.ex
@@ -0,0 +1,54 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.Search do
+  import Ecto.Query
+
+  alias Pleroma.Repo
+  alias Pleroma.User
+
+  @page_size 50
+
+  def user(%{query: term} = params) when is_nil(term) or term == "" do
+    query = maybe_filtered_query(params)
+
+    paginated_query =
+      maybe_filtered_query(params)
+      |> paginate(params[:page] || 1, params[:page_size] || @page_size)
+
+    count = query |> Repo.aggregate(:count, :id)
+
+    results = Repo.all(paginated_query)
+
+    {:ok, results, count}
+  end
+
+  def user(%{query: term} = params) when is_binary(term) do
+    search_query = from(u in maybe_filtered_query(params), where: ilike(u.nickname, ^"%#{term}%"))
+
+    count = search_query |> Repo.aggregate(:count, :id)
+
+    results =
+      search_query
+      |> paginate(params[:page] || 1, params[:page_size] || @page_size)
+      |> Repo.all()
+
+    {:ok, results, count}
+  end
+
+  defp maybe_filtered_query(params) do
+    from(u in User, order_by: u.nickname)
+    |> User.maybe_local_user_query(params[:local])
+    |> User.maybe_external_user_query(params[:external])
+    |> User.maybe_active_user_query(params[:active])
+    |> User.maybe_deactivated_user_query(params[:deactivated])
+  end
+
+  defp paginate(query, page, page_size) do
+    from(u in query,
+      limit: ^page_size,
+      offset: ^((page - 1) * page_size)
+    )
+  end
+end
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index b5f79c3bf..25b990677 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -6,7 +6,6 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.Activity
   alias Pleroma.Formatter
   alias Pleroma.Object
-  alias Pleroma.Repo
   alias Pleroma.ThreadMute
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -64,8 +63,9 @@ defmodule Pleroma.Web.CommonAPI do
   end
 
   def delete(activity_id, user) do
-    with %Activity{data: %{"object" => %{"id" => object_id}}} <- Repo.get(Activity, activity_id),
-         %Object{} = object <- Object.normalize(object_id),
+    with %Activity{data: %{"object" => _}} = activity <-
+           Activity.get_by_id_with_object(activity_id),
+         %Object{} = object <- Object.normalize(activity),
          true <- User.superuser?(user) || user.ap_id == object.data["actor"],
          {:ok, _} <- unpin(activity_id, user),
          {:ok, delete} <- ActivityPub.delete(object) do
@@ -75,7 +75,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def repeat(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]),
+         object <- Object.normalize(activity),
          nil <- Utils.get_existing_announce(user.ap_id, object) do
       ActivityPub.announce(user, object)
     else
@@ -86,7 +86,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def unrepeat(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]) do
+         object <- Object.normalize(activity) do
       ActivityPub.unannounce(user, object)
     else
       _ ->
@@ -96,7 +96,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def favorite(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]),
+         object <- Object.normalize(activity),
          nil <- Utils.get_existing_like(user.ap_id, object) do
       ActivityPub.like(user, object)
     else
@@ -107,7 +107,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def unfavorite(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]) do
+         object <- Object.normalize(activity) do
       ActivityPub.unlike(user, object)
     else
       _ ->
@@ -142,7 +142,8 @@ defmodule Pleroma.Web.CommonAPI do
            make_content_html(
              status,
              attachments,
-             data
+             data,
+             visibility
            ),
          {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility),
          context <- make_context(in_reply_to),
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index b7513ef28..f596f703b 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -17,13 +17,14 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   # This is a hack for twidere.
   def get_by_id_or_ap_id(id) do
-    activity = Repo.get(Activity, id) || Activity.get_create_by_object_ap_id(id)
+    activity =
+      Activity.get_by_id_with_object(id) || Activity.get_create_by_object_ap_id_with_object(id)
 
     activity &&
       if activity.data["type"] == "Create" do
         activity
       else
-        Activity.get_create_by_object_ap_id(activity.data["object"])
+        Activity.get_create_by_object_ap_id_with_object(activity.data["object"])
       end
   end
 
@@ -101,7 +102,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   def make_content_html(
         status,
         attachments,
-        data
+        data,
+        visibility
       ) do
     no_attachment_links =
       data
@@ -110,8 +112,15 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
     content_type = get_content_type(data["content_type"])
 
+    options =
+      if visibility == "direct" && Config.get([:instance, :safe_dm_mentions]) do
+        [safe_mention: true]
+      else
+        []
+      end
+
     status
-    |> format_input(content_type)
+    |> format_input(content_type, options)
     |> maybe_add_attachments(attachments, no_attachment_links)
     |> maybe_add_nsfw_tag(data)
   end
@@ -294,10 +303,10 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   def maybe_notify_mentioned_recipients(
         recipients,
-        %Activity{data: %{"to" => _to, "type" => type} = data} = _activity
+        %Activity{data: %{"to" => _to, "type" => type} = data} = activity
       )
       when type == "Create" do
-    object = Object.normalize(data["object"])
+    object = Object.normalize(activity)
 
     object_data =
       cond do
@@ -344,4 +353,33 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   end
 
   def get_report_statuses(_, _), do: {:ok, nil}
+
+  # DEPRECATED mostly, context objects are now created at insertion time.
+  def context_to_conversation_id(context) do
+    with %Object{id: id} <- Object.get_cached_by_ap_id(context) do
+      id
+    else
+      _e ->
+        changeset = Object.context_mapping(context)
+
+        case Repo.insert(changeset) do
+          {:ok, %{id: id}} ->
+            id
+
+          # This should be solved by an upsert, but it seems ecto
+          # has problems accessing the constraint inside the jsonb.
+          {:error, _} ->
+            Object.get_cached_by_ap_id(context).id
+        end
+    end
+  end
+
+  def conversation_id_to_context(id) do
+    with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do
+      context
+    else
+      _e ->
+        {:error, "No such conversation"}
+    end
+  end
 end
diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index 5e690ddb8..c47328e13 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -4,7 +4,6 @@
 
 defmodule Pleroma.Web.Federator do
   alias Pleroma.Activity
-  alias Pleroma.Jobs
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
@@ -31,39 +30,39 @@ defmodule Pleroma.Web.Federator do
   # Client API
 
   def incoming_doc(doc) do
-    Jobs.enqueue(:federator_incoming, __MODULE__, [:incoming_doc, doc])
+    PleromaJobQueue.enqueue(:federator_incoming, __MODULE__, [:incoming_doc, doc])
   end
 
   def incoming_ap_doc(params) do
-    Jobs.enqueue(:federator_incoming, __MODULE__, [:incoming_ap_doc, params])
+    PleromaJobQueue.enqueue(:federator_incoming, __MODULE__, [:incoming_ap_doc, params])
   end
 
   def publish(activity, priority \\ 1) do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish, activity], priority)
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish, activity], priority)
   end
 
   def publish_single_ap(params) do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish_single_ap, params])
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_single_ap, params])
   end
 
   def publish_single_websub(websub) do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish_single_websub, websub])
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_single_websub, websub])
   end
 
   def verify_websub(websub) do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:verify_websub, websub])
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:verify_websub, websub])
   end
 
   def request_subscription(sub) do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:request_subscription, sub])
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:request_subscription, sub])
   end
 
   def refresh_subscriptions do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:refresh_subscriptions])
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:refresh_subscriptions])
   end
 
   def publish_single_salmon(params) do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish_single_salmon, params])
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_single_salmon, params])
   end
 
   # Job Worker Callbacks
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex
index 54cb6c97a..08ea5f967 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex
@@ -2,61 +2,49 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do
   import Ecto.Query
   import Ecto.Changeset
 
-  alias Pleroma.Repo
+  alias Pleroma.Activity
+  alias Pleroma.Notification
+  alias Pleroma.Pagination
   alias Pleroma.User
 
-  @default_limit 20
-
   def get_followers(user, params \\ %{}) do
     user
     |> User.get_followers_query()
-    |> paginate(params)
-    |> Repo.all()
+    |> Pagination.fetch_paginated(params)
   end
 
   def get_friends(user, params \\ %{}) do
     user
     |> User.get_friends_query()
-    |> paginate(params)
-    |> Repo.all()
+    |> Pagination.fetch_paginated(params)
   end
 
-  def paginate(query, params \\ %{}) do
+  def get_notifications(user, params \\ %{}) do
     options = cast_params(params)
 
-    query
-    |> restrict(:max_id, options)
-    |> restrict(:since_id, options)
-    |> restrict(:limit, options)
-    |> order_by([u], fragment("? desc nulls last", u.id))
+    user
+    |> Notification.for_user_query()
+    |> restrict(:exclude_types, options)
+    |> Pagination.fetch_paginated(params)
   end
 
-  def cast_params(params) do
+  defp cast_params(params) do
     param_types = %{
-      max_id: :string,
-      since_id: :string,
-      limit: :integer
+      exclude_types: {:array, :string}
     }
 
     changeset = cast({%{}, param_types}, params, Map.keys(param_types))
     changeset.changes
   end
 
-  defp restrict(query, :max_id, %{max_id: max_id}) do
-    query
-    |> where([q], q.id < ^max_id)
-  end
-
-  defp restrict(query, :since_id, %{since_id: since_id}) do
-    query
-    |> where([q], q.id > ^since_id)
-  end
-
-  defp restrict(query, :limit, options) do
-    limit = Map.get(options, :limit, @default_limit)
+  defp restrict(query, :exclude_types, %{exclude_types: mastodon_types = [_ | _]}) do
+    ap_types =
+      mastodon_types
+      |> Enum.map(&Activity.from_mastodon_notification_type/1)
+      |> Enum.filter(& &1)
 
     query
-    |> limit(^limit)
+    |> where([q, a], not fragment("? @> ARRAY[?->>'type']::varchar[]", ^ap_types, a.data))
   end
 
   defp restrict(query, _, _), do: query
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 952aa2453..eee4e7678 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -18,6 +18,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.AccountView
+  alias Pleroma.Web.MastodonAPI.AppView
   alias Pleroma.Web.MastodonAPI.FilterView
   alias Pleroma.Web.MastodonAPI.ListView
   alias Pleroma.Web.MastodonAPI.MastodonAPI
@@ -51,16 +52,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     with cs <- App.register_changeset(%App{}, app_attrs),
          false <- cs.changes[:client_name] == @local_mastodon_name,
          {:ok, app} <- Repo.insert(cs) do
-      res = %{
-        id: app.id |> to_string,
-        name: app.client_name,
-        client_id: app.client_id,
-        client_secret: app.client_secret,
-        redirect_uri: app.redirect_uris,
-        website: app.website
-      }
-
-      json(conn, res)
+      conn
+      |> put_view(AppView)
+      |> render("show.json", %{app: app})
     end
   end
 
@@ -132,6 +126,14 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     json(conn, account)
   end
 
+  def verify_app_credentials(%{assigns: %{user: _user, token: token}} = conn, _) do
+    with %Token{app: %App{} = app} <- Repo.preload(token, :app) do
+      conn
+      |> put_view(AppView)
+      |> render("short.json", %{app: app})
+    end
+  end
+
   def user(%{assigns: %{user: for_user}} = conn, %{"id" => nickname_or_id}) do
     with %User{} = user <- User.get_cached_by_nickname_or_id(nickname_or_id),
          true <- User.auth_active?(user) || user.id == for_user.id || User.superuser?(for_user) do
@@ -161,6 +163,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       },
       stats: Stats.get_stats(),
       thumbnail: Web.base_url() <> "/instance/thumbnail.jpeg",
+      languages: ["en"],
+      registrations: Pleroma.Config.get([:instance, :registrations_open]),
+      # Extra (not present in Mastodon):
       max_toot_chars: Keyword.get(instance, :limit)
     }
 
@@ -502,7 +507,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def notifications(%{assigns: %{user: user}} = conn, params) do
-    notifications = Notification.for_user(user, params)
+    notifications = MastodonAPI.get_notifications(user, params)
 
     conn
     |> add_link_headers(:notifications, notifications)
@@ -944,12 +949,14 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def favourites(%{assigns: %{user: user}} = conn, params) do
-    activities =
+    params =
       params
       |> Map.put("type", "Create")
       |> Map.put("favorited_by", user.ap_id)
       |> Map.put("blocking_user", user)
-      |> ActivityPub.fetch_public_activities()
+
+    activities =
+      ActivityPub.fetch_activities([], params)
       |> Enum.reverse()
 
     conn
diff --git a/lib/pleroma/web/mastodon_api/views/app_view.ex b/lib/pleroma/web/mastodon_api/views/app_view.ex
new file mode 100644
index 000000000..f52b693a6
--- /dev/null
+++ b/lib/pleroma/web/mastodon_api/views/app_view.ex
@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MastodonAPI.AppView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.Web.OAuth.App
+
+  @vapid_key :web_push_encryption
+             |> Application.get_env(:vapid_details, [])
+             |> Keyword.get(:public_key)
+
+  def render("show.json", %{app: %App{} = app}) do
+    %{
+      id: app.id |> to_string,
+      name: app.client_name,
+      client_id: app.client_id,
+      client_secret: app.client_secret,
+      redirect_uri: app.redirect_uris,
+      website: app.website
+    }
+    |> with_vapid_key()
+  end
+
+  def render("short.json", %{app: %App{website: webiste, client_name: name}}) do
+    %{
+      name: name,
+      website: webiste
+    }
+    |> with_vapid_key()
+  end
+
+  defp with_vapid_key(data) do
+    if @vapid_key do
+      Map.put(data, "vapid_key", @vapid_key)
+    else
+      data
+    end
+  end
+end
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 209119dd5..200bb453d 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -46,6 +46,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     end
   end
 
+  defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id),
+    do: context_id
+
+  defp get_context_id(%{data: %{"context" => context}}) when is_binary(context),
+    do: Utils.context_to_conversation_id(context)
+
+  defp get_context_id(_), do: nil
+
   def render("index.json", opts) do
     replied_to_activities = get_replied_to_activities(opts.activities)
 
@@ -166,7 +174,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       content: content,
       created_at: created_at,
       reblogs_count: announcement_count,
-      replies_count: 0,
+      replies_count: object["repliesCount"] || 0,
       favourites_count: like_count,
       reblogged: present?(repeated),
       favourited: present?(favorited),
@@ -186,7 +194,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       language: nil,
       emojis: build_emojis(activity.data["object"]["emoji"]),
       pleroma: %{
-        local: activity.local
+        local: activity.local,
+        conversation_id: get_context_id(activity)
       }
     }
   end
diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
index 8c775ce24..216a962bd 100644
--- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
+++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
@@ -124,6 +124,9 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
         end,
         if Keyword.get(instance, :allow_relay) do
           "relay"
+        end,
+        if Keyword.get(instance, :safe_dm_mentions) do
+          "safe_dm_mentions"
         end
       ]
       |> Enum.filter(& &1)
diff --git a/lib/pleroma/web/oauth/authorization.ex b/lib/pleroma/web/oauth/authorization.ex
index a80543adf..3461f9983 100644
--- a/lib/pleroma/web/oauth/authorization.ex
+++ b/lib/pleroma/web/oauth/authorization.ex
@@ -16,7 +16,7 @@ defmodule Pleroma.Web.OAuth.Authorization do
   schema "oauth_authorizations" do
     field(:token, :string)
     field(:scopes, {:array, :string}, default: [])
-    field(:valid_until, :naive_datetime)
+    field(:valid_until, :naive_datetime_usec)
     field(:used, :boolean, default: false)
     belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index d151efe9e..ebb3dd253 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -83,14 +83,18 @@ defmodule Pleroma.Web.OAuth.OAuthController do
       end
     else
       {scopes_issue, _} when scopes_issue in [:unsupported_scopes, :missing_scopes] ->
+        # Per https://github.com/tootsuite/mastodon/blob/
+        #   51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L39
         conn
-        |> put_flash(:error, "Permissions not specified.")
+        |> put_flash(:error, "This action is outside the authorized scopes")
         |> put_status(:unauthorized)
         |> authorize(auth_params)
 
       {:auth_active, false} ->
+        # Per https://github.com/tootsuite/mastodon/blob/
+        #   51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L76
         conn
-        |> put_flash(:error, "Account confirmation pending.")
+        |> put_flash(:error, "Your login is missing a confirmed e-mail address")
         |> put_status(:forbidden)
         |> authorize(auth_params)
 
@@ -149,9 +153,11 @@ defmodule Pleroma.Web.OAuth.OAuthController do
       json(conn, response)
     else
       {:auth_active, false} ->
+        # Per https://github.com/tootsuite/mastodon/blob/
+        #   51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L76
         conn
         |> put_status(:forbidden)
-        |> json(%{error: "Account confirmation pending"})
+        |> json(%{error: "Your login is missing a confirmed e-mail address"})
 
       _error ->
         put_status(conn, 400)
diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex
index 2b074b470..a8b06db36 100644
--- a/lib/pleroma/web/oauth/token.ex
+++ b/lib/pleroma/web/oauth/token.ex
@@ -17,7 +17,7 @@ defmodule Pleroma.Web.OAuth.Token do
     field(:token, :string)
     field(:refresh_token, :string)
     field(:scopes, {:array, :string}, default: [])
-    field(:valid_until, :naive_datetime)
+    field(:valid_until, :naive_datetime_usec)
     belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex
index 770a71a0a..db995ec77 100644
--- a/lib/pleroma/web/ostatus/handlers/note_handler.ex
+++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex
@@ -106,7 +106,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do
   # TODO: Clean this up a bit.
   def handle_note(entry, doc \\ nil) do
     with id <- XML.string_from_xpath("//id", entry),
-         activity when is_nil(activity) <- Activity.get_create_by_object_ap_id(id),
+         activity when is_nil(activity) <- Activity.get_create_by_object_ap_id_with_object(id),
          [author] <- :xmerl_xpath.string('//author[1]', doc),
          {:ok, actor} <- OStatus.find_make_or_update_user(author),
          content_html <- OStatus.get_content(entry),
diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex
index 266f86bf4..9a34d7ad5 100644
--- a/lib/pleroma/web/ostatus/ostatus.ex
+++ b/lib/pleroma/web/ostatus/ostatus.ex
@@ -23,8 +23,8 @@ defmodule Pleroma.Web.OStatus do
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Websub
 
-  def is_representable?(%Activity{data: data}) do
-    object = Object.normalize(data["object"])
+  def is_representable?(%Activity{} = activity) do
+    object = Object.normalize(activity)
 
     cond do
       is_nil(object) ->
@@ -119,7 +119,7 @@ defmodule Pleroma.Web.OStatus do
 
   def make_share(entry, doc, retweeted_activity) do
     with {:ok, actor} <- find_make_or_update_user(doc),
-         %Object{} = object <- Object.normalize(retweeted_activity.data["object"]),
+         %Object{} = object <- Object.normalize(retweeted_activity),
          id when not is_nil(id) <- string_from_xpath("/entry/id", entry),
          {:ok, activity, _object} = ActivityPub.announce(actor, object, id, false) do
       {:ok, activity}
@@ -137,7 +137,7 @@ defmodule Pleroma.Web.OStatus do
 
   def make_favorite(entry, doc, favorited_activity) do
     with {:ok, actor} <- find_make_or_update_user(doc),
-         %Object{} = object <- Object.normalize(favorited_activity.data["object"]),
+         %Object{} = object <- Object.normalize(favorited_activity),
          id when not is_nil(id) <- string_from_xpath("/entry/id", entry),
          {:ok, activity, _object} = ActivityPub.like(actor, object, id, false) do
       {:ok, activity}
@@ -159,7 +159,7 @@ defmodule Pleroma.Web.OStatus do
     Logger.debug("Trying to get entry from db")
 
     with id when not is_nil(id) <- string_from_xpath("//activity:object[1]/id", entry),
-         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id_with_object(id) do
       {:ok, activity}
     else
       _ ->
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index 0579a5f3d..2fb6ce41b 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -102,7 +102,8 @@ defmodule Pleroma.Web.OStatus.OStatusController do
       ActivityPubController.call(conn, :object)
     else
       with id <- o_status_url(conn, :object, uuid),
-           {_, %Activity{} = activity} <- {:activity, Activity.get_create_by_object_ap_id(id)},
+           {_, %Activity{} = activity} <-
+             {:activity, Activity.get_create_by_object_ap_id_with_object(id)},
            {_, true} <- {:public?, Visibility.is_public?(activity)},
            %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
         case get_format(conn) do
@@ -148,13 +149,13 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   end
 
   def notice(conn, %{"id" => id}) do
-    with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id(id)},
+    with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)},
          {_, true} <- {:public?, Visibility.is_public?(activity)},
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       case format = get_format(conn) do
         "html" ->
           if activity.data["type"] == "Create" do
-            %Object{} = object = Object.normalize(activity.data["object"])
+            %Object{} = object = Object.normalize(activity)
 
             Fallback.RedirectController.redirector_with_meta(conn, %{
               activity_id: activity.id,
@@ -191,9 +192,9 @@ defmodule Pleroma.Web.OStatus.OStatusController do
 
   # Returns an HTML embedded <audio> or <video> player suitable for embed iframes.
   def notice_player(conn, %{"id" => id}) do
-    with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id),
+    with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id_with_object(id),
          true <- Visibility.is_public?(activity),
-         %Object{} = object <- Object.normalize(activity.data["object"]),
+         %Object{} = object <- Object.normalize(activity),
          %{data: %{"attachment" => [%{"url" => [url | _]} | _]}} <- object,
          true <- String.starts_with?(url["mediaType"], ["audio", "video"]) do
       conn
@@ -219,7 +220,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
          %Activity{data: %{"type" => "Create"}} = activity,
          _user
        ) do
-    object = Object.normalize(activity.data["object"])
+    object = Object.normalize(activity)
 
     conn
     |> put_resp_header("content-type", "application/activity+json")
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 92c61ff51..f67aaf58b 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -21,9 +21,9 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   defp validate_page_url(%URI{}), do: :ok
   defp validate_page_url(_), do: :error
 
-  def fetch_data_for_activity(%Activity{} = activity) do
+  def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
     with true <- Pleroma.Config.get([:rich_media, :enabled]),
-         %Object{} = object <- Object.normalize(activity.data["object"]),
+         %Object{} = object <- Object.normalize(activity),
          {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),
          :ok <- validate_page_url(page_url),
          {:ok, rich_media} <- Parser.parse(page_url) do
@@ -32,4 +32,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do
       _ -> %{}
     end
   end
+
+  def fetch_data_for_activity(_), do: %{}
 end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 6d000b7f5..64e1a2bd8 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -140,6 +140,7 @@ defmodule Pleroma.Web.Router do
     pipe_through([:admin_api, :oauth_write])
 
     get("/users", AdminAPIController, :list_users)
+    get("/users/:nickname", AdminAPIController, :user_show)
     delete("/user", AdminAPIController, :user_delete)
     patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation)
     post("/user", AdminAPIController, :user_create)
@@ -217,6 +218,7 @@ defmodule Pleroma.Web.Router do
       get("/accounts/search", MastodonAPIController, :account_search)
 
       get("/accounts/:id/lists", MastodonAPIController, :account_lists)
+      get("/accounts/:id/identity_proofs", MastodonAPIController, :empty_array)
 
       get("/follow_requests", MastodonAPIController, :follow_requests)
       get("/blocks", MastodonAPIController, :blocks)
@@ -331,6 +333,7 @@ defmodule Pleroma.Web.Router do
     get("/instance", MastodonAPIController, :masto_instance)
     get("/instance/peers", MastodonAPIController, :peers)
     post("/apps", MastodonAPIController, :create_app)
+    get("/apps/verify_credentials", MastodonAPIController, :verify_app_credentials)
     get("/custom_emojis", MastodonAPIController, :custom_emojis)
 
     get("/statuses/:id/card", MastodonAPIController, :status_card)
diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex
index 7425bfb54..592749b42 100644
--- a/lib/pleroma/web/streamer.ex
+++ b/lib/pleroma/web/streamer.ex
@@ -202,7 +202,7 @@ defmodule Pleroma.Web.Streamer do
         mutes = user.info.mutes || []
         reblog_mutes = user.info.muted_reblogs || []
 
-        parent = Object.normalize(item.data["object"])
+        parent = Object.normalize(item)
 
         unless is_nil(parent) or item.actor in blocks or item.actor in mutes or
                  item.actor in reblog_mutes or not ActivityPub.contain_activity(item, user) or
diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
index 320ec778c..faa733fec 100644
--- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex
+++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
@@ -197,7 +197,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
           vapidPublicKey: vapid_public_key,
           accountActivationRequired:
             if(Keyword.get(instance, :account_activation_required, false), do: "1", else: "0"),
-          invitesEnabled: if(Keyword.get(instance, :invites_enabled, false), do: "1", else: "0")
+          invitesEnabled: if(Keyword.get(instance, :invites_enabled, false), do: "1", else: "0"),
+          safeDMMentionsEnabled:
+            if(Pleroma.Config.get([:instance, :safe_dm_mentions]), do: "1", else: "0")
         }
 
         pleroma_fe =
diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
deleted file mode 100644
index 55c612ddd..000000000
--- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex
+++ /dev/null
@@ -1,15 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-# FIXME: Remove this module?
-# THIS MODULE IS DEPRECATED! DON'T USE IT!
-# USE THE Pleroma.Web.TwitterAPI.Views.ActivityView MODULE!
-defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
-  def to_map(activity, opts) do
-    Pleroma.Web.TwitterAPI.ActivityView.render(
-      "activity.json",
-      Map.put(opts, :activity, activity)
-    )
-  end
-end
diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex
index d57100491..9978c7f64 100644
--- a/lib/pleroma/web/twitter_api/twitter_api.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api.ex
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
   alias Pleroma.Activity
   alias Pleroma.Mailer
-  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.UserEmail
@@ -282,35 +281,6 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
     _activities = Repo.all(q)
   end
 
-  # DEPRECATED mostly, context objects are now created at insertion time.
-  def context_to_conversation_id(context) do
-    with %Object{id: id} <- Object.get_cached_by_ap_id(context) do
-      id
-    else
-      _e ->
-        changeset = Object.context_mapping(context)
-
-        case Repo.insert(changeset) do
-          {:ok, %{id: id}} ->
-            id
-
-          # This should be solved by an upsert, but it seems ecto
-          # has problems accessing the constraint inside the jsonb.
-          {:error, _} ->
-            Object.get_cached_by_ap_id(context).id
-        end
-    end
-  end
-
-  def conversation_id_to_context(id) do
-    with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do
-      context
-    else
-      _e ->
-        {:error, "No such conversation"}
-    end
-  end
-
   def get_external_profile(for_user, uri) do
     with %User{} = user <- User.get_or_fetch(uri) do
       {:ok, UserView.render("show.json", %{user: user, for: for_user})}
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index 6ea0b110b..62cce18dc 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -16,6 +16,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.TwitterAPI.ActivityView
   alias Pleroma.Web.TwitterAPI.NotificationView
@@ -278,7 +279,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   end
 
   def fetch_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with context when is_binary(context) <- TwitterAPI.conversation_id_to_context(id),
+    with context when is_binary(context) <- Utils.conversation_id_to_context(id),
          activities <-
            ActivityPub.fetch_activities_for_context(context, %{
              "blocking_user" => user,
diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex
index 4926f007e..aa1d41fa2 100644
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@@ -15,7 +15,6 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.TwitterAPI.ActivityView
   alias Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter
-  alias Pleroma.Web.TwitterAPI.TwitterAPI
   alias Pleroma.Web.TwitterAPI.UserView
 
   import Ecto.Query
@@ -78,7 +77,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
   defp get_context_id(%{data: %{"context" => context}}, options) do
     cond do
       id = options[:context_ids][context] -> id
-      true -> TwitterAPI.context_to_conversation_id(context)
+      true -> Utils.context_to_conversation_id(context)
     end
   end
 
@@ -267,6 +266,8 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
         content
         |> String.replace(~r/<br\s?\/?>/, "\n")
         |> HTML.get_cached_stripped_html_for_object(activity, __MODULE__)
+      else
+        ""
       end
 
     reply_parent = Activity.get_in_reply_to_activity(activity)
diff --git a/lib/pleroma/web/websub/websub_client_subscription.ex b/lib/pleroma/web/websub/websub_client_subscription.ex
index 969ee0684..77703c496 100644
--- a/lib/pleroma/web/websub/websub_client_subscription.ex
+++ b/lib/pleroma/web/websub/websub_client_subscription.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do
   schema "websub_client_subscriptions" do
     field(:topic, :string)
     field(:secret, :string)
-    field(:valid_until, :naive_datetime)
+    field(:valid_until, :naive_datetime_usec)
     field(:state, :string)
     field(:subscribers, {:array, :string}, default: [])
     field(:hub, :string)