Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into feature/jobs

# Conflicts:
#	lib/pleroma/web/activity_pub/activity_pub.ex
#	lib/pleroma/web/federator/federator.ex

27 files changed, 492 insertions, 101 deletions

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index e44c48c35..d2523c045 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -6,11 +6,13 @@ defmodule Pleroma.Application do
   use Application
   import Supervisor.Spec
 
-  @name "Pleroma"
+  @name Mix.Project.config()[:name]
   @version Mix.Project.config()[:version]
+  @repository Mix.Project.config()[:source_url]
   def name, do: @name
   def version, do: @version
   def named_version(), do: @name <> " " <> @version
+  def repository, do: @repository
 
   def user_agent() do
     info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex
index dc50682ee..0eb1833aa 100644
--- a/lib/pleroma/config/deprecation_warnings.ex
+++ b/lib/pleroma/config/deprecation_warnings.ex
@@ -12,6 +12,13 @@ defmodule Pleroma.Config.DeprecationWarnings do
       You are using the old configuration mechanism for the frontend. Please check config.md.
       """)
     end
+
+    if Pleroma.Config.get(:mrf_hellthread, :threshold) do
+      Logger.warn("""
+      !!!DEPRECATION WARNING!!!
+      You are using the old configuration mechanism for the hellthread filter. Please check config.md.
+      """)
+    end
   end
 
   def warn do
diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex
new file mode 100644
index 000000000..5e107f4c9
--- /dev/null
+++ b/lib/pleroma/instances.ex
@@ -0,0 +1,36 @@
+defmodule Pleroma.Instances do
+  @moduledoc "Instances context."
+
+  @adapter Pleroma.Instances.Instance
+
+  defdelegate filter_reachable(urls_or_hosts), to: @adapter
+  defdelegate reachable?(url_or_host), to: @adapter
+  defdelegate set_reachable(url_or_host), to: @adapter
+  defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter
+
+  def set_consistently_unreachable(url_or_host),
+    do: set_unreachable(url_or_host, reachability_datetime_threshold())
+
+  def reachability_datetime_threshold do
+    federation_reachability_timeout_days =
+      Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 0
+
+    if federation_reachability_timeout_days > 0 do
+      NaiveDateTime.add(
+        NaiveDateTime.utc_now(),
+        -federation_reachability_timeout_days * 24 * 3600,
+        :second
+      )
+    else
+      ~N[0000-01-01 00:00:00]
+    end
+  end
+
+  def host(url_or_host) when is_binary(url_or_host) do
+    if url_or_host =~ ~r/^http/i do
+      URI.parse(url_or_host).host
+    else
+      url_or_host
+    end
+  end
+end
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
new file mode 100644
index 000000000..4a4ca26dd
--- /dev/null
+++ b/lib/pleroma/instances/instance.ex
@@ -0,0 +1,113 @@
+defmodule Pleroma.Instances.Instance do
+  @moduledoc "Instance."
+
+  alias Pleroma.Instances
+  alias Pleroma.Instances.Instance
+
+  use Ecto.Schema
+
+  import Ecto.{Query, Changeset}
+
+  alias Pleroma.Repo
+
+  schema "instances" do
+    field(:host, :string)
+    field(:unreachable_since, :naive_datetime)
+
+    timestamps()
+  end
+
+  defdelegate host(url_or_host), to: Instances
+
+  def changeset(struct, params \\ %{}) do
+    struct
+    |> cast(params, [:host, :unreachable_since])
+    |> validate_required([:host])
+    |> unique_constraint(:host)
+  end
+
+  def filter_reachable([]), do: %{}
+
+  def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
+    hosts =
+      urls_or_hosts
+      |> Enum.map(&(&1 && host(&1)))
+      |> Enum.filter(&(to_string(&1) != ""))
+
+    unreachable_since_by_host =
+      Repo.all(
+        from(i in Instance,
+          where: i.host in ^hosts,
+          select: {i.host, i.unreachable_since}
+        )
+      )
+      |> Map.new(& &1)
+
+    reachability_datetime_threshold = Instances.reachability_datetime_threshold()
+
+    for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
+      host = host(entry)
+      unreachable_since = unreachable_since_by_host[host]
+
+      if !unreachable_since ||
+           NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do
+        {entry, unreachable_since}
+      end
+    end
+    |> Enum.filter(& &1)
+    |> Map.new(& &1)
+  end
+
+  def reachable?(url_or_host) when is_binary(url_or_host) do
+    !Repo.one(
+      from(i in Instance,
+        where:
+          i.host == ^host(url_or_host) and
+            i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
+        select: true
+      )
+    )
+  end
+
+  def reachable?(_), do: true
+
+  def set_reachable(url_or_host) when is_binary(url_or_host) do
+    with host <- host(url_or_host),
+         %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
+      {:ok, _instance} =
+        existing_record
+        |> changeset(%{unreachable_since: nil})
+        |> Repo.update()
+    end
+  end
+
+  def set_reachable(_), do: {:error, nil}
+
+  def set_unreachable(url_or_host, unreachable_since \\ nil)
+
+  def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do
+    unreachable_since = unreachable_since || DateTime.utc_now()
+    host = host(url_or_host)
+    existing_record = Repo.get_by(Instance, %{host: host})
+
+    changes = %{unreachable_since: unreachable_since}
+
+    cond do
+      is_nil(existing_record) ->
+        %Instance{}
+        |> changeset(Map.put(changes, :host, host))
+        |> Repo.insert()
+
+      existing_record.unreachable_since &&
+          NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
+        {:ok, existing_record}
+
+      true ->
+        existing_record
+        |> changeset(changes)
+        |> Repo.update()
+    end
+  end
+
+  def set_unreachable(_, _), do: {:error, nil}
+end
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index 707a61f14..7b46a3b05 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -31,8 +31,8 @@ defmodule Pleroma.Object do
     Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
   end
 
-  def normalize(obj) when is_map(obj), do: Object.get_by_ap_id(obj["id"])
-  def normalize(ap_id) when is_binary(ap_id), do: Object.get_by_ap_id(ap_id)
+  def normalize(%{"id" => ap_id}), do: normalize(ap_id)
+  def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
   def normalize(_), do: nil
 
   # Owned objects can only be mutated by their owner
@@ -42,24 +42,18 @@ defmodule Pleroma.Object do
   # Legacy objects can be mutated by anybody
   def authorize_mutation(%Object{}, %User{}), do: true
 
-  if Mix.env() == :test do
-    def get_cached_by_ap_id(ap_id) do
-      get_by_ap_id(ap_id)
-    end
-  else
-    def get_cached_by_ap_id(ap_id) do
-      key = "object:#{ap_id}"
-
-      Cachex.fetch!(:object_cache, key, fn _ ->
-        object = get_by_ap_id(ap_id)
-
-        if object do
-          {:commit, object}
-        else
-          {:ignore, object}
-        end
-      end)
-    end
+  def get_cached_by_ap_id(ap_id) do
+    key = "object:#{ap_id}"
+
+    Cachex.fetch!(:object_cache, key, fn _ ->
+      object = get_by_ap_id(ap_id)
+
+      if object do
+        {:commit, object}
+      else
+        {:ignore, object}
+      end
+    end)
   end
 
   def context_mapping(context) do
@@ -90,4 +84,17 @@ defmodule Pleroma.Object do
       {:ok, object}
     end
   end
+
+  def set_cache(%Object{data: %{"id" => ap_id}} = object) do
+    Cachex.put(:object_cache, "object:#{ap_id}", object)
+    {:ok, object}
+  end
+
+  def update_and_set_cache(changeset) do
+    with {:ok, object} <- Repo.update(changeset) do
+      set_cache(object)
+    else
+      e -> e
+    end
+  end
 end
diff --git a/lib/pleroma/plugs/instance_static.ex b/lib/pleroma/plugs/instance_static.ex
index af2f6f331..11f108de7 100644
--- a/lib/pleroma/plugs/instance_static.ex
+++ b/lib/pleroma/plugs/instance_static.ex
@@ -21,7 +21,7 @@ defmodule Pleroma.Plugs.InstanceStatic do
     end
   end
 
-  @only ~w(index.html static emoji packs sounds images instance favicon.png)
+  @only ~w(index.html static emoji packs sounds images instance favicon.png sw.js sw-pleroma.js)
 
   def init(opts) do
     opts
diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
index 0a19e737b..ce2a1b696 100644
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@@ -124,10 +124,10 @@ defmodule Pleroma.Upload do
 
           :pleroma, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Mogrify]]
 
-          :pleroma, Pleroma.Upload.Filter.Mogrify, args: "strip"
+          :pleroma, Pleroma.Upload.Filter.Mogrify, args: ["strip", "auto-orient"]
         """)
 
-        Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: "strip")
+        Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: ["strip", "auto-orient"])
         Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Mogrify])
       else
         opts
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index bd797db40..33630ac7c 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -39,6 +39,7 @@ defmodule Pleroma.User do
     field(:follower_address, :string)
     field(:search_rank, :float, virtual: true)
     field(:tags, {:array, :string}, default: [])
+    field(:bookmarks, {:array, :string}, default: [])
     field(:last_refreshed_at, :naive_datetime)
     has_many(:notifications, Notification)
     embeds_one(:info, Pleroma.User.Info)
@@ -314,7 +315,16 @@ defmodule Pleroma.User do
     q =
       from(u in User,
         where: u.id == ^follower.id,
-        update: [set: [following: fragment("array_cat(?, ?)", u.following, ^followed_addresses)]]
+        update: [
+          set: [
+            following:
+              fragment(
+                "array(select distinct unnest (array_cat(?, ?)))",
+                u.following,
+                ^followed_addresses
+              )
+          ]
+        ]
       )
 
     {1, [follower]} = Repo.update_all(q, [], returning: true)
@@ -1161,6 +1171,22 @@ defmodule Pleroma.User do
     updated_user
   end
 
+  def bookmark(%User{} = user, status_id) do
+    bookmarks = Enum.uniq(user.bookmarks ++ [status_id])
+    update_bookmarks(user, bookmarks)
+  end
+
+  def unbookmark(%User{} = user, status_id) do
+    bookmarks = Enum.uniq(user.bookmarks -- [status_id])
+    update_bookmarks(user, bookmarks)
+  end
+
+  def update_bookmarks(%User{} = user, bookmarks) do
+    user
+    |> change(%{bookmarks: bookmarks})
+    |> update_and_set_cache
+  end
+
   defp normalize_tags(tags) do
     [tags]
     |> List.flatten()
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 6d784717e..bdc9456dd 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
-  alias Pleroma.{Activity, Repo, Object, Upload, User, Notification}
+  alias Pleroma.{Activity, Repo, Object, Upload, User, Notification, Instances}
   alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF}
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Federator
@@ -734,7 +734,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def publish(actor, activity) do
-    followers =
+    remote_followers =
       if actor.follower_address in activity.recipients do
         {:ok, followers} = User.get_followers(actor)
         followers |> Enum.filter(&(!&1.local))
@@ -747,24 +747,26 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
     json = Jason.encode!(data)
 
-    (Pleroma.Web.Salmon.remote_users(activity) ++ followers)
+    (Pleroma.Web.Salmon.remote_users(activity) ++ remote_followers)
     |> Enum.filter(fn user -> User.ap_enabled?(user) end)
     |> Enum.map(fn %{info: %{source_data: data}} ->
       (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"]
     end)
     |> Enum.uniq()
     |> Enum.filter(fn inbox -> should_federate?(inbox, public) end)
-    |> Enum.each(fn inbox ->
+    |> Instances.filter_reachable()
+    |> Enum.each(fn {inbox, unreachable_since} ->
       Federator.publish_single_ap(%{
         inbox: inbox,
         json: json,
         actor: actor,
-        id: activity.data["id"]
+        id: activity.data["id"],
+        unreachable_since: unreachable_since
       })
     end)
   end
 
-  def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
+  def publish_one(%{inbox: inbox, json: json, actor: actor, id: id} = params) do
     Logger.info("Federating #{id} to #{inbox}")
     host = URI.parse(inbox).host
 
@@ -777,15 +779,26 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         digest: digest
       })
 
-    @httpoison.post(
-      inbox,
-      json,
-      [
-        {"Content-Type", "application/activity+json"},
-        {"signature", signature},
-        {"digest", digest}
-      ]
-    )
+    with {:ok, %{status: code}} when code in 200..299 <-
+           result =
+             @httpoison.post(
+               inbox,
+               json,
+               [
+                 {"Content-Type", "application/activity+json"},
+                 {"signature", signature},
+                 {"digest", digest}
+               ]
+             ) do
+      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
+        do: Instances.set_reachable(inbox)
+
+      result
+    else
+      {_post_result, response} ->
+        unless params[:unreachable_since], do: Instances.set_unreachable(inbox)
+        {:error, response}
+    end
   end
 
   # TODO:
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 04c6fef3f..96ac51862 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   use Pleroma.Web, :controller
+
   alias Pleroma.{Activity, User, Object}
   alias Pleroma.Web.ActivityPub.{ObjectView, UserView}
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -17,6 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   action_fallback(:errors)
 
   plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
+  plug(:set_requester_reachable when action in [:inbox])
   plug(:relay_active? when action in [:relay])
 
   def relay_active?(conn, _) do
@@ -289,4 +291,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     |> put_status(500)
     |> json("error")
   end
+
+  defp set_requester_reachable(%Plug.Conn{} = conn, _) do
+    with actor <- conn.params["actor"],
+         true <- is_binary(actor) do
+      Pleroma.Instances.set_reachable(actor)
+    end
+
+    conn
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
index a3f516ae7..4c6e612b2 100644
--- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
@@ -3,20 +3,46 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
+  alias Pleroma.User
   @behaviour Pleroma.Web.ActivityPub.MRF
 
+  defp delist_message(message) do
+    follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address
+
+    message
+    |> Map.put("to", [follower_collection])
+    |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"])
+  end
+
   @impl true
-  def filter(%{"type" => "Create"} = object) do
-    threshold = Pleroma.Config.get([:mrf_hellthread, :threshold])
-    recipients = (object["to"] || []) ++ (object["cc"] || [])
-
-    if length(recipients) > threshold do
-      {:reject, nil}
-    else
-      {:ok, object}
+  def filter(%{"type" => "Create"} = message) do
+    delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold])
+
+    reject_threshold =
+      Pleroma.Config.get(
+        [:mrf_hellthread, :reject_threshold],
+        Pleroma.Config.get([:mrf_hellthread, :threshold])
+      )
+
+    recipients = (message["to"] || []) ++ (message["cc"] || [])
+
+    cond do
+      length(recipients) > reject_threshold and reject_threshold > 0 ->
+        {:reject, nil}
+
+      length(recipients) > delist_threshold and delist_threshold > 0 ->
+        if Enum.member?(message["to"], "https://www.w3.org/ns/activitystreams#Public") or
+             Enum.member?(message["cc"], "https://www.w3.org/ns/activitystreams#Public") do
+          {:ok, delist_message(message)}
+        else
+          {:ok, message}
+        end
+
+      true ->
+        {:ok, message}
     end
   end
 
   @impl true
-  def filter(object), do: {:ok, object}
+  def filter(message), do: {:ok, message}
 end
diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
index 3d254498c..83086dcec 100644
--- a/lib/pleroma/web/activity_pub/utils.ex
+++ b/lib/pleroma/web/activity_pub/utils.ex
@@ -285,7 +285,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
            |> Map.put("#{property}_count", length(element))
            |> Map.put("#{property}s", element),
          changeset <- Changeset.change(object, data: new_data),
-         {:ok, object} <- Repo.update(changeset),
+         {:ok, object} <- Object.update_and_set_cache(changeset),
          _ <- update_object_in_activities(object) do
       {:ok, object}
     end
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 0b4ce9cc4..3eed047ca 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -25,7 +25,7 @@ defmodule Pleroma.Web.Endpoint do
     at: "/",
     from: :pleroma,
     only:
-      ~w(index.html static finmoji emoji packs sounds images instance sw.js favicon.png schemas doc)
+      ~w(index.html static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc)
   )
 
   # Code reloading can be explicitly enabled under the
@@ -82,4 +82,8 @@ defmodule Pleroma.Web.Endpoint do
     port = System.get_env("PORT") || raise "expected the PORT environment variable to be set"
     {:ok, Keyword.put(config, :http, [:inet6, port: port])}
   end
+
+  def websocket_url do
+    String.replace_leading(url(), "http", "ws")
+  end
 end
diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index 4d03b4622..37100c9e6 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.User
   alias Pleroma.Activity
   alias Pleroma.Jobs
-  alias Pleroma.Web.{WebFinger, Websub}
+  alias Pleroma.Web.{WebFinger, Websub, Salmon}
   alias Pleroma.Web.Federator.RetryQueue
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
@@ -58,6 +58,10 @@ defmodule Pleroma.Web.Federator do
     Jobs.enqueue(:federator_out, __MODULE__, [:refresh_subscriptions])
   end
 
+  def publish_single_salmon(params) do
+    Jobs.enqueue(:federator_out, __MODULE__, [:publish_single_salmon, params])
+  end
+
   # Job Worker Callbacks
 
   def perform(:refresh_subscriptions) do
@@ -145,6 +149,10 @@ defmodule Pleroma.Web.Federator do
     end
   end
 
+  def perform(:publish_single_salmon, params) do
+    Salmon.send_to_user(params)
+  end
+
   def perform(:publish_single_ap, params) do
     case ActivityPub.publish_one(params) do
       {:ok, _} ->
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index a92645ca3..7f3fbff4a 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -138,7 +138,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})",
       email: Keyword.get(instance, :email),
       urls: %{
-        streaming_api: String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws")
+        streaming_api: Pleroma.Web.Endpoint.websocket_url()
       },
       stats: Stats.get_stats(),
       thumbnail: Web.base_url() <> "/instance/thumbnail.jpeg",
@@ -423,6 +423,28 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
+  def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
+    with %Activity{} = activity <- Repo.get(Activity, id),
+         %User{} = user <- User.get_by_nickname(user.nickname),
+         true <- ActivityPub.visible_for_user?(activity, user),
+         {:ok, user} <- User.bookmark(user, activity.data["object"]["id"]) do
+      conn
+      |> put_view(StatusView)
+      |> try_render("status.json", %{activity: activity, for: user, as: :activity})
+    end
+  end
+
+  def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
+    with %Activity{} = activity <- Repo.get(Activity, id),
+         %User{} = user <- User.get_by_nickname(user.nickname),
+         true <- ActivityPub.visible_for_user?(activity, user),
+         {:ok, user} <- User.unbookmark(user, activity.data["object"]["id"]) do
+      conn
+      |> put_view(StatusView)
+      |> try_render("status.json", %{activity: activity, for: user, as: :activity})
+    end
+  end
+
   def notifications(%{assigns: %{user: user}} = conn, params) do
     notifications = Notification.for_user(user, params)
 
@@ -859,6 +881,19 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     |> render("index.json", %{activities: activities, for: user, as: :activity})
   end
 
+  def bookmarks(%{assigns: %{user: user}} = conn, _) do
+    user = Repo.get(User, user.id)
+
+    activities =
+      user.bookmarks
+      |> Enum.map(fn id -> Activity.get_create_by_object_ap_id(id) end)
+      |> Enum.reverse()
+
+    conn
+    |> put_view(StatusView)
+    |> render("index.json", %{activities: activities, for: user, as: :activity})
+  end
+
   def get_lists(%{assigns: %{user: user}} = conn, opts) do
     lists = Pleroma.List.for_user(user, opts)
     res = ListView.render("lists.json", lists: lists)
@@ -870,7 +905,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       res = ListView.render("list.json", list: list)
       json(conn, res)
     else
-      _e -> json(conn, "error")
+      _e ->
+        conn
+        |> put_status(404)
+        |> json(%{error: "Record not found"})
     end
   end
 
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index b14ca9f5d..d5b7e68c7 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -87,6 +87,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       favourites_count: 0,
       reblogged: false,
       favourited: false,
+      bookmarked: false,
       muted: false,
       pinned: pinned?(activity, user),
       sensitive: false,
@@ -121,6 +122,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
     repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || [])
     favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || [])
+    bookmarked = opts[:for] && object["id"] in opts[:for].bookmarks
 
     attachment_data = object["attachment"] || []
     attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)
@@ -157,6 +159,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       favourites_count: like_count,
       reblogged: present?(repeated),
       favourited: present?(favorited),
+      bookmarked: present?(bookmarked),
       muted: false,
       pinned: pinned?(activity, user),
       sensitive: sensitive,
diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
index 11b97164d..21694a5ee 100644
--- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
+++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
@@ -19,6 +19,10 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
         %{
           rel: "http://nodeinfo.diaspora.software/ns/schema/2.0",
           href: Web.base_url() <> "/nodeinfo/2.0.json"
+        },
+        %{
+          rel: "http://nodeinfo.diaspora.software/ns/schema/2.1",
+          href: Web.base_url() <> "/nodeinfo/2.1.json"
         }
       ]
     }
@@ -26,8 +30,9 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
     json(conn, response)
   end
 
-  # Schema definition: https://github.com/jhass/nodeinfo/blob/master/schemas/2.0/schema.json
-  def nodeinfo(conn, %{"version" => "2.0"}) do
+  # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field
+  # under software.
+  def raw_nodeinfo() do
     instance = Application.get_env(:pleroma, :instance)
     media_proxy = Application.get_env(:pleroma, :media_proxy)
     suggestions = Application.get_env(:pleroma, :suggestions)
@@ -98,10 +103,10 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
       ]
       |> Enum.filter(& &1)
 
-    response = %{
+    %{
       version: "2.0",
       software: %{
-        name: Pleroma.Application.name(),
+        name: Pleroma.Application.name() |> String.downcase(),
         version: Pleroma.Application.version()
       },
       protocols: ["ostatus", "activitypub"],
@@ -142,12 +147,37 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
         restrictedNicknames: Pleroma.Config.get([Pleroma.User, :restricted_nicknames])
       }
     }
+  end
 
+  # Schema definition: https://github.com/jhass/nodeinfo/blob/master/schemas/2.0/schema.json
+  # and https://github.com/jhass/nodeinfo/blob/master/schemas/2.1/schema.json
+  def nodeinfo(conn, %{"version" => "2.0"}) do
     conn
     |> put_resp_header(
       "content-type",
       "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#; charset=utf-8"
     )
+    |> json(raw_nodeinfo())
+  end
+
+  def nodeinfo(conn, %{"version" => "2.1"}) do
+    raw_response = raw_nodeinfo()
+
+    updated_software =
+      raw_response
+      |> Map.get(:software)
+      |> Map.put(:repository, Pleroma.Application.repository())
+
+    response =
+      raw_response
+      |> Map.put(:software, updated_software)
+      |> Map.put(:version, "2.1")
+
+    conn
+    |> put_resp_header(
+      "content-type",
+      "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.1#; charset=utf-8"
+    )
     |> json(response)
   end
 
diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex
index a3155b79d..a20ca17bb 100644
--- a/lib/pleroma/web/ostatus/ostatus.ex
+++ b/lib/pleroma/web/ostatus/ostatus.ex
@@ -48,6 +48,9 @@ defmodule Pleroma.Web.OStatus do
 
   def handle_incoming(xml_string) do
     with doc when doc != :error <- parse_document(xml_string) do
+      with {:ok, actor_user} <- find_make_or_update_user(doc),
+           do: Pleroma.Instances.set_reachable(actor_user.ap_id)
+
       entries = :xmerl_xpath.string('//entry', doc)
 
       activities =
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index ffd3a24dc..9ad3d3bd1 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -14,6 +14,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Web.ActivityPub.ActivityPub
 
   plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming])
+
   action_fallback(:errors)
 
   def feed_redirect(conn, %{"nickname" => nickname}) do
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 71fdddef9..521fa7ee0 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -7,7 +7,8 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   alias Pleroma.Web.RichMedia.Parser
 
   def fetch_data_for_activity(%Activity{} = activity) do
-    with %Object{} = object <- Object.normalize(activity.data["object"]),
+    with true <- Pleroma.Config.get([:rich_media, :enabled]),
+         %Object{} = object <- Object.normalize(activity.data["object"]),
          {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),
          {:ok, rich_media} <- Parser.parse(page_url) do
       %{page_url: page_url, rich_media: rich_media}
diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index e67ecc47d..32dec9887 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -30,7 +30,7 @@ defmodule Pleroma.Web.RichMedia.Parser do
     try do
       {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: [pool: :media])
 
-      html |> maybe_parse() |> get_parsed_data()
+      html |> maybe_parse() |> clean_parsed_data() |> check_parsed_data()
     rescue
       e ->
         {:error, "Parsing error: #{inspect(e)}"}
@@ -46,11 +46,33 @@ defmodule Pleroma.Web.RichMedia.Parser do
     end)
   end
 
-  defp get_parsed_data(%{title: title} = data) when is_binary(title) and byte_size(title) > 0 do
+  defp check_parsed_data(%{title: title} = data) when is_binary(title) and byte_size(title) > 0 do
     {:ok, data}
   end
 
-  defp get_parsed_data(data) do
+  defp check_parsed_data(data) do
     {:error, "Found metadata was invalid or incomplete: #{inspect(data)}"}
   end
+
+  defp string_is_valid_unicode(data) when is_binary(data) do
+    data
+    |> :unicode.characters_to_binary()
+    |> clean_string()
+  end
+
+  defp string_is_valid_unicode(data), do: {:ok, data}
+
+  defp clean_string({:error, _, _}), do: {:error, "Invalid data"}
+  defp clean_string(data), do: {:ok, data}
+
+  defp clean_parsed_data(data) do
+    data
+    |> Enum.reject(fn {_, val} ->
+      case string_is_valid_unicode(val) do
+        {:ok, _} -> false
+        _ -> true
+      end
+    end)
+    |> Map.new()
+  end
 end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index bfa10451a..c6b4d37ab 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -185,6 +185,7 @@ defmodule Pleroma.Web.Router do
     get("/timelines/direct", MastodonAPIController, :dm_timeline)
 
     get("/favourites", MastodonAPIController, :favourites)
+    get("/bookmarks", MastodonAPIController, :bookmarks)
 
     post("/statuses", MastodonAPIController, :post_status)
     delete("/statuses/:id", MastodonAPIController, :delete_status)
@@ -195,6 +196,8 @@ defmodule Pleroma.Web.Router do
     post("/statuses/:id/unfavourite", MastodonAPIController, :unfav_status)
     post("/statuses/:id/pin", MastodonAPIController, :pin_status)
     post("/statuses/:id/unpin", MastodonAPIController, :unpin_status)
+    post("/statuses/:id/bookmark", MastodonAPIController, :bookmark_status)
+    post("/statuses/:id/unbookmark", MastodonAPIController, :unbookmark_status)
 
     post("/notifications/clear", MastodonAPIController, :clear_notifications)
     post("/notifications/dismiss", MastodonAPIController, :dismiss_notification)
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index e41657da1..3db0c07fd 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.Salmon do
   @httpoison Application.get_env(:pleroma, :httpoison)
 
   use Bitwise
+  alias Pleroma.Instances
   alias Pleroma.Web.XML
   alias Pleroma.Web.OStatus.ActivityRepresenter
   alias Pleroma.User
@@ -161,25 +162,31 @@ defmodule Pleroma.Web.Salmon do
     |> Enum.filter(fn user -> user && !user.local end)
   end
 
-  # push an activity to remote accounts
-  #
-  defp send_to_user(%{info: %{salmon: salmon}}, feed, poster),
-    do: send_to_user(salmon, feed, poster)
+  @doc "Pushes an activity to remote account."
+  def send_to_user(%{recipient: %{info: %{salmon: salmon}}} = params),
+    do: send_to_user(Map.put(params, :recipient, salmon))
 
-  defp send_to_user(url, feed, poster) when is_binary(url) do
-    with {:ok, %{status: code}} <-
+  def send_to_user(%{recipient: url, feed: feed, poster: poster} = params) when is_binary(url) do
+    with {:ok, %{status: code}} when code in 200..299 <-
            poster.(
              url,
              feed,
              [{"Content-Type", "application/magic-envelope+xml"}]
            ) do
+      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
+        do: Instances.set_reachable(url)
+
       Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
+      :ok
     else
-      e -> Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
+      e ->
+        unless params[:unreachable_since], do: Instances.set_reachable(url)
+        Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
+        :error
     end
   end
 
-  defp send_to_user(_, _, _), do: nil
+  def send_to_user(_), do: :noop
 
   @supported_activities [
     "Create",
@@ -209,12 +216,23 @@ defmodule Pleroma.Web.Salmon do
       {:ok, private, _} = keys_from_pem(keys)
       {:ok, feed} = encode(private, feed)
 
-      remote_users(activity)
+      remote_users = remote_users(activity)
+
+      salmon_urls = Enum.map(remote_users, & &1.info.salmon)
+      reachable_urls_metadata = Instances.filter_reachable(salmon_urls)
+      reachable_urls = Map.keys(reachable_urls_metadata)
+
+      remote_users
+      |> Enum.filter(&(&1.info.salmon in reachable_urls))
       |> Enum.each(fn remote_user ->
-        Task.start(fn ->
-          Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
-          send_to_user(remote_user, feed, poster)
-        end)
+        Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
+
+        Pleroma.Web.Federator.publish_single_salmon(%{
+          recipient: remote_user,
+          feed: feed,
+          poster: poster,
+          unreachable_since: reachable_urls_metadata[remote_user.info.salmon]
+        })
       end)
     end
   end
diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex
index 2e96c1509..8dd3284d6 100644
--- a/lib/pleroma/web/templates/layout/app.html.eex
+++ b/lib/pleroma/web/templates/layout/app.html.eex
@@ -1,7 +1,8 @@
 <!DOCTYPE html>
 <html>
   <head>
-    <meta charset=utf-8 />
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1,minimal-ui" />
     <title>
     <%= Application.get_env(:pleroma, :instance)[:name] %>
     </title>
diff --git a/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex b/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex
index 0862412ea..9a725e420 100644
--- a/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex
+++ b/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex
@@ -1,23 +1,28 @@
 <!DOCTYPE html>
 <html lang='en'>
 <head>
+<meta charset='utf-8'>
+<meta content='width=device-width, initial-scale=1' name='viewport'>
 <title>
 <%= Application.get_env(:pleroma, :instance)[:name] %>
 </title>
-<meta charset='utf-8'>
-<meta content='width=device-width, initial-scale=1' name='viewport'>
 <link rel="icon" type="image/png" href="/favicon.png"/>
-<link rel="stylesheet" media="all" href="/packs/common.css" />
-<link rel="stylesheet" media="all" href="/packs/default.css" />
+<script crossorigin='anonymous' src="/packs/locales.js"></script>
+<script crossorigin='anonymous' src="/packs/locales/glitch/en.js"></script>
 
-<script src="/packs/common.js"></script>
-<script src="/packs/locale_en.js"></script>
-<link as='script' crossorigin='anonymous' href='/packs/features/getting_started.js' rel='preload'>
-<link as='script' crossorigin='anonymous' href='/packs/features/compose.js' rel='preload'>
-<link as='script' crossorigin='anonymous' href='/packs/features/home_timeline.js' rel='preload'>
-<link as='script' crossorigin='anonymous' href='/packs/features/notifications.js' rel='preload'>
+<link rel='preload' as='script' crossorigin='anonymous' href='/packs/features/getting_started.js'>
+<link rel='preload' as='script' crossorigin='anonymous' href='/packs/features/compose.js'>
+<link rel='preload' as='script' crossorigin='anonymous' href='/packs/features/home_timeline.js'>
+<link rel='preload' as='script' crossorigin='anonymous' href='/packs/features/notifications.js'>
 <script id='initial-state' type='application/json'><%= raw @initial_state %></script>
-<script src="/packs/application.js"></script>
+
+<script src="/packs/core/common.js"></script>
+<link rel="stylesheet" media="all" href="/packs/core/common.css" />
+
+<script src="/packs/flavours/glitch/common.js"></script>
+<link rel="stylesheet" media="all" href="/packs/flavours/glitch/common.css" />
+
+<script src="/packs/flavours/glitch/home.js"></script>
 </head>
 <body class='app-body no-reduce-motion system-font'>
   <div class='app-holder' data-props='{&quot;locale&quot;:&quot;en&quot;}' id='mastodon'>
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index 652ffd92c..82845f13d 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.Websub do
   alias Ecto.Changeset
   alias Pleroma.Repo
+  alias Pleroma.Instances
   alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription}
   alias Pleroma.Web.OStatus.FeedRepresenter
   alias Pleroma.Web.{XML, Endpoint, OStatus, Federator}
@@ -53,28 +54,34 @@ defmodule Pleroma.Web.Websub do
   ]
   def publish(topic, user, %{data: %{"type" => type}} = activity)
       when type in @supported_activities do
-    # TODO: Only send to still valid subscriptions.
+    response =
+      user
+      |> FeedRepresenter.to_simple_form([activity], [user])
+      |> :xmerl.export_simple(:xmerl_xml)
+      |> to_string
+
     query =
       from(
         sub in WebsubServerSubscription,
         where: sub.topic == ^topic and sub.state == "active",
-        where: fragment("? > NOW()", sub.valid_until)
+        where: fragment("? > (NOW() at time zone 'UTC')", sub.valid_until)
       )
 
     subscriptions = Repo.all(query)
 
-    Enum.each(subscriptions, fn sub ->
-      response =
-        user
-        |> FeedRepresenter.to_simple_form([activity], [user])
-        |> :xmerl.export_simple(:xmerl_xml)
-        |> to_string
+    callbacks = Enum.map(subscriptions, & &1.callback)
+    reachable_callbacks_metadata = Instances.filter_reachable(callbacks)
+    reachable_callbacks = Map.keys(reachable_callbacks_metadata)
 
+    subscriptions
+    |> Enum.filter(&(&1.callback in reachable_callbacks))
+    |> Enum.each(fn sub ->
       data = %{
         xml: response,
         topic: topic,
         callback: sub.callback,
-        secret: sub.secret
+        secret: sub.secret,
+        unreachable_since: reachable_callbacks_metadata[sub.callback]
       }
 
       Federator.publish_single_websub(data)
@@ -263,11 +270,11 @@ defmodule Pleroma.Web.Websub do
     end)
   end
 
-  def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) do
+  def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret} = params) do
     signature = sign(secret || "", xml)
     Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
 
-    with {:ok, %{status: code}} <-
+    with {:ok, %{status: code}} when code in 200..299 <-
            @httpoison.post(
              callback,
              xml,
@@ -276,12 +283,16 @@ defmodule Pleroma.Web.Websub do
                {"X-Hub-Signature", "sha1=#{signature}"}
              ]
            ) do
+      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
+        do: Instances.set_reachable(callback)
+
       Logger.info(fn -> "Pushed to #{callback}, code #{code}" end)
       {:ok, code}
     else
-      e ->
-        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end)
-        {:error, e}
+      {_post_result, response} ->
+        unless params[:unreachable_since], do: Instances.set_reachable(callback)
+        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(response)}" end)
+        {:error, response}
     end
   end
 end
diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex
index eb10227cb..41bbc0369 100644
--- a/lib/pleroma/web/websub/websub_controller.ex
+++ b/lib/pleroma/web/websub/websub_controller.ex
@@ -4,9 +4,11 @@
 
 defmodule Pleroma.Web.Websub.WebsubController do
   use Pleroma.Web, :controller
+
   alias Pleroma.{Repo, User}
   alias Pleroma.Web.{Websub, Federator}
   alias Pleroma.Web.Websub.WebsubClientSubscription
+
   require Logger
 
   plug(