Merge develop to 770-add-emoji-tags

Merge conflict in test/web/mastodon_api/mastodon_api_controller_test.exs

14 files changed, 201 insertions, 55 deletions

diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index 1ba452275..8f8d86a11 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -81,6 +81,14 @@ defmodule Mix.Tasks.Pleroma.Instance do
 
       email = Common.get_option(options, :admin_email, "What is your admin email address?")
 
+      indexable =
+        Common.get_option(
+          options,
+          :indexable,
+          "Do you want search engines to index your site? (y/n)",
+          "y"
+        ) === "y"
+
       dbhost =
         Common.get_option(options, :dbhost, "What is the hostname of your database?", "localhost")
 
@@ -142,6 +150,8 @@ defmodule Mix.Tasks.Pleroma.Instance do
       Mix.shell().info("Writing #{psql_path}.")
       File.write(psql_path, result_psql)
 
+      write_robots_txt(indexable)
+
       Mix.shell().info(
         "\n" <>
           """
@@ -163,4 +173,28 @@ defmodule Mix.Tasks.Pleroma.Instance do
       )
     end
   end
+
+  defp write_robots_txt(indexable) do
+    robots_txt =
+      EEx.eval_file(
+        Path.expand("robots_txt.eex", __DIR__),
+        indexable: indexable
+      )
+
+    static_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static/")
+
+    unless File.exists?(static_dir) do
+      File.mkdir_p!(static_dir)
+    end
+
+    robots_txt_path = Path.join(static_dir, "robots.txt")
+
+    if File.exists?(robots_txt_path) do
+      File.cp!(robots_txt_path, "#{robots_txt_path}.bak")
+      Mix.shell().info("Backing up existing robots.txt to #{robots_txt_path}.bak")
+    end
+
+    File.write(robots_txt_path, robots_txt)
+    Mix.shell().info("Writing #{robots_txt_path}.")
+  end
 end
diff --git a/lib/mix/tasks/pleroma/robots_txt.eex b/lib/mix/tasks/pleroma/robots_txt.eex
new file mode 100644
index 000000000..1af3c47ee
--- /dev/null
+++ b/lib/mix/tasks/pleroma/robots_txt.eex
@@ -0,0 +1,2 @@
+User-Agent: *
+Disallow: <%= if indexable, do: "", else: "/" %>
diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex
index 5b152d926..7f1dbe28c 100644
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@@ -28,27 +28,39 @@ defmodule Pleroma.HTML do
   def filter_tags(html), do: filter_tags(html, nil)
   def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags)
 
-  def get_cached_scrubbed_html_for_object(content, scrubbers, object, module) do
-    key = "#{module}#{generate_scrubber_signature(scrubbers)}|#{object.id}"
-    Cachex.fetch!(:scrubber_cache, key, fn _key -> ensure_scrubbed_html(content, scrubbers) end)
+  def get_cached_scrubbed_html_for_activity(content, scrubbers, activity, key \\ "") do
+    key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
+
+    Cachex.fetch!(:scrubber_cache, key, fn _key ->
+      ensure_scrubbed_html(content, scrubbers, activity.data["object"]["fake"] || false)
+    end)
   end
 
-  def get_cached_stripped_html_for_object(content, object, module) do
-    get_cached_scrubbed_html_for_object(
+  def get_cached_stripped_html_for_activity(content, activity, key) do
+    get_cached_scrubbed_html_for_activity(
       content,
       HtmlSanitizeEx.Scrubber.StripTags,
-      object,
-      module
+      activity,
+      key
     )
   end
 
   def ensure_scrubbed_html(
         content,
-        scrubbers
+        scrubbers,
+        false = _fake
       ) do
     {:commit, filter_tags(content, scrubbers)}
   end
 
+  def ensure_scrubbed_html(
+        content,
+        scrubbers,
+        true = _fake
+      ) do
+    {:ignore, filter_tags(content, scrubbers)}
+  end
+
   defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
     generate_scrubber_signature([scrubber])
   end
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index 8a670645d..013d62157 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -44,6 +44,11 @@ defmodule Pleroma.Object do
   # Use this whenever possible, especially when walking graphs in an O(N) loop!
   def normalize(%Activity{object: %Object{} = object}), do: object
 
+  # A hack for fake activities
+  def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}) do
+    %Object{id: "pleroma:fake_object_id", data: data}
+  end
+
   # Catch and log Object.normalize() calls where the Activity's child object is not
   # preloaded.
   def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}) do
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 6e1ed7ec9..f217e7bac 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -113,15 +113,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def decrease_replies_count_if_reply(_object), do: :noop
 
-  def insert(map, local \\ true) when is_map(map) do
+  def insert(map, local \\ true, fake \\ false) when is_map(map) do
     with nil <- Activity.normalize(map),
-         map <- lazy_put_activity_defaults(map),
+         map <- lazy_put_activity_defaults(map, fake),
          :ok <- check_actor_is_active(map["actor"]),
          {_, true} <- {:remote_limit_error, check_remote_limit(map)},
          {:ok, map} <- MRF.filter(map),
+         {recipients, _, _} = get_recipients(map),
+         {:fake, false, map, recipients} <- {:fake, fake, map, recipients},
          {:ok, object} <- insert_full_object(map) do
-      {recipients, _, _} = get_recipients(map)
-
       {:ok, activity} =
         Repo.insert(%Activity{
           data: map,
@@ -146,8 +146,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       stream_out(activity)
       {:ok, activity}
     else
-      %Activity{} = activity -> {:ok, activity}
-      error -> {:error, error}
+      %Activity{} = activity ->
+        {:ok, activity}
+
+      {:fake, true, map, recipients} ->
+        activity = %Activity{
+          data: map,
+          local: local,
+          actor: map["actor"],
+          recipients: recipients,
+          id: "pleroma:fakeid"
+        }
+
+        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+        {:ok, activity}
+
+      error ->
+        {:error, error}
     end
   end
 
@@ -190,7 +205,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def create(%{to: to, actor: actor, context: context, object: object} = params) do
+  def create(%{to: to, actor: actor, context: context, object: object} = params, fake \\ false) do
     additional = params[:additional] || %{}
     # only accept false as false value
     local = !(params[:local] == false)
@@ -201,13 +216,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
              %{to: to, actor: actor, published: published, context: context, object: object},
              additional
            ),
-         {:ok, activity} <- insert(create_data, local),
+         {:ok, activity} <- insert(create_data, local, fake),
+         {:fake, false, activity} <- {:fake, fake, activity},
          _ <- increase_replies_count_if_reply(create_data),
          # Changing note count prior to enqueuing federation task in order to avoid
          # race conditions on updating user.info
          {:ok, _actor} <- increase_note_count_if_public(actor, activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
+    else
+      {:fake, true, activity} ->
+        {:ok, activity}
     end
   end
 
diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
index 77841278a..32545937e 100644
--- a/lib/pleroma/web/activity_pub/utils.ex
+++ b/lib/pleroma/web/activity_pub/utils.ex
@@ -175,18 +175,26 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   Adds an id and a published data if they aren't there,
   also adds it to an included object
   """
-  def lazy_put_activity_defaults(map) do
-    %{data: %{"id" => context}, id: context_id} = create_context(map["context"])
-
+  def lazy_put_activity_defaults(map, fake \\ false) do
     map =
-      map
-      |> Map.put_new_lazy("id", &generate_activity_id/0)
-      |> Map.put_new_lazy("published", &make_date/0)
-      |> Map.put_new("context", context)
-      |> Map.put_new("context_id", context_id)
+      unless fake do
+        %{data: %{"id" => context}, id: context_id} = create_context(map["context"])
+
+        map
+        |> Map.put_new_lazy("id", &generate_activity_id/0)
+        |> Map.put_new_lazy("published", &make_date/0)
+        |> Map.put_new("context", context)
+        |> Map.put_new("context_id", context_id)
+      else
+        map
+        |> Map.put_new("id", "pleroma:fakeid")
+        |> Map.put_new_lazy("published", &make_date/0)
+        |> Map.put_new("context", "pleroma:fakecontext")
+        |> Map.put_new("context_id", -1)
+      end
 
     if is_map(map["object"]) do
-      object = lazy_put_object_defaults(map["object"], map)
+      object = lazy_put_object_defaults(map["object"], map, fake)
       %{map | "object" => object}
     else
       map
@@ -196,7 +204,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   @doc """
   Adds an id and published date if they aren't there.
   """
-  def lazy_put_object_defaults(map, activity \\ %{}) do
+  def lazy_put_object_defaults(map, activity \\ %{}, fake)
+
+  def lazy_put_object_defaults(map, activity, true = _fake) do
+    map
+    |> Map.put_new_lazy("published", &make_date/0)
+    |> Map.put_new("id", "pleroma:fake_object_id")
+    |> Map.put_new("context", activity["context"])
+    |> Map.put_new("fake", true)
+    |> Map.put_new("context_id", activity["context_id"])
+  end
+
+  def lazy_put_object_defaults(map, activity, _fake) do
     map
     |> Map.put_new_lazy("id", &generate_object_id/0)
     |> Map.put_new_lazy("published", &make_date/0)
@@ -404,13 +423,15 @@ defmodule Pleroma.Web.ActivityPub.Utils do
             activity.data
           ),
         where: activity.actor == ^follower_id,
+        # this is to use the index
         where:
           fragment(
-            "? @> ?",
+            "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
             activity.data,
-            ^%{object: followed_id}
+            activity.data,
+            ^followed_id
           ),
-        order_by: [desc: :id],
+        order_by: [fragment("? desc nulls last", activity.id)],
         limit: 1
       )
 
@@ -567,13 +588,15 @@ defmodule Pleroma.Web.ActivityPub.Utils do
             activity.data
           ),
         where: activity.actor == ^blocker_id,
+        # this is to use the index
         where:
           fragment(
-            "? @> ?",
+            "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
+            activity.data,
             activity.data,
-            ^%{object: blocked_id}
+            ^blocked_id
           ),
-        order_by: [desc: :id],
+        order_by: [fragment("? desc nulls last", activity.id)],
         limit: 1
       )
 
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index b3a09e49e..78bf31893 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -25,6 +25,26 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     |> json(nickname)
   end
 
+  def user_follow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do
+    with %User{} = follower <- User.get_by_nickname(follower_nick),
+         %User{} = followed <- User.get_by_nickname(followed_nick) do
+      User.follow(follower, followed)
+    end
+
+    conn
+    |> json("ok")
+  end
+
+  def user_unfollow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do
+    with %User{} = follower <- User.get_by_nickname(follower_nick),
+         %User{} = followed <- User.get_by_nickname(followed_nick) do
+      User.unfollow(follower, followed)
+    end
+
+    conn
+    |> json("ok")
+  end
+
   def user_create(
         conn,
         %{"nickname" => nickname, "email" => email, "password" => password}
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index f910eb1f9..74babdf14 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -172,13 +172,16 @@ defmodule Pleroma.Web.CommonAPI do
              end)
            ) do
       res =
-        ActivityPub.create(%{
-          to: to,
-          actor: user,
-          context: context,
-          object: object,
-          additional: %{"cc" => cc, "directMessage" => visibility == "direct"}
-        })
+        ActivityPub.create(
+          %{
+            to: to,
+            actor: user,
+            context: context,
+            object: object,
+            additional: %{"cc" => cc, "directMessage" => visibility == "direct"}
+          },
+          Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false
+        )
 
       res
     end
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index aafeb7248..051db6c79 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -15,6 +15,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.MediaProxy
 
+  require Logger
+
   # This is a hack for twidere.
   def get_by_id_or_ap_id(id) do
     activity =
@@ -240,15 +242,21 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     Strftime.strftime!(date, "%a %b %d %H:%M:%S %z %Y")
   end
 
-  def date_to_asctime(date) do
-    with {:ok, date, _offset} <- date |> DateTime.from_iso8601() do
+  def date_to_asctime(date) when is_binary(date) do
+    with {:ok, date, _offset} <- DateTime.from_iso8601(date) do
       format_asctime(date)
     else
       _e ->
+        Logger.warn("Date #{date} in wrong format, must be ISO 8601")
         ""
     end
   end
 
+  def date_to_asctime(date) do
+    Logger.warn("Date #{date} in wrong format, must be ISO 8601")
+    ""
+  end
+
   def to_masto_date(%NaiveDateTime{} = date) do
     date
     |> NaiveDateTime.to_iso8601()
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index bdd1cf434..98943e5d1 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1092,9 +1092,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def index(%{assigns: %{user: user}} = conn, _params) do
-    token =
-      conn
-      |> get_session(:oauth_token)
+    token = get_session(conn, :oauth_token)
 
     if user && token do
       mastodon_emoji = mastodonized_emoji()
@@ -1122,7 +1120,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
             auto_play_gif: false,
             display_sensitive_media: false,
             reduce_motion: false,
-            max_toot_chars: limit
+            max_toot_chars: limit,
+            mascot: "/images/pleroma-fox-tan-smol.png"
           },
           rights: %{
             delete_others_notice: present?(user.info.is_moderator),
@@ -1194,6 +1193,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       |> render("index.html", %{initial_state: initial_state, flavour: flavour})
     else
       conn
+      |> put_session(:return_to, conn.request_path)
       |> redirect(to: "/web/login")
     end
   end
@@ -1278,12 +1278,20 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
           scope: Enum.join(app.scopes, " ")
         )
 
-      conn
-      |> redirect(to: path)
+      redirect(conn, to: path)
     end
   end
 
-  defp local_mastodon_root_path(conn), do: mastodon_api_path(conn, :index, ["getting-started"])
+  defp local_mastodon_root_path(conn) do
+    case get_session(conn, :return_to) do
+      nil ->
+        mastodon_api_path(conn, :index, ["getting-started"])
+
+      return_to ->
+        delete_session(conn, :return_to)
+        return_to
+    end
+  end
 
   defp get_or_make_app do
     find_attrs = %{client_name: @local_mastodon_name, redirect_uris: "."}
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 200bb453d..4c0b53bdd 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -147,10 +147,18 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     content =
       object
       |> render_content()
-      |> HTML.get_cached_scrubbed_html_for_object(
+      |> HTML.get_cached_scrubbed_html_for_activity(
         User.html_filter_policy(opts[:for]),
         activity,
-        __MODULE__
+        "mastoapi:content"
+      )
+
+    summary =
+      (object["summary"] || "")
+      |> HTML.get_cached_scrubbed_html_for_activity(
+        User.html_filter_policy(opts[:for]),
+        activity,
+        "mastoapi:summary"
       )
 
     card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity))
@@ -182,7 +190,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user),
       pinned: pinned?(activity, user),
       sensitive: sensitive,
-      spoiler_text: object["summary"] || "",
+      spoiler_text: summary,
       visibility: get_visibility(object),
       media_attachments: attachments,
       mentions: mentions,
diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex
index 23bbde1a6..58385a3d1 100644
--- a/lib/pleroma/web/metadata/utils.ex
+++ b/lib/pleroma/web/metadata/utils.ex
@@ -12,7 +12,7 @@ defmodule Pleroma.Web.Metadata.Utils do
     # html content comes from DB already encoded, decode first and scrub after
     |> HtmlEntities.decode()
     |> String.replace(~r/<br\s?\/?>/, " ")
-    |> HTML.get_cached_stripped_html_for_object(object, __MODULE__)
+    |> HTML.get_cached_stripped_html_for_activity(object, "metadata")
     |> Formatter.demojify()
     |> Formatter.truncate()
   end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 605a327fc..1c752e44c 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -140,8 +140,12 @@ defmodule Pleroma.Web.Router do
   scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do
     pipe_through([:admin_api, :oauth_write])
 
+    post("/user/follow", AdminAPIController, :user_follow)
+    post("/user/unfollow", AdminAPIController, :user_unfollow)
+
     get("/users", AdminAPIController, :list_users)
     get("/users/:nickname", AdminAPIController, :user_show)
+
     delete("/user", AdminAPIController, :user_delete)
     patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation)
     post("/user", AdminAPIController, :user_create)
diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex
index aa1d41fa2..433322eb8 100644
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@@ -254,10 +254,10 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
 
     html =
       content
-      |> HTML.get_cached_scrubbed_html_for_object(
+      |> HTML.get_cached_scrubbed_html_for_activity(
         User.html_filter_policy(opts[:for]),
         activity,
-        __MODULE__
+        "twitterapi:content"
       )
       |> Formatter.emojify(object["emoji"])
 
@@ -265,7 +265,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
       if content do
         content
         |> String.replace(~r/<br\s?\/?>/, "\n")
-        |> HTML.get_cached_stripped_html_for_object(activity, __MODULE__)
+        |> HTML.get_cached_stripped_html_for_activity(activity, "twitterapi:content")
       else
         ""
       end