[#923] Merge remote-tracking branch 'remotes/upstream/develop' into twitter_oauth

# Conflicts:
#	mix.exs

14 files changed, 214 insertions, 95 deletions

diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index 1ba452275..8f8d86a11 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -81,6 +81,14 @@ defmodule Mix.Tasks.Pleroma.Instance do
 
       email = Common.get_option(options, :admin_email, "What is your admin email address?")
 
+      indexable =
+        Common.get_option(
+          options,
+          :indexable,
+          "Do you want search engines to index your site? (y/n)",
+          "y"
+        ) === "y"
+
       dbhost =
         Common.get_option(options, :dbhost, "What is the hostname of your database?", "localhost")
 
@@ -142,6 +150,8 @@ defmodule Mix.Tasks.Pleroma.Instance do
       Mix.shell().info("Writing #{psql_path}.")
       File.write(psql_path, result_psql)
 
+      write_robots_txt(indexable)
+
       Mix.shell().info(
         "\n" <>
           """
@@ -163,4 +173,28 @@ defmodule Mix.Tasks.Pleroma.Instance do
       )
     end
   end
+
+  defp write_robots_txt(indexable) do
+    robots_txt =
+      EEx.eval_file(
+        Path.expand("robots_txt.eex", __DIR__),
+        indexable: indexable
+      )
+
+    static_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static/")
+
+    unless File.exists?(static_dir) do
+      File.mkdir_p!(static_dir)
+    end
+
+    robots_txt_path = Path.join(static_dir, "robots.txt")
+
+    if File.exists?(robots_txt_path) do
+      File.cp!(robots_txt_path, "#{robots_txt_path}.bak")
+      Mix.shell().info("Backing up existing robots.txt to #{robots_txt_path}.bak")
+    end
+
+    File.write(robots_txt_path, robots_txt)
+    Mix.shell().info("Writing #{robots_txt_path}.")
+  end
 end
diff --git a/lib/mix/tasks/pleroma/robots_txt.eex b/lib/mix/tasks/pleroma/robots_txt.eex
new file mode 100644
index 000000000..1af3c47ee
--- /dev/null
+++ b/lib/mix/tasks/pleroma/robots_txt.eex
@@ -0,0 +1,2 @@
+User-Agent: *
+Disallow: <%= if indexable, do: "", else: "/" %>
diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex
index 2487b4ab5..0d0bea8c0 100644
--- a/lib/mix/tasks/pleroma/user.ex
+++ b/lib/mix/tasks/pleroma/user.ex
@@ -32,6 +32,10 @@ defmodule Mix.Tasks.Pleroma.User do
 
       mix pleroma.user rm NICKNAME
 
+  ## Delete the user's activities.
+
+      mix pleroma.user delete_activities NICKNAME
+
   ## Deactivate or activate the user's account.
 
       mix pleroma.user toggle_activated NICKNAME
@@ -303,6 +307,18 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
+  def run(["delete_activities", nickname]) do
+    Common.start_pleroma()
+
+    with %User{local: true} = user <- User.get_by_nickname(nickname) do
+      User.delete_user_activities(user)
+      Mix.shell().info("User #{nickname} statuses deleted.")
+    else
+      _ ->
+        Mix.shell().error("No local user #{nickname}")
+    end
+  end
+
   defp set_moderator(user, value) do
     info_cng = User.Info.admin_api_update(user.info, %{is_moderator: value})
 
diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex
index 5b152d926..1e48749a8 100644
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@@ -28,9 +28,13 @@ defmodule Pleroma.HTML do
   def filter_tags(html), do: filter_tags(html, nil)
   def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags)
 
+  # TODO: rename object to activity because that's what it is really working with
   def get_cached_scrubbed_html_for_object(content, scrubbers, object, module) do
     key = "#{module}#{generate_scrubber_signature(scrubbers)}|#{object.id}"
-    Cachex.fetch!(:scrubber_cache, key, fn _key -> ensure_scrubbed_html(content, scrubbers) end)
+
+    Cachex.fetch!(:scrubber_cache, key, fn _key ->
+      ensure_scrubbed_html(content, scrubbers, object.data["object"]["fake"] || false)
+    end)
   end
 
   def get_cached_stripped_html_for_object(content, object, module) do
@@ -44,11 +48,20 @@ defmodule Pleroma.HTML do
 
   def ensure_scrubbed_html(
         content,
-        scrubbers
+        scrubbers,
+        false = _fake
       ) do
     {:commit, filter_tags(content, scrubbers)}
   end
 
+  def ensure_scrubbed_html(
+        content,
+        scrubbers,
+        true = _fake
+      ) do
+    {:ignore, filter_tags(content, scrubbers)}
+  end
+
   defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
     generate_scrubber_signature([scrubber])
   end
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index 8a670645d..013d62157 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -44,6 +44,11 @@ defmodule Pleroma.Object do
   # Use this whenever possible, especially when walking graphs in an O(N) loop!
   def normalize(%Activity{object: %Object{} = object}), do: object
 
+  # A hack for fake activities
+  def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}) do
+    %Object{id: "pleroma:fake_object_id", data: data}
+  end
+
   # Catch and log Object.normalize() calls where the Activity's child object is not
   # preloaded.
   def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}) do
diff --git a/lib/pleroma/plugs/user_fetcher_plug.ex b/lib/pleroma/plugs/user_fetcher_plug.ex
index 5a77f6833..4089aa958 100644
--- a/lib/pleroma/plugs/user_fetcher_plug.ex
+++ b/lib/pleroma/plugs/user_fetcher_plug.ex
@@ -3,9 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.UserFetcherPlug do
-  alias Pleroma.Repo
   alias Pleroma.User
-
   import Plug.Conn
 
   def init(options) do
@@ -14,26 +12,10 @@ defmodule Pleroma.Plugs.UserFetcherPlug do
 
   def call(conn, _options) do
     with %{auth_credentials: %{username: username}} <- conn.assigns,
-         {:ok, %User{} = user} <- user_fetcher(username) do
-      conn
-      |> assign(:auth_user, user)
+         %User{} = user <- User.get_by_nickname_or_email(username) do
+      assign(conn, :auth_user, user)
     else
       _ -> conn
     end
   end
-
-  defp user_fetcher(username_or_email) do
-    {
-      :ok,
-      cond do
-        # First, try logging in as if it was a name
-        user = Repo.get_by(User, %{nickname: username_or_email}) ->
-          user
-
-        # If we get nil, we try using it as an email
-        user = Repo.get_by(User, %{email: username_or_email}) ->
-          user
-      end
-    }
-  end
 end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index cd1815b85..05f56c01e 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1096,28 +1096,27 @@ defmodule Pleroma.User do
     # Remove all relationships
     {:ok, followers} = User.get_followers(user)
 
-    followers
-    |> Enum.each(fn follower -> User.unfollow(follower, user) end)
+    Enum.each(followers, fn follower -> User.unfollow(follower, user) end)
 
     {:ok, friends} = User.get_friends(user)
 
-    friends
-    |> Enum.each(fn followed -> User.unfollow(user, followed) end)
+    Enum.each(friends, fn followed -> User.unfollow(user, followed) end)
 
-    query =
-      from(a in Activity, where: a.actor == ^user.ap_id)
-      |> Activity.with_preloaded_object()
+    delete_user_activities(user)
+  end
 
-    Repo.all(query)
-    |> Enum.each(fn activity ->
-      case activity.data["type"] do
-        "Create" ->
-          ActivityPub.delete(Object.normalize(activity))
+  def delete_user_activities(%User{ap_id: ap_id} = user) do
+    Activity
+    |> where(actor: ^ap_id)
+    |> Activity.with_preloaded_object()
+    |> Repo.all()
+    |> Enum.each(fn
+      %{data: %{"type" => "Create"}} = activity ->
+        activity |> Object.normalize() |> ActivityPub.delete()
 
-        # TODO: Do something with likes, follows, repeats.
-        _ ->
-          "Doing nothing"
-      end
+      # TODO: Do something with likes, follows, repeats.
+      _ ->
+        "Doing nothing"
     end)
 
     {:ok, user}
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 6e1ed7ec9..f217e7bac 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -113,15 +113,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def decrease_replies_count_if_reply(_object), do: :noop
 
-  def insert(map, local \\ true) when is_map(map) do
+  def insert(map, local \\ true, fake \\ false) when is_map(map) do
     with nil <- Activity.normalize(map),
-         map <- lazy_put_activity_defaults(map),
+         map <- lazy_put_activity_defaults(map, fake),
          :ok <- check_actor_is_active(map["actor"]),
          {_, true} <- {:remote_limit_error, check_remote_limit(map)},
          {:ok, map} <- MRF.filter(map),
+         {recipients, _, _} = get_recipients(map),
+         {:fake, false, map, recipients} <- {:fake, fake, map, recipients},
          {:ok, object} <- insert_full_object(map) do
-      {recipients, _, _} = get_recipients(map)
-
       {:ok, activity} =
         Repo.insert(%Activity{
           data: map,
@@ -146,8 +146,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       stream_out(activity)
       {:ok, activity}
     else
-      %Activity{} = activity -> {:ok, activity}
-      error -> {:error, error}
+      %Activity{} = activity ->
+        {:ok, activity}
+
+      {:fake, true, map, recipients} ->
+        activity = %Activity{
+          data: map,
+          local: local,
+          actor: map["actor"],
+          recipients: recipients,
+          id: "pleroma:fakeid"
+        }
+
+        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+        {:ok, activity}
+
+      error ->
+        {:error, error}
     end
   end
 
@@ -190,7 +205,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def create(%{to: to, actor: actor, context: context, object: object} = params) do
+  def create(%{to: to, actor: actor, context: context, object: object} = params, fake \\ false) do
     additional = params[:additional] || %{}
     # only accept false as false value
     local = !(params[:local] == false)
@@ -201,13 +216,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
              %{to: to, actor: actor, published: published, context: context, object: object},
              additional
            ),
-         {:ok, activity} <- insert(create_data, local),
+         {:ok, activity} <- insert(create_data, local, fake),
+         {:fake, false, activity} <- {:fake, fake, activity},
          _ <- increase_replies_count_if_reply(create_data),
          # Changing note count prior to enqueuing federation task in order to avoid
          # race conditions on updating user.info
          {:ok, _actor} <- increase_note_count_if_public(actor, activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
+    else
+      {:fake, true, activity} ->
+        {:ok, activity}
     end
   end
 
diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
index 77841278a..32545937e 100644
--- a/lib/pleroma/web/activity_pub/utils.ex
+++ b/lib/pleroma/web/activity_pub/utils.ex
@@ -175,18 +175,26 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   Adds an id and a published data if they aren't there,
   also adds it to an included object
   """
-  def lazy_put_activity_defaults(map) do
-    %{data: %{"id" => context}, id: context_id} = create_context(map["context"])
-
+  def lazy_put_activity_defaults(map, fake \\ false) do
     map =
-      map
-      |> Map.put_new_lazy("id", &generate_activity_id/0)
-      |> Map.put_new_lazy("published", &make_date/0)
-      |> Map.put_new("context", context)
-      |> Map.put_new("context_id", context_id)
+      unless fake do
+        %{data: %{"id" => context}, id: context_id} = create_context(map["context"])
+
+        map
+        |> Map.put_new_lazy("id", &generate_activity_id/0)
+        |> Map.put_new_lazy("published", &make_date/0)
+        |> Map.put_new("context", context)
+        |> Map.put_new("context_id", context_id)
+      else
+        map
+        |> Map.put_new("id", "pleroma:fakeid")
+        |> Map.put_new_lazy("published", &make_date/0)
+        |> Map.put_new("context", "pleroma:fakecontext")
+        |> Map.put_new("context_id", -1)
+      end
 
     if is_map(map["object"]) do
-      object = lazy_put_object_defaults(map["object"], map)
+      object = lazy_put_object_defaults(map["object"], map, fake)
       %{map | "object" => object}
     else
       map
@@ -196,7 +204,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   @doc """
   Adds an id and published date if they aren't there.
   """
-  def lazy_put_object_defaults(map, activity \\ %{}) do
+  def lazy_put_object_defaults(map, activity \\ %{}, fake)
+
+  def lazy_put_object_defaults(map, activity, true = _fake) do
+    map
+    |> Map.put_new_lazy("published", &make_date/0)
+    |> Map.put_new("id", "pleroma:fake_object_id")
+    |> Map.put_new("context", activity["context"])
+    |> Map.put_new("fake", true)
+    |> Map.put_new("context_id", activity["context_id"])
+  end
+
+  def lazy_put_object_defaults(map, activity, _fake) do
     map
     |> Map.put_new_lazy("id", &generate_object_id/0)
     |> Map.put_new_lazy("published", &make_date/0)
@@ -404,13 +423,15 @@ defmodule Pleroma.Web.ActivityPub.Utils do
             activity.data
           ),
         where: activity.actor == ^follower_id,
+        # this is to use the index
         where:
           fragment(
-            "? @> ?",
+            "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
             activity.data,
-            ^%{object: followed_id}
+            activity.data,
+            ^followed_id
           ),
-        order_by: [desc: :id],
+        order_by: [fragment("? desc nulls last", activity.id)],
         limit: 1
       )
 
@@ -567,13 +588,15 @@ defmodule Pleroma.Web.ActivityPub.Utils do
             activity.data
           ),
         where: activity.actor == ^blocker_id,
+        # this is to use the index
         where:
           fragment(
-            "? @> ?",
+            "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
+            activity.data,
             activity.data,
-            ^%{object: blocked_id}
+            ^blocked_id
           ),
-        order_by: [desc: :id],
+        order_by: [fragment("? desc nulls last", activity.id)],
         limit: 1
       )
 
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 25b990677..745d1839b 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -172,13 +172,16 @@ defmodule Pleroma.Web.CommonAPI do
              end)
            ) do
       res =
-        ActivityPub.create(%{
-          to: to,
-          actor: user,
-          context: context,
-          object: object,
-          additional: %{"cc" => cc, "directMessage" => visibility == "direct"}
-        })
+        ActivityPub.create(
+          %{
+            to: to,
+            actor: user,
+            context: context,
+            object: object,
+            additional: %{"cc" => cc, "directMessage" => visibility == "direct"}
+          },
+          Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false
+        )
 
       res
     end
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 40cea3090..9cd8b3758 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -15,6 +15,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.MediaProxy
 
+  require Logger
+
   # This is a hack for twidere.
   def get_by_id_or_ap_id(id) do
     activity =
@@ -240,15 +242,21 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     Strftime.strftime!(date, "%a %b %d %H:%M:%S %z %Y")
   end
 
-  def date_to_asctime(date) do
-    with {:ok, date, _offset} <- date |> DateTime.from_iso8601() do
+  def date_to_asctime(date) when is_binary(date) do
+    with {:ok, date, _offset} <- DateTime.from_iso8601(date) do
       format_asctime(date)
     else
       _e ->
+        Logger.warn("Date #{date} in wrong format, must be ISO 8601")
         ""
     end
   end
 
+  def date_to_asctime(date) do
+    Logger.warn("Date #{date} in wrong format, must be ISO 8601")
+    ""
+  end
+
   def to_masto_date(%NaiveDateTime{} = date) do
     date
     |> NaiveDateTime.to_iso8601()
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 0141186d8..89fd7629a 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -755,7 +755,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def follow(%{assigns: %{user: follower}} = conn, %{"uri" => uri}) do
-    with %User{} = followed <- Repo.get_by(User, nickname: uri),
+    with %User{} = followed <- User.get_by_nickname(uri),
          {:ok, follower, followed, _} <- CommonAPI.follow(follower, followed) do
       conn
       |> put_view(AccountView)
@@ -1121,7 +1121,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
             auto_play_gif: false,
             display_sensitive_media: false,
             reduce_motion: false,
-            max_toot_chars: limit
+            max_toot_chars: limit,
+            mascot: "/images/pleroma-fox-tan-smol.png"
           },
           rights: %{
             delete_others_notice: present?(user.info.is_moderator),
diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
index e817f0d79..3cdd7a2f2 100644
--- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex
+++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   require Logger
 
   alias Comeonin.Pbkdf2
+  alias Pleroma.Activity
   alias Pleroma.Emoji
   alias Pleroma.Notification
   alias Pleroma.PasswordResetToken
@@ -73,23 +74,39 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   end
 
   def remote_follow(%{assigns: %{user: user}} = conn, %{"acct" => acct}) do
-    {err, followee} = OStatus.find_or_make_user(acct)
-    avatar = User.avatar_url(followee)
-    name = followee.nickname
-    id = followee.id
-
-    if !!user do
-      conn
-      |> render("follow.html", %{error: err, acct: acct, avatar: avatar, name: name, id: id})
+    if is_status?(acct) do
+      {:ok, object} = ActivityPub.fetch_object_from_id(acct)
+      %Activity{id: activity_id} = Activity.get_create_by_object_ap_id(object.data["id"])
+      redirect(conn, to: "/notice/#{activity_id}")
     else
-      conn
-      |> render("follow_login.html", %{
-        error: false,
-        acct: acct,
-        avatar: avatar,
-        name: name,
-        id: id
-      })
+      {err, followee} = OStatus.find_or_make_user(acct)
+      avatar = User.avatar_url(followee)
+      name = followee.nickname
+      id = followee.id
+
+      if !!user do
+        conn
+        |> render("follow.html", %{error: err, acct: acct, avatar: avatar, name: name, id: id})
+      else
+        conn
+        |> render("follow_login.html", %{
+          error: false,
+          acct: acct,
+          avatar: avatar,
+          name: name,
+          id: id
+        })
+      end
+    end
+  end
+
+  defp is_status?(acct) do
+    case ActivityPub.fetch_and_contain_remote_object_from_id(acct) do
+      {:ok, %{"type" => type}} when type in ["Article", "Note", "Video", "Page", "Question"] ->
+        true
+
+      _ ->
+        false
     end
   end
 
diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex
index d0e58e71b..9b081a316 100644
--- a/lib/pleroma/web/twitter_api/twitter_api.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api.ex
@@ -227,12 +227,9 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
         end
 
       %{"screen_name" => nickname} ->
-        case target = Repo.get_by(User, nickname: nickname) do
-          nil ->
-            {:error, "No user with such screen_name"}
-
-          _ ->
-            {:ok, target}
+        case User.get_by_nickname(nickname) do
+          nil -> {:error, "No user with such screen_name"}
+          target -> {:ok, target}
         end
 
       _ ->