Merge branch 'release/2.2.0' into 'stable'v2.2.0

Release/2.2.0

See merge request pleroma/secteam/pleroma!19

16 files changed, 203 insertions, 48 deletions

diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex
index fe9b0d16c..49ba2aae4 100644
--- a/lib/mix/pleroma.ex
+++ b/lib/mix/pleroma.ex
@@ -18,6 +18,7 @@ defmodule Mix.Pleroma do
   @doc "Common functions to be reused in mix tasks"
   def start_pleroma do
     Pleroma.Config.Holder.save_default()
+    Pleroma.Config.Oban.warn()
     Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
 
     if Pleroma.Config.get(:env) != :test do
diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex
index dd2b9c8f2..a607d5d4f 100644
--- a/lib/mix/tasks/pleroma/benchmark.ex
+++ b/lib/mix/tasks/pleroma/benchmark.ex
@@ -91,20 +91,17 @@ defmodule Mix.Tasks.Pleroma.Benchmark do
         "Without conn and without pool" => fn ->
           {:ok, %Tesla.Env{}} =
             Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              adapter: [pool: :no_pool, receive_conn: false]
+              pool: :no_pool,
+              receive_conn: false
             )
         end,
         "Without conn and with pool" => fn ->
           {:ok, %Tesla.Env{}} =
-            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              adapter: [receive_conn: false]
-            )
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false)
         end,
         "With reused conn and without pool" => fn ->
           {:ok, %Tesla.Env{}} =
-            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              adapter: [pool: :no_pool]
-            )
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool)
         end,
         "With reused conn and with pool" => fn ->
           {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex
index 904c5a74b..18f99318d 100644
--- a/lib/mix/tasks/pleroma/config.ex
+++ b/lib/mix/tasks/pleroma/config.ex
@@ -32,7 +32,8 @@ defmodule Mix.Tasks.Pleroma.Config do
 
   @spec migrate_to_db(Path.t() | nil) :: any()
   def migrate_to_db(file_path \\ nil) do
-    if Pleroma.Config.get([:configurable_from_database]) do
+    with true <- Pleroma.Config.get([:configurable_from_database]),
+         :ok <- Pleroma.Config.DeprecationWarnings.warn() do
       config_file =
         if file_path do
           file_path
@@ -46,7 +47,8 @@ defmodule Mix.Tasks.Pleroma.Config do
 
       do_migrate_to_db(config_file)
     else
-      migration_error()
+      :error -> deprecation_error()
+      _ -> migration_error()
     end
   end
 
@@ -120,6 +122,10 @@ defmodule Mix.Tasks.Pleroma.Config do
     )
   end
 
+  defp deprecation_error do
+    shell_error("Migration is not allowed until all deprecation warnings have been resolved.")
+  end
+
   if Code.ensure_loaded?(Config.Reader) do
     defp config_header, do: "import Config\r\n\r\n"
     defp read_file(config_file), do: Config.Reader.read_imports!(config_file)
diff --git a/lib/mix/tasks/pleroma/count_statuses.ex b/lib/mix/tasks/pleroma/count_statuses.ex
index e1e8195dd..8761d8f17 100644
--- a/lib/mix/tasks/pleroma/count_statuses.ex
+++ b/lib/mix/tasks/pleroma/count_statuses.ex
@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.CountStatuses do
   @shortdoc "Re-counts statuses for all users"
 
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index 7d8f00b08..a01c36ece 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -99,7 +99,7 @@ defmodule Mix.Tasks.Pleroma.Database do
       where: fragment("(?)->>'likes' is not null", object.data),
       select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)}
     )
-    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Pleroma.Repo.chunk_stream(100, :batches)
     |> Stream.each(fn objects ->
       ids =
         objects
@@ -133,8 +133,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)
 
     Pleroma.Activity
-    |> join(:left, [a], u in assoc(a, :expiration))
-    |> join(:inner, [a, _u], o in Object,
+    |> join(:inner, [a], o in Object,
       on:
         fragment(
           "(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
@@ -144,14 +143,20 @@ defmodule Mix.Tasks.Pleroma.Database do
         )
     )
     |> where(local: true)
-    |> where([a, u], is_nil(u))
     |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
-    |> where([_a, _u, o], fragment("?->>'type' = 'Note'", o.data))
-    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> where([_a, o], fragment("?->>'type' = 'Note'", o.data))
+    |> Pleroma.Repo.chunk_stream(100, :batches)
     |> Stream.each(fn activities ->
       Enum.each(activities, fn activity ->
-        expires_at = Timex.shift(activity.inserted_at, days: days)
-        Pleroma.ActivityExpiration.create(activity, expires_at, false)
+        expires_at =
+          activity.inserted_at
+          |> DateTime.from_naive!("Etc/UTC")
+          |> Timex.shift(days: days)
+
+        Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
+          activity_id: activity.id,
+          expires_at: expires_at
+        })
       end)
     end)
     |> Stream.run()
diff --git a/lib/mix/tasks/pleroma/digest.ex b/lib/mix/tasks/pleroma/digest.ex
index 3595f912d..cac148b88 100644
--- a/lib/mix/tasks/pleroma/digest.ex
+++ b/lib/mix/tasks/pleroma/digest.ex
@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Digest do
   use Mix.Task
   import Mix.Pleroma
diff --git a/lib/mix/tasks/pleroma/docs.ex b/lib/mix/tasks/pleroma/docs.ex
index 6088fc71d..ad5c37fc9 100644
--- a/lib/mix/tasks/pleroma/docs.ex
+++ b/lib/mix/tasks/pleroma/docs.ex
@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Docs do
   use Mix.Task
   import Mix.Pleroma
diff --git a/lib/mix/tasks/pleroma/ecto/ecto.ex b/lib/mix/tasks/pleroma/ecto.ex
index 3363cd45f..3363cd45f 100644
--- a/lib/mix/tasks/pleroma/ecto/ecto.ex
+++ b/lib/mix/tasks/pleroma/ecto.ex
diff --git a/lib/mix/tasks/pleroma/email.ex b/lib/mix/tasks/pleroma/email.ex
index d3fac6ec8..bc5facc09 100644
--- a/lib/mix/tasks/pleroma/email.ex
+++ b/lib/mix/tasks/pleroma/email.ex
@@ -1,12 +1,16 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Email do
   use Mix.Task
   import Mix.Pleroma
 
-  @shortdoc "Simple Email test"
+  @shortdoc "Email administrative tasks"
   @moduledoc File.read!("docs/administration/CLI_tasks/email.md")
 
   def run(["test" | args]) do
-    Mix.Pleroma.start_pleroma()
+    start_pleroma()
 
     {options, [], []} =
       OptionParser.parse(
@@ -21,4 +25,20 @@ defmodule Mix.Tasks.Pleroma.Email do
 
     shell_info("Test email has been sent to #{inspect(email.to)} from #{inspect(email.from)}")
   end
+
+  def run(["resend_confirmation_emails"]) do
+    start_pleroma()
+
+    shell_info("Sending emails to all unconfirmed users")
+
+    Pleroma.User.Query.build(%{
+      local: true,
+      deactivated: false,
+      confirmation_pending: true,
+      invisible: false
+    })
+    |> Pleroma.Repo.chunk_stream(500)
+    |> Stream.each(&Pleroma.User.try_send_confirmation_email(&1))
+    |> Stream.run()
+  end
 end
diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex
index 8f52ee98d..1750373f9 100644
--- a/lib/mix/tasks/pleroma/emoji.ex
+++ b/lib/mix/tasks/pleroma/emoji.ex
@@ -183,7 +183,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
 
     IO.puts("Downloading the pack and generating SHA256")
 
-    binary_archive = Tesla.get!(client(), src).body
+    {:ok, %{body: binary_archive}} = Pleroma.HTTP.get(src)
     archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16()
 
     IO.puts("SHA256 is #{archive_sha}")
@@ -252,7 +252,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
   end
 
   defp fetch("http" <> _ = from) do
-    with {:ok, %{body: body}} <- Tesla.get(client(), from) do
+    with {:ok, %{body: body}} <- Pleroma.HTTP.get(from) do
       {:ok, body}
     end
   end
@@ -271,13 +271,5 @@ defmodule Mix.Tasks.Pleroma.Emoji do
     )
   end
 
-  defp client do
-    middleware = [
-      {Tesla.Middleware.FollowRedirects, [max_redirects: 3]}
-    ]
-
-    Tesla.client(middleware)
-  end
-
   defp default_manifest, do: Pleroma.Config.get!([:emoji, :default_manifest])
 end
diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex
index 484af6da7..cbce81ab9 100644
--- a/lib/mix/tasks/pleroma/frontend.ex
+++ b/lib/mix/tasks/pleroma/frontend.ex
@@ -69,7 +69,7 @@ defmodule Mix.Tasks.Pleroma.Frontend do
 
     fe_label = "#{frontend} (#{ref})"
 
-    tmp_dir = Path.join(dest, "tmp")
+    tmp_dir = Path.join([instance_static_dir, "frontends", "tmp"])
 
     with {_, :ok} <-
            {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, options[:file])},
@@ -124,9 +124,7 @@ defmodule Mix.Tasks.Pleroma.Frontend do
     url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
 
     with {:ok, %{status: 200, body: zip_body}} <-
-           Pleroma.HTTP.get(url, [],
-             adapter: [pool: :media, timeout: 120_000, recv_timeout: 120_000]
-           ) do
+           Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
       unzip(zip_body, dest)
     else
       e -> {:error, e}
@@ -135,6 +133,7 @@ defmodule Mix.Tasks.Pleroma.Frontend do
 
   defp install_frontend(frontend_info, source, dest) do
     from = frontend_info["build_dir"] || "dist"
+    File.rm_rf!(dest)
     File.mkdir_p!(dest)
     File.cp_r!(Path.join([source, from]), dest)
     :ok
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index 91440b453..fc21ae062 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -33,7 +33,10 @@ defmodule Mix.Tasks.Pleroma.Instance do
           uploads_dir: :string,
           static_dir: :string,
           listen_ip: :string,
-          listen_port: :string
+          listen_port: :string,
+          strip_uploads: :string,
+          anonymize_uploads: :string,
+          dedupe_uploads: :string
         ],
         aliases: [
           o: :output,
@@ -158,6 +161,30 @@ defmodule Mix.Tasks.Pleroma.Instance do
         )
         |> Path.expand()
 
+      strip_uploads =
+        get_option(
+          options,
+          :strip_uploads,
+          "Do you want to strip location (GPS) data from uploaded images? (y/n)",
+          "y"
+        ) === "y"
+
+      anonymize_uploads =
+        get_option(
+          options,
+          :anonymize_uploads,
+          "Do you want to anonymize the filenames of uploads? (y/n)",
+          "n"
+        ) === "y"
+
+      dedupe_uploads =
+        get_option(
+          options,
+          :dedupe_uploads,
+          "Do you want to deduplicate uploaded files? (y/n)",
+          "n"
+        ) === "y"
+
       Config.put([:instance, :static_dir], static_dir)
 
       secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
@@ -188,7 +215,13 @@ defmodule Mix.Tasks.Pleroma.Instance do
           uploads_dir: uploads_dir,
           rum_enabled: rum_enabled,
           listen_ip: listen_ip,
-          listen_port: listen_port
+          listen_port: listen_port,
+          upload_filters:
+            upload_filters(%{
+              strip: strip_uploads,
+              anonymize: anonymize_uploads,
+              dedupe: dedupe_uploads
+            })
         )
 
       result_psql =
@@ -247,4 +280,31 @@ defmodule Mix.Tasks.Pleroma.Instance do
     File.write(robots_txt_path, robots_txt)
     shell_info("Writing #{robots_txt_path}.")
   end
+
+  defp upload_filters(filters) when is_map(filters) do
+    enabled_filters =
+      if filters.strip do
+        [Pleroma.Upload.Filter.ExifTool]
+      else
+        []
+      end
+
+    enabled_filters =
+      if filters.anonymize do
+        enabled_filters ++ [Pleroma.Upload.Filter.AnonymizeFilename]
+      else
+        enabled_filters
+      end
+
+    enabled_filters =
+      if filters.dedupe do
+        enabled_filters ++ [Pleroma.Upload.Filter.Dedupe]
+      else
+        enabled_filters
+      end
+
+    enabled_filters
+  end
+
+  defp upload_filters(_), do: []
 end
diff --git a/lib/mix/tasks/pleroma/notification_settings.ex b/lib/mix/tasks/pleroma/notification_settings.ex
index 00f5ba7bf..f99275de1 100644
--- a/lib/mix/tasks/pleroma/notification_settings.ex
+++ b/lib/mix/tasks/pleroma/notification_settings.ex
@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.NotificationSettings do
   @shortdoc "Enable&Disable privacy option for push notifications"
   @moduledoc """
diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex
index a6d8d6c1c..bb808ca47 100644
--- a/lib/mix/tasks/pleroma/relay.ex
+++ b/lib/mix/tasks/pleroma/relay.ex
@@ -21,10 +21,19 @@ defmodule Mix.Tasks.Pleroma.Relay do
     end
   end
 
-  def run(["unfollow", target]) do
+  def run(["unfollow", target | rest]) do
     start_pleroma()
 
-    with {:ok, _activity} <- Relay.unfollow(target) do
+    {options, [], []} =
+      OptionParser.parse(
+        rest,
+        strict: [force: :boolean],
+        aliases: [f: :force]
+      )
+
+    force = Keyword.get(options, :force, false)
+
+    with {:ok, _activity} <- Relay.unfollow(target, %{force: force}) do
       # put this task to sleep to allow the genserver to push out the messages
       :timer.sleep(500)
     else
diff --git a/lib/mix/tasks/pleroma/robotstxt.ex b/lib/mix/tasks/pleroma/robots_txt.ex
index 24f08180e..24f08180e 100644
--- a/lib/mix/tasks/pleroma/robotstxt.ex
+++ b/lib/mix/tasks/pleroma/robots_txt.ex
diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex
index 01824aa18..a8d251411 100644
--- a/lib/mix/tasks/pleroma/user.ex
+++ b/lib/mix/tasks/pleroma/user.ex
@@ -179,7 +179,7 @@ defmodule Mix.Tasks.Pleroma.User do
     start_pleroma()
 
     Pleroma.User.Query.build(%{nickname: "@#{instance}"})
-    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
@@ -196,17 +196,24 @@ defmodule Mix.Tasks.Pleroma.User do
       OptionParser.parse(
         rest,
         strict: [
-          moderator: :boolean,
           admin: :boolean,
-          locked: :boolean
+          confirmed: :boolean,
+          locked: :boolean,
+          moderator: :boolean
         ]
       )
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
       user =
-        case Keyword.get(options, :moderator) do
+        case Keyword.get(options, :admin) do
           nil -> user
-          value -> set_moderator(user, value)
+          value -> set_admin(user, value)
+        end
+
+      user =
+        case Keyword.get(options, :confirmed) do
+          nil -> user
+          value -> set_confirmed(user, value)
         end
 
       user =
@@ -216,9 +223,9 @@ defmodule Mix.Tasks.Pleroma.User do
         end
 
       _user =
-        case Keyword.get(options, :admin) do
+        case Keyword.get(options, :moderator) do
           nil -> user
-          value -> set_admin(user, value)
+          value -> set_moderator(user, value)
         end
     else
       _ ->
@@ -353,6 +360,42 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
+  def run(["confirm_all"]) do
+    start_pleroma()
+
+    Pleroma.User.Query.build(%{
+      local: true,
+      deactivated: false,
+      is_moderator: false,
+      is_admin: false,
+      invisible: false
+    })
+    |> Pleroma.Repo.chunk_stream(500, :batches)
+    |> Stream.each(fn users ->
+      users
+      |> Enum.each(fn user -> User.need_confirmation(user, false) end)
+    end)
+    |> Stream.run()
+  end
+
+  def run(["unconfirm_all"]) do
+    start_pleroma()
+
+    Pleroma.User.Query.build(%{
+      local: true,
+      deactivated: false,
+      is_moderator: false,
+      is_admin: false,
+      invisible: false
+    })
+    |> Pleroma.Repo.chunk_stream(500, :batches)
+    |> Stream.each(fn users ->
+      users
+      |> Enum.each(fn user -> User.need_confirmation(user, true) end)
+    end)
+    |> Stream.run()
+  end
+
   def run(["sign_out", nickname]) do
     start_pleroma()
 
@@ -370,13 +413,13 @@ defmodule Mix.Tasks.Pleroma.User do
     start_pleroma()
 
     Pleroma.User.Query.build(%{local: true})
-    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
         shell_info(
           "#{user.nickname} moderator: #{user.is_moderator}, admin: #{user.is_admin}, locked: #{
-            user.locked
+            user.is_locked
           }, deactivated: #{user.deactivated}"
         )
       end)
@@ -404,10 +447,17 @@ defmodule Mix.Tasks.Pleroma.User do
   defp set_locked(user, value) do
     {:ok, user} =
       user
-      |> Changeset.change(%{locked: value})
+      |> Changeset.change(%{is_locked: value})
       |> User.update_and_set_cache()
 
-    shell_info("Locked status of #{user.nickname}: #{user.locked}")
+    shell_info("Locked status of #{user.nickname}: #{user.is_locked}")
+    user
+  end
+
+  defp set_confirmed(user, value) do
+    {:ok, user} = User.need_confirmation(user, !value)
+
+    shell_info("Confirmation pending status of #{user.nickname}: #{user.confirmation_pending}")
     user
   end
 end