Merge remote-tracking branch 'origin/develop' into feature/embeddable-posts

7 files changed, 185 insertions, 89 deletions

diff --git a/lib/mix/tasks/pleroma/app.ex b/lib/mix/tasks/pleroma/app.ex
new file mode 100644
index 000000000..463e2449f
--- /dev/null
+++ b/lib/mix/tasks/pleroma/app.ex
@@ -0,0 +1,49 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.App do
+  @moduledoc File.read!("docs/administration/CLI_tasks/oauth_app.md")
+  use Mix.Task
+
+  import Mix.Pleroma
+
+  @shortdoc "Creates trusted OAuth App"
+
+  def run(["create" | options]) do
+    start_pleroma()
+
+    {opts, _} =
+      OptionParser.parse!(options,
+        strict: [name: :string, redirect_uri: :string, scopes: :string],
+        aliases: [n: :name, r: :redirect_uri, s: :scopes]
+      )
+
+    scopes =
+      if opts[:scopes] do
+        String.split(opts[:scopes], ",")
+      else
+        ["read", "write", "follow", "push"]
+      end
+
+    params = %{
+      client_name: opts[:name],
+      redirect_uris: opts[:redirect_uri],
+      trusted: true,
+      scopes: scopes
+    }
+
+    with {:ok, app} <- Pleroma.Web.OAuth.App.create(params) do
+      shell_info("#{app.client_name} successfully created:")
+      shell_info("App client_id: " <> app.client_id)
+      shell_info("App client_secret: " <> app.client_secret)
+    else
+      {:error, changeset} ->
+        shell_error("Creating failed:")
+
+        Enum.each(Pleroma.Web.OAuth.App.errors(changeset), fn {key, error} ->
+          shell_error("#{key}: #{error}")
+        end)
+    end
+  end
+end
diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex
index a4885b70c..dd2b9c8f2 100644
--- a/lib/mix/tasks/pleroma/benchmark.ex
+++ b/lib/mix/tasks/pleroma/benchmark.ex
@@ -74,4 +74,43 @@ defmodule Mix.Tasks.Pleroma.Benchmark do
       inputs: inputs
     )
   end
+
+  def run(["adapters"]) do
+    start_pleroma()
+
+    :ok =
+      Pleroma.Gun.Conn.open(
+        "https://httpbin.org/stream-bytes/1500",
+        :gun_connections
+      )
+
+    Process.sleep(1_500)
+
+    Benchee.run(
+      %{
+        "Without conn and without pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [pool: :no_pool, receive_conn: false]
+            )
+        end,
+        "Without conn and with pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [receive_conn: false]
+            )
+        end,
+        "With reused conn and without pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [pool: :no_pool]
+            )
+        end,
+        "With reused conn and with pool" => fn ->
+          {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
+        end
+      },
+      parallel: 10
+    )
+  end
 end
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index 778de162f..82e2abdcb 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -4,6 +4,7 @@
 
 defmodule Mix.Tasks.Pleroma.Database do
   alias Pleroma.Conversation
+  alias Pleroma.Maintenance
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
@@ -34,13 +35,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     )
 
     if Keyword.get(options, :vacuum) do
-      Logger.info("Runnning VACUUM FULL")
-
-      Repo.query!(
-        "vacuum full;",
-        [],
-        timeout: :infinity
-      )
+      Maintenance.vacuum("full")
     end
   end
 
@@ -94,13 +89,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     |> Repo.delete_all(timeout: :infinity)
 
     if Keyword.get(options, :vacuum) do
-      Logger.info("Runnning VACUUM FULL")
-
-      Repo.query!(
-        "vacuum full;",
-        [],
-        timeout: :infinity
-      )
+      Maintenance.vacuum("full")
     end
   end
 
@@ -135,4 +124,10 @@ defmodule Mix.Tasks.Pleroma.Database do
     end)
     |> Stream.run()
   end
+
+  def run(["vacuum", args]) do
+    start_pleroma()
+
+    Maintenance.vacuum(args)
+  end
 end
diff --git a/lib/mix/tasks/pleroma/digest.ex b/lib/mix/tasks/pleroma/digest.ex
index 7d09e70c5..3595f912d 100644
--- a/lib/mix/tasks/pleroma/digest.ex
+++ b/lib/mix/tasks/pleroma/digest.ex
@@ -1,5 +1,6 @@
 defmodule Mix.Tasks.Pleroma.Digest do
   use Mix.Task
+  import Mix.Pleroma
 
   @shortdoc "Manages digest emails"
   @moduledoc File.read!("docs/administration/CLI_tasks/digest.md")
@@ -22,12 +23,10 @@ defmodule Mix.Tasks.Pleroma.Digest do
     with %Swoosh.Email{} = email <- Pleroma.Emails.UserEmail.digest_email(patched_user) do
       {:ok, _} = Pleroma.Emails.Mailer.deliver(email)
 
-      Mix.shell().info("Digest email have been sent to #{nickname} (#{user.email})")
+      shell_info("Digest email have been sent to #{nickname} (#{user.email})")
     else
       _ ->
-        Mix.shell().info(
-          "Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}"
-        )
+        shell_info("Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}")
     end
   end
 end
diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex
index 2b03a3009..29a5fa99c 100644
--- a/lib/mix/tasks/pleroma/emoji.ex
+++ b/lib/mix/tasks/pleroma/emoji.ex
@@ -4,18 +4,18 @@
 
 defmodule Mix.Tasks.Pleroma.Emoji do
   use Mix.Task
+  import Mix.Pleroma
 
   @shortdoc "Manages emoji packs"
   @moduledoc File.read!("docs/administration/CLI_tasks/emoji.md")
 
   def run(["ls-packs" | args]) do
-    Mix.Pleroma.start_pleroma()
-    Application.ensure_all_started(:hackney)
+    start_pleroma()
 
     {options, [], []} = parse_global_opts(args)
 
-    manifest =
-      fetch_manifest(if options[:manifest], do: options[:manifest], else: default_manifest())
+    url_or_path = options[:manifest] || default_manifest()
+    manifest = fetch_and_decode(url_or_path)
 
     Enum.each(manifest, fn {name, info} ->
       to_print = [
@@ -36,19 +36,18 @@ defmodule Mix.Tasks.Pleroma.Emoji do
   end
 
   def run(["get-packs" | args]) do
-    Mix.Pleroma.start_pleroma()
-    Application.ensure_all_started(:hackney)
+    start_pleroma()
 
     {options, pack_names, []} = parse_global_opts(args)
 
-    manifest_url = if options[:manifest], do: options[:manifest], else: default_manifest()
+    url_or_path = options[:manifest] || default_manifest()
 
-    manifest = fetch_manifest(manifest_url)
+    manifest = fetch_and_decode(url_or_path)
 
     for pack_name <- pack_names do
       if Map.has_key?(manifest, pack_name) do
         pack = manifest[pack_name]
-        src_url = pack["src"]
+        src = pack["src"]
 
         IO.puts(
           IO.ANSI.format([
@@ -58,11 +57,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do
             :normal,
             " from ",
             :underline,
-            src_url
+            src
           ])
         )
 
-        binary_archive = Tesla.get!(client(), src_url).body
+        {:ok, binary_archive} = fetch(src)
         archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16()
 
         sha_status_text = ["SHA256 of ", :bright, pack_name, :normal, " source file is ", :bright]
@@ -75,8 +74,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do
           raise "Bad SHA256 for #{pack_name}"
         end
 
-        # The url specified in files should be in the same directory
-        files_url = Path.join(Path.dirname(manifest_url), pack["files"])
+        # The location specified in files should be in the same directory
+        files_loc =
+          url_or_path
+          |> Path.dirname()
+          |> Path.join(pack["files"])
 
         IO.puts(
           IO.ANSI.format([
@@ -86,11 +88,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do
             :normal,
             " from ",
             :underline,
-            files_url
+            files_loc
           ])
         )
 
-        files = Tesla.get!(client(), files_url).body |> Jason.decode!()
+        files = fetch_and_decode(files_loc)
 
         IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name]))
 
@@ -134,38 +136,51 @@ defmodule Mix.Tasks.Pleroma.Emoji do
     end
   end
 
-  def run(["gen-pack", src]) do
-    Application.ensure_all_started(:hackney)
+  def run(["gen-pack" | args]) do
+    start_pleroma()
+
+    {opts, [src], []} =
+      OptionParser.parse(
+        args,
+        strict: [
+          name: :string,
+          license: :string,
+          homepage: :string,
+          description: :string,
+          files: :string,
+          extensions: :string
+        ]
+      )
 
     proposed_name = Path.basename(src) |> Path.rootname()
-    name = String.trim(IO.gets("Pack name [#{proposed_name}]: "))
-    # If there's no name, use the default one
-    name = if String.length(name) > 0, do: name, else: proposed_name
-
-    license = String.trim(IO.gets("License: "))
-    homepage = String.trim(IO.gets("Homepage: "))
-    description = String.trim(IO.gets("Description: "))
+    name = get_option(opts, :name, "Pack name:", proposed_name)
+    license = get_option(opts, :license, "License:")
+    homepage = get_option(opts, :homepage, "Homepage:")
+    description = get_option(opts, :description, "Description:")
 
-    proposed_files_name = "#{name}.json"
-    files_name = String.trim(IO.gets("Save file list to [#{proposed_files_name}]: "))
-    files_name = if String.length(files_name) > 0, do: files_name, else: proposed_files_name
+    proposed_files_name = "#{name}_files.json"
+    files_name = get_option(opts, :files, "Save file list to:", proposed_files_name)
 
     default_exts = [".png", ".gif"]
-    default_exts_str = Enum.join(default_exts, " ")
 
-    exts =
-      String.trim(
-        IO.gets("Emoji file extensions (separated with spaces) [#{default_exts_str}]: ")
+    custom_exts =
+      get_option(
+        opts,
+        :extensions,
+        "Emoji file extensions (separated with spaces):",
+        Enum.join(default_exts, " ")
       )
+      |> String.split(" ", trim: true)
 
     exts =
-      if String.length(exts) > 0 do
-        String.split(exts, " ")
-        |> Enum.filter(fn e -> e |> String.trim() |> String.length() > 0 end)
-      else
+      if MapSet.equal?(MapSet.new(default_exts), MapSet.new(custom_exts)) do
         default_exts
+      else
+        custom_exts
       end
 
+    IO.puts("Using #{Enum.join(exts, " ")} extensions")
+
     IO.puts("Downloading the pack and generating SHA256")
 
     binary_archive = Tesla.get!(client(), src).body
@@ -195,14 +210,16 @@ defmodule Mix.Tasks.Pleroma.Emoji do
     IO.puts("""
 
     #{files_name} has been created and contains the list of all found emojis in the pack.
-    Please review the files in the remove those not needed.
+    Please review the files in the pack and remove those not needed.
     """)
 
-    if File.exists?("index.json") do
-      existing_data = File.read!("index.json") |> Jason.decode!()
+    pack_file = "#{name}.json"
+
+    if File.exists?(pack_file) do
+      existing_data = File.read!(pack_file) |> Jason.decode!()
 
       File.write!(
-        "index.json",
+        pack_file,
         Jason.encode!(
           Map.merge(
             existing_data,
@@ -212,24 +229,28 @@ defmodule Mix.Tasks.Pleroma.Emoji do
         )
       )
 
-      IO.puts("index.json file has been update with the #{name} pack")
+      IO.puts("#{pack_file} has been updated with the #{name} pack")
     else
-      File.write!("index.json", Jason.encode!(pack_json, pretty: true))
+      File.write!(pack_file, Jason.encode!(pack_json, pretty: true))
 
-      IO.puts("index.json has been created with the #{name} pack")
+      IO.puts("#{pack_file} has been created with the #{name} pack")
     end
   end
 
-  defp fetch_manifest(from) do
-    Jason.decode!(
-      if String.starts_with?(from, "http") do
-        Tesla.get!(client(), from).body
-      else
-        File.read!(from)
-      end
-    )
+  defp fetch_and_decode(from) do
+    with {:ok, json} <- fetch(from) do
+      Jason.decode!(json)
+    end
   end
 
+  defp fetch("http" <> _ = from) do
+    with {:ok, %{body: body}} <- Tesla.get(client(), from) do
+      {:ok, body}
+    end
+  end
+
+  defp fetch(path), do: File.read(path)
+
   defp parse_global_opts(args) do
     OptionParser.parse(
       args,
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index bc842a59f..86409738a 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -147,6 +147,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
           "What directory should media uploads go in (when using the local uploader)?",
           Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads])
         )
+        |> Path.expand()
 
       static_dir =
         get_option(
@@ -155,6 +156,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
           "What directory should custom public files be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)?",
           Pleroma.Config.get([:instance, :static_dir])
         )
+        |> Path.expand()
 
       Config.put([:instance, :static_dir], static_dir)
 
@@ -204,7 +206,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
       shell_info("Writing the postgres script to #{psql_path}.")
       File.write(psql_path, result_psql)
 
-      write_robots_txt(indexable, template_dir)
+      write_robots_txt(static_dir, indexable, template_dir)
 
       shell_info(
         "\n All files successfully written! Refer to the installation instructions for your platform for next steps."
@@ -224,15 +226,13 @@ defmodule Mix.Tasks.Pleroma.Instance do
     end
   end
 
-  defp write_robots_txt(indexable, template_dir) do
+  defp write_robots_txt(static_dir, indexable, template_dir) do
     robots_txt =
       EEx.eval_file(
         template_dir <> "/robots_txt.eex",
         indexable: indexable
       )
 
-    static_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static/")
-
     unless File.exists?(static_dir) do
       File.mkdir_p!(static_dir)
     end
diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex
index 40dd9bdc0..3635c02bc 100644
--- a/lib/mix/tasks/pleroma/user.ex
+++ b/lib/mix/tasks/pleroma/user.ex
@@ -8,6 +8,8 @@ defmodule Mix.Tasks.Pleroma.User do
   alias Ecto.Changeset
   alias Pleroma.User
   alias Pleroma.UserInviteToken
+  alias Pleroma.Web.ActivityPub.Builder
+  alias Pleroma.Web.ActivityPub.Pipeline
 
   @shortdoc "Manages Pleroma users"
   @moduledoc File.read!("docs/administration/CLI_tasks/user.md")
@@ -96,8 +98,9 @@ defmodule Mix.Tasks.Pleroma.User do
   def run(["rm", nickname]) do
     start_pleroma()
 
-    with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
-      User.perform(:delete, user)
+    with %User{local: true} = user <- User.get_cached_by_nickname(nickname),
+         {:ok, delete_data, _} <- Builder.delete(user, user.ap_id),
+         {:ok, _delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do
       shell_info("User #{nickname} deleted.")
     else
       _ -> shell_error("No local user #{nickname}")
@@ -141,28 +144,18 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
-  def run(["unsubscribe", nickname]) do
+  def run(["deactivate", nickname]) do
     start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
       shell_info("Deactivating #{user.nickname}")
       User.deactivate(user)
-
-      user
-      |> User.get_friends()
-      |> Enum.each(fn friend ->
-        user = User.get_cached_by_id(user.id)
-
-        shell_info("Unsubscribing #{friend.nickname} from #{user.nickname}")
-        User.unfollow(user, friend)
-      end)
-
       :timer.sleep(500)
 
       user = User.get_cached_by_id(user.id)
 
-      if Enum.empty?(User.get_friends(user)) do
-        shell_info("Successfully unsubscribed all followers from #{user.nickname}")
+      if Enum.empty?(Enum.filter(User.get_friends(user), & &1.local)) do
+        shell_info("Successfully unsubscribed all local followers from #{user.nickname}")
       end
     else
       _ ->
@@ -170,7 +163,7 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
-  def run(["unsubscribe_all_from_instance", instance]) do
+  def run(["deactivate_all_from_instance", instance]) do
     start_pleroma()
 
     Pleroma.User.Query.build(%{nickname: "@#{instance}"})
@@ -178,7 +171,7 @@ defmodule Mix.Tasks.Pleroma.User do
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
-        run(["unsubscribe", user.nickname])
+        run(["deactivate", user.nickname])
       end)
     end)
     |> Stream.run()