Merge remote-tracking branch 'origin/develop' into conversations_three

5 files changed, 447 insertions, 21 deletions

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
new file mode 100644
index 000000000..ab9a3a7ff
--- /dev/null
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -0,0 +1,51 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Database do
+  alias Mix.Tasks.Pleroma.Common
+  require Logger
+  use Mix.Task
+
+  @shortdoc "A collection of database related tasks"
+  @moduledoc """
+   A collection of database related tasks
+
+   ## Replace embedded objects with their references
+
+   Replaces embedded objects with references to them in the `objects` table. Only needs to be ran once. The reason why this is not a migration is because it could significantly increase the database size after being ran, however after this `VACUUM FULL` will be able to reclaim about 20% (really depends on what is in the database, your mileage may vary) of the db size before the migration.
+
+       mix pleroma.database remove_embedded_objects
+
+    Options:
+    - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
+  """
+  def run(["remove_embedded_objects" | args]) do
+    {options, [], []} =
+      OptionParser.parse(
+        args,
+        strict: [
+          vacuum: :boolean
+        ]
+      )
+
+    Common.start_pleroma()
+    Logger.info("Removing embedded objects")
+
+    Pleroma.Repo.query!(
+      "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
+      [],
+      timeout: :infinity
+    )
+
+    if Keyword.get(options, :vacuum) do
+      Logger.info("Runnning VACUUM FULL")
+
+      Pleroma.Repo.query!(
+        "vacuum full;",
+        [],
+        timeout: :infinity
+      )
+    end
+  end
+end
diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex
new file mode 100644
index 000000000..cced73226
--- /dev/null
+++ b/lib/mix/tasks/pleroma/emoji.ex
@@ -0,0 +1,293 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Emoji do
+  use Mix.Task
+
+  @shortdoc "Manages emoji packs"
+  @moduledoc """
+  Manages emoji packs
+
+  ## ls-packs
+
+      mix pleroma.emoji ls-packs [OPTION...]
+
+  Lists the emoji packs and metadata specified in the manifest.
+
+  ### Options
+
+  - `-m, --manifest PATH/URL` - path to a custom manifest, it can
+    either be an URL starting with `http`, in that case the
+    manifest will be fetched from that address, or a local path
+
+  ## get-packs
+
+      mix pleroma.emoji get-packs [OPTION...] PACKS
+
+  Fetches, verifies and installs the specified PACKS from the
+  manifest into the `STATIC-DIR/emoji/PACK-NAME`
+
+  ### Options
+
+  - `-m, --manifest PATH/URL` - same as ls-packs
+
+  ## gen-pack
+
+      mix pleroma.emoji gen-pack PACK-URL
+
+  Creates a new manifest entry and a file list from the specified
+  remote pack file. Currently, only .zip archives are recognized
+  as remote pack files and packs are therefore assumed to be zip
+  archives. This command is intended to run interactively and will
+  first ask you some basic questions about the pack, then download
+  the remote file and generate an SHA256 checksum for it, then
+  generate an emoji file list for you.
+
+  The manifest entry will either be written to a newly created
+  `index.json` file or appended to the existing one, *replacing*
+  the old pack with the same name if it was in the file previously.
+
+  The file list will be written to the file specified previously,
+  *replacing* that file. You _should_ check that the file list doesn't
+  contain anything you don't need in the pack, that is, anything that is
+  not an emoji (the whole pack is downloaded, but only emoji files
+  are extracted).
+  """
+
+  @default_manifest Pleroma.Config.get!([:emoji, :default_manifest])
+
+  def run(["ls-packs" | args]) do
+    Application.ensure_all_started(:hackney)
+
+    {options, [], []} = parse_global_opts(args)
+
+    manifest =
+      fetch_manifest(if options[:manifest], do: options[:manifest], else: @default_manifest)
+
+    Enum.each(manifest, fn {name, info} ->
+      to_print = [
+        {"Name", name},
+        {"Homepage", info["homepage"]},
+        {"Description", info["description"]},
+        {"License", info["license"]},
+        {"Source", info["src"]}
+      ]
+
+      for {param, value} <- to_print do
+        IO.puts(IO.ANSI.format([:bright, param, :normal, ": ", value]))
+      end
+
+      # A newline
+      IO.puts("")
+    end)
+  end
+
+  def run(["get-packs" | args]) do
+    Application.ensure_all_started(:hackney)
+
+    {options, pack_names, []} = parse_global_opts(args)
+
+    manifest_url = if options[:manifest], do: options[:manifest], else: @default_manifest
+
+    manifest = fetch_manifest(manifest_url)
+
+    for pack_name <- pack_names do
+      if Map.has_key?(manifest, pack_name) do
+        pack = manifest[pack_name]
+        src_url = pack["src"]
+
+        IO.puts(
+          IO.ANSI.format([
+            "Downloading ",
+            :bright,
+            pack_name,
+            :normal,
+            " from ",
+            :underline,
+            src_url
+          ])
+        )
+
+        binary_archive = Tesla.get!(src_url).body
+        archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16()
+
+        sha_status_text = ["SHA256 of ", :bright, pack_name, :normal, " source file is ", :bright]
+
+        if archive_sha == String.upcase(pack["src_sha256"]) do
+          IO.puts(IO.ANSI.format(sha_status_text ++ [:green, "OK"]))
+        else
+          IO.puts(IO.ANSI.format(sha_status_text ++ [:red, "BAD"]))
+
+          raise "Bad SHA256 for #{pack_name}"
+        end
+
+        # The url specified in files should be in the same directory
+        files_url = Path.join(Path.dirname(manifest_url), pack["files"])
+
+        IO.puts(
+          IO.ANSI.format([
+            "Fetching the file list for ",
+            :bright,
+            pack_name,
+            :normal,
+            " from ",
+            :underline,
+            files_url
+          ])
+        )
+
+        files = Tesla.get!(files_url).body |> Poison.decode!()
+
+        IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name]))
+
+        pack_path =
+          Path.join([
+            Pleroma.Config.get!([:instance, :static_dir]),
+            "emoji",
+            pack_name
+          ])
+
+        files_to_unzip =
+          Enum.map(
+            files,
+            fn {_, f} -> to_charlist(f) end
+          )
+
+        {:ok, _} =
+          :zip.unzip(binary_archive,
+            cwd: pack_path,
+            file_list: files_to_unzip
+          )
+
+        IO.puts(IO.ANSI.format(["Writing emoji.txt for ", :bright, pack_name]))
+
+        emoji_txt_str =
+          Enum.map(
+            files,
+            fn {shortcode, path} ->
+              emojo_path = Path.join("/emoji/#{pack_name}", path)
+              "#{shortcode}, #{emojo_path}"
+            end
+          )
+          |> Enum.join("\n")
+
+        File.write!(Path.join(pack_path, "emoji.txt"), emoji_txt_str)
+      else
+        IO.puts(IO.ANSI.format([:bright, :red, "No pack named \"#{pack_name}\" found"]))
+      end
+    end
+  end
+
+  def run(["gen-pack", src]) do
+    Application.ensure_all_started(:hackney)
+
+    proposed_name = Path.basename(src) |> Path.rootname()
+    name = String.trim(IO.gets("Pack name [#{proposed_name}]: "))
+    # If there's no name, use the default one
+    name = if String.length(name) > 0, do: name, else: proposed_name
+
+    license = String.trim(IO.gets("License: "))
+    homepage = String.trim(IO.gets("Homepage: "))
+    description = String.trim(IO.gets("Description: "))
+
+    proposed_files_name = "#{name}.json"
+    files_name = String.trim(IO.gets("Save file list to [#{proposed_files_name}]: "))
+    files_name = if String.length(files_name) > 0, do: files_name, else: proposed_files_name
+
+    default_exts = [".png", ".gif"]
+    default_exts_str = Enum.join(default_exts, " ")
+
+    exts =
+      String.trim(
+        IO.gets("Emoji file extensions (separated with spaces) [#{default_exts_str}]: ")
+      )
+
+    exts =
+      if String.length(exts) > 0 do
+        String.split(exts, " ")
+        |> Enum.filter(fn e -> e |> String.trim() |> String.length() > 0 end)
+      else
+        default_exts
+      end
+
+    IO.puts("Downloading the pack and generating SHA256")
+
+    binary_archive = Tesla.get!(src).body
+    archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16()
+
+    IO.puts("SHA256 is #{archive_sha}")
+
+    pack_json = %{
+      name => %{
+        license: license,
+        homepage: homepage,
+        description: description,
+        src: src,
+        src_sha256: archive_sha,
+        files: files_name
+      }
+    }
+
+    tmp_pack_dir = Path.join(System.tmp_dir!(), "emoji-pack-#{name}")
+
+    {:ok, _} =
+      :zip.unzip(
+        binary_archive,
+        cwd: tmp_pack_dir
+      )
+
+    emoji_map = Pleroma.Emoji.make_shortcode_to_file_map(tmp_pack_dir, exts)
+
+    File.write!(files_name, Poison.encode!(emoji_map, pretty: true))
+
+    IO.puts("""
+
+    #{files_name} has been created and contains the list of all found emojis in the pack.
+    Please review the files in the remove those not needed.
+    """)
+
+    if File.exists?("index.json") do
+      existing_data = File.read!("index.json") |> Poison.decode!()
+
+      File.write!(
+        "index.json",
+        Poison.encode!(
+          Map.merge(
+            existing_data,
+            pack_json
+          ),
+          pretty: true
+        )
+      )
+
+      IO.puts("index.json file has been update with the #{name} pack")
+    else
+      File.write!("index.json", Poison.encode!(pack_json, pretty: true))
+
+      IO.puts("index.json has been created with the #{name} pack")
+    end
+  end
+
+  defp fetch_manifest(from) do
+    Poison.decode!(
+      if String.starts_with?(from, "http") do
+        Tesla.get!(from).body
+      else
+        File.read!(from)
+      end
+    )
+  end
+
+  defp parse_global_opts(args) do
+    OptionParser.parse(
+      args,
+      strict: [
+        manifest: :string
+      ],
+      aliases: [
+        m: :manifest
+      ]
+    )
+  end
+end
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index 8f8d86a11..6cee8d630 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -24,10 +24,12 @@ defmodule Mix.Tasks.Pleroma.Instance do
   - `--domain DOMAIN` - the domain of your instance
   - `--instance-name INSTANCE_NAME` - the name of your instance
   - `--admin-email ADMIN_EMAIL` - the email address of the instance admin
+  - `--notify-email NOTIFY_EMAIL` - email address for notifications
   - `--dbhost HOSTNAME` - the hostname of the PostgreSQL database to use
   - `--dbname DBNAME` - the name of the database to use
   - `--dbuser DBUSER` - the user (aka role) to use for the database connection
   - `--dbpass DBPASS` - the password to use for the database connection
+  - `--indexable Y/N` - Allow/disallow indexing site by search engines
   """
 
   def run(["gen" | rest]) do
@@ -41,10 +43,12 @@ defmodule Mix.Tasks.Pleroma.Instance do
           domain: :string,
           instance_name: :string,
           admin_email: :string,
+          notify_email: :string,
           dbhost: :string,
           dbname: :string,
           dbuser: :string,
-          dbpass: :string
+          dbpass: :string,
+          indexable: :string
         ],
         aliases: [
           o: :output,
@@ -61,7 +65,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
     will_overwrite = Enum.filter(paths, &File.exists?/1)
     proceed? = Enum.empty?(will_overwrite) or Keyword.get(options, :force, false)
 
-    unless not proceed? do
+    if proceed? do
       [domain, port | _] =
         String.split(
           Common.get_option(
@@ -81,6 +85,14 @@ defmodule Mix.Tasks.Pleroma.Instance do
 
       email = Common.get_option(options, :admin_email, "What is your admin email address?")
 
+      notify_email =
+        Common.get_option(
+          options,
+          :notify_email,
+          "What email address do you want to use for sending email notifications?",
+          email
+        )
+
       indexable =
         Common.get_option(
           options,
@@ -122,6 +134,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
           domain: domain,
           port: port,
           email: email,
+          notify_email: notify_email,
           name: name,
           dbhost: dbhost,
           dbname: dbname,
diff --git a/lib/mix/tasks/pleroma/sample_config.eex b/lib/mix/tasks/pleroma/sample_config.eex
index 1c935c0d8..52bd57cb7 100644
--- a/lib/mix/tasks/pleroma/sample_config.eex
+++ b/lib/mix/tasks/pleroma/sample_config.eex
@@ -13,6 +13,7 @@ config :pleroma, Pleroma.Web.Endpoint,
 config :pleroma, :instance,
   name: "<%= name %>",
   email: "<%= email %>",
+  notify_email: "<%= notify_email %>",
   limit: 5000,
   registrations_open: true,
   dedupe_media: false
@@ -75,4 +76,3 @@ config :web_push_encryption, :vapid_details,
 #  storage_url: "https://swift-endpoint.prodider.com/v1/AUTH_<tenant>/<container>",
 #  object_url: "https://cdn-endpoint.provider.com/<container>"
 #
-
diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex
index 0d0bea8c0..9e2523b18 100644
--- a/lib/mix/tasks/pleroma/user.ex
+++ b/lib/mix/tasks/pleroma/user.ex
@@ -7,6 +7,7 @@ defmodule Mix.Tasks.Pleroma.User do
   import Ecto.Changeset
   alias Mix.Tasks.Pleroma.Common
   alias Pleroma.User
+  alias Pleroma.UserInviteToken
 
   @shortdoc "Manages Pleroma users"
   @moduledoc """
@@ -26,7 +27,19 @@ defmodule Mix.Tasks.Pleroma.User do
 
   ## Generate an invite link.
 
-      mix pleroma.user invite
+      mix pleroma.user invite [OPTION...]
+
+    Options:
+    - `--expires_at DATE` - last day on which token is active (e.g. "2019-04-05")
+    - `--max_use NUMBER` - maximum numbers of token uses
+
+  ## List generated invites
+
+      mix pleroma.user invites
+
+  ## Revoke invite
+
+      mix pleroma.user revoke_invite TOKEN OR TOKEN_ID
 
   ## Delete the user's account.
 
@@ -113,7 +126,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     proceed? = assume_yes? or Mix.shell().yes?("Continue?")
 
-    unless not proceed? do
+    if proceed? do
       Common.start_pleroma()
 
       params = %{
@@ -149,7 +162,7 @@ defmodule Mix.Tasks.Pleroma.User do
   def run(["rm", nickname]) do
     Common.start_pleroma()
 
-    with %User{local: true} = user <- User.get_by_nickname(nickname) do
+    with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
       User.delete(user)
       Mix.shell().info("User #{nickname} deleted.")
     else
@@ -161,7 +174,7 @@ defmodule Mix.Tasks.Pleroma.User do
   def run(["toggle_activated", nickname]) do
     Common.start_pleroma()
 
-    with %User{} = user <- User.get_by_nickname(nickname) do
+    with %User{} = user <- User.get_cached_by_nickname(nickname) do
       {:ok, user} = User.deactivate(user, !user.info.deactivated)
 
       Mix.shell().info(
@@ -176,7 +189,7 @@ defmodule Mix.Tasks.Pleroma.User do
   def run(["reset_password", nickname]) do
     Common.start_pleroma()
 
-    with %User{local: true} = user <- User.get_by_nickname(nickname),
+    with %User{local: true} = user <- User.get_cached_by_nickname(nickname),
          {:ok, token} <- Pleroma.PasswordResetToken.create_token(user) do
       Mix.shell().info("Generated password reset token for #{user.nickname}")
 
@@ -198,14 +211,14 @@ defmodule Mix.Tasks.Pleroma.User do
   def run(["unsubscribe", nickname]) do
     Common.start_pleroma()
 
-    with %User{} = user <- User.get_by_nickname(nickname) do
+    with %User{} = user <- User.get_cached_by_nickname(nickname) do
       Mix.shell().info("Deactivating #{user.nickname}")
       User.deactivate(user)
 
       {:ok, friends} = User.get_friends(user)
 
       Enum.each(friends, fn friend ->
-        user = User.get_by_id(user.id)
+        user = User.get_cached_by_id(user.id)
 
         Mix.shell().info("Unsubscribing #{friend.nickname} from #{user.nickname}")
         User.unfollow(user, friend)
@@ -213,7 +226,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
       :timer.sleep(500)
 
-      user = User.get_by_id(user.id)
+      user = User.get_cached_by_id(user.id)
 
       if Enum.empty?(user.following) do
         Mix.shell().info("Successfully unsubscribed all followers from #{user.nickname}")
@@ -237,7 +250,7 @@ defmodule Mix.Tasks.Pleroma.User do
         ]
       )
 
-    with %User{local: true} = user <- User.get_by_nickname(nickname) do
+    with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
       user =
         case Keyword.get(options, :moderator) do
           nil -> user
@@ -264,7 +277,7 @@ defmodule Mix.Tasks.Pleroma.User do
   def run(["tag", nickname | tags]) do
     Common.start_pleroma()
 
-    with %User{} = user <- User.get_by_nickname(nickname) do
+    with %User{} = user <- User.get_cached_by_nickname(nickname) do
       user = user |> User.tag(tags)
 
       Mix.shell().info("Tags of #{user.nickname}: #{inspect(tags)}")
@@ -277,7 +290,7 @@ defmodule Mix.Tasks.Pleroma.User do
   def run(["untag", nickname | tags]) do
     Common.start_pleroma()
 
-    with %User{} = user <- User.get_by_nickname(nickname) do
+    with %User{} = user <- User.get_cached_by_nickname(nickname) do
       user = user |> User.untag(tags)
 
       Mix.shell().info("Tags of #{user.nickname}: #{inspect(tags)}")
@@ -287,30 +300,86 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
-  def run(["invite"]) do
+  def run(["invite" | rest]) do
+    {options, [], []} =
+      OptionParser.parse(rest,
+        strict: [
+          expires_at: :string,
+          max_use: :integer
+        ]
+      )
+
+    options =
+      options
+      |> Keyword.update(:expires_at, {:ok, nil}, fn
+        nil -> {:ok, nil}
+        val -> Date.from_iso8601(val)
+      end)
+      |> Enum.into(%{})
+
     Common.start_pleroma()
 
-    with {:ok, token} <- Pleroma.UserInviteToken.create_token() do
-      Mix.shell().info("Generated user invite token")
+    with {:ok, val} <- options[:expires_at],
+         options = Map.put(options, :expires_at, val),
+         {:ok, invite} <- UserInviteToken.create_invite(options) do
+      Mix.shell().info(
+        "Generated user invite token " <> String.replace(invite.invite_type, "_", " ")
+      )
 
       url =
         Pleroma.Web.Router.Helpers.redirect_url(
           Pleroma.Web.Endpoint,
           :registration_page,
-          token.token
+          invite.token
         )
 
       IO.puts(url)
     else
-      _ ->
-        Mix.shell().error("Could not create invite token.")
+      error ->
+        Mix.shell().error("Could not create invite token: #{inspect(error)}")
+    end
+  end
+
+  def run(["invites"]) do
+    Common.start_pleroma()
+
+    Mix.shell().info("Invites list:")
+
+    UserInviteToken.list_invites()
+    |> Enum.each(fn invite ->
+      expire_info =
+        with expires_at when not is_nil(expires_at) <- invite.expires_at do
+          " | Expires at: #{Date.to_string(expires_at)}"
+        end
+
+      using_info =
+        with max_use when not is_nil(max_use) <- invite.max_use do
+          " | Max use: #{max_use}    Left use: #{max_use - invite.uses}"
+        end
+
+      Mix.shell().info(
+        "ID: #{invite.id} | Token: #{invite.token} | Token type: #{invite.invite_type} | Used: #{
+          invite.used
+        }#{expire_info}#{using_info}"
+      )
+    end)
+  end
+
+  def run(["revoke_invite", token]) do
+    Common.start_pleroma()
+
+    with {:ok, invite} <- UserInviteToken.find_by_token(token),
+         {:ok, _} <- UserInviteToken.update_invite(invite, %{used: true}) do
+      Mix.shell().info("Invite for token #{token} was revoked.")
+    else
+      _ -> Mix.shell().error("No invite found with token #{token}")
     end
   end
 
   def run(["delete_activities", nickname]) do
     Common.start_pleroma()
 
-    with %User{local: true} = user <- User.get_by_nickname(nickname) do
+    with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
       User.delete_user_activities(user)
       Mix.shell().info("User #{nickname} statuses deleted.")
     else