diff options
Diffstat (limited to 'lib')
282 files changed, 7028 insertions, 3594 deletions
diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 3ad6edbfb..49ba2aae4 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -3,15 +3,54 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Pleroma do + @apps [ + :restarter, + :ecto, + :ecto_sql, + :postgrex, + :db_connection, + :cachex, + :flake_id, + :swoosh, + :timex + ] + @cachex_children ["object", "user", "scrubber"] @doc "Common functions to be reused in mix tasks" def start_pleroma do + Pleroma.Config.Holder.save_default() + Pleroma.Config.Oban.warn() Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) if Pleroma.Config.get(:env) != :test do Application.put_env(:logger, :console, level: :debug) end - {:ok, _} = Application.ensure_all_started(:pleroma) + adapter = Application.get_env(:tesla, :adapter) + + apps = + if adapter == Tesla.Adapter.Gun do + [:gun | @apps] + else + [:hackney | @apps] + end + + Enum.each(apps, &Application.ensure_all_started/1) + + children = + [ + Pleroma.Repo, + {Pleroma.Config.TransferTask, false}, + Pleroma.Web.Endpoint, + {Oban, Pleroma.Config.get(Oban)} + ] ++ + http_children(adapter) + + cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, [])) + + Supervisor.start_link(children ++ cachex_children, + strategy: :one_for_one, + name: Pleroma.Supervisor + ) if Pleroma.Config.get(:env) not in [:test, :benchmark] do pleroma_rebooted?() @@ -82,4 +121,11 @@ defmodule Mix.Pleroma do def escape_sh_path(path) do ~S(') <> String.replace(path, ~S('), ~S(\')) <> ~S(') end + + defp http_children(Tesla.Adapter.Gun) do + Pleroma.Gun.ConnectionPool.children() ++ + [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}] + end + + defp http_children(_), do: [] end diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index dd2b9c8f2..a607d5d4f 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -91,20 +91,17 @@ defmodule Mix.Tasks.Pleroma.Benchmark do "Without conn and without pool" => fn -> {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], - adapter: [pool: :no_pool, receive_conn: false] + pool: :no_pool, + receive_conn: false ) end, "Without conn and with pool" => fn -> {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], - adapter: [receive_conn: false] - ) + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false) end, "With reused conn and without pool" => fn -> {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], - adapter: [pool: :no_pool] - ) + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool) end, "With reused conn and with pool" => fn -> {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500") diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index 5c9ef6904..904c5a74b 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -52,6 +52,7 @@ defmodule Mix.Tasks.Pleroma.Config do defp do_migrate_to_db(config_file) do if File.exists?(config_file) do + shell_info("Migrating settings from file: #{Path.expand(config_file)}") Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;") Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;") @@ -72,8 +73,7 @@ defmodule Mix.Tasks.Pleroma.Config do group |> Pleroma.Config.Loader.filter_group(settings) |> Enum.each(fn {key, value} -> - key = inspect(key) - {:ok, _} = ConfigDB.update_or_create(%{group: inspect(group), key: key, value: value}) + {:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value}) shell_info("Settings for key #{key} migrated.") end) @@ -83,7 +83,7 @@ defmodule Mix.Tasks.Pleroma.Config do defp migrate_from_db(opts) do if Pleroma.Config.get([:configurable_from_database]) do - env = opts[:env] || "prod" + env = opts[:env] || Pleroma.Config.get(:env) config_path = if Pleroma.Config.get(:release) do @@ -105,6 +105,10 @@ defmodule Mix.Tasks.Pleroma.Config do :ok = File.close(file) System.cmd("mix", ["format", config_path]) + + shell_info( + "Database configuration settings have been exported to config/#{env}.exported_from_db.secret.exs" + ) else migration_error() end @@ -112,7 +116,7 @@ defmodule Mix.Tasks.Pleroma.Config do defp migration_error do shell_error( - "Migration is not allowed in config. You can change this behavior by setting `configurable_from_database` to true." + "Migration is not allowed in config. You can change this behavior by setting `config :pleroma, configurable_from_database: true`" ) end @@ -131,12 +135,9 @@ defmodule Mix.Tasks.Pleroma.Config do end defp write(config, file) do - value = - config.value - |> ConfigDB.from_binary() - |> inspect(limit: :infinity) + value = inspect(config.value, limit: :infinity) - IO.write(file, "config #{config.group}, #{config.key}, #{value}\r\n\r\n") + IO.write(file, "config #{inspect(config.group)}, #{inspect(config.key)}, #{value}\r\n\r\n") config end diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 82e2abdcb..7f1108dcf 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -10,6 +10,7 @@ defmodule Mix.Tasks.Pleroma.Database do alias Pleroma.User require Logger require Pleroma.Constants + import Ecto.Query import Mix.Pleroma use Mix.Task @@ -53,8 +54,6 @@ defmodule Mix.Tasks.Pleroma.Database do end def run(["prune_objects" | args]) do - import Ecto.Query - {options, [], []} = OptionParser.parse( args, @@ -94,8 +93,6 @@ defmodule Mix.Tasks.Pleroma.Database do end def run(["fix_likes_collections"]) do - import Ecto.Query - start_pleroma() from(object in Object, @@ -130,4 +127,38 @@ defmodule Mix.Tasks.Pleroma.Database do Maintenance.vacuum(args) end + + def run(["ensure_expiration"]) do + start_pleroma() + days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365) + + Pleroma.Activity + |> join(:inner, [a], o in Object, + on: + fragment( + "(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')", + o.data, + a.data, + a.data + ) + ) + |> where(local: true) + |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) + |> where([_a, o], fragment("?->>'type' = 'Note'", o.data)) + |> Pleroma.RepoStreamer.chunk_stream(100) + |> Stream.each(fn activities -> + Enum.each(activities, fn activity -> + expires_at = + activity.inserted_at + |> DateTime.from_naive!("Etc/UTC") + |> Timex.shift(days: days) + + Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ + activity_id: activity.id, + expires_at: expires_at + }) + end) + end) + |> Stream.run() + end end diff --git a/lib/mix/tasks/pleroma/ecto/migrate.ex b/lib/mix/tasks/pleroma/ecto/migrate.ex index bc8ed29fb..e903bd171 100644 --- a/lib/mix/tasks/pleroma/ecto/migrate.ex +++ b/lib/mix/tasks/pleroma/ecto/migrate.ex @@ -41,6 +41,10 @@ defmodule Mix.Tasks.Pleroma.Ecto.Migrate do load_pleroma() {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) + if Application.get_env(:pleroma, Pleroma.Repo)[:ssl] do + Application.ensure_all_started(:ssl) + end + opts = if opts[:to] || opts[:step] || opts[:all], do: opts, diff --git a/lib/mix/tasks/pleroma/ecto/rollback.ex b/lib/mix/tasks/pleroma/ecto/rollback.ex index f43bd0b98..3dba952cb 100644 --- a/lib/mix/tasks/pleroma/ecto/rollback.ex +++ b/lib/mix/tasks/pleroma/ecto/rollback.ex @@ -40,6 +40,10 @@ defmodule Mix.Tasks.Pleroma.Ecto.Rollback do load_pleroma() {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) + if Application.get_env(:pleroma, Pleroma.Repo)[:ssl] do + Application.ensure_all_started(:ssl) + end + opts = if opts[:to] || opts[:step] || opts[:all], do: opts, diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index f4eaeac98..1750373f9 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -15,7 +15,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do {options, [], []} = parse_global_opts(args) url_or_path = options[:manifest] || default_manifest() - manifest = fetch_and_decode(url_or_path) + manifest = fetch_and_decode!(url_or_path) Enum.each(manifest, fn {name, info} -> to_print = [ @@ -42,7 +42,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do url_or_path = options[:manifest] || default_manifest() - manifest = fetch_and_decode(url_or_path) + manifest = fetch_and_decode!(url_or_path) for pack_name <- pack_names do if Map.has_key?(manifest, pack_name) do @@ -92,7 +92,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do ]) ) - files = fetch_and_decode(files_loc) + files = fetch_and_decode!(files_loc) IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name])) @@ -183,7 +183,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do IO.puts("Downloading the pack and generating SHA256") - binary_archive = Tesla.get!(client(), src).body + {:ok, %{body: binary_archive}} = Pleroma.HTTP.get(src) archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16() IO.puts("SHA256 is #{archive_sha}") @@ -243,14 +243,16 @@ defmodule Mix.Tasks.Pleroma.Emoji do IO.puts("Emoji packs have been reloaded.") end - defp fetch_and_decode(from) do + defp fetch_and_decode!(from) do with {:ok, json} <- fetch(from) do Jason.decode!(json) + else + {:error, error} -> raise "#{from} cannot be fetched. Error: #{error} occur." end end defp fetch("http" <> _ = from) do - with {:ok, %{body: body}} <- Tesla.get(client(), from) do + with {:ok, %{body: body}} <- Pleroma.HTTP.get(from) do {:ok, body} end end @@ -269,13 +271,5 @@ defmodule Mix.Tasks.Pleroma.Emoji do ) end - defp client do - middleware = [ - {Tesla.Middleware.FollowRedirects, [max_redirects: 3]} - ] - - Tesla.client(middleware) - end - defp default_manifest, do: Pleroma.Config.get!([:emoji, :default_manifest]) end diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex new file mode 100644 index 000000000..cbce81ab9 --- /dev/null +++ b/lib/mix/tasks/pleroma/frontend.ex @@ -0,0 +1,141 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Frontend do + use Mix.Task + + import Mix.Pleroma + + @shortdoc "Manages bundled Pleroma frontends" + + @moduledoc File.read!("docs/administration/CLI_tasks/frontend.md") + + def run(["install", "none" | _args]) do + shell_info("Skipping frontend installation because none was requested") + "none" + end + + def run(["install", frontend | args]) do + log_level = Logger.level() + Logger.configure(level: :warn) + start_pleroma() + + {options, [], []} = + OptionParser.parse( + args, + strict: [ + ref: :string, + static_dir: :string, + build_url: :string, + build_dir: :string, + file: :string + ] + ) + + instance_static_dir = + with nil <- options[:static_dir] do + Pleroma.Config.get!([:instance, :static_dir]) + end + + cmd_frontend_info = %{ + "name" => frontend, + "ref" => options[:ref], + "build_url" => options[:build_url], + "build_dir" => options[:build_dir] + } + + config_frontend_info = Pleroma.Config.get([:frontends, :available, frontend], %{}) + + frontend_info = + Map.merge(config_frontend_info, cmd_frontend_info, fn _key, config, cmd -> + # This only overrides things that are actually set + cmd || config + end) + + ref = frontend_info["ref"] + + unless ref do + raise "No ref given or configured" + end + + dest = + Path.join([ + instance_static_dir, + "frontends", + frontend, + ref + ]) + + fe_label = "#{frontend} (#{ref})" + + tmp_dir = Path.join([instance_static_dir, "frontends", "tmp"]) + + with {_, :ok} <- + {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, options[:file])}, + shell_info("Installing #{fe_label} to #{dest}"), + :ok <- install_frontend(frontend_info, tmp_dir, dest) do + File.rm_rf!(tmp_dir) + shell_info("Frontend #{fe_label} installed to #{dest}") + + Logger.configure(level: log_level) + else + {:download_or_unzip, _} -> + shell_info("Could not download or unzip the frontend") + + _e -> + shell_info("Could not install the frontend") + end + end + + defp download_or_unzip(frontend_info, temp_dir, file) do + if file do + with {:ok, zip} <- File.read(Path.expand(file)) do + unzip(zip, temp_dir) + end + else + download_build(frontend_info, temp_dir) + end + end + + def unzip(zip, dest) do + with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do + File.rm_rf!(dest) + File.mkdir_p!(dest) + + Enum.each(unzipped, fn {filename, data} -> + path = filename + + new_file_path = Path.join(dest, path) + + new_file_path + |> Path.dirname() + |> File.mkdir_p!() + + File.write!(new_file_path, data) + end) + + :ok + end + end + + defp download_build(frontend_info, dest) do + shell_info("Downloading pre-built bundle for #{frontend_info["name"]}") + url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"]) + + with {:ok, %{status: 200, body: zip_body}} <- + Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do + unzip(zip_body, dest) + else + e -> {:error, e} + end + end + + defp install_frontend(frontend_info, source, dest) do + from = frontend_info["build_dir"] || "dist" + File.rm_rf!(dest) + File.mkdir_p!(dest) + File.cp_r!(Path.join([source, from]), dest) + :ok + end +end diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 86409738a..91440b453 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -145,7 +145,7 @@ defmodule Mix.Tasks.Pleroma.Instance do options, :uploads_dir, "What directory should media uploads go in (when using the local uploader)?", - Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads]) + Config.get([Pleroma.Uploaders.Local, :uploads]) ) |> Path.expand() @@ -154,7 +154,7 @@ defmodule Mix.Tasks.Pleroma.Instance do options, :static_dir, "What directory should custom public files be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)?", - Pleroma.Config.get([:instance, :static_dir]) + Config.get([:instance, :static_dir]) ) |> Path.expand() diff --git a/lib/mix/tasks/pleroma/notification_settings.ex b/lib/mix/tasks/pleroma/notification_settings.ex index 7d65f0587..00f5ba7bf 100644 --- a/lib/mix/tasks/pleroma/notification_settings.ex +++ b/lib/mix/tasks/pleroma/notification_settings.ex @@ -3,8 +3,8 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do @moduledoc """ Example: - > mix pleroma.notification_settings --privacy-option=false --nickname-users="parallel588" # set false only for parallel588 user - > mix pleroma.notification_settings --privacy-option=true # set true for all users + > mix pleroma.notification_settings --hide-notification-contents=false --nickname-users="parallel588" # set false only for parallel588 user + > mix pleroma.notification_settings --hide-notification-contents=true # set true for all users """ @@ -19,16 +19,16 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do OptionParser.parse( args, strict: [ - privacy_option: :boolean, + hide_notification_contents: :boolean, email_users: :string, nickname_users: :string ] ) - privacy_option = Keyword.get(options, :privacy_option) + hide_notification_contents = Keyword.get(options, :hide_notification_contents) - if not is_nil(privacy_option) do - privacy_option + if not is_nil(hide_notification_contents) do + hide_notification_contents |> build_query(options) |> Pleroma.Repo.update_all([]) end @@ -36,15 +36,15 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do shell_info("Done") end - defp build_query(privacy_option, options) do + defp build_query(hide_notification_contents, options) do query = from(u in Pleroma.User, update: [ set: [ notification_settings: fragment( - "jsonb_set(notification_settings, '{privacy_option}', ?)", - ^privacy_option + "jsonb_set(notification_settings, '{hide_notification_contents}', ?)", + ^hide_notification_contents ) ] ] diff --git a/lib/mix/tasks/pleroma/refresh_counter_cache.ex b/lib/mix/tasks/pleroma/refresh_counter_cache.ex index 15b4dbfa6..efcbaa3b1 100644 --- a/lib/mix/tasks/pleroma/refresh_counter_cache.ex +++ b/lib/mix/tasks/pleroma/refresh_counter_cache.ex @@ -17,30 +17,53 @@ defmodule Mix.Tasks.Pleroma.RefreshCounterCache do def run([]) do Mix.Pleroma.start_pleroma() - ["public", "unlisted", "private", "direct"] - |> Enum.each(fn visibility -> - count = status_visibility_count_query(visibility) - name = "status_visibility_#{visibility}" - CounterCache.set(name, count) - Mix.Pleroma.shell_info("Set #{name} to #{count}") + instances = + Activity + |> distinct([a], true) + |> select([a], fragment("split_part(?, '/', 3)", a.actor)) + |> Repo.all() + + instances + |> Enum.with_index(1) + |> Enum.each(fn {instance, i} -> + counters = instance_counters(instance) + CounterCache.set(instance, counters) + + Mix.Pleroma.shell_info( + "[#{i}/#{length(instances)}] Setting #{instance} counters: #{inspect(counters)}" + ) end) Mix.Pleroma.shell_info("Done") end - defp status_visibility_count_query(visibility) do + defp instance_counters(instance) do + counters = %{"public" => 0, "unlisted" => 0, "private" => 0, "direct" => 0} + Activity - |> where( + |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) + |> where([a], fragment("split_part(?, '/', 3) = ?", a.actor, ^instance)) + |> select( + [a], + {fragment( + "activity_visibility(?, ?, ?)", + a.actor, + a.recipients, + a.data + ), count(a.id)} + ) + |> group_by( [a], fragment( - "activity_visibility(?, ?, ?) = ?", + "activity_visibility(?, ?, ?)", a.actor, a.recipients, - a.data, - ^visibility + a.data ) ) - |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) - |> Repo.aggregate(:count, :id, timeout: :timer.minutes(30)) + |> Repo.all(timeout: :timer.minutes(30)) + |> Enum.reduce(counters, fn {visibility, count}, acc -> + Map.put(acc, visibility, count) + end) end end diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex index c3312507e..a6d8d6c1c 100644 --- a/lib/mix/tasks/pleroma/relay.ex +++ b/lib/mix/tasks/pleroma/relay.ex @@ -35,10 +35,16 @@ defmodule Mix.Tasks.Pleroma.Relay do def run(["list"]) do start_pleroma() - with {:ok, list} <- Relay.list(true) do - list |> Enum.each(&shell_info(&1)) + with {:ok, list} <- Relay.list() do + Enum.each(list, &print_relay_url/1) else {:error, e} -> shell_error("Error while fetching relay subscription list: #{inspect(e)}") end end + + defp print_relay_url(%{followed_back: false} = relay) do + shell_info("#{relay.actor} - no Accept received (relay didn't follow back)") + end + + defp print_relay_url(relay), do: shell_info(relay.actor) end diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index bca7e87bf..01824aa18 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -232,7 +232,7 @@ defmodule Mix.Tasks.Pleroma.User do with %User{} = user <- User.get_cached_by_nickname(nickname) do user = user |> User.tag(tags) - shell_info("Tags of #{user.nickname}: #{inspect(tags)}") + shell_info("Tags of #{user.nickname}: #{inspect(user.tags)}") else _ -> shell_error("Could not change user tags for #{nickname}") @@ -245,7 +245,7 @@ defmodule Mix.Tasks.Pleroma.User do with %User{} = user <- User.get_cached_by_nickname(nickname) do user = user |> User.untag(tags) - shell_info("Tags of #{user.nickname}: #{inspect(tags)}") + shell_info("Tags of #{user.nickname}: #{inspect(user.tags)}") else _ -> shell_error("Could not change user tags for #{nickname}") diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index c3cea8d2a..17af04257 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Activity do alias Pleroma.Activity alias Pleroma.Activity.Queries - alias Pleroma.ActivityExpiration alias Pleroma.Bookmark alias Pleroma.Notification alias Pleroma.Object @@ -60,8 +59,6 @@ defmodule Pleroma.Activity do # typical case. has_one(:object, Object, on_delete: :nothing, foreign_key: :id) - has_one(:expiration, ActivityExpiration, on_delete: :delete_all) - timestamps() end @@ -304,14 +301,14 @@ defmodule Pleroma.Activity do |> Repo.all() end - def follow_requests_for_actor(%Pleroma.User{ap_id: ap_id}) do + def follow_requests_for_actor(%User{ap_id: ap_id}) do ap_id |> Queries.by_object_id() |> Queries.by_type("Follow") |> where([a], fragment("? ->> 'state' = 'pending'", a.data)) end - def following_requests_for_actor(%Pleroma.User{ap_id: ap_id}) do + def following_requests_for_actor(%User{ap_id: ap_id}) do Queries.by_type("Follow") |> where([a], fragment("?->>'state' = 'pending'", a.data)) |> where([a], a.actor == ^ap_id) @@ -340,4 +337,10 @@ defmodule Pleroma.Activity do _ -> nil end end + + @spec pinned_by_actor?(Activity.t()) :: boolean() + def pinned_by_actor?(%Activity{} = activity) do + actor = user_actor(activity) + activity.id in actor.pinned_activities + end end diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex deleted file mode 100644 index db9c88d84..000000000 --- a/lib/pleroma/activity_expiration.ex +++ /dev/null @@ -1,67 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ActivityExpiration do - use Ecto.Schema - - alias Pleroma.Activity - alias Pleroma.ActivityExpiration - alias Pleroma.Repo - - import Ecto.Changeset - import Ecto.Query - - @type t :: %__MODULE__{} - @min_activity_lifetime :timer.hours(1) - - schema "activity_expirations" do - belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType) - field(:scheduled_at, :naive_datetime) - end - - def changeset(%ActivityExpiration{} = expiration, attrs) do - expiration - |> cast(attrs, [:scheduled_at]) - |> validate_required([:scheduled_at]) - |> validate_scheduled_at() - end - - def get_by_activity_id(activity_id) do - ActivityExpiration - |> where([exp], exp.activity_id == ^activity_id) - |> Repo.one() - end - - def create(%Activity{} = activity, scheduled_at) do - %ActivityExpiration{activity_id: activity.id} - |> changeset(%{scheduled_at: scheduled_at}) - |> Repo.insert() - end - - def due_expirations(offset \\ 0) do - naive_datetime = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(offset, :millisecond) - - ActivityExpiration - |> where([exp], exp.scheduled_at < ^naive_datetime) - |> Repo.all() - end - - def validate_scheduled_at(changeset) do - validate_change(changeset, :scheduled_at, fn _, scheduled_at -> - if not expires_late_enough?(scheduled_at) do - [scheduled_at: "an ephemeral activity must live for at least one hour"] - else - [] - end - end) - end - - def expires_late_enough?(scheduled_at) do - now = NaiveDateTime.utc_now() - diff = NaiveDateTime.diff(scheduled_at, now, :millisecond) - diff > @min_activity_lifetime - end -end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9c74fa00e..cd7a856d0 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -22,26 +22,42 @@ defmodule Pleroma.Application do def repository, do: @repository def user_agent do - case Config.get([:http, :user_agent], :default) do - :default -> - info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>" - named_version() <> "; " <> info - - custom -> - custom + if Process.whereis(Pleroma.Web.Endpoint) do + case Config.get([:http, :user_agent], :default) do + :default -> + info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>" + named_version() <> "; " <> info + + custom -> + custom + end + else + # fallback, if endpoint is not started yet + "Pleroma Data Loader" end end # See http://elixir-lang.org/docs/stable/elixir/Application.html # for more information on OTP Applications def start(_type, _args) do - Pleroma.Config.Holder.save_default() + # Scrubbers are compiled at runtime and therefore will cause a conflict + # every time the application is restarted, so we disable module + # conflicts at runtime + Code.compiler_options(ignore_module_conflict: true) + # Disable warnings_as_errors at runtime, it breaks Phoenix live reload + # due to protocol consolidation warnings + Code.compiler_options(warnings_as_errors: false) + Pleroma.Telemetry.Logger.attach() + Config.Holder.save_default() Pleroma.HTML.compile_scrubbers() + Pleroma.Config.Oban.warn() Config.DeprecationWarnings.warn() Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled() - Pleroma.Repo.check_migrations_applied!() + Pleroma.ApplicationRequirements.verify!() setup_instrumenters() load_custom_modules() + check_system_commands() + Pleroma.Docs.JSON.compile() adapter = Application.get_env(:tesla, :adapter) @@ -149,7 +165,8 @@ defmodule Pleroma.Application do build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), - build_cachex("failed_proxy_url", limit: 2500) + build_cachex("failed_proxy_url", limit: 2500), + build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) ] end @@ -162,7 +179,8 @@ defmodule Pleroma.Application do defp seconds_valid_interval, do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid])) - defp build_cachex(type, opts), + @spec build_cachex(String.t(), keyword()) :: map() + def build_cachex(type, opts), do: %{ id: String.to_atom("cachex_" <> type), start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]}, @@ -217,9 +235,7 @@ defmodule Pleroma.Application do # start hackney and gun pools in tests defp http_children(_, :test) do - hackney_options = Config.get([:hackney_pools, :federation]) - hackney_pool = :hackney_pool.child_spec(:federation, hackney_options) - [hackney_pool, Pleroma.Pool.Supervisor] + http_children(Tesla.Adapter.Hackney, nil) ++ http_children(Tesla.Adapter.Gun, nil) end defp http_children(Tesla.Adapter.Hackney, _) do @@ -238,7 +254,27 @@ defmodule Pleroma.Application do end end - defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor] + defp http_children(Tesla.Adapter.Gun, _) do + Pleroma.Gun.ConnectionPool.children() ++ + [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}] + end defp http_children(_, _), do: [] + + defp check_system_commands do + filters = Config.get([Pleroma.Upload, :filters]) + + check_filter = fn filter, command_required -> + with true <- filter in filters, + false <- Pleroma.Utils.command_available?(command_required) do + Logger.error( + "#{filter} is specified in list of Pleroma.Upload filters, but the #{command_required} command is not found" + ) + end + end + + check_filter.(Pleroma.Upload.Filters.Exiftool, "exiftool") + check_filter.(Pleroma.Upload.Filters.Mogrify, "mogrify") + check_filter.(Pleroma.Upload.Filters.Mogrifun, "mogrify") + end end diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex new file mode 100644 index 000000000..16f62b6f5 --- /dev/null +++ b/lib/pleroma/application_requirements.ex @@ -0,0 +1,143 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ApplicationRequirements do + @moduledoc """ + The module represents the collection of validations to runs before start server. + """ + + defmodule VerifyError, do: defexception([:message]) + + import Ecto.Query + + require Logger + + @spec verify!() :: :ok | VerifyError.t() + def verify! do + :ok + |> check_confirmation_accounts! + |> check_migrations_applied!() + |> check_welcome_message_config!() + |> check_rum!() + |> handle_result() + end + + defp handle_result(:ok), do: :ok + defp handle_result({:error, message}), do: raise(VerifyError, message: message) + + defp check_welcome_message_config!(:ok) do + if Pleroma.Config.get([:welcome, :email, :enabled], false) and + not Pleroma.Emails.Mailer.enabled?() do + Logger.error(""" + To send welcome email do you need to enable mail. + \nconfig :pleroma, Pleroma.Emails.Mailer, enabled: true + """) + + {:error, "The mail disabled."} + else + :ok + end + end + + defp check_welcome_message_config!(result), do: result + + # Checks account confirmation email + # + def check_confirmation_accounts!(:ok) do + if Pleroma.Config.get([:instance, :account_activation_required]) && + not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do + Logger.error( + "Account activation enabled, but no Mailer settings enabled.\nPlease set config :pleroma, :instance, account_activation_required: false\nOtherwise setup and enable Mailer." + ) + + {:error, + "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails."} + else + :ok + end + end + + def check_confirmation_accounts!(result), do: result + + # Checks for pending migrations. + # + def check_migrations_applied!(:ok) do + unless Pleroma.Config.get( + [:i_am_aware_this_may_cause_data_loss, :disable_migration_check], + false + ) do + {_, res, _} = + Ecto.Migrator.with_repo(Pleroma.Repo, fn repo -> + down_migrations = + Ecto.Migrator.migrations(repo) + |> Enum.reject(fn + {:up, _, _} -> true + {:down, _, _} -> false + end) + + if length(down_migrations) > 0 do + down_migrations_text = + Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end) + + Logger.error( + "The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true" + ) + + {:error, "Unapplied Migrations detected"} + else + :ok + end + end) + + res + else + :ok + end + end + + def check_migrations_applied!(result), do: result + + # Checks for settings of RUM indexes. + # + defp check_rum!(:ok) do + {_, res, _} = + Ecto.Migrator.with_repo(Pleroma.Repo, fn repo -> + migrate = + from(o in "columns", + where: o.table_name == "objects", + where: o.column_name == "fts_content" + ) + |> repo.exists?(prefix: "information_schema") + + setting = Pleroma.Config.get([:database, :rum_enabled], false) + + do_check_rum!(setting, migrate) + end) + + res + end + + defp check_rum!(result), do: result + + defp do_check_rum!(setting, migrate) do + case {setting, migrate} do + {true, false} -> + Logger.error( + "Use `RUM` index is enabled, but were not applied migrations for it.\nIf you want to start Pleroma anyway, set\nconfig :pleroma, :database, rum_enabled: false\nOtherwise apply the following migrations:\n`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`" + ) + + {:error, "Unapplied RUM Migrations detected"} + + {false, true} -> + Logger.error( + "Detected applied migrations to use `RUM` index, but `RUM` isn't enable in settings.\nIf you want to use `RUM`, set\nconfig :pleroma, :database, rum_enabled: true\nOtherwise roll `RUM` migrations back.\n`mix ecto.rollback --migrations-path priv/repo/optional_migrations/rum_indexing/`" + ) + + {:error, "RUM Migrations detected"} + + _ -> + :ok + end + end +end diff --git a/lib/pleroma/captcha/kocaptcha.ex b/lib/pleroma/captcha/kocaptcha.ex index 6bc2fa158..337506647 100644 --- a/lib/pleroma/captcha/kocaptcha.ex +++ b/lib/pleroma/captcha/kocaptcha.ex @@ -21,7 +21,8 @@ defmodule Pleroma.Captcha.Kocaptcha do type: :kocaptcha, token: json_resp["token"], url: endpoint <> json_resp["url"], - answer_data: json_resp["md5"] + answer_data: json_resp["md5"], + seconds_valid: Pleroma.Config.get([Pleroma.Captcha, :seconds_valid]) } end end diff --git a/lib/pleroma/captcha/native.ex b/lib/pleroma/captcha/native.ex index a90631d61..8d604d2b2 100644 --- a/lib/pleroma/captcha/native.ex +++ b/lib/pleroma/captcha/native.ex @@ -17,7 +17,8 @@ defmodule Pleroma.Captcha.Native do type: :native, token: token(), url: "data:image/png;base64," <> Base.encode64(img_binary), - answer_data: answer_data + answer_data: answer_data, + seconds_valid: Pleroma.Config.get([Pleroma.Captcha, :seconds_valid]) } end end diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index cc80deff5..97f877595 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -11,12 +11,10 @@ defmodule Pleroma.Config do def get([key], default), do: get(key, default) - def get([parent_key | keys], default) do - case :pleroma - |> Application.get_env(parent_key) - |> get_in(keys) do - nil -> default - any -> any + def get([_ | _] = path, default) do + case fetch(path) do + {:ok, value} -> value + :error -> default end end @@ -34,6 +32,24 @@ defmodule Pleroma.Config do end end + def fetch(key) when is_atom(key), do: fetch([key]) + + def fetch([root_key | keys]) do + Enum.reduce_while(keys, Application.fetch_env(:pleroma, root_key), fn + key, {:ok, config} when is_map(config) or is_list(config) -> + case Access.fetch(config, key) do + :error -> + {:halt, :error} + + value -> + {:cont, value} + end + + _key, _config -> + {:halt, :error} + end) + end + def put([key], value), do: put(key, value) def put([parent_key | keys], value) do @@ -50,18 +66,31 @@ defmodule Pleroma.Config do def delete([key]), do: delete(key) - def delete([parent_key | keys]) do - {_, parent} = - Application.get_env(:pleroma, parent_key) - |> get_and_update_in(keys, fn _ -> :pop end) + def delete([parent_key | keys] = path) do + with {:ok, _} <- fetch(path) do + {_, parent} = + parent_key + |> get() + |> get_and_update_in(keys, fn _ -> :pop end) - Application.put_env(:pleroma, parent_key, parent) + Application.put_env(:pleroma, parent_key, parent) + end end def delete(key) do Application.delete_env(:pleroma, key) end + def restrict_unauthenticated_access?(resource, kind) do + setting = get([:restrict_unauthenticated, resource, kind]) + + if setting in [nil, :if_instance_is_private] do + !get!([:instance, :public]) + else + setting + end + end + def oauth_consumer_strategies, do: get([:auth, :oauth_consumer_strategies], []) def oauth_consumer_enabled?, do: oauth_consumer_strategies() != [] diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index 2b43d4c36..e5b7811aa 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -6,7 +6,7 @@ defmodule Pleroma.ConfigDB do use Ecto.Schema import Ecto.Changeset - import Ecto.Query + import Ecto.Query, only: [select: 3] import Pleroma.Web.Gettext alias __MODULE__ @@ -14,16 +14,6 @@ defmodule Pleroma.ConfigDB do @type t :: %__MODULE__{} - @full_key_update [ - {:pleroma, :ecto_repos}, - {:quack, :meta}, - {:mime, :types}, - {:cors_plug, [:max_age, :methods, :expose, :headers]}, - {:auto_linker, :opts}, - {:swarm, :node_blacklist}, - {:logger, :backends} - ] - @full_subkey_update [ {:pleroma, :assets, :mascots}, {:pleroma, :emoji, :groups}, @@ -32,14 +22,10 @@ defmodule Pleroma.ConfigDB do {:pleroma, :mrf_keyword, :replace} ] - @regex ~r/^~r(?'delimiter'[\/|"'([{<]{1})(?'pattern'.+)[\/|"')\]}>]{1}(?'modifier'[uismxfU]*)/u - - @delimiters ["/", "|", "\"", "'", {"(", ")"}, {"[", "]"}, {"{", "}"}, {"<", ">"}] - schema "config" do - field(:key, :string) - field(:group, :string) - field(:value, :binary) + field(:key, Pleroma.EctoType.Config.Atom) + field(:group, Pleroma.EctoType.Config.Atom) + field(:value, Pleroma.EctoType.Config.BinaryValue) field(:db, {:array, :string}, virtual: true, default: []) timestamps() @@ -51,10 +37,6 @@ defmodule Pleroma.ConfigDB do |> select([c], {c.group, c.key, c.value}) |> Repo.all() |> Enum.reduce([], fn {group, key, value}, acc -> - group = ConfigDB.from_string(group) - key = ConfigDB.from_string(key) - value = from_binary(value) - Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}])) end) end @@ -64,50 +46,41 @@ defmodule Pleroma.ConfigDB do @spec changeset(ConfigDB.t(), map()) :: Changeset.t() def changeset(config, params \\ %{}) do - params = Map.put(params, :value, transform(params[:value])) - config |> cast(params, [:key, :group, :value]) |> validate_required([:key, :group, :value]) |> unique_constraint(:key, name: :config_group_key_index) end - @spec create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} - def create(params) do + defp create(params) do %ConfigDB{} |> changeset(params) |> Repo.insert() end - @spec update(ConfigDB.t(), map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} - def update(%ConfigDB{} = config, %{value: value}) do + defp update(%ConfigDB{} = config, %{value: value}) do config |> changeset(%{value: value}) |> Repo.update() end - @spec get_db_keys(ConfigDB.t()) :: [String.t()] - def get_db_keys(%ConfigDB{} = config) do - config.value - |> ConfigDB.from_binary() - |> get_db_keys(config.key) - end - @spec get_db_keys(keyword(), any()) :: [String.t()] def get_db_keys(value, key) do - if Keyword.keyword?(value) do - value |> Keyword.keys() |> Enum.map(&convert(&1)) - else - [convert(key)] - end + keys = + if Keyword.keyword?(value) do + Keyword.keys(value) + else + [key] + end + + Enum.map(keys, &to_json_types(&1)) end @spec merge_group(atom(), atom(), keyword(), keyword()) :: keyword() def merge_group(group, key, old_value, new_value) do - new_keys = to_map_set(new_value) + new_keys = to_mapset(new_value) - intersect_keys = - old_value |> to_map_set() |> MapSet.intersection(new_keys) |> MapSet.to_list() + intersect_keys = old_value |> to_mapset() |> MapSet.intersection(new_keys) |> MapSet.to_list() merged_value = ConfigDB.merge(old_value, new_value) @@ -120,12 +93,10 @@ defmodule Pleroma.ConfigDB do [] end) |> List.flatten() - |> Enum.reduce(merged_value, fn subkey, acc -> - Keyword.put(acc, subkey, new_value[subkey]) - end) + |> Enum.reduce(merged_value, &Keyword.put(&2, &1, new_value[&1])) end - defp to_map_set(keyword) do + defp to_mapset(keyword) do keyword |> Keyword.keys() |> MapSet.new() @@ -159,57 +130,54 @@ defmodule Pleroma.ConfigDB do @spec update_or_create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} def update_or_create(params) do + params = Map.put(params, :value, to_elixir_types(params[:value])) search_opts = Map.take(params, [:group, :key]) with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts), - {:partial_update, true, config} <- - {:partial_update, can_be_partially_updated?(config), config}, - old_value <- from_binary(config.value), - transformed_value <- do_transform(params[:value]), - {:can_be_merged, true, config} <- {:can_be_merged, is_list(transformed_value), config}, - new_value <- - merge_group( - ConfigDB.from_string(config.group), - ConfigDB.from_string(config.key), - old_value, - transformed_value - ) do - ConfigDB.update(config, %{value: new_value}) + {_, true, config} <- {:partial_update, can_be_partially_updated?(config), config}, + {_, true, config} <- + {:can_be_merged, is_list(params[:value]) and is_list(config.value), config} do + new_value = merge_group(config.group, config.key, config.value, params[:value]) + update(config, %{value: new_value}) else {reason, false, config} when reason in [:partial_update, :can_be_merged] -> - ConfigDB.update(config, params) + update(config, params) nil -> - ConfigDB.create(params) + create(params) end end defp can_be_partially_updated?(%ConfigDB{} = config), do: not only_full_update?(config) - defp only_full_update?(%ConfigDB{} = config) do - config_group = ConfigDB.from_string(config.group) - config_key = ConfigDB.from_string(config.key) - - Enum.any?(@full_key_update, fn - {group, key} when is_list(key) -> - config_group == group and config_key in key - - {group, key} -> - config_group == group and config_key == key + defp only_full_update?(%ConfigDB{group: group, key: key}) do + full_key_update = [ + {:pleroma, :ecto_repos}, + {:quack, :meta}, + {:mime, :types}, + {:cors_plug, [:max_age, :methods, :expose, :headers]}, + {:swarm, :node_blacklist}, + {:logger, :backends} + ] + + Enum.any?(full_key_update, fn + {s_group, s_key} -> + group == s_group and ((is_list(s_key) and key in s_key) or key == s_key) end) end - @spec delete(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} + @spec delete(ConfigDB.t() | map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} + def delete(%ConfigDB{} = config), do: Repo.delete(config) + def delete(params) do search_opts = Map.delete(params, :subkeys) with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts), {config, sub_keys} when is_list(sub_keys) <- {config, params[:subkeys]}, - old_value <- from_binary(config.value), - keys <- Enum.map(sub_keys, &do_transform_string(&1)), - {:partial_remove, config, new_value} when new_value != [] <- - {:partial_remove, config, Keyword.drop(old_value, keys)} do - ConfigDB.update(config, %{value: new_value}) + keys <- Enum.map(sub_keys, &string_to_elixir_types(&1)), + {_, config, new_value} when new_value != [] <- + {:partial_remove, config, Keyword.drop(config.value, keys)} do + update(config, %{value: new_value}) else {:partial_remove, config, []} -> Repo.delete(config) @@ -225,37 +193,32 @@ defmodule Pleroma.ConfigDB do end end - @spec from_binary(binary()) :: term() - def from_binary(binary), do: :erlang.binary_to_term(binary) - - @spec from_binary_with_convert(binary()) :: any() - def from_binary_with_convert(binary) do - binary - |> from_binary() - |> do_convert() + @spec to_json_types(term()) :: map() | list() | boolean() | String.t() + def to_json_types(entity) when is_list(entity) do + Enum.map(entity, &to_json_types/1) end - @spec from_string(String.t()) :: atom() | no_return() - def from_string(string), do: do_transform_string(string) + def to_json_types(%Regex{} = entity), do: inspect(entity) - @spec convert(any()) :: any() - def convert(entity), do: do_convert(entity) - - defp do_convert(entity) when is_list(entity) do - for v <- entity, into: [], do: do_convert(v) + def to_json_types(entity) when is_map(entity) do + Map.new(entity, fn {k, v} -> {to_json_types(k), to_json_types(v)} end) end - defp do_convert(%Regex{} = entity), do: inspect(entity) + def to_json_types({:args, args}) when is_list(args) do + arguments = + Enum.map(args, fn + arg when is_tuple(arg) -> inspect(arg) + arg -> to_json_types(arg) + end) - defp do_convert(entity) when is_map(entity) do - for {k, v} <- entity, into: %{}, do: {do_convert(k), do_convert(v)} + %{"tuple" => [":args", arguments]} end - defp do_convert({:proxy_url, {type, :localhost, port}}) do - %{"tuple" => [":proxy_url", %{"tuple" => [do_convert(type), "localhost", port]}]} + def to_json_types({:proxy_url, {type, :localhost, port}}) do + %{"tuple" => [":proxy_url", %{"tuple" => [to_json_types(type), "localhost", port]}]} end - defp do_convert({:proxy_url, {type, host, port}}) when is_tuple(host) do + def to_json_types({:proxy_url, {type, host, port}}) when is_tuple(host) do ip = host |> :inet_parse.ntoa() @@ -264,66 +227,64 @@ defmodule Pleroma.ConfigDB do %{ "tuple" => [ ":proxy_url", - %{"tuple" => [do_convert(type), ip, port]} + %{"tuple" => [to_json_types(type), ip, port]} ] } end - defp do_convert({:proxy_url, {type, host, port}}) do + def to_json_types({:proxy_url, {type, host, port}}) do %{ "tuple" => [ ":proxy_url", - %{"tuple" => [do_convert(type), to_string(host), port]} + %{"tuple" => [to_json_types(type), to_string(host), port]} ] } end - defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]} + def to_json_types({:partial_chain, entity}), + do: %{"tuple" => [":partial_chain", inspect(entity)]} - defp do_convert(entity) when is_tuple(entity) do + def to_json_types(entity) when is_tuple(entity) do value = entity |> Tuple.to_list() - |> do_convert() + |> to_json_types() %{"tuple" => value} end - defp do_convert(entity) when is_boolean(entity) or is_number(entity) or is_nil(entity) do + def to_json_types(entity) when is_binary(entity), do: entity + + def to_json_types(entity) when is_boolean(entity) or is_number(entity) or is_nil(entity) do entity end - defp do_convert(entity) - when is_atom(entity) and entity in [:"tlsv1.1", :"tlsv1.2", :"tlsv1.3"] do + def to_json_types(entity) when entity in [:"tlsv1.1", :"tlsv1.2", :"tlsv1.3"] do ":#{entity}" end - defp do_convert(entity) when is_atom(entity), do: inspect(entity) + def to_json_types(entity) when is_atom(entity), do: inspect(entity) - defp do_convert(entity) when is_binary(entity), do: entity + @spec to_elixir_types(boolean() | String.t() | map() | list()) :: term() + def to_elixir_types(%{"tuple" => [":args", args]}) when is_list(args) do + arguments = + Enum.map(args, fn arg -> + if String.contains?(arg, ["{", "}"]) do + {elem, []} = Code.eval_string(arg) + elem + else + to_elixir_types(arg) + end + end) - @spec transform(any()) :: binary() | no_return() - def transform(entity) when is_binary(entity) or is_map(entity) or is_list(entity) do - entity - |> do_transform() - |> to_binary() + {:args, arguments} end - def transform(entity), do: to_binary(entity) - - @spec transform_with_out_binary(any()) :: any() - def transform_with_out_binary(entity), do: do_transform(entity) - - @spec to_binary(any()) :: binary() - def to_binary(entity), do: :erlang.term_to_binary(entity) - - defp do_transform(%Regex{} = entity), do: entity - - defp do_transform(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}) do - {:proxy_url, {do_transform_string(type), parse_host(host), port}} + def to_elixir_types(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}) do + {:proxy_url, {string_to_elixir_types(type), parse_host(host), port}} end - defp do_transform(%{"tuple" => [":partial_chain", entity]}) do + def to_elixir_types(%{"tuple" => [":partial_chain", entity]}) do {partial_chain, []} = entity |> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "") @@ -332,25 +293,51 @@ defmodule Pleroma.ConfigDB do {:partial_chain, partial_chain} end - defp do_transform(%{"tuple" => entity}) do - Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end) + def to_elixir_types(%{"tuple" => entity}) do + Enum.reduce(entity, {}, &Tuple.append(&2, to_elixir_types(&1))) end - defp do_transform(entity) when is_map(entity) do - for {k, v} <- entity, into: %{}, do: {do_transform(k), do_transform(v)} + def to_elixir_types(entity) when is_map(entity) do + Map.new(entity, fn {k, v} -> {to_elixir_types(k), to_elixir_types(v)} end) end - defp do_transform(entity) when is_list(entity) do - for v <- entity, into: [], do: do_transform(v) + def to_elixir_types(entity) when is_list(entity) do + Enum.map(entity, &to_elixir_types/1) end - defp do_transform(entity) when is_binary(entity) do + def to_elixir_types(entity) when is_binary(entity) do entity |> String.trim() - |> do_transform_string() + |> string_to_elixir_types() end - defp do_transform(entity), do: entity + def to_elixir_types(entity), do: entity + + @spec string_to_elixir_types(String.t()) :: + atom() | Regex.t() | module() | String.t() | no_return() + def string_to_elixir_types("~r" <> _pattern = regex) do + pattern = + ~r/^~r(?'delimiter'[\/|"'([{<]{1})(?'pattern'.+)[\/|"')\]}>]{1}(?'modifier'[uismxfU]*)/u + + delimiters = ["/", "|", "\"", "'", {"(", ")"}, {"[", "]"}, {"{", "}"}, {"<", ">"}] + + with %{"modifier" => modifier, "pattern" => pattern, "delimiter" => regex_delimiter} <- + Regex.named_captures(pattern, regex), + {:ok, {leading, closing}} <- find_valid_delimiter(delimiters, pattern, regex_delimiter), + {result, _} <- Code.eval_string("~r#{leading}#{pattern}#{closing}#{modifier}") do + result + end + end + + def string_to_elixir_types(":" <> atom), do: String.to_atom(atom) + + def string_to_elixir_types(value) do + if module_name?(value) do + String.to_existing_atom("Elixir." <> value) + else + value + end + end defp parse_host("localhost"), do: :localhost @@ -387,27 +374,8 @@ defmodule Pleroma.ConfigDB do end end - defp do_transform_string("~r" <> _pattern = regex) do - with %{"modifier" => modifier, "pattern" => pattern, "delimiter" => regex_delimiter} <- - Regex.named_captures(@regex, regex), - {:ok, {leading, closing}} <- find_valid_delimiter(@delimiters, pattern, regex_delimiter), - {result, _} <- Code.eval_string("~r#{leading}#{pattern}#{closing}#{modifier}") do - result - end - end - - defp do_transform_string(":" <> atom), do: String.to_atom(atom) - - defp do_transform_string(value) do - if is_module_name?(value) do - String.to_existing_atom("Elixir." <> value) - else - value - end - end - - @spec is_module_name?(String.t()) :: boolean() - def is_module_name?(string) do + @spec module_name?(String.t()) :: boolean() + def module_name?(string) do Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or string in ["Oban", "Ueberauth", "ExSyslogger"] end diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index b68ded01f..412d55a77 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -3,9 +3,23 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Config.DeprecationWarnings do + alias Pleroma.Config + require Logger alias Pleroma.Config + @type config_namespace() :: atom() | [atom()] + @type config_map() :: {config_namespace(), config_namespace(), String.t()} + + @mrf_config_map [ + {[:instance, :rewrite_policy], [:mrf, :policies], + "\n* `config :pleroma, :instance, rewrite_policy` is now `config :pleroma, :mrf, policies`"}, + {[:instance, :mrf_transparency], [:mrf, :transparency], + "\n* `config :pleroma, :instance, mrf_transparency` is now `config :pleroma, :mrf, transparency`"}, + {[:instance, :mrf_transparency_exclusions], [:mrf, :transparency_exclusions], + "\n* `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`"} + ] + def check_hellthread_threshold do if Config.get([:mrf_hellthread, :threshold]) do Logger.warn(""" @@ -39,5 +53,126 @@ defmodule Pleroma.Config.DeprecationWarnings do def warn do check_hellthread_threshold() mrf_user_allowlist() + check_old_mrf_config() + check_media_proxy_whitelist_config() + check_welcome_message_config() + check_gun_pool_options() + check_activity_expiration_config() + end + + def check_welcome_message_config do + instance_config = Pleroma.Config.get([:instance]) + + use_old_config = + Keyword.has_key?(instance_config, :welcome_user_nickname) or + Keyword.has_key?(instance_config, :welcome_message) + + if use_old_config do + Logger.error(""" + !!!DEPRECATION WARNING!!! + Your config is using the old namespace for Welcome messages configuration. You need to change to the new namespace: + \n* `config :pleroma, :instance, welcome_user_nickname` is now `config :pleroma, :welcome, :direct_message, :sender_nickname` + \n* `config :pleroma, :instance, welcome_message` is now `config :pleroma, :welcome, :direct_message, :message` + """) + end + end + + def check_old_mrf_config do + warning_preface = """ + !!!DEPRECATION WARNING!!! + Your config is using old namespaces for MRF configuration. They should work for now, but you are advised to change to new namespaces to prevent possible issues later: + """ + + move_namespace_and_warn(@mrf_config_map, warning_preface) + end + + @spec move_namespace_and_warn([config_map()], String.t()) :: :ok | nil + def move_namespace_and_warn(config_map, warning_preface) do + warning = + Enum.reduce(config_map, "", fn + {old, new, err_msg}, acc -> + old_config = Config.get(old) + + if old_config do + Config.put(new, old_config) + acc <> err_msg + else + acc + end + end) + + if warning != "" do + Logger.warn(warning_preface <> warning) + end + end + + @spec check_media_proxy_whitelist_config() :: :ok | nil + def check_media_proxy_whitelist_config do + whitelist = Config.get([:media_proxy, :whitelist]) + + if Enum.any?(whitelist, &(not String.starts_with?(&1, "http"))) do + Logger.warn(""" + !!!DEPRECATION WARNING!!! + Your config is using old format (only domain) for MediaProxy whitelist option. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later. + """) + end + end + + def check_gun_pool_options do + pool_config = Config.get(:connections_pool) + + if timeout = pool_config[:await_up_timeout] do + Logger.warn(""" + !!!DEPRECATION WARNING!!! + Your config is using old setting name `await_up_timeout` instead of `connect_timeout`. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later. + """) + + Config.put(:connections_pool, Keyword.put_new(pool_config, :connect_timeout, timeout)) + end + + pools_configs = Config.get(:pools) + + warning_preface = """ + !!!DEPRECATION WARNING!!! + Your config is using old setting name `timeout` instead of `recv_timeout` in pool settings. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later. + """ + + updated_config = + Enum.reduce(pools_configs, [], fn {pool_name, config}, acc -> + if timeout = config[:timeout] do + Keyword.put(acc, pool_name, Keyword.put_new(config, :recv_timeout, timeout)) + else + acc + end + end) + + if updated_config != [] do + pool_warnings = + updated_config + |> Keyword.keys() + |> Enum.map(fn pool_name -> + "\n* `:timeout` options in #{pool_name} pool is now `:recv_timeout`" + end) + + Logger.warn(Enum.join([warning_preface | pool_warnings])) + + Config.put(:pools, updated_config) + end + end + + @spec check_activity_expiration_config() :: :ok | nil + def check_activity_expiration_config do + warning_preface = """ + !!!DEPRECATION WARNING!!! + Your config is using old namespace for activity expiration configuration. Setting should work for now, but you are advised to change to new namespace to prevent possible issues later: + """ + + move_namespace_and_warn( + [ + {Pleroma.ActivityExpiration, Pleroma.Workers.PurgeExpiredActivity, + "\n* `config :pleroma, Pleroma.ActivityExpiration` is now `config :pleroma, Pleroma.Workers.PurgeExpiredActivity`"} + ], + warning_preface + ) end end diff --git a/lib/pleroma/config/helpers.ex b/lib/pleroma/config/helpers.ex new file mode 100644 index 000000000..3dce40ea0 --- /dev/null +++ b/lib/pleroma/config/helpers.ex @@ -0,0 +1,17 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Config.Helpers do + alias Pleroma.Config + + def instance_name, do: Config.get([:instance, :name]) + + defp instance_notify_email do + Config.get([:instance, :notify_email]) || Config.get([:instance, :email]) + end + + def sender do + {instance_name(), instance_notify_email()} + end +end diff --git a/lib/pleroma/config/loader.ex b/lib/pleroma/config/loader.ex index 0f3ecf1ed..64e7de6df 100644 --- a/lib/pleroma/config/loader.ex +++ b/lib/pleroma/config/loader.ex @@ -12,6 +12,11 @@ defmodule Pleroma.Config.Loader do :swarm ] + @reject_groups [ + :postgrex, + :tesla + ] + if Code.ensure_loaded?(Config.Reader) do @reader Config.Reader @@ -47,7 +52,8 @@ defmodule Pleroma.Config.Loader do @spec filter_group(atom(), keyword()) :: keyword() def filter_group(group, configs) do Enum.reject(configs[group], fn {key, _v} -> - key in @reject_keys or (group == :phoenix and key == :serve_endpoints) or group == :postgrex + key in @reject_keys or group in @reject_groups or + (group == :phoenix and key == :serve_endpoints) end) end end diff --git a/lib/pleroma/config/oban.ex b/lib/pleroma/config/oban.ex new file mode 100644 index 000000000..9f601b1a3 --- /dev/null +++ b/lib/pleroma/config/oban.ex @@ -0,0 +1,34 @@ +defmodule Pleroma.Config.Oban do + require Logger + + def warn do + oban_config = Pleroma.Config.get(Oban) + + crontab = + [ + Pleroma.Workers.Cron.StatsWorker, + Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker, + Pleroma.Workers.Cron.ClearOauthTokenWorker + ] + |> Enum.reduce(oban_config[:crontab], fn removed_worker, acc -> + with acc when is_list(acc) <- acc, + setting when is_tuple(setting) <- + Enum.find(acc, fn {_, worker} -> worker == removed_worker end) do + """ + !!!OBAN CONFIG WARNING!!! + You are using old workers in Oban crontab settings, which were removed. + Please, remove setting from crontab in your config file (prod.secret.exs): #{ + inspect(setting) + } + """ + |> Logger.warn() + + List.delete(acc, setting) + else + _ -> acc + end + end) + + Pleroma.Config.put(Oban, Keyword.put(oban_config, :crontab, crontab)) + end +end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index c02b70e96..a0d7b7d71 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -28,15 +28,11 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, Pleroma.Captcha, [:seconds_valid]}, {:pleroma, Pleroma.Upload, [:proxy_remote]}, {:pleroma, :instance, [:upload_limit]}, - {:pleroma, :email_notifications, [:digest]}, - {:pleroma, :oauth2, [:clean_expired_tokens]}, - {:pleroma, Pleroma.ActivityExpiration, [:enabled]}, - {:pleroma, Pleroma.ScheduledActivity, [:enabled]}, {:pleroma, :gopher, [:enabled]} ] - def start_link(_) do - load_and_update_env() + def start_link(restart_pleroma? \\ true) do + load_and_update_env([], restart_pleroma?) if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) :ignore end @@ -48,7 +44,7 @@ defmodule Pleroma.Config.TransferTask do {logger, other} = (Repo.all(ConfigDB) ++ deleted_settings) - |> Enum.map(&transform_and_merge/1) + |> Enum.map(&merge_with_default/1) |> Enum.split_with(fn {group, _, _, _} -> group in [:logger, :quack] end) logger @@ -92,11 +88,7 @@ defmodule Pleroma.Config.TransferTask do end end - defp transform_and_merge(%{group: group, key: key, value: value} = setting) do - group = ConfigDB.from_string(group) - key = ConfigDB.from_string(key) - value = ConfigDB.from_binary(value) - + defp merge_with_default(%{group: group, key: key, value: value} = setting) do default = Config.Holder.default_config(group, key) merged = diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index ce7bd2396..8bc3e85d6 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -162,10 +162,13 @@ defmodule Pleroma.Conversation.Participation do for_user(user, params) |> Enum.map(fn participation -> activity_id = - ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{ - user: user, - blocking_user: user - }) + ActivityPub.fetch_latest_direct_activity_id_for_context( + participation.conversation.ap_id, + %{ + user: user, + blocking_user: user + } + ) %{ participation diff --git a/lib/pleroma/counter_cache.ex b/lib/pleroma/counter_cache.ex index 4d348a413..ebd1f603d 100644 --- a/lib/pleroma/counter_cache.ex +++ b/lib/pleroma/counter_cache.ex @@ -10,32 +10,70 @@ defmodule Pleroma.CounterCache do import Ecto.Query schema "counter_cache" do - field(:name, :string) - field(:count, :integer) + field(:instance, :string) + field(:public, :integer) + field(:unlisted, :integer) + field(:private, :integer) + field(:direct, :integer) end def changeset(struct, params) do struct - |> cast(params, [:name, :count]) - |> validate_required([:name]) - |> unique_constraint(:name) + |> cast(params, [:instance, :public, :unlisted, :private, :direct]) + |> validate_required([:instance]) + |> unique_constraint(:instance) end - def get_as_map(names) when is_list(names) do + def get_by_instance(instance) do CounterCache - |> where([cc], cc.name in ^names) - |> Repo.all() - |> Enum.group_by(& &1.name, & &1.count) - |> Map.new(fn {k, v} -> {k, hd(v)} end) + |> select([c], %{ + "public" => c.public, + "unlisted" => c.unlisted, + "private" => c.private, + "direct" => c.direct + }) + |> where([c], c.instance == ^instance) + |> Repo.one() + |> case do + nil -> %{"public" => 0, "unlisted" => 0, "private" => 0, "direct" => 0} + val -> val + end end - def set(name, count) do + def get_sum do + CounterCache + |> select([c], %{ + "public" => type(sum(c.public), :integer), + "unlisted" => type(sum(c.unlisted), :integer), + "private" => type(sum(c.private), :integer), + "direct" => type(sum(c.direct), :integer) + }) + |> Repo.one() + end + + def set(instance, values) do + params = + Enum.reduce( + ["public", "private", "unlisted", "direct"], + %{"instance" => instance}, + fn param, acc -> + Map.put_new(acc, param, Map.get(values, param, 0)) + end + ) + %CounterCache{} - |> changeset(%{"name" => name, "count" => count}) + |> changeset(params) |> Repo.insert( - on_conflict: [set: [count: count]], + on_conflict: [ + set: [ + public: params["public"], + private: params["private"], + unlisted: params["unlisted"], + direct: params["direct"] + ] + ], returning: true, - conflict_target: :name + conflict_target: :instance ) end end diff --git a/lib/pleroma/docs/generator.ex b/lib/pleroma/docs/generator.ex index e0fc8cd02..a671a6278 100644 --- a/lib/pleroma/docs/generator.ex +++ b/lib/pleroma/docs/generator.ex @@ -6,16 +6,21 @@ defmodule Pleroma.Docs.Generator do implementation.process(descriptions) end - @spec list_modules_in_dir(String.t(), String.t()) :: [module()] - def list_modules_in_dir(dir, start) do - with {:ok, files} <- File.ls(dir) do - files - |> Enum.filter(&String.ends_with?(&1, ".ex")) - |> Enum.map(fn filename -> - module = filename |> String.trim_trailing(".ex") |> Macro.camelize() - String.to_atom(start <> module) - end) - end + @spec list_behaviour_implementations(behaviour :: module()) :: [module()] + def list_behaviour_implementations(behaviour) do + :code.all_loaded() + |> Enum.filter(fn {module, _} -> + # This shouldn't be needed as all modules are expected to have module_info/1, + # but in test enviroments some transient modules `:elixir_compiler_XX` + # are loaded for some reason (where XX is a random integer). + if function_exported?(module, :module_info, 1) do + module.module_info(:attributes) + |> Keyword.get_values(:behaviour) + |> List.flatten() + |> Enum.member?(behaviour) + end + end) + |> Enum.map(fn {module, _} -> module end) end @doc """ @@ -87,6 +92,12 @@ defmodule Pleroma.Docs.Generator do else: string end + defp format_suggestions({:list_behaviour_implementations, behaviour}) do + behaviour + |> list_behaviour_implementations() + |> format_suggestions() + end + defp format_suggestions([]), do: [] defp format_suggestions([suggestion | tail]) do diff --git a/lib/pleroma/docs/json.ex b/lib/pleroma/docs/json.ex index d1cf1f487..feeb4320e 100644 --- a/lib/pleroma/docs/json.ex +++ b/lib/pleroma/docs/json.ex @@ -1,5 +1,19 @@ defmodule Pleroma.Docs.JSON do @behaviour Pleroma.Docs.Generator + @external_resource "config/description.exs" + @raw_config Pleroma.Config.Loader.read("config/description.exs") + @raw_descriptions @raw_config[:pleroma][:config_description] + @term __MODULE__.Compiled + + @spec compile :: :ok + def compile do + :persistent_term.put(@term, Pleroma.Docs.Generator.convert_to_strings(@raw_descriptions)) + end + + @spec compiled_descriptions :: Map.t() + def compiled_descriptions do + :persistent_term.get(@term) + end @spec process(keyword()) :: {:ok, String.t()} def process(descriptions) do @@ -13,11 +27,4 @@ defmodule Pleroma.Docs.JSON do {:ok, path} end end - - def compile do - with config <- Pleroma.Config.Loader.read("config/description.exs") do - config[:pleroma][:config_description] - |> Pleroma.Docs.Generator.convert_to_strings() - end - end end diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index 68b106499..da3f20f43 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -68,6 +68,11 @@ defmodule Pleroma.Docs.Markdown do IO.write(file, " #{list_mark}`#{inspect(suggestion)}`\n") end + defp print_suggestions(file, {:list_behaviour_implementations, behaviour}) do + suggestions = Pleroma.Docs.Generator.list_behaviour_implementations(behaviour) + print_suggestions(file, suggestions) + end + defp print_suggestions(_file, nil), do: nil defp print_suggestions(_file, ""), do: nil diff --git a/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/date_time.ex index 4f412fcde..d852c0abd 100644 --- a/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/date_time.ex @@ -1,4 +1,8 @@ -defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.DateTime do +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime do @moduledoc """ The AP standard defines the date fields in AP as xsd:DateTime. Elixir's DateTime can't parse this, but it can parse the related iso8601. This diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex new file mode 100644 index 000000000..4aacc5c88 --- /dev/null +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Emoji do + use Ecto.Type + + def type, do: :map + + def cast(data) when is_map(data) do + has_invalid_emoji? = + Enum.find(data, fn + {name, uri} when is_binary(name) and is_binary(uri) -> + # based on ObjectValidators.Uri.cast() + case URI.parse(uri) do + %URI{host: nil} -> true + %URI{host: ""} -> true + %URI{scheme: scheme} when scheme in ["https", "http"] -> false + _ -> true + end + + {_name, _uri} -> + true + end) + + if has_invalid_emoji?, do: :error, else: {:ok, data} + end + + def cast(_data), do: :error + + def dump(data), do: {:ok, data} + + def load(data), do: {:ok, data} +end diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/object_id.ex index f71f76370..8034235b0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/object_id.ex @@ -1,4 +1,8 @@ -defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID do use Ecto.Type def type, do: :string diff --git a/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex index 408e0f6ee..205527a96 100644 --- a/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex @@ -1,7 +1,11 @@ -defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.Recipients do +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do use Ecto.Type - alias Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID + alias Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID def type, do: {:array, ObjectID} diff --git a/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/safe_text.ex index 95c948123..7f0405c7b 100644 --- a/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/safe_text.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.SafeText do +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.SafeText do use Ecto.Type alias Pleroma.HTML diff --git a/lib/pleroma/web/activity_pub/object_validators/types/uri.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/uri.ex index 24845bcc0..2054c26be 100644 --- a/lib/pleroma/web/activity_pub/object_validators/types/uri.ex +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/uri.ex @@ -1,4 +1,8 @@ -defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.Uri do +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Uri do use Ecto.Type def type, do: :string diff --git a/lib/pleroma/ecto_type/config/atom.ex b/lib/pleroma/ecto_type/config/atom.ex new file mode 100644 index 000000000..df565d432 --- /dev/null +++ b/lib/pleroma/ecto_type/config/atom.ex @@ -0,0 +1,26 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.Config.Atom do + use Ecto.Type + + def type, do: :atom + + def cast(key) when is_atom(key) do + {:ok, key} + end + + def cast(key) when is_binary(key) do + {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)} + end + + def cast(_), do: :error + + def load(key) do + {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)} + end + + def dump(key) when is_atom(key), do: {:ok, inspect(key)} + def dump(_), do: :error +end diff --git a/lib/pleroma/ecto_type/config/binary_value.ex b/lib/pleroma/ecto_type/config/binary_value.ex new file mode 100644 index 000000000..bbd2608c5 --- /dev/null +++ b/lib/pleroma/ecto_type/config/binary_value.ex @@ -0,0 +1,27 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.Config.BinaryValue do + use Ecto.Type + + def type, do: :term + + def cast(value) when is_binary(value) do + if String.valid?(value) do + {:ok, value} + else + {:ok, :erlang.binary_to_term(value)} + end + end + + def cast(value), do: {:ok, value} + + def load(value) when is_binary(value) do + {:ok, :erlang.binary_to_term(value)} + end + + def dump(value) do + {:ok, :erlang.term_to_binary(value)} + end +end diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex index 55f61024e..c27ad1065 100644 --- a/lib/pleroma/emails/admin_email.ex +++ b/lib/pleroma/emails/admin_email.ex @@ -8,9 +8,10 @@ defmodule Pleroma.Emails.AdminEmail do import Swoosh.Email alias Pleroma.Config + alias Pleroma.HTML alias Pleroma.Web.Router.Helpers - defp instance_config, do: Pleroma.Config.get(:instance) + defp instance_config, do: Config.get(:instance) defp instance_name, do: instance_config()[:name] defp instance_notify_email do @@ -72,6 +73,8 @@ defmodule Pleroma.Emails.AdminEmail do <p>Reported Account: <a href="#{user_url(account)}">#{account.nickname}</a></p> #{comment_html} #{statuses_html} + <p> + <a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/reports/index">View Reports in AdminFE</a> """ new() @@ -80,4 +83,18 @@ defmodule Pleroma.Emails.AdminEmail do |> subject("#{instance_name()} Report") |> html_body(html_body) end + + def new_unapproved_registration(to, account) do + html_body = """ + <p>New account for review: <a href="#{user_url(account)}">@#{account.nickname}</a></p> + <blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote> + <a href="#{Pleroma.Web.base_url()}/pleroma/admin">Visit AdminFE</a> + """ + + new() + |> to({to.name, to.email}) + |> from({instance_name(), instance_notify_email()}) + |> subject("New account up for review on #{instance_name()} (@#{account.nickname})") + |> html_body(html_body) + end end diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index dfadc10b3..1d8c72ae9 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -12,17 +12,22 @@ defmodule Pleroma.Emails.UserEmail do alias Pleroma.Web.Endpoint alias Pleroma.Web.Router - defp instance_name, do: Config.get([:instance, :name]) - - defp sender do - email = Config.get([:instance, :notify_email]) || Config.get([:instance, :email]) - {instance_name(), email} - end + import Pleroma.Config.Helpers, only: [instance_name: 0, sender: 0] defp recipient(email, nil), do: email defp recipient(email, name), do: {name, email} defp recipient(%User{} = user), do: recipient(user.email, user.name) + @spec welcome(User.t(), map()) :: Swoosh.Email.t() + def welcome(user, opts \\ %{}) do + new() + |> to(recipient(user)) + |> from(Map.get(opts, :sender, sender())) + |> subject(Map.get(opts, :subject, "Welcome to #{instance_name()}!")) + |> html_body(Map.get(opts, :html, "Welcome to #{instance_name()}!")) + |> text_body(Map.get(opts, :text, "Welcome to #{instance_name()}!")) + end + def password_reset_email(user, token) when is_binary(token) do password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token) @@ -102,25 +107,34 @@ defmodule Pleroma.Emails.UserEmail do |> Enum.filter(&(&1.activity.data["type"] == "Create")) |> Enum.map(fn notification -> object = Pleroma.Object.normalize(notification.activity) - object = update_in(object.data["content"], &format_links/1) - %{ - data: notification, - object: object, - from: User.get_by_ap_id(notification.activity.actor) - } + if not is_nil(object) do + object = update_in(object.data["content"], &format_links/1) + + %{ + data: notification, + object: object, + from: User.get_by_ap_id(notification.activity.actor) + } + end end) + |> Enum.filter(& &1) followers = notifications |> Enum.filter(&(&1.activity.data["type"] == "Follow")) |> Enum.map(fn notification -> - %{ - data: notification, - object: Pleroma.Object.normalize(notification.activity), - from: User.get_by_ap_id(notification.activity.actor) - } + from = User.get_by_ap_id(notification.activity.actor) + + if not is_nil(from) do + %{ + data: notification, + object: Pleroma.Object.normalize(notification.activity), + from: User.get_by_ap_id(notification.activity.actor) + } + end end) + |> Enum.filter(& &1) unless Enum.empty?(mentions) do styling = Config.get([__MODULE__, :styling]) diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex index 3de2dc762..03a6bca0b 100644 --- a/lib/pleroma/emoji/loader.ex +++ b/lib/pleroma/emoji/loader.ex @@ -108,7 +108,7 @@ defmodule Pleroma.Emoji.Loader do if File.exists?(emoji_txt) do load_from_file(emoji_txt, emoji_groups) else - extensions = Pleroma.Config.get([:emoji, :pack_extensions]) + extensions = Config.get([:emoji, :pack_extensions]) Logger.info( "No emoji.txt found for pack \"#{pack_name}\", assuming all #{ diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 14a5185be..d076ae312 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Emoji.Pack do - @derive {Jason.Encoder, only: [:files, :pack]} + @derive {Jason.Encoder, only: [:files, :pack, :files_count]} defstruct files: %{}, + files_count: 0, pack_file: nil, path: nil, pack: %{}, @@ -8,6 +9,7 @@ defmodule Pleroma.Emoji.Pack do @type t() :: %__MODULE__{ files: %{String.t() => Path.t()}, + files_count: non_neg_integer(), pack_file: Path.t(), path: Path.t(), pack: map(), @@ -16,7 +18,7 @@ defmodule Pleroma.Emoji.Pack do alias Pleroma.Emoji - @spec create(String.t()) :: :ok | {:error, File.posix()} | {:error, :empty_values} + @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values} def create(name) do with :ok <- validate_not_empty([name]), dir <- Path.join(emoji_path(), name), @@ -26,10 +28,28 @@ defmodule Pleroma.Emoji.Pack do end end - @spec show(String.t()) :: {:ok, t()} | {:error, atom()} - def show(name) do + defp paginate(entities, 1, page_size), do: Enum.take(entities, page_size) + + defp paginate(entities, page, page_size) do + entities + |> Enum.chunk_every(page_size) + |> Enum.at(page - 1) + end + + @spec show(keyword()) :: {:ok, t()} | {:error, atom()} + def show(opts) do + name = opts[:name] + with :ok <- validate_not_empty([name]), {:ok, pack} <- load_pack(name) do + shortcodes = + pack.files + |> Map.keys() + |> Enum.sort() + |> paginate(opts[:page], opts[:page_size]) + + pack = Map.put(pack, :files, Map.take(pack.files, shortcodes)) + {:ok, validate_pack(pack)} end end @@ -120,10 +140,10 @@ defmodule Pleroma.Emoji.Pack do end end - @spec list_local() :: {:ok, map()} - def list_local do + @spec list_local(keyword()) :: {:ok, map(), non_neg_integer()} + def list_local(opts) do with {:ok, results} <- list_packs_dir() do - packs = + all_packs = results |> Enum.map(fn name -> case load_pack(name) do @@ -132,9 +152,13 @@ defmodule Pleroma.Emoji.Pack do end end) |> Enum.reject(&is_nil/1) + + packs = + all_packs + |> paginate(opts[:page], opts[:page_size]) |> Map.new(fn pack -> {pack.name, validate_pack(pack)} end) - {:ok, packs} + {:ok, packs, length(all_packs)} end end @@ -146,7 +170,7 @@ defmodule Pleroma.Emoji.Pack do end end - @spec download(String.t(), String.t(), String.t()) :: :ok | {:error, atom()} + @spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()} def download(name, url, as) do uri = url |> String.trim() |> URI.parse() @@ -197,7 +221,12 @@ defmodule Pleroma.Emoji.Pack do |> Map.put(:path, Path.dirname(pack_file)) |> Map.put(:name, name) - {:ok, pack} + files_count = + pack.files + |> Map.keys() + |> length() + + {:ok, Map.put(pack, :files_count, files_count)} else {:error, :not_found} end @@ -296,7 +325,9 @@ defmodule Pleroma.Emoji.Pack do # Otherwise, they'd have to download it from external-src pack.pack["share-files"] && Enum.all?(pack.files, fn {_, file} -> - File.exists?(Path.join(pack.path, file)) + pack.path + |> Path.join(file) + |> File.exists?() end) end @@ -440,7 +471,7 @@ defmodule Pleroma.Emoji.Pack do # with the API so it should be sufficient with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)}, {:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do - {:ok, results} + {:ok, Enum.sort(results)} else {:create_dir, {:error, e}} -> {:error, :create_dir, e} {:ls, {:error, e}} -> {:error, :ls, e} diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index 4d61b3650..5d6df9530 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -34,10 +34,18 @@ defmodule Pleroma.Filter do Repo.one(query) end - def get_filters(%User{id: user_id} = _user) do + def get_active(query) do + from(f in query, where: is_nil(f.expires_at) or f.expires_at > ^NaiveDateTime.utc_now()) + end + + def get_irreversible(query) do + from(f in query, where: f.hide) + end + + def get_filters(query \\ __MODULE__, %User{id: user_id}) do query = from( - f in Pleroma.Filter, + f in query, where: f.user_id == ^user_id, order_by: [desc: :id] ) @@ -95,4 +103,34 @@ defmodule Pleroma.Filter do |> validate_required([:phrase, :context]) |> Repo.update() end + + def compose_regex(user_or_filters, format \\ :postgres) + + def compose_regex(%User{} = user, format) do + __MODULE__ + |> get_active() + |> get_irreversible() + |> get_filters(user) + |> compose_regex(format) + end + + def compose_regex([_ | _] = filters, format) do + phrases = + filters + |> Enum.map(& &1.phrase) + |> Enum.join("|") + + case format do + :postgres -> + "\\y(#{phrases})\\y" + + :re -> + ~r/\b#{phrases}\b/i + + _ -> + nil + end + end + + def compose_regex(_, _), do: nil end diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index 093b1f405..2039a259d 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -95,7 +95,11 @@ defmodule Pleroma.FollowingRelationship do |> where([r], r.state == ^:follow_accept) end - def followers_ap_ids(%User{} = user, from_ap_ids \\ nil) do + def followers_ap_ids(user, from_ap_ids \\ nil) + + def followers_ap_ids(_, []), do: [] + + def followers_ap_ids(%User{} = user, from_ap_ids) do query = user |> followers_query() @@ -124,6 +128,7 @@ defmodule Pleroma.FollowingRelationship do |> join(:inner, [r], f in assoc(r, :follower)) |> where([r], r.state == ^:follow_pending) |> where([r], r.following_id == ^id) + |> where([r, f], f.deactivated != true) |> select([r, f], f) |> Repo.all() end @@ -259,4 +264,12 @@ defmodule Pleroma.FollowingRelationship do end end) end + + @spec following_ap_ids(User.t()) :: [String.t()] + def following_ap_ids(%User{} = user) do + user + |> following_query() + |> select([r, u], u.ap_id) + |> Repo.all() + end end diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 02a93a8dc..0c450eae4 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -10,11 +10,15 @@ defmodule Pleroma.Formatter do @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ - @auto_linker_config hashtag: true, - hashtag_handler: &Pleroma.Formatter.hashtag_handler/4, - mention: true, - mention_handler: &Pleroma.Formatter.mention_handler/4, - scheme: true + defp linkify_opts do + Pleroma.Config.get(Pleroma.Formatter) ++ + [ + hashtag: true, + hashtag_handler: &Pleroma.Formatter.hashtag_handler/4, + mention: true, + mention_handler: &Pleroma.Formatter.mention_handler/4 + ] + end def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do case User.get_cached_by_nickname(nickname) do @@ -80,19 +84,19 @@ defmodule Pleroma.Formatter do @spec linkify(String.t(), keyword()) :: {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]} def linkify(text, options \\ []) do - options = options ++ @auto_linker_config + options = linkify_opts() ++ options if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text) acc = %{mentions: MapSet.new(), tags: MapSet.new()} - {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options) - {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options) + {text_mentions, %{mentions: mentions}} = Linkify.link_map(mentions, acc, options) + {text_rest, %{tags: tags}} = Linkify.link_map(rest, acc, options) {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)} else acc = %{mentions: MapSet.new(), tags: MapSet.new()} - {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options) + {text, %{mentions: mentions, tags: tags}} = Linkify.link_map(text, acc, options) {text, MapSet.to_list(mentions), MapSet.to_list(tags)} end @@ -111,9 +115,9 @@ defmodule Pleroma.Formatter do if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text) - AutoLinker.link(mentions, options) <> AutoLinker.link(rest, options) + Linkify.link(mentions, options) <> Linkify.link(rest, options) else - AutoLinker.link(text, options) + Linkify.link(text, options) end end diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex index 3d56d50a9..e9f54c4c0 100644 --- a/lib/pleroma/gopher/server.ex +++ b/lib/pleroma/gopher/server.ex @@ -96,16 +96,18 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do def response("/main/public") do posts = - ActivityPub.fetch_public_activities(%{"type" => ["Create"], "local_only" => true}) - |> render_activities + %{type: ["Create"], local_only: true} + |> ActivityPub.fetch_public_activities() + |> render_activities() info("Welcome to the Public Timeline!") <> posts <> ".\r\n" end def response("/main/all") do posts = - ActivityPub.fetch_public_activities(%{"type" => ["Create"]}) - |> render_activities + %{type: ["Create"]} + |> ActivityPub.fetch_public_activities() + |> render_activities() info("Welcome to the Federated Timeline!") <> posts <> ".\r\n" end @@ -130,13 +132,14 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do def response("/users/" <> nickname) do with %User{} = user <- User.get_cached_by_nickname(nickname) do params = %{ - "type" => ["Create"], - "actor_id" => user.ap_id + type: ["Create"], + actor_id: user.ap_id } activities = - ActivityPub.fetch_public_activities(params) - |> render_activities + params + |> ActivityPub.fetch_public_activities() + |> render_activities() info("Posts by #{user.nickname}") <> activities <> ".\r\n" else diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index f51cd7db8..09be74392 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -19,7 +19,8 @@ defmodule Pleroma.Gun.API do :tls_opts, :tcp_opts, :socks_opts, - :ws_opts + :ws_opts, + :supervise ] @impl Gun diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index cd25a2e74..75b1ffc0a 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -3,85 +3,33 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun.Conn do - @moduledoc """ - Struct for gun connection data - """ alias Pleroma.Gun - alias Pleroma.Pool.Connections require Logger - @type gun_state :: :up | :down - @type conn_state :: :active | :idle - - @type t :: %__MODULE__{ - conn: pid(), - gun_state: gun_state(), - conn_state: conn_state(), - used_by: [pid()], - last_reference: pos_integer(), - crf: float(), - retries: pos_integer() - } - - defstruct conn: nil, - gun_state: :open, - conn_state: :init, - used_by: [], - last_reference: 0, - crf: 1, - retries: 0 - - @spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil - def open(url, name, opts \\ []) - def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts) - - def open(%URI{} = uri, name, opts) do + def open(%URI{} = uri, opts) do pool_opts = Pleroma.Config.get([:connections_pool], []) opts = opts |> Enum.into(%{}) - |> Map.put_new(:retry, pool_opts[:retry] || 1) - |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000) - |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + |> Map.put_new(:connect_timeout, pool_opts[:connect_timeout] || 5_000) + |> Map.put_new(:supervise, false) |> maybe_add_tls_opts(uri) - key = "#{uri.scheme}:#{uri.host}:#{uri.port}" - - max_connections = pool_opts[:max_connections] || 250 - - conn_pid = - if Connections.count(name) < max_connections do - do_open(uri, opts) - else - close_least_used_and_do_open(name, uri, opts) - end - - if is_pid(conn_pid) do - conn = %Pleroma.Gun.Conn{ - conn: conn_pid, - gun_state: :up, - conn_state: :active, - last_reference: :os.system_time(:second) - } - - :ok = Gun.set_owner(conn_pid, Process.whereis(name)) - Connections.add_conn(name, key, conn) - end + do_open(uri, opts) end defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts - defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do + defp maybe_add_tls_opts(opts, %URI{scheme: "https"}) do tls_opts = [ verify: :verify_peer, cacertfile: CAStore.file_path(), depth: 20, reuse_sessions: false, - verify_fun: - {&:ssl_verify_hostname.verify_fun/3, - [check_hostname: Pleroma.HTTP.Connection.format_host(host)]} + log_level: :warning, + customize_hostname_check: [match_fun: :public_key.pkix_verify_hostname_match_fun(:https)] ] tls_opts = @@ -102,10 +50,10 @@ defmodule Pleroma.Gun.Conn do with open_opts <- Map.delete(opts, :tls_opts), {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), - {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]), + {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]), stream <- Gun.connect(conn, connect_opts), {:response, :fin, 200, _} <- Gun.await(conn, stream) do - conn + {:ok, conn} else error -> Logger.warn( @@ -140,8 +88,8 @@ defmodule Pleroma.Gun.Conn do |> Map.put(:socks_opts, socks_opts) with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), - {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do - conn + {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do + {:ok, conn} else error -> Logger.warn( @@ -155,11 +103,11 @@ defmodule Pleroma.Gun.Conn do end defp do_open(%URI{host: host, port: port} = uri, opts) do - host = Pleroma.HTTP.Connection.parse_host(host) + host = Pleroma.HTTP.AdapterHelper.parse_host(host) with {:ok, conn} <- Gun.open(host, port, opts), - {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do - conn + {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do + {:ok, conn} else error -> Logger.warn( @@ -171,7 +119,7 @@ defmodule Pleroma.Gun.Conn do end defp destination_opts(%URI{host: host, port: port}) do - host = Pleroma.HTTP.Connection.parse_host(host) + host = Pleroma.HTTP.AdapterHelper.parse_host(host) %{host: host, port: port} end @@ -181,17 +129,6 @@ defmodule Pleroma.Gun.Conn do defp add_http2_opts(opts, _, _), do: opts - defp close_least_used_and_do_open(name, uri, opts) do - with [{key, conn} | _conns] <- Connections.get_unused_conns(name), - :ok <- Gun.close(conn.conn) do - Connections.remove_conn(name, key) - - do_open(uri, opts) - else - [] -> {:error, :pool_overflowed} - end - end - def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do "#{scheme}://#{host}#{path}" end diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex new file mode 100644 index 000000000..f34602b73 --- /dev/null +++ b/lib/pleroma/gun/connection_pool.ex @@ -0,0 +1,82 @@ +defmodule Pleroma.Gun.ConnectionPool do + @registry __MODULE__ + + alias Pleroma.Gun.ConnectionPool.WorkerSupervisor + + def children do + [ + {Registry, keys: :unique, name: @registry}, + Pleroma.Gun.ConnectionPool.WorkerSupervisor + ] + end + + @spec get_conn(URI.t(), keyword()) :: {:ok, pid()} | {:error, term()} + def get_conn(uri, opts) do + key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + + case Registry.lookup(@registry, key) do + # The key has already been registered, but connection is not up yet + [{worker_pid, nil}] -> + get_gun_pid_from_worker(worker_pid, true) + + [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] -> + GenServer.call(worker_pid, :add_client) + {:ok, gun_pid} + + [] -> + # :gun.set_owner fails in :connected state for whatevever reason, + # so we open the connection in the process directly and send it's pid back + # We trust gun to handle timeouts by itself + case WorkerSupervisor.start_worker([key, uri, opts, self()]) do + {:ok, worker_pid} -> + get_gun_pid_from_worker(worker_pid, false) + + {:error, {:already_started, worker_pid}} -> + get_gun_pid_from_worker(worker_pid, true) + + err -> + err + end + end + end + + defp get_gun_pid_from_worker(worker_pid, register) do + # GenServer.call will block the process for timeout length if + # the server crashes on startup (which will happen if gun fails to connect) + # so instead we use cast + monitor + + ref = Process.monitor(worker_pid) + if register, do: GenServer.cast(worker_pid, {:add_client, self()}) + + receive do + {:conn_pid, pid} -> + Process.demonitor(ref) + {:ok, pid} + + {:DOWN, ^ref, :process, ^worker_pid, reason} -> + case reason do + {:shutdown, {:error, _} = error} -> error + {:shutdown, error} -> {:error, error} + _ -> {:error, reason} + end + end + end + + @spec release_conn(pid()) :: :ok + def release_conn(conn_pid) do + # :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _, _, _}}} when gun_pid == conn_pid -> + # worker_pid end) + query_result = + Registry.select(@registry, [ + {{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]} + ]) + + case query_result do + [worker_pid] -> + GenServer.call(worker_pid, :remove_client) + + [] -> + :ok + end + end +end diff --git a/lib/pleroma/gun/connection_pool/reclaimer.ex b/lib/pleroma/gun/connection_pool/reclaimer.ex new file mode 100644 index 000000000..cea800882 --- /dev/null +++ b/lib/pleroma/gun/connection_pool/reclaimer.ex @@ -0,0 +1,85 @@ +defmodule Pleroma.Gun.ConnectionPool.Reclaimer do + use GenServer, restart: :temporary + + @registry Pleroma.Gun.ConnectionPool + + def start_monitor do + pid = + case :gen_server.start(__MODULE__, [], name: {:via, Registry, {@registry, "reclaimer"}}) do + {:ok, pid} -> + pid + + {:error, {:already_registered, pid}} -> + pid + end + + {pid, Process.monitor(pid)} + end + + @impl true + def init(_) do + {:ok, nil, {:continue, :reclaim}} + end + + @impl true + def handle_continue(:reclaim, _) do + max_connections = Pleroma.Config.get([:connections_pool, :max_connections]) + + reclaim_max = + [:connections_pool, :reclaim_multiplier] + |> Pleroma.Config.get() + |> Kernel.*(max_connections) + |> round + |> max(1) + + :telemetry.execute([:pleroma, :connection_pool, :reclaim, :start], %{}, %{ + max_connections: max_connections, + reclaim_max: reclaim_max + }) + + # :ets.fun2ms( + # fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] -> + # {worker_pid, crf, last_reference} end) + unused_conns = + Registry.select( + @registry, + [ + {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]} + ] + ) + + case unused_conns do + [] -> + :telemetry.execute( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: 0}, + %{ + max_connections: max_connections + } + ) + + {:stop, :no_unused_conns, nil} + + unused_conns -> + reclaimed = + unused_conns + |> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} -> + crf1 <= crf2 and last_reference1 <= last_reference2 + end) + |> Enum.take(reclaim_max) + + reclaimed + |> Enum.each(fn {pid, _, _} -> + DynamicSupervisor.terminate_child(Pleroma.Gun.ConnectionPool.WorkerSupervisor, pid) + end) + + :telemetry.execute( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: Enum.count(reclaimed)}, + %{max_connections: max_connections} + ) + + {:stop, :normal, nil} + end + end +end diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex new file mode 100644 index 000000000..c36332817 --- /dev/null +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -0,0 +1,146 @@ +defmodule Pleroma.Gun.ConnectionPool.Worker do + alias Pleroma.Gun + use GenServer, restart: :temporary + + @registry Pleroma.Gun.ConnectionPool + + def start_link([key | _] = opts) do + GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {@registry, key}}) + end + + @impl true + def init([_key, _uri, _opts, _client_pid] = opts) do + {:ok, nil, {:continue, {:connect, opts}}} + end + + @impl true + def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do + with {:ok, conn_pid} <- Gun.Conn.open(uri, opts), + Process.link(conn_pid) do + time = :erlang.monotonic_time(:millisecond) + + {_, _} = + Registry.update_value(@registry, key, fn _ -> + {conn_pid, [client_pid], 1, time} + end) + + send(client_pid, {:conn_pid, conn_pid}) + + {:noreply, + %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}}, + :hibernate} + else + err -> + {:stop, {:shutdown, err}, nil} + end + end + + @impl true + def handle_cast({:add_client, client_pid}, state) do + case handle_call(:add_client, {client_pid, nil}, state) do + {:reply, conn_pid, state, :hibernate} -> + send(client_pid, {:conn_pid, conn_pid}) + {:noreply, state, :hibernate} + end + end + + @impl true + def handle_cast({:remove_client, client_pid}, state) do + case handle_call(:remove_client, {client_pid, nil}, state) do + {:reply, _, state, :hibernate} -> + {:noreply, state, :hibernate} + end + end + + @impl true + def handle_call(:add_client, {client_pid, _}, %{key: key} = state) do + time = :erlang.monotonic_time(:millisecond) + + {{conn_pid, _, _, _}, _} = + Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} -> + {conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time} + end) + + state = + if state.timer != nil do + Process.cancel_timer(state[:timer]) + %{state | timer: nil} + else + state + end + + ref = Process.monitor(client_pid) + + state = put_in(state.client_monitors[client_pid], ref) + {:reply, conn_pid, state, :hibernate} + end + + @impl true + def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do + {{_conn_pid, used_by, _crf, _last_reference}, _} = + Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} -> + {conn_pid, List.delete(used_by, client_pid), crf, last_reference} + end) + + {ref, state} = pop_in(state.client_monitors[client_pid]) + # DOWN message can receive right after `remove_client` call and cause worker to terminate + state = + if is_nil(ref) do + state + else + Process.demonitor(ref) + + timer = + if used_by == [] do + max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000) + Process.send_after(self(), :idle_close, max_idle) + else + nil + end + + %{state | timer: timer} + end + + {:reply, :ok, state, :hibernate} + end + + @impl true + def handle_info(:idle_close, state) do + # Gun monitors the owner process, and will close the connection automatically + # when it's terminated + {:stop, :normal, state} + end + + @impl true + def handle_info({:gun_up, _pid, _protocol}, state) do + {:noreply, state, :hibernate} + end + + # Gracefully shutdown if the connection got closed without any streams left + @impl true + def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do + {:stop, :normal, state} + end + + # Otherwise, wait for retry + @impl true + def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) do + {:noreply, state, :hibernate} + end + + @impl true + def handle_info({:DOWN, _ref, :process, pid, reason}, state) do + :telemetry.execute( + [:pleroma, :connection_pool, :client_death], + %{client_pid: pid, reason: reason}, + %{key: state.key} + ) + + handle_cast({:remove_client, pid}, state) + end + + # LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478 + defp crf(time_delta, prev_crf) do + 1 + :math.pow(0.5, 0.0001 * time_delta) * prev_crf + end +end diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex new file mode 100644 index 000000000..39615c956 --- /dev/null +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -0,0 +1,45 @@ +defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do + @moduledoc "Supervisor for pool workers. Does not do anything except enforce max connection limit" + + use DynamicSupervisor + + def start_link(opts) do + DynamicSupervisor.start_link(__MODULE__, opts, name: __MODULE__) + end + + def init(_opts) do + DynamicSupervisor.init( + strategy: :one_for_one, + max_children: Pleroma.Config.get([:connections_pool, :max_connections]) + ) + end + + def start_worker(opts, retry \\ false) do + case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do + {:error, :max_children} -> + if retry or free_pool() == :error do + :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) + {:error, :pool_full} + else + start_worker(opts, true) + end + + res -> + res + end + end + + defp free_pool do + wait_for_reclaimer_finish(Pleroma.Gun.ConnectionPool.Reclaimer.start_monitor()) + end + + defp wait_for_reclaimer_finish({pid, mon}) do + receive do + {:DOWN, ^mon, :process, ^pid, :no_unused_conns} -> + :error + + {:DOWN, ^mon, :process, ^pid, :normal} -> + :ok + end + end +end diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index d78c5f202..43e9145be 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -100,20 +100,27 @@ defmodule Pleroma.HTML do end) end - def extract_first_external_url(_, nil), do: {:error, "No content"} + def extract_first_external_url_from_object(%{data: %{"content" => content}} = object) + when is_binary(content) do + unless object.data["fake"] do + key = "URL|#{object.id}" + + Cachex.fetch!(:scrubber_cache, key, fn _key -> + {:commit, {:ok, extract_first_external_url(content)}} + end) + else + {:ok, extract_first_external_url(content)} + end + end - def extract_first_external_url(object, content) do - key = "URL|#{object.id}" + def extract_first_external_url_from_object(_), do: {:error, :no_content} - Cachex.fetch!(:scrubber_cache, key, fn _key -> - result = - content - |> Floki.parse_fragment!() - |> Floki.filter_out("a.mention,a.hashtag,a[rel~=\"tag\"]") - |> Floki.attribute("a", "href") - |> Enum.at(0) - - {:commit, {:ok, result}} - end) + def extract_first_external_url(content) do + content + |> Floki.parse_fragment!() + |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])") + |> Enum.take(1) + |> Floki.attribute("href") + |> Enum.at(0) end end diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 510722ff9..08b51578a 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -3,32 +3,28 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.AdapterHelper do - alias Pleroma.HTTP.Connection + @moduledoc """ + Configure Tesla.Client with default and customized adapter options. + """ + @defaults [pool: :federation, connect_timeout: 5_000, recv_timeout: 5_000] + + @type proxy_type() :: :socks4 | :socks5 + @type host() :: charlist() | :inet.ip_address() + + alias Pleroma.HTTP.AdapterHelper + require Logger @type proxy :: {Connection.host(), pos_integer()} | {Connection.proxy_type(), Connection.host(), pos_integer()} @callback options(keyword(), URI.t()) :: keyword() - @callback after_request(keyword()) :: :ok - - @spec options(keyword(), URI.t()) :: keyword() - def options(opts, _uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) - maybe_add_proxy(opts, format_proxy(proxy)) - end - - @spec maybe_get_conn(URI.t(), keyword()) :: keyword() - def maybe_get_conn(_uri, opts), do: opts - - @spec after_request(keyword()) :: :ok - def after_request(_opts), do: :ok @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil def format_proxy(nil), do: nil def format_proxy(proxy_url) do - case Connection.parse_proxy(proxy_url) do + case parse_proxy(proxy_url) do {:ok, host, port} -> {host, port} {:ok, type, host, port} -> {type, host, port} _ -> nil @@ -38,4 +34,88 @@ defmodule Pleroma.HTTP.AdapterHelper do @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() def maybe_add_proxy(opts, nil), do: opts def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) + + @doc """ + Merge default connection & adapter options with received ones. + """ + + @spec options(URI.t(), keyword()) :: keyword() + def options(%URI{} = uri, opts \\ []) do + @defaults + |> Keyword.merge(opts) + |> adapter_helper().options(uri) + end + + defp adapter, do: Application.get_env(:tesla, :adapter) + + defp adapter_helper do + case adapter() do + Tesla.Adapter.Gun -> AdapterHelper.Gun + Tesla.Adapter.Hackney -> AdapterHelper.Hackney + _ -> AdapterHelper.Default + end + end + + @spec parse_proxy(String.t() | tuple() | nil) :: + {:ok, host(), pos_integer()} + | {:ok, proxy_type(), host(), pos_integer()} + | {:error, atom()} + | nil + + def parse_proxy(nil), do: nil + + def parse_proxy(proxy) when is_binary(proxy) do + with [host, port] <- String.split(proxy, ":"), + {port, ""} <- Integer.parse(port) do + {:ok, parse_host(host), port} + else + {_, _} -> + Logger.warn("Parsing port failed #{inspect(proxy)}") + {:error, :invalid_proxy_port} + + :error -> + Logger.warn("Parsing port failed #{inspect(proxy)}") + {:error, :invalid_proxy_port} + + _ -> + Logger.warn("Parsing proxy failed #{inspect(proxy)}") + {:error, :invalid_proxy} + end + end + + def parse_proxy(proxy) when is_tuple(proxy) do + with {type, host, port} <- proxy do + {:ok, type, parse_host(host), port} + else + _ -> + Logger.warn("Parsing proxy failed #{inspect(proxy)}") + {:error, :invalid_proxy} + end + end + + @spec parse_host(String.t() | atom() | charlist()) :: charlist() | :inet.ip_address() + def parse_host(host) when is_list(host), do: host + def parse_host(host) when is_atom(host), do: to_charlist(host) + + def parse_host(host) when is_binary(host) do + host = to_charlist(host) + + case :inet.parse_address(host) do + {:error, :einval} -> host + {:ok, ip} -> ip + end + end + + @spec format_host(String.t()) :: charlist() + def format_host(host) do + host_charlist = to_charlist(host) + + case :inet.parse_address(host_charlist) do + {:error, :einval} -> + :idna.encode(host_charlist) + + {:ok, _ip} -> + host_charlist + end + end end diff --git a/lib/pleroma/http/adapter_helper/default.ex b/lib/pleroma/http/adapter_helper/default.ex new file mode 100644 index 000000000..e13441316 --- /dev/null +++ b/lib/pleroma/http/adapter_helper/default.ex @@ -0,0 +1,14 @@ +defmodule Pleroma.HTTP.AdapterHelper.Default do + alias Pleroma.HTTP.AdapterHelper + + @behaviour Pleroma.HTTP.AdapterHelper + + @spec options(keyword(), URI.t()) :: keyword() + def options(opts, _uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + AdapterHelper.maybe_add_proxy(opts, AdapterHelper.format_proxy(proxy)) + end + + @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} + def get_conn(_uri, opts), do: {:ok, opts} +end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index ead7cdc6b..1dbb71362 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -5,73 +5,78 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @behaviour Pleroma.HTTP.AdapterHelper + alias Pleroma.Config alias Pleroma.HTTP.AdapterHelper - alias Pleroma.Pool.Connections require Logger @defaults [ - connect_timeout: 5_000, - domain_lookup_timeout: 5_000, - tls_handshake_timeout: 5_000, retry: 1, - retry_timeout: 1000, - await_up_timeout: 5_000 + retry_timeout: 1_000 ] + @type pool() :: :federation | :upload | :media | :default + @spec options(keyword(), URI.t()) :: keyword() def options(incoming_opts \\ [], %URI{} = uri) do proxy = - Pleroma.Config.get([:http, :proxy_url]) + [:http, :proxy_url] + |> Config.get() |> AdapterHelper.format_proxy() - config_opts = Pleroma.Config.get([:http, :adapter], []) + config_opts = Config.get([:http, :adapter], []) @defaults |> Keyword.merge(config_opts) |> add_scheme_opts(uri) |> AdapterHelper.maybe_add_proxy(proxy) - |> maybe_get_conn(uri, incoming_opts) - end - - @spec after_request(keyword()) :: :ok - def after_request(opts) do - if opts[:conn] && opts[:body_as] != :chunks do - Connections.checkout(opts[:conn], self(), :gun_connections) - end - - :ok + |> Keyword.merge(incoming_opts) + |> put_timeout() end defp add_scheme_opts(opts, %{scheme: "http"}), do: opts defp add_scheme_opts(opts, %{scheme: "https"}) do - opts - |> Keyword.put(:certificates_verification, true) - |> Keyword.put(:tls_opts, log_level: :warning) + Keyword.put(opts, :certificates_verification, true) + end + + defp put_timeout(opts) do + {recv_timeout, opts} = Keyword.pop(opts, :recv_timeout, pool_timeout(opts[:pool])) + # this is the timeout to receive a message from Gun + # `:timeout` key is used in Tesla + Keyword.put(opts, :timeout, recv_timeout) end - defp maybe_get_conn(adapter_opts, uri, incoming_opts) do - {receive_conn?, opts} = - adapter_opts - |> Keyword.merge(incoming_opts) - |> Keyword.pop(:receive_conn, true) - - if Connections.alive?(:gun_connections) and receive_conn? do - checkin_conn(uri, opts) - else - opts - end + @spec pool_timeout(pool()) :: non_neg_integer() + def pool_timeout(pool) do + default = Config.get([:pools, :default, :recv_timeout], 5_000) + + Config.get([:pools, pool, :recv_timeout], default) end - defp checkin_conn(uri, opts) do - case Connections.checkin(uri, :gun_connections) do - nil -> - Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts]) - opts + @prefix Pleroma.Gun.ConnectionPool + def limiter_setup do + wait = Config.get([:connections_pool, :connection_acquisition_wait]) + retries = Config.get([:connections_pool, :connection_acquisition_retries]) + + :pools + |> Config.get([]) + |> Enum.each(fn {name, opts} -> + max_running = Keyword.get(opts, :size, 50) + max_waiting = Keyword.get(opts, :max_waiting, 10) - conn when is_pid(conn) -> - Keyword.merge(opts, conn: conn, close_conn: false) - end + result = + ConcurrentLimiter.new(:"#{@prefix}.#{name}", max_running, max_waiting, + wait: wait, + max_retries: retries + ) + + case result do + :ok -> :ok + {:error, :existing} -> :ok + end + end) + + :ok end end diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index 3972a03a9..ef84553c1 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -2,11 +2,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do @behaviour Pleroma.HTTP.AdapterHelper @defaults [ - connect_timeout: 10_000, - recv_timeout: 20_000, follow_redirect: true, - force_redirect: true, - pool: :federation + force_redirect: true ] @spec options(keyword(), URI.t()) :: keyword() @@ -19,10 +16,21 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do |> Keyword.merge(config_opts) |> Keyword.merge(connection_opts) |> add_scheme_opts(uri) + |> maybe_add_with_body() |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) end + defp add_scheme_opts(opts, %URI{scheme: "https"}) do + Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1]) + end + defp add_scheme_opts(opts, _), do: opts - def after_request(_), do: :ok + defp maybe_add_with_body(opts) do + if opts[:max_body] do + Keyword.put(opts, :with_body, true) + else + opts + end + end end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex deleted file mode 100644 index ebacf7902..000000000 --- a/lib/pleroma/http/connection.ex +++ /dev/null @@ -1,124 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.Connection do - @moduledoc """ - Configure Tesla.Client with default and customized adapter options. - """ - - alias Pleroma.Config - alias Pleroma.HTTP.AdapterHelper - - require Logger - - @defaults [pool: :federation] - - @type ip_address :: ipv4_address() | ipv6_address() - @type ipv4_address :: {0..255, 0..255, 0..255, 0..255} - @type ipv6_address :: - {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535} - @type proxy_type() :: :socks4 | :socks5 - @type host() :: charlist() | ip_address() - - @doc """ - Merge default connection & adapter options with received ones. - """ - - @spec options(URI.t(), keyword()) :: keyword() - def options(%URI{} = uri, opts \\ []) do - @defaults - |> pool_timeout() - |> Keyword.merge(opts) - |> adapter_helper().options(uri) - end - - defp pool_timeout(opts) do - {config_key, default} = - if adapter() == Tesla.Adapter.Gun do - {:pools, Config.get([:pools, :default, :timeout])} - else - {:hackney_pools, 10_000} - end - - timeout = Config.get([config_key, opts[:pool], :timeout], default) - - Keyword.merge(opts, timeout: timeout) - end - - @spec after_request(keyword()) :: :ok - def after_request(opts), do: adapter_helper().after_request(opts) - - defp adapter, do: Application.get_env(:tesla, :adapter) - - defp adapter_helper do - case adapter() do - Tesla.Adapter.Gun -> AdapterHelper.Gun - Tesla.Adapter.Hackney -> AdapterHelper.Hackney - _ -> AdapterHelper - end - end - - @spec parse_proxy(String.t() | tuple() | nil) :: - {:ok, host(), pos_integer()} - | {:ok, proxy_type(), host(), pos_integer()} - | {:error, atom()} - | nil - - def parse_proxy(nil), do: nil - - def parse_proxy(proxy) when is_binary(proxy) do - with [host, port] <- String.split(proxy, ":"), - {port, ""} <- Integer.parse(port) do - {:ok, parse_host(host), port} - else - {_, _} -> - Logger.warn("Parsing port failed #{inspect(proxy)}") - {:error, :invalid_proxy_port} - - :error -> - Logger.warn("Parsing port failed #{inspect(proxy)}") - {:error, :invalid_proxy_port} - - _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") - {:error, :invalid_proxy} - end - end - - def parse_proxy(proxy) when is_tuple(proxy) do - with {type, host, port} <- proxy do - {:ok, type, parse_host(host), port} - else - _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") - {:error, :invalid_proxy} - end - end - - @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address() - def parse_host(host) when is_list(host), do: host - def parse_host(host) when is_atom(host), do: to_charlist(host) - - def parse_host(host) when is_binary(host) do - host = to_charlist(host) - - case :inet.parse_address(host) do - {:error, :einval} -> host - {:ok, ip} -> ip - end - end - - @spec format_host(String.t()) :: charlist() - def format_host(host) do - host_charlist = to_charlist(host) - - case :inet.parse_address(host_charlist) do - {:error, :einval} -> - :idna.encode(host_charlist) - - {:ok, _ip} -> - host_charlist - end - end -end diff --git a/lib/pleroma/http/ex_aws.ex b/lib/pleroma/http/ex_aws.ex new file mode 100644 index 000000000..5cac3532f --- /dev/null +++ b/lib/pleroma/http/ex_aws.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.ExAws do + @moduledoc false + + @behaviour ExAws.Request.HttpClient + + alias Pleroma.HTTP + + @impl true + def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do + http_opts = Keyword.put_new(http_opts, :pool, :upload) + + case HTTP.request(method, url, body, headers, http_opts) do + {:ok, env} -> + {:ok, %{status_code: env.status, headers: env.headers, body: env.body}} + + {:error, reason} -> + {:error, %{reason: reason}} + end + end +end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 583b56484..052597191 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -7,7 +7,7 @@ defmodule Pleroma.HTTP do Wrapper for `Tesla.request/2`. """ - alias Pleroma.HTTP.Connection + alias Pleroma.HTTP.AdapterHelper alias Pleroma.HTTP.Request alias Pleroma.HTTP.RequestBuilder, as: Builder alias Tesla.Client @@ -16,6 +16,7 @@ defmodule Pleroma.HTTP do require Logger @type t :: __MODULE__ + @type method() :: :get | :post | :put | :delete | :head @doc """ Performs GET request. @@ -28,6 +29,9 @@ defmodule Pleroma.HTTP do def get(nil, _, _), do: nil def get(url, headers, options), do: request(:get, url, "", headers, options) + @spec head(Request.url(), Request.headers(), keyword()) :: {:ok, Env.t()} | {:error, any()} + def head(url, headers \\ [], options \\ []), do: request(:head, url, "", headers, options) + @doc """ Performs POST request. @@ -42,7 +46,7 @@ defmodule Pleroma.HTTP do Builds and performs http request. # Arguments: - `method` - :get, :post, :put, :delete + `method` - :get, :post, :put, :delete, :head `url` - full url `body` - request body `headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]` @@ -52,52 +56,26 @@ defmodule Pleroma.HTTP do `{:ok, %Tesla.Env{}}` or `{:error, error}` """ - @spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) :: + @spec request(method(), Request.url(), String.t(), Request.headers(), keyword()) :: {:ok, Env.t()} | {:error, any()} def request(method, url, body, headers, options) when is_binary(url) do uri = URI.parse(url) - adapter_opts = Connection.options(uri, options[:adapter] || []) + adapter_opts = AdapterHelper.options(uri, options || []) + options = put_in(options[:adapter], adapter_opts) params = options[:params] || [] request = build_request(method, headers, options, url, body, params) adapter = Application.get_env(:tesla, :adapter) - client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter) - - pid = Process.whereis(adapter_opts[:pool]) - pool_alive? = - if adapter == Tesla.Adapter.Gun && pid do - Process.alive?(pid) - else - false - end + client = Tesla.client(adapter_middlewares(adapter), adapter) - request_opts = + maybe_limit( + fn -> + request(client, request) + end, + adapter, adapter_opts - |> Enum.into(%{}) - |> Map.put(:env, Pleroma.Config.get([:env])) - |> Map.put(:pool_alive?, pool_alive?) - - response = request(client, request, request_opts) - - Connection.after_request(adapter_opts) - - response - end - - @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()} - def request(%Client{} = client, request, %{env: :test}), do: request(client, request) - - def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request) - - def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request) - - def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do - :poolboy.transaction( - pool, - &Pleroma.Pool.Request.execute(&1, client, request, timeout), - timeout ) end @@ -114,4 +92,19 @@ defmodule Pleroma.HTTP do |> Builder.add_param(:query, :query, params) |> Builder.convert_to_keyword() end + + @prefix Pleroma.Gun.ConnectionPool + defp maybe_limit(fun, Tesla.Adapter.Gun, opts) do + ConcurrentLimiter.limit(:"#{@prefix}.#{opts[:pool] || :default}", fun) + end + + defp maybe_limit(fun, _, _) do + fun.() + end + + defp adapter_middlewares(Tesla.Adapter.Gun) do + [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool] + end + + defp adapter_middlewares(_), do: [] end diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 2fc876d92..8a44a001d 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -34,10 +34,12 @@ defmodule Pleroma.HTTP.RequestBuilder do @spec headers(Request.t(), Request.headers()) :: Request.t() def headers(request, headers) do headers_list = - if Pleroma.Config.get([:http, :send_user_agent]) do + with true <- Pleroma.Config.get([:http, :send_user_agent]), + nil <- Enum.find(headers, fn {key, _val} -> String.downcase(key) == "user-agent" end) do [{"user-agent", Pleroma.Application.user_agent()} | headers] else - headers + _ -> + headers end %{request | headers: headers_list} diff --git a/lib/pleroma/http/tzdata.ex b/lib/pleroma/http/tzdata.ex new file mode 100644 index 000000000..09cfdadf7 --- /dev/null +++ b/lib/pleroma/http/tzdata.ex @@ -0,0 +1,29 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Tzdata do + @moduledoc false + + @behaviour Tzdata.HTTPClient + + alias Pleroma.HTTP + + @impl true + def get(url, headers, options) do + options = Keyword.put_new(options, :pool, :default) + + with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do + {:ok, {env.status, env.headers, env.body}} + end + end + + @impl true + def head(url, headers, options) do + options = Keyword.put_new(options, :pool, :default) + + with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do + {:ok, {env.status, env.headers}} + end + end +end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 74458c09a..8bf53c090 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -14,9 +14,13 @@ defmodule Pleroma.Instances.Instance do import Ecto.Query import Ecto.Changeset + require Logger + schema "instances" do field(:host, :string) field(:unreachable_since, :naive_datetime_usec) + field(:favicon, :string) + field(:favicon_updated_at, :naive_datetime) timestamps() end @@ -25,7 +29,7 @@ defmodule Pleroma.Instances.Instance do def changeset(struct, params \\ %{}) do struct - |> cast(params, [:host, :unreachable_since]) + |> cast(params, [:host, :unreachable_since, :favicon, :favicon_updated_at]) |> validate_required([:host]) |> unique_constraint(:host) end @@ -120,4 +124,59 @@ defmodule Pleroma.Instances.Instance do end defp parse_datetime(datetime), do: datetime + + def get_or_update_favicon(%URI{host: host} = instance_uri) do + existing_record = Repo.get_by(Instance, %{host: host}) + now = NaiveDateTime.utc_now() + + if existing_record && existing_record.favicon_updated_at && + NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do + existing_record.favicon + else + favicon = scrape_favicon(instance_uri) + + if existing_record do + existing_record + |> changeset(%{favicon: favicon, favicon_updated_at: now}) + |> Repo.update() + else + %Instance{} + |> changeset(%{host: host, favicon: favicon, favicon_updated_at: now}) + |> Repo.insert() + end + + favicon + end + rescue + e -> + Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}") + nil + end + + defp scrape_favicon(%URI{} = instance_uri) do + try do + with {:ok, %Tesla.Env{body: html}} <- + Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], + adapter: [pool: :media] + ), + favicon_rel <- + html + |> Floki.parse_document!() + |> Floki.attribute("link[rel=icon]", "href") + |> List.first(), + favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(), + true <- is_binary(favicon) do + favicon + else + _ -> nil + end + rescue + e -> + Logger.warn( + "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}" + ) + + nil + end + end end diff --git a/lib/pleroma/job_queue_monitor.ex b/lib/pleroma/job_queue_monitor.ex index 2ecf261f3..c255a61ec 100644 --- a/lib/pleroma/job_queue_monitor.ex +++ b/lib/pleroma/job_queue_monitor.ex @@ -15,8 +15,8 @@ defmodule Pleroma.JobQueueMonitor do @impl true def init(state) do - :telemetry.attach("oban-monitor-failure", [:oban, :failure], &handle_event/4, nil) - :telemetry.attach("oban-monitor-success", [:oban, :success], &handle_event/4, nil) + :telemetry.attach("oban-monitor-failure", [:oban, :job, :exception], &handle_event/4, nil) + :telemetry.attach("oban-monitor-success", [:oban, :job, :stop], &handle_event/4, nil) {:ok, state} end @@ -25,8 +25,11 @@ defmodule Pleroma.JobQueueMonitor do GenServer.call(__MODULE__, :stats) end - def handle_event([:oban, status], %{duration: duration}, meta, _) do - GenServer.cast(__MODULE__, {:process_event, status, duration, meta}) + def handle_event([:oban, :job, event], %{duration: duration}, meta, _) do + GenServer.cast( + __MODULE__, + {:process_event, mapping_status(event), duration, meta} + ) end @impl true @@ -75,4 +78,7 @@ defmodule Pleroma.JobQueueMonitor do |> Map.update!(:processed_jobs, &(&1 + 1)) |> Map.update!(status, &(&1 + 1)) end + + defp mapping_status(:stop), do: :success + defp mapping_status(:exception), do: :failure end diff --git a/lib/pleroma/mfa/token.ex b/lib/pleroma/mfa/token.ex index 0b2449971..69b64c0e8 100644 --- a/lib/pleroma/mfa/token.ex +++ b/lib/pleroma/mfa/token.ex @@ -10,10 +10,11 @@ defmodule Pleroma.MFA.Token do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.OAuth.Authorization - alias Pleroma.Web.OAuth.Token, as: OAuthToken @expires 300 + @type t() :: %__MODULE__{} + schema "mfa_tokens" do field(:token, :string) field(:valid_until, :naive_datetime_usec) @@ -24,6 +25,7 @@ defmodule Pleroma.MFA.Token do timestamps() end + @spec get_by_token(String.t()) :: {:ok, t()} | {:error, :not_found} def get_by_token(token) do from( t in __MODULE__, @@ -33,33 +35,40 @@ defmodule Pleroma.MFA.Token do |> Repo.find_resource() end - def validate(token) do - with {:fetch_token, {:ok, token}} <- {:fetch_token, get_by_token(token)}, - {:expired, false} <- {:expired, is_expired?(token)} do + @spec validate(String.t()) :: {:ok, t()} | {:error, :not_found} | {:error, :expired_token} + def validate(token_str) do + with {:ok, token} <- get_by_token(token_str), + false <- expired?(token) do {:ok, token} - else - {:expired, _} -> {:error, :expired_token} - {:fetch_token, _} -> {:error, :not_found} - error -> {:error, error} end end - def create_token(%User{} = user) do - %__MODULE__{} - |> change - |> assign_user(user) - |> put_token - |> put_valid_until - |> Repo.insert() + defp expired?(%__MODULE__{valid_until: valid_until}) do + with true <- NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0 do + {:error, :expired_token} + end + end + + @spec create(User.t(), Authorization.t() | nil) :: {:ok, t()} | {:error, Ecto.Changeset.t()} + def create(user, authorization \\ nil) do + with {:ok, token} <- do_create(user, authorization) do + Pleroma.Workers.PurgeExpiredToken.enqueue(%{ + token_id: token.id, + valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"), + mod: __MODULE__ + }) + + {:ok, token} + end end - def create_token(user, authorization) do + defp do_create(user, authorization) do %__MODULE__{} - |> change + |> change() |> assign_user(user) - |> assign_authorization(authorization) - |> put_token - |> put_valid_until + |> maybe_assign_authorization(authorization) + |> put_token() + |> put_valid_until() |> Repo.insert() end @@ -69,15 +78,19 @@ defmodule Pleroma.MFA.Token do |> validate_required([:user]) end - defp assign_authorization(changeset, authorization) do + defp maybe_assign_authorization(changeset, %Authorization{} = authorization) do changeset |> put_assoc(:authorization, authorization) |> validate_required([:authorization]) end + defp maybe_assign_authorization(changeset, _), do: changeset + defp put_token(changeset) do + token = Pleroma.Web.OAuth.Token.Utils.generate_token() + changeset - |> change(%{token: OAuthToken.Utils.generate_token()}) + |> change(%{token: token}) |> validate_required([:token]) |> unique_constraint(:token) end @@ -89,18 +102,4 @@ defmodule Pleroma.MFA.Token do |> change(%{valid_until: expires_in}) |> validate_required([:valid_until]) end - - def is_expired?(%__MODULE__{valid_until: valid_until}) do - NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0 - end - - def is_expired?(_), do: false - - def delete_expired_tokens do - from( - q in __MODULE__, - where: fragment("?", q.valid_until) < ^Timex.now() - ) - |> Repo.delete_all() - end end diff --git a/lib/pleroma/migration_helper/notification_backfill.ex b/lib/pleroma/migration_helper/notification_backfill.ex index 09647d12a..d260e62ca 100644 --- a/lib/pleroma/migration_helper/notification_backfill.ex +++ b/lib/pleroma/migration_helper/notification_backfill.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.MigrationHelper.NotificationBackfill do - alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -18,25 +17,34 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do ) query - |> Repo.all() + |> Repo.chunk_stream(100) |> Enum.each(fn notification -> type = notification.activity |> type_from_activity() notification - |> Notification.changeset(%{type: type}) + |> Ecto.Changeset.change(%{type: type}) |> Repo.update() end) end + defp get_by_ap_id(ap_id) do + q = + from(u in User, + select: u.id + ) + + Repo.get_by(q, ap_id: ap_id) + end + # This is copied over from Notifications to keep this stable. defp type_from_activity(%{data: %{"type" => type}} = activity) do case type do "Follow" -> accepted_function = fn activity -> - with %User{} = follower <- User.get_by_ap_id(activity.data["actor"]), - %User{} = followed <- User.get_by_ap_id(activity.data["object"]) do + with %User{} = follower <- get_by_ap_id(activity.data["actor"]), + %User{} = followed <- get_by_ap_id(activity.data["object"]) do Pleroma.FollowingRelationship.following?(follower, followed) end end diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index 7aacd9d80..31c9afe2a 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -413,6 +413,17 @@ defmodule Pleroma.ModerationLog do def get_log_entry_message(%ModerationLog{ data: %{ "actor" => %{"nickname" => actor_nickname}, + "action" => "approve", + "subject" => users + } + }) do + "@#{actor_nickname} approved users: #{users_to_nicknames_string(users)}" + end + + @spec get_log_entry_message(ModerationLog) :: String.t() + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, "nicknames" => nicknames, "tags" => tags, "action" => "tag" diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 9ee9606be..8868a910e 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Notification do alias Pleroma.Repo alias Pleroma.ThreadMute alias Pleroma.User + alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.Push alias Pleroma.Web.Streamer @@ -130,6 +131,7 @@ defmodule Pleroma.Notification do |> preload([n, a, o], activity: {a, object: o}) |> exclude_notification_muted(user, exclude_notification_muted_opts) |> exclude_blocked(user, exclude_blocked_opts) + |> exclude_filtered(user) |> exclude_visibility(opts) end @@ -158,6 +160,20 @@ defmodule Pleroma.Notification do |> where([n, a, o, tm], is_nil(tm.user_id)) end + defp exclude_filtered(query, user) do + case Pleroma.Filter.compose_regex(user) do + nil -> + query + + regex -> + from([_n, a, o] in query, + where: + fragment("not(?->>'content' ~* ?)", o.data, ^regex) or + fragment("?->>'actor' = ?", o.data, ^user.ap_id) + ) + end + end + @valid_visibilities ~w[direct unlisted public private] defp exclude_visibility(query, %{exclude_visibilities: visibility}) @@ -337,6 +353,7 @@ defmodule Pleroma.Notification do end end + @spec create_notifications(Activity.t(), keyword()) :: {:ok, [Notification.t()] | []} def create_notifications(activity, options \\ []) def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity, options) do @@ -367,6 +384,7 @@ defmodule Pleroma.Notification do do_send = do_send && user in enabled_receivers create_notification(activity, user, do_send) end) + |> Enum.reject(&is_nil/1) {:ok, notifications} end @@ -424,6 +442,7 @@ defmodule Pleroma.Notification do |> Multi.insert(:notification, %Notification{ user_id: user.id, activity: activity, + seen: mark_as_read?(activity, user), type: type_from_activity(activity) }) |> Marker.multi_set_last_read_id(user, "notifications") @@ -480,6 +499,10 @@ defmodule Pleroma.Notification do end end + def get_potential_receiver_ap_ids(%{data: %{"type" => "Follow", "object" => object_id}}) do + [object_id] + end + def get_potential_receiver_ap_ids(activity) do [] |> Utils.maybe_notify_to_recipients(activity) @@ -550,11 +573,9 @@ defmodule Pleroma.Notification do [ :self, :invisible, - :followers, - :follows, - :non_followers, - :non_follows, - :recently_followed + :block_from_strangers, + :recently_followed, + :filtered ] |> Enum.find(&skip?(&1, activity, user)) end @@ -573,45 +594,15 @@ defmodule Pleroma.Notification do end def skip?( - :followers, - %Activity{} = activity, - %User{notification_settings: %{followers: false}} = user - ) do - actor = activity.data["actor"] - follower = User.get_cached_by_ap_id(actor) - User.following?(follower, user) - end - - def skip?( - :non_followers, + :block_from_strangers, %Activity{} = activity, - %User{notification_settings: %{non_followers: false}} = user + %User{notification_settings: %{block_from_strangers: true}} = user ) do actor = activity.data["actor"] follower = User.get_cached_by_ap_id(actor) !User.following?(follower, user) end - def skip?( - :follows, - %Activity{} = activity, - %User{notification_settings: %{follows: false}} = user - ) do - actor = activity.data["actor"] - followed = User.get_cached_by_ap_id(actor) - User.following?(user, followed) - end - - def skip?( - :non_follows, - %Activity{} = activity, - %User{notification_settings: %{non_follows: false}} = user - ) do - actor = activity.data["actor"] - followed = User.get_cached_by_ap_id(actor) - !User.following?(user, followed) - end - # To do: consider defining recency in hours and checking FollowingRelationship with a single SQL def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do actor = activity.data["actor"] @@ -623,8 +614,33 @@ defmodule Pleroma.Notification do end) end + def skip?(:filtered, %{data: %{"type" => type}}, _) when type in ["Follow", "Move"], do: false + + def skip?(:filtered, activity, user) do + object = Object.normalize(activity) + + cond do + is_nil(object) -> + false + + object.data["actor"] == user.ap_id -> + false + + not is_nil(regex = Pleroma.Filter.compose_regex(user, :re)) -> + Regex.match?(regex, object.data["content"]) + + true -> + false + end + end + def skip?(_, _, _), do: false + def mark_as_read?(activity, target_user) do + user = Activity.user_actor(activity) + User.mutes_user?(target_user, user) || CommonAPI.thread_muted?(target_user, activity) + end + def for_user_and_activity(user, activity) do from(n in __MODULE__, where: n.user_id == ^user.id, @@ -632,4 +648,16 @@ defmodule Pleroma.Notification do ) |> Repo.one() end + + @spec mark_context_as_read(User.t(), String.t()) :: {integer(), nil | [term()]} + def mark_context_as_read(%User{id: id}, context) do + from( + n in Notification, + join: a in assoc(n, :activity), + where: n.user_id == ^id, + where: n.seen == false, + where: fragment("?->>'context'", a.data) == ^context + ) + |> Repo.update_all(set: [seen: true]) + end end diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 546c4ea01..052ad413b 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -255,6 +255,10 @@ defmodule Pleroma.Object do end end + defp poll_is_multiple?(%Object{data: %{"anyOf" => [_ | _]}}), do: true + + defp poll_is_multiple?(_), do: false + def decrease_replies_count(ap_id) do Object |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id))) @@ -281,10 +285,10 @@ defmodule Pleroma.Object do def increase_vote_count(ap_id, name, actor) do with %Object{} = object <- Object.normalize(ap_id), "Question" <- object.data["type"] do - multiple = Map.has_key?(object.data, "anyOf") + key = if poll_is_multiple?(object), do: "anyOf", else: "oneOf" options = - (object.data["anyOf"] || object.data["oneOf"] || []) + object.data[key] |> Enum.map(fn %{"name" => ^name} = option -> Kernel.update_in(option["replies"]["totalItems"], &(&1 + 1)) @@ -296,11 +300,8 @@ defmodule Pleroma.Object do voters = [actor | object.data["voters"] || []] |> Enum.uniq() data = - if multiple do - Map.put(object.data, "anyOf", options) - else - Map.put(object.data, "oneOf", options) - end + object.data + |> Map.put(key, options) |> Map.put("voters", voters) object diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index 99608b8a5..29cb3d672 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -44,18 +44,11 @@ defmodule Pleroma.Object.Containment do nil end - # TODO: We explicitly allow 'tag' URIs through, due to references to legacy OStatus - # objects being present in the test suite environment. Once these objects are - # removed, please also remove this. - if Mix.env() == :test do - defp compare_uris(_, %URI{scheme: "tag"}), do: :ok - end - defp compare_uris(%URI{host: host} = _id_uri, %URI{host: host} = _other_uri), do: :ok defp compare_uris(_id_uri, _other_uri), do: :error @doc """ - Checks that an imported AP object's actor matches the domain it came from. + Checks that an imported AP object's actor matches the host it came from. """ def contain_origin(_id, %{"actor" => nil}), do: :error diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 263ded5dd..1de2ce6c3 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.Repo alias Pleroma.Signature alias Pleroma.Web.ActivityPub.InternalFetchActor + alias Pleroma.Web.ActivityPub.ObjectValidator alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.Federator @@ -23,21 +24,38 @@ defmodule Pleroma.Object.Fetcher do Ecto.Changeset.put_change(changeset, :updated_at, updated_at) end - defp maybe_reinject_internal_fields(data, %{data: %{} = old_data}) do + defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields()) - Map.merge(data, internal_fields) + Map.merge(new_data, internal_fields) end - defp maybe_reinject_internal_fields(data, _), do: data + defp maybe_reinject_internal_fields(_, new_data), do: new_data @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()} - defp reinject_object(struct, data) do - Logger.debug("Reinjecting object #{data["id"]}") + defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do + Logger.debug("Reinjecting object #{new_data["id"]}") - with data <- Transmogrifier.fix_object(data), - data <- maybe_reinject_internal_fields(data, struct), - changeset <- Object.change(struct, %{data: data}), + with data <- maybe_reinject_internal_fields(object, new_data), + {:ok, data, _} <- ObjectValidator.validate(data, %{}), + changeset <- Object.change(object, %{data: data}), + changeset <- touch_changeset(changeset), + {:ok, object} <- Repo.insert_or_update(changeset), + {:ok, object} <- Object.set_cache(object) do + {:ok, object} + else + e -> + Logger.error("Error while processing object: #{inspect(e)}") + {:error, e} + end + end + + defp reinject_object(%Object{} = object, new_data) do + Logger.debug("Reinjecting object #{new_data["id"]}") + + with new_data <- Transmogrifier.fix_object(new_data), + data <- maybe_reinject_internal_fields(object, new_data), + changeset <- Object.change(object, %{data: data}), changeset <- touch_changeset(changeset), {:ok, object} <- Repo.insert_or_update(changeset), {:ok, object} <- Object.set_cache(object) do @@ -51,8 +69,8 @@ defmodule Pleroma.Object.Fetcher do def refetch_object(%Object{data: %{"id" => id}} = object) do with {:local, false} <- {:local, Object.local?(object)}, - {:ok, data} <- fetch_and_contain_remote_object_from_id(id), - {:ok, object} <- reinject_object(object, data) do + {:ok, new_data} <- fetch_and_contain_remote_object_from_id(id), + {:ok, object} <- reinject_object(object, new_data) do {:ok, object} else {:local, true} -> {:ok, object} @@ -83,8 +101,8 @@ defmodule Pleroma.Object.Fetcher do {:transmogrifier, {:error, {:reject, nil}}} -> {:reject, nil} - {:transmogrifier, _} -> - {:error, "Transmogrifier failure."} + {:transmogrifier, _} = e -> + {:error, e} {:object, data, nil} -> reinject_object(%Object{}, data) @@ -106,8 +124,8 @@ defmodule Pleroma.Object.Fetcher do defp prepare_activity_params(data) do %{ "type" => "Create", - "to" => data["to"], - "cc" => data["cc"], + "to" => data["to"] || [], + "cc" => data["cc"] || [], # Should we seriously keep this attributedTo thing? "actor" => data["actor"] || data["attributedTo"], "object" => data @@ -124,6 +142,10 @@ defmodule Pleroma.Object.Fetcher do {:error, "Object has been deleted"} -> nil + {:reject, reason} -> + Logger.info("Rejected #{id} while fetching: #{inspect(reason)}") + nil + e -> Logger.error("Error while fetching #{id}: #{inspect(e)}") nil @@ -141,12 +163,12 @@ defmodule Pleroma.Object.Fetcher do date: date }) - [{"signature", signature}] + {"signature", signature} end defp sign_fetch(headers, id, date) do if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do - headers ++ make_signature(id, date) + [make_signature(id, date) | headers] else headers end @@ -154,7 +176,7 @@ defmodule Pleroma.Object.Fetcher do defp maybe_date_fetch(headers, date) do if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do - headers ++ [{"date", date}] + [{"date", date} | headers] else headers end diff --git a/lib/pleroma/pagination.ex b/lib/pleroma/pagination.ex index 1b99e44f9..9a3795769 100644 --- a/lib/pleroma/pagination.ex +++ b/lib/pleroma/pagination.ex @@ -64,6 +64,12 @@ defmodule Pleroma.Pagination do @spec paginate(Ecto.Query.t(), map(), type(), atom() | nil) :: [Ecto.Schema.t()] def paginate(query, options, method \\ :keyset, table_binding \\ nil) + def paginate(list, options, _method, _table_binding) when is_list(list) do + offset = options[:offset] || 0 + limit = options[:limit] || 0 + Enum.slice(list, offset, limit) + end + def paginate(query, options, :keyset, table_binding) do query |> restrict(:min_id, options, table_binding) diff --git a/lib/pleroma/plugs/admin_secret_authentication_plug.ex b/lib/pleroma/plugs/admin_secret_authentication_plug.ex index b4b47a31f..2e54df47a 100644 --- a/lib/pleroma/plugs/admin_secret_authentication_plug.ex +++ b/lib/pleroma/plugs/admin_secret_authentication_plug.ex @@ -4,6 +4,9 @@ defmodule Pleroma.Plugs.AdminSecretAuthenticationPlug do import Plug.Conn + + alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.Plugs.RateLimiter alias Pleroma.User def init(options) do @@ -11,7 +14,10 @@ defmodule Pleroma.Plugs.AdminSecretAuthenticationPlug do end def secret_token do - Pleroma.Config.get(:admin_token) + case Pleroma.Config.get(:admin_token) do + blank when blank in [nil, ""] -> nil + token -> token + end end def call(%{assigns: %{user: %User{}}} = conn, _), do: conn @@ -26,9 +32,9 @@ defmodule Pleroma.Plugs.AdminSecretAuthenticationPlug do def authenticate(%{params: %{"admin_token" => admin_token}} = conn) do if admin_token == secret_token() do - assign(conn, :user, %User{is_admin: true}) + assign_admin_user(conn) else - conn + handle_bad_token(conn) end end @@ -36,8 +42,19 @@ defmodule Pleroma.Plugs.AdminSecretAuthenticationPlug do token = secret_token() case get_req_header(conn, "x-admin-token") do - [^token] -> assign(conn, :user, %User{is_admin: true}) - _ -> conn + blank when blank in [[], [""]] -> conn + [^token] -> assign_admin_user(conn) + _ -> handle_bad_token(conn) end end + + defp assign_admin_user(conn) do + conn + |> assign(:user, %User{is_admin: true}) + |> OAuthScopesPlug.skip_plug() + end + + defp handle_bad_token(conn) do + RateLimiter.call(conn, name: :authentication) + end end diff --git a/lib/pleroma/plugs/frontend_static.ex b/lib/pleroma/plugs/frontend_static.ex new file mode 100644 index 000000000..11a0d5382 --- /dev/null +++ b/lib/pleroma/plugs/frontend_static.ex @@ -0,0 +1,55 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Plugs.FrontendStatic do + require Pleroma.Constants + + @moduledoc """ + This is a shim to call `Plug.Static` but with runtime `from` configuration`. It dispatches to the different frontends. + """ + @behaviour Plug + + def file_path(path, frontend_type \\ :primary) do + if configuration = Pleroma.Config.get([:frontends, frontend_type]) do + instance_static_path = Pleroma.Config.get([:instance, :static_dir], "instance/static") + + Path.join([ + instance_static_path, + "frontends", + configuration["name"], + configuration["ref"], + path + ]) + else + nil + end + end + + def init(opts) do + opts + |> Keyword.put(:from, "__unconfigured_frontend_static_plug") + |> Plug.Static.init() + |> Map.put(:frontend_type, opts[:frontend_type]) + end + + def call(conn, opts) do + frontend_type = Map.get(opts, :frontend_type, :primary) + path = file_path("", frontend_type) + + if path do + conn + |> call_static(opts, path) + else + conn + end + end + + defp call_static(conn, opts, from) do + opts = + opts + |> Map.put(:from, from) + + Plug.Static.call(conn, opts) + end +end diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 1420a9611..c363b193b 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -69,10 +69,11 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" + # Strict multimedia CSP enforcement only when MediaProxy is enabled {img_src, media_src} = if Config.get([:media_proxy, :enabled]) && !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do - sources = get_proxy_and_attachment_sources() + sources = build_csp_multimedia_source_list() {[img_src, sources], [media_src, sources]} else {[img_src, " https:"], [media_src, " https:"]} @@ -81,14 +82,14 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] connect_src = - if Pleroma.Config.get(:env) == :dev do + if Config.get(:env) == :dev do [connect_src, " http://localhost:3035/"] else connect_src end script_src = - if Pleroma.Config.get(:env) == :dev do + if Config.get(:env) == :dev do "script-src 'self' 'unsafe-eval'" else "script-src 'self'" @@ -107,38 +108,64 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do |> :erlang.iolist_to_binary() end - defp get_proxy_and_attachment_sources do + defp build_csp_from_whitelist([], acc), do: acc + + defp build_csp_from_whitelist([last], acc) do + [build_csp_param_from_whitelist(last) | acc] + end + + defp build_csp_from_whitelist([head | tail], acc) do + build_csp_from_whitelist(tail, [[?\s, build_csp_param_from_whitelist(head)] | acc]) + end + + # TODO: use `build_csp_param/1` after removing support bare domains for media proxy whitelist + defp build_csp_param_from_whitelist("http" <> _ = url) do + build_csp_param(url) + end + + defp build_csp_param_from_whitelist(url), do: url + + defp build_csp_multimedia_source_list do media_proxy_whitelist = - Enum.reduce(Config.get([:media_proxy, :whitelist]), [], fn host, acc -> - add_source(acc, host) - end) - - media_proxy_base_url = - if Config.get([:media_proxy, :base_url]), - do: URI.parse(Config.get([:media_proxy, :base_url])).host - - upload_base_url = - if Config.get([Pleroma.Upload, :base_url]), - do: URI.parse(Config.get([Pleroma.Upload, :base_url])).host - - s3_endpoint = - if Config.get([Pleroma.Upload, :uploader]) == Pleroma.Uploaders.S3, - do: URI.parse(Config.get([Pleroma.Uploaders.S3, :public_endpoint])).host - - [] - |> add_source(media_proxy_base_url) - |> add_source(upload_base_url) - |> add_source(s3_endpoint) + [:media_proxy, :whitelist] + |> Config.get() + |> build_csp_from_whitelist([]) + + captcha_method = Config.get([Pleroma.Captcha, :method]) + captcha_endpoint = Config.get([captcha_method, :endpoint]) + + base_endpoints = + [ + [:media_proxy, :base_url], + [Pleroma.Upload, :base_url], + [Pleroma.Uploaders.S3, :public_endpoint] + ] + |> Enum.map(&Config.get/1) + + [captcha_endpoint | base_endpoints] + |> Enum.map(&build_csp_param/1) + |> Enum.reduce([], &add_source(&2, &1)) |> add_source(media_proxy_whitelist) end defp add_source(iodata, nil), do: iodata + defp add_source(iodata, []), do: iodata defp add_source(iodata, source), do: [[?\s, source] | iodata] defp add_csp_param(csp_iodata, nil), do: csp_iodata defp add_csp_param(csp_iodata, param), do: [[param, ?;] | csp_iodata] + defp build_csp_param(nil), do: nil + + defp build_csp_param(url) when is_binary(url) do + %{host: host, scheme: scheme} = URI.parse(url) + + if scheme do + [scheme, "://", host] + end + end + def warn_if_disabled do unless Config.get([:http_security, :enabled]) do Logger.warn(" diff --git a/lib/pleroma/plugs/instance_static.ex b/lib/pleroma/plugs/instance_static.ex index 7516f75c3..0fb57e422 100644 --- a/lib/pleroma/plugs/instance_static.ex +++ b/lib/pleroma/plugs/instance_static.ex @@ -16,28 +16,24 @@ defmodule Pleroma.Plugs.InstanceStatic do instance_path = Path.join(Pleroma.Config.get([:instance, :static_dir], "instance/static/"), path) - if File.exists?(instance_path) do - instance_path - else + frontend_path = Pleroma.Plugs.FrontendStatic.file_path(path, :primary) + + (File.exists?(instance_path) && instance_path) || + (frontend_path && File.exists?(frontend_path) && frontend_path) || Path.join(Application.app_dir(:pleroma, "priv/static/"), path) - end end def init(opts) do opts |> Keyword.put(:from, "__unconfigured_instance_static_plug") - |> Keyword.put(:at, "/__unconfigured_instance_static_plug") |> Plug.Static.init() end for only <- Pleroma.Constants.static_only_files() do - at = Plug.Router.Utils.split("/") - def call(%{request_path: "/" <> unquote(only) <> _} = conn, opts) do call_static( conn, opts, - unquote(at), Pleroma.Config.get([:instance, :static_dir], "instance/static") ) end @@ -47,11 +43,10 @@ defmodule Pleroma.Plugs.InstanceStatic do conn end - defp call_static(conn, opts, at, from) do + defp call_static(conn, opts, from) do opts = opts |> Map.put(:from, from) - |> Map.put(:at, at) Plug.Static.call(conn, opts) end diff --git a/lib/pleroma/plugs/remote_ip.ex b/lib/pleroma/plugs/remote_ip.ex index 2eca4f8f6..0ac9050d0 100644 --- a/lib/pleroma/plugs/remote_ip.ex +++ b/lib/pleroma/plugs/remote_ip.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Plugs.RemoteIp do This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration. """ + import Plug.Conn + @behaviour Plug @headers ~w[ @@ -26,11 +28,12 @@ defmodule Pleroma.Plugs.RemoteIp do def init(_), do: nil - def call(conn, _) do + def call(%{remote_ip: original_remote_ip} = conn, _) do config = Pleroma.Config.get(__MODULE__, []) if Keyword.get(config, :enabled, false) do - RemoteIp.call(conn, remote_ip_opts(config)) + %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts(config)) + assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip) else conn end diff --git a/lib/pleroma/plugs/static_fe_plug.ex b/lib/pleroma/plugs/static_fe_plug.ex index 156e6788e..143665c71 100644 --- a/lib/pleroma/plugs/static_fe_plug.ex +++ b/lib/pleroma/plugs/static_fe_plug.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Plugs.StaticFEPlug do def init(options), do: options def call(conn, _) do - if enabled?() and accepts_html?(conn) do + if enabled?() and requires_html?(conn) do conn |> StaticFEController.call(:show) |> halt() @@ -20,10 +20,7 @@ defmodule Pleroma.Plugs.StaticFEPlug do defp enabled?, do: Pleroma.Config.get([:static_fe, :enabled], false) - defp accepts_html?(conn) do - case get_req_header(conn, "accept") do - [accept | _] -> String.contains?(accept, "text/html") - _ -> false - end + defp requires_html?(conn) do + Phoenix.Controller.get_format(conn) == "html" end end diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex index 94147e0c4..40984cfc0 100644 --- a/lib/pleroma/plugs/uploaded_media.ex +++ b/lib/pleroma/plugs/uploaded_media.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Plugs.UploadedMedia do import Pleroma.Web.Gettext require Logger + alias Pleroma.Web.MediaProxy + @behaviour Plug # no slashes @path "media" @@ -35,8 +37,7 @@ defmodule Pleroma.Plugs.UploadedMedia do %{query_params: %{"name" => name}} = conn -> name = String.replace(name, "\"", "\\\"") - conn - |> put_resp_header("content-disposition", "filename=\"#{name}\"") + put_resp_header(conn, "content-disposition", "filename=\"#{name}\"") conn -> conn @@ -47,7 +48,8 @@ defmodule Pleroma.Plugs.UploadedMedia do with uploader <- Keyword.fetch!(config, :uploader), proxy_remote = Keyword.get(config, :proxy_remote, false), - {:ok, get_method} <- uploader.get_file(file) do + {:ok, get_method} <- uploader.get_file(file), + false <- media_is_banned(conn, get_method) do get_media(conn, get_method, proxy_remote, opts) else _ -> @@ -59,6 +61,14 @@ defmodule Pleroma.Plugs.UploadedMedia do def call(conn, _opts), do: conn + defp media_is_banned(%{request_path: path} = _conn, {:static_dir, _}) do + MediaProxy.in_banned_urls(Pleroma.Web.base_url() <> path) + end + + defp media_is_banned(_, {:url, url}), do: MediaProxy.in_banned_urls(url) + + defp media_is_banned(_, _), do: false + defp get_media(conn, {:static_dir, directory}, _, opts) do static_opts = Map.get(opts, :static_plug_opts) diff --git a/lib/pleroma/plugs/user_is_admin_plug.ex b/lib/pleroma/plugs/user_is_admin_plug.ex index 2748102df..488a61d1d 100644 --- a/lib/pleroma/plugs/user_is_admin_plug.ex +++ b/lib/pleroma/plugs/user_is_admin_plug.ex @@ -7,37 +7,18 @@ defmodule Pleroma.Plugs.UserIsAdminPlug do import Plug.Conn alias Pleroma.User - alias Pleroma.Web.OAuth def init(options) do options end - def call(%{assigns: %{user: %User{is_admin: true}} = assigns} = conn, _) do - token = assigns[:token] - - cond do - not Pleroma.Config.enforce_oauth_admin_scope_usage?() -> - conn - - token && OAuth.Scopes.contains_admin_scopes?(token.scopes) -> - # Note: checking for _any_ admin scope presence, not necessarily fitting requested action. - # Thus, controller must explicitly invoke OAuthScopesPlug to verify scope requirements. - # Admin might opt out of admin scope for some apps to block any admin actions from them. - conn - - true -> - fail(conn) - end + def call(%{assigns: %{user: %User{is_admin: true}}} = conn, _) do + conn end def call(conn, _) do - fail(conn) - end - - defp fail(conn) do conn - |> render_error(:forbidden, "User is not an admin or OAuth admin scope is not granted.") + |> render_error(:forbidden, "User is not an admin.") |> halt() end end diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex deleted file mode 100644 index acafe1bea..000000000 --- a/lib/pleroma/pool/connections.ex +++ /dev/null @@ -1,283 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Pool.Connections do - use GenServer - - alias Pleroma.Config - alias Pleroma.Gun - - require Logger - - @type domain :: String.t() - @type conn :: Pleroma.Gun.Conn.t() - - @type t :: %__MODULE__{ - conns: %{domain() => conn()}, - opts: keyword() - } - - defstruct conns: %{}, opts: [] - - @spec start_link({atom(), keyword()}) :: {:ok, pid()} - def start_link({name, opts}) do - GenServer.start_link(__MODULE__, opts, name: name) - end - - @impl true - def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}} - - @spec checkin(String.t() | URI.t(), atom()) :: pid() | nil - def checkin(url, name) - def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) - - def checkin(%URI{} = uri, name) do - timeout = Config.get([:connections_pool, :checkin_timeout], 250) - - GenServer.call(name, {:checkin, uri}, timeout) - end - - @spec alive?(atom()) :: boolean() - def alive?(name) do - if pid = Process.whereis(name) do - Process.alive?(pid) - else - false - end - end - - @spec get_state(atom()) :: t() - def get_state(name) do - GenServer.call(name, :state) - end - - @spec count(atom()) :: pos_integer() - def count(name) do - GenServer.call(name, :count) - end - - @spec get_unused_conns(atom()) :: [{domain(), conn()}] - def get_unused_conns(name) do - GenServer.call(name, :unused_conns) - end - - @spec checkout(pid(), pid(), atom()) :: :ok - def checkout(conn, pid, name) do - GenServer.cast(name, {:checkout, conn, pid}) - end - - @spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok - def add_conn(name, key, conn) do - GenServer.cast(name, {:add_conn, key, conn}) - end - - @spec remove_conn(atom(), String.t()) :: :ok - def remove_conn(name, key) do - GenServer.cast(name, {:remove_conn, key}) - end - - @impl true - def handle_cast({:add_conn, key, conn}, state) do - state = put_in(state.conns[key], conn) - - Process.monitor(conn.conn) - {:noreply, state} - end - - @impl true - def handle_cast({:checkout, conn_pid, pid}, state) do - state = - with true <- Process.alive?(conn_pid), - {key, conn} <- find_conn(state.conns, conn_pid), - used_by <- List.keydelete(conn.used_by, pid, 0) do - conn_state = if used_by == [], do: :idle, else: conn.conn_state - - put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) - else - false -> - Logger.debug("checkout for closed conn #{inspect(conn_pid)}") - state - - nil -> - Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state") - state - end - - {:noreply, state} - end - - @impl true - def handle_cast({:remove_conn, key}, state) do - state = put_in(state.conns, Map.delete(state.conns, key)) - {:noreply, state} - end - - @impl true - def handle_call({:checkin, uri}, from, state) do - key = "#{uri.scheme}:#{uri.host}:#{uri.port}" - - case state.conns[key] do - %{conn: pid, gun_state: :up} = conn -> - time = :os.system_time(:second) - last_reference = time - conn.last_reference - crf = crf(last_reference, 100, conn.crf) - - state = - put_in(state.conns[key], %{ - conn - | last_reference: time, - crf: crf, - conn_state: :active, - used_by: [from | conn.used_by] - }) - - {:reply, pid, state} - - %{gun_state: :down} -> - {:reply, nil, state} - - nil -> - {:reply, nil, state} - end - end - - @impl true - def handle_call(:state, _from, state), do: {:reply, state, state} - - @impl true - def handle_call(:count, _from, state) do - {:reply, Enum.count(state.conns), state} - end - - @impl true - def handle_call(:unused_conns, _from, state) do - unused_conns = - state.conns - |> Enum.filter(&filter_conns/1) - |> Enum.sort(&sort_conns/2) - - {:reply, unused_conns, state} - end - - defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true - defp filter_conns(_), do: false - - defp sort_conns({_, c1}, {_, c2}) do - c1.crf <= c2.crf and c1.last_reference <= c2.last_reference - end - - @impl true - def handle_info({:gun_up, conn_pid, _protocol}, state) do - %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) - - host = - case :inet.ntoa(host) do - {:error, :einval} -> host - ip -> ip - end - - key = "#{scheme}:#{host}:#{port}" - - state = - with {key, conn} <- find_conn(state.conns, conn_pid, key), - {true, key} <- {Process.alive?(conn_pid), key} do - put_in(state.conns[key], %{ - conn - | gun_state: :up, - conn_state: :active, - retries: 0 - }) - else - {false, key} -> - put_in( - state.conns, - Map.delete(state.conns, key) - ) - - nil -> - :ok = Gun.close(conn_pid) - - state - end - - {:noreply, state} - end - - @impl true - def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do - retries = Config.get([:connections_pool, :retry], 1) - # we can't get info on this pid, because pid is dead - state = - with {key, conn} <- find_conn(state.conns, conn_pid), - {true, key} <- {Process.alive?(conn_pid), key} do - if conn.retries == retries do - :ok = Gun.close(conn.conn) - - put_in( - state.conns, - Map.delete(state.conns, key) - ) - else - put_in(state.conns[key], %{ - conn - | gun_state: :down, - retries: conn.retries + 1 - }) - end - else - {false, key} -> - put_in( - state.conns, - Map.delete(state.conns, key) - ) - - nil -> - Logger.debug(":gun_down for conn which isn't found in state") - - state - end - - {:noreply, state} - end - - @impl true - def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do - Logger.debug("received DOWN message for #{inspect(conn_pid)} reason -> #{inspect(reason)}") - - state = - with {key, conn} <- find_conn(state.conns, conn_pid) do - Enum.each(conn.used_by, fn {pid, _ref} -> - Process.exit(pid, reason) - end) - - put_in( - state.conns, - Map.delete(state.conns, key) - ) - else - nil -> - Logger.debug(":DOWN for conn which isn't found in state") - - state - end - - {:noreply, state} - end - - defp find_conn(conns, conn_pid) do - Enum.find(conns, fn {_key, conn} -> - conn.conn == conn_pid - end) - end - - defp find_conn(conns, conn_pid, conn_key) do - Enum.find(conns, fn {key, conn} -> - key == conn_key and conn.conn == conn_pid - end) - end - - def crf(current, steps, crf) do - 1 + :math.pow(0.5, current / steps) * crf - end -end diff --git a/lib/pleroma/pool/pool.ex b/lib/pleroma/pool/pool.ex deleted file mode 100644 index 21a6fbbc5..000000000 --- a/lib/pleroma/pool/pool.ex +++ /dev/null @@ -1,22 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Pool do - def child_spec(opts) do - poolboy_opts = - opts - |> Keyword.put(:worker_module, Pleroma.Pool.Request) - |> Keyword.put(:name, {:local, opts[:name]}) - |> Keyword.put(:size, opts[:size]) - |> Keyword.put(:max_overflow, opts[:max_overflow]) - - %{ - id: opts[:id] || {__MODULE__, make_ref()}, - start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]}, - restart: :permanent, - shutdown: 5000, - type: :worker - } - end -end diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex deleted file mode 100644 index 3fb930db7..000000000 --- a/lib/pleroma/pool/request.ex +++ /dev/null @@ -1,65 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Pool.Request do - use GenServer - - require Logger - - def start_link(args) do - GenServer.start_link(__MODULE__, args) - end - - @impl true - def init(_), do: {:ok, []} - - @spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) :: - {:ok, Tesla.Env.t()} | {:error, any()} - def execute(pid, client, request, timeout) do - GenServer.call(pid, {:execute, client, request}, timeout) - end - - @impl true - def handle_call({:execute, client, request}, _from, state) do - response = Pleroma.HTTP.request(client, request) - - {:reply, response, state} - end - - @impl true - def handle_info({:gun_data, _conn, _stream, _, _}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_up, _conn, _protocol}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_error, _conn, _stream, _error}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do - {:noreply, state} - end - - @impl true - def handle_info(msg, state) do - Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}") - {:noreply, state} - end -end diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex deleted file mode 100644 index faf646cb2..000000000 --- a/lib/pleroma/pool/supervisor.ex +++ /dev/null @@ -1,42 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Pool.Supervisor do - use Supervisor - - alias Pleroma.Config - alias Pleroma.Pool - - def start_link(args) do - Supervisor.start_link(__MODULE__, args, name: __MODULE__) - end - - def init(_) do - conns_child = %{ - id: Pool.Connections, - start: - {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]} - } - - Supervisor.init([conns_child | pools()], strategy: :one_for_one) - end - - defp pools do - pools = Config.get(:pools) - - pools = - if Config.get([Pleroma.Upload, :proxy_remote]) == false do - Keyword.delete(pools, :upload) - else - pools - end - - for {pool_name, pool_opts} <- pools do - pool_opts - |> Keyword.put(:id, {Pool, pool_name}) - |> Keyword.put(:name, pool_name) - |> Pool.child_spec() - end - end -end diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index f62138466..f317e4d58 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -8,11 +8,10 @@ defmodule Pleroma.Repo do adapter: Ecto.Adapters.Postgres, migration_timestamps: [type: :naive_datetime_usec] + import Ecto.Query require Logger - defmodule Instrumenter do - use Prometheus.EctoInstrumenter - end + defmodule Instrumenter, do: use(Prometheus.EctoInstrumenter) @doc """ Dynamically loads the repository url from the @@ -50,36 +49,30 @@ defmodule Pleroma.Repo do end end - def check_migrations_applied!() do - unless Pleroma.Config.get( - [:i_am_aware_this_may_cause_data_loss, :disable_migration_check], - false - ) do - Ecto.Migrator.with_repo(__MODULE__, fn repo -> - down_migrations = - Ecto.Migrator.migrations(repo) - |> Enum.reject(fn - {:up, _, _} -> true - {:down, _, _} -> false - end) - - if length(down_migrations) > 0 do - down_migrations_text = - Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end) - - Logger.error( - "The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true" - ) + def chunk_stream(query, chunk_size) do + # We don't actually need start and end funcitons of resource streaming, + # but it seems to be the only way to not fetch records one-by-one and + # have individual records be the elements of the stream, instead of + # lists of records + Stream.resource( + fn -> 0 end, + fn + last_id -> + query + |> order_by(asc: :id) + |> where([r], r.id > ^last_id) + |> limit(^chunk_size) + |> all() + |> case do + [] -> + {:halt, last_id} - raise Pleroma.Repo.UnappliedMigrationsError - end - end) - else - :ok - end + records -> + last_id = List.last(records).id + {records, last_id} + end + end, + fn _ -> :ok end + ) end end - -defmodule Pleroma.Repo.UnappliedMigrationsError do - defexception message: "Unapplied Migrations detected" -end diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex index e84118a90..ad988fac3 100644 --- a/lib/pleroma/reverse_proxy/client/hackney.ex +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -7,6 +7,7 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do @impl true def request(method, url, headers, body, opts \\ []) do + opts = Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1]) :hackney.request(method, url, headers, body, opts) end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index e81ea8bde..4b118eec2 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -5,6 +5,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @behaviour Pleroma.ReverseProxy.Client + alias Pleroma.Gun.ConnectionPool + @type headers() :: [{String.t(), String.t()}] @type status() :: pos_integer() @@ -26,11 +28,13 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do url, body, headers, - Keyword.put(opts, :adapter, opts) + opts ) do if is_map(response.body) and method != :head do {:ok, response.status, response.headers, response.body} else + conn_pid = response.opts[:adapter][:conn] + ConnectionPool.release_conn(conn_pid) {:ok, response.status, response.headers} end else @@ -41,15 +45,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @impl true @spec stream_body(map()) :: {:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return() - def stream_body(%{pid: pid, opts: opts, fin: true}) do - # if connection was reused, but in tesla were redirects, - # tesla returns new opened connection, which must be closed manually - if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid) - # if there were redirects we need to checkout old conn - conn = opts[:old_conn] || opts[:conn] - - if conn, do: :ok = Pleroma.Pool.Connections.checkout(conn, self(), :gun_connections) - + def stream_body(%{pid: pid, fin: true}) do + ConnectionPool.release_conn(pid) :done end @@ -74,8 +71,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @impl true @spec close(map) :: :ok | no_return() def close(%{pid: pid}) do - adapter = check_adapter() - adapter.close(pid) + ConnectionPool.release_conn(pid) end defp check_adapter do diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 4bbeb493c..0de4e2309 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -3,12 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy do + @range_headers ~w(range if-range) @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++ - ~w(if-unmodified-since if-none-match if-range range) + ~w(if-unmodified-since if-none-match) ++ @range_headers @resp_cache_headers ~w(etag date last-modified) @keep_resp_headers @resp_cache_headers ++ - ~w(content-type content-disposition content-encoding content-range) ++ - ~w(accept-ranges vary) + ~w(content-length content-type content-disposition content-encoding) ++ + ~w(content-range accept-ranges vary) @default_cache_control_header "public, max-age=1209600" @valid_resp_codes [200, 206, 304] @max_read_duration :timer.seconds(30) @@ -164,12 +165,17 @@ defmodule Pleroma.ReverseProxy do {:ok, code, _, _} -> {:error, {:invalid_http_response, code}} + {:ok, code, _} -> + {:error, {:invalid_http_response, code}} + {:error, error} -> {:error, error} end end defp response(conn, client, url, status, headers, opts) do + Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}") + result = conn |> put_resp_headers(build_resp_headers(headers, opts)) @@ -220,7 +226,9 @@ defmodule Pleroma.ReverseProxy do end end - defp head_response(conn, _url, code, headers, opts) do + defp head_response(conn, url, code, headers, opts) do + Logger.debug("#{__MODULE__} #{code} #{url} #{inspect(headers)}") + conn |> put_resp_headers(build_resp_headers(headers, opts)) |> send_resp(code, "") @@ -262,20 +270,33 @@ defmodule Pleroma.ReverseProxy do headers |> downcase_headers() |> Enum.filter(fn {k, _} -> k in @keep_req_headers end) - |> (fn headers -> - headers = headers ++ Keyword.get(opts, :req_headers, []) - - if Keyword.get(opts, :keep_user_agent, false) do - List.keystore( - headers, - "user-agent", - 0, - {"user-agent", Pleroma.Application.user_agent()} - ) - else - headers - end - end).() + |> build_req_range_or_encoding_header(opts) + |> build_req_user_agent_header(opts) + |> Keyword.merge(Keyword.get(opts, :req_headers, [])) + end + + # Disable content-encoding if any @range_headers are requested (see #1823). + defp build_req_range_or_encoding_header(headers, _opts) do + range? = Enum.any?(headers, fn {header, _} -> Enum.member?(@range_headers, header) end) + + if range? && List.keymember?(headers, "accept-encoding", 0) do + List.keydelete(headers, "accept-encoding", 0) + else + headers + end + end + + defp build_req_user_agent_header(headers, opts) do + if Keyword.get(opts, :keep_user_agent, false) do + List.keystore( + headers, + "user-agent", + 0, + {"user-agent", Pleroma.Application.user_agent()} + ) + else + headers + end end defp build_resp_headers(headers, opts) do @@ -283,7 +304,7 @@ defmodule Pleroma.ReverseProxy do |> Enum.filter(fn {k, _} -> k in @keep_resp_headers end) |> build_resp_cache_headers(opts) |> build_resp_content_disposition_header(opts) - |> (fn headers -> headers ++ Keyword.get(opts, :resp_headers, []) end).() + |> Keyword.merge(Keyword.get(opts, :resp_headers, [])) end defp build_resp_cache_headers(headers, _opts) do diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index d01728361..3aa6909d2 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -5,10 +5,10 @@ defmodule Pleroma.Signature do @behaviour HTTPSignatures.Adapter + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Keys alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.ObjectValidators.Types def key_id_to_actor_id(key_id) do uri = @@ -24,7 +24,7 @@ defmodule Pleroma.Signature do maybe_ap_id = URI.to_string(uri) - case Types.ObjectID.cast(maybe_ap_id) do + case ObjectValidators.ObjectID.cast(maybe_ap_id) do {:ok, ap_id} -> {:ok, ap_id} diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 6b3a8a41f..e5c9c668b 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -3,12 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Stats do + use GenServer + import Ecto.Query + alias Pleroma.CounterCache alias Pleroma.Repo alias Pleroma.User - use GenServer + @interval :timer.seconds(60) def start_link(_) do GenServer.start_link( @@ -18,6 +21,12 @@ defmodule Pleroma.Stats do ) end + @impl true + def init(_args) do + if Pleroma.Config.get(:env) == :test, do: :ok = Ecto.Adapters.SQL.Sandbox.checkout(Repo) + {:ok, nil, {:continue, :calculate_stats}} + end + @doc "Performs update stats" def force_update do GenServer.call(__MODULE__, :force_update) @@ -29,7 +38,11 @@ defmodule Pleroma.Stats do end @doc "Returns stats data" - @spec get_stats() :: %{domain_count: integer(), status_count: integer(), user_count: integer()} + @spec get_stats() :: %{ + domain_count: non_neg_integer(), + status_count: non_neg_integer(), + user_count: non_neg_integer() + } def get_stats do %{stats: stats} = GenServer.call(__MODULE__, :get_state) @@ -44,25 +57,14 @@ defmodule Pleroma.Stats do peers end - def init(_args) do - {:ok, calculate_stat_data()} - end - - def handle_call(:force_update, _from, _state) do - new_stats = calculate_stat_data() - {:reply, new_stats, new_stats} - end - - def handle_call(:get_state, _from, state) do - {:reply, state, state} - end - - def handle_cast(:run_update, _state) do - new_stats = calculate_stat_data() - - {:noreply, new_stats} - end - + @spec calculate_stat_data() :: %{ + peers: list(), + stats: %{ + domain_count: non_neg_integer(), + status_count: non_neg_integer(), + user_count: non_neg_integer() + } + } def calculate_stat_data do peers = from( @@ -97,20 +99,44 @@ defmodule Pleroma.Stats do } end - def get_status_visibility_count do - counter_cache = - CounterCache.get_as_map([ - "status_visibility_public", - "status_visibility_private", - "status_visibility_unlisted", - "status_visibility_direct" - ]) + @spec get_status_visibility_count(String.t() | nil) :: map() + def get_status_visibility_count(instance \\ nil) do + if is_nil(instance) do + CounterCache.get_sum() + else + CounterCache.get_by_instance(instance) + end + end - %{ - public: counter_cache["status_visibility_public"] || 0, - unlisted: counter_cache["status_visibility_unlisted"] || 0, - private: counter_cache["status_visibility_private"] || 0, - direct: counter_cache["status_visibility_direct"] || 0 - } + @impl true + def handle_continue(:calculate_stats, _) do + stats = calculate_stat_data() + Process.send_after(self(), :run_update, @interval) + {:noreply, stats} + end + + @impl true + def handle_call(:force_update, _from, _state) do + new_stats = calculate_stat_data() + {:reply, new_stats, new_stats} + end + + @impl true + def handle_call(:get_state, _from, state) do + {:reply, state, state} + end + + @impl true + def handle_cast(:run_update, _state) do + new_stats = calculate_stat_data() + + {:noreply, new_stats} + end + + @impl true + def handle_info(:run_update, _) do + new_stats = calculate_stat_data() + Process.send_after(self(), :run_update, @interval) + {:noreply, new_stats} end end diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex new file mode 100644 index 000000000..4cacae02f --- /dev/null +++ b/lib/pleroma/telemetry/logger.ex @@ -0,0 +1,76 @@ +defmodule Pleroma.Telemetry.Logger do + @moduledoc "Transforms Pleroma telemetry events to logs" + + require Logger + + @events [ + [:pleroma, :connection_pool, :reclaim, :start], + [:pleroma, :connection_pool, :reclaim, :stop], + [:pleroma, :connection_pool, :provision_failure], + [:pleroma, :connection_pool, :client_death] + ] + def attach do + :telemetry.attach_many("pleroma-logger", @events, &handle_event/4, []) + end + + # Passing anonymous functions instead of strings to logger is intentional, + # that way strings won't be concatenated if the message is going to be thrown + # out anyway due to higher log level configured + + def handle_event( + [:pleroma, :connection_pool, :reclaim, :start], + _, + %{max_connections: max_connections, reclaim_max: reclaim_max}, + _ + ) do + Logger.debug(fn -> + "Connection pool is exhausted (reached #{max_connections} connections). Starting idle connection cleanup to reclaim as much as #{ + reclaim_max + } connections" + end) + end + + def handle_event( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: 0}, + _, + _ + ) do + Logger.error(fn -> + "Connection pool failed to reclaim any connections due to all of them being in use. It will have to drop requests for opening connections to new hosts" + end) + end + + def handle_event( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: reclaimed_count}, + _, + _ + ) do + Logger.debug(fn -> "Connection pool cleaned up #{reclaimed_count} idle connections" end) + end + + def handle_event( + [:pleroma, :connection_pool, :provision_failure], + %{opts: [key | _]}, + _, + _ + ) do + Logger.error(fn -> + "Connection pool had to refuse opening a connection to #{key} due to connection limit exhaustion" + end) + end + + def handle_event( + [:pleroma, :connection_pool, :client_death], + %{client_pid: client_pid, reason: reason}, + %{key: key}, + _ + ) do + Logger.warn(fn -> + "Pool worker for #{key}: Client #{inspect(client_pid)} died before releasing the connection with #{ + inspect(reason) + }" + end) + end +end diff --git a/lib/pleroma/tesla/middleware/connection_pool.ex b/lib/pleroma/tesla/middleware/connection_pool.ex new file mode 100644 index 000000000..056e736ce --- /dev/null +++ b/lib/pleroma/tesla/middleware/connection_pool.ex @@ -0,0 +1,50 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Tesla.Middleware.ConnectionPool do + @moduledoc """ + Middleware to get/release connections from `Pleroma.Gun.ConnectionPool` + """ + + @behaviour Tesla.Middleware + + alias Pleroma.Gun.ConnectionPool + + @impl Tesla.Middleware + def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do + uri = URI.parse(url) + + # Avoid leaking connections when the middleware is called twice + # with body_as: :chunks. We assume only the middleware can set + # opts[:adapter][:conn] + if opts[:adapter][:conn] do + ConnectionPool.release_conn(opts[:adapter][:conn]) + end + + case ConnectionPool.get_conn(uri, opts[:adapter]) do + {:ok, conn_pid} -> + adapter_opts = Keyword.merge(opts[:adapter], conn: conn_pid, close_conn: false) + opts = Keyword.put(opts, :adapter, adapter_opts) + env = %{env | opts: opts} + + case Tesla.run(env, next) do + {:ok, env} -> + unless opts[:adapter][:body_as] == :chunks do + ConnectionPool.release_conn(conn_pid) + {_, res} = pop_in(env.opts[:adapter][:conn]) + {:ok, res} + else + {:ok, env} + end + + err -> + ConnectionPool.release_conn(conn_pid) + err + end + + err -> + err + end + end +end diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index a0ba2f4c0..db2cc1dae 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -56,6 +56,15 @@ defmodule Pleroma.Upload do } defstruct [:id, :name, :tempfile, :content_type, :path] + defp get_description(opts, upload) do + case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do + {description, _} when is_binary(description) -> description + {_, :filename} -> upload.name + {_, str} when is_binary(str) -> str + _ -> "" + end + end + @spec store(source, options :: [option()]) :: {:ok, Map.t()} | {:error, any()} @doc "Store a file. If using a `Plug.Upload{}` as the source, be sure to use `Majic.Plug` to ensure its content_type and filename is correct." def store(upload, opts \\ []) do @@ -64,6 +73,10 @@ defmodule Pleroma.Upload do with {:ok, upload} <- prepare_upload(upload, opts), upload = %__MODULE__{upload | path: upload.path || "#{upload.id}/#{upload.name}"}, {:ok, upload} <- Pleroma.Upload.Filter.filter(opts.filters, upload), + description = get_description(opts, upload), + {_, true} <- + {:description_limit, + String.length(description) <= Pleroma.Config.get([:instance, :description_limit])}, {:ok, url_spec} <- Pleroma.Uploaders.Uploader.put_file(opts.uploader, upload) do {:ok, %{ @@ -76,9 +89,12 @@ defmodule Pleroma.Upload do "href" => url_from_spec(upload, opts.base_url, url_spec) } ], - "name" => Map.get(opts, :description) || upload.name + "name" => description }} else + {:description_limit, _} -> + {:error, :description_too_long} + {:error, error} -> Logger.error( "#{__MODULE__} store (using #{inspect(opts.uploader)}) failed: #{inspect(error)}" diff --git a/lib/pleroma/upload/filter.ex b/lib/pleroma/upload/filter.ex index dbdadc97f..661135634 100644 --- a/lib/pleroma/upload/filter.ex +++ b/lib/pleroma/upload/filter.ex @@ -15,7 +15,11 @@ defmodule Pleroma.Upload.Filter do require Logger - @callback filter(Pleroma.Upload.t()) :: :ok | {:ok, Pleroma.Upload.t()} | {:error, any()} + @callback filter(Pleroma.Upload.t()) :: + {:ok, :filtered} + | {:ok, :noop} + | {:ok, :filtered, Pleroma.Upload.t()} + | {:error, any()} @spec filter([module()], Pleroma.Upload.t()) :: {:ok, Pleroma.Upload.t()} | {:error, any()} @@ -25,10 +29,13 @@ defmodule Pleroma.Upload.Filter do def filter([filter | rest], upload) do case filter.filter(upload) do - :ok -> + {:ok, :filtered} -> filter(rest, upload) - {:ok, upload} -> + {:ok, :filtered, upload} -> + filter(rest, upload) + + {:ok, :noop} -> filter(rest, upload) error -> diff --git a/lib/pleroma/upload/filter/anonymize_filename.ex b/lib/pleroma/upload/filter/anonymize_filename.ex index 07ead8203..fc456e4f4 100644 --- a/lib/pleroma/upload/filter/anonymize_filename.ex +++ b/lib/pleroma/upload/filter/anonymize_filename.ex @@ -16,9 +16,11 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do def filter(%Upload{name: name} = upload) do extension = List.last(String.split(name, ".")) name = predefined_name(extension) || random(extension) - {:ok, %Upload{upload | name: name}} + {:ok, :filtered, %Upload{upload | name: name}} end + def filter(_), do: {:ok, :noop} + @spec predefined_name(String.t()) :: String.t() | nil defp predefined_name(extension) do with name when not is_nil(name) <- Config.get([__MODULE__, :text]), diff --git a/lib/pleroma/upload/filter/dedupe.ex b/lib/pleroma/upload/filter/dedupe.ex index 41218a918..86cbc8996 100644 --- a/lib/pleroma/upload/filter/dedupe.ex +++ b/lib/pleroma/upload/filter/dedupe.ex @@ -17,8 +17,8 @@ defmodule Pleroma.Upload.Filter.Dedupe do |> Base.encode16(case: :lower) filename = shasum <> "." <> extension - {:ok, %Upload{upload | id: shasum, path: filename}} + {:ok, :filtered, %Upload{upload | id: shasum, path: filename}} end - def filter(_), do: :ok + def filter(_), do: {:ok, :noop} end diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex new file mode 100644 index 000000000..1fd0cfdaa --- /dev/null +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -0,0 +1,30 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.Exiftool do + @moduledoc """ + Strips GPS related EXIF tags and overwrites the file in place. + Also strips or replaces filesystem metadata e.g., timestamps. + """ + @behaviour Pleroma.Upload.Filter + + @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()} + + # webp is not compatible with exiftool at this time + def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop} + + def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do + try do + case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do + {_response, 0} -> {:ok, :filtered} + {error, 1} -> {:error, error} + end + rescue + _e in ErlangError -> + {:error, "exiftool command not found"} + end + end + + def filter(_), do: {:ok, :noop} +end diff --git a/lib/pleroma/upload/filter/mogrifun.ex b/lib/pleroma/upload/filter/mogrifun.ex index 7d95577a4..363e5cf0f 100644 --- a/lib/pleroma/upload/filter/mogrifun.ex +++ b/lib/pleroma/upload/filter/mogrifun.ex @@ -6,6 +6,10 @@ defmodule Pleroma.Upload.Filter.Mogrifun do @behaviour Pleroma.Upload.Filter alias Pleroma.Upload.Filter + @moduledoc """ + This module is just an example of an Upload filter. It's not supposed to be used in production. + """ + @filters [ {"implode", "1"}, {"-raise", "20"}, @@ -34,11 +38,16 @@ defmodule Pleroma.Upload.Filter.Mogrifun do [{"fill", "yellow"}, {"tint", "40"}] ] + @spec filter(Pleroma.Upload.t()) :: {:ok, atom()} | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - Filter.Mogrify.do_filter(file, [Enum.random(@filters)]) - - :ok + try do + Filter.Mogrify.do_filter(file, [Enum.random(@filters)]) + {:ok, :filtered} + rescue + _e in ErlangError -> + {:error, "mogrify command not found"} + end end - def filter(_), do: :ok + def filter(_), do: {:ok, :noop} end diff --git a/lib/pleroma/upload/filter/mogrify.ex b/lib/pleroma/upload/filter/mogrify.ex index 2eb758006..71968fd9c 100644 --- a/lib/pleroma/upload/filter/mogrify.ex +++ b/lib/pleroma/upload/filter/mogrify.ex @@ -8,14 +8,18 @@ defmodule Pleroma.Upload.Filter.Mogrify do @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} @type conversions :: conversion() | [conversion()] + @spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - filters = Pleroma.Config.get!([__MODULE__, :args]) - - do_filter(file, filters) - :ok + try do + do_filter(file, Pleroma.Config.get!([__MODULE__, :args])) + {:ok, :filtered} + rescue + _e in ErlangError -> + {:error, "mogrify command not found"} + end end - def filter(_), do: :ok + def filter(_), do: {:ok, :noop} def do_filter(file, filters) do file diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex index a13ff23b6..6dbef9085 100644 --- a/lib/pleroma/uploaders/s3.ex +++ b/lib/pleroma/uploaders/s3.ex @@ -46,12 +46,23 @@ defmodule Pleroma.Uploaders.S3 do op = if streaming do - upload.tempfile - |> ExAws.S3.Upload.stream_file() - |> ExAws.S3.upload(bucket, s3_name, [ - {:acl, :public_read}, - {:content_type, upload.content_type} - ]) + op = + upload.tempfile + |> ExAws.S3.Upload.stream_file() + |> ExAws.S3.upload(bucket, s3_name, [ + {:acl, :public_read}, + {:content_type, upload.content_type} + ]) + + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + # set s3 upload timeout to respect :upload pool timeout + # timeout should be slightly larger, so s3 can retry upload on fail + timeout = Pleroma.HTTP.AdapterHelper.Gun.pool_timeout(:upload) + 1_000 + opts = Keyword.put(op.opts, :timeout, timeout) + Map.put(op, :opts, opts) + else + op + end else {:ok, file_data} = File.read(upload.tempfile) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 52ac9052b..e73d19964 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -14,6 +14,7 @@ defmodule Pleroma.User do alias Pleroma.Config alias Pleroma.Conversation.Participation alias Pleroma.Delivery + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Emoji alias Pleroma.FollowingRelationship alias Pleroma.Formatter @@ -30,7 +31,6 @@ defmodule Pleroma.User do alias Pleroma.Web alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Builder - alias Pleroma.Web.ActivityPub.ObjectValidators.Types alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI @@ -42,7 +42,12 @@ defmodule Pleroma.User do require Logger @type t :: %__MODULE__{} - @type account_status :: :active | :deactivated | :password_reset_pending | :confirmation_pending + @type account_status :: + :active + | :deactivated + | :password_reset_pending + | :confirmation_pending + | :approval_pending @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true} # credo:disable-for-next-line Credo.Check.Readability.MaxLineLength @@ -78,7 +83,8 @@ defmodule Pleroma.User do ] schema "users" do - field(:bio, :string) + field(:bio, :string, default: "") + field(:raw_bio, :string) field(:email, :string) field(:name, :string) field(:nickname, :string) @@ -88,7 +94,7 @@ defmodule Pleroma.User do field(:keys, :string) field(:public_key, :string) field(:ap_id, :string) - field(:avatar, :map) + field(:avatar, :map, default: %{}) field(:local, :boolean, default: true) field(:follower_address, :string) field(:following_address, :string) @@ -105,6 +111,8 @@ defmodule Pleroma.User do field(:locked, :boolean, default: false) field(:confirmation_pending, :boolean, default: false) field(:password_reset_pending, :boolean, default: false) + field(:approval_pending, :boolean, default: false) + field(:registration_reason, :string, default: nil) field(:confirmation_token, :string, default: nil) field(:default_scope, :string, default: "public") field(:domain_blocks, {:array, :string}, default: []) @@ -114,8 +122,8 @@ defmodule Pleroma.User do field(:is_moderator, :boolean, default: false) field(:is_admin, :boolean, default: false) field(:show_role, :boolean, default: true) - field(:settings, :map, default: nil) - field(:uri, Types.Uri, default: nil) + field(:mastofe_settings, :map, default: nil) + field(:uri, ObjectValidators.Uri, default: nil) field(:hide_followers_count, :boolean, default: false) field(:hide_follows_count, :boolean, default: false) field(:hide_followers, :boolean, default: false) @@ -137,6 +145,7 @@ defmodule Pleroma.User do field(:also_known_as, {:array, :string}, default: []) field(:inbox, :string) field(:shared_inbox, :string) + field(:accepts_chat_messages, :boolean, default: nil) embeds_one( :notification_settings, @@ -238,6 +247,13 @@ defmodule Pleroma.User do end end + defdelegate following_count(user), to: FollowingRelationship + defdelegate following(user), to: FollowingRelationship + defdelegate following?(follower, followed), to: FollowingRelationship + defdelegate following_ap_ids(user), to: FollowingRelationship + defdelegate get_follow_requests(user), to: FollowingRelationship + defdelegate search(query, opts \\ []), to: User.Search + @doc """ Dumps Flake Id to SQL-compatible format (16-byte UUID). E.g. "9pQtDGXuq4p3VlcJEm" -> <<0, 0, 1, 110, 179, 218, 42, 92, 213, 41, 44, 227, 95, 213, 0, 0>> @@ -260,39 +276,65 @@ defmodule Pleroma.User do @spec account_status(User.t()) :: account_status() def account_status(%User{deactivated: true}), do: :deactivated def account_status(%User{password_reset_pending: true}), do: :password_reset_pending + def account_status(%User{approval_pending: true}), do: :approval_pending def account_status(%User{confirmation_pending: true}) do - case Config.get([:instance, :account_activation_required]) do - true -> :confirmation_pending - _ -> :active + if Config.get([:instance, :account_activation_required]) do + :confirmation_pending + else + :active end end def account_status(%User{}), do: :active - @spec visible_for?(User.t(), User.t() | nil) :: boolean() - def visible_for?(user, for_user \\ nil) + @spec visible_for(User.t(), User.t() | nil) :: + :visible + | :invisible + | :restricted_unauthenticated + | :deactivated + | :confirmation_pending + def visible_for(user, for_user \\ nil) - def visible_for?(%User{invisible: true}, _), do: false + def visible_for(%User{invisible: true}, _), do: :invisible - def visible_for?(%User{id: user_id}, %User{id: user_id}), do: true + def visible_for(%User{id: user_id}, %User{id: user_id}), do: :visible - def visible_for?(%User{local: local} = user, nil) do - cfg_key = - if local, - do: :local, - else: :remote + def visible_for(%User{} = user, nil) do + if restrict_unauthenticated?(user) do + :restrict_unauthenticated + else + visible_account_status(user) + end + end - if Config.get([:restrict_unauthenticated, :profiles, cfg_key]), - do: false, - else: account_status(user) == :active + def visible_for(%User{} = user, for_user) do + if superuser?(for_user) do + :visible + else + visible_account_status(user) + end end - def visible_for?(%User{} = user, for_user) do - account_status(user) == :active || superuser?(for_user) + def visible_for(_, _), do: :invisible + + defp restrict_unauthenticated?(%User{local: true}) do + Config.restrict_unauthenticated_access?(:profiles, :local) end - def visible_for?(_, _), do: false + defp restrict_unauthenticated?(%User{local: _}) do + Config.restrict_unauthenticated_access?(:profiles, :remote) + end + + defp visible_account_status(user) do + status = account_status(user) + + if status in [:active, :password_reset_pending] do + :visible + else + status + end + end @spec superuser?(User.t()) :: boolean() def superuser?(%User{local: true, is_admin: true}), do: true @@ -337,8 +379,6 @@ defmodule Pleroma.User do from(u in query, where: u.deactivated != ^true) end - defdelegate following_count(user), to: FollowingRelationship - defp truncate_fields_param(params) do if Map.has_key?(params, :fields) do Map.put(params, :fields, Enum.map(params[:fields], &truncate_field/1)) @@ -364,8 +404,8 @@ defmodule Pleroma.User do defp fix_follower_address(params), do: params def remote_user_changeset(struct \\ %User{local: false}, params) do - bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) - name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) + bio_limit = Config.get([:instance, :user_bio_length], 5000) + name_limit = Config.get([:instance, :user_name_length], 100) name = case params[:name] do @@ -412,7 +452,8 @@ defmodule Pleroma.User do :discoverable, :invisible, :actor_type, - :also_known_as + :also_known_as, + :accepts_chat_messages ] ) |> validate_required([:name, :ap_id]) @@ -424,14 +465,15 @@ defmodule Pleroma.User do end def update_changeset(struct, params \\ %{}) do - bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) - name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) + bio_limit = Config.get([:instance, :user_bio_length], 5000) + name_limit = Config.get([:instance, :user_name_length], 100) struct |> cast( params, [ :bio, + :raw_bio, :name, :emoji, :avatar, @@ -456,13 +498,15 @@ defmodule Pleroma.User do :pleroma_settings_store, :discoverable, :actor_type, - :also_known_as + :also_known_as, + :accepts_chat_messages ] ) |> unique_constraint(:nickname) |> validate_format(:nickname, local_nickname_regex()) |> validate_length(:bio, max: bio_limit) |> validate_length(:name, min: 1, max: name_limit) + |> validate_inclusion(:actor_type, ["Person", "Service"]) |> put_fields() |> put_emoji() |> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)}) @@ -501,11 +545,21 @@ defmodule Pleroma.User do end defp put_emoji(changeset) do - bio = get_change(changeset, :bio) - name = get_change(changeset, :name) + emojified_fields = [:bio, :name, :raw_fields] + + if Enum.any?(changeset.changes, fn {k, _} -> k in emojified_fields end) do + bio = Emoji.Formatter.get_emoji_map(get_field(changeset, :bio)) + name = Emoji.Formatter.get_emoji_map(get_field(changeset, :name)) + + emoji = Map.merge(bio, name) + + emoji = + changeset + |> get_field(:raw_fields) + |> Enum.reduce(emoji, fn x, acc -> + Map.merge(acc, Emoji.Formatter.get_emoji_map(x["name"] <> x["value"])) + end) - if bio || name do - emoji = Map.merge(Emoji.Formatter.get_emoji_map(bio), Emoji.Formatter.get_emoji_map(name)) put_change(changeset, :emoji, emoji) else changeset @@ -513,14 +567,11 @@ defmodule Pleroma.User do end defp put_change_if_present(changeset, map_field, value_function) do - if value = get_change(changeset, map_field) do - with {:ok, new_value} <- value_function.(value) do - put_change(changeset, map_field, new_value) - else - _ -> changeset - end + with {:ok, value} <- fetch_change(changeset, map_field), + {:ok, new_value} <- value_function.(value) do + put_change(changeset, map_field, new_value) else - changeset + _ -> changeset end end @@ -594,29 +645,88 @@ defmodule Pleroma.User do @spec force_password_reset(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} def force_password_reset(user), do: update_password_reset_pending(user, true) + # Used to auto-register LDAP accounts which won't have a password hash stored locally + def register_changeset_ldap(struct, params = %{password: password}) + when is_nil(password) do + params = Map.put_new(params, :accepts_chat_messages, true) + + params = + if Map.has_key?(params, :email) do + Map.put_new(params, :email, params[:email]) + else + params + end + + struct + |> cast(params, [ + :name, + :nickname, + :email, + :accepts_chat_messages + ]) + |> validate_required([:name, :nickname]) + |> unique_constraint(:nickname) + |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames])) + |> validate_format(:nickname, local_nickname_regex()) + |> put_ap_id() + |> unique_constraint(:ap_id) + |> put_following_and_follower_address() + end + def register_changeset(struct, params \\ %{}, opts \\ []) do - bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) - name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) + bio_limit = Config.get([:instance, :user_bio_length], 5000) + name_limit = Config.get([:instance, :user_name_length], 100) + reason_limit = Config.get([:instance, :registration_reason_length], 500) + params = Map.put_new(params, :accepts_chat_messages, true) need_confirmation? = if is_nil(opts[:need_confirmation]) do - Pleroma.Config.get([:instance, :account_activation_required]) + Config.get([:instance, :account_activation_required]) else opts[:need_confirmation] end + need_approval? = + if is_nil(opts[:need_approval]) do + Config.get([:instance, :account_approval_required]) + else + opts[:need_approval] + end + struct |> confirmation_changeset(need_confirmation: need_confirmation?) - |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation, :emoji]) + |> approval_changeset(need_approval: need_approval?) + |> cast(params, [ + :bio, + :raw_bio, + :email, + :name, + :nickname, + :password, + :password_confirmation, + :emoji, + :accepts_chat_messages, + :registration_reason + ]) |> validate_required([:name, :nickname, :password, :password_confirmation]) |> validate_confirmation(:password) |> unique_constraint(:email) + |> validate_format(:email, @email_regex) + |> validate_change(:email, fn :email, email -> + valid? = + Config.get([User, :email_blacklist]) + |> Enum.all?(fn blacklisted_domain -> + !String.ends_with?(email, ["@" <> blacklisted_domain, "." <> blacklisted_domain]) + end) + + if valid?, do: [], else: [email: "Invalid email"] + end) |> unique_constraint(:nickname) - |> validate_exclusion(:nickname, Pleroma.Config.get([User, :restricted_nicknames])) + |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames])) |> validate_format(:nickname, local_nickname_regex()) - |> validate_format(:email, @email_regex) |> validate_length(:bio, max: bio_limit) |> validate_length(:name, min: 1, max: name_limit) + |> validate_length(:registration_reason, max: reason_limit) |> maybe_validate_required_email(opts[:external]) |> put_password_hash |> put_ap_id() @@ -627,7 +737,7 @@ defmodule Pleroma.User do def maybe_validate_required_email(changeset, true), do: changeset def maybe_validate_required_email(changeset, _) do - if Pleroma.Config.get([:instance, :account_activation_required]) do + if Config.get([:instance, :account_activation_required]) do validate_required(changeset, [:email]) else changeset @@ -647,7 +757,7 @@ defmodule Pleroma.User do end defp autofollow_users(user) do - candidates = Pleroma.Config.get([:instance, :autofollowed_nicknames]) + candidates = Config.get([:instance, :autofollowed_nicknames]) autofollowed_users = User.Query.build(%{nickname: candidates, local: true, deactivated: false}) @@ -666,27 +776,62 @@ defmodule Pleroma.User do def post_register_action(%User{} = user) do with {:ok, user} <- autofollow_users(user), {:ok, user} <- set_cache(user), - {:ok, _} <- User.WelcomeMessage.post_welcome_message_to_user(user), + {:ok, _} <- send_welcome_email(user), + {:ok, _} <- send_welcome_message(user), + {:ok, _} <- send_welcome_chat_message(user), {:ok, _} <- try_send_confirmation_email(user) do {:ok, user} end end - def try_send_confirmation_email(%User{} = user) do - if user.confirmation_pending && - Pleroma.Config.get([:instance, :account_activation_required]) do - user - |> Pleroma.Emails.UserEmail.account_confirmation_email() - |> Pleroma.Emails.Mailer.deliver_async() + def send_welcome_message(user) do + if User.WelcomeMessage.enabled?() do + User.WelcomeMessage.post_message(user) + {:ok, :enqueued} + else + {:ok, :noop} + end + end + def send_welcome_chat_message(user) do + if User.WelcomeChatMessage.enabled?() do + User.WelcomeChatMessage.post_message(user) {:ok, :enqueued} else {:ok, :noop} end end - def try_send_confirmation_email(users) do - Enum.each(users, &try_send_confirmation_email/1) + def send_welcome_email(%User{email: email} = user) when is_binary(email) do + if User.WelcomeEmail.enabled?() do + User.WelcomeEmail.send_email(user) + {:ok, :enqueued} + else + {:ok, :noop} + end + end + + def send_welcome_email(_), do: {:ok, :noop} + + @spec try_send_confirmation_email(User.t()) :: {:ok, :enqueued | :noop} + def try_send_confirmation_email(%User{confirmation_pending: true} = user) do + if Config.get([:instance, :account_activation_required]) do + send_confirmation_email(user) + {:ok, :enqueued} + else + {:ok, :noop} + end + end + + def try_send_confirmation_email(_), do: {:ok, :noop} + + @spec send_confirmation_email(Uset.t()) :: User.t() + def send_confirmation_email(%User{} = user) do + user + |> Pleroma.Emails.UserEmail.account_confirmation_email() + |> Pleroma.Emails.Mailer.deliver_async() + + user end def needs_update?(%User{local: true}), do: false @@ -728,10 +873,8 @@ defmodule Pleroma.User do set_cache(follower) end - defdelegate following(user), to: FollowingRelationship - def follow(%User{} = follower, %User{} = followed, state \\ :follow_accept) do - deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked]) + deny_follow_blocked = Config.get([:user, :deny_follow_blocked]) cond do followed.deactivated -> @@ -747,7 +890,6 @@ defmodule Pleroma.User do follower |> update_following_count() - |> set_cache() end end @@ -776,7 +918,6 @@ defmodule Pleroma.User do {:ok, follower} = follower |> update_following_count() - |> set_cache() {:ok, follower, followed} @@ -785,8 +926,6 @@ defmodule Pleroma.User do end end - defdelegate following?(follower, followed), to: FollowingRelationship - @doc "Returns follow state as Pleroma.FollowingRelationship.State value" def get_follow_state(%User{} = follower, %User{} = following) do following_relationship = FollowingRelationship.get(follower, following) @@ -934,7 +1073,7 @@ defmodule Pleroma.User do end def get_cached_by_nickname_or_id(nickname_or_id, opts \\ []) do - restrict_to_local = Pleroma.Config.get([:instance, :limit_to_local_content]) + restrict_to_local = Config.get([:instance, :limit_to_local_content]) cond do is_integer(nickname_or_id) or FlakeId.flake_id?(nickname_or_id) -> @@ -986,31 +1125,31 @@ defmodule Pleroma.User do User.Query.build(%{followers: user, deactivated: false}) end - def get_followers_query(user, page) do + def get_followers_query(%User{} = user, page) do user |> get_followers_query(nil) |> User.Query.paginate(page, 20) end @spec get_followers_query(User.t()) :: Ecto.Query.t() - def get_followers_query(user), do: get_followers_query(user, nil) + def get_followers_query(%User{} = user), do: get_followers_query(user, nil) @spec get_followers(User.t(), pos_integer() | nil) :: {:ok, list(User.t())} - def get_followers(user, page \\ nil) do + def get_followers(%User{} = user, page \\ nil) do user |> get_followers_query(page) |> Repo.all() end @spec get_external_followers(User.t(), pos_integer() | nil) :: {:ok, list(User.t())} - def get_external_followers(user, page \\ nil) do + def get_external_followers(%User{} = user, page \\ nil) do user |> get_followers_query(page) |> User.Query.build(%{external: true}) |> Repo.all() end - def get_followers_ids(user, page \\ nil) do + def get_followers_ids(%User{} = user, page \\ nil) do user |> get_followers_query(page) |> select([u], u.id) @@ -1022,37 +1161,35 @@ defmodule Pleroma.User do User.Query.build(%{friends: user, deactivated: false}) end - def get_friends_query(user, page) do + def get_friends_query(%User{} = user, page) do user |> get_friends_query(nil) |> User.Query.paginate(page, 20) end @spec get_friends_query(User.t()) :: Ecto.Query.t() - def get_friends_query(user), do: get_friends_query(user, nil) + def get_friends_query(%User{} = user), do: get_friends_query(user, nil) - def get_friends(user, page \\ nil) do + def get_friends(%User{} = user, page \\ nil) do user |> get_friends_query(page) |> Repo.all() end - def get_friends_ap_ids(user) do + def get_friends_ap_ids(%User{} = user) do user |> get_friends_query(nil) |> select([u], u.ap_id) |> Repo.all() end - def get_friends_ids(user, page \\ nil) do + def get_friends_ids(%User{} = user, page \\ nil) do user |> get_friends_query(page) |> select([u], u.id) |> Repo.all() end - defdelegate get_follow_requests(user), to: FollowingRelationship - def increase_note_count(%User{} = user) do User |> where(id: ^user.id) @@ -1128,35 +1265,25 @@ defmodule Pleroma.User do ]) end + @spec update_follower_count(User.t()) :: {:ok, User.t()} def update_follower_count(%User{} = user) do - if user.local or !Pleroma.Config.get([:instance, :external_user_synchronization]) do - follower_count_query = - User.Query.build(%{followers: user, deactivated: false}) - |> select([u], %{count: count(u.id)}) - - User - |> where(id: ^user.id) - |> join(:inner, [u], s in subquery(follower_count_query)) - |> update([u, s], - set: [follower_count: s.count] - ) - |> select([u], u) - |> Repo.update_all([]) - |> case do - {1, [user]} -> set_cache(user) - _ -> {:error, user} - end + if user.local or !Config.get([:instance, :external_user_synchronization]) do + follower_count = FollowingRelationship.follower_count(user) + + user + |> follow_information_changeset(%{follower_count: follower_count}) + |> update_and_set_cache else {:ok, maybe_fetch_follow_information(user)} end end - @spec update_following_count(User.t()) :: User.t() + @spec update_following_count(User.t()) :: {:ok, User.t()} def update_following_count(%User{local: false} = user) do - if Pleroma.Config.get([:instance, :external_user_synchronization]) do - maybe_fetch_follow_information(user) + if Config.get([:instance, :external_user_synchronization]) do + {:ok, maybe_fetch_follow_information(user)} else - user + {:ok, user} end end @@ -1165,7 +1292,7 @@ defmodule Pleroma.User do user |> follow_information_changeset(%{following_count: following_count}) - |> Repo.update!() + |> update_and_set_cache() end def set_unread_conversation_count(%User{local: true} = user) do @@ -1240,7 +1367,7 @@ defmodule Pleroma.User do end def subscribe(%User{} = subscriber, %User{} = target) do - deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked]) + deny_follow_blocked = Config.get([:user, :deny_follow_blocked]) if blocks?(target, subscriber) and deny_follow_blocked do {:error, "Could not subscribe: #{target.nickname} is blocking you"} @@ -1286,7 +1413,8 @@ defmodule Pleroma.User do unsubscribe(blocked, blocker) - if following?(blocked, blocker), do: unfollow(blocked, blocker) + unfollowing_blocked = Config.get([:activitypub, :unfollow_blocked], true) + if unfollowing_blocked && following?(blocked, blocker), do: unfollow(blocked, blocker) {:ok, blocker} = update_follower_count(blocker) {:ok, blocker, _} = Participation.mark_all_as_read(blocker, blocked) @@ -1433,6 +1561,19 @@ defmodule Pleroma.User do end end + def approve(users) when is_list(users) do + Repo.transaction(fn -> + Enum.map(users, fn user -> + with {:ok, user} <- approve(user), do: user + end) + end) + end + + def approve(%User{} = user) do + change(user, approval_pending: false) + |> update_and_set_cache() + end + def update_notification_settings(%User{} = user, settings) do user |> cast(%{notification_settings: settings}, []) @@ -1441,6 +1582,49 @@ defmodule Pleroma.User do |> update_and_set_cache() end + @spec purge_user_changeset(User.t()) :: Changeset.t() + def purge_user_changeset(user) do + # "Right to be forgotten" + # https://gdpr.eu/right-to-be-forgotten/ + change(user, %{ + bio: "", + raw_bio: nil, + email: nil, + name: nil, + password_hash: nil, + keys: nil, + public_key: nil, + avatar: %{}, + tags: [], + last_refreshed_at: nil, + last_digest_emailed_at: nil, + banner: %{}, + background: %{}, + note_count: 0, + follower_count: 0, + following_count: 0, + locked: false, + confirmation_pending: false, + password_reset_pending: false, + approval_pending: false, + registration_reason: nil, + confirmation_token: nil, + domain_blocks: [], + deactivated: true, + ap_enabled: false, + is_moderator: false, + is_admin: false, + mastofe_settings: nil, + mascot: nil, + emoji: %{}, + pleroma_settings_store: %{}, + fields: [], + raw_fields: [], + discoverable: false, + also_known_as: [] + }) + end + def delete(users) when is_list(users) do for user <- users, do: delete(user) end @@ -1459,12 +1643,17 @@ defmodule Pleroma.User do defp delete_or_deactivate(%User{local: true} = user) do status = account_status(user) - if status == :confirmation_pending do - delete_and_invalidate_cache(user) - else - user - |> change(%{deactivated: true, email: nil}) - |> update_and_set_cache() + case status do + :confirmation_pending -> + delete_and_invalidate_cache(user) + + :approval_pending -> + delete_and_invalidate_cache(user) + + _ -> + user + |> purge_user_changeset() + |> update_and_set_cache() end end @@ -1504,8 +1693,7 @@ defmodule Pleroma.User do blocked_identifiers, fn blocked_identifier -> with {:ok, %User{} = blocked} <- get_or_fetch(blocked_identifier), - {:ok, _user_block} <- block(blocker, blocked), - {:ok, _} <- ActivityPub.block(blocker, blocked) do + {:ok, _block} <- CommonAPI.block(blocker, blocked) do blocked else err -> @@ -1523,7 +1711,7 @@ defmodule Pleroma.User do fn followed_identifier -> with {:ok, %User{} = followed} <- get_or_fetch(followed_identifier), {:ok, follower} <- maybe_direct_follow(follower, followed), - {:ok, _} <- ActivityPub.follow(follower, followed) do + {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do followed else err -> @@ -1631,7 +1819,7 @@ defmodule Pleroma.User do Pleroma.HTML.Scrubber.TwitterText end - def html_filter_policy(_), do: Pleroma.Config.get([:markup, :scrub_policy]) + def html_filter_policy(_), do: Config.get([:markup, :scrub_policy]) def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id) @@ -1813,7 +2001,7 @@ defmodule Pleroma.User do end defp local_nickname_regex do - if Pleroma.Config.get([:instance, :extended_nickname_format]) do + if Config.get([:instance, :extended_nickname_format]) do @extended_local_nickname_regex else @strict_local_nickname_regex @@ -1941,8 +2129,8 @@ defmodule Pleroma.User do def get_mascot(%{mascot: mascot}) when is_nil(mascot) do # use instance-default - config = Pleroma.Config.get([:assets, :mascots]) - default_mascot = Pleroma.Config.get([:assets, :default_mascot]) + config = Config.get([:assets, :mascots]) + default_mascot = Config.get([:assets, :default_mascot]) mascot = Keyword.get(config, default_mascot) %{ @@ -1974,8 +2162,6 @@ defmodule Pleroma.User do |> Repo.all() end - defdelegate search(query, opts \\ []), to: User.Search - defp put_password_hash( %Ecto.Changeset{valid?: true, changes: %{password: password}} = changeset ) do @@ -2037,7 +2223,7 @@ defmodule Pleroma.User do def validate_fields(changeset, remote? \\ false) do limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields - limit = Pleroma.Config.get([:instance, limit_name], 0) + limit = Config.get([:instance, limit_name], 0) changeset |> validate_length(:fields, max: limit) @@ -2051,8 +2237,8 @@ defmodule Pleroma.User do end defp valid_field?(%{"name" => name, "value" => value}) do - name_limit = Pleroma.Config.get([:instance, :account_field_name_length], 255) - value_limit = Pleroma.Config.get([:instance, :account_field_value_length], 255) + name_limit = Config.get([:instance, :account_field_name_length], 255) + value_limit = Config.get([:instance, :account_field_value_length], 255) is_binary(name) && is_binary(value) && String.length(name) <= name_limit && String.length(value) <= value_limit @@ -2062,10 +2248,10 @@ defmodule Pleroma.User do defp truncate_field(%{"name" => name, "value" => value}) do {name, _chopped} = - String.split_at(name, Pleroma.Config.get([:instance, :account_field_name_length], 255)) + String.split_at(name, Config.get([:instance, :account_field_name_length], 255)) {value, _chopped} = - String.split_at(value, Pleroma.Config.get([:instance, :account_field_value_length], 255)) + String.split_at(value, Config.get([:instance, :account_field_value_length], 255)) %{"name" => name, "value" => value} end @@ -2095,8 +2281,8 @@ defmodule Pleroma.User do def mastodon_settings_update(user, settings) do user - |> cast(%{settings: settings}, [:settings]) - |> validate_required([:settings]) + |> cast(%{mastofe_settings: settings}, [:mastofe_settings]) + |> validate_required([:mastofe_settings]) |> update_and_set_cache() end @@ -2118,11 +2304,22 @@ defmodule Pleroma.User do cast(user, params, [:confirmation_pending, :confirmation_token]) end + @spec approval_changeset(User.t(), keyword()) :: Changeset.t() + def approval_changeset(user, need_approval: need_approval?) do + params = if need_approval?, do: %{approval_pending: true}, else: %{approval_pending: false} + cast(user, params, [:approval_pending]) + end + def add_pinnned_activity(user, %Pleroma.Activity{id: id}) do if id not in user.pinned_activities do - max_pinned_statuses = Pleroma.Config.get([:instance, :max_pinned_statuses], 0) + max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0) params = %{pinned_activities: user.pinned_activities ++ [id]} + # if pinned activity was scheduled for deletion, we remove job + if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(id) do + Oban.cancel_job(expiration.id) + end + user |> cast(params, [:pinned_activities]) |> validate_length(:pinned_activities, @@ -2135,9 +2332,19 @@ defmodule Pleroma.User do |> update_and_set_cache() end - def remove_pinnned_activity(user, %Pleroma.Activity{id: id}) do + def remove_pinnned_activity(user, %Pleroma.Activity{id: id, data: data}) do params = %{pinned_activities: List.delete(user.pinned_activities, id)} + # if pinned activity was scheduled for deletion, we reschedule it for deletion + if data["expires_at"] do + {:ok, expires_at, _} = DateTime.from_iso8601(data["expires_at"]) + + Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ + activity_id: id, + expires_at: expires_at + }) + end + user |> cast(params, [:pinned_activities]) |> update_and_set_cache() diff --git a/lib/pleroma/user/notification_setting.ex b/lib/pleroma/user/notification_setting.ex index 4bd55e139..7d9e8a000 100644 --- a/lib/pleroma/user/notification_setting.ex +++ b/lib/pleroma/user/notification_setting.ex @@ -10,21 +10,15 @@ defmodule Pleroma.User.NotificationSetting do @primary_key false embedded_schema do - field(:followers, :boolean, default: true) - field(:follows, :boolean, default: true) - field(:non_follows, :boolean, default: true) - field(:non_followers, :boolean, default: true) - field(:privacy_option, :boolean, default: false) + field(:block_from_strangers, :boolean, default: false) + field(:hide_notification_contents, :boolean, default: false) end def changeset(schema, params) do schema |> cast(prepare_attrs(params), [ - :followers, - :follows, - :non_follows, - :non_followers, - :privacy_option + :block_from_strangers, + :hide_notification_contents ]) end diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index 66ffe9090..d618432ff 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -42,6 +42,7 @@ defmodule Pleroma.User.Query do external: boolean(), active: boolean(), deactivated: boolean(), + need_approval: boolean(), is_admin: boolean(), is_moderator: boolean(), super_users: boolean(), @@ -129,6 +130,7 @@ defmodule Pleroma.User.Query do defp compose_query({:active, _}, query) do User.restrict_deactivated(query) |> where([u], not is_nil(u.nickname)) + |> where([u], u.approval_pending == false) end defp compose_query({:legacy_active, _}, query) do @@ -146,6 +148,10 @@ defmodule Pleroma.User.Query do |> where([u], not is_nil(u.nickname)) end + defp compose_query({:need_approval, _}, query) do + where(query, [u], u.approval_pending) + end + defp compose_query({:followers, %User{id: id}}, query) do query |> where([u], u.id != ^id) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index cec59c372..7babd47ea 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -52,6 +52,7 @@ defmodule Pleroma.User.Search do |> base_query(following) |> filter_blocked_user(for_user) |> filter_invisible_users() + |> filter_internal_users() |> filter_blocked_domains(for_user) |> fts_search(query_string) |> trigram_rank(query_string) @@ -68,11 +69,15 @@ defmodule Pleroma.User.Search do u in query, where: fragment( + # The fragment must _exactly_ match `users_fts_index`, otherwise the index won't work """ - (to_tsvector('simple', ?) || to_tsvector('simple', ?)) @@ to_tsquery('simple', ?) + ( + setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') || + setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B') + ) @@ to_tsquery('simple', ?) """, - u.name, u.nickname, + u.name, ^query_string ) ) @@ -87,28 +92,40 @@ defmodule Pleroma.User.Search do |> Enum.join(" | ") end + # Considers nickname match, localized nickname match, name match; preferences nickname match defp trigram_rank(query, query_string) do from( u in query, select_merge: %{ search_rank: fragment( - "similarity(?, trim(? || ' ' || coalesce(?, '')))", + """ + similarity(?, ?) + + similarity(?, regexp_replace(?, '@.+', '')) + + similarity(?, trim(coalesce(?, ''))) + """, ^query_string, u.nickname, + ^query_string, + u.nickname, + ^query_string, u.name ) } ) end - defp base_query(_user, false), do: User - defp base_query(user, true), do: User.get_followers_query(user) + defp base_query(%User{} = user, true), do: User.get_friends_query(user) + defp base_query(_user, _following), do: User defp filter_invisible_users(query) do from(q in query, where: q.invisible == false) end + defp filter_internal_users(query) do + from(q in query, where: q.actor_type != "Application") + end + defp filter_blocked_user(query, %User{} = blocker) do query |> join(:left, [u], b in Pleroma.UserRelationship, diff --git a/lib/pleroma/user/welcome_chat_message.ex b/lib/pleroma/user/welcome_chat_message.ex new file mode 100644 index 000000000..3e7d1f424 --- /dev/null +++ b/lib/pleroma/user/welcome_chat_message.ex @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.User.WelcomeChatMessage do + alias Pleroma.Config + alias Pleroma.User + alias Pleroma.Web.CommonAPI + + @spec enabled?() :: boolean() + def enabled?, do: Config.get([:welcome, :chat_message, :enabled], false) + + @spec post_message(User.t()) :: {:ok, Pleroma.Activity.t() | nil} + def post_message(user) do + [:welcome, :chat_message, :sender_nickname] + |> Config.get(nil) + |> fetch_sender() + |> do_post(user, welcome_message()) + end + + defp do_post(%User{} = sender, recipient, message) + when is_binary(message) do + CommonAPI.post_chat_message( + sender, + recipient, + message + ) + end + + defp do_post(_sender, _recipient, _message), do: {:ok, nil} + + defp fetch_sender(nickname) when is_binary(nickname) do + with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do + user + else + _ -> nil + end + end + + defp fetch_sender(_), do: nil + + defp welcome_message do + Config.get([:welcome, :chat_message, :message], nil) + end +end diff --git a/lib/pleroma/user/welcome_email.ex b/lib/pleroma/user/welcome_email.ex new file mode 100644 index 000000000..5322000d4 --- /dev/null +++ b/lib/pleroma/user/welcome_email.ex @@ -0,0 +1,62 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.User.WelcomeEmail do + @moduledoc """ + The module represents the functions to send welcome email. + """ + + alias Pleroma.Config + alias Pleroma.Emails + alias Pleroma.User + + import Pleroma.Config.Helpers, only: [instance_name: 0] + + @spec enabled?() :: boolean() + def enabled?, do: Config.get([:welcome, :email, :enabled], false) + + @spec send_email(User.t()) :: {:ok, Oban.Job.t()} + def send_email(%User{} = user) do + user + |> Emails.UserEmail.welcome(email_options(user)) + |> Emails.Mailer.deliver_async() + end + + defp email_options(user) do + bindings = [user: user, instance_name: instance_name()] + + %{} + |> add_sender(Config.get([:welcome, :email, :sender], nil)) + |> add_option(:subject, bindings) + |> add_option(:html, bindings) + |> add_option(:text, bindings) + end + + defp add_option(opts, option, bindings) do + [:welcome, :email, option] + |> Config.get(nil) + |> eval_string(bindings) + |> merge_options(opts, option) + end + + defp add_sender(opts, {_name, _email} = sender) do + merge_options(sender, opts, :sender) + end + + defp add_sender(opts, sender) when is_binary(sender) do + add_sender(opts, {instance_name(), sender}) + end + + defp add_sender(opts, _), do: opts + + defp merge_options(nil, options, _option), do: options + + defp merge_options(value, options, option) do + Map.merge(options, %{option => value}) + end + + defp eval_string(nil, _), do: nil + defp eval_string("", _), do: nil + defp eval_string(str, bindings), do: EEx.eval_string(str, bindings) +end diff --git a/lib/pleroma/user/welcome_message.ex b/lib/pleroma/user/welcome_message.ex index f8f520285..86e1c0678 100644 --- a/lib/pleroma/user/welcome_message.ex +++ b/lib/pleroma/user/welcome_message.ex @@ -3,32 +3,45 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.User.WelcomeMessage do + alias Pleroma.Config alias Pleroma.User alias Pleroma.Web.CommonAPI - def post_welcome_message_to_user(user) do - with %User{} = sender_user <- welcome_user(), - message when is_binary(message) <- welcome_message() do - CommonAPI.post(sender_user, %{ + @spec enabled?() :: boolean() + def enabled?, do: Config.get([:welcome, :direct_message, :enabled], false) + + @spec post_message(User.t()) :: {:ok, Pleroma.Activity.t() | nil} + def post_message(user) do + [:welcome, :direct_message, :sender_nickname] + |> Config.get(nil) + |> fetch_sender() + |> do_post(user, welcome_message()) + end + + defp do_post(%User{} = sender, %User{nickname: nickname}, message) + when is_binary(message) do + CommonAPI.post( + sender, + %{ visibility: "direct", - status: "@#{user.nickname}\n#{message}" - }) - else - _ -> {:ok, nil} - end + status: "@#{nickname}\n#{message}" + } + ) end - defp welcome_user do - with nickname when is_binary(nickname) <- - Pleroma.Config.get([:instance, :welcome_user_nickname]), - %User{local: true} = user <- User.get_cached_by_nickname(nickname) do + defp do_post(_sender, _recipient, _message), do: {:ok, nil} + + defp fetch_sender(nickname) when is_binary(nickname) do + with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do user else _ -> nil end end + defp fetch_sender(_), do: nil + defp welcome_message do - Pleroma.Config.get([:instance, :welcome_message]) + Config.get([:welcome, :direct_message, :message], nil) end end diff --git a/lib/pleroma/utils.ex b/lib/pleroma/utils.ex index 6b8e3accf..21d1159be 100644 --- a/lib/pleroma/utils.ex +++ b/lib/pleroma/utils.ex @@ -9,4 +9,19 @@ defmodule Pleroma.Utils do |> Enum.map(&Path.join(dir, &1)) |> Kernel.ParallelCompiler.compile() end + + @doc """ + POSIX-compliant check if command is available in the system + + ## Examples + iex> command_available?("git") + true + iex> command_available?("wrongcmd") + false + + """ + @spec command_available?(String.t()) :: boolean() + def command_available?(command) do + match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"])) + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c9dc6135c..66a9f78a3 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -5,11 +5,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Activity alias Pleroma.Activity.Ir.Topics - alias Pleroma.ActivityExpiration alias Pleroma.Config alias Pleroma.Constants alias Pleroma.Conversation alias Pleroma.Conversation.Participation + alias Pleroma.Filter alias Pleroma.Maps alias Pleroma.Notification alias Pleroma.Object @@ -65,7 +65,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp check_remote_limit(_), do: true - defp increase_note_count_if_public(actor, object) do + def increase_note_count_if_public(actor, object) do if is_public?(object), do: User.increase_note_count(actor), else: {:ok, actor} end @@ -84,17 +84,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp increase_replies_count_if_reply(_create_data), do: :noop - defp increase_poll_votes_if_vote(%{ - "object" => %{"inReplyTo" => reply_ap_id, "name" => name}, - "type" => "Create", - "actor" => actor - }) do - Object.increase_vote_count(reply_ap_id, name, actor) - end - - defp increase_poll_votes_if_vote(_create_data), do: :noop - - @object_types ["ChatMessage"] + @object_types ~w[ChatMessage Question Answer Audio Event] @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} def persist(%{"type" => type} = object, meta) when type in @object_types do with {:ok, object} <- Object.create(object) do @@ -111,7 +101,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do local: local, recipients: recipients, actor: object["actor"] - }) do + }), + # TODO: add tests for expired activities, when Note type will be supported in new pipeline + {:ok, _} <- maybe_create_activity_expiration(activity) do {:ok, activity, meta} end end @@ -120,23 +112,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do with nil <- Activity.normalize(map), map <- lazy_put_activity_defaults(map, fake), - true <- bypass_actor_check || check_actor_is_active(map["actor"]), - {_, true} <- {:remote_limit_error, check_remote_limit(map)}, + {_, true} <- {:actor_check, bypass_actor_check || check_actor_is_active(map["actor"])}, + {_, true} <- {:remote_limit_pass, check_remote_limit(map)}, {:ok, map} <- MRF.filter(map), {recipients, _, _} = get_recipients(map), {:fake, false, map, recipients} <- {:fake, fake, map, recipients}, {:containment, :ok} <- {:containment, Containment.contain_child(map)}, - {:ok, map, object} <- insert_full_object(map) do - {:ok, activity} = - %Activity{ - data: map, - local: local, - actor: map["actor"], - recipients: recipients - } - |> Repo.insert() - |> maybe_create_activity_expiration() - + {:ok, map, object} <- insert_full_object(map), + {:ok, activity} <- insert_activity_with_expiration(map, local, recipients) do # Splice in the child object if we have one. activity = Maps.put_if_present(activity, :object, object) @@ -147,6 +130,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do %Activity{} = activity -> {:ok, activity} + {:actor_check, _} -> + {:error, false} + + {:containment, _} = error -> + error + + {:error, _} = error -> + error + {:fake, true, map, recipients} -> activity = %Activity{ data: map, @@ -159,8 +151,24 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) {:ok, activity} - error -> - {:error, error} + {:remote_limit_pass, _} -> + {:error, :remote_limit} + + {:reject, reason} -> + {:error, reason} + end + end + + defp insert_activity_with_expiration(data, local, recipients) do + struct = %Activity{ + data: data, + local: local, + actor: data["actor"], + recipients: recipients + } + + with {:ok, activity} <- Repo.insert(struct) do + maybe_create_activity_expiration(activity) end end @@ -173,13 +181,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do stream_out_participations(participations) end - defp maybe_create_activity_expiration({:ok, %{data: %{"expires_at" => expires_at}} = activity}) do - with {:ok, _} <- ActivityExpiration.create(activity, expires_at) do + defp maybe_create_activity_expiration( + %{data: %{"expires_at" => %DateTime{} = expires_at}} = activity + ) do + with {:ok, _job} <- + Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ + activity_id: activity.id, + expires_at: expires_at + }) do {:ok, activity} end end - defp maybe_create_activity_expiration(result), do: result + defp maybe_create_activity_expiration(activity), do: {:ok, activity} defp create_or_bump_conversation(activity, actor) do with {:ok, conversation} <- Conversation.create_or_bump_for(activity), @@ -210,7 +224,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do conversation = Repo.preload(conversation, :participations) last_activity_id = - fetch_latest_activity_id_for_context(conversation.ap_id, %{ + fetch_latest_direct_activity_id_for_context(conversation.ap_id, %{ user: user, blocking_user: user }) @@ -257,7 +271,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do with {:ok, activity} <- insert(create_data, local, fake), {:fake, false, activity} <- {:fake, fake, activity}, _ <- increase_replies_count_if_reply(create_data), - _ <- increase_poll_votes_if_vote(create_data), {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity}, {:ok, _actor} <- increase_note_count_if_public(actor, activity), _ <- notify_and_stream(activity), @@ -295,76 +308,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - @spec accept(map()) :: {:ok, Activity.t()} | {:error, any()} - def accept(params) do - accept_or_reject("Accept", params) - end - - @spec reject(map()) :: {:ok, Activity.t()} | {:error, any()} - def reject(params) do - accept_or_reject("Reject", params) - end - - @spec accept_or_reject(String.t(), map()) :: {:ok, Activity.t()} | {:error, any()} - defp accept_or_reject(type, %{to: to, actor: actor, object: object} = params) do - local = Map.get(params, :local, true) - activity_id = Map.get(params, :activity_id, nil) - - data = - %{"to" => to, "type" => type, "actor" => actor.ap_id, "object" => object} - |> Maps.put_if_present("id", activity_id) - - with {:ok, activity} <- insert(data, local), - _ <- notify_and_stream(activity), - :ok <- maybe_federate(activity) do - {:ok, activity} - end - end - - @spec update(map()) :: {:ok, Activity.t()} | {:error, any()} - def update(%{to: to, cc: cc, actor: actor, object: object} = params) do - local = !(params[:local] == false) - activity_id = params[:activity_id] - - data = - %{ - "to" => to, - "cc" => cc, - "type" => "Update", - "actor" => actor, - "object" => object - } - |> Maps.put_if_present("id", activity_id) - - with {:ok, activity} <- insert(data, local), - _ <- notify_and_stream(activity), - :ok <- maybe_federate(activity) do - {:ok, activity} - end - end - - @spec follow(User.t(), User.t(), String.t() | nil, boolean(), keyword()) :: - {:ok, Activity.t()} | {:error, any()} - def follow(follower, followed, activity_id \\ nil, local \\ true, opts \\ []) do - with {:ok, result} <- - Repo.transaction(fn -> do_follow(follower, followed, activity_id, local, opts) end) do - result - end - end - - defp do_follow(follower, followed, activity_id, local, opts) do - skip_notify_and_stream = Keyword.get(opts, :skip_notify_and_stream, false) - data = make_follow_data(follower, followed, activity_id) - - with {:ok, activity} <- insert(data, local), - _ <- skip_notify_and_stream || notify_and_stream(activity), - :ok <- maybe_federate(activity) do - {:ok, activity} - else - {:error, error} -> Repo.rollback(error) - end - end - @spec unfollow(User.t(), User.t(), String.t() | nil, boolean()) :: {:ok, Activity.t()} | nil | {:error, any()} def unfollow(follower, followed, activity_id \\ nil, local \\ true) do @@ -388,33 +331,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - @spec block(User.t(), User.t(), String.t() | nil, boolean()) :: - {:ok, Activity.t()} | {:error, any()} - def block(blocker, blocked, activity_id \\ nil, local \\ true) do - with {:ok, result} <- - Repo.transaction(fn -> do_block(blocker, blocked, activity_id, local) end) do - result - end - end - - defp do_block(blocker, blocked, activity_id, local) do - unfollow_blocked = Config.get([:activitypub, :unfollow_blocked]) - - if unfollow_blocked and fetch_latest_follow(blocker, blocked) do - unfollow(blocker, blocked, nil, local) - end - - block_data = make_block_data(blocker, blocked, activity_id) - - with {:ok, activity} <- insert(block_data, local), - _ <- notify_and_stream(activity), - :ok <- maybe_federate(activity) do - {:ok, activity} - else - {:error, error} -> Repo.rollback(error) - end - end - @spec flag(map()) :: {:ok, Activity.t()} | {:error, any()} def flag( %{ @@ -495,6 +411,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> maybe_set_thread_muted_field(opts) |> restrict_blocked(opts) |> restrict_recipients(recipients, opts[:user]) + |> restrict_filtered(opts) |> where( [activity], fragment( @@ -517,11 +434,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Repo.all() end - @spec fetch_latest_activity_id_for_context(String.t(), keyword() | map()) :: + @spec fetch_latest_direct_activity_id_for_context(String.t(), keyword() | map()) :: FlakeId.Ecto.CompatType.t() | nil - def fetch_latest_activity_id_for_context(context, opts \\ %{}) do + def fetch_latest_direct_activity_id_for_context(context, opts \\ %{}) do context |> fetch_activities_for_context_query(Map.merge(%{skip_preload: true}, opts)) + |> restrict_visibility(%{visibility: "direct"}) |> limit(1) |> select([a], a.id) |> Repo.one() @@ -833,7 +751,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_media(query, %{only_media: true}) do from( - [_activity, object] in query, + [activity, object] in query, + where: fragment("(?)->>'type' = ?", activity.data, "Create"), where: fragment("not (?)->'attachment' = (?)", object.data, ^[]) ) end @@ -1008,6 +927,26 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_instance(query, _), do: query + defp restrict_filtered(query, %{user: %User{} = user}) do + case Filter.compose_regex(user) do + nil -> + query + + regex -> + from([activity, object] in query, + where: + fragment("not(?->>'content' ~* ?)", object.data, ^regex) or + activity.actor == ^user.ap_id + ) + end + end + + defp restrict_filtered(query, %{blocking_user: %User{} = user}) do + restrict_filtered(query, %{user: user}) + end + + defp restrict_filtered(query, _), do: query + defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query defp exclude_poll_votes(query, _) do @@ -1138,6 +1077,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_favorited_by(opts) |> restrict_blocked(restrict_blocked_opts) |> restrict_muted(restrict_muted_opts) + |> restrict_filtered(opts) |> restrict_media(opts) |> restrict_visibility(opts) |> restrict_thread_visibility(opts, config) @@ -1146,6 +1086,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_muted_reblogs(restrict_muted_reblogs_opts) |> restrict_instance(opts) |> restrict_announce_object_actor(opts) + |> restrict_filtered(opts) |> Activity.restrict_deactivated_users() |> exclude_poll_votes(opts) |> exclude_chat_messages(opts) @@ -1271,6 +1212,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) locked = data["manuallyApprovesFollowers"] || false + capabilities = data["capabilities"] || %{} + accepts_chat_messages = capabilities["acceptsChatMessages"] data = Transmogrifier.maybe_fix_user_object(data) discoverable = data["discoverable"] || false invisible = data["invisible"] || false @@ -1304,12 +1247,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do name: data["name"], follower_address: data["followers"], following_address: data["following"], - bio: data["summary"], + bio: data["summary"] || "", actor_type: actor_type, also_known_as: Map.get(data, "alsoKnownAs", []), public_key: public_key, inbox: data["inbox"], - shared_inbox: shared_inbox + shared_inbox: shared_inbox, + accepts_chat_messages: accepts_chat_messages } # nickname can be nil because of virtual actors @@ -1412,12 +1356,40 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}") {:error, e} + {:error, {:reject, reason} = e} -> + Logger.info("Rejected user #{ap_id}: #{inspect(reason)}") + {:error, e} + {:error, e} -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") {:error, e} end end + def maybe_handle_clashing_nickname(data) do + with nickname when is_binary(nickname) <- data[:nickname], + %User{} = old_user <- User.get_by_nickname(nickname), + {_, false} <- {:ap_id_comparison, data[:ap_id] == old_user.ap_id} do + Logger.info( + "Found an old user for #{nickname}, the old ap id is #{old_user.ap_id}, new one is #{ + data[:ap_id] + }, renaming." + ) + + old_user + |> User.remote_user_changeset(%{nickname: "#{old_user.id}.#{old_user.nickname}"}) + |> User.update_and_set_cache() + else + {:ap_id_comparison, true} -> + Logger.info( + "Found an old user for #{data[:nickname]}, but the ap id #{data[:ap_id]} is the same as the new user. Race condition? Not changing anything." + ) + + _ -> + nil + end + end + def make_user_from_ap_id(ap_id) do user = User.get_cached_by_ap_id(ap_id) @@ -1430,6 +1402,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> User.remote_user_changeset(data) |> User.update_and_set_cache() else + maybe_handle_clashing_nickname(data) + data |> User.remote_user_changeset() |> Repo.insert() diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index e2a5fb9e9..27bf6347d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -401,21 +401,30 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do defp handle_user_activity( %User{} = user, - %{"type" => "Create", "object" => %{"type" => "Note"}} = params + %{"type" => "Create", "object" => %{"type" => "Note"} = object} = params ) do - object = - params["object"] - |> Map.merge(Map.take(params, ["to", "cc"])) - |> Map.put("attributedTo", user.ap_id()) - |> Transmogrifier.fix_object() - - ActivityPub.create(%{ - to: params["to"], - actor: user, - context: object["context"], - object: object, - additional: Map.take(params, ["cc"]) - }) + content = if is_binary(object["content"]), do: object["content"], else: "" + name = if is_binary(object["name"]), do: object["name"], else: "" + summary = if is_binary(object["summary"]), do: object["summary"], else: "" + length = String.length(content <> name <> summary) + + if length > Pleroma.Config.get([:instance, :limit]) do + {:error, dgettext("errors", "Note is over the character limit")} + else + object = + object + |> Map.merge(Map.take(params, ["to", "cc"])) + |> Map.put("attributedTo", user.ap_id()) + |> Transmogrifier.fix_object() + + ActivityPub.create(%{ + to: params["to"], + actor: user, + context: object["context"], + object: object, + additional: Map.take(params, ["cc"]) + }) + end end defp handle_user_activity(%User{} = user, %{"type" => "Delete"} = params) do @@ -516,7 +525,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do {new_user, for_user} end - # TODO: Add support for "object" field @doc """ Endpoint based on <https://www.w3.org/wiki/SocialCG/ActivityPub/MediaUpload> @@ -527,6 +535,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do Response: - HTTP Code: 201 Created - HTTP Body: ActivityPub object to be inserted into another's `attachment` field + + Note: Will not point to a URL with a `Location` header because no standalone Activity has been created. """ def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do with {:ok, object} <- diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index 1aac62c69..9a7b7d9de 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -14,6 +14,41 @@ defmodule Pleroma.Web.ActivityPub.Builder do require Pleroma.Constants + def accept_or_reject(actor, activity, type) do + data = %{ + "id" => Utils.generate_activity_id(), + "actor" => actor.ap_id, + "type" => type, + "object" => activity.data["id"], + "to" => [activity.actor] + } + + {:ok, data, []} + end + + @spec reject(User.t(), Activity.t()) :: {:ok, map(), keyword()} + def reject(actor, rejected_activity) do + accept_or_reject(actor, rejected_activity, "Reject") + end + + @spec accept(User.t(), Activity.t()) :: {:ok, map(), keyword()} + def accept(actor, accepted_activity) do + accept_or_reject(actor, accepted_activity, "Accept") + end + + @spec follow(User.t(), User.t()) :: {:ok, map(), keyword()} + def follow(follower, followed) do + data = %{ + "id" => Utils.generate_activity_id(), + "actor" => follower.ap_id, + "type" => "Follow", + "object" => followed.ap_id, + "to" => [followed.ap_id] + } + + {:ok, data, []} + end + @spec emoji_react(User.t(), Object.t(), String.t()) :: {:ok, map(), keyword()} def emoji_react(actor, object, emoji) do with {:ok, data, meta} <- object_action(actor, object) do @@ -67,6 +102,13 @@ defmodule Pleroma.Web.ActivityPub.Builder do end def create(actor, object, recipients) do + context = + if is_map(object) do + object["context"] + else + nil + end + {:ok, %{ "id" => Utils.generate_activity_id(), @@ -75,7 +117,8 @@ defmodule Pleroma.Web.ActivityPub.Builder do "object" => object, "type" => "Create", "published" => DateTime.utc_now() |> DateTime.to_iso8601() - }, []} + } + |> Pleroma.Maps.put_if_present("context", context), []} end def chat_message(actor, recipient, content, opts \\ []) do @@ -102,6 +145,22 @@ defmodule Pleroma.Web.ActivityPub.Builder do end end + def answer(user, object, name) do + {:ok, + %{ + "type" => "Answer", + "actor" => user.ap_id, + "attributedTo" => user.ap_id, + "cc" => [object.data["actor"]], + "to" => [], + "name" => name, + "inReplyTo" => object.data["id"], + "context" => object.data["context"], + "published" => DateTime.utc_now() |> DateTime.to_iso8601(), + "id" => Utils.generate_object_id() + }, []} + end + @spec tombstone(String.t(), String.t()) :: {:ok, map(), keyword()} def tombstone(actor, id) do {:ok, @@ -123,13 +182,40 @@ defmodule Pleroma.Web.ActivityPub.Builder do end end + # Retricted to user updates for now, always public + @spec update(User.t(), Object.t()) :: {:ok, map(), keyword()} + def update(actor, object) do + to = [Pleroma.Constants.as_public(), actor.follower_address] + + {:ok, + %{ + "id" => Utils.generate_activity_id(), + "type" => "Update", + "actor" => actor.ap_id, + "object" => object, + "to" => to + }, []} + end + + @spec block(User.t(), User.t()) :: {:ok, map(), keyword()} + def block(blocker, blocked) do + {:ok, + %{ + "id" => Utils.generate_activity_id(), + "type" => "Block", + "actor" => blocker.ap_id, + "object" => blocked.ap_id, + "to" => [blocked.ap_id] + }, []} + end + @spec announce(User.t(), Object.t(), keyword()) :: {:ok, map(), keyword()} def announce(actor, object, options \\ []) do public? = Keyword.get(options, :public, false) to = cond do - actor.ap_id == Relay.relay_ap_id() -> + actor.ap_id == Relay.ap_id() -> [actor.follower_address] public? -> diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 5a4a76085..206d6af52 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do def filter(%{} = object), do: get_policies() |> filter(object) def get_policies do - Pleroma.Config.get([:instance, :rewrite_policy], []) |> get_policies() + Pleroma.Config.get([:mrf, :policies], []) |> get_policies() end defp get_policies(policy) when is_atom(policy), do: [policy] @@ -51,7 +51,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do get_policies() |> Enum.map(fn policy -> to_string(policy) |> String.split(".") |> List.last() end) - exclusions = Pleroma.Config.get([:instance, :mrf_transparency_exclusions]) + exclusions = Pleroma.Config.get([:mrf, :transparency_exclusions]) base = %{ diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex index 8e47f1e02..bee47b4ed 100644 --- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex @@ -21,8 +21,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do @impl true def describe, do: {:ok, %{}} - defp local?(%{"id" => id}) do - String.starts_with?(id, Pleroma.Web.Endpoint.url()) + defp local?(%{"actor" => actor}) do + String.starts_with?(actor, Pleroma.Web.Endpoint.url()) end defp note?(activity) do @@ -31,10 +31,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do defp maybe_add_expiration(activity) do days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365) - expires_at = NaiveDateTime.utc_now() |> Timex.shift(days: days) + expires_at = DateTime.utc_now() |> Timex.shift(days: days) with %{"expires_at" => existing_expires_at} <- activity, - :lt <- NaiveDateTime.compare(existing_expires_at, expires_at) do + :lt <- DateTime.compare(existing_expires_at, expires_at) do activity else _ -> Map.put(activity, "expires_at", expires_at) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 0270b96ae..b96388489 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -60,7 +60,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do if score < 0.8 do {:ok, message} else - {:reject, nil} + {:reject, "[AntiFollowbotPolicy] Scored #{actor_id} as #{score}"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index 9e7800997..b22464111 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -27,23 +27,25 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do @impl true def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message) do - with {:ok, %User{} = u} <- User.get_or_fetch_by_ap_id(actor), + with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor), {:contains_links, true} <- {:contains_links, contains_links?(object)}, {:old_user, true} <- {:old_user, old_user?(u)} do {:ok, message} else + {:ok, %User{local: true}} -> + {:ok, message} + {:contains_links, false} -> {:ok, message} {:old_user, false} -> - {:reject, nil} + {:reject, "[AntiLinkSpamPolicy] User has no posts nor followers"} {:error, _} -> - {:reject, nil} + {:reject, "[AntiLinkSpamPolicy] Failed to get or fetch user by ap_id"} e -> - Logger.warn("[MRF anti-link-spam] WTF: unhandled error #{inspect(e)}") - {:reject, nil} + {:reject, "[AntiLinkSpamPolicy] Unhandled error #{inspect(e)}"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index 2627a0007..3bf70b894 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -27,7 +27,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do def filter_by_summary(_in_reply_to, child), do: child - def filter(%{"type" => "Create", "object" => child_object} = object) do + def filter(%{"type" => "Create", "object" => child_object} = object) + when is_map(child_object) do child = child_object["inReplyTo"] |> Object.normalize(child_object["inReplyTo"]) diff --git a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex new file mode 100644 index 000000000..ea9c3d3f5 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do + alias Pleroma.User + @behaviour Pleroma.Web.ActivityPub.MRF + @moduledoc "Remove bot posts from federated timeline" + + require Pleroma.Constants + + defp check_by_actor_type(user), do: user.actor_type in ["Application", "Service"] + defp check_by_nickname(user), do: Regex.match?(~r/bot@|ebooks@/i, user.nickname) + + defp check_if_bot(user), do: check_by_actor_type(user) or check_by_nickname(user) + + @impl true + def filter( + %{ + "type" => "Create", + "to" => to, + "cc" => cc, + "actor" => actor, + "object" => object + } = message + ) do + user = User.get_cached_by_ap_id(actor) + isbot = check_if_bot(user) + + if isbot and Enum.member?(to, Pleroma.Constants.as_public()) do + to = List.delete(to, Pleroma.Constants.as_public()) ++ [user.follower_address] + cc = List.delete(cc, user.follower_address) ++ [Pleroma.Constants.as_public()] + + object = + object + |> Map.put("to", to) + |> Map.put("cc", cc) + + message = + message + |> Map.put("to", to) + |> Map.put("cc", cc) + |> Map.put("object", object) + + {:ok, message} + else + {:ok, message} + end + end + + @impl true + def filter(message), do: {:ok, message} + + @impl true + def describe, do: {:ok, %{}} +end diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 1764bc789..9ba07b4e3 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -13,8 +13,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do defp delist_message(message, threshold) when threshold > 0 do follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address + to = message["to"] || [] + cc = message["cc"] || [] - follower_collection? = Enum.member?(message["to"] ++ message["cc"], follower_collection) + follower_collection? = Enum.member?(to ++ cc, follower_collection) message = case get_recipient_count(message) do @@ -41,7 +43,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do defp reject_message(message, threshold) when threshold > 0 do with {_, recipients} <- get_recipient_count(message) do if recipients > threshold do - {:reject, nil} + {:reject, "[HellthreadPolicy] #{recipients} recipients is over the limit of #{threshold}"} else {:ok, message} end @@ -71,7 +73,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do end @impl true - def filter(%{"type" => "Create"} = message) do + def filter(%{"type" => "Create", "object" => %{"type" => object_type}} = message) + when object_type in ~w{Note Article} do reject_threshold = Pleroma.Config.get( [:mrf_hellthread, :reject_threshold], @@ -84,7 +87,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do {:ok, message} <- delist_message(message, delist_threshold) do {:ok, message} else - _e -> {:reject, nil} + e -> e end end diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 88b0d2b39..15e09dcf0 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern -> string_matches?(content, pattern) or string_matches?(summary, pattern) end) do - {:reject, nil} + {:reject, "[KeywordPolicy] Matches with rejected keyword"} else {:ok, message} end @@ -89,8 +89,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do {:ok, message} <- check_replace(message) do {:ok, message} else - _e -> - {:reject, nil} + {:reject, nil} -> {:reject, "[KeywordPolicy] "} + {:reject, _} = e -> e + _e -> {:reject, "[KeywordPolicy] "} end end diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index dfab105a3..98d595469 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -13,22 +13,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger @options [ - pool: :media + pool: :media, + recv_timeout: 10_000 ] def perform(:prefetch, url) do Logger.debug("Prefetching #{inspect(url)}") - opts = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - Keyword.put(@options, :recv_timeout, 10_000) - else - @options - end - url |> MediaProxy.url() - |> HTTP.get([], adapter: opts) + |> HTTP.get([], @options) end def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do diff --git a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex index 06f003921..7910ca131 100644 --- a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex @@ -12,8 +12,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do reject_actors = Pleroma.Config.get([:mrf_mention, :actors], []) recipients = (message["to"] || []) ++ (message["cc"] || []) - if Enum.any?(recipients, fn recipient -> Enum.member?(reject_actors, recipient) end) do - {:reject, nil} + if rejected_mention = + Enum.find(recipients, fn recipient -> Enum.member?(reject_actors, recipient) end) do + {:reject, "[MentionPolicy] Rejected for mention of #{rejected_mention}"} else {:ok, message} end diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex index b0ccb63c8..d45d2d7e3 100644 --- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do defp check_reject(message, actions) do if :reject in actions do - {:reject, nil} + {:reject, "[ObjectAgePolicy]"} else {:ok, message} end @@ -37,8 +37,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do defp check_delist(message, actions) do if :delist in actions do with %User{} = user <- User.get_cached_by_ap_id(message["actor"]) do - to = List.delete(message["to"], Pleroma.Constants.as_public()) ++ [user.follower_address] - cc = List.delete(message["cc"], user.follower_address) ++ [Pleroma.Constants.as_public()] + to = + List.delete(message["to"] || [], Pleroma.Constants.as_public()) ++ + [user.follower_address] + + cc = + List.delete(message["cc"] || [], user.follower_address) ++ + [Pleroma.Constants.as_public()] message = message @@ -47,9 +52,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do {:ok, message} else - # Unhandleable error: somebody is messing around, just drop the message. _e -> - {:reject, nil} + {:reject, "[ObjectAgePolicy] Unhandled error"} end else {:ok, message} @@ -59,8 +63,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do defp check_strip_followers(message, actions) do if :strip_followers in actions do with %User{} = user <- User.get_cached_by_ap_id(message["actor"]) do - to = List.delete(message["to"], user.follower_address) - cc = List.delete(message["cc"], user.follower_address) + to = List.delete(message["to"] || [], user.follower_address) + cc = List.delete(message["cc"] || [], user.follower_address) message = message @@ -69,9 +73,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do {:ok, message} else - # Unhandleable error: somebody is messing around, just drop the message. _e -> - {:reject, nil} + {:reject, "[ObjectAgePolicy] Unhandled error"} end else {:ok, message} @@ -98,7 +101,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do @impl true def describe do mrf_object_age = - Pleroma.Config.get(:mrf_object_age) + Config.get(:mrf_object_age) |> Enum.into(%{}) {:ok, %{mrf_object_age: mrf_object_age}} diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index 3092f3272..0b9ed2224 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do {:ok, object} true -> - {:reject, nil} + {:reject, "[RejectNonPublic] visibility: #{visibility}"} end end @@ -47,5 +47,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do @impl true def describe, - do: {:ok, %{mrf_rejectnonpublic: Pleroma.Config.get(:mrf_rejectnonpublic) |> Enum.into(%{})}} + do: {:ok, %{mrf_rejectnonpublic: Config.get(:mrf_rejectnonpublic) |> Enum.into(%{})}} end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index b7dcb1b86..bb193475a 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -3,33 +3,36 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do - alias Pleroma.User - alias Pleroma.Web.ActivityPub.MRF @moduledoc "Filter activities depending on their origin instance" @behaviour Pleroma.Web.ActivityPub.MRF + alias Pleroma.Config + alias Pleroma.FollowingRelationship + alias Pleroma.User + alias Pleroma.Web.ActivityPub.MRF + require Pleroma.Constants defp check_accept(%{host: actor_host} = _actor_info, object) do accepts = - Pleroma.Config.get([:mrf_simple, :accept]) + Config.get([:mrf_simple, :accept]) |> MRF.subdomains_regex() cond do accepts == [] -> {:ok, object} - actor_host == Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, object} + actor_host == Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, object} MRF.subdomain_match?(accepts, actor_host) -> {:ok, object} - true -> {:reject, nil} + true -> {:reject, "[SimplePolicy] host not in accept list"} end end defp check_reject(%{host: actor_host} = _actor_info, object) do rejects = - Pleroma.Config.get([:mrf_simple, :reject]) + Config.get([:mrf_simple, :reject]) |> MRF.subdomains_regex() if MRF.subdomain_match?(rejects, actor_host) do - {:reject, nil} + {:reject, "[SimplePolicy] host in reject list"} else {:ok, object} end @@ -41,7 +44,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do ) when length(child_attachment) > 0 do media_removal = - Pleroma.Config.get([:mrf_simple, :media_removal]) + Config.get([:mrf_simple, :media_removal]) |> MRF.subdomains_regex() object = @@ -65,7 +68,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do } = object ) do media_nsfw = - Pleroma.Config.get([:mrf_simple, :media_nsfw]) + Config.get([:mrf_simple, :media_nsfw]) |> MRF.subdomains_regex() object = @@ -85,7 +88,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do timeline_removal = - Pleroma.Config.get([:mrf_simple, :federated_timeline_removal]) + Config.get([:mrf_simple, :federated_timeline_removal]) |> MRF.subdomains_regex() object = @@ -106,13 +109,42 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} end + defp intersection(list1, list2) do + list1 -- list1 -- list2 + end + + defp check_followers_only(%{host: actor_host} = _actor_info, object) do + followers_only = + Config.get([:mrf_simple, :followers_only]) + |> MRF.subdomains_regex() + + object = + with true <- MRF.subdomain_match?(followers_only, actor_host), + user <- User.get_cached_by_ap_id(object["actor"]) do + # Don't use Map.get/3 intentionally, these must not be nil + fixed_to = object["to"] || [] + fixed_cc = object["cc"] || [] + + to = FollowingRelationship.followers_ap_ids(user, fixed_to) + cc = FollowingRelationship.followers_ap_ids(user, fixed_cc) + + object + |> Map.put("to", intersection([user.follower_address | to], fixed_to)) + |> Map.put("cc", intersection([user.follower_address | cc], fixed_cc)) + else + _ -> object + end + + {:ok, object} + end + defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do report_removal = - Pleroma.Config.get([:mrf_simple, :report_removal]) + Config.get([:mrf_simple, :report_removal]) |> MRF.subdomains_regex() if MRF.subdomain_match?(report_removal, actor_host) do - {:reject, nil} + {:reject, "[SimplePolicy] host in report_removal list"} else {:ok, object} end @@ -122,7 +154,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do avatar_removal = - Pleroma.Config.get([:mrf_simple, :avatar_removal]) + Config.get([:mrf_simple, :avatar_removal]) |> MRF.subdomains_regex() if MRF.subdomain_match?(avatar_removal, actor_host) do @@ -136,7 +168,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do banner_removal = - Pleroma.Config.get([:mrf_simple, :banner_removal]) + Config.get([:mrf_simple, :banner_removal]) |> MRF.subdomains_regex() if MRF.subdomain_match?(banner_removal, actor_host) do @@ -153,11 +185,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do %{host: actor_host} = URI.parse(actor) reject_deletes = - Pleroma.Config.get([:mrf_simple, :reject_deletes]) + Config.get([:mrf_simple, :reject_deletes]) |> MRF.subdomains_regex() if MRF.subdomain_match?(reject_deletes, actor_host) do - {:reject, nil} + {:reject, "[SimplePolicy] host in reject_deletes list"} else {:ok, object} end @@ -172,10 +204,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} <- check_media_removal(actor_info, object), {:ok, object} <- check_media_nsfw(actor_info, object), {:ok, object} <- check_ftl_removal(actor_info, object), + {:ok, object} <- check_followers_only(actor_info, object), {:ok, object} <- check_report_removal(actor_info, object) do {:ok, object} else - _e -> {:reject, nil} + {:reject, nil} -> {:reject, "[SimplePolicy]"} + {:reject, _} = e -> e + _ -> {:reject, "[SimplePolicy]"} end end @@ -189,7 +224,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} <- check_banner_removal(actor_info, object) do {:ok, object} else - _e -> {:reject, nil} + {:reject, nil} -> {:reject, "[SimplePolicy]"} + {:reject, _} = e -> e + _ -> {:reject, "[SimplePolicy]"} end end @@ -197,10 +234,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do @impl true def describe do - exclusions = Pleroma.Config.get([:instance, :mrf_transparency_exclusions]) + exclusions = Config.get([:mrf, :transparency_exclusions]) mrf_simple = - Pleroma.Config.get(:mrf_simple) + Config.get(:mrf_simple) |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn v -> v in exclusions end)} end) |> Enum.into(%{}) diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index c310462cb..febabda08 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -134,12 +134,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do if user.local == true do {:ok, message} else - {:reject, nil} + {:reject, + "[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-remote-subscription"} end end - defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow"}), - do: {:reject, nil} + defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow", "actor" => actor}), + do: {:reject, "[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-any-subscription"} defp process_tag(_, message), do: {:ok, message} diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex index 651aed70f..1a28f2ba2 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do if actor in allow_list do {:ok, object} else - {:reject, nil} + {:reject, "[UserAllowListPolicy] #{actor} not in the list"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index 6167a74e2..a6c545570 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -11,22 +11,26 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do with {:ok, _} <- filter(child_message) do {:ok, message} else - {:reject, nil} -> - {:reject, nil} + {:reject, _} = e -> e end end def filter(%{"type" => message_type} = message) do with accepted_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :accept]), rejected_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :reject]), - true <- - Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, message_type), - false <- - length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type), + {_, true} <- + {:accepted, + Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, message_type)}, + {_, false} <- + {:rejected, + length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type)}, {:ok, _} <- filter(message["object"]) do {:ok, message} else - _ -> {:reject, nil} + {:reject, _} = e -> e + {:accepted, _} -> {:reject, "[VocabularyPolicy] #{message_type} not in accept list"} + {:rejected, _} -> {:reject, "[VocabularyPolicy] #{message_type} in reject list"} + _ -> {:reject, "[VocabularyPolicy]"} end end diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index c01c5f780..b77c06395 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -9,26 +9,102 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do the system. """ + alias Pleroma.Activity + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object alias Pleroma.User + alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.EventValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.Types + alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()} def validate(object, meta) + def validate(%{"type" => type} = object, meta) + when type in ~w[Accept Reject] do + with {:ok, object} <- + object + |> AcceptRejectValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + + def validate(%{"type" => "Event"} = object, meta) do + with {:ok, object} <- + object + |> EventValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + + def validate(%{"type" => "Follow"} = object, meta) do + with {:ok, object} <- + object + |> FollowValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + + def validate(%{"type" => "Block"} = block_activity, meta) do + with {:ok, block_activity} <- + block_activity + |> BlockValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + block_activity = stringify_keys(block_activity) + outgoing_blocks = Pleroma.Config.get([:activitypub, :outgoing_blocks]) + + meta = + if !outgoing_blocks do + Keyword.put(meta, :do_not_federate, true) + else + meta + end + + {:ok, block_activity, meta} + end + end + + def validate(%{"type" => "Update"} = update_activity, meta) do + with {:ok, update_activity} <- + update_activity + |> UpdateValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + update_activity = stringify_keys(update_activity) + {:ok, update_activity, meta} + end + end + def validate(%{"type" => "Undo"} = object, meta) do with {:ok, object} <- object |> UndoValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do object = stringify_keys(object) + undone_object = Activity.get_by_ap_id(object["object"]) + + meta = + meta + |> Keyword.put(:object_data, undone_object.data) + {:ok, object, meta} end end @@ -63,17 +139,50 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end end + def validate(%{"type" => "Question"} = object, meta) do + with {:ok, object} <- + object + |> QuestionValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + + def validate(%{"type" => "Audio"} = object, meta) do + with {:ok, object} <- + object + |> AudioValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + + def validate(%{"type" => "Answer"} = object, meta) do + with {:ok, object} <- + object + |> AnswerValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + def validate(%{"type" => "EmojiReact"} = object, meta) do with {:ok, object} <- object |> EmojiReactValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object |> Map.from_struct()) + object = stringify_keys(object) {:ok, object, meta} end end - def validate(%{"type" => "Create", "object" => object} = create_activity, meta) do + def validate( + %{"type" => "Create", "object" => %{"type" => "ChatMessage"} = object} = create_activity, + meta + ) do with {:ok, object_data} <- cast_and_apply(object), meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, create_activity} <- @@ -85,12 +194,28 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end end + def validate( + %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity, + meta + ) + when objtype in ~w[Question Answer Audio Event] do + with {:ok, object_data} <- cast_and_apply(object), + meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), + {:ok, create_activity} <- + create_activity + |> CreateGenericValidator.cast_and_validate(meta) + |> Ecto.Changeset.apply_action(:insert) do + create_activity = stringify_keys(create_activity) + {:ok, create_activity, meta} + end + end + def validate(%{"type" => "Announce"} = object, meta) do with {:ok, object} <- object |> AnnounceValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object |> Map.from_struct()) + object = stringify_keys(object) {:ok, object, meta} end end @@ -99,8 +224,25 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do ChatMessageValidator.cast_and_apply(object) end + def cast_and_apply(%{"type" => "Question"} = object) do + QuestionValidator.cast_and_apply(object) + end + + def cast_and_apply(%{"type" => "Answer"} = object) do + AnswerValidator.cast_and_apply(object) + end + + def cast_and_apply(%{"type" => "Audio"} = object) do + AudioValidator.cast_and_apply(object) + end + + def cast_and_apply(%{"type" => "Event"} = object) do + EventValidator.cast_and_apply(object) + end + def cast_and_apply(o), do: {:error, {:validator_not_set, o}} + # is_struct/1 isn't present in Elixir 1.8.x def stringify_keys(%{__struct__: _} = object) do object |> Map.from_struct() @@ -120,7 +262,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def stringify_keys(object), do: object def fetch_actor(object) do - with {:ok, actor} <- Types.ObjectID.cast(object["actor"]) do + with {:ok, actor} <- ObjectValidators.ObjectID.cast(object["actor"]) do User.get_or_fetch_by_ap_id(actor) end end diff --git a/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex new file mode 100644 index 000000000..179beda58 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator do + use Ecto.Schema + + alias Pleroma.Activity + alias Pleroma.EctoType.ActivityPub.ObjectValidators + + import Ecto.Changeset + import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + + @primary_key false + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:type, :string) + field(:object, ObjectValidators.ObjectID) + field(:actor, ObjectValidators.ObjectID) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + end + + def cast_data(data) do + %__MODULE__{} + |> cast(data, __schema__(:fields)) + end + + def validate_data(cng) do + cng + |> validate_required([:id, :type, :actor, :to, :cc, :object]) + |> validate_inclusion(:type, ["Accept", "Reject"]) + |> validate_actor_presence() + |> validate_object_presence(allowed_types: ["Follow"]) + |> validate_accept_reject_rights() + end + + def cast_and_validate(data) do + data + |> cast_data + |> validate_data + end + + def validate_accept_reject_rights(cng) do + with object_id when is_binary(object_id) <- get_field(cng, :object), + %Activity{data: %{"object" => followed_actor}} <- Activity.get_by_ap_id(object_id), + true <- followed_actor == get_field(cng, :actor) do + cng + else + _e -> + cng + |> add_error(:actor, "can't accept or reject the given activity") + end + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex index 40f861f47..6f757f49c 100644 --- a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex @@ -5,9 +5,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object alias Pleroma.User - alias Pleroma.Web.ActivityPub.ObjectValidators.Types alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility @@ -19,14 +19,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) + field(:id, ObjectValidators.ObjectID, primary_key: true) field(:type, :string) - field(:object, Types.ObjectID) - field(:actor, Types.ObjectID) + field(:object, ObjectValidators.ObjectID) + field(:actor, ObjectValidators.ObjectID) field(:context, :string, autogenerate: {Utils, :generate_context_id, []}) - field(:to, Types.Recipients, default: []) - field(:cc, Types.Recipients, default: []) - field(:published, Types.DateTime) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:published, ObjectValidators.DateTime) end def cast_and_validate(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex new file mode 100644 index 000000000..b9fbaf4f6 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex @@ -0,0 +1,62 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + + import Ecto.Changeset + + @primary_key false + @derive Jason.Encoder + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:bto, ObjectValidators.Recipients, default: []) + field(:bcc, ObjectValidators.Recipients, default: []) + field(:type, :string) + field(:name, :string) + field(:inReplyTo, ObjectValidators.ObjectID) + field(:attributedTo, ObjectValidators.ObjectID) + + # TODO: Remove actor on objects + field(:actor, ObjectValidators.ObjectID) + end + + def cast_and_apply(data) do + data + |> cast_data() + |> apply_action(:insert) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + def cast_data(data) do + %__MODULE__{} + |> changeset(data) + end + + def changeset(struct, data) do + struct + |> cast(data, __schema__(:fields)) + end + + def validate_data(data_cng) do + data_cng + |> validate_inclusion(:type, ["Answer"]) + |> validate_required([:id, :inReplyTo, :name, :attributedTo, :actor]) + |> CommonValidations.validate_any_presence([:cc, :to]) + |> CommonValidations.validate_fields_match([:actor, :attributedTo]) + |> CommonValidations.validate_actor_presence() + |> CommonValidations.validate_host_match() + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index f53bb02be..c8b148280 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -41,34 +41,34 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do end def fix_media_type(data) do - data = - data - |> Map.put_new("mediaType", data["mimeType"]) + data = Map.put_new(data, "mediaType", data["mimeType"]) if MIME.valid?(data["mediaType"]) do data else - data - |> Map.put("mediaType", "application/octet-stream") + Map.put(data, "mediaType", "application/octet-stream") end end - def fix_url(data) do - case data["url"] do - url when is_binary(url) -> - data - |> Map.put( - "url", - [ - %{ - "href" => url, - "type" => "Link", - "mediaType" => data["mediaType"] - } - ] - ) - - _ -> + defp handle_href(href, mediaType) do + [ + %{ + "href" => href, + "type" => "Link", + "mediaType" => mediaType + } + ] + end + + defp fix_url(data) do + cond do + is_binary(data["url"]) -> + Map.put(data, "url", handle_href(data["url"], data["mediaType"])) + + is_binary(data["href"]) and data["url"] == nil -> + Map.put(data, "url", handle_href(data["href"], data["mediaType"])) + + true -> data end end diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex new file mode 100644 index 000000000..1a97c504a --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex @@ -0,0 +1,107 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Ecto.Changeset + + @primary_key false + @derive Jason.Encoder + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:bto, ObjectValidators.Recipients, default: []) + field(:bcc, ObjectValidators.Recipients, default: []) + # TODO: Write type + field(:tag, {:array, :map}, default: []) + field(:type, :string) + field(:content, :string) + field(:context, :string) + + # TODO: Remove actor on objects + field(:actor, ObjectValidators.ObjectID) + + field(:attributedTo, ObjectValidators.ObjectID) + field(:summary, :string) + field(:published, ObjectValidators.DateTime) + field(:emoji, ObjectValidators.Emoji, default: %{}) + field(:sensitive, :boolean, default: false) + embeds_many(:attachment, AttachmentValidator) + field(:replies_count, :integer, default: 0) + field(:like_count, :integer, default: 0) + field(:announcement_count, :integer, default: 0) + field(:inReplyTo, :string) + field(:url, ObjectValidators.Uri) + # short identifier for PleromaFE to group statuses by context + field(:context_id, :integer) + + field(:likes, {:array, :string}, default: []) + field(:announcements, {:array, :string}, default: []) + end + + def cast_and_apply(data) do + data + |> cast_data + |> apply_action(:insert) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + def cast_data(data) do + %__MODULE__{} + |> changeset(data) + end + + defp fix_url(%{"url" => url} = data) when is_list(url) do + attachment = + Enum.find(url, fn x -> is_map(x) and String.starts_with?(x["mimeType"], "audio/") end) + + link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end) + + data + |> Map.put("attachment", [attachment]) + |> Map.put("url", link_element["href"]) + end + + defp fix_url(data), do: data + + defp fix(data) do + data + |> CommonFixes.fix_defaults() + |> CommonFixes.fix_attribution() + |> Transmogrifier.fix_emoji() + |> fix_url() + end + + def changeset(struct, data) do + data = fix(data) + + struct + |> cast(data, __schema__(:fields) -- [:attachment]) + |> cast_embed(:attachment) + end + + def validate_data(data_cng) do + data_cng + |> validate_inclusion(:type, ["Audio"]) + |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment]) + |> CommonValidations.validate_any_presence([:cc, :to]) + |> CommonValidations.validate_fields_match([:actor, :attributedTo]) + |> CommonValidations.validate_actor_presence() + |> CommonValidations.validate_host_match() + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex new file mode 100644 index 000000000..1dde77198 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex @@ -0,0 +1,42 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + + import Ecto.Changeset + import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + + @primary_key false + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:type, :string) + field(:actor, ObjectValidators.ObjectID) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:object, ObjectValidators.ObjectID) + end + + def cast_data(data) do + %__MODULE__{} + |> cast(data, __schema__(:fields)) + end + + def validate_data(cng) do + cng + |> validate_required([:id, :type, :actor, :to, :cc, :object]) + |> validate_inclusion(:type, ["Block"]) + |> validate_actor_presence() + |> validate_actor_presence(field_name: :object) + end + + def cast_and_validate(data) do + data + |> cast_data + |> validate_data + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex index 138736f23..6acd4a771 100644 --- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex @@ -5,9 +5,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.Types import Ecto.Changeset import Pleroma.Web.ActivityPub.Transmogrifier, only: [fix_emoji: 1] @@ -16,13 +16,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do @derive Jason.Encoder embedded_schema do - field(:id, Types.ObjectID, primary_key: true) - field(:to, Types.Recipients, default: []) + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:to, ObjectValidators.Recipients, default: []) field(:type, :string) - field(:content, Types.SafeText) - field(:actor, Types.ObjectID) - field(:published, Types.DateTime) - field(:emoji, :map, default: %{}) + field(:content, ObjectValidators.SafeText) + field(:actor, ObjectValidators.ObjectID) + field(:published, ObjectValidators.DateTime) + field(:emoji, ObjectValidators.Emoji, default: %{}) embeds_one(:attachment, AttachmentValidator) end @@ -93,12 +93,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do - If both users are in our system - If at least one of the users in this ChatMessage is a local user - If the recipient is not blocking the actor + - If the recipient is explicitly not accepting chat messages """ def validate_local_concern(cng) do with actor_ap <- get_field(cng, :actor), {_, %User{} = actor} <- {:find_actor, User.get_cached_by_ap_id(actor_ap)}, {_, %User{} = recipient} <- {:find_recipient, User.get_cached_by_ap_id(get_field(cng, :to) |> hd())}, + {_, false} <- {:not_accepting_chats?, recipient.accepts_chat_messages == false}, {_, false} <- {:blocking_actor?, User.blocks?(recipient, actor)}, {_, true} <- {:local?, Enum.any?([actor, recipient], & &1.local)} do cng @@ -107,6 +109,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do cng |> add_error(:actor, "actor is blocked by recipient") + {:not_accepting_chats?, true} -> + cng + |> add_error(:to, "recipient does not accept chat messages") + {:local?, false} -> cng |> add_error(:actor, "actor and recipient are both remote") diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex new file mode 100644 index 000000000..720213d73 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do + alias Pleroma.Web.ActivityPub.Utils + + # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults + def fix_defaults(data) do + %{data: %{"id" => context}, id: context_id} = + Utils.create_context(data["context"] || data["conversation"]) + + data + |> Map.put("context", context) + |> Map.put("context_id", context_id) + end + + def fix_attribution(data) do + data + |> Map.put_new("actor", data["attributedTo"]) + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index aeef31945..603d87b8e 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do alias Pleroma.Object alias Pleroma.User - def validate_recipients_presence(cng, fields \\ [:to, :cc]) do + def validate_any_presence(cng, fields) do non_empty = fields |> Enum.map(fn field -> get_field(cng, field) end) @@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do fields |> Enum.reduce(cng, fn field, cng -> cng - |> add_error(field, "no recipients in any field") + |> add_error(field, "none of #{inspect(fields)} present") end) end end @@ -34,10 +34,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do cng |> validate_change(field_name, fn field_name, actor -> - if User.get_cached_by_ap_id(actor) do - [] - else - [{field_name, "can't find user"}] + case User.get_cached_by_ap_id(actor) do + %User{deactivated: true} -> + [{field_name, "user is deactivated"}] + + %User{} -> + [] + + _ -> + [{field_name, "can't find user"}] end end) end @@ -77,4 +82,60 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do if actor_cng.valid?, do: actor_cng, else: object_cng end + + def validate_host_match(cng, fields \\ [:id, :actor]) do + if same_domain?(cng, fields) do + cng + else + fields + |> Enum.reduce(cng, fn field, cng -> + cng + |> add_error(field, "hosts of #{inspect(fields)} aren't matching") + end) + end + end + + def validate_fields_match(cng, fields) do + if map_unique?(cng, fields) do + cng + else + fields + |> Enum.reduce(cng, fn field, cng -> + cng + |> add_error(field, "Fields #{inspect(fields)} aren't matching") + end) + end + end + + defp map_unique?(cng, fields, func \\ & &1) do + Enum.reduce_while(fields, nil, fn field, acc -> + value = + cng + |> get_field(field) + |> func.() + + case {value, acc} do + {value, nil} -> {:cont, value} + {value, value} -> {:cont, value} + _ -> {:halt, false} + end + end) + end + + def same_domain?(cng, fields \\ [:actor, :object]) do + map_unique?(cng, fields, fn value -> URI.parse(value).host end) + end + + # This figures out if a user is able to create, delete or modify something + # based on the domain and superuser status + def validate_modification_rights(cng) do + actor = User.get_cached_by_ap_id(get_field(cng, :actor)) + + if User.superuser?(actor) || same_domain?(cng) do + cng + else + cng + |> add_error(:actor, "is not allowed to modify object") + end + end end diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex index fc582400b..7269f9ff0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex @@ -7,9 +7,9 @@ # - doesn't embed, will only get the object id defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object - alias Pleroma.Web.ActivityPub.ObjectValidators.Types import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @@ -17,11 +17,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) - field(:actor, Types.ObjectID) + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:actor, ObjectValidators.ObjectID) field(:type, :string) - field(:to, Types.Recipients, default: []) - field(:object, Types.ObjectID) + field(:to, ObjectValidators.Recipients, default: []) + field(:object, ObjectValidators.ObjectID) end def cast_and_apply(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex new file mode 100644 index 000000000..b3dbeea57 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -0,0 +1,144 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +# Code based on CreateChatMessageValidator +# NOTES +# - doesn't embed, will only get the object id +defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Object + + import Ecto.Changeset + import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + + @primary_key false + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:actor, ObjectValidators.ObjectID) + field(:type, :string) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:object, ObjectValidators.ObjectID) + field(:expires_at, ObjectValidators.DateTime) + + # Should be moved to object, done for CommonAPI.Utils.make_context + field(:context, :string) + end + + def cast_data(data, meta \\ []) do + data = fix(data, meta) + + %__MODULE__{} + |> changeset(data) + end + + def cast_and_apply(data) do + data + |> cast_data + |> apply_action(:insert) + end + + def cast_and_validate(data, meta \\ []) do + data + |> cast_data(meta) + |> validate_data(meta) + end + + def changeset(struct, data) do + struct + |> cast(data, __schema__(:fields)) + end + + defp fix_context(data, meta) do + if object = meta[:object_data] do + Map.put_new(data, "context", object["context"]) + else + data + end + end + + defp fix_addressing(data, meta) do + if object = meta[:object_data] do + data + |> Map.put_new("to", object["to"] || []) + |> Map.put_new("cc", object["cc"] || []) + else + data + end + end + + defp fix(data, meta) do + data + |> fix_context(meta) + |> fix_addressing(meta) + end + + def validate_data(cng, meta \\ []) do + cng + |> validate_required([:actor, :type, :object]) + |> validate_inclusion(:type, ["Create"]) + |> validate_actor_presence() + |> validate_any_presence([:to, :cc]) + |> validate_actors_match(meta) + |> validate_context_match(meta) + |> validate_object_nonexistence() + |> validate_object_containment() + end + + def validate_object_containment(cng) do + actor = get_field(cng, :actor) + + cng + |> validate_change(:object, fn :object, object_id -> + %URI{host: object_id_host} = URI.parse(object_id) + %URI{host: actor_host} = URI.parse(actor) + + if object_id_host == actor_host do + [] + else + [{:object, "The host of the object id doesn't match with the host of the actor"}] + end + end) + end + + def validate_object_nonexistence(cng) do + cng + |> validate_change(:object, fn :object, object_id -> + if Object.get_cached_by_ap_id(object_id) do + [{:object, "The object to create already exists"}] + else + [] + end + end) + end + + def validate_actors_match(cng, meta) do + attributed_to = meta[:object_data]["attributedTo"] || meta[:object_data]["actor"] + + cng + |> validate_change(:actor, fn :actor, actor -> + if actor == attributed_to do + [] + else + [{:actor, "Actor doesn't match with object attributedTo"}] + end + end) + end + + def validate_context_match(cng, %{object_data: %{"context" => object_context}}) do + cng + |> validate_change(:context, fn :context, context -> + if context == object_context do + [] + else + [{:context, "context field not matching between Create and object (#{object_context})"}] + end + end) + end + + def validate_context_match(cng, _), do: cng +end diff --git a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex index 926804ce7..9b9743c4a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex @@ -5,22 +5,21 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.Types import Ecto.Changeset @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) - field(:actor, Types.ObjectID) + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:actor, ObjectValidators.ObjectID) field(:type, :string) - field(:to, {:array, :string}) - field(:cc, {:array, :string}) - field(:bto, {:array, :string}, default: []) - field(:bcc, {:array, :string}, default: []) - + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:bto, ObjectValidators.Recipients, default: []) + field(:bcc, ObjectValidators.Recipients, default: []) embeds_one(:object, NoteValidator) end diff --git a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex index e5d08eb5c..2634e8d4d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex @@ -6,8 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do use Ecto.Schema alias Pleroma.Activity - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ObjectValidators.Types + alias Pleroma.EctoType.ActivityPub.ObjectValidators import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @@ -15,13 +14,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) + field(:id, ObjectValidators.ObjectID, primary_key: true) field(:type, :string) - field(:actor, Types.ObjectID) - field(:to, Types.Recipients, default: []) - field(:cc, Types.Recipients, default: []) - field(:deleted_activity_id, Types.ObjectID) - field(:object, Types.ObjectID) + field(:actor, ObjectValidators.ObjectID) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:deleted_activity_id, ObjectValidators.ObjectID) + field(:object, ObjectValidators.ObjectID) end def cast_data(data) do @@ -59,7 +58,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do |> validate_required([:id, :type, :actor, :to, :cc, :object]) |> validate_inclusion(:type, ["Delete"]) |> validate_actor_presence() - |> validate_deletion_rights() + |> validate_modification_rights() |> validate_object_or_user_presence(allowed_types: @deletable_types) |> add_deleted_activity_id() end @@ -68,31 +67,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do !same_domain?(cng) end - defp same_domain?(cng) do - actor_uri = - cng - |> get_field(:actor) - |> URI.parse() - - object_uri = - cng - |> get_field(:object) - |> URI.parse() - - object_uri.host == actor_uri.host - end - - def validate_deletion_rights(cng) do - actor = User.get_cached_by_ap_id(get_field(cng, :actor)) - - if User.superuser?(actor) || same_domain?(cng) do - cng - else - cng - |> add_error(:actor, "is not allowed to delete object") - end - end - def cast_and_validate(data) do data |> cast_data diff --git a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex index e87519c59..336c92d35 100644 --- a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object - alias Pleroma.Web.ActivityPub.ObjectValidators.Types import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @@ -14,14 +14,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) + field(:id, ObjectValidators.ObjectID, primary_key: true) field(:type, :string) - field(:object, Types.ObjectID) - field(:actor, Types.ObjectID) + field(:object, ObjectValidators.ObjectID) + field(:actor, ObjectValidators.ObjectID) field(:context, :string) field(:content, :string) - field(:to, {:array, :string}, default: []) - field(:cc, {:array, :string}, default: []) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) end def cast_and_validate(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex new file mode 100644 index 000000000..0b4c99dc0 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex @@ -0,0 +1,97 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Ecto.Changeset + + @primary_key false + @derive Jason.Encoder + + # Extends from NoteValidator + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:bto, ObjectValidators.Recipients, default: []) + field(:bcc, ObjectValidators.Recipients, default: []) + # TODO: Write type + field(:tag, {:array, :map}, default: []) + field(:type, :string) + + field(:name, :string) + field(:summary, :string) + field(:content, :string) + + field(:context, :string) + # short identifier for PleromaFE to group statuses by context + field(:context_id, :integer) + + # TODO: Remove actor on objects + field(:actor, ObjectValidators.ObjectID) + + field(:attributedTo, ObjectValidators.ObjectID) + field(:published, ObjectValidators.DateTime) + field(:emoji, ObjectValidators.Emoji, default: %{}) + field(:sensitive, :boolean, default: false) + embeds_many(:attachment, AttachmentValidator) + field(:replies_count, :integer, default: 0) + field(:like_count, :integer, default: 0) + field(:announcement_count, :integer, default: 0) + field(:inReplyTo, ObjectValidators.ObjectID) + field(:url, ObjectValidators.Uri) + + field(:likes, {:array, ObjectValidators.ObjectID}, default: []) + field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) + end + + def cast_and_apply(data) do + data + |> cast_data + |> apply_action(:insert) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + def cast_data(data) do + %__MODULE__{} + |> changeset(data) + end + + defp fix(data) do + data + |> CommonFixes.fix_defaults() + |> CommonFixes.fix_attribution() + |> Transmogrifier.fix_emoji() + end + + def changeset(struct, data) do + data = fix(data) + + struct + |> cast(data, __schema__(:fields) -- [:attachment]) + |> cast_embed(:attachment) + end + + def validate_data(data_cng) do + data_cng + |> validate_inclusion(:type, ["Event"]) + |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) + |> CommonValidations.validate_any_presence([:cc, :to]) + |> CommonValidations.validate_fields_match([:actor, :attributedTo]) + |> CommonValidations.validate_actor_presence() + |> CommonValidations.validate_host_match() + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex new file mode 100644 index 000000000..ca2724616 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex @@ -0,0 +1,44 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + + import Ecto.Changeset + import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + + @primary_key false + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:type, :string) + field(:actor, ObjectValidators.ObjectID) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:object, ObjectValidators.ObjectID) + field(:state, :string, default: "pending") + end + + def cast_data(data) do + %__MODULE__{} + |> cast(data, __schema__(:fields)) + end + + def validate_data(cng) do + cng + |> validate_required([:id, :type, :actor, :to, :cc, :object]) + |> validate_inclusion(:type, ["Follow"]) + |> validate_inclusion(:state, ~w{pending reject accept}) + |> validate_actor_presence() + |> validate_actor_presence(field_name: :object) + end + + def cast_and_validate(data) do + data + |> cast_data + |> validate_data + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex index 034f25492..493e4c247 100644 --- a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object - alias Pleroma.Web.ActivityPub.ObjectValidators.Types alias Pleroma.Web.ActivityPub.Utils import Ecto.Changeset @@ -15,13 +15,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) + field(:id, ObjectValidators.ObjectID, primary_key: true) field(:type, :string) - field(:object, Types.ObjectID) - field(:actor, Types.ObjectID) + field(:object, ObjectValidators.ObjectID) + field(:actor, ObjectValidators.ObjectID) field(:context, :string) - field(:to, Types.Recipients, default: []) - field(:cc, Types.Recipients, default: []) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) end def cast_and_validate(data) do @@ -67,7 +67,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do with {[], []} <- {to, cc}, %Object{data: %{"actor" => actor}} <- Object.get_cached_by_ap_id(object), - {:ok, actor} <- Types.ObjectID.cast(actor) do + {:ok, actor} <- ObjectValidators.ObjectID.cast(actor) do cng |> put_change(:to, [actor]) else diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex index 462a5620a..ab4469a59 100644 --- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex @@ -5,44 +5,46 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do use Ecto.Schema - alias Pleroma.Web.ActivityPub.ObjectValidators.Types + alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) - field(:to, {:array, :string}, default: []) - field(:cc, {:array, :string}, default: []) - field(:bto, {:array, :string}, default: []) - field(:bcc, {:array, :string}, default: []) + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:bto, ObjectValidators.Recipients, default: []) + field(:bcc, ObjectValidators.Recipients, default: []) # TODO: Write type field(:tag, {:array, :map}, default: []) field(:type, :string) + + field(:name, :string) + field(:summary, :string) field(:content, :string) + field(:context, :string) - field(:actor, Types.ObjectID) - field(:attributedTo, Types.ObjectID) - field(:summary, :string) - field(:published, Types.DateTime) - # TODO: Write type - field(:emoji, :map, default: %{}) + # short identifier for PleromaFE to group statuses by context + field(:context_id, :integer) + + field(:actor, ObjectValidators.ObjectID) + field(:attributedTo, ObjectValidators.ObjectID) + field(:published, ObjectValidators.DateTime) + field(:emoji, ObjectValidators.Emoji, default: %{}) field(:sensitive, :boolean, default: false) # TODO: Write type field(:attachment, {:array, :map}, default: []) field(:replies_count, :integer, default: 0) field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) - field(:inRepyTo, :string) - field(:uri, Types.Uri) + field(:inReplyTo, ObjectValidators.ObjectID) + field(:url, ObjectValidators.Uri) field(:likes, {:array, :string}, default: []) field(:announcements, {:array, :string}, default: []) - - # see if needed - field(:conversation, :string) - field(:context_id, :string) end def cast_and_validate(data) do @@ -51,7 +53,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do |> validate_data() end + defp fix(data) do + data + |> Transmogrifier.fix_emoji() + end + def cast_data(data) do + data = fix(data) + %__MODULE__{} |> cast(data, __schema__(:fields)) end diff --git a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex new file mode 100644 index 000000000..478b3b5cf --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator do + use Ecto.Schema + + import Ecto.Changeset + + @primary_key false + + embedded_schema do + field(:name, :string) + + embeds_one :replies, Replies, primary_key: false do + field(:totalItems, :integer) + field(:type, :string) + end + + field(:type, :string) + end + + def changeset(struct, data) do + struct + |> cast(data, [:name, :type]) + |> cast_embed(:replies, with: &replies_changeset/2) + |> validate_inclusion(:type, ["Note"]) + |> validate_required([:name, :type]) + end + + def replies_changeset(struct, data) do + struct + |> cast(data, [:totalItems, :type]) + |> validate_inclusion(:type, ["Collection"]) + |> validate_required([:type]) + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex new file mode 100644 index 000000000..934d3c1ea --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -0,0 +1,112 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Ecto.Changeset + + @primary_key false + @derive Jason.Encoder + + # Extends from NoteValidator + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:bto, ObjectValidators.Recipients, default: []) + field(:bcc, ObjectValidators.Recipients, default: []) + # TODO: Write type + field(:tag, {:array, :map}, default: []) + field(:type, :string) + field(:content, :string) + field(:context, :string) + + # TODO: Remove actor on objects + field(:actor, ObjectValidators.ObjectID) + + field(:attributedTo, ObjectValidators.ObjectID) + field(:summary, :string) + field(:published, ObjectValidators.DateTime) + field(:emoji, ObjectValidators.Emoji, default: %{}) + field(:sensitive, :boolean, default: false) + embeds_many(:attachment, AttachmentValidator) + field(:replies_count, :integer, default: 0) + field(:like_count, :integer, default: 0) + field(:announcement_count, :integer, default: 0) + field(:inReplyTo, ObjectValidators.ObjectID) + field(:url, ObjectValidators.Uri) + # short identifier for PleromaFE to group statuses by context + field(:context_id, :integer) + + field(:likes, {:array, :string}, default: []) + field(:announcements, {:array, :string}, default: []) + + field(:closed, ObjectValidators.DateTime) + field(:voters, {:array, ObjectValidators.ObjectID}, default: []) + embeds_many(:anyOf, QuestionOptionsValidator) + embeds_many(:oneOf, QuestionOptionsValidator) + end + + def cast_and_apply(data) do + data + |> cast_data + |> apply_action(:insert) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + def cast_data(data) do + %__MODULE__{} + |> changeset(data) + end + + defp fix_closed(data) do + cond do + is_binary(data["closed"]) -> data + is_binary(data["endTime"]) -> Map.put(data, "closed", data["endTime"]) + true -> Map.drop(data, ["closed"]) + end + end + + defp fix(data) do + data + |> CommonFixes.fix_defaults() + |> CommonFixes.fix_attribution() + |> Transmogrifier.fix_emoji() + |> fix_closed() + end + + def changeset(struct, data) do + data = fix(data) + + struct + |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment]) + |> cast_embed(:attachment) + |> cast_embed(:anyOf) + |> cast_embed(:oneOf) + end + + def validate_data(data_cng) do + data_cng + |> validate_inclusion(:type, ["Question"]) + |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) + |> CommonValidations.validate_any_presence([:cc, :to]) + |> CommonValidations.validate_fields_match([:actor, :attributedTo]) + |> CommonValidations.validate_actor_presence() + |> CommonValidations.validate_any_presence([:oneOf, :anyOf]) + |> CommonValidations.validate_host_match() + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex index d0ba418e8..8cae94467 100644 --- a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator do use Ecto.Schema alias Pleroma.Activity - alias Pleroma.Web.ActivityPub.ObjectValidators.Types + alias Pleroma.EctoType.ActivityPub.ObjectValidators import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @@ -14,12 +14,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator do @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) + field(:id, ObjectValidators.ObjectID, primary_key: true) field(:type, :string) - field(:object, Types.ObjectID) - field(:actor, Types.ObjectID) - field(:to, {:array, :string}, default: []) - field(:cc, {:array, :string}, default: []) + field(:object, ObjectValidators.ObjectID) + field(:actor, ObjectValidators.ObjectID) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) end def cast_and_validate(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex new file mode 100644 index 000000000..b4ba5ede0 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex @@ -0,0 +1,59 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + + import Ecto.Changeset + import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + + @primary_key false + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:type, :string) + field(:actor, ObjectValidators.ObjectID) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + # In this case, we save the full object in this activity instead of just a + # reference, so we can always see what was actually changed by this. + field(:object, :map) + end + + def cast_data(data) do + %__MODULE__{} + |> cast(data, __schema__(:fields)) + end + + def validate_data(cng) do + cng + |> validate_required([:id, :type, :actor, :to, :cc, :object]) + |> validate_inclusion(:type, ["Update"]) + |> validate_actor_presence() + |> validate_updating_rights() + end + + def cast_and_validate(data) do + data + |> cast_data + |> validate_data + end + + # For now we only support updating users, and here the rule is easy: + # object id == actor id + def validate_updating_rights(cng) do + with actor = get_field(cng, :actor), + object = get_field(cng, :object), + {:ok, object_id} <- ObjectValidators.ObjectID.cast(object), + true <- actor == object_id do + cng + else + _e -> + cng + |> add_error(:object, "Can't be updated by this actor") + end + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex index 47e231150..881030f38 100644 --- a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex @@ -1,15 +1,19 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do use Ecto.Schema - alias Pleroma.Web.ActivityPub.ObjectValidators.Types + alias Pleroma.EctoType.ActivityPub.ObjectValidators import Ecto.Changeset @primary_key false embedded_schema do field(:type, :string) - field(:href, Types.Uri) - field(:mediaType, :string) + field(:href, ObjectValidators.Uri) + field(:mediaType, :string, default: "application/octet-stream") end def changeset(struct, data) do diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 6875c47f6..36e325c37 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -52,6 +52,13 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do do_not_federate = meta[:do_not_federate] || !Config.get([:instance, :federating]) if !do_not_federate && local do + activity = + if object = Keyword.get(meta, :object_data) do + %{activity | data: Map.put(activity.data, "object", object)} + else + activity + end + Federator.publish(activity) {:ok, :federated} else diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index b70cbd043..d88f7f3ee 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -49,7 +49,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do """ def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do Logger.debug("Federating #{id} to #{inbox}") - %{host: host, path: path} = URI.parse(inbox) + + uri = URI.parse(inbox) digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64()) @@ -57,8 +58,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do signature = Pleroma.Signature.sign(actor, %{ - "(request-target)": "post #{path}", - host: host, + "(request-target)": "post #{uri.path}", + host: signature_host(uri), "content-length": byte_size(json), digest: digest, date: date @@ -76,8 +77,9 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {"digest", digest} ] ) do - if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since], - do: Instances.set_reachable(inbox) + if not Map.has_key?(params, :unreachable_since) || params[:unreachable_since] do + Instances.set_reachable(inbox) + end result else @@ -96,6 +98,14 @@ defmodule Pleroma.Web.ActivityPub.Publisher do |> publish_one() end + defp signature_host(%URI{port: port, scheme: scheme, host: host}) do + if port == URI.default_port(scheme) do + host + else + "#{host}:#{port}" + end + end + defp should_federate?(inbox, public) do if public do true diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index 484178edd..b65710a94 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -10,25 +10,19 @@ defmodule Pleroma.Web.ActivityPub.Relay do alias Pleroma.Web.CommonAPI require Logger - @relay_nickname "relay" + @nickname "relay" - def get_actor do - actor = - relay_ap_id() - |> User.get_or_create_service_actor_by_ap_id(@relay_nickname) + @spec ap_id() :: String.t() + def ap_id, do: "#{Pleroma.Web.Endpoint.url()}/#{@nickname}" - actor - end - - def relay_ap_id do - "#{Pleroma.Web.Endpoint.url()}/relay" - end + @spec get_actor() :: User.t() | nil + def get_actor, do: User.get_or_create_service_actor_by_ap_id(ap_id(), @nickname) @spec follow(String.t()) :: {:ok, Activity.t()} | {:error, any()} def follow(target_instance) do with %User{} = local_user <- get_actor(), {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance), - {:ok, activity} <- ActivityPub.follow(local_user, target_user) do + {:ok, _, _, activity} <- CommonAPI.follow(local_user, target_user) do Logger.info("relay: followed instance: #{target_instance}; id=#{activity.data["id"]}") {:ok, activity} else @@ -61,34 +55,38 @@ defmodule Pleroma.Web.ActivityPub.Relay do def publish(_), do: {:error, "Not implemented"} - @spec list(boolean()) :: {:ok, [String.t()]} | {:error, any()} - def list(with_not_accepted \\ false) do + @spec list() :: {:ok, [%{actor: String.t(), followed_back: boolean()}]} | {:error, any()} + def list do with %User{} = user <- get_actor() do accepted = user - |> User.following() - |> Enum.map(fn entry -> URI.parse(entry).host end) - |> Enum.uniq() - - list = - if with_not_accepted do - without_accept = - user - |> Pleroma.Activity.following_requests_for_actor() - |> Enum.map(fn a -> URI.parse(a.data["object"]).host <> " (no Accept received)" end) - |> Enum.uniq() + |> following() + |> Enum.map(fn actor -> %{actor: actor, followed_back: true} end) - accepted ++ without_accept - else - accepted - end + without_accept = + user + |> Pleroma.Activity.following_requests_for_actor() + |> Enum.map(fn activity -> %{actor: activity.data["object"], followed_back: false} end) + |> Enum.uniq() - {:ok, list} + {:ok, accepted ++ without_accept} else error -> format_error(error) end end + @spec following() :: [String.t()] + def following do + get_actor() + |> following() + end + + defp following(user) do + user + |> User.following_ap_ids() + |> Enum.uniq() + end + defp format_error({:error, error}), do: format_error(error) defp format_error(error) do diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 1a1cc675c..46a8be767 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -6,20 +6,157 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do collection, and so on. """ alias Pleroma.Activity + alias Pleroma.Activity.Ir.Topics alias Pleroma.Chat alias Pleroma.Chat.MessageReference + alias Pleroma.FollowingRelationship alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Push alias Pleroma.Web.Streamer + alias Pleroma.Workers.BackgroundWorker + + require Logger def handle(object, meta \\ []) + # Task this handles + # - Follows + # - Sends a notification + def handle( + %{ + data: %{ + "actor" => actor, + "type" => "Accept", + "object" => follow_activity_id + } + } = object, + meta + ) do + with %Activity{actor: follower_id} = follow_activity <- + Activity.get_by_ap_id(follow_activity_id), + %User{} = followed <- User.get_cached_by_ap_id(actor), + %User{} = follower <- User.get_cached_by_ap_id(follower_id), + {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"), + {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_accept) do + Notification.update_notification_type(followed, follow_activity) + User.update_follower_count(followed) + User.update_following_count(follower) + end + + {:ok, object, meta} + end + + # Task this handles + # - Rejects all existing follow activities for this person + # - Updates the follow state + # - Dismisses notification + def handle( + %{ + data: %{ + "actor" => actor, + "type" => "Reject", + "object" => follow_activity_id + } + } = object, + meta + ) do + with %Activity{actor: follower_id} = follow_activity <- + Activity.get_by_ap_id(follow_activity_id), + %User{} = followed <- User.get_cached_by_ap_id(actor), + %User{} = follower <- User.get_cached_by_ap_id(follower_id), + {:ok, _follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject") do + FollowingRelationship.update(follower, followed, :follow_reject) + Notification.dismiss(follow_activity) + end + + {:ok, object, meta} + end + + # Tasks this handle + # - Follows if possible + # - Sends a notification + # - Generates accept or reject if appropriate + def handle( + %{ + data: %{ + "id" => follow_id, + "type" => "Follow", + "object" => followed_user, + "actor" => following_user + } + } = object, + meta + ) do + with %User{} = follower <- User.get_cached_by_ap_id(following_user), + %User{} = followed <- User.get_cached_by_ap_id(followed_user), + {_, {:ok, _}, _, _} <- + {:following, User.follow(follower, followed, :follow_pending), follower, followed} do + if followed.local && !followed.locked do + {:ok, accept_data, _} = Builder.accept(followed, object) + {:ok, _activity, _} = Pipeline.common_pipeline(accept_data, local: true) + end + else + {:following, {:error, _}, _follower, followed} -> + {:ok, reject_data, _} = Builder.reject(followed, object) + {:ok, _activity, _} = Pipeline.common_pipeline(reject_data, local: true) + + _ -> + nil + end + + {:ok, notifications} = Notification.create_notifications(object, do_send: false) + + meta = + meta + |> add_notifications(notifications) + + updated_object = Activity.get_by_ap_id(follow_id) + + {:ok, updated_object, meta} + end + + # Tasks this handles: + # - Unfollow and block + def handle( + %{data: %{"type" => "Block", "object" => blocked_user, "actor" => blocking_user}} = + object, + meta + ) do + with %User{} = blocker <- User.get_cached_by_ap_id(blocking_user), + %User{} = blocked <- User.get_cached_by_ap_id(blocked_user) do + User.block(blocker, blocked) + end + + {:ok, object, meta} + end + + # Tasks this handles: + # - Update the user + # + # For a local user, we also get a changeset with the full information, so we + # can update non-federating, non-activitypub settings as well. + def handle(%{data: %{"type" => "Update", "object" => updated_object}} = object, meta) do + if changeset = Keyword.get(meta, :user_update_changeset) do + changeset + |> User.update_and_set_cache() + else + {:ok, new_user_data} = ActivityPub.user_data_from_user_object(updated_object) + + User.get_by_ap_id(updated_object["id"]) + |> User.remote_user_changeset(new_user_data) + |> User.update_and_set_cache() + end + + {:ok, object, meta} + end + # Tasks this handles: # - Add like to object # - Set up notification @@ -35,10 +172,22 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # Tasks this handles # - Actually create object # - Rollback if we couldn't create it + # - Increase the user note count + # - Increase the reply count + # - Increase replies count + # - Set up ActivityExpiration # - Set up notifications def handle(%{data: %{"type" => "Create"}} = activity, meta) do - with {:ok, _object, meta} <- handle_object_creation(meta[:object_data], meta) do + with {:ok, object, meta} <- handle_object_creation(meta[:object_data], meta), + %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do {:ok, notifications} = Notification.create_notifications(activity, do_send: false) + {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object) + + if in_reply_to = object.data["inReplyTo"] do + Object.increase_replies_count(in_reply_to) + end + + BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id}) meta = meta @@ -62,7 +211,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if !User.is_internal_user?(user) do Notification.create_notifications(object) - ActivityPub.stream_out(object) + + object + |> Topics.get_activity_topics() + |> Streamer.stream(object) end {:ok, object, meta} @@ -96,13 +248,15 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # - Stream out the activity def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, meta) do deleted_object = - Object.normalize(deleted_object, false) || User.get_cached_by_ap_id(deleted_object) + Object.normalize(deleted_object, false) || + User.get_cached_by_ap_id(deleted_object) result = case deleted_object do %Object{} -> with {:ok, deleted_object, activity} <- Object.delete(deleted_object), - %User{} = user <- User.get_cached_by_ap_id(deleted_object.data["actor"]) do + {_, actor} when is_binary(actor) <- {:actor, deleted_object.data["actor"]}, + %User{} = user <- User.get_cached_by_ap_id(actor) do User.remove_pinnned_activity(user, activity) {:ok, user} = ActivityPub.decrease_note_count_if_public(user, deleted_object) @@ -116,6 +270,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do ActivityPub.stream_out(object) ActivityPub.stream_out_participations(deleted_object, user) :ok + else + {:actor, _} -> + Logger.error("The object doesn't have an actor: #{inspect(deleted_object)}") + :no_object_actor end %User{} -> @@ -165,19 +323,44 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end end + def handle_object_creation(%{"type" => "Answer"} = object_map, meta) do + with {:ok, object, meta} <- Pipeline.common_pipeline(object_map, meta) do + Object.increase_vote_count( + object.data["inReplyTo"], + object.data["name"], + object.data["actor"] + ) + + {:ok, object, meta} + end + end + + def handle_object_creation(%{"type" => objtype} = object, meta) + when objtype in ~w[Audio Question Event] do + with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do + {:ok, object, meta} + end + end + # Nothing to do - def handle_object_creation(object) do - {:ok, object} + def handle_object_creation(object, meta) do + {:ok, object, meta} end - def handle_undoing(%{data: %{"type" => "Like"}} = object) do - with %Object{} = liked_object <- Object.get_by_ap_id(object.data["object"]), - {:ok, _} <- Utils.remove_like_from_object(object, liked_object), - {:ok, _} <- Repo.delete(object) do - :ok + defp undo_like(nil, object), do: delete_object(object) + + defp undo_like(%Object{} = liked_object, object) do + with {:ok, _} <- Utils.remove_like_from_object(object, liked_object) do + delete_object(object) end end + def handle_undoing(%{data: %{"type" => "Like"}} = object) do + object.data["object"] + |> Object.get_by_ap_id() + |> undo_like(object) + end + def handle_undoing(%{data: %{"type" => "EmojiReact"}} = object) do with %Object{} = reacted_object <- Object.get_by_ap_id(object.data["object"]), {:ok, _} <- Utils.remove_emoji_reaction_from_object(object, reacted_object), @@ -207,6 +390,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do def handle_undoing(object), do: {:error, ["don't know how to handle", object]} + @spec delete_object(Object.t()) :: :ok | {:error, Ecto.Changeset.t()} + defp delete_object(object) do + with {:ok, _} <- Repo.delete(object), do: :ok + end + defp send_notifications(meta) do Keyword.get(meta, :notifications, []) |> Enum.each(fn notification -> diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 985921aa0..af4384213 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -8,9 +8,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do """ alias Pleroma.Activity alias Pleroma.EarmarkRenderer - alias Pleroma.FollowingRelationship + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Maps - alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Object.Containment alias Pleroma.Repo @@ -18,7 +17,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.ObjectValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.Types alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility @@ -62,15 +60,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_summary(object), do: Map.put(object, "summary", "") def fix_addressing_list(map, field) do + addrs = map[field] + cond do - is_binary(map[field]) -> - Map.put(map, field, [map[field]]) + is_list(addrs) -> + Map.put(map, field, Enum.filter(addrs, &is_binary/1)) - is_nil(map[field]) -> - Map.put(map, field, []) + is_binary(addrs) -> + Map.put(map, field, [addrs]) true -> - map + Map.put(map, field, []) end end @@ -155,7 +155,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def fix_actor(%{"attributedTo" => actor} = object) do - Map.put(object, "actor", Containment.get_actor(%{"actor" => actor})) + actor = Containment.get_actor(%{"actor" => actor}) + + # TODO: Remove actor field for Objects + object + |> Map.put("actor", actor) + |> Map.put("attributedTo", actor) end def fix_in_reply_to(object, options \\ []) @@ -163,7 +168,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) when not is_nil(in_reply_to) do in_reply_to_id = prepare_in_reply_to(in_reply_to) - object = Map.put(object, "inReplyToAtomUri", in_reply_to_id) depth = (options[:depth] || 0) + 1 if Federator.allowed_thread_distance?(depth) do @@ -171,12 +175,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do %Activity{} <- Activity.get_create_by_object_ap_id(replied_object.data["id"]) do object |> Map.put("inReplyTo", replied_object.data["id"]) - |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id) - |> Map.put("conversation", replied_object.data["context"] || object["conversation"]) |> Map.put("context", replied_object.data["context"] || object["conversation"]) + |> Map.drop(["conversation", "inReplyToAtomUri"]) else e -> - Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") + Logger.warn("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") object end else @@ -207,7 +210,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do object |> Map.put("context", context) - |> Map.put("conversation", context) + |> Map.drop(["conversation"]) end def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do @@ -233,18 +236,28 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do is_map(url) && is_binary(url["href"]) -> url["href"] is_binary(data["url"]) -> data["url"] is_binary(data["href"]) -> data["href"] + true -> nil end - attachment_url = - %{"href" => href} - |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("type", Map.get(url || %{}, "type")) + if href do + attachment_url = + %{ + "href" => href, + "type" => Map.get(url || %{}, "type", "Link") + } + |> Maps.put_if_present("mediaType", media_type) - %{"url" => [attachment_url]} - |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("type", data["type"]) - |> Maps.put_if_present("name", data["name"]) + %{ + "url" => [attachment_url], + "type" => data["type"] || "Document" + } + |> Maps.put_if_present("mediaType", media_type) + |> Maps.put_if_present("name", data["name"]) + else + nil + end end) + |> Enum.filter(& &1) Map.put(object, "attachment", attachments) end @@ -261,14 +274,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Map.put(object, "url", url["href"]) end - def fix_url(%{"type" => object_type, "url" => url} = object) - when object_type in ["Video", "Audio"] and is_list(url) do - first_element = Enum.at(url, 0) + def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do + attachment = + Enum.find(url, fn x -> + media_type = x["mediaType"] || x["mimeType"] || "" + + is_map(x) and String.starts_with?(media_type, "video/") + end) - link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end) + link_element = + Enum.find(url, fn x -> is_map(x) and (x["mediaType"] || x["mimeType"]) == "text/html" end) object - |> Map.put("attachment", [first_element]) + |> Map.put("attachment", [attachment]) |> Map.put("url", link_element["href"]) end @@ -298,9 +316,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Map.put(mapping, name, data["icon"]["url"]) end) - # we merge mastodon and pleroma emoji into a single mapping, to allow for both wire formats - emoji = Map.merge(object["emoji"] || %{}, emoji) - Map.put(object, "emoji", emoji) end @@ -368,32 +383,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do defp fix_content(object), do: object - defp mastodon_follow_hack(%{"id" => id, "actor" => follower_id}, followed) do - with true <- id =~ "follows", - %User{local: true} = follower <- User.get_cached_by_ap_id(follower_id), - %Activity{} = activity <- Utils.fetch_latest_follow(follower, followed) do - {:ok, activity} - else - _ -> {:error, nil} - end - end - - defp mastodon_follow_hack(_, _), do: {:error, nil} - - defp get_follow_activity(follow_object, followed) do - with object_id when not is_nil(object_id) <- Utils.get_ap_id(follow_object), - {_, %Activity{} = activity} <- {:activity, Activity.get_by_ap_id(object_id)} do - {:ok, activity} - else - # Can't find the activity. This might a Mastodon 2.3 "Accept" - {:activity, nil} -> - mastodon_follow_hack(follow_object, followed) - - _ -> - {:error, nil} - end - end - # Reduce the object list to find the reported user. defp get_reported(objects) do Enum.reduce_while(objects, nil, fn ap_id, _ -> @@ -405,6 +394,29 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end) end + # Compatibility wrapper for Mastodon votes + defp handle_create(%{"object" => %{"type" => "Answer"}} = data, _user) do + handle_incoming(data) + end + + defp handle_create(%{"object" => object} = data, user) do + %{ + to: data["to"], + object: object, + actor: user, + context: object["context"], + local: false, + published: data["published"], + additional: + Map.take(data, [ + "cc", + "directMessage", + "id" + ]) + } + |> ActivityPub.create() + end + def handle_incoming(data, options \\ []) # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them @@ -443,33 +455,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, options ) - when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer", "Audio"] do + when objtype in ~w{Article Note Video Page} do actor = Containment.get_actor(data) - data = - Map.put(data, "actor", actor) - |> fix_addressing - with nil <- Activity.get_create_by_object_ap_id(object["id"]), - {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do - object = fix_object(object, options) - - params = %{ - to: data["to"], - object: object, - actor: user, - context: object["conversation"], - local: false, - published: data["published"], - additional: - Map.take(data, [ - "cc", - "directMessage", - "id" - ]) - } + {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(actor) do + data = + data + |> Map.put("object", fix_object(object, options)) + |> Map.put("actor", actor) + |> fix_addressing() - with {:ok, created_activity} <- ActivityPub.create(params) do + with {:ok, created_activity} <- handle_create(data, user) do reply_depth = (options[:depth] || 0) + 1 if Federator.allowed_thread_distance?(reply_depth) do @@ -520,120 +517,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def handle_incoming( - %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data, - _options - ) do - with %User{local: true} = followed <- - User.get_cached_by_ap_id(Containment.get_actor(%{"actor" => followed})), - {:ok, %User{} = follower} <- - User.get_or_fetch_by_ap_id(Containment.get_actor(%{"actor" => follower})), - {:ok, activity} <- - ActivityPub.follow(follower, followed, id, false, skip_notify_and_stream: true) do - with deny_follow_blocked <- Pleroma.Config.get([:user, :deny_follow_blocked]), - {_, false} <- {:user_blocked, User.blocks?(followed, follower) && deny_follow_blocked}, - {_, false} <- {:user_locked, User.locked?(followed)}, - {_, {:ok, follower}} <- {:follow, User.follow(follower, followed)}, - {_, {:ok, _}} <- - {:follow_state_update, Utils.update_follow_state_for_all(activity, "accept")}, - {:ok, _relationship} <- - FollowingRelationship.update(follower, followed, :follow_accept) do - ActivityPub.accept(%{ - to: [follower.ap_id], - actor: followed, - object: data, - local: true - }) - else - {:user_blocked, true} -> - {:ok, _} = Utils.update_follow_state_for_all(activity, "reject") - {:ok, _relationship} = FollowingRelationship.update(follower, followed, :follow_reject) - - ActivityPub.reject(%{ - to: [follower.ap_id], - actor: followed, - object: data, - local: true - }) - - {:follow, {:error, _}} -> - {:ok, _} = Utils.update_follow_state_for_all(activity, "reject") - {:ok, _relationship} = FollowingRelationship.update(follower, followed, :follow_reject) - - ActivityPub.reject(%{ - to: [follower.ap_id], - actor: followed, - object: data, - local: true - }) - - {:user_locked, true} -> - {:ok, _relationship} = FollowingRelationship.update(follower, followed, :follow_pending) - :noop - end - - ActivityPub.notify_and_stream(activity) - {:ok, activity} - else - _e -> - :error - end - end - - def handle_incoming( - %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => id} = data, - _options - ) do - with actor <- Containment.get_actor(data), - {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor), - {:ok, follow_activity} <- get_follow_activity(follow_object, followed), - {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"), - %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]), - {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_accept) do - User.update_follower_count(followed) - User.update_following_count(follower) - - Notification.update_notification_type(followed, follow_activity) - - ActivityPub.accept(%{ - to: follow_activity.data["to"], - type: "Accept", - actor: followed, - object: follow_activity.data["id"], - local: false, - activity_id: id - }) - else - _e -> - :error - end - end - - def handle_incoming( - %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => id} = data, - _options - ) do - with actor <- Containment.get_actor(data), - {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor), - {:ok, follow_activity} <- get_follow_activity(follow_object, followed), - {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject"), - %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]), - {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_reject), - {:ok, activity} <- - ActivityPub.reject(%{ - to: follow_activity.data["to"], - type: "Reject", - actor: followed, - object: follow_activity.data["id"], - local: false, - activity_id: id - }) do - {:ok, activity} - else - _e -> :error - end - end - @misskey_reactions %{ "like" => "👍", "love" => "❤️", @@ -663,9 +546,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Create", "object" => %{"type" => "ChatMessage"}} = data, + %{"type" => "Create", "object" => %{"type" => objtype}} = data, _options - ) do + ) + when objtype in ~w{Question Answer ChatMessage Audio Event} do with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do {:ok, activity} @@ -673,7 +557,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming(%{"type" => type} = data, _options) - when type in ["Like", "EmojiReact", "Announce"] do + when type in ~w{Like EmojiReact Announce} do with :ok <- ObjectValidator.fetch_actor_and_object(data), {:ok, activity, _meta} <- Pipeline.common_pipeline(data, local: false) do @@ -684,35 +568,14 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Update", "object" => %{"type" => object_type} = object, "actor" => actor_id} = - data, + %{"type" => type} = data, _options ) - when object_type in [ - "Person", - "Application", - "Service", - "Organization" - ] do - with %User{ap_id: ^actor_id} = actor <- User.get_cached_by_ap_id(object["id"]) do - {:ok, new_user_data} = ActivityPub.user_data_from_user_object(object) - - actor - |> User.remote_user_changeset(new_user_data) - |> User.update_and_set_cache() - - ActivityPub.update(%{ - local: false, - to: data["to"] || [], - cc: data["cc"] || [], - object: object, - actor: actor_id, - activity_id: data["id"] - }) - else - e -> - Logger.error(e) - :error + when type in ~w{Update Block Follow Accept Reject} do + with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), + {:ok, activity, _} <- + Pipeline.common_pipeline(data, local: false) do + {:ok, activity} end end @@ -720,12 +583,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do %{"type" => "Delete"} = data, _options ) do - with {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do + with {:ok, activity, _} <- + Pipeline.common_pipeline(data, local: false) do {:ok, activity} else {:error, {:validate_object, _}} = e -> # Check if we have a create activity for this - with {:ok, object_id} <- Types.ObjectID.cast(data["object"]), + with {:ok, object_id} <- ObjectValidators.ObjectID.cast(data["object"]), %Activity{data: %{"actor" => actor}} <- Activity.create_by_object_ap_id(object_id) |> Repo.one(), # We have one, insert a tombstone and retry @@ -789,21 +653,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data, - _options - ) do - with %User{local: true} = blocked = User.get_cached_by_ap_id(blocked), - {:ok, %User{} = blocker} = User.get_or_fetch_by_ap_id(blocker), - {:ok, activity} <- ActivityPub.block(blocker, blocked, id, false) do - User.unfollow(blocker, blocked) - User.block(blocker, blocked) - {:ok, activity} - else - _e -> :error - end - end - - def handle_incoming( %{ "type" => "Move", "actor" => origin_actor, diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index dfae602df..713b0ca1f 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -719,15 +719,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do case Activity.get_by_ap_id_with_object(id) do %Activity{} = activity -> + activity_actor = User.get_by_ap_id(activity.object.data["actor"]) + %{ "type" => "Note", "id" => activity.data["id"], "content" => activity.object.data["content"], "published" => activity.object.data["published"], "actor" => - AccountView.render("show.json", %{ - user: User.get_by_ap_id(activity.object.data["actor"]) - }) + AccountView.render( + "show.json", + %{user: activity_actor, skip_visibility_check: true} + ) } _ -> diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 4a02b09a1..3a4564912 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -81,6 +81,15 @@ defmodule Pleroma.Web.ActivityPub.UserView do fields = Enum.map(user.fields, &Map.put(&1, "type", "PropertyValue")) + capabilities = + if is_boolean(user.accepts_chat_messages) do + %{ + "acceptsChatMessages" => user.accepts_chat_messages + } + else + %{} + end + %{ "id" => user.ap_id, "type" => user.actor_type, @@ -101,7 +110,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do "endpoints" => endpoints, "attachment" => fields, "tag" => emoji_tags, - "discoverable" => user.discoverable + "discoverable" => user.discoverable, + "capabilities" => capabilities } |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user)) diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index 453a6842e..5c349bb7a 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -47,6 +47,10 @@ defmodule Pleroma.Web.ActivityPub.Visibility do @spec visible_for_user?(Activity.t(), User.t() | nil) :: boolean() def visible_for_user?(%{actor: ap_id}, %User{ap_id: ap_id}), do: true + def visible_for_user?(nil, _), do: false + + def visible_for_user?(%{data: %{"listMessage" => _}}, nil), do: false + def visible_for_user?(%{data: %{"listMessage" => list_ap_id}} = activity, %User{} = user) do user.ap_id in activity.data["to"] || list_ap_id @@ -54,15 +58,10 @@ defmodule Pleroma.Web.ActivityPub.Visibility do |> Pleroma.List.member?(user) end - def visible_for_user?(%{data: %{"listMessage" => _}}, nil), do: false - def visible_for_user?(%{local: local} = activity, nil) do - cfg_key = - if local, - do: :local, - else: :remote + cfg_key = if local, do: :local, else: :remote - if Pleroma.Config.get([:restrict_unauthenticated, :activities, cfg_key]), + if Pleroma.Config.restrict_unauthenticated_access?(:activities, cfg_key), do: false, else: is_public?(activity) end diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index 5cbf0dd4f..f5e4d49f9 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -44,6 +44,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do :user_toggle_activation, :user_activate, :user_deactivate, + :user_approve, :tag_users, :untag_users, :right_add, @@ -111,8 +112,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do action: "delete" }) - conn - |> json(nicknames) + json(conn, nicknames) end def user_follow(%{assigns: %{user: admin}} = conn, %{ @@ -131,8 +131,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do }) end - conn - |> json("ok") + json(conn, "ok") end def user_unfollow(%{assigns: %{user: admin}} = conn, %{ @@ -151,8 +150,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do }) end - conn - |> json("ok") + json(conn, "ok") end def users_create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do @@ -191,8 +189,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do action: "create" }) - conn - |> json(res) + json(conn, res) {:error, id, changeset, _} -> res = @@ -210,8 +207,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end end - def user_show(conn, %{"nickname" => nickname}) do - with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do + def user_show(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do conn |> put_view(AccountView) |> render("show.json", %{user: user}) @@ -237,11 +234,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> render("index.json", %{activities: activities, as: :activity}) end - def list_user_statuses(conn, %{"nickname" => nickname} = params) do + def list_user_statuses(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} = params) do with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true godmode = params["godmode"] == "true" || params["godmode"] == true - with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do + with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do {_, page_size} = page_params(params) activities = @@ -307,6 +304,21 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> render("index.json", %{users: Keyword.values(updated_users)}) end + def user_approve(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + users = Enum.map(nicknames, &User.get_cached_by_nickname/1) + {:ok, updated_users} = User.approve(users) + + ModerationLog.insert_log(%{ + actor: admin, + subject: users, + action: "approve" + }) + + conn + |> put_view(AccountView) + |> render("index.json", %{users: updated_users}) + end + def tag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do with {:ok, _} <- User.tag(nicknames, tags) do ModerationLog.insert_log(%{ @@ -349,12 +361,16 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do with {:ok, users, count} <- Search.user(Map.merge(search_params, filters)) do json( conn, - AccountView.render("index.json", users: users, count: count, page_size: page_size) + AccountView.render("index.json", + users: users, + count: count, + page_size: page_size + ) ) end end - @filters ~w(local external active deactivated is_admin is_moderator) + @filters ~w(local external active deactivated need_approval is_admin is_moderator) @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{} defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{} @@ -363,8 +379,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do filters |> String.split(",") |> Enum.filter(&Enum.member?(@filters, &1)) - |> Enum.map(&String.to_atom(&1)) - |> Enum.into(%{}, &{&1, true}) + |> Map.new(&{String.to_existing_atom(&1), true}) end def right_add_multiple(%{assigns: %{user: admin}} = conn, %{ @@ -530,7 +545,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do @doc "Show a given user's credentials" def show_user_credentials(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do - with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do + with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do conn |> put_view(AccountView) |> render("credentials.json", %{user: user, for: admin}) @@ -568,10 +583,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do {:error, changeset} -> errors = Map.new(changeset.errors, fn {key, {error, _}} -> {key, error} end) - json(conn, %{errors: errors}) + {:errors, errors} _ -> - json(conn, %{error: "Unable to update user."}) + {:error, :not_found} end end @@ -616,42 +631,36 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do def reload_emoji(conn, _params) do Pleroma.Emoji.reload() - conn |> json("ok") + json(conn, "ok") end def confirm_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do - users = nicknames |> Enum.map(&User.get_cached_by_nickname/1) + users = Enum.map(nicknames, &User.get_cached_by_nickname/1) User.toggle_confirmation(users) - ModerationLog.insert_log(%{ - actor: admin, - subject: users, - action: "confirm_email" - }) + ModerationLog.insert_log(%{actor: admin, subject: users, action: "confirm_email"}) - conn |> json("") + json(conn, "") end def resend_confirmation_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do - users = nicknames |> Enum.map(&User.get_cached_by_nickname/1) - - User.try_send_confirmation_email(users) + users = + Enum.map(nicknames, fn nickname -> + nickname + |> User.get_cached_by_nickname() + |> User.send_confirmation_email() + end) - ModerationLog.insert_log(%{ - actor: admin, - subject: users, - action: "resend_confirmation_email" - }) + ModerationLog.insert_log(%{actor: admin, subject: users, action: "resend_confirmation_email"}) - conn |> json("") + json(conn, "") end - def stats(conn, _) do - count = Stats.get_status_visibility_count() + def stats(conn, params) do + counters = Stats.get_status_visibility_count(params["instance"]) - conn - |> json(%{"status_visibility" => count}) + json(conn, %{"status_visibility" => counters}) end defp page_params(params) do diff --git a/lib/pleroma/web/admin_api/controllers/config_controller.ex b/lib/pleroma/web/admin_api/controllers/config_controller.ex index d6e2019bc..0df13007f 100644 --- a/lib/pleroma/web/admin_api/controllers/config_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/config_controller.ex @@ -9,8 +9,6 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do alias Pleroma.ConfigDB alias Pleroma.Plugs.OAuthScopesPlug - @descriptions Pleroma.Docs.JSON.compile() - plug(Pleroma.Web.ApiSpec.CastAndValidate) plug(OAuthScopesPlug, %{scopes: ["write"], admin: true} when action == :update) @@ -25,7 +23,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.ConfigOperation def descriptions(conn, _params) do - descriptions = Enum.filter(@descriptions, &whitelisted_config?/1) + descriptions = Enum.filter(Pleroma.Docs.JSON.compiled_descriptions(), &whitelisted_config?/1) json(conn, descriptions) end @@ -33,7 +31,11 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do def show(conn, %{only_db: true}) do with :ok <- configurable_from_database() do configs = Pleroma.Repo.all(ConfigDB) - render(conn, "index.json", %{configs: configs}) + + render(conn, "index.json", %{ + configs: configs, + need_reboot: Restarter.Pleroma.need_reboot?() + }) end end @@ -61,17 +63,20 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do value end - %{ - group: ConfigDB.convert(group), - key: ConfigDB.convert(key), - value: ConfigDB.convert(merged_value) + %ConfigDB{ + group: group, + key: key, + value: merged_value } |> Pleroma.Maps.put_if_present(:db, db) end) end) |> List.flatten() - json(conn, %{configs: merged, need_reboot: Restarter.Pleroma.need_reboot?()}) + render(conn, "index.json", %{ + configs: merged, + need_reboot: Restarter.Pleroma.need_reboot?() + }) end end @@ -91,24 +96,17 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do {deleted, updated} = results - |> Enum.map(fn {:ok, config} -> - Map.put(config, :db, ConfigDB.get_db_keys(config)) - end) - |> Enum.split_with(fn config -> - Ecto.get_meta(config, :state) == :deleted + |> Enum.map(fn {:ok, %{key: key, value: value} = config} -> + Map.put(config, :db, ConfigDB.get_db_keys(value, key)) end) + |> Enum.split_with(&(Ecto.get_meta(&1, :state) == :deleted)) Config.TransferTask.load_and_update_env(deleted, false) if not Restarter.Pleroma.need_reboot?() do changed_reboot_settings? = (updated ++ deleted) - |> Enum.any?(fn config -> - group = ConfigDB.from_string(config.group) - key = ConfigDB.from_string(config.key) - value = ConfigDB.from_binary(config.value) - Config.TransferTask.pleroma_need_restart?(group, key, value) - end) + |> Enum.any?(&Config.TransferTask.pleroma_need_restart?(&1.group, &1.key, &1.value)) if changed_reboot_settings?, do: Restarter.Pleroma.need_reboot() end diff --git a/lib/pleroma/web/admin_api/controllers/fallback_controller.ex b/lib/pleroma/web/admin_api/controllers/fallback_controller.ex index 82965936d..34d90db07 100644 --- a/lib/pleroma/web/admin_api/controllers/fallback_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/fallback_controller.ex @@ -17,6 +17,12 @@ defmodule Pleroma.Web.AdminAPI.FallbackController do |> json(%{error: reason}) end + def call(conn, {:errors, errors}) do + conn + |> put_status(:bad_request) + |> json(%{errors: errors}) + end + def call(conn, {:param_cast, _}) do conn |> put_status(:bad_request) diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex new file mode 100644 index 000000000..131e22d78 --- /dev/null +++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex @@ -0,0 +1,74 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do + use Pleroma.Web, :controller + + alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.Web.ApiSpec.Admin, as: Spec + alias Pleroma.Web.MediaProxy + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["read:media_proxy_caches"], admin: true} when action in [:index] + ) + + plug( + OAuthScopesPlug, + %{scopes: ["write:media_proxy_caches"], admin: true} when action in [:purge, :delete] + ) + + action_fallback(Pleroma.Web.AdminAPI.FallbackController) + + defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation + + def index(%{assigns: %{user: _}} = conn, params) do + entries = fetch_entries(params) + urls = paginate_entries(entries, params.page, params.page_size) + + render(conn, "index.json", + urls: urls, + page_size: params.page_size, + count: length(entries) + ) + end + + defp fetch_entries(params) do + MediaProxy.cache_table() + |> Cachex.stream!(Cachex.Query.create(true, :key)) + |> filter_entries(params[:query]) + end + + defp filter_entries(stream, query) when is_binary(query) do + regex = ~r/#{query}/i + + stream + |> Enum.filter(fn url -> String.match?(url, regex) end) + |> Enum.to_list() + end + + defp filter_entries(stream, _), do: Enum.to_list(stream) + + defp paginate_entries(entries, page, page_size) do + offset = page_size * (page - 1) + Enum.slice(entries, offset, page_size) + end + + def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do + MediaProxy.remove_from_banned_urls(urls) + json(conn, %{}) + end + + def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do + MediaProxy.Invalidation.purge(urls) + + if ban do + MediaProxy.put_in_banned_urls(urls) + end + + json(conn, %{}) + end +end diff --git a/lib/pleroma/web/admin_api/controllers/relay_controller.ex b/lib/pleroma/web/admin_api/controllers/relay_controller.ex index cf9f3a14b..95d06dde7 100644 --- a/lib/pleroma/web/admin_api/controllers/relay_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/relay_controller.ex @@ -39,7 +39,7 @@ defmodule Pleroma.Web.AdminAPI.RelayController do target: target }) - json(conn, target) + json(conn, %{actor: target, followed_back: target in Relay.following()}) else _ -> conn diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index e1e929632..9c477feab 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -77,7 +77,10 @@ defmodule Pleroma.Web.AdminAPI.AccountView do "roles" => User.roles(user), "tags" => user.tags || [], "confirmation_pending" => user.confirmation_pending, - "url" => user.uri || user.ap_id + "approval_pending" => user.approval_pending, + "url" => user.uri || user.ap_id, + "registration_reason" => user.registration_reason, + "actor_type" => user.actor_type } end @@ -105,7 +108,7 @@ defmodule Pleroma.Web.AdminAPI.AccountView do end def merge_account_views(%User{} = user) do - MastodonAPI.AccountView.render("show.json", %{user: user}) + MastodonAPI.AccountView.render("show.json", %{user: user, skip_visibility_check: true}) |> Map.merge(AdminAPI.AccountView.render("show.json", %{user: user})) end diff --git a/lib/pleroma/web/admin_api/views/config_view.ex b/lib/pleroma/web/admin_api/views/config_view.ex index 587ef760e..d2d8b5907 100644 --- a/lib/pleroma/web/admin_api/views/config_view.ex +++ b/lib/pleroma/web/admin_api/views/config_view.ex @@ -5,23 +5,20 @@ defmodule Pleroma.Web.AdminAPI.ConfigView do use Pleroma.Web, :view + alias Pleroma.ConfigDB + def render("index.json", %{configs: configs} = params) do - map = %{ - configs: render_many(configs, __MODULE__, "show.json", as: :config) + %{ + configs: render_many(configs, __MODULE__, "show.json", as: :config), + need_reboot: params[:need_reboot] } - - if params[:need_reboot] do - Map.put(map, :need_reboot, true) - else - map - end end def render("show.json", %{config: config}) do map = %{ - key: config.key, - group: config.group, - value: Pleroma.ConfigDB.from_binary_with_convert(config.value) + key: ConfigDB.to_json_types(config.key), + group: ConfigDB.to_json_types(config.group), + value: ConfigDB.to_json_types(config.value) } if config.db != [] do diff --git a/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex new file mode 100644 index 000000000..a803bda0b --- /dev/null +++ b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex @@ -0,0 +1,15 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do + use Pleroma.Web, :view + + def render("index.json", %{urls: urls, page_size: page_size, count: count}) do + %{ + urls: urls, + count: count, + page_size: page_size + } + end +end diff --git a/lib/pleroma/web/api_spec/cast_and_validate.ex b/lib/pleroma/web/api_spec/cast_and_validate.ex index bd9026237..fbfc27d6f 100644 --- a/lib/pleroma/web/api_spec/cast_and_validate.ex +++ b/lib/pleroma/web/api_spec/cast_and_validate.ex @@ -40,7 +40,7 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do |> List.first() _ -> - nil + "application/json" end private_data = Map.put(private_data, :operation_id, operation_id) diff --git a/lib/pleroma/web/api_spec/helpers.ex b/lib/pleroma/web/api_spec/helpers.ex index a9cfe0fed..2a7f1a706 100644 --- a/lib/pleroma/web/api_spec/helpers.ex +++ b/lib/pleroma/web/api_spec/helpers.ex @@ -29,6 +29,10 @@ defmodule Pleroma.Web.ApiSpec.Helpers do } end + def admin_api_params do + [Operation.parameter(:admin_token, :query, :string, "Allows authorization via admin token.")] + end + def pagination_params do [ Operation.parameter(:max_id, :query, :string, "Return items older than this ID"), @@ -40,6 +44,12 @@ defmodule Pleroma.Web.ApiSpec.Helpers do "Return the newest items newer than this ID" ), Operation.parameter( + :offset, + :query, + %Schema{type: :integer, default: 0}, + "Return items past this number of items" + ), + Operation.parameter( :limit, :query, %Schema{type: :integer, default: 20}, diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 20572f8ea..aaebc9b5c 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -61,7 +61,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do description: "Update the user's display and preferences.", operationId: "AccountController.update_credentials", security: [%{"oAuth" => ["write:accounts"]}], - requestBody: request_body("Parameters", update_creadentials_request(), required: true), + requestBody: request_body("Parameters", update_credentials_request(), required: true), responses: %{ 200 => Operation.response("Account", "application/json", Account), 403 => Operation.response("Error", "application/json", ApiError) @@ -102,6 +102,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do parameters: [%Reference{"$ref": "#/components/parameters/accountIdOrNickname"}], responses: %{ 200 => Operation.response("Account", "application/json", Account), + 401 => Operation.response("Error", "application/json", ApiError), 404 => Operation.response("Error", "application/json", ApiError) } } @@ -142,6 +143,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do ] ++ pagination_params(), responses: %{ 200 => Operation.response("Statuses", "application/json", array_of_statuses()), + 401 => Operation.response("Error", "application/json", ApiError), 404 => Operation.response("Error", "application/json", ApiError) } } @@ -157,6 +159,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do "Accounts which follow the given account, if network is not hidden by the account owner.", parameters: [ %Reference{"$ref": "#/components/parameters/accountIdOrNickname"}, + Operation.parameter(:id, :query, :string, "ID of the resource owner"), with_relationships_param() | pagination_params() ], responses: %{ @@ -175,6 +178,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do "Accounts which the given account is following, if network is not hidden by the account owner.", parameters: [ %Reference{"$ref": "#/components/parameters/accountIdOrNickname"}, + Operation.parameter(:id, :query, :string, "ID of the resource owner"), with_relationships_param() | pagination_params() ], responses: %{200 => Operation.response("Accounts", "application/json", array_of_accounts())} @@ -201,14 +205,23 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do security: [%{"oAuth" => ["follow", "write:follows"]}], description: "Follow the given account", parameters: [ - %Reference{"$ref": "#/components/parameters/accountIdOrNickname"}, - Operation.parameter( - :reblogs, - :query, - BooleanLike, - "Receive this account's reblogs in home timeline? Defaults to true." - ) + %Reference{"$ref": "#/components/parameters/accountIdOrNickname"} ], + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + properties: %{ + reblogs: %Schema{ + type: :boolean, + description: "Receive this account's reblogs in home timeline? Defaults to true.", + default: true + } + } + }, + required: false + ), responses: %{ 200 => Operation.response("Relationship", "application/json", AccountRelationship), 400 => Operation.response("Error", "application/json", ApiError), @@ -436,27 +449,45 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do } end + # Note: this is a token response (if login succeeds!), but there's no oauth operation file yet. defp create_response do %Schema{ title: "AccountCreateResponse", description: "Response schema for an account", type: :object, properties: %{ + # The response when auto-login on create succeeds (token is issued): token_type: %Schema{type: :string}, access_token: %Schema{type: :string}, - scope: %Schema{type: :array, items: %Schema{type: :string}}, - created_at: %Schema{type: :integer, format: :"date-time"} + refresh_token: %Schema{type: :string}, + scope: %Schema{type: :string}, + created_at: %Schema{type: :integer, format: :"date-time"}, + me: %Schema{type: :string}, + expires_in: %Schema{type: :integer}, + # + # The response when registration succeeds but auto-login fails (no token): + identifier: %Schema{type: :string}, + message: %Schema{type: :string} }, + required: [], + # Note: example of successful registration with failed login response: + # example: %{ + # "identifier" => "missing_confirmed_email", + # "message" => "You have been registered. Please check your email for further instructions." + # }, example: %{ + "token_type" => "Bearer", "access_token" => "i9hAVVzGld86Pl5JtLtizKoXVvtTlSCJvwaugCxvZzk", + "refresh_token" => "i9hAVVzGld86Pl5JtLtizKoXVvtTlSCJvwaugCxvZzz", "created_at" => 1_585_918_714, - "scope" => ["read", "write", "follow", "push"], - "token_type" => "Bearer" + "expires_in" => 600, + "scope" => "read write follow push", + "me" => "https://gensokyo.2hu/users/raymoo" } } end - defp update_creadentials_request do + defp update_credentials_request do %Schema{ title: "AccountUpdateCredentialsRequest", description: "POST body for creating an account", @@ -490,6 +521,11 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do nullable: true, description: "Whether manual approval of follow requests is required." }, + accepts_chat_messages: %Schema{ + allOf: [BooleanLike], + nullable: true, + description: "Whether the user accepts receiving chat messages." + }, fields_attributes: %Schema{ nullable: true, oneOf: [ diff --git a/lib/pleroma/web/api_spec/operations/admin/config_operation.ex b/lib/pleroma/web/api_spec/operations/admin/config_operation.ex index 7b38a2ef4..3a8380797 100644 --- a/lib/pleroma/web/api_spec/operations/admin/config_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/config_operation.ex @@ -26,6 +26,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ConfigOperation do %Schema{type: :boolean, default: false}, "Get only saved in database settings" ) + | admin_api_params() ], security: [%{"oAuth" => ["read"]}], responses: %{ @@ -41,6 +42,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ConfigOperation do summary: "Update config settings", operationId: "AdminAPI.ConfigController.update", security: [%{"oAuth" => ["write"]}], + parameters: admin_api_params(), requestBody: request_body("Parameters", %Schema{ type: :object, @@ -73,6 +75,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ConfigOperation do summary: "Get JSON with config descriptions.", operationId: "AdminAPI.ConfigController.descriptions", security: [%{"oAuth" => ["read"]}], + parameters: admin_api_params(), responses: %{ 200 => Operation.response("Config Descriptions", "application/json", %Schema{ diff --git a/lib/pleroma/web/api_spec/operations/admin/invite_operation.ex b/lib/pleroma/web/api_spec/operations/admin/invite_operation.ex index d3af9db49..801024d75 100644 --- a/lib/pleroma/web/api_spec/operations/admin/invite_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/invite_operation.ex @@ -20,6 +20,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.InviteOperation do summary: "Get a list of generated invites", operationId: "AdminAPI.InviteController.index", security: [%{"oAuth" => ["read:invites"]}], + parameters: admin_api_params(), responses: %{ 200 => Operation.response("Invites", "application/json", %Schema{ @@ -51,6 +52,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.InviteOperation do summary: "Create an account registration invite token", operationId: "AdminAPI.InviteController.create", security: [%{"oAuth" => ["write:invites"]}], + parameters: admin_api_params(), requestBody: request_body("Parameters", %Schema{ type: :object, @@ -71,6 +73,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.InviteOperation do summary: "Revoke invite by token", operationId: "AdminAPI.InviteController.revoke", security: [%{"oAuth" => ["write:invites"]}], + parameters: admin_api_params(), requestBody: request_body( "Parameters", @@ -97,6 +100,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.InviteOperation do summary: "Sends registration invite via email", operationId: "AdminAPI.InviteController.email", security: [%{"oAuth" => ["write:invites"]}], + parameters: admin_api_params(), requestBody: request_body( "Parameters", diff --git a/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex new file mode 100644 index 000000000..ab45d6633 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex @@ -0,0 +1,121 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Admin", "MediaProxyCache"], + summary: "Fetch a paginated list of all banned MediaProxy URLs in Cachex", + operationId: "AdminAPI.MediaProxyCacheController.index", + security: [%{"oAuth" => ["read:media_proxy_caches"]}], + parameters: [ + Operation.parameter( + :query, + :query, + %Schema{type: :string, default: nil}, + "Page" + ), + Operation.parameter( + :page, + :query, + %Schema{type: :integer, default: 1}, + "Page" + ), + Operation.parameter( + :page_size, + :query, + %Schema{type: :integer, default: 50}, + "Number of statuses to return" + ) + | admin_api_params() + ], + responses: %{ + 200 => + Operation.response( + "Array of banned MediaProxy URLs in Cachex", + "application/json", + %Schema{ + type: :object, + properties: %{ + count: %Schema{type: :integer}, + page_size: %Schema{type: :integer}, + urls: %Schema{ + type: :array, + items: %Schema{ + type: :string, + format: :uri, + description: "MediaProxy URLs" + } + } + } + } + ) + } + } + end + + def delete_operation do + %Operation{ + tags: ["Admin", "MediaProxyCache"], + summary: "Remove a banned MediaProxy URL from Cachex", + operationId: "AdminAPI.MediaProxyCacheController.delete", + security: [%{"oAuth" => ["write:media_proxy_caches"]}], + parameters: admin_api_params(), + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + required: [:urls], + properties: %{ + urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}} + } + }, + required: true + ), + responses: %{ + 200 => empty_object_response(), + 400 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def purge_operation do + %Operation{ + tags: ["Admin", "MediaProxyCache"], + summary: "Purge and optionally ban a MediaProxy URL", + operationId: "AdminAPI.MediaProxyCacheController.purge", + security: [%{"oAuth" => ["write:media_proxy_caches"]}], + parameters: admin_api_params(), + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + required: [:urls], + properties: %{ + urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}}, + ban: %Schema{type: :boolean, default: true} + } + }, + required: true + ), + responses: %{ + 200 => empty_object_response(), + 400 => Operation.response("Error", "application/json", ApiError) + } + } + end +end diff --git a/lib/pleroma/web/api_spec/operations/admin/oauth_app_operation.ex b/lib/pleroma/web/api_spec/operations/admin/oauth_app_operation.ex index fbc9f80d7..a75f3e622 100644 --- a/lib/pleroma/web/api_spec/operations/admin/oauth_app_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/oauth_app_operation.ex @@ -36,6 +36,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do %Schema{type: :integer, default: 50}, "Number of apps to return" ) + | admin_api_params() ], responses: %{ 200 => @@ -72,6 +73,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do summary: "Create OAuth App", operationId: "AdminAPI.OAuthAppController.create", requestBody: request_body("Parameters", create_request()), + parameters: admin_api_params(), security: [%{"oAuth" => ["write"]}], responses: %{ 200 => Operation.response("App", "application/json", oauth_app()), @@ -85,7 +87,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do tags: ["Admin", "oAuth Apps"], summary: "Update OAuth App", operationId: "AdminAPI.OAuthAppController.update", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], security: [%{"oAuth" => ["write"]}], requestBody: request_body("Parameters", update_request()), responses: %{ @@ -103,7 +105,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do tags: ["Admin", "oAuth Apps"], summary: "Delete OAuth App", operationId: "AdminAPI.OAuthAppController.delete", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], security: [%{"oAuth" => ["write"]}], responses: %{ 204 => no_content_response(), diff --git a/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex b/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex index 7672cb467..e06b2d164 100644 --- a/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex @@ -19,6 +19,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do summary: "List Relays", operationId: "AdminAPI.RelayController.index", security: [%{"oAuth" => ["read"]}], + parameters: admin_api_params(), responses: %{ 200 => Operation.response("Response", "application/json", %Schema{ @@ -26,8 +27,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do properties: %{ relays: %Schema{ type: :array, - items: %Schema{type: :string}, - example: ["lain.com", "mstdn.io"] + items: relay() } } }) @@ -41,19 +41,10 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do summary: "Follow a Relay", operationId: "AdminAPI.RelayController.follow", security: [%{"oAuth" => ["write:follows"]}], - requestBody: - request_body("Parameters", %Schema{ - type: :object, - properties: %{ - relay_url: %Schema{type: :string, format: :uri} - } - }), + parameters: admin_api_params(), + requestBody: request_body("Parameters", relay_url()), responses: %{ - 200 => - Operation.response("Status", "application/json", %Schema{ - type: :string, - example: "http://mastodon.example.org/users/admin" - }) + 200 => Operation.response("Status", "application/json", relay()) } } end @@ -64,13 +55,8 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do summary: "Unfollow a Relay", operationId: "AdminAPI.RelayController.unfollow", security: [%{"oAuth" => ["write:follows"]}], - requestBody: - request_body("Parameters", %Schema{ - type: :object, - properties: %{ - relay_url: %Schema{type: :string, format: :uri} - } - }), + parameters: admin_api_params(), + requestBody: request_body("Parameters", relay_url()), responses: %{ 200 => Operation.response("Status", "application/json", %Schema{ @@ -80,4 +66,29 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do } } end + + defp relay do + %Schema{ + type: :object, + properties: %{ + actor: %Schema{ + type: :string, + example: "https://example.com/relay" + }, + followed_back: %Schema{ + type: :boolean, + description: "Is relay followed back by this actor?" + } + } + } + end + + defp relay_url do + %Schema{ + type: :object, + properties: %{ + relay_url: %Schema{type: :string, format: :uri} + } + } + end end diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex index 15e78bfaf..3bb7ec49e 100644 --- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex @@ -48,6 +48,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do %Schema{type: :integer, default: 50}, "Number number of log entries per page" ) + | admin_api_params() ], responses: %{ 200 => @@ -71,7 +72,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do tags: ["Admin", "Reports"], summary: "Get an individual report", operationId: "AdminAPI.ReportController.show", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], security: [%{"oAuth" => ["read:reports"]}], responses: %{ 200 => Operation.response("Report", "application/json", report()), @@ -86,6 +87,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do summary: "Change the state of one or multiple reports", operationId: "AdminAPI.ReportController.update", security: [%{"oAuth" => ["write:reports"]}], + parameters: admin_api_params(), requestBody: request_body("Parameters", update_request(), required: true), responses: %{ 204 => no_content_response(), @@ -100,7 +102,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do tags: ["Admin", "Reports"], summary: "Create report note", operationId: "AdminAPI.ReportController.notes_create", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], requestBody: request_body("Parameters", %Schema{ type: :object, @@ -124,6 +126,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do parameters: [ Operation.parameter(:report_id, :path, :string, "Report ID"), Operation.parameter(:id, :path, :string, "Note ID") + | admin_api_params() ], security: [%{"oAuth" => ["write:reports"]}], responses: %{ diff --git a/lib/pleroma/web/api_spec/operations/admin/status_operation.ex b/lib/pleroma/web/api_spec/operations/admin/status_operation.ex index 745399b4b..c105838a4 100644 --- a/lib/pleroma/web/api_spec/operations/admin/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/status_operation.ex @@ -55,6 +55,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.StatusOperation do %Schema{type: :integer, default: 50}, "Number of statuses to return" ) + | admin_api_params() ], responses: %{ 200 => @@ -71,7 +72,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.StatusOperation do tags: ["Admin", "Statuses"], summary: "Show Status", operationId: "AdminAPI.StatusController.show", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], security: [%{"oAuth" => ["read:statuses"]}], responses: %{ 200 => Operation.response("Status", "application/json", status()), @@ -85,7 +86,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.StatusOperation do tags: ["Admin", "Statuses"], summary: "Change the scope of an individual reported status", operationId: "AdminAPI.StatusController.update", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], security: [%{"oAuth" => ["write:statuses"]}], requestBody: request_body("Parameters", update_request(), required: true), responses: %{ @@ -100,7 +101,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.StatusOperation do tags: ["Admin", "Statuses"], summary: "Delete an individual reported status", operationId: "AdminAPI.StatusController.delete", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], security: [%{"oAuth" => ["write:statuses"]}], responses: %{ 200 => empty_object_response(), diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex index cf299bfc2..b1a0d26ab 100644 --- a/lib/pleroma/web/api_spec/operations/chat_operation.ex +++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex @@ -300,11 +300,11 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do "content" => "Check this out :firefox:", "id" => "13", "chat_id" => "1", - "actor_id" => "someflakeid", + "account_id" => "someflakeid", "unread" => false }, %{ - "actor_id" => "someflakeid", + "account_id" => "someflakeid", "content" => "Whats' up?", "id" => "12", "chat_id" => "1", diff --git a/lib/pleroma/web/api_spec/operations/domain_block_operation.ex b/lib/pleroma/web/api_spec/operations/domain_block_operation.ex index 049bcf931..1e0da8209 100644 --- a/lib/pleroma/web/api_spec/operations/domain_block_operation.ex +++ b/lib/pleroma/web/api_spec/operations/domain_block_operation.ex @@ -31,6 +31,7 @@ defmodule Pleroma.Web.ApiSpec.DomainBlockOperation do } end + # Supporting domain query parameter is deprecated in Mastodon API def create_operation do %Operation{ tags: ["domain_blocks"], @@ -45,11 +46,13 @@ defmodule Pleroma.Web.ApiSpec.DomainBlockOperation do """, operationId: "DomainBlockController.create", requestBody: domain_block_request(), + parameters: [Operation.parameter(:domain, :query, %Schema{type: :string}, "Domain name")], security: [%{"oAuth" => ["follow", "write:blocks"]}], responses: %{200 => empty_object_response()} } end + # Supporting domain query parameter is deprecated in Mastodon API def delete_operation do %Operation{ tags: ["domain_blocks"], @@ -57,6 +60,7 @@ defmodule Pleroma.Web.ApiSpec.DomainBlockOperation do description: "Remove a domain block, if it exists in the user's array of blocked domains.", operationId: "DomainBlockController.delete", requestBody: domain_block_request(), + parameters: [Operation.parameter(:domain, :query, %Schema{type: :string}, "Domain name")], security: [%{"oAuth" => ["follow", "write:blocks"]}], responses: %{ 200 => Operation.response("Empty object", "application/json", %Schema{type: :object}) @@ -71,10 +75,9 @@ defmodule Pleroma.Web.ApiSpec.DomainBlockOperation do type: :object, properties: %{ domain: %Schema{type: :string} - }, - required: [:domain] + } }, - required: true, + required: false, example: %{ "domain" => "facebook.com" } diff --git a/lib/pleroma/web/api_spec/operations/list_operation.ex b/lib/pleroma/web/api_spec/operations/list_operation.ex index c88ed5dd0..15039052e 100644 --- a/lib/pleroma/web/api_spec/operations/list_operation.ex +++ b/lib/pleroma/web/api_spec/operations/list_operation.ex @@ -114,7 +114,7 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do description: "Add accounts to the given list.", operationId: "ListController.add_to_list", parameters: [id_param()], - requestBody: add_remove_accounts_request(), + requestBody: add_remove_accounts_request(true), security: [%{"oAuth" => ["write:lists"]}], responses: %{ 200 => Operation.response("Empty object", "application/json", %Schema{type: :object}) @@ -127,8 +127,16 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do tags: ["Lists"], summary: "Remove accounts from list", operationId: "ListController.remove_from_list", - parameters: [id_param()], - requestBody: add_remove_accounts_request(), + parameters: [ + id_param(), + Operation.parameter( + :account_ids, + :query, + %Schema{type: :array, items: %Schema{type: :string}}, + "Array of account IDs" + ) + ], + requestBody: add_remove_accounts_request(false), security: [%{"oAuth" => ["write:lists"]}], responses: %{ 200 => Operation.response("Empty object", "application/json", %Schema{type: :object}) @@ -171,7 +179,7 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do ) end - defp add_remove_accounts_request do + defp add_remove_accounts_request(required) when is_boolean(required) do request_body( "Parameters", %Schema{ @@ -180,9 +188,9 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do properties: %{ account_ids: %Schema{type: :array, description: "Array of account IDs", items: FlakeID} }, - required: [:account_ids] + required: required && [:account_ids] }, - required: true + required: required ) end end diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index c966b553a..f09be64cb 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -163,6 +163,13 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do description: "Status that was the object of the notification, e.g. in mentions, reblogs, favourites, or polls.", nullable: true + }, + pleroma: %Schema{ + type: :object, + properties: %{ + is_seen: %Schema{type: :boolean}, + is_muted: %Schema{type: :boolean} + } } }, example: %{ @@ -170,7 +177,8 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do "type" => "mention", "created_at" => "2019-11-23T07:49:02.064Z", "account" => Account.schema().example, - "status" => Status.schema().example + "status" => Status.schema().example, + "pleroma" => %{"is_seen" => false, "is_muted" => false} } } end @@ -183,7 +191,6 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do "favourite", "reblog", "mention", - "poll", "pleroma:emoji_reaction", "pleroma:chat_mention", "move", diff --git a/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex index 90922c064..97836b2eb 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex @@ -4,7 +4,6 @@ defmodule Pleroma.Web.ApiSpec.PleromaAccountOperation do alias OpenApiSpex.Operation - alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.Schemas.AccountRelationship alias Pleroma.Web.ApiSpec.Schemas.ApiError alias Pleroma.Web.ApiSpec.Schemas.FlakeID @@ -40,48 +39,6 @@ defmodule Pleroma.Web.ApiSpec.PleromaAccountOperation do } end - def update_avatar_operation do - %Operation{ - tags: ["Accounts"], - summary: "Set/clear user avatar image", - operationId: "PleromaAPI.AccountController.update_avatar", - requestBody: - request_body("Parameters", update_avatar_or_background_request(), required: true), - security: [%{"oAuth" => ["write:accounts"]}], - responses: %{ - 200 => update_response(), - 403 => Operation.response("Forbidden", "application/json", ApiError) - } - } - end - - def update_banner_operation do - %Operation{ - tags: ["Accounts"], - summary: "Set/clear user banner image", - operationId: "PleromaAPI.AccountController.update_banner", - requestBody: request_body("Parameters", update_banner_request(), required: true), - security: [%{"oAuth" => ["write:accounts"]}], - responses: %{ - 200 => update_response() - } - } - end - - def update_background_operation do - %Operation{ - tags: ["Accounts"], - summary: "Set/clear user background image", - operationId: "PleromaAPI.AccountController.update_background", - security: [%{"oAuth" => ["write:accounts"]}], - requestBody: - request_body("Parameters", update_avatar_or_background_request(), required: true), - responses: %{ - 200 => update_response() - } - } - end - def favourites_operation do %Operation{ tags: ["Accounts"], @@ -136,52 +93,4 @@ defmodule Pleroma.Web.ApiSpec.PleromaAccountOperation do required: true ) end - - defp update_avatar_or_background_request do - %Schema{ - title: "PleromaAccountUpdateAvatarOrBackgroundRequest", - type: :object, - properties: %{ - img: %Schema{ - nullable: true, - type: :string, - format: :binary, - description: "Image encoded using `multipart/form-data` or an empty string to clear" - } - } - } - end - - defp update_banner_request do - %Schema{ - title: "PleromaAccountUpdateBannerRequest", - type: :object, - properties: %{ - banner: %Schema{ - type: :string, - nullable: true, - format: :binary, - description: "Image encoded using `multipart/form-data` or an empty string to clear" - } - } - } - end - - defp update_response do - Operation.response("PleromaAccountUpdateResponse", "application/json", %Schema{ - type: :object, - properties: %{ - url: %Schema{ - type: :string, - format: :uri, - nullable: true, - description: "Image URL" - } - }, - example: %{ - "url" => - "https://cofe.party/media/9d0add56-bcb6-4c0f-8225-cbbd0b6dd773/13eadb6972c9ccd3f4ffa3b8196f0e0d38b4d2f27594457c52e52946c054cd9a.gif" - } - }) - end end diff --git a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex index 567688ff5..b2b4f8713 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex @@ -33,6 +33,20 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiPackOperation do tags: ["Emoji Packs"], summary: "Lists local custom emoji packs", operationId: "PleromaAPI.EmojiPackController.index", + parameters: [ + Operation.parameter( + :page, + :query, + %Schema{type: :integer, default: 1}, + "Page" + ), + Operation.parameter( + :page_size, + :query, + %Schema{type: :integer, default: 50}, + "Number of emoji packs to return" + ) + ], responses: %{ 200 => emoji_packs_response() } @@ -44,7 +58,21 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiPackOperation do tags: ["Emoji Packs"], summary: "Show emoji pack", operationId: "PleromaAPI.EmojiPackController.show", - parameters: [name_param()], + parameters: [ + name_param(), + Operation.parameter( + :page, + :query, + %Schema{type: :integer, default: 1}, + "Page" + ), + Operation.parameter( + :page_size, + :query, + %Schema{type: :integer, default: 30}, + "Number of emoji to return" + ) + ], responses: %{ 200 => Operation.response("Emoji Pack", "application/json", emoji_pack()), 400 => Operation.response("Bad Request", "application/json", ApiError), diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 0b7fad793..5bd4619d5 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -84,7 +84,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do operationId: "StatusController.delete", parameters: [id_param()], responses: %{ - 200 => empty_object_response(), + 200 => status_response(), 403 => Operation.response("Forbidden", "application/json", ApiError), 404 => Operation.response("Not Found", "application/json", ApiError) } diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index d54e2158d..ca79f0747 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -40,33 +40,72 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do pleroma: %Schema{ type: :object, properties: %{ - allow_following_move: %Schema{type: :boolean}, - background_image: %Schema{type: :string, nullable: true}, + allow_following_move: %Schema{ + type: :boolean, + description: "whether the user allows automatically follow moved following accounts" + }, + background_image: %Schema{type: :string, nullable: true, format: :uri}, chat_token: %Schema{type: :string}, - confirmation_pending: %Schema{type: :boolean}, + confirmation_pending: %Schema{ + type: :boolean, + description: + "whether the user account is waiting on email confirmation to be activated" + }, hide_favorites: %Schema{type: :boolean}, - hide_followers_count: %Schema{type: :boolean}, - hide_followers: %Schema{type: :boolean}, - hide_follows_count: %Schema{type: :boolean}, - hide_follows: %Schema{type: :boolean}, - is_admin: %Schema{type: :boolean}, - is_moderator: %Schema{type: :boolean}, + hide_followers_count: %Schema{ + type: :boolean, + description: "whether the user has follower stat hiding enabled" + }, + hide_followers: %Schema{ + type: :boolean, + description: "whether the user has follower hiding enabled" + }, + hide_follows_count: %Schema{ + type: :boolean, + description: "whether the user has follow stat hiding enabled" + }, + hide_follows: %Schema{ + type: :boolean, + description: "whether the user has follow hiding enabled" + }, + is_admin: %Schema{ + type: :boolean, + description: "whether the user is an admin of the local instance" + }, + is_moderator: %Schema{ + type: :boolean, + description: "whether the user is a moderator of the local instance" + }, skip_thread_containment: %Schema{type: :boolean}, - tags: %Schema{type: :array, items: %Schema{type: :string}}, - unread_conversation_count: %Schema{type: :integer}, + tags: %Schema{ + type: :array, + items: %Schema{type: :string}, + description: + "List of tags being used for things like extra roles or moderation(ie. marking all media as nsfw all)." + }, + unread_conversation_count: %Schema{ + type: :integer, + description: "The count of unread conversations. Only returned to the account owner." + }, notification_settings: %Schema{ type: :object, properties: %{ - followers: %Schema{type: :boolean}, - follows: %Schema{type: :boolean}, - non_followers: %Schema{type: :boolean}, - non_follows: %Schema{type: :boolean}, - privacy_option: %Schema{type: :boolean} + block_from_strangers: %Schema{type: :boolean}, + hide_notification_contents: %Schema{type: :boolean} } }, relationship: AccountRelationship, settings_store: %Schema{ - type: :object + type: :object, + description: + "A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials`" + }, + accepts_chat_messages: %Schema{type: :boolean, nullable: true}, + favicon: %Schema{ + type: :string, + format: :uri, + nullable: true, + description: "Favicon image of the user's instance" } } }, @@ -74,16 +113,32 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do type: :object, properties: %{ fields: %Schema{type: :array, items: AccountField}, - note: %Schema{type: :string}, + note: %Schema{ + type: :string, + description: + "Plaintext version of the bio without formatting applied by the backend, used for editing the bio." + }, privacy: VisibilityScope, sensitive: %Schema{type: :boolean}, pleroma: %Schema{ type: :object, properties: %{ actor_type: ActorType, - discoverable: %Schema{type: :boolean}, - no_rich_text: %Schema{type: :boolean}, - show_role: %Schema{type: :boolean} + discoverable: %Schema{ + type: :boolean, + description: + "whether the user allows discovery of the account in search results and other services." + }, + no_rich_text: %Schema{ + type: :boolean, + description: + "whether the HTML tags for rich-text formatting are stripped from all statuses requested from the API." + }, + show_role: %Schema{ + type: :boolean, + description: + "whether the user wants their role (e.g admin, moderator) to be shown" + } } } } @@ -118,16 +173,14 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do "is_admin" => false, "is_moderator" => false, "skip_thread_containment" => false, + "accepts_chat_messages" => true, "chat_token" => "SFMyNTY.g3QAAAACZAAEZGF0YW0AAAASOXRLaTNlc2JHN09RZ1oyOTIwZAAGc2lnbmVkbgYARNplS3EB.Mb_Iaqew2bN1I1o79B_iP7encmVCpTKC4OtHZRxdjKc", "unread_conversation_count" => 0, "tags" => [], "notification_settings" => %{ - "followers" => true, - "follows" => true, - "non_followers" => true, - "non_follows" => true, - "privacy_option" => false + "block_from_strangers" => false, + "hide_notification_contents" => false }, "relationship" => %{ "blocked_by" => false, diff --git a/lib/pleroma/web/api_spec/schemas/chat_message.ex b/lib/pleroma/web/api_spec/schemas/chat_message.ex index 3ee85aa76..bbf2a4427 100644 --- a/lib/pleroma/web/api_spec/schemas/chat_message.ex +++ b/lib/pleroma/web/api_spec/schemas/chat_message.ex @@ -19,13 +19,46 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do content: %Schema{type: :string, nullable: true}, created_at: %Schema{type: :string, format: :"date-time"}, emojis: %Schema{type: :array}, - attachment: %Schema{type: :object, nullable: true} + attachment: %Schema{type: :object, nullable: true}, + card: %Schema{ + type: :object, + nullable: true, + description: "Preview card for links included within status content", + required: [:url, :title, :description, :type], + properties: %{ + type: %Schema{ + type: :string, + enum: ["link", "photo", "video", "rich"], + description: "The type of the preview card" + }, + provider_name: %Schema{ + type: :string, + nullable: true, + description: "The provider of the original resource" + }, + provider_url: %Schema{ + type: :string, + format: :uri, + description: "A link to the provider of the original resource" + }, + url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, + image: %Schema{ + type: :string, + nullable: true, + format: :uri, + description: "Preview thumbnail" + }, + title: %Schema{type: :string, description: "Title of linked resource"}, + description: %Schema{type: :string, description: "Description of preview"} + } + } }, example: %{ "account_id" => "someflakeid", "chat_id" => "1", "content" => "hey you again", "created_at" => "2020-04-21T15:06:45.000Z", + "card" => nil, "emojis" => [ %{ "static_url" => "https://dontbulling.me/emoji/Firefox.gif", diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 8b87cb25b..947e42890 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -62,6 +62,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do } }, content: %Schema{type: :string, format: :html, description: "HTML-encoded status content"}, + text: %Schema{ + type: :string, + description: "Original unformatted content in plain text", + nullable: true + }, created_at: %Schema{ type: :string, format: "date-time", @@ -184,6 +189,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do thread_muted: %Schema{ type: :boolean, description: "`true` if the thread the post belongs to is muted" + }, + parent_visible: %Schema{ + type: :boolean, + description: "`true` if the parent post is visible to the user" } } }, diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index f63a66c03..402ab428b 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -28,10 +28,6 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do %User{} = user <- ldap_user(name, password) do {:ok, user} else - {:error, {:ldap_connection_error, _}} -> - # When LDAP is unavailable, try default authenticator - @base.get_user(conn) - {:ldap, _} -> @base.get_user(conn) @@ -92,7 +88,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do user _ -> - register_user(connection, base, uid, name, password) + register_user(connection, base, uid, name) end error -> @@ -100,34 +96,31 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do end end - defp register_user(connection, base, uid, name, password) do + defp register_user(connection, base, uid, name) do case :eldap.search(connection, [ {:base, to_charlist(base)}, {:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))}, {:scope, :eldap.wholeSubtree()}, - {:attributes, ['mail', 'email']}, {:timeout, @search_timeout} ]) do {:ok, {:eldap_search_result, [{:eldap_entry, _, attributes}], _}} -> - with {_, [mail]} <- List.keyfind(attributes, 'mail', 0) do - params = %{ - email: :erlang.list_to_binary(mail), - name: name, - nickname: name, - password: password, - password_confirmation: password - } - - changeset = User.register_changeset(%User{}, params) - - case User.register(changeset) do - {:ok, user} -> user - error -> error + params = %{ + name: name, + nickname: name, + password: nil + } + + params = + case List.keyfind(attributes, 'mail', 0) do + {_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail)) + _ -> params end - else - _ -> - Logger.error("Could not find LDAP attribute mail: #{inspect(attributes)}") - {:error, :ldap_registration_missing_attributes} + + changeset = User.register_changeset_ldap(%User{}, params) + + case User.register(changeset) do + {:ok, user} -> user + error -> error end error -> diff --git a/lib/pleroma/web/auth/pleroma_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex index 200ca03dc..c611b3e09 100644 --- a/lib/pleroma/web/auth/pleroma_authenticator.ex +++ b/lib/pleroma/web/auth/pleroma_authenticator.ex @@ -68,7 +68,7 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do nickname = value([registration_attrs["nickname"], Registration.nickname(registration)]) email = value([registration_attrs["email"], Registration.email(registration)]) name = value([registration_attrs["name"], Registration.name(registration)]) || nickname - bio = value([registration_attrs["bio"], Registration.description(registration)]) + bio = value([registration_attrs["bio"], Registration.description(registration)]) || "" random_password = :crypto.strong_rand_bytes(64) |> Base.encode64() diff --git a/lib/pleroma/web/chat_channel.ex b/lib/pleroma/web/chat_channel.ex index bce27897f..3b1469c19 100644 --- a/lib/pleroma/web/chat_channel.ex +++ b/lib/pleroma/web/chat_channel.ex @@ -4,8 +4,10 @@ defmodule Pleroma.Web.ChatChannel do use Phoenix.Channel + alias Pleroma.User alias Pleroma.Web.ChatChannel.ChatChannelState + alias Pleroma.Web.MastodonAPI.AccountView def join("chat:public", _message, socket) do send(self(), :after_join) @@ -22,9 +24,9 @@ defmodule Pleroma.Web.ChatChannel do if String.length(text) in 1..Pleroma.Config.get([:instance, :chat_limit]) do author = User.get_cached_by_nickname(user_name) - author = Pleroma.Web.MastodonAPI.AccountView.render("show.json", user: author) + author_json = AccountView.render("show.json", user: author, skip_visibility_check: true) - message = ChatChannelState.add_message(%{text: text, author: author}) + message = ChatChannelState.add_message(%{text: text, author: author_json}) broadcast!(socket, "new_msg", message) end diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 9bcb9f587..548f76609 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -186,6 +186,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do draft.poll ) |> Map.put("emoji", emoji) + |> Map.put("source", draft.status) %__MODULE__{draft | object: object} end @@ -201,7 +202,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do additional = case draft.expires_at do - %NaiveDateTime{} = expires_at -> Map.put(additional, "expires_at", expires_at) + %DateTime{} = expires_at -> Map.put(additional, "expires_at", expires_at) _ -> additional end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 04e081a8e..500c3883e 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -4,11 +4,8 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Activity - alias Pleroma.ActivityExpiration alias Pleroma.Conversation.Participation - alias Pleroma.FollowingRelationship alias Pleroma.Formatter - alias Pleroma.Notification alias Pleroma.Object alias Pleroma.ThreadMute alias Pleroma.User @@ -25,6 +22,13 @@ defmodule Pleroma.Web.CommonAPI do require Pleroma.Constants require Logger + def block(blocker, blocked) do + with {:ok, block_data, _} <- Builder.block(blocker, blocked), + {:ok, block, _} <- Pipeline.common_pipeline(block_data, local: true) do + {:ok, block} + end + end + def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ []) do with maybe_attachment <- opts[:media_id] && Object.get_by_id(opts[:media_id]), :ok <- validate_chat_content_length(content, !!maybe_attachment), @@ -94,10 +98,14 @@ defmodule Pleroma.Web.CommonAPI do def follow(follower, followed) do timeout = Pleroma.Config.get([:activitypub, :follow_handshake_timeout]) - with {:ok, follower} <- User.maybe_direct_follow(follower, followed), - {:ok, activity} <- ActivityPub.follow(follower, followed), + with {:ok, follow_data, _} <- Builder.follow(follower, followed), + {:ok, activity, _} <- Pipeline.common_pipeline(follow_data, local: true), {:ok, follower, followed} <- User.wait_and_refresh(timeout, follower, followed) do - {:ok, follower, followed, activity} + if activity.data["state"] == "reject" do + {:error, :rejected} + else + {:ok, follower, followed, activity} + end end end @@ -111,33 +119,16 @@ defmodule Pleroma.Web.CommonAPI do def accept_follow_request(follower, followed) do with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed), - {:ok, follower} <- User.follow(follower, followed), - {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"), - {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_accept), - {:ok, _activity} <- - ActivityPub.accept(%{ - to: [follower.ap_id], - actor: followed, - object: follow_activity.data["id"], - type: "Accept" - }) do - Notification.update_notification_type(followed, follow_activity) + {:ok, accept_data, _} <- Builder.accept(followed, follow_activity), + {:ok, _activity, _} <- Pipeline.common_pipeline(accept_data, local: true) do {:ok, follower} end end def reject_follow_request(follower, followed) do with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed), - {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject"), - {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_reject), - {:ok, _notifications} <- Notification.dismiss(follow_activity), - {:ok, _activity} <- - ActivityPub.reject(%{ - to: [follower.ap_id], - actor: followed, - object: follow_activity.data["id"], - type: "Reject" - }) do + {:ok, reject_data, _} <- Builder.reject(followed, follow_activity), + {:ok, _activity, _} <- Pipeline.common_pipeline(reject_data, local: true) do {:ok, follower} end end @@ -297,18 +288,19 @@ defmodule Pleroma.Web.CommonAPI do {:ok, options, choices} <- normalize_and_validate_choices(choices, object) do answer_activities = Enum.map(choices, fn index -> - answer_data = make_answer_data(user, object, Enum.at(options, index)["name"]) - - {:ok, activity} = - ActivityPub.create(%{ - to: answer_data["to"], - actor: user, - context: object.data["context"], - object: answer_data, - additional: %{"cc" => answer_data["cc"]} - }) - - activity + {:ok, answer_object, _meta} = + Builder.answer(user, object, Enum.at(options, index)["name"]) + + {:ok, activity_data, _meta} = Builder.create(user, answer_object, []) + + {:ok, activity, _meta} = + activity_data + |> Map.put("cc", answer_object["cc"]) + |> Map.put("context", answer_object["context"]) + |> Pipeline.common_pipeline(local: true) + + # TODO: Do preload of Pleroma.Object in Pipeline + Activity.normalize(activity.data) end) object = Object.get_cached_by_ap_id(object.data["id"]) @@ -329,8 +321,13 @@ defmodule Pleroma.Web.CommonAPI do end end - defp get_options_and_max_count(%{data: %{"anyOf" => any_of}}), do: {any_of, Enum.count(any_of)} - defp get_options_and_max_count(%{data: %{"oneOf" => one_of}}), do: {one_of, 1} + defp get_options_and_max_count(%{data: %{"anyOf" => any_of}}) + when is_list(any_of) and any_of != [], + do: {any_of, Enum.count(any_of)} + + defp get_options_and_max_count(%{data: %{"oneOf" => one_of}}) + when is_list(one_of) and one_of != [], + do: {one_of, 1} defp normalize_and_validate_choices(choices, object) do choices = Enum.map(choices, fn i -> if is_binary(i), do: String.to_integer(i), else: i end) @@ -383,9 +380,9 @@ defmodule Pleroma.Web.CommonAPI do def check_expiry_date({:ok, nil} = res), do: res def check_expiry_date({:ok, in_seconds}) do - expiry = NaiveDateTime.utc_now() |> NaiveDateTime.add(in_seconds) + expiry = DateTime.add(DateTime.utc_now(), in_seconds) - if ActivityExpiration.expires_late_enough?(expiry) do + if Pleroma.Workers.PurgeExpiredActivity.expires_late_enough?(expiry) do {:ok, expiry} else {:error, "Expiry date is too soon"} @@ -454,7 +451,8 @@ defmodule Pleroma.Web.CommonAPI do end def add_mute(user, activity) do - with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]) do + with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]), + _ <- Pleroma.Notification.mark_context_as_read(user, activity.data["context"]) do {:ok, activity} else {:error, _} -> {:error, dgettext("errors", "conversation is already muted")} @@ -467,7 +465,7 @@ defmodule Pleroma.Web.CommonAPI do end def thread_muted?(%User{id: user_id}, %{data: %{"context" => context}}) - when is_binary("context") do + when is_binary(context) do ThreadMute.exists?(user_id, context) end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 15594125f..9d7b24eb2 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -143,7 +143,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do def make_poll_data(%{poll: %{options: options, expires_in: expires_in}} = data) when is_list(options) do - limits = Pleroma.Config.get([:instance, :poll_limits]) + limits = Config.get([:instance, :poll_limits]) with :ok <- validate_poll_expiration(expires_in, limits), :ok <- validate_poll_options_amount(options, limits), @@ -502,7 +502,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do def make_report_content_html(nil), do: {:ok, {nil, [], []}} def make_report_content_html(comment) do - max_size = Pleroma.Config.get([:instance, :max_report_comment_size], 1000) + max_size = Config.get([:instance, :max_report_comment_size], 1000) if String.length(comment) <= max_size do {:ok, format_input(comment, "text/plain")} @@ -548,23 +548,12 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end - def make_answer_data(%User{ap_id: ap_id}, object, name) do - %{ - "type" => "Answer", - "actor" => ap_id, - "cc" => [object.data["actor"]], - "to" => [], - "name" => name, - "inReplyTo" => object.data["id"] - } - end - def validate_character_limit("" = _full_payload, [] = _attachments) do {:error, dgettext("errors", "Cannot post an empty status without attachments")} end def validate_character_limit(full_payload, _attachments) do - limit = Pleroma.Config.get([:instance, :limit]) + limit = Config.get([:instance, :limit]) length = String.length(full_payload) if length <= limit do diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index 69946fb81..6445966e0 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -18,6 +18,12 @@ defmodule Pleroma.Web.ControllerHelper do def truthy_param?(value), do: not falsy_param?(value) + def json_response(conn, status, _) when status in [204, :no_content] do + conn + |> put_resp_header("content-type", "application/json") + |> send_resp(status, "") + end + def json_response(conn, status, json) do conn |> put_status(status) diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 226d42c2c..8b153763d 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -28,6 +28,29 @@ defmodule Pleroma.Web.Endpoint do } ) + # Careful! No `only` restriction here, as we don't know what frontends contain. + plug(Pleroma.Plugs.FrontendStatic, + at: "/", + frontend_type: :primary, + gzip: true, + cache_control_for_etags: @static_cache_control, + headers: %{ + "cache-control" => @static_cache_control + } + ) + + plug(Plug.Static.IndexHtml, at: "/pleroma/admin/") + + plug(Pleroma.Plugs.FrontendStatic, + at: "/pleroma/admin", + frontend_type: :admin, + gzip: true, + cache_control_for_etags: @static_cache_control, + headers: %{ + "cache-control" => @static_cache_control + } + ) + # Serve at "/" the static files from "priv/static" directory. # # You should set gzip to true if you are running phoenix.digest @@ -45,8 +68,6 @@ defmodule Pleroma.Web.Endpoint do } ) - plug(Plug.Static.IndexHtml, at: "/pleroma/admin/") - plug(Plug.Static, at: "/pleroma/admin/", from: {:pleroma, "priv/static/adminfe/"} diff --git a/lib/pleroma/web/fallback_redirect_controller.ex b/lib/pleroma/web/fallback_redirect_controller.ex index 0d9d578fc..431ad5485 100644 --- a/lib/pleroma/web/fallback_redirect_controller.ex +++ b/lib/pleroma/web/fallback_redirect_controller.ex @@ -9,6 +9,7 @@ defmodule Fallback.RedirectController do alias Pleroma.User alias Pleroma.Web.Metadata + alias Pleroma.Web.Preload def api_not_implemented(conn, _params) do conn @@ -16,16 +17,7 @@ defmodule Fallback.RedirectController do |> json(%{error: "Not implemented"}) end - def redirector(conn, _params, code \\ 200) - - # redirect to admin section - # /pleroma/admin -> /pleroma/admin/ - # - def redirector(conn, %{"path" => ["pleroma", "admin"]} = _, _code) do - redirect(conn, to: "/pleroma/admin/") - end - - def redirector(conn, _params, code) do + def redirector(conn, _params, code \\ 200) do conn |> put_resp_content_type("text/html") |> send_file(code, index_file_path()) @@ -43,28 +35,33 @@ defmodule Fallback.RedirectController do def redirector_with_meta(conn, params) do {:ok, index_content} = File.read(index_file_path()) - tags = - try do - Metadata.build_tags(params) - rescue - e -> - Logger.error( - "Metadata rendering for #{conn.request_path} failed.\n" <> - Exception.format(:error, e, __STACKTRACE__) - ) - - "" - end + tags = build_tags(conn, params) + preloads = preload_data(conn, params) - response = String.replace(index_content, "<!--server-generated-meta-->", tags) + response = + index_content + |> String.replace("<!--server-generated-meta-->", tags <> preloads) conn |> put_resp_content_type("text/html") |> send_resp(200, response) end - def index_file_path do - Pleroma.Plugs.InstanceStatic.file_path("index.html") + def redirector_with_preload(conn, %{"path" => ["pleroma", "admin"]}) do + redirect(conn, to: "/pleroma/admin/") + end + + def redirector_with_preload(conn, params) do + {:ok, index_content} = File.read(index_file_path()) + preloads = preload_data(conn, params) + + response = + index_content + |> String.replace("<!--server-generated-meta-->", preloads) + + conn + |> put_resp_content_type("text/html") + |> send_resp(200, response) end def registration_page(conn, params) do @@ -76,4 +73,36 @@ defmodule Fallback.RedirectController do |> put_status(204) |> text("") end + + defp index_file_path do + Pleroma.Plugs.InstanceStatic.file_path("index.html") + end + + defp build_tags(conn, params) do + try do + Metadata.build_tags(params) + rescue + e -> + Logger.error( + "Metadata rendering for #{conn.request_path} failed.\n" <> + Exception.format(:error, e, __STACKTRACE__) + ) + + "" + end + end + + defp preload_data(conn, params) do + try do + Preload.build_tags(conn, params) + rescue + e -> + Logger.error( + "Preloading for #{conn.request_path} failed.\n" <> + Exception.format(:error, e, __STACKTRACE__) + ) + + "" + end + end end diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex index 39b2a766a..93a8294b7 100644 --- a/lib/pleroma/web/feed/tag_controller.ex +++ b/lib/pleroma/web/feed/tag_controller.ex @@ -9,7 +9,15 @@ defmodule Pleroma.Web.Feed.TagController do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.Feed.FeedView - def feed(conn, %{"tag" => raw_tag} = params) do + def feed(conn, params) do + unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do + render_feed(conn, params) + else + render_error(conn, :not_found, "Not found") + end + end + + def render_feed(conn, %{"tag" => raw_tag} = params) do {format, tag} = parse_tag(raw_tag) activities = diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index d56f43818..71eb1ea7e 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -37,7 +37,15 @@ defmodule Pleroma.Web.Feed.UserController do end end - def feed(conn, %{"nickname" => nickname} = params) do + def feed(conn, params) do + unless Pleroma.Config.restrict_unauthenticated_access?(:profiles, :local) do + render_feed(conn, params) + else + errors(conn, {:error, :not_found}) + end + end + + def render_feed(conn, %{"nickname" => nickname} = params) do format = get_format(conn) format = @@ -47,7 +55,7 @@ defmodule Pleroma.Web.Feed.UserController do "atom" end - with {_, %User{} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do + with {_, %User{local: true} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do activities = %{ type: ["Create"], @@ -71,6 +79,7 @@ defmodule Pleroma.Web.Feed.UserController do render_error(conn, :not_found, "Not found") end + def errors(conn, {:fetch_user, %User{local: false}}), do: errors(conn, {:error, :not_found}) def errors(conn, {:fetch_user, nil}), do: errors(conn, {:error, :not_found}) def errors(conn, _) do diff --git a/lib/pleroma/web/masto_fe_controller.ex b/lib/pleroma/web/masto_fe_controller.ex index d0d8bc8eb..43ec70021 100644 --- a/lib/pleroma/web/masto_fe_controller.ex +++ b/lib/pleroma/web/masto_fe_controller.ex @@ -49,7 +49,7 @@ defmodule Pleroma.Web.MastoFEController do |> render("manifest.json") end - @doc "PUT /api/web/settings" + @doc "PUT /api/web/settings: Backend-obscure settings blob for MastoFE, don't parse/reuse elsewhere" def put_settings(%{assigns: %{user: user}} = conn, %{"data" => settings} = _params) do with {:ok, _} <- User.mastodon_settings_update(user, settings) do json(conn, %{}) diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 7cdd8f458..95d8452df 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -20,12 +20,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do alias Pleroma.Plugs.RateLimiter alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Builder + alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.ListView alias Pleroma.Web.MastodonAPI.MastodonAPI alias Pleroma.Web.MastodonAPI.MastodonAPIController alias Pleroma.Web.MastodonAPI.StatusView - alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.OAuth.OAuthController + alias Pleroma.Web.OAuth.OAuthView alias Pleroma.Web.TwitterAPI.TwitterAPI plug(Pleroma.Web.ApiSpec.CastAndValidate) @@ -97,16 +100,34 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do def create(%{assigns: %{app: app}, body_params: params} = conn, _params) do with :ok <- validate_email_param(params), :ok <- TwitterAPI.validate_captcha(app, params), - {:ok, user} <- TwitterAPI.register_user(params, need_confirmation: true), - {:ok, token} <- Token.create_token(app, user, %{scopes: app.scopes}) do - json(conn, %{ - token_type: "Bearer", - access_token: token.token, - scope: app.scopes, - created_at: Token.Utils.format_created_at(token) - }) + {:ok, user} <- TwitterAPI.register_user(params), + {_, {:ok, token}} <- + {:login, OAuthController.login(user, app, app.scopes)} do + json(conn, OAuthView.render("token.json", %{user: user, token: token})) else - {:error, error} -> json_response(conn, :bad_request, %{error: error}) + {:login, {:account_status, :confirmation_pending}} -> + json_response(conn, :ok, %{ + message: "You have been registered. Please check your email for further instructions.", + identifier: "missing_confirmed_email" + }) + + {:login, {:account_status, :approval_pending}} -> + json_response(conn, :ok, %{ + message: + "You have been registered. You'll be able to log in once your account is approved.", + identifier: "awaiting_approval" + }) + + {:login, _} -> + json_response(conn, :ok, %{ + message: + "You have been registered. Some post-registration steps may be pending. " <> + "Please log in manually.", + identifier: "manual_login_required" + }) + + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) end end @@ -146,6 +167,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do |> Enum.filter(fn {_, value} -> not is_nil(value) end) |> Enum.into(%{}) + # We use an empty string as a special value to reset + # avatars, banners, backgrounds + user_image_value = fn + "" -> {:ok, nil} + value -> {:ok, value} + end + user_params = [ :no_rich_text, @@ -158,16 +186,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do :show_role, :skip_thread_containment, :allow_following_move, - :discoverable + :discoverable, + :accepts_chat_messages ] |> Enum.reduce(%{}, fn key, acc -> Maps.put_if_present(acc, key, params[key], &{:ok, truthy_param?(&1)}) end) |> Maps.put_if_present(:name, params[:display_name]) |> Maps.put_if_present(:bio, params[:note]) - |> Maps.put_if_present(:avatar, params[:avatar]) - |> Maps.put_if_present(:banner, params[:header]) - |> Maps.put_if_present(:background, params[:pleroma_background_image]) + |> Maps.put_if_present(:raw_bio, params[:note]) + |> Maps.put_if_present(:avatar, params[:avatar], user_image_value) + |> Maps.put_if_present(:banner, params[:header], user_image_value) + |> Maps.put_if_present(:background, params[:pleroma_background_image], user_image_value) |> Maps.put_if_present( :raw_fields, params[:fields_attributes], @@ -176,36 +206,44 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do |> Maps.put_if_present(:pleroma_settings_store, params[:pleroma_settings_store]) |> Maps.put_if_present(:default_scope, params[:default_scope]) |> Maps.put_if_present(:default_scope, params["source"]["privacy"]) + |> Maps.put_if_present(:actor_type, params[:bot], fn bot -> + if bot, do: {:ok, "Service"}, else: {:ok, "Person"} + end) |> Maps.put_if_present(:actor_type, params[:actor_type]) - changeset = User.update_changeset(user, user_params) - - with {:ok, user} <- User.update_and_set_cache(changeset) do - user - |> build_update_activity_params() - |> ActivityPub.update() - - render(conn, "show.json", user: user, for: user, with_pleroma_settings: true) + # What happens here: + # + # We want to update the user through the pipeline, but the ActivityPub + # update information is not quite enough for this, because this also + # contains local settings that don't federate and don't even appear + # in the Update activity. + # + # So we first build the normal local changeset, then apply it to the + # user data, but don't persist it. With this, we generate the object + # data for our update activity. We feed this and the changeset as meta + # inforation into the pipeline, where they will be properly updated and + # federated. + with changeset <- User.update_changeset(user, user_params), + {:ok, unpersisted_user} <- Ecto.Changeset.apply_action(changeset, :update), + updated_object <- + Pleroma.Web.ActivityPub.UserView.render("user.json", user: unpersisted_user) + |> Map.delete("@context"), + {:ok, update_data, []} <- Builder.update(user, updated_object), + {:ok, _update, _} <- + Pipeline.common_pipeline(update_data, + local: true, + user_update_changeset: changeset + ) do + render(conn, "show.json", + user: unpersisted_user, + for: unpersisted_user, + with_pleroma_settings: true + ) else _e -> render_error(conn, :forbidden, "Invalid request") end end - # Hotfix, handling will be redone with the pipeline - defp build_update_activity_params(user) do - object = - Pleroma.Web.ActivityPub.UserView.render("user.json", user: user) - |> Map.delete("@context") - - %{ - local: true, - to: [user.follower_address], - cc: [], - object: object, - actor: user.ap_id - } - end - defp normalize_fields_attributes(fields) do if Enum.all?(fields, &is_tuple/1) do Enum.map(fields, fn {_, v} -> v end) @@ -230,17 +268,17 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do @doc "GET /api/v1/accounts/:id" def show(%{assigns: %{user: for_user}} = conn, %{id: nickname_or_id}) do with %User{} = user <- User.get_cached_by_nickname_or_id(nickname_or_id, for: for_user), - true <- User.visible_for?(user, for_user) do + :visible <- User.visible_for(user, for_user) do render(conn, "show.json", user: user, for: for_user) else - _e -> render_error(conn, :not_found, "Can't find user") + error -> user_visibility_error(conn, error) end end @doc "GET /api/v1/accounts/:id/statuses" def statuses(%{assigns: %{user: reading_user}} = conn, params) do with %User{} = user <- User.get_cached_by_nickname_or_id(params.id, for: reading_user), - true <- User.visible_for?(user, reading_user) do + :visible <- User.visible_for(user, reading_user) do params = params |> Map.delete(:tagged) @@ -257,7 +295,17 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do as: :activity ) else - _e -> render_error(conn, :not_found, "Can't find user") + error -> user_visibility_error(conn, error) + end + end + + defp user_visibility_error(conn, error) do + case error do + :restrict_unauthenticated -> + render_error(conn, :unauthorized, "This API requires an authenticated user") + + _ -> + render_error(conn, :not_found, "Can't find user") end end @@ -325,7 +373,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do {:error, "Can not follow yourself"} end - def follow(%{assigns: %{user: follower, account: followed}} = conn, params) do + def follow(%{body_params: params, assigns: %{user: follower, account: followed}} = conn, _) do with {:ok, follower} <- MastodonAPI.follow(follower, followed, params) do render(conn, "relationship.json", user: follower, target: followed) else @@ -364,8 +412,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do @doc "POST /api/v1/accounts/:id/block" def block(%{assigns: %{user: blocker, account: blocked}} = conn, _params) do - with {:ok, _user_block} <- User.block(blocker, blocked), - {:ok, _activity} <- ActivityPub.block(blocker, blocked) do + with {:ok, _activity} <- CommonAPI.block(blocker, blocked) do render(conn, "relationship.json", user: blocker, target: blocked) else {:error, message} -> json_response(conn, :forbidden, %{error: message}) diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex index 753b3db3e..9f09550e1 100644 --- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex @@ -59,17 +59,11 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do def password_reset(conn, params) do nickname_or_email = params["email"] || params["nickname"] - with {:ok, _} <- TwitterAPI.password_reset(nickname_or_email) do - conn - |> put_status(:no_content) - |> json("") - else - {:error, "unknown user"} -> - send_resp(conn, :not_found, "") - - {:error, _} -> - send_resp(conn, :bad_request, "") - end + TwitterAPI.password_reset(nickname_or_email) + + conn + |> put_status(:no_content) + |> json("") end defp local_mastodon_root_path(conn) do diff --git a/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex b/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex index 825b231ab..9c2d093cd 100644 --- a/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex @@ -32,9 +32,19 @@ defmodule Pleroma.Web.MastodonAPI.DomainBlockController do json(conn, %{}) end + def create(%{assigns: %{user: blocker}} = conn, %{domain: domain}) do + User.block_domain(blocker, domain) + json(conn, %{}) + end + @doc "DELETE /api/v1/domain_blocks" def delete(%{assigns: %{user: blocker}, body_params: %{domain: domain}} = conn, _params) do User.unblock_domain(blocker, domain) json(conn, %{}) end + + def delete(%{assigns: %{user: blocker}} = conn, %{domain: domain}) do + User.unblock_domain(blocker, domain) + json(conn, %{}) + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/list_controller.ex b/lib/pleroma/web/mastodon_api/controllers/list_controller.ex index acdc76fd2..5daeaa780 100644 --- a/lib/pleroma/web/mastodon_api/controllers/list_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/list_controller.ex @@ -74,7 +74,7 @@ defmodule Pleroma.Web.MastodonAPI.ListController do # DELETE /api/v1/lists/:id/accounts def remove_from_list( - %{assigns: %{list: list}, body_params: %{account_ids: account_ids}} = conn, + %{assigns: %{list: list}, params: %{account_ids: account_ids}} = conn, _ ) do Enum.each(account_ids, fn account_id -> @@ -86,6 +86,10 @@ defmodule Pleroma.Web.MastodonAPI.ListController do json(conn, %{}) end + def remove_from_list(%{body_params: params} = conn, _) do + remove_from_list(%{conn | params: params}, %{}) + end + defp list_by_id_and_user(%{assigns: %{user: user}, params: %{id: id}} = conn, _) do case Pleroma.List.get(id, user) do %Pleroma.List{} = list -> assign(conn, :list, list) diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 3be0ca095..5a983db39 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -44,6 +44,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do def search(conn, params), do: do_search(:v1, conn, params) defp do_search(version, %{assigns: %{user: user}} = conn, %{q: query} = params) do + query = String.trim(query) options = search_options(params, user) timeout = Keyword.get(Repo.config(), :timeout, 15_000) default_values = %{"statuses" => [], "accounts" => [], "hashtags" => []} @@ -92,7 +93,6 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do AccountView.render("index.json", users: accounts, for: options[:for_user], - as: :user, embed_relationships: options[:embed_relationships] ) end @@ -107,21 +107,21 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do ) end - defp resource_search(:v2, "hashtags", query, _options) do + defp resource_search(:v2, "hashtags", query, options) do tags_path = Web.base_url() <> "/tag/" query - |> prepare_tags() + |> prepare_tags(options) |> Enum.map(fn tag -> %{name: tag, url: tags_path <> tag} end) end - defp resource_search(:v1, "hashtags", query, _options) do - prepare_tags(query) + defp resource_search(:v1, "hashtags", query, options) do + prepare_tags(query, options) end - defp prepare_tags(query, add_joined_tag \\ true) do + defp prepare_tags(query, options) do tags = query |> preprocess_uri_query() @@ -139,13 +139,20 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end) - if Enum.empty?(explicit_tags) && add_joined_tag do - tags - |> Kernel.++([joined_tag(tags)]) - |> Enum.uniq_by(&String.downcase/1) - else - tags - end + tags = + if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do + add_joined_tag(tags) + else + tags + end + + Pleroma.Pagination.paginate(tags, options) + end + + defp add_joined_tag(tags) do + tags + |> Kernel.++([joined_tag(tags)]) + |> Enum.uniq_by(&String.downcase/1) end # If `query` is a URI, returns last component of its path, otherwise returns `query` diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 468b44b67..ecfa38489 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -172,6 +172,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do with_direct_conversation_id: true ) else + {:error, {:reject, message}} -> + conn + |> put_status(:unprocessable_entity) + |> json(%{error: message}) + {:error, message} -> conn |> put_status(:unprocessable_entity) @@ -200,11 +205,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do @doc "DELETE /api/v1/statuses/:id" def delete(%{assigns: %{user: user}} = conn, %{id: id}) do - with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do - json(conn, %{}) + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + {:ok, %Activity{}} <- CommonAPI.delete(id, user) do + try_render(conn, "show.json", + activity: activity, + for: user, + with_direct_conversation_id: true, + with_source: true + ) else - {:error, :not_found} = e -> e - _e -> render_error(conn, :forbidden, "Can't delete this post") + _e -> {:error, :not_found} end end @@ -304,7 +314,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do @doc "GET /api/v1/statuses/:id/favourited_by" def favourited_by(%{assigns: %{user: user}} = conn, %{id: id}) do - with %Activity{} = activity <- Activity.get_by_id_with_object(id), + with true <- Pleroma.Config.get([:instance, :show_reactions]), + %Activity{} = activity <- Activity.get_by_id_with_object(id), {:visible, true} <- {:visible, Visibility.visible_for_user?(activity, user)}, %Object{data: %{"likes" => likes}} <- Object.normalize(activity) do users = diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index 4bdd46d7e..5272790d3 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2, add_link_headers: 3] + alias Pleroma.Config alias Pleroma.Pagination alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug alias Pleroma.Plugs.OAuthScopesPlug @@ -88,21 +89,20 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do ) end + defp restrict_unauthenticated?(true = _local_only) do + Config.restrict_unauthenticated_access?(:timelines, :local) + end + + defp restrict_unauthenticated?(_) do + Config.restrict_unauthenticated_access?(:timelines, :federated) + end + # GET /api/v1/timelines/public def public(%{assigns: %{user: user}} = conn, params) do local_only = params[:local] - cfg_key = - if local_only do - :local - else - :federated - end - - restrict? = Pleroma.Config.get([:restrict_unauthenticated, :timelines, cfg_key]) - - if restrict? and is_nil(user) do - render_error(conn, :unauthorized, "authorization required for timeline view") + if is_nil(user) and restrict_unauthenticated?(local_only) do + fail_on_bad_auth(conn) else activities = params @@ -123,6 +123,10 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do end end + defp fail_on_bad_auth(conn) do + render_error(conn, :unauthorized, "authorization required for timeline view") + end + defp hashtag_fetching(params, user, local_only) do tags = [params[:tag], params[:any]] @@ -157,15 +161,20 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do # GET /api/v1/timelines/tag/:tag def hashtag(%{assigns: %{user: user}} = conn, params) do local_only = params[:local] - activities = hashtag_fetching(params, user, local_only) - conn - |> add_link_headers(activities, %{"local" => local_only}) - |> render("index.json", - activities: activities, - for: user, - as: :activity - ) + if is_nil(user) and restrict_unauthenticated?(local_only) do + fail_on_bad_auth(conn) + else + activities = hashtag_fetching(params, user, local_only) + + conn + |> add_link_headers(activities, %{"local" => local_only}) + |> render("index.json", + activities: activities, + for: user, + as: :activity + ) + end end # GET /api/v1/timelines/list/:list_id @@ -173,11 +182,10 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do with %Pleroma.List{title: _title, following: following} <- Pleroma.List.get(id, user) do params = params - |> Map.new(fn {key, value} -> {to_string(key), value} end) - |> Map.put("type", "Create") - |> Map.put("blocking_user", user) - |> Map.put("user", user) - |> Map.put("muting_user", user) + |> Map.put(:type, "Create") + |> Map.put(:blocking_user, user) + |> Map.put(:user, user) + |> Map.put(:muting_user, user) # we must filter the following list for the user to avoid leaking statuses the user # does not actually have permission to see (for more info, peruse security issue #270). diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 9fc06bf9d..d2a30a548 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -27,21 +27,40 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do UserRelationship.view_relationships_option(reading_user, users) end - opts = Map.put(opts, :relationships, relationships_opt) + opts = + opts + |> Map.merge(%{relationships: relationships_opt, as: :user}) + |> Map.delete(:users) users |> render_many(AccountView, "show.json", opts) |> Enum.filter(&Enum.any?/1) end - def render("show.json", %{user: user} = opts) do - if User.visible_for?(user, opts[:for]) do + @doc """ + Renders specified user account. + :skip_visibility_check option skips visibility check and renders any user (local or remote) + regardless of [:pleroma, :restrict_unauthenticated] setting. + :for option specifies the requester and can be a User record or nil. + Only use `user: user, for: user` when `user` is the actual requester of own profile. + """ + def render("show.json", %{user: _user, skip_visibility_check: true} = opts) do + do_render("show.json", opts) + end + + def render("show.json", %{user: user, for: for_user_or_nil} = opts) do + if User.visible_for(user, for_user_or_nil) == :visible do do_render("show.json", opts) else %{} end end + def render("show.json", _) do + raise "In order to prevent account accessibility issues, " <> + ":skip_visibility_check or :for option is required." + end + def render("mention.json", %{user: user}) do %{ id: to_string(user.id), @@ -179,7 +198,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do 0 end - bot = user.actor_type in ["Application", "Service"] + bot = user.actor_type == "Service" emojis = Enum.map(user.emoji, fn {shortcode, raw_url} -> @@ -204,6 +223,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do %{} end + favicon = + if Pleroma.Config.get([:instances_favicons, :enabled]) do + user + |> Map.get(:ap_id, "") + |> URI.parse() + |> URI.merge("/") + |> Pleroma.Instances.Instance.get_or_update_favicon() + |> MediaProxy.url() + else + nil + end + %{ id: to_string(user.id), username: username_from_nickname(user.nickname), @@ -214,7 +245,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do followers_count: followers_count, following_count: following_count, statuses_count: user.note_count, - note: user.bio || "", + note: user.bio, url: user.uri || user.ap_id, avatar: image, avatar_static: image, @@ -224,7 +255,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do fields: user.fields, bot: bot, source: %{ - note: prepare_user_bio(user), + note: user.raw_bio || "", sensitive: false, fields: user.raw_fields, pleroma: %{ @@ -245,7 +276,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do hide_favorites: user.hide_favorites, relationship: relationship, skip_thread_containment: user.skip_thread_containment, - background_image: image_url(user.background) |> MediaProxy.url() + background_image: image_url(user.background) |> MediaProxy.url(), + accepts_chat_messages: user.accepts_chat_messages, + favicon: favicon } } |> maybe_put_role(user, opts[:for]) @@ -260,17 +293,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do |> maybe_put_unread_notification_count(user, opts[:for]) end - defp prepare_user_bio(%User{bio: ""}), do: "" - - defp prepare_user_bio(%User{bio: bio}) when is_binary(bio) do - bio - |> String.replace(~r(<br */?>), "\n") - |> Pleroma.HTML.strip_tags() - |> HtmlEntities.decode() - end - - defp prepare_user_bio(_), do: "" - defp username_from_nickname(string) when is_binary(string) do hd(String.split(string, "@")) end diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index fbe618377..a91994915 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -23,10 +23,13 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do last_activity_id = with nil <- participation.last_activity_id do - ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{ - user: user, - blocking_user: user - }) + ActivityPub.fetch_latest_direct_activity_id_for_context( + participation.conversation.ap_id, + %{ + user: user, + blocking_user: user + } + ) end activity = Activity.get_by_id_with_object(last_activity_id) @@ -35,7 +38,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do %{ id: participation.id |> to_string(), - accounts: render(AccountView, "index.json", users: users, as: :user), + accounts: render(AccountView, "index.json", users: users, for: user), unread: !participation.read, last_status: render(StatusView, "show.json", diff --git a/lib/pleroma/web/mastodon_api/views/filter_view.ex b/lib/pleroma/web/mastodon_api/views/filter_view.ex index aeff646f5..c37f624e0 100644 --- a/lib/pleroma/web/mastodon_api/views/filter_view.ex +++ b/lib/pleroma/web/mastodon_api/views/filter_view.ex @@ -25,7 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterView do context: filter.context, expires_at: expires_at, irreversible: filter.hide, - whole_word: false + whole_word: filter.whole_word } end end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index c498fe632..ea2d3aa9c 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -23,9 +23,10 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do streaming_api: Pleroma.Web.Endpoint.websocket_url() }, stats: Pleroma.Stats.get_stats(), - thumbnail: instance_thumbnail(), + thumbnail: Keyword.get(instance, :instance_thumbnail), languages: ["en"], registrations: Keyword.get(instance, :registrations_open), + approval_required: Keyword.get(instance, :account_approval_required), # Extra (not present in Mastodon): max_toot_chars: Keyword.get(instance, :limit), poll_limits: Keyword.get(instance, :poll_limits), @@ -34,10 +35,15 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do background_upload_limit: Keyword.get(instance, :background_upload_limit), banner_upload_limit: Keyword.get(instance, :banner_upload_limit), background_image: Keyword.get(instance, :background_image), + chat_limit: Keyword.get(instance, :chat_limit), + description_limit: Keyword.get(instance, :description_limit), pleroma: %{ metadata: %{ + account_activation_required: Keyword.get(instance, :account_activation_required), features: features(), - federation: federation() + federation: federation(), + fields_limits: fields_limits(), + post_formats: Config.get([:instance, :allowed_post_formats]) }, vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) } @@ -78,7 +84,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do def federation do quarantined = Config.get([:instance, :quarantined_instances], []) - if Config.get([:instance, :mrf_transparency]) do + if Config.get([:mrf, :transparency]) do {:ok, data} = MRF.describe() data @@ -89,8 +95,12 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do |> Map.put(:enabled, Config.get([:instance, :federating])) end - defp instance_thumbnail do - Pleroma.Config.get([:instance, :instance_thumbnail]) || - "#{Pleroma.Web.base_url()}/instance/thumbnail.jpeg" + def fields_limits do + %{ + max_fields: Config.get([:instance, :max_account_fields]), + max_remote_fields: Config.get([:instance, :max_remote_account_fields]), + name_length: Config.get([:instance, :account_field_name_length]), + value_length: Config.get([:instance, :account_field_value_length]) + } end end diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 3865be280..c97e6d32f 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -84,12 +84,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do # Note: :relationships contain user mutes (needed for :muted flag in :status) status_render_opts = %{relationships: opts[:relationships]} - - account = - AccountView.render( - "show.json", - %{user: actor, for: reading_user} - ) + account = AccountView.render("show.json", %{user: actor, for: reading_user}) response = %{ id: to_string(notification.id), @@ -97,6 +92,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at), account: account, pleroma: %{ + is_muted: User.mutes?(reading_user, actor), is_seen: notification.seen } } diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex index 59a5deb28..1208dc9a0 100644 --- a/lib/pleroma/web/mastodon_api/views/poll_view.ex +++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex @@ -28,10 +28,10 @@ defmodule Pleroma.Web.MastodonAPI.PollView do def render("show.json", %{object: object} = params) do case object.data do - %{"anyOf" => options} when is_list(options) -> + %{"anyOf" => [_ | _] = options} -> render(__MODULE__, "show.json", Map.merge(params, %{multiple: true, options: options})) - %{"oneOf" => options} when is_list(options) -> + %{"oneOf" => [_ | _] = options} -> render(__MODULE__, "show.json", Map.merge(params, %{multiple: false, options: options})) _ -> @@ -40,15 +40,13 @@ defmodule Pleroma.Web.MastodonAPI.PollView do end defp end_time_and_expired(object) do - case object.data["closed"] || object.data["endTime"] do - end_time when is_binary(end_time) -> - end_time = NaiveDateTime.from_iso8601!(end_time) - expired = NaiveDateTime.compare(end_time, NaiveDateTime.utc_now()) == :lt + if object.data["closed"] do + end_time = NaiveDateTime.from_iso8601!(object.data["closed"]) + expired = NaiveDateTime.compare(end_time, NaiveDateTime.utc_now()) == :lt - {Utils.to_masto_date(end_time), expired} - - _ -> - {nil, false} + {Utils.to_masto_date(end_time), expired} + else + {nil, false} end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 2c49bedb3..ca42917fc 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do require Pleroma.Constants alias Pleroma.Activity - alias Pleroma.ActivityExpiration alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo @@ -21,7 +20,18 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.MediaProxy - import Pleroma.Web.ActivityPub.Visibility, only: [get_visibility: 1] + import Pleroma.Web.ActivityPub.Visibility, only: [get_visibility: 1, visible_for_user?: 2] + + # This is a naive way to do this, just spawning a process per activity + # to fetch the preview. However it should be fine considering + # pagination is restricted to 40 activities at a time + defp fetch_rich_media_for_activities(activities) do + Enum.each(activities, fn activity -> + spawn(fn -> + Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) + end) + end) + end # TODO: Add cached version. defp get_replied_to_activities([]), do: %{} @@ -80,6 +90,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list activities = Enum.filter(opts.activities, & &1) + + # Start fetching rich media before doing anything else, so that later calls to get the cards + # only block for timeout in the worst case, as opposed to + # length(activities_with_links) * timeout + fetch_rich_media_for_activities(activities) replied_to_activities = get_replied_to_activities(activities) parent_activities = @@ -229,8 +244,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do expires_at = with true <- client_posted_this_activity, - %ActivityExpiration{scheduled_at: scheduled_at} <- - ActivityExpiration.get_by_activity_id(activity.id) do + %Oban.Job{scheduled_at: scheduled_at} <- + Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) do scheduled_at else _ -> nil @@ -297,13 +312,17 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do emoji_reactions = with %{data: %{"reactions" => emoji_reactions}} <- object do - Enum.map(emoji_reactions, fn [emoji, users] -> - %{ - name: emoji, - count: length(users), - me: !!(opts[:for] && opts[:for].ap_id in users) - } + Enum.map(emoji_reactions, fn + [emoji, users] when is_list(users) -> + build_emoji_map(emoji, users, opts[:for]) + + {emoji, users} when is_list(users) -> + build_emoji_map(emoji, users, opts[:for]) + + _ -> + nil end) + |> Enum.reject(&is_nil/1) else _ -> [] end @@ -333,6 +352,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reblog: nil, card: card, content: content_html, + text: opts[:with_source] && object.data["source"], created_at: created_at, reblogs_count: announcement_count, replies_count: object.data["repliesCount"] || 0, @@ -364,7 +384,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do expires_at: expires_at, direct_conversation_id: direct_conversation_id, thread_muted: thread_muted?, - emoji_reactions: emoji_reactions + emoji_reactions: emoji_reactions, + parent_visible: visible_for_user?(reply_to, opts[:for]) } } end @@ -467,23 +488,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end end - def render_content(%{data: %{"type" => object_type}} = object) - when object_type in ["Video", "Event", "Audio"] do - with name when not is_nil(name) and name != "" <- object.data["name"] do - "<p><a href=\"#{object.data["id"]}\">#{name}</a></p>#{object.data["content"]}" - else - _ -> object.data["content"] || "" - end - end + def render_content(%{data: %{"name" => name}} = object) when not is_nil(name) and name != "" do + url = object.data["url"] || object.data["id"] - def render_content(%{data: %{"type" => object_type}} = object) - when object_type in ["Article", "Page"] do - with summary when not is_nil(summary) and summary != "" <- object.data["name"], - url when is_bitstring(url) <- object.data["url"] do - "<p><a href=\"#{url}\">#{summary}</a></p>#{object.data["content"]}" - else - _ -> object.data["content"] || "" - end + "<p><a href=\"#{url}\">#{name}</a></p>#{object.data["content"]}" end def render_content(object), do: object.data["content"] || "" @@ -543,4 +551,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do defp pinned?(%Activity{id: id}, %User{pinned_activities: pinned_activities}), do: id in pinned_activities + + defp build_emoji_map(emoji, users, current_user) do + %{ + name: emoji, + count: length(users), + me: !!(current_user && current_user.ap_id in users) + } + end end diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 94e4595d8..cf923ded8 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -37,12 +37,12 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do else {:error, :bad_topic} -> Logger.debug("#{__MODULE__} bad topic #{inspect(req)}") - {:ok, req} = :cowboy_req.reply(404, req) + req = :cowboy_req.reply(404, req) {:ok, req, state} {:error, :unauthorized} -> Logger.debug("#{__MODULE__} authentication error: #{inspect(req)}") - {:ok, req} = :cowboy_req.reply(401, req) + req = :cowboy_req.reply(401, req) {:ok, req, state} end end @@ -64,7 +64,9 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do {:ok, %{state | timer: timer()}} end - # We never receive messages. + # We only receive pings for now + def websocket_handle(:ping, state), do: {:ok, state} + def websocket_handle(frame, state) do Logger.error("#{__MODULE__} received frame: #{inspect(frame)}") {:ok, state} @@ -98,6 +100,10 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do {:reply, :ping, %{state | timer: nil, count: 0}, :hibernate} end + # State can be `[]` only in case we terminate before switching to websocket, + # we already log errors for these cases in `init/1`, so just do nothing here + def terminate(_reason, _req, []), do: :ok + def terminate(reason, _req, state) do Logger.debug( "#{__MODULE__} terminating websocket connection for user #{ diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index c037ff13e..5808861e6 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -5,22 +5,34 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do @moduledoc false - @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()} + @callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()} alias Pleroma.Config + alias Pleroma.Web.MediaProxy - @spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()} + @spec enabled?() :: boolean() + def enabled?, do: Config.get([:media_proxy, :invalidation, :enabled]) + + @spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()} def purge(urls) do - [:media_proxy, :invalidation, :enabled] - |> Config.get() - |> do_purge(urls) + prepared_urls = prepare_urls(urls) + + if enabled?() do + do_purge(prepared_urls) + else + {:ok, prepared_urls} + end end - defp do_purge(true, urls) do + defp do_purge(urls) do provider = Config.get([:media_proxy, :invalidation, :provider]) options = Config.get(provider) provider.purge(urls, options) end - defp do_purge(_, _), do: :ok + def prepare_urls(urls) do + urls + |> List.wrap() + |> Enum.map(&MediaProxy.url/1) + end end diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex index 07248df6e..bb81d8888 100644 --- a/lib/pleroma/web/media_proxy/invalidations/http.ex +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -9,10 +9,10 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, opts) do - method = Map.get(opts, :method, :purge) - headers = Map.get(opts, :headers, []) - options = Map.get(opts, :options, []) + def purge(urls, opts \\ []) do + method = Keyword.get(opts, :method, :purge) + headers = Keyword.get(opts, :headers, []) + options = Keyword.get(opts, :options, []) Logger.debug("Running cache purge: #{inspect(urls)}") @@ -22,7 +22,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do end end) - {:ok, "success"} + {:ok, urls} end defp do_purge(method, url, headers, options) do diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index 6be782132..d32ffc50b 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -10,32 +10,34 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, %{script_path: script_path} = _options) do + def purge(urls, opts \\ []) do args = urls |> List.wrap() |> Enum.uniq() |> Enum.join(" ") - path = Path.expand(script_path) - - Logger.debug("Running cache purge: #{inspect(urls)}, #{path}") - - case do_purge(path, [args]) do - {result, exit_status} when exit_status > 0 -> - Logger.error("Error while cache purge: #{inspect(result)}") - {:error, inspect(result)} - - _ -> - {:ok, "success"} - end + opts + |> Keyword.get(:script_path) + |> do_purge([args]) + |> handle_result(urls) end - def purge(_, _), do: {:error, "not found script path"} - - defp do_purge(path, args) do + defp do_purge(script_path, args) when is_binary(script_path) do + path = Path.expand(script_path) + Logger.debug("Running cache purge: #{inspect(args)}, #{inspect(path)}") System.cmd(path, args) rescue - error -> {inspect(error), 1} + error -> error + end + + defp do_purge(_, _), do: {:error, "not found script path"} + + defp handle_result({_result, 0}, urls), do: {:ok, urls} + defp handle_result({:error, error}, urls), do: handle_result(error, urls) + + defp handle_result(error, _) do + Logger.error("Error while cache purge: #{inspect(error)}") + {:error, inspect(error)} end end diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index b2b524524..e18dd8224 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -6,20 +6,56 @@ defmodule Pleroma.Web.MediaProxy do alias Pleroma.Config alias Pleroma.Upload alias Pleroma.Web + alias Pleroma.Web.MediaProxy.Invalidation @base64_opts [padding: false] + @cache_table :banned_urls_cache + + def cache_table, do: @cache_table + + @spec in_banned_urls(String.t()) :: boolean() + def in_banned_urls(url), do: elem(Cachex.exists?(@cache_table, url(url)), 1) + + def remove_from_banned_urls(urls) when is_list(urls) do + Cachex.execute!(@cache_table, fn cache -> + Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1)) + end) + end + + def remove_from_banned_urls(url) when is_binary(url) do + Cachex.del(@cache_table, url(url)) + end + + def put_in_banned_urls(urls) when is_list(urls) do + Cachex.execute!(@cache_table, fn cache -> + Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true)) + end) + end + + def put_in_banned_urls(url) when is_binary(url) do + Cachex.put(@cache_table, url(url), true) + end def url(url) when is_nil(url) or url == "", do: nil def url("/" <> _ = url), do: url def url(url) do - if disabled?() or local?(url) or whitelisted?(url) do + if disabled?() or not url_proxiable?(url) do url else encode_url(url) end end + @spec url_proxiable?(String.t()) :: boolean() + def url_proxiable?(url) do + if local?(url) or whitelisted?(url) do + false + else + true + end + end + defp disabled?, do: !Config.get([:media_proxy, :enabled], false) defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url()) @@ -27,22 +63,28 @@ defmodule Pleroma.Web.MediaProxy do defp whitelisted?(url) do %{host: domain} = URI.parse(url) - mediaproxy_whitelist = Config.get([:media_proxy, :whitelist]) + mediaproxy_whitelist_domains = + [:media_proxy, :whitelist] + |> Config.get() + |> Enum.map(&maybe_get_domain_from_url/1) - upload_base_url_domain = - if !is_nil(Config.get([Upload, :base_url])) do - [URI.parse(Config.get([Upload, :base_url])).host] + whitelist_domains = + if base_url = Config.get([Upload, :base_url]) do + %{host: base_domain} = URI.parse(base_url) + [base_domain | mediaproxy_whitelist_domains] else - [] + mediaproxy_whitelist_domains end - whitelist = mediaproxy_whitelist ++ upload_base_url_domain + domain in whitelist_domains + end - Enum.any?(whitelist, fn pattern -> - String.equivalent?(domain, pattern) - end) + defp maybe_get_domain_from_url("http" <> _ = url) do + URI.parse(url).host end + defp maybe_get_domain_from_url(domain), do: domain + def encode_url(url) do base64 = Base.url_encode64(url, @base64_opts) @@ -73,7 +115,7 @@ defmodule Pleroma.Web.MediaProxy do def build_url(sig_base64, url_base64, filename \\ nil) do [ - Pleroma.Config.get([:media_proxy, :base_url], Web.base_url()), + Config.get([:media_proxy, :base_url], Web.base_url()), "proxy", sig_base64, url_base64, diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 4657a4383..9a64b0ef3 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -14,10 +14,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), + {_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)}, :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else - false -> + error when error in [false, {:in_banned_urls, true}] -> send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404)) {:error, :invalid_signature} -> diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index a9f70c43e..0f2d8d1e7 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -7,8 +7,9 @@ defmodule Pleroma.Web.Metadata do def build_tags(params) do providers = [ + Pleroma.Web.Metadata.Providers.RelMe, Pleroma.Web.Metadata.Providers.RestrictIndexing - | Pleroma.Config.get([__MODULE__, :providers], []) + | activated_providers() ] Enum.reduce(providers, "", fn parser, acc -> @@ -42,4 +43,12 @@ defmodule Pleroma.Web.Metadata do def activity_nsfw?(_) do false end + + defp activated_providers do + unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do + [Pleroma.Web.Metadata.Providers.Feed | Pleroma.Config.get([__MODULE__, :providers], [])] + else + [] + end + end end diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 68c871e71..bb1b23208 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -61,7 +61,7 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do @impl Provider def build_tags(%{user: user}) do - with truncated_bio = Utils.scrub_html_and_truncate(user.bio || "") do + with truncated_bio = Utils.scrub_html_and_truncate(user.bio) do [ {:meta, [ diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex index 5d08ce422..df34b033f 100644 --- a/lib/pleroma/web/metadata/twitter_card.ex +++ b/lib/pleroma/web/metadata/twitter_card.ex @@ -40,7 +40,7 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do @impl Provider def build_tags(%{user: user}) do - with truncated_bio = Utils.scrub_html_and_truncate(user.bio || "") do + with truncated_bio = Utils.scrub_html_and_truncate(user.bio) do [ title_tag(user), {:meta, [property: "twitter:description", content: truncated_bio], []}, diff --git a/lib/pleroma/web/nodeinfo/nodeinfo.ex b/lib/pleroma/web/nodeinfo/nodeinfo.ex new file mode 100644 index 000000000..47fa46376 --- /dev/null +++ b/lib/pleroma/web/nodeinfo/nodeinfo.ex @@ -0,0 +1,91 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Nodeinfo.Nodeinfo do + alias Pleroma.Config + alias Pleroma.Stats + alias Pleroma.User + alias Pleroma.Web.Federator.Publisher + alias Pleroma.Web.MastodonAPI.InstanceView + + # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field + # under software. + def get_nodeinfo("2.0") do + stats = Stats.get_stats() + + staff_accounts = + User.all_superusers() + |> Enum.map(fn u -> u.ap_id end) + + federation = InstanceView.federation() + features = InstanceView.features() + + %{ + version: "2.0", + software: %{ + name: Pleroma.Application.name() |> String.downcase(), + version: Pleroma.Application.version() + }, + protocols: Publisher.gather_nodeinfo_protocol_names(), + services: %{ + inbound: [], + outbound: [] + }, + openRegistrations: Config.get([:instance, :registrations_open]), + usage: %{ + users: %{ + total: Map.get(stats, :user_count, 0) + }, + localPosts: Map.get(stats, :status_count, 0) + }, + metadata: %{ + nodeName: Config.get([:instance, :name]), + nodeDescription: Config.get([:instance, :description]), + private: !Config.get([:instance, :public], true), + suggestions: %{ + enabled: false + }, + staffAccounts: staff_accounts, + federation: federation, + pollLimits: Config.get([:instance, :poll_limits]), + postFormats: Config.get([:instance, :allowed_post_formats]), + uploadLimits: %{ + general: Config.get([:instance, :upload_limit]), + avatar: Config.get([:instance, :avatar_upload_limit]), + banner: Config.get([:instance, :banner_upload_limit]), + background: Config.get([:instance, :background_upload_limit]) + }, + fieldsLimits: %{ + maxFields: Config.get([:instance, :max_account_fields]), + maxRemoteFields: Config.get([:instance, :max_remote_account_fields]), + nameLength: Config.get([:instance, :account_field_name_length]), + valueLength: Config.get([:instance, :account_field_value_length]) + }, + accountActivationRequired: Config.get([:instance, :account_activation_required], false), + invitesEnabled: Config.get([:instance, :invites_enabled], false), + mailerEnabled: Config.get([Pleroma.Emails.Mailer, :enabled], false), + features: features, + restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]), + skipThreadContainment: Config.get([:instance, :skip_thread_containment], false) + } + } + end + + def get_nodeinfo("2.1") do + raw_response = get_nodeinfo("2.0") + + updated_software = + raw_response + |> Map.get(:software) + |> Map.put(:repository, Pleroma.Application.repository()) + + raw_response + |> Map.put(:software, updated_software) + |> Map.put(:version, "2.1") + end + + def get_nodeinfo(_version) do + {:error, :missing} + end +end diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 721b599d4..8c7a9e565 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -5,12 +5,8 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do use Pleroma.Web, :controller - alias Pleroma.Config - alias Pleroma.Stats - alias Pleroma.User alias Pleroma.Web - alias Pleroma.Web.Federator.Publisher - alias Pleroma.Web.MastodonAPI.InstanceView + alias Pleroma.Web.Nodeinfo.Nodeinfo def schemas(conn, _params) do response = %{ @@ -29,102 +25,20 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do json(conn, response) end - # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field - # under software. - def raw_nodeinfo do - stats = Stats.get_stats() - - staff_accounts = - User.all_superusers() - |> Enum.map(fn u -> u.ap_id end) - - features = InstanceView.features() - federation = InstanceView.federation() - - %{ - version: "2.0", - software: %{ - name: Pleroma.Application.name() |> String.downcase(), - version: Pleroma.Application.version() - }, - protocols: Publisher.gather_nodeinfo_protocol_names(), - services: %{ - inbound: [], - outbound: [] - }, - openRegistrations: Config.get([:instance, :registrations_open]), - usage: %{ - users: %{ - total: Map.get(stats, :user_count, 0) - }, - localPosts: Map.get(stats, :status_count, 0) - }, - metadata: %{ - nodeName: Config.get([:instance, :name]), - nodeDescription: Config.get([:instance, :description]), - private: !Config.get([:instance, :public], true), - suggestions: %{ - enabled: false - }, - staffAccounts: staff_accounts, - federation: federation, - pollLimits: Config.get([:instance, :poll_limits]), - postFormats: Config.get([:instance, :allowed_post_formats]), - uploadLimits: %{ - general: Config.get([:instance, :upload_limit]), - avatar: Config.get([:instance, :avatar_upload_limit]), - banner: Config.get([:instance, :banner_upload_limit]), - background: Config.get([:instance, :background_upload_limit]) - }, - fieldsLimits: %{ - maxFields: Config.get([:instance, :max_account_fields]), - maxRemoteFields: Config.get([:instance, :max_remote_account_fields]), - nameLength: Config.get([:instance, :account_field_name_length]), - valueLength: Config.get([:instance, :account_field_value_length]) - }, - accountActivationRequired: Config.get([:instance, :account_activation_required], false), - invitesEnabled: Config.get([:instance, :invites_enabled], false), - mailerEnabled: Config.get([Pleroma.Emails.Mailer, :enabled], false), - features: features, - restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]), - skipThreadContainment: Config.get([:instance, :skip_thread_containment], false) - } - } - end - # Schema definition: https://github.com/jhass/nodeinfo/blob/master/schemas/2.0/schema.json # and https://github.com/jhass/nodeinfo/blob/master/schemas/2.1/schema.json - def nodeinfo(conn, %{"version" => "2.0"}) do - conn - |> put_resp_header( - "content-type", - "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#; charset=utf-8" - ) - |> json(raw_nodeinfo()) - end - - def nodeinfo(conn, %{"version" => "2.1"}) do - raw_response = raw_nodeinfo() - - updated_software = - raw_response - |> Map.get(:software) - |> Map.put(:repository, Pleroma.Application.repository()) - - response = - raw_response - |> Map.put(:software, updated_software) - |> Map.put(:version, "2.1") - - conn - |> put_resp_header( - "content-type", - "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.1#; charset=utf-8" - ) - |> json(response) - end - - def nodeinfo(conn, _) do - render_error(conn, :not_found, "Nodeinfo schema version not handled") + def nodeinfo(conn, %{"version" => version}) do + case Nodeinfo.get_nodeinfo(version) do + {:error, :missing} -> + render_error(conn, :not_found, "Nodeinfo schema version not handled") + + node_info -> + conn + |> put_resp_header( + "content-type", + "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#; charset=utf-8" + ) + |> json(node_info) + end end end diff --git a/lib/pleroma/web/oauth/mfa_controller.ex b/lib/pleroma/web/oauth/mfa_controller.ex index 53e19f82e..f102c93e7 100644 --- a/lib/pleroma/web/oauth/mfa_controller.ex +++ b/lib/pleroma/web/oauth/mfa_controller.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.OAuth.MFAController do alias Pleroma.Web.Auth.TOTPAuthenticator alias Pleroma.Web.OAuth.MFAView, as: View alias Pleroma.Web.OAuth.OAuthController + alias Pleroma.Web.OAuth.OAuthView alias Pleroma.Web.OAuth.Token plug(:fetch_session when action in [:show, :verify]) @@ -74,7 +75,7 @@ defmodule Pleroma.Web.OAuth.MFAController do {:ok, %{user: user, authorization: auth}} <- MFA.Token.validate(mfa_token), {:ok, _} <- validates_challenge(user, params), {:ok, token} <- Token.exchange_token(app, auth) do - json(conn, Token.Response.build(user, token)) + json(conn, OAuthView.render("token.json", %{user: user, token: token})) else _error -> conn diff --git a/lib/pleroma/web/oauth/mfa_view.ex b/lib/pleroma/web/oauth/mfa_view.ex index 41d5578dc..5d87db268 100644 --- a/lib/pleroma/web/oauth/mfa_view.ex +++ b/lib/pleroma/web/oauth/mfa_view.ex @@ -5,4 +5,13 @@ defmodule Pleroma.Web.OAuth.MFAView do use Pleroma.Web, :view import Phoenix.HTML.Form + alias Pleroma.MFA + + def render("mfa_response.json", %{token: token, user: user}) do + %{ + error: "mfa_required", + mfa_token: token.token, + supported_challenge_types: MFA.supported_methods(user) + } + end end diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index c557778ca..26e68be42 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -17,6 +17,8 @@ defmodule Pleroma.Web.OAuth.OAuthController do alias Pleroma.Web.OAuth.App alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.MFAController + alias Pleroma.Web.OAuth.MFAView + alias Pleroma.Web.OAuth.OAuthView alias Pleroma.Web.OAuth.Scopes alias Pleroma.Web.OAuth.Token alias Pleroma.Web.OAuth.Token.Strategy.RefreshToken @@ -74,6 +76,13 @@ defmodule Pleroma.Web.OAuth.OAuthController do available_scopes = (app && app.scopes) || [] scopes = Scopes.fetch_scopes(params, available_scopes) + scopes = + if scopes == [] do + available_scopes + else + scopes + end + # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template render(conn, Authenticator.auth_template(), %{ response_type: params["response_type"], @@ -136,7 +145,10 @@ defmodule Pleroma.Web.OAuth.OAuthController do def after_create_authorization(%Plug.Conn{} = conn, %Authorization{} = auth, %{ "authorization" => %{"redirect_uri" => @oob_token_redirect_uri} }) do - render(conn, "oob_authorization_created.html", %{auth: auth}) + # Enforcing the view to reuse the template when calling from other controllers + conn + |> put_view(OAuthView) + |> render("oob_authorization_created.html", %{auth: auth}) end def after_create_authorization(%Plug.Conn{} = conn, %Authorization{} = auth, %{ @@ -188,7 +200,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do {:mfa_required, user, auth, _}, params ) do - {:ok, token} = MFA.Token.create_token(user, auth) + {:ok, token} = MFA.Token.create(user, auth) data = %{ "mfa_token" => token.token, @@ -233,9 +245,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do with {:ok, app} <- Token.Utils.fetch_app(conn), {:ok, %{user: user} = token} <- Token.get_by_refresh_token(app, token), {:ok, token} <- RefreshToken.grant(token) do - response_attrs = %{created_at: Token.Utils.format_created_at(token)} - - json(conn, Token.Response.build(user, token, response_attrs)) + json(conn, OAuthView.render("token.json", %{user: user, token: token})) else _error -> render_invalid_credentials_error(conn) end @@ -247,9 +257,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do {:ok, auth} <- Authorization.get_by_token(app, fixed_token), %User{} = user <- User.get_cached_by_id(auth.user_id), {:ok, token} <- Token.exchange_token(app, auth) do - response_attrs = %{created_at: Token.Utils.format_created_at(token)} - - json(conn, Token.Response.build(user, token, response_attrs)) + json(conn, OAuthView.render("token.json", %{user: user, token: token})) else error -> handle_token_exchange_error(conn, error) @@ -262,12 +270,9 @@ defmodule Pleroma.Web.OAuth.OAuthController do ) do with {:ok, %User{} = user} <- Authenticator.get_user(conn), {:ok, app} <- Token.Utils.fetch_app(conn), - {:account_status, :active} <- {:account_status, User.account_status(user)}, - {:ok, scopes} <- validate_scopes(app, params), - {:ok, auth} <- Authorization.create_authorization(app, user, scopes), - {:mfa_required, _, _, false} <- {:mfa_required, user, auth, MFA.require?(user)}, - {:ok, token} <- Token.exchange_token(app, auth) do - json(conn, Token.Response.build(user, token)) + requested_scopes <- Scopes.fetch_scopes(params, app.scopes), + {:ok, token} <- login(user, app, requested_scopes) do + json(conn, OAuthView.render("token.json", %{user: user, token: token})) else error -> handle_token_exchange_error(conn, error) @@ -290,7 +295,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do with {:ok, app} <- Token.Utils.fetch_app(conn), {:ok, auth} <- Authorization.create_authorization(app, %User{}), {:ok, token} <- Token.exchange_token(app, auth) do - json(conn, Token.Response.build_for_client_credentials(token)) + json(conn, OAuthView.render("token.json", %{token: token})) else _error -> handle_token_exchange_error(conn, :invalid_credentails) @@ -339,6 +344,16 @@ defmodule Pleroma.Web.OAuth.OAuthController do ) end + defp handle_token_exchange_error(%Plug.Conn{} = conn, {:account_status, :approval_pending}) do + render_error( + conn, + :forbidden, + "Your account is awaiting approval.", + %{}, + "awaiting_approval" + ) + end + defp handle_token_exchange_error(%Plug.Conn{} = conn, _error) do render_invalid_credentials_error(conn) end @@ -514,6 +529,8 @@ defmodule Pleroma.Web.OAuth.OAuthController do end end + defp do_create_authorization(conn, auth_attrs, user \\ nil) + defp do_create_authorization( %Plug.Conn{} = conn, %{ @@ -523,19 +540,37 @@ defmodule Pleroma.Web.OAuth.OAuthController do "redirect_uri" => redirect_uri } = auth_attrs }, - user \\ nil + user ) do with {_, {:ok, %User{} = user}} <- {:get_user, (user && {:ok, user}) || Authenticator.get_user(conn)}, %App{} = app <- Repo.get_by(App, client_id: client_id), true <- redirect_uri in String.split(app.redirect_uris), - {:ok, scopes} <- validate_scopes(app, auth_attrs), - {:account_status, :active} <- {:account_status, User.account_status(user)}, - {:ok, auth} <- Authorization.create_authorization(app, user, scopes) do + requested_scopes <- Scopes.fetch_scopes(auth_attrs, app.scopes), + {:ok, auth} <- do_create_authorization(user, app, requested_scopes) do {:ok, auth, user} end end + defp do_create_authorization(%User{} = user, %App{} = app, requested_scopes) + when is_list(requested_scopes) do + with {:account_status, :active} <- {:account_status, User.account_status(user)}, + {:ok, scopes} <- validate_scopes(app, requested_scopes), + {:ok, auth} <- Authorization.create_authorization(app, user, scopes) do + {:ok, auth} + end + end + + # Note: intended to be a private function but opened for AccountController that logs in on signup + @doc "If checks pass, creates authorization and token for given user, app and requested scopes." + def login(%User{} = user, %App{} = app, requested_scopes) when is_list(requested_scopes) do + with {:ok, auth} <- do_create_authorization(user, app, requested_scopes), + {:mfa_required, _, _, false} <- {:mfa_required, user, auth, MFA.require?(user)}, + {:ok, token} <- Token.exchange_token(app, auth) do + {:ok, token} + end + end + # Special case: Local MastodonFE defp redirect_uri(%Plug.Conn{} = conn, "."), do: auth_url(conn, :login) @@ -547,17 +582,20 @@ defmodule Pleroma.Web.OAuth.OAuthController do do: put_session(conn, :registration_id, registration_id) defp build_and_response_mfa_token(user, auth) do - with {:ok, token} <- MFA.Token.create_token(user, auth) do - Token.Response.build_for_mfa_token(user, token) + with {:ok, token} <- MFA.Token.create(user, auth) do + MFAView.render("mfa_response.json", %{token: token, user: user}) end end - @spec validate_scopes(App.t(), map()) :: + @spec validate_scopes(App.t(), map() | list()) :: {:ok, list()} | {:error, :missing_scopes | :unsupported_scopes} - defp validate_scopes(%App{} = app, params) do - params - |> Scopes.fetch_scopes(app.scopes) - |> Scopes.validate(app.scopes) + defp validate_scopes(%App{} = app, params) when is_map(params) do + requested_scopes = Scopes.fetch_scopes(params, app.scopes) + validate_scopes(app, requested_scopes) + end + + defp validate_scopes(%App{} = app, requested_scopes) when is_list(requested_scopes) do + Scopes.validate(requested_scopes, app.scopes) end def default_redirect_uri(%App{} = app) do diff --git a/lib/pleroma/web/oauth/oauth_view.ex b/lib/pleroma/web/oauth/oauth_view.ex index 94ddaf913..f55247ebd 100644 --- a/lib/pleroma/web/oauth/oauth_view.ex +++ b/lib/pleroma/web/oauth/oauth_view.ex @@ -5,4 +5,26 @@ defmodule Pleroma.Web.OAuth.OAuthView do use Pleroma.Web, :view import Phoenix.HTML.Form + + alias Pleroma.Web.OAuth.Token.Utils + + def render("token.json", %{token: token} = opts) do + response = %{ + token_type: "Bearer", + access_token: token.token, + refresh_token: token.refresh_token, + expires_in: expires_in(), + scope: Enum.join(token.scopes, " "), + created_at: Utils.format_created_at(token) + } + + if user = opts[:user] do + response + |> Map.put(:me, user.ap_id) + else + response + end + end + + defp expires_in, do: Pleroma.Config.get([:oauth2, :token_expires_in], 600) end diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 08bb7326d..de37998f2 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -50,7 +50,7 @@ defmodule Pleroma.Web.OAuth.Token do true <- auth.app_id == app.id do user = if auth.user_id, do: User.get_cached_by_id(auth.user_id), else: %User{} - create_token( + create( app, user, %{scopes: auth.scopes} @@ -83,8 +83,22 @@ defmodule Pleroma.Web.OAuth.Token do |> validate_required([:valid_until]) end - @spec create_token(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Changeset.t()} - def create_token(%App{} = app, %User{} = user, attrs \\ %{}) do + @spec create(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Changeset.t()} + def create(%App{} = app, %User{} = user, attrs \\ %{}) do + with {:ok, token} <- do_create(app, user, attrs) do + if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do + Pleroma.Workers.PurgeExpiredToken.enqueue(%{ + token_id: token.id, + valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"), + mod: __MODULE__ + }) + end + + {:ok, token} + end + end + + defp do_create(app, user, attrs) do %__MODULE__{user_id: user.id, app_id: app.id} |> cast(%{scopes: attrs[:scopes] || app.scopes}, [:scopes]) |> validate_required([:scopes, :app_id]) @@ -105,11 +119,6 @@ defmodule Pleroma.Web.OAuth.Token do |> Repo.delete_all() end - def delete_expired_tokens do - Query.get_expired_tokens() - |> Repo.delete_all() - end - def get_user_tokens(%User{id: user_id}) do Query.get_by_user(user_id) |> Query.preload([:app]) diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex deleted file mode 100644 index e3aa4eb7e..000000000 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ /dev/null @@ -1,38 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OAuth.Token.CleanWorker do - @moduledoc """ - The module represents functions to clean an expired OAuth and MFA tokens. - """ - use GenServer - - @ten_seconds 10_000 - @one_day 86_400_000 - - alias Pleroma.MFA - alias Pleroma.Web.OAuth - alias Pleroma.Workers.BackgroundWorker - - def start_link(_), do: GenServer.start_link(__MODULE__, %{}) - - def init(_) do - Process.send_after(self(), :perform, @ten_seconds) - {:ok, nil} - end - - @doc false - def handle_info(:perform, state) do - BackgroundWorker.enqueue("clean_expired_tokens", %{}) - interval = Pleroma.Config.get([:oauth2, :clean_expired_tokens_interval], @one_day) - - Process.send_after(self(), :perform, interval) - {:noreply, state} - end - - def perform(:clean) do - OAuth.Token.delete_expired_tokens() - MFA.Token.delete_expired_tokens() - end -end diff --git a/lib/pleroma/web/oauth/token/query.ex b/lib/pleroma/web/oauth/token/query.ex index 93d6e26ed..fd6d9b112 100644 --- a/lib/pleroma/web/oauth/token/query.ex +++ b/lib/pleroma/web/oauth/token/query.ex @@ -33,12 +33,6 @@ defmodule Pleroma.Web.OAuth.Token.Query do from(q in query, where: q.id == ^id) end - @spec get_expired_tokens(query, DateTime.t() | nil) :: query - def get_expired_tokens(query \\ Token, date \\ nil) do - expired_date = date || Timex.now() - from(q in query, where: fragment("?", q.valid_until) < ^expired_date) - end - @spec get_by_user(query, String.t()) :: query def get_by_user(query \\ Token, user_id) do from(q in query, where: q.user_id == ^user_id) diff --git a/lib/pleroma/web/oauth/token/response.ex b/lib/pleroma/web/oauth/token/response.ex deleted file mode 100644 index 0e72c31e9..000000000 --- a/lib/pleroma/web/oauth/token/response.ex +++ /dev/null @@ -1,45 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OAuth.Token.Response do - @moduledoc false - - alias Pleroma.MFA - alias Pleroma.User - alias Pleroma.Web.OAuth.Token.Utils - - @doc false - def build(%User{} = user, token, opts \\ %{}) do - %{ - token_type: "Bearer", - access_token: token.token, - refresh_token: token.refresh_token, - expires_in: expires_in(), - scope: Enum.join(token.scopes, " "), - me: user.ap_id - } - |> Map.merge(opts) - end - - def build_for_client_credentials(token) do - %{ - token_type: "Bearer", - access_token: token.token, - refresh_token: token.refresh_token, - created_at: Utils.format_created_at(token), - expires_in: expires_in(), - scope: Enum.join(token.scopes, " ") - } - end - - def build_for_mfa_token(user, mfa_token) do - %{ - error: "mfa_required", - mfa_token: mfa_token.token, - supported_challenge_types: MFA.supported_methods(user) - } - end - - defp expires_in, do: Pleroma.Config.get([:oauth2, :token_expires_in], 600) -end diff --git a/lib/pleroma/web/oauth/token/strategy/refresh_token.ex b/lib/pleroma/web/oauth/token/strategy/refresh_token.ex index debc29b0b..625b0fde2 100644 --- a/lib/pleroma/web/oauth/token/strategy/refresh_token.ex +++ b/lib/pleroma/web/oauth/token/strategy/refresh_token.ex @@ -46,7 +46,7 @@ defmodule Pleroma.Web.OAuth.Token.Strategy.RefreshToken do defp create_access_token({:error, error}, _), do: {:error, error} defp create_access_token({:ok, token}, %{app: app, user: user} = token_params) do - Token.create_token(app, user, add_refresh_token(token_params, token.refresh_token)) + Token.create(app, user, add_refresh_token(token_params, token.refresh_token)) end defp add_refresh_token(params, token) do diff --git a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex index c76cbfc48..a0ed41d96 100644 --- a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do import Pleroma.Web.ControllerHelper, only: [json_response: 3, add_link_headers: 2, assign_account_by_id: 2] - alias Ecto.Changeset alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.Plugs.RateLimiter @@ -42,17 +41,6 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do plug( OAuthScopesPlug, - %{scopes: ["write:accounts"]} - # Note: the following actions are not permission-secured in Mastodon: - when action in [ - :update_avatar, - :update_banner, - :update_background - ] - ) - - plug( - OAuthScopesPlug, %{scopes: ["read:favourites"], fallback: :proceed_unauthenticated} when action == :favourites ) @@ -73,56 +61,6 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do end end - @doc "PATCH /api/v1/pleroma/accounts/update_avatar" - def update_avatar(%{assigns: %{user: user}, body_params: %{img: ""}} = conn, _) do - {:ok, _user} = - user - |> Changeset.change(%{avatar: nil}) - |> User.update_and_set_cache() - - json(conn, %{url: nil}) - end - - def update_avatar(%{assigns: %{user: user}, body_params: params} = conn, _params) do - {:ok, %{data: data}} = ActivityPub.upload(params, type: :avatar) - {:ok, _user} = user |> Changeset.change(%{avatar: data}) |> User.update_and_set_cache() - %{"url" => [%{"href" => href} | _]} = data - - json(conn, %{url: href}) - end - - @doc "PATCH /api/v1/pleroma/accounts/update_banner" - def update_banner(%{assigns: %{user: user}, body_params: %{banner: ""}} = conn, _) do - with {:ok, _user} <- User.update_banner(user, %{}) do - json(conn, %{url: nil}) - end - end - - def update_banner(%{assigns: %{user: user}, body_params: params} = conn, _) do - with {:ok, object} <- ActivityPub.upload(%{img: params[:banner]}, type: :banner), - {:ok, _user} <- User.update_banner(user, object.data) do - %{"url" => [%{"href" => href} | _]} = object.data - - json(conn, %{url: href}) - end - end - - @doc "PATCH /api/v1/pleroma/accounts/update_background" - def update_background(%{assigns: %{user: user}, body_params: %{img: ""}} = conn, _) do - with {:ok, _user} <- User.update_background(user, %{}) do - json(conn, %{url: nil}) - end - end - - def update_background(%{assigns: %{user: user}, body_params: params} = conn, _) do - with {:ok, object} <- ActivityPub.upload(params, type: :background), - {:ok, _user} <- User.update_background(user, object.data) do - %{"url" => [%{"href" => href} | _]} = object.data - - json(conn, %{url: href}) - end - end - @doc "GET /api/v1/pleroma/accounts/:id/favourites" def favourites(%{assigns: %{account: %{hide_favorites: true}}} = conn, _params) do render_error(conn, :forbidden, "Can't get favorites") diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex index c8ef3d915..e8a1746d4 100644 --- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex @@ -89,11 +89,11 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do cm_ref <- MessageReference.for_chat_and_object(chat, message) do conn |> put_view(MessageReferenceView) - |> render("show.json", for: user, chat_message_reference: cm_ref) + |> render("show.json", chat_message_reference: cm_ref) end end - def mark_message_as_read(%{assigns: %{user: %{id: user_id} = user}} = conn, %{ + def mark_message_as_read(%{assigns: %{user: %{id: user_id}}} = conn, %{ id: chat_id, message_id: message_id }) do @@ -104,12 +104,15 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do {:ok, cm_ref} <- MessageReference.mark_as_read(cm_ref) do conn |> put_view(MessageReferenceView) - |> render("show.json", for: user, chat_message_reference: cm_ref) + |> render("show.json", chat_message_reference: cm_ref) end end def mark_as_read( - %{body_params: %{last_read_id: last_read_id}, assigns: %{user: %{id: user_id}}} = conn, + %{ + body_params: %{last_read_id: last_read_id}, + assigns: %{user: %{id: user_id}} + } = conn, %{id: id} ) do with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id), @@ -121,7 +124,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do end end - def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{id: id} = params) do + def messages(%{assigns: %{user: %{id: user_id}}} = conn, %{id: id} = params) do with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id) do cm_refs = chat @@ -130,7 +133,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do conn |> put_view(MessageReferenceView) - |> render("index.json", for: user, chat_message_references: cm_refs) + |> render("index.json", chat_message_references: cm_refs) else _ -> conn diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex index d1efdeb5d..657f46324 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex @@ -21,8 +21,8 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do ] ) - @skip_plugs [Pleroma.Plugs.OAuthScopesPlug, Pleroma.Plugs.ExpectPublicOrAuthenticatedCheckPlug] - plug(:skip_plug, @skip_plugs when action in [:archive, :show, :list]) + @skip_plugs [Pleroma.Plugs.OAuthScopesPlug, Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug] + plug(:skip_plug, @skip_plugs when action in [:index, :show, :archive]) defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaEmojiPackOperation @@ -37,14 +37,14 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do end end - def index(conn, _params) do + def index(conn, params) do emoji_path = [:instance, :static_dir] |> Pleroma.Config.get!() |> Path.join("emoji") - with {:ok, packs} <- Pack.list_local() do - json(conn, packs) + with {:ok, packs, count} <- Pack.list_local(page: params.page, page_size: params.page_size) do + json(conn, %{packs: packs, count: count}) else {:error, :create_dir, e} -> conn @@ -60,10 +60,10 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do end end - def show(conn, %{name: name}) do + def show(conn, %{name: name, page: page, page_size: page_size}) do name = String.trim(name) - with {:ok, pack} <- Pack.show(name) do + with {:ok, pack} <- Pack.show(name: name, page: page, page_size: page_size) do json(conn, pack) else {:error, :not_found} -> diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex index 19dcffdf3..7f9254c13 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex @@ -25,7 +25,8 @@ defmodule Pleroma.Web.PleromaAPI.EmojiReactionController do action_fallback(Pleroma.Web.MastodonAPI.FallbackController) def index(%{assigns: %{user: user}} = conn, %{id: activity_id} = params) do - with %Activity{} = activity <- Activity.get_by_id_with_object(activity_id), + with true <- Pleroma.Config.get([:instance, :show_reactions]), + %Activity{} = activity <- Activity.get_by_id_with_object(activity_id), %Object{data: %{"reactions" => reactions}} when is_list(reactions) <- Object.normalize(activity) do reactions = filter(reactions, params) diff --git a/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex index f2112a86e..d4e08b50d 100644 --- a/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex +++ b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do %{ chat_message_reference: %{ id: id, - object: %{data: chat_message}, + object: %{data: chat_message} = object, chat_id: chat_id, unread: unread } @@ -30,7 +30,12 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do attachment: chat_message["attachment"] && StatusView.render("attachment.json", attachment: chat_message["attachment"]), - unread: unread + unread: unread, + card: + StatusView.render( + "card.json", + Pleroma.Web.RichMedia.Helpers.fetch_data_for_object(object) + ) } end diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex index 1c996da11..04dc20d51 100644 --- a/lib/pleroma/web/pleroma_api/views/chat_view.ex +++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex @@ -15,10 +15,11 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do def render("show.json", %{chat: %Chat{} = chat} = opts) do recipient = User.get_cached_by_ap_id(chat.recipient) last_message = opts[:last_message] || MessageReference.last_message_for_chat(chat) + account_view_opts = account_view_opts(opts, recipient) %{ id: chat.id |> to_string(), - account: AccountView.render("show.json", Map.put(opts, :user, recipient)), + account: AccountView.render("show.json", account_view_opts), unread: MessageReference.unread_count_for_chat(chat), last_message: last_message && @@ -27,7 +28,17 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do } end - def render("index.json", %{chats: chats}) do - render_many(chats, __MODULE__, "show.json") + def render("index.json", %{chats: chats} = opts) do + render_many(chats, __MODULE__, "show.json", Map.delete(opts, :chats)) + end + + defp account_view_opts(opts, recipient) do + account_view_opts = Map.put(opts, :user, recipient) + + if Map.has_key?(account_view_opts, :for) do + account_view_opts + else + Map.put(account_view_opts, :skip_visibility_check, true) + end end end diff --git a/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex b/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex index 84d2d303d..e0f98b50a 100644 --- a/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex +++ b/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiReactionView do %{ name: emoji, count: length(users), - accounts: render(AccountView, "index.json", users: users, for: user, as: :user), + accounts: render(AccountView, "index.json", users: users, for: user), me: !!(user && user.ap_id in user_ap_ids) } end diff --git a/lib/pleroma/web/preload.ex b/lib/pleroma/web/preload.ex new file mode 100644 index 000000000..90e454468 --- /dev/null +++ b/lib/pleroma/web/preload.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Preload do + alias Phoenix.HTML + require Logger + + def build_tags(_conn, params) do + preload_data = + Enum.reduce(Pleroma.Config.get([__MODULE__, :providers], []), %{}, fn parser, acc -> + terms = + params + |> parser.generate_terms() + |> Enum.map(fn {k, v} -> {k, Base.encode64(Jason.encode!(v))} end) + |> Enum.into(%{}) + + Map.merge(acc, terms) + end) + + rendered_html = + preload_data + |> Jason.encode!() + |> build_script_tag() + |> HTML.safe_to_string() + + rendered_html + end + + def build_script_tag(content) do + HTML.Tag.content_tag(:script, HTML.raw(content), + id: "initial-results", + type: "application/json" + ) + end +end diff --git a/lib/pleroma/web/preload/instance.ex b/lib/pleroma/web/preload/instance.ex new file mode 100644 index 000000000..50d1f3382 --- /dev/null +++ b/lib/pleroma/web/preload/instance.ex @@ -0,0 +1,50 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Preload.Providers.Instance do + alias Pleroma.Plugs.InstanceStatic + alias Pleroma.Web.MastodonAPI.InstanceView + alias Pleroma.Web.Nodeinfo.Nodeinfo + alias Pleroma.Web.Preload.Providers.Provider + + @behaviour Provider + @instance_url "/api/v1/instance" + @panel_url "/instance/panel.html" + @nodeinfo_url "/nodeinfo/2.0.json" + + @impl Provider + def generate_terms(_params) do + %{} + |> build_info_tag() + |> build_panel_tag() + |> build_nodeinfo_tag() + end + + defp build_info_tag(acc) do + info_data = InstanceView.render("show.json", %{}) + + Map.put(acc, @instance_url, info_data) + end + + defp build_panel_tag(acc) do + instance_path = InstanceStatic.file_path(@panel_url |> to_string()) + + if File.exists?(instance_path) do + panel_data = File.read!(instance_path) + Map.put(acc, @panel_url, panel_data) + else + acc + end + end + + defp build_nodeinfo_tag(acc) do + case Nodeinfo.get_nodeinfo("2.0") do + {:error, _} -> + acc + + nodeinfo_data -> + Map.put(acc, @nodeinfo_url, nodeinfo_data) + end + end +end diff --git a/lib/pleroma/web/preload/provider.ex b/lib/pleroma/web/preload/provider.ex new file mode 100644 index 000000000..7ef595a34 --- /dev/null +++ b/lib/pleroma/web/preload/provider.ex @@ -0,0 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Preload.Providers.Provider do + @callback generate_terms(map()) :: map() +end diff --git a/lib/pleroma/web/preload/timelines.ex b/lib/pleroma/web/preload/timelines.ex new file mode 100644 index 000000000..b279a865d --- /dev/null +++ b/lib/pleroma/web/preload/timelines.ex @@ -0,0 +1,39 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Preload.Providers.Timelines do + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Web.Preload.Providers.Provider + + @behaviour Provider + @public_url "/api/v1/timelines/public" + + @impl Provider + def generate_terms(params) do + build_public_tag(%{}, params) + end + + def build_public_tag(acc, params) do + if Pleroma.Config.restrict_unauthenticated_access?(:timelines, :federated) do + acc + else + Map.put(acc, @public_url, public_timeline(params)) + end + end + + defp public_timeline(%{"path" => ["main", "all"]}), do: get_public_timeline(false) + + defp public_timeline(_params), do: get_public_timeline(true) + + defp get_public_timeline(local_only) do + activities = + ActivityPub.fetch_public_activities(%{ + type: ["Create"], + local_only: local_only + }) + + StatusView.render("index.json", activities: activities, for: nil, as: :activity) + end +end diff --git a/lib/pleroma/web/preload/user.ex b/lib/pleroma/web/preload/user.ex new file mode 100644 index 000000000..b3d2e9b8d --- /dev/null +++ b/lib/pleroma/web/preload/user.ex @@ -0,0 +1,26 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Preload.Providers.User do + alias Pleroma.User + alias Pleroma.Web.MastodonAPI.AccountView + alias Pleroma.Web.Preload.Providers.Provider + + @behaviour Provider + @account_url_base "/api/v1/accounts" + + @impl Provider + def generate_terms(%{user: user}) do + build_accounts_tag(%{}, user) + end + + def generate_terms(_params), do: %{} + + def build_accounts_tag(acc, %User{} = user) do + account_data = AccountView.render("show.json", %{user: user, for: user}) + Map.put(acc, "#{@account_url_base}/#{user.id}", account_data) + end + + def build_accounts_tag(acc, _), do: acc +end diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index cdb827e76..16368485e 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -104,7 +104,7 @@ defmodule Pleroma.Web.Push.Impl do def build_content( %{ - user: %{notification_settings: %{privacy_option: true}} + user: %{notification_settings: %{hide_notification_contents: true}} } = notification, _actor, _object, diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index 8e2b51508..28f75b18d 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -5,7 +5,8 @@ defmodule Pleroma.Web.RelMe do @options [ pool: :media, - max_body: 2_000_000 + max_body: 2_000_000, + recv_timeout: 2_000 ] if Pleroma.Config.get(:env) == :test do @@ -23,18 +24,8 @@ defmodule Pleroma.Web.RelMe do def parse(_), do: {:error, "No URL provided"} defp parse_url(url) do - opts = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - Keyword.merge(@options, - recv_timeout: 2_000, - with_body: true - ) - else - @options - end - with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <- - Pleroma.HTTP.get(url, [], adapter: opts), + Pleroma.HTTP.get(url, [], @options), {:ok, html_tree} <- Floki.parse_document(html), data <- Floki.attribute(html_tree, "link[rel~=me]", "href") ++ diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 1729141e9..bd7f03cbe 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -9,12 +9,18 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Object alias Pleroma.Web.RichMedia.Parser + @options [ + pool: :media, + max_body: 2_000_000, + recv_timeout: 2_000 + ] + @spec validate_page_url(URI.t() | binary()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do - validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld] + validate_tld = Config.get([Pleroma.Formatter, :validate_tld]) page_url - |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld) + |> Linkify.Parser.url?(validate_tld: validate_tld) |> parse_uri(page_url) end @@ -49,11 +55,11 @@ defmodule Pleroma.Web.RichMedia.Helpers do |> hd end - def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do + def fetch_data_for_object(object) do with true <- Config.get([:rich_media, :enabled]), - %Object{} = object <- Object.normalize(activity), false <- object.data["sensitive"] || false, - {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]), + {:ok, page_url} <- + HTML.extract_first_external_url_from_object(object), :ok <- validate_page_url(page_url), {:ok, rich_media} <- Parser.parse(page_url) do %{page_url: page_url, rich_media: rich_media} @@ -62,10 +68,25 @@ defmodule Pleroma.Web.RichMedia.Helpers do end end + def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do + with true <- Config.get([:rich_media, :enabled]), + %Object{} = object <- Object.normalize(activity) do + fetch_data_for_object(object) + else + _ -> %{} + end + end + def fetch_data_for_activity(_), do: %{} def perform(:fetch, %Activity{} = activity) do fetch_data_for_activity(activity) :ok end + + def rich_media_get(url) do + headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] + + Pleroma.HTTP.get(url, headers, @options) + end end diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index ef5ead2da..5727fda18 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -3,10 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser do - @options [ - pool: :media, - max_body: 2_000_000 - ] + require Logger defp parsers do Pleroma.Config.get([:rich_media, :parsers]) @@ -15,17 +12,34 @@ defmodule Pleroma.Web.RichMedia.Parser do def parse(nil), do: {:error, "No URL provided"} if Pleroma.Config.get(:env) == :test do + @spec parse(String.t()) :: {:ok, map()} | {:error, any()} def parse(url), do: parse_url(url) else + @spec parse(String.t()) :: {:ok, map()} | {:error, any()} def parse(url) do - try do - Cachex.fetch!(:rich_media_cache, url, fn _ -> - {:commit, parse_url(url)} - end) - |> set_ttl_based_on_image(url) - rescue - e -> - {:error, "Cachex error: #{inspect(e)}"} + with {:ok, data} <- get_cached_or_parse(url), + {:ok, _} <- set_ttl_based_on_image(data, url) do + {:ok, data} + else + {:error, {:invalid_metadata, data}} = e -> + Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end) + e + + error -> + Logger.error(fn -> "Rich media error for #{url}: #{inspect(error)}" end) + error + end + end + + defp get_cached_or_parse(url) do + case Cachex.fetch!(:rich_media_cache, url, fn _ -> {:commit, parse_url(url)} end) do + {:ok, _data} = res -> + res + + {:error, _} = e -> + ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000) + Cachex.expire(:rich_media_cache, url, ttl) + e end end end @@ -52,19 +66,26 @@ defmodule Pleroma.Web.RichMedia.Parser do config :pleroma, :rich_media, ttl_setters: [MyModule] """ - def set_ttl_based_on_image({:ok, data}, url) do - with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url), - ttl when is_number(ttl) <- get_ttl_from_image(data, url) do - Cachex.expire_at(:rich_media_cache, url, ttl * 1000) - {:ok, data} - else + @spec set_ttl_based_on_image(map(), String.t()) :: + {:ok, Integer.t() | :noop} | {:error, :no_key} + def set_ttl_based_on_image(data, url) do + case get_ttl_from_image(data, url) do + {:ok, ttl} when is_number(ttl) -> + ttl = ttl * 1000 + + case Cachex.expire_at(:rich_media_cache, url, ttl) do + {:ok, true} -> {:ok, ttl} + {:ok, false} -> {:error, :no_key} + end + _ -> - {:ok, data} + {:ok, :noop} end end defp get_ttl_from_image(data, url) do - Pleroma.Config.get([:rich_media, :ttl_setters]) + [:rich_media, :ttl_setters] + |> Pleroma.Config.get() |> Enum.reduce({:ok, nil}, fn module, {:ok, _ttl} -> module.ttl(data, url) @@ -74,34 +95,17 @@ defmodule Pleroma.Web.RichMedia.Parser do end) end - defp parse_url(url) do - opts = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - Keyword.merge(@options, - recv_timeout: 2_000, - with_body: true - ) - else - @options - end - - try do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: opts) - + def parse_url(url) do + with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url), + {:ok, html} <- Floki.parse_document(html) do html - |> parse_html() |> maybe_parse() |> Map.put("url", url) |> clean_parsed_data() |> check_parsed_data() - rescue - e -> - {:error, "Parsing error: #{inspect(e)} #{inspect(__STACKTRACE__)}"} end end - defp parse_html(html), do: Floki.parse_document!(html) - defp maybe_parse(html) do Enum.reduce_while(parsers(), %{}, fn parser, acc -> case parser.parse(html, acc) do @@ -117,7 +121,7 @@ defmodule Pleroma.Web.RichMedia.Parser do end defp check_parsed_data(data) do - {:error, "Found metadata was invalid or incomplete: #{inspect(data)}"} + {:error, {:invalid_metadata, data}} end defp clean_parsed_data(data) do diff --git a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex index 6bdeac89c..1fe6729c3 100644 --- a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex +++ b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do end defp get_oembed_data(url) do - with {:ok, %Tesla.Env{body: json}} <- Pleroma.HTTP.get(url, [], adapter: [pool: :media]) do + with {:ok, %Tesla.Env{body: json}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url) do Jason.decode(json) end end diff --git a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex index 0dc1efdaf..c5aaea2d4 100644 --- a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex +++ b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex @@ -10,20 +10,15 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do |> parse_query_params() |> format_query_params() |> get_expiration_timestamp() + else + {:error, "Not aws signed url #{inspect(image)}"} end end - defp is_aws_signed_url(""), do: nil - defp is_aws_signed_url(nil), do: nil - - defp is_aws_signed_url(image) when is_binary(image) do + defp is_aws_signed_url(image) when is_binary(image) and image != "" do %URI{host: host, query: query} = URI.parse(image) - if String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires") do - image - else - nil - end + String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires") end defp is_aws_signed_url(_), do: nil @@ -46,6 +41,6 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do |> Map.get("X-Amz-Date") |> Timex.parse("{ISO:Basic:Z}") - Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires")) + {:ok, Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))} end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 57570b672..c6433cc53 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -138,6 +138,7 @@ defmodule Pleroma.Web.Router do patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation) patch("/users/activate", AdminAPIController, :user_activate) patch("/users/deactivate", AdminAPIController, :user_deactivate) + patch("/users/approve", AdminAPIController, :user_approve) put("/users/tag", AdminAPIController, :tag_users) delete("/users/tag", AdminAPIController, :untag_users) @@ -209,6 +210,10 @@ defmodule Pleroma.Web.Router do post("/oauth_app", OAuthAppController, :create) patch("/oauth_app/:id", OAuthAppController, :update) delete("/oauth_app/:id", OAuthAppController, :delete) + + get("/media_proxy_caches", MediaProxyCacheController, :index) + post("/media_proxy_caches/delete", MediaProxyCacheController, :delete) + post("/media_proxy_caches/purge", MediaProxyCacheController, :purge) end scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do @@ -324,10 +329,6 @@ defmodule Pleroma.Web.Router do delete("/statuses/:id/reactions/:emoji", EmojiReactionController, :delete) post("/notifications/read", NotificationController, :mark_as_read) - patch("/accounts/update_avatar", AccountController, :update_avatar) - patch("/accounts/update_banner", AccountController, :update_banner) - patch("/accounts/update_background", AccountController, :update_background) - get("/mascot", MascotController, :show) put("/mascot", MascotController, :update) @@ -463,6 +464,7 @@ defmodule Pleroma.Web.Router do scope "/api/web", Pleroma.Web do pipe_through(:authenticated_api) + # Backend-obscure settings blob for MastoFE, don't parse/reuse elsewhere put("/settings", MastoFEController, :put_settings) end @@ -511,10 +513,6 @@ defmodule Pleroma.Web.Router do scope "/api", Pleroma.Web do pipe_through(:config) - get("/help/test", TwitterAPI.UtilController, :help_test) - post("/help/test", TwitterAPI.UtilController, :help_test) - get("/statusnet/config", TwitterAPI.UtilController, :config) - get("/statusnet/version", TwitterAPI.UtilController, :version) get("/pleroma/frontend_configurations", TwitterAPI.UtilController, :frontend_configurations) end @@ -721,7 +719,7 @@ defmodule Pleroma.Web.Router do get("/registration/:token", RedirectController, :registration_page) get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta) get("/api*path", RedirectController, :api_not_implemented) - get("/*path", RedirectController, :redirector) + get("/*path", RedirectController, :redirector_with_preload) options("/*path", RedirectController, :empty) end diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex index d1d2c9b9c..d1d70e556 100644 --- a/lib/pleroma/web/streamer/streamer.ex +++ b/lib/pleroma/web/streamer/streamer.ex @@ -104,7 +104,9 @@ defmodule Pleroma.Web.Streamer do :ok end - def filtered_by_user?(%User{} = user, %Activity{} = item) do + def filtered_by_user?(user, item, streamed_type \\ :activity) + + def filtered_by_user?(%User{} = user, %Activity{} = item, streamed_type) do %{block: blocked_ap_ids, mute: muted_ap_ids, reblog_mute: reblog_muted_ap_ids} = User.outgoing_relationships_ap_ids(user, [:block, :mute, :reblog_mute]) @@ -116,6 +118,9 @@ defmodule Pleroma.Web.Streamer do true <- Enum.all?([blocked_ap_ids, muted_ap_ids], &(item.actor not in &1)), true <- item.data["type"] != "Announce" || item.actor not in reblog_muted_ap_ids, + true <- + !(streamed_type == :activity && item.data["type"] == "Announce" && + parent.data["actor"] == user.ap_id), true <- Enum.all?([blocked_ap_ids, muted_ap_ids], &(parent.data["actor"] not in &1)), true <- MapSet.disjoint?(recipients, recipient_blocks), %{host: item_host} <- URI.parse(item.actor), @@ -130,8 +135,8 @@ defmodule Pleroma.Web.Streamer do end end - def filtered_by_user?(%User{} = user, %Notification{activity: activity}) do - filtered_by_user?(user, activity) + def filtered_by_user?(%User{} = user, %Notification{activity: activity}, _) do + filtered_by_user?(user, activity, :notification) end defp do_stream("direct", item) do diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index 5836ec1e0..51603fe0c 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -37,7 +37,7 @@ } a { - color: color: #d8a070; + color: #d8a070; text-decoration: none; } diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex index 750f65386..5ab59b57b 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex @@ -10,7 +10,7 @@ <%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %> <div class="input"> <%= label f, :code, "Recovery code" %> - <%= text_input f, :code %> + <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, spellcheck: false] %> <%= hidden_input f, :mfa_token, value: @mfa_token %> <%= hidden_input f, :state, value: @state %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex index af6e546b0..af85777eb 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex @@ -10,7 +10,7 @@ <%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %> <div class="input"> <%= label f, :code, "Authentication code" %> - <%= text_input f, :code %> + <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, pattern: "[0-9]*", spellcheck: false] %> <%= hidden_input f, :mfa_token, value: @mfa_token %> <%= hidden_input f, :state, value: @state %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex index 8443d906b..ffabe29a6 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex @@ -1,2 +1,2 @@ <h1>Successfully authorized</h1> -<h2>Token code is <%= @auth.token %></h2> +<h2>Token code is <br><%= @auth.token %></h2> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex index 961aad976..82785c4b9 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex @@ -1,2 +1,2 @@ <h1>Authorization exists</h1> -<h2>Access token is <%= @token.token %></h2> +<h2>Access token is <br><%= @token.token %></h2> diff --git a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex index 521dc9322..072d889e2 100644 --- a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex @@ -135,7 +135,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do end defp handle_follow_error(conn, {:mfa_required, followee, user, _} = _) do - {:ok, %{token: token}} = MFA.Token.create_token(user) + {:ok, %{token: token}} = MFA.Token.create(user) render(conn, "follow_mfa.html", %{followee: followee, mfa_token: token, error: false}) end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index fd2aee175..f02c4075c 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -13,7 +13,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Notification alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.User - alias Pleroma.Web alias Pleroma.Web.CommonAPI alias Pleroma.Web.WebFinger @@ -41,12 +40,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do plug(OAuthScopesPlug, %{scopes: ["write:notifications"]} when action == :notifications_read) - plug(Pleroma.Plugs.SetFormatPlug when action in [:config, :version]) - - def help_test(conn, _params) do - json(conn, "ok") - end - def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do with %User{} = user <- User.get_cached_by_nickname(nick), avatar = User.avatar_url(user) do @@ -88,90 +81,14 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end - def config(%{assigns: %{format: "xml"}} = conn, _params) do - instance = Pleroma.Config.get(:instance) - - response = """ - <config> - <site> - <name>#{Keyword.get(instance, :name)}</name> - <site>#{Web.base_url()}</site> - <textlimit>#{Keyword.get(instance, :limit)}</textlimit> - <closed>#{!Keyword.get(instance, :registrations_open)}</closed> - </site> - </config> - """ - - conn - |> put_resp_content_type("application/xml") - |> send_resp(200, response) - end - - def config(conn, _params) do - instance = Pleroma.Config.get(:instance) - - vapid_public_key = Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) - - uploadlimit = %{ - uploadlimit: to_string(Keyword.get(instance, :upload_limit)), - avatarlimit: to_string(Keyword.get(instance, :avatar_upload_limit)), - backgroundlimit: to_string(Keyword.get(instance, :background_upload_limit)), - bannerlimit: to_string(Keyword.get(instance, :banner_upload_limit)) - } - - data = %{ - name: Keyword.get(instance, :name), - description: Keyword.get(instance, :description), - server: Web.base_url(), - textlimit: to_string(Keyword.get(instance, :limit)), - uploadlimit: uploadlimit, - closed: bool_to_val(Keyword.get(instance, :registrations_open), "0", "1"), - private: bool_to_val(Keyword.get(instance, :public, true), "0", "1"), - vapidPublicKey: vapid_public_key, - accountActivationRequired: - bool_to_val(Keyword.get(instance, :account_activation_required, false)), - invitesEnabled: bool_to_val(Keyword.get(instance, :invites_enabled, false)), - safeDMMentionsEnabled: bool_to_val(Pleroma.Config.get([:instance, :safe_dm_mentions])) - } - - managed_config = Keyword.get(instance, :managed_config) - - data = - if managed_config do - pleroma_fe = Pleroma.Config.get([:frontend_configurations, :pleroma_fe]) - Map.put(data, "pleromafe", pleroma_fe) - else - data - end - - json(conn, %{site: data}) - end - - defp bool_to_val(true), do: "1" - defp bool_to_val(_), do: "0" - defp bool_to_val(true, val, _), do: val - defp bool_to_val(_, _, val), do: val - def frontend_configurations(conn, _params) do config = - Pleroma.Config.get(:frontend_configurations, %{}) + Config.get(:frontend_configurations, %{}) |> Enum.into(%{}) json(conn, config) end - def version(%{assigns: %{format: "xml"}} = conn, _params) do - version = Pleroma.Application.named_version() - - conn - |> put_resp_content_type("application/xml") - |> send_resp(200, "<version>#{version}</version>") - end - - def version(conn, _params) do - json(conn, Pleroma.Application.named_version()) - end - def emoji(conn, _params) do emoji = Enum.reduce(Emoji.get_all(), %{}, fn {code, %Emoji{file: file, tags: tags}}, acc -> diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 5cfb385ac..5d7948507 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -19,6 +19,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do |> Map.put(:nickname, params[:username]) |> Map.put(:name, Map.get(params, :fullname, params[:username])) |> Map.put(:password_confirmation, params[:password]) + |> Map.put(:registration_reason, params[:reason]) if Pleroma.Config.get([:instance, :registrations_open]) do create_user(params, opts) @@ -44,6 +45,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do case User.register(changeset) do {:ok, user} -> + maybe_notify_admins(user) {:ok, user} {:error, changeset} -> @@ -56,9 +58,21 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do end end + defp maybe_notify_admins(%User{} = account) do + if Pleroma.Config.get([:instance, :account_approval_required]) do + User.all_superusers() + |> Enum.filter(fn user -> not is_nil(user.email) end) + |> Enum.each(fn superuser -> + superuser + |> Pleroma.Emails.AdminEmail.new_unapproved_registration(account) + |> Pleroma.Emails.Mailer.deliver_async() + end) + end + end + def password_reset(nickname_or_email) do with true <- is_binary(nickname_or_email), - %User{local: true, email: email} = user when is_binary(email) <- + %User{local: true, email: email, deactivated: false} = user when is_binary(email) <- User.get_by_nickname_or_email(nickname_or_email), {:ok, token_record} <- Pleroma.PasswordResetToken.create_token(user) do user @@ -67,17 +81,8 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do {:ok, :enqueued} else - false -> - {:error, "bad user identifier"} - - %User{local: true, email: nil} -> + _ -> {:ok, :noop} - - %User{local: false} -> - {:error, "remote user"} - - nil -> - {:error, "unknown user"} end end diff --git a/lib/pleroma/web/twitter_api/views/util_view.ex b/lib/pleroma/web/twitter_api/views/util_view.ex index 52054e020..d3bdb4f62 100644 --- a/lib/pleroma/web/twitter_api/views/util_view.ex +++ b/lib/pleroma/web/twitter_api/views/util_view.ex @@ -5,4 +5,18 @@ defmodule Pleroma.Web.TwitterAPI.UtilView do use Pleroma.Web, :view import Phoenix.HTML.Form + alias Pleroma.Web + + def status_net_config(instance) do + """ + <config> + <site> + <name>#{Keyword.get(instance, :name)}</name> + <site>#{Web.base_url()}</site> + <textlimit>#{Keyword.get(instance, :limit)}</textlimit> + <closed>#{!Keyword.get(instance, :registrations_open)}</closed> + </site> + </config> + """ + end end diff --git a/lib/pleroma/web/views/masto_fe_view.ex b/lib/pleroma/web/views/masto_fe_view.ex index c3096006e..b1669d198 100644 --- a/lib/pleroma/web/views/masto_fe_view.ex +++ b/lib/pleroma/web/views/masto_fe_view.ex @@ -9,36 +9,6 @@ defmodule Pleroma.Web.MastoFEView do alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.CustomEmojiView - @default_settings %{ - onboarded: true, - home: %{ - shows: %{ - reblog: true, - reply: true - } - }, - notifications: %{ - alerts: %{ - follow: true, - favourite: true, - reblog: true, - mention: true - }, - shows: %{ - follow: true, - favourite: true, - reblog: true, - mention: true - }, - sounds: %{ - follow: true, - favourite: true, - reblog: true, - mention: true - } - } - } - def initial_state(token, user, custom_emojis) do limit = Config.get([:instance, :limit]) @@ -86,7 +56,7 @@ defmodule Pleroma.Web.MastoFEView do "video\/mp4" ] }, - settings: user.settings || @default_settings, + settings: user.mastofe_settings || %{}, push_subscription: nil, accounts: %{user.id => render(AccountView, "show.json", user: user, for: user)}, custom_emojis: render(CustomEmojiView, "index.json", custom_emojis: custom_emojis), diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 71ccf251a..6629f5356 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -136,12 +136,12 @@ defmodule Pleroma.Web.WebFinger do def find_lrdd_template(domain) do with {:ok, %{status: status, body: body}} when status in 200..299 <- - HTTP.get("http://#{domain}/.well-known/host-meta", []) do + HTTP.get("http://#{domain}/.well-known/host-meta") do get_template_from_xml(body) else _ -> with {:ok, %{body: body, status: status}} when status in 200..299 <- - HTTP.get("https://#{domain}/.well-known/host-meta", []) do + HTTP.get("https://#{domain}/.well-known/host-meta") do get_template_from_xml(body) else e -> {:error, "Can't find LRDD template: #{inspect(e)}"} @@ -149,6 +149,18 @@ defmodule Pleroma.Web.WebFinger do end end + defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do + case find_lrdd_template(domain) do + {:ok, template} -> + String.replace(template, "{uri}", encoded_account) + + _ -> + "https://#{domain}/.well-known/webfinger?resource=#{encoded_account}" + end + end + + defp get_address_from_domain(_, _), do: nil + @spec finger(String.t()) :: {:ok, map()} | {:error, any()} def finger(account) do account = String.trim_leading(account, "@") @@ -163,16 +175,8 @@ defmodule Pleroma.Web.WebFinger do encoded_account = URI.encode("acct:#{account}") - address = - case find_lrdd_template(domain) do - {:ok, template} -> - String.replace(template, "{uri}", encoded_account) - - _ -> - "https://#{domain}/.well-known/webfinger?resource=#{encoded_account}" - end - - with response <- + with address when is_binary(address) <- get_address_from_domain(domain, encoded_account), + response <- HTTP.get( address, [{"accept", "application/xrd+xml,application/jrd+json"}] diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 49352db2a..58226b395 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -11,20 +11,25 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup" @impl Oban.Worker - def perform( - %{ + def perform(%Job{ + args: %{ "op" => "cleanup_attachments", "object" => %{"data" => %{"attachment" => [_ | _] = attachments, "actor" => actor}} - }, - _job - ) do - hrefs = - Enum.flat_map(attachments, fn attachment -> - Enum.map(attachment["url"], & &1["href"]) - end) + } + }) do + attachments + |> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end) + |> fetch_objects + |> prepare_objects(actor, Enum.map(attachments, & &1["name"])) + |> filter_objects + |> do_clean - names = Enum.map(attachments, & &1["name"]) + {:ok, :success} + end + def perform(%Job{args: %{"op" => "cleanup_attachments", "object" => _object}}), do: {:ok, :skip} + + defp do_clean({object_ids, attachment_urls}) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) prefix = @@ -39,68 +44,70 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do "/" ) - # find all objects for copies of the attachments, name and actor doesn't matter here - object_ids_and_hrefs = - from(o in Object, - where: - fragment( - "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", - o.data, - o.data, - ^hrefs - ) - ) - # The query above can be time consumptive on large instances until we - # refactor how uploads are stored - |> Repo.all(timeout: :infinity) - # we should delete 1 object for any given attachment, but don't delete - # files if there are more than 1 object for it - |> Enum.reduce(%{}, fn %{ - id: id, - data: %{ - "url" => [%{"href" => href}], - "actor" => obj_actor, - "name" => name - } - }, - acc -> - Map.update(acc, href, %{id: id, count: 1}, fn val -> - case obj_actor == actor and name in names do - true -> - # set id of the actor's object that will be deleted - %{val | id: id, count: val.count + 1} - - false -> - # another actor's object, just increase count to not delete file - %{val | count: val.count + 1} - end - end) - end) - |> Enum.map(fn {href, %{id: id, count: count}} -> - # only delete files that have single instance - with 1 <- count do - href - |> String.trim_leading("#{base_url}/#{prefix}") - |> uploader.delete_file() - - {id, href} - else - _ -> {id, nil} - end - end) + Enum.each(attachment_urls, fn href -> + href + |> String.trim_leading("#{base_url}/#{prefix}") + |> uploader.delete_file() + end) - object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end) + delete_objects(object_ids) + end - from(o in Object, where: o.id in ^object_ids) - |> Repo.delete_all() + defp delete_objects([_ | _] = object_ids) do + Repo.delete_all(from(o in Object, where: o.id in ^object_ids)) + end - object_ids_and_hrefs - |> Enum.filter(fn {_, href} -> not is_nil(href) end) - |> Enum.map(&elem(&1, 1)) - |> Pleroma.Web.MediaProxy.Invalidation.purge() + defp delete_objects(_), do: :ok - {:ok, :success} + # we should delete 1 object for any given attachment, but don't delete + # files if there are more than 1 object for it + defp filter_objects(objects) do + Enum.reduce(objects, {[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} -> + with 1 <- count do + {ids ++ [id], hrefs ++ [href]} + else + _ -> {ids ++ [id], hrefs} + end + end) + end + + defp prepare_objects(objects, actor, names) do + objects + |> Enum.reduce(%{}, fn %{ + id: id, + data: %{ + "url" => [%{"href" => href}], + "actor" => obj_actor, + "name" => name + } + }, + acc -> + Map.update(acc, href, %{id: id, count: 1}, fn val -> + case obj_actor == actor and name in names do + true -> + # set id of the actor's object that will be deleted + %{val | id: id, count: val.count + 1} + + false -> + # another actor's object, just increase count to not delete file + %{val | count: val.count + 1} + end + end) + end) end - def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} + defp fetch_objects(hrefs) do + from(o in Object, + where: + fragment( + "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", + o.data, + o.data, + ^hrefs + ) + ) + # The query above can be time consumptive on large instances until we + # refactor how uploads are stored + |> Repo.all(timeout: :infinity) + end end diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 57c3a9c3a..cec5a7462 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -11,59 +11,59 @@ defmodule Pleroma.Workers.BackgroundWorker do @impl Oban.Worker - def perform(%{"op" => "deactivate_user", "user_id" => user_id, "status" => status}, _job) do + def perform(%Job{args: %{"op" => "deactivate_user", "user_id" => user_id, "status" => status}}) do user = User.get_cached_by_id(user_id) User.perform(:deactivate_async, user, status) end - def perform(%{"op" => "delete_user", "user_id" => user_id}, _job) do + def perform(%Job{args: %{"op" => "delete_user", "user_id" => user_id}}) do user = User.get_cached_by_id(user_id) User.perform(:delete, user) end - def perform(%{"op" => "force_password_reset", "user_id" => user_id}, _job) do + def perform(%Job{args: %{"op" => "force_password_reset", "user_id" => user_id}}) do user = User.get_cached_by_id(user_id) User.perform(:force_password_reset, user) end - def perform( - %{ + def perform(%Job{ + args: %{ "op" => "blocks_import", "blocker_id" => blocker_id, "blocked_identifiers" => blocked_identifiers - }, - _job - ) do + } + }) do blocker = User.get_cached_by_id(blocker_id) {:ok, User.perform(:blocks_import, blocker, blocked_identifiers)} end - def perform( - %{ + def perform(%Job{ + args: %{ "op" => "follow_import", "follower_id" => follower_id, "followed_identifiers" => followed_identifiers - }, - _job - ) do + } + }) do follower = User.get_cached_by_id(follower_id) {:ok, User.perform(:follow_import, follower, followed_identifiers)} end - def perform(%{"op" => "media_proxy_preload", "message" => message}, _job) do + def perform(%Job{args: %{"op" => "media_proxy_preload", "message" => message}}) do MediaProxyWarmingPolicy.perform(:preload, message) end - def perform(%{"op" => "media_proxy_prefetch", "url" => url}, _job) do + def perform(%Job{args: %{"op" => "media_proxy_prefetch", "url" => url}}) do MediaProxyWarmingPolicy.perform(:prefetch, url) end - def perform(%{"op" => "fetch_data_for_activity", "activity_id" => activity_id}, _job) do + def perform(%Job{args: %{"op" => "fetch_data_for_activity", "activity_id" => activity_id}}) do activity = Activity.get_by_id(activity_id) Pleroma.Web.RichMedia.Helpers.perform(:fetch, activity) end - def perform(%{"op" => "move_following", "origin_id" => origin_id, "target_id" => target_id}, _) do + def perform(%Job{ + args: %{"op" => "move_following", "origin_id" => origin_id, "target_id" => target_id} + }) do origin = User.get_cached_by_id(origin_id) target = User.get_cached_by_id(target_id) diff --git a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex deleted file mode 100644 index a4c3b9516..000000000 --- a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex +++ /dev/null @@ -1,23 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.ClearOauthTokenWorker do - @moduledoc """ - The worker to cleanup expired oAuth tokens. - """ - - use Oban.Worker, queue: "background" - - alias Pleroma.Config - alias Pleroma.Web.OAuth.Token - - @impl Oban.Worker - def perform(_opts, _job) do - if Config.get([:oauth2, :clean_expired_tokens], false) do - Token.delete_expired_tokens() - else - :ok - end - end -end diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex index 7f09ff3cf..0c56f00fb 100644 --- a/lib/pleroma/workers/cron/digest_emails_worker.ex +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do require Logger @impl Oban.Worker - def perform(_opts, _job) do + def perform(_job) do config = Config.get([:email_notifications, :digest]) if config[:active] do @@ -37,9 +37,9 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do ) |> Repo.all() |> send_emails - else - :ok end + + :ok end def send_emails(users) do diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex index 5c816b3fe..8bbaed83d 100644 --- a/lib/pleroma/workers/cron/new_users_digest_worker.ex +++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do use Pleroma.Workers.WorkerHelper, queue: "new_users_digest" @impl Oban.Worker - def perform(_args, _job) do + def perform(_job) do if Pleroma.Config.get([Pleroma.Emails.NewUsersDigestEmail, :enabled]) do today = NaiveDateTime.utc_now() |> Timex.beginning_of_day() @@ -55,11 +55,9 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do |> Repo.all() |> Enum.map(&Pleroma.Emails.NewUsersDigestEmail.new_users(&1, users_and_statuses)) |> Enum.each(&Pleroma.Emails.Mailer.deliver/1) - else - :ok end - else - :ok end + + :ok end end diff --git a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex deleted file mode 100644 index 84b3b84de..000000000 --- a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex +++ /dev/null @@ -1,48 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker do - @moduledoc """ - The worker to purge expired activities. - """ - - use Oban.Worker, queue: "activity_expiration" - - alias Pleroma.Activity - alias Pleroma.ActivityExpiration - alias Pleroma.Config - alias Pleroma.User - alias Pleroma.Web.CommonAPI - - require Logger - - @interval :timer.minutes(1) - - @impl Oban.Worker - def perform(_opts, _job) do - if Config.get([ActivityExpiration, :enabled]) do - Enum.each(ActivityExpiration.due_expirations(@interval), &delete_activity/1) - else - :ok - end - end - - def delete_activity(%ActivityExpiration{activity_id: activity_id}) do - with {:activity, %Activity{} = activity} <- - {:activity, Activity.get_by_id_with_object(activity_id)}, - {:user, %User{} = user} <- {:user, User.get_by_ap_id(activity.object.data["actor"])} do - CommonAPI.delete(activity.id, user) - else - {:activity, _} -> - Logger.error( - "#{__MODULE__} Couldn't delete expired activity: not found activity ##{activity_id}" - ) - - {:user, _} -> - Logger.error( - "#{__MODULE__} Couldn't delete expired activity: not found actorof ##{activity_id}" - ) - end - end -end diff --git a/lib/pleroma/workers/cron/stats_worker.ex b/lib/pleroma/workers/cron/stats_worker.ex deleted file mode 100644 index e9b8d59c4..000000000 --- a/lib/pleroma/workers/cron/stats_worker.ex +++ /dev/null @@ -1,16 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.StatsWorker do - @moduledoc """ - The worker to update peers statistics. - """ - - use Oban.Worker, queue: "background" - - @impl Oban.Worker - def perform(_opts, _job) do - Pleroma.Stats.do_collect() - end -end diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index 6955338a5..32273cfa5 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.MailerWorker do use Pleroma.Workers.WorkerHelper, queue: "mailer" @impl Oban.Worker - def perform(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}, _job) do + def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do encoded_email |> Base.decode64!() |> :erlang.binary_to_term() diff --git a/lib/pleroma/workers/publisher_worker.ex b/lib/pleroma/workers/publisher_worker.ex index daf79efc0..e739c3cd0 100644 --- a/lib/pleroma/workers/publisher_worker.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -8,17 +8,17 @@ defmodule Pleroma.Workers.PublisherWorker do use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" - def backoff(attempt) when is_integer(attempt) do + def backoff(%Job{attempt: attempt}) when is_integer(attempt) do Pleroma.Workers.WorkerHelper.sidekiq_backoff(attempt, 5) end @impl Oban.Worker - def perform(%{"op" => "publish", "activity_id" => activity_id}, _job) do + def perform(%Job{args: %{"op" => "publish", "activity_id" => activity_id}}) do activity = Activity.get_by_id(activity_id) Federator.perform(:publish, activity) end - def perform(%{"op" => "publish_one", "module" => module_name, "params" => params}, _job) do + def perform(%Job{args: %{"op" => "publish_one", "module" => module_name, "params" => params}}) do params = Map.new(params, fn {k, v} -> {String.to_atom(k), v} end) Federator.perform(:publish_one, String.to_atom(module_name), params) end diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex new file mode 100644 index 000000000..c168890a2 --- /dev/null +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -0,0 +1,72 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.PurgeExpiredActivity do + @moduledoc """ + Worker which purges expired activity. + """ + + use Oban.Worker, queue: :activity_expiration, max_attempts: 1 + + import Ecto.Query + + alias Pleroma.Activity + + @spec enqueue(map()) :: + {:ok, Oban.Job.t()} + | {:error, :expired_activities_disabled} + | {:error, :expiration_too_close} + def enqueue(args) do + with true <- enabled?() do + {scheduled_at, args} = Map.pop(args, :expires_at) + + args + |> new(scheduled_at: scheduled_at) + |> Oban.insert() + end + end + + @impl true + def perform(%Oban.Job{args: %{"activity_id" => id}}) do + with %Activity{} = activity <- find_activity(id), + %Pleroma.User{} = user <- find_user(activity.object.data["actor"]) do + Pleroma.Web.CommonAPI.delete(activity.id, user) + end + end + + defp enabled? do + with false <- Pleroma.Config.get([__MODULE__, :enabled], false) do + {:error, :expired_activities_disabled} + end + end + + defp find_activity(id) do + with nil <- Activity.get_by_id_with_object(id) do + {:error, :activity_not_found} + end + end + + defp find_user(ap_id) do + with nil <- Pleroma.User.get_by_ap_id(ap_id) do + {:error, :user_not_found} + end + end + + def get_expiration(id) do + from(j in Oban.Job, + where: j.state == "scheduled", + where: j.queue == "activity_expiration", + where: fragment("?->>'activity_id' = ?", j.args, ^id) + ) + |> Pleroma.Repo.one() + end + + @spec expires_late_enough?(DateTime.t()) :: boolean() + def expires_late_enough?(scheduled_at) do + now = DateTime.utc_now() + diff = DateTime.diff(scheduled_at, now, :millisecond) + min_lifetime = Pleroma.Config.get([__MODULE__, :min_lifetime], 600) + diff > :timer.seconds(min_lifetime) + end +end diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex new file mode 100644 index 000000000..a81e0cd28 --- /dev/null +++ b/lib/pleroma/workers/purge_expired_token.ex @@ -0,0 +1,29 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.PurgeExpiredToken do + @moduledoc """ + Worker which purges expired OAuth tokens + """ + + use Oban.Worker, queue: :token_expiration, max_attempts: 1 + + @spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) :: + {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()} + def enqueue(args) do + {scheduled_at, args} = Map.pop(args, :valid_until) + + args + |> __MODULE__.new(scheduled_at: scheduled_at) + |> Oban.insert() + end + + @impl true + def perform(%Oban.Job{args: %{"token_id" => id, "mod" => module}}) do + module + |> String.to_existing_atom() + |> Pleroma.Repo.get(id) + |> Pleroma.Repo.delete() + end +end diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index f7a7124f3..1b97af1a8 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Workers.ReceiverWorker do use Pleroma.Workers.WorkerHelper, queue: "federator_incoming" @impl Oban.Worker - def perform(%{"op" => "incoming_ap_doc", "params" => params}, _job) do + def perform(%Job{args: %{"op" => "incoming_ap_doc", "params" => params}}) do Federator.perform(:incoming_ap_doc, params) end end diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index ec6534f21..27e2e3386 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -8,13 +8,7 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher" @impl Oban.Worker - def perform( - %{ - "op" => "fetch_remote", - "id" => id - } = args, - _job - ) do + def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do {:ok, _object} = Fetcher.fetch_object_from_id(id, depth: args["depth"]) end end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index 97d1efbfb..dd9986fe4 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do require Logger @impl Oban.Worker - def perform(%{"activity_id" => activity_id}, _job) do + def perform(%Job{args: %{"activity_id" => activity_id}}) do if Config.get([ScheduledActivity, :enabled]) do case Pleroma.Repo.get(ScheduledActivity, activity_id) do %ScheduledActivity{} = scheduled_activity -> diff --git a/lib/pleroma/workers/transmogrifier_worker.ex b/lib/pleroma/workers/transmogrifier_worker.ex index 11239ca5e..15f36375c 100644 --- a/lib/pleroma/workers/transmogrifier_worker.ex +++ b/lib/pleroma/workers/transmogrifier_worker.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Workers.TransmogrifierWorker do use Pleroma.Workers.WorkerHelper, queue: "transmogrifier" @impl Oban.Worker - def perform(%{"op" => "user_upgrade", "user_id" => user_id}, _job) do + def perform(%Job{args: %{"op" => "user_upgrade", "user_id" => user_id}}) do user = User.get_cached_by_id(user_id) Pleroma.Web.ActivityPub.Transmogrifier.perform(:user_upgrade, user) end diff --git a/lib/pleroma/workers/web_pusher_worker.ex b/lib/pleroma/workers/web_pusher_worker.ex index 58ad25e39..0cfdc6a6f 100644 --- a/lib/pleroma/workers/web_pusher_worker.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.WebPusherWorker do use Pleroma.Workers.WorkerHelper, queue: "web_push" @impl Oban.Worker - def perform(%{"op" => "web_push", "notification_id" => notification_id}, _job) do + def perform(%Job{args: %{"op" => "web_push", "notification_id" => notification_id}}) do notification = Notification |> Repo.get(notification_id) diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index d1f90c35b..7d1289be2 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -32,6 +32,8 @@ defmodule Pleroma.Workers.WorkerHelper do queue: unquote(queue), max_attempts: 1 + alias Oban.Job + def enqueue(op, params, worker_args \\ []) do params = Map.merge(%{"op" => op}, params) queue_atom = String.to_atom(unquote(queue)) @@ -39,7 +41,7 @@ defmodule Pleroma.Workers.WorkerHelper do unquote(caller_module) |> apply(:new, [params, worker_args]) - |> Pleroma.Repo.insert() + |> Oban.insert() end end end |