From 514c899275a32e6ef63305f9424c50344d41b12e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 11 Feb 2020 10:12:57 +0300 Subject: adding gun adapter --- lib/mix/tasks/pleroma/benchmark.ex | 39 ++ lib/mix/tasks/pleroma/emoji.ex | 9 +- lib/pleroma/application.ex | 90 +++-- lib/pleroma/config/config_db.ex | 11 - lib/pleroma/config/transfer_task.ex | 43 ++- lib/pleroma/gun/api.ex | 26 ++ lib/pleroma/gun/api/mock.ex | 151 ++++++++ lib/pleroma/gun/conn.ex | 29 ++ lib/pleroma/gun/gun.ex | 45 +++ lib/pleroma/http/adapter.ex | 64 ++++ lib/pleroma/http/adapter/gun.ex | 123 ++++++ lib/pleroma/http/adapter/hackney.ex | 41 ++ lib/pleroma/http/connection.ex | 113 ++++-- lib/pleroma/http/http.ex | 154 +++++--- lib/pleroma/http/request.ex | 23 ++ lib/pleroma/http/request_builder.ex | 105 ++---- lib/pleroma/object/fetcher.ex | 6 +- lib/pleroma/otp_version.ex | 63 ++++ lib/pleroma/pool/connections.ex | 415 +++++++++++++++++++++ lib/pleroma/pool/pool.ex | 22 ++ lib/pleroma/pool/request.ex | 72 ++++ lib/pleroma/pool/supervisor.ex | 36 ++ lib/pleroma/reverse_proxy/client.ex | 26 +- lib/pleroma/reverse_proxy/client/hackney.ex | 24 ++ lib/pleroma/reverse_proxy/client/tesla.ex | 87 +++++ lib/pleroma/reverse_proxy/reverse_proxy.ex | 20 +- .../activity_pub/mrf/media_proxy_warming_policy.ex | 14 +- lib/pleroma/web/rel_me.ex | 18 +- lib/pleroma/web/rich_media/parser.ex | 18 +- lib/pleroma/web/web_finger/web_finger.ex | 2 +- 30 files changed, 1647 insertions(+), 242 deletions(-) create mode 100644 lib/pleroma/gun/api.ex create mode 100644 lib/pleroma/gun/api/mock.ex create mode 100644 lib/pleroma/gun/conn.ex create mode 100644 lib/pleroma/gun/gun.ex create mode 100644 lib/pleroma/http/adapter.ex create mode 100644 lib/pleroma/http/adapter/gun.ex create mode 100644 lib/pleroma/http/adapter/hackney.ex create mode 100644 lib/pleroma/http/request.ex create mode 100644 lib/pleroma/otp_version.ex create mode 100644 lib/pleroma/pool/connections.ex create mode 100644 lib/pleroma/pool/pool.ex create mode 100644 lib/pleroma/pool/request.ex create mode 100644 lib/pleroma/pool/supervisor.ex create mode 100644 lib/pleroma/reverse_proxy/client/hackney.ex create mode 100644 lib/pleroma/reverse_proxy/client/tesla.ex (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index 84dccf7f3..01e079136 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -74,4 +74,43 @@ defmodule Mix.Tasks.Pleroma.Benchmark do inputs: inputs ) end + + def run(["adapters"]) do + start_pleroma() + + :ok = + Pleroma.Pool.Connections.open_conn( + "https://httpbin.org/stream-bytes/1500", + :gun_connections + ) + + Process.sleep(1_500) + + Benchee.run( + %{ + "Without conn and without pool" => fn -> + {:ok, %Tesla.Env{}} = + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], + adapter: [pool: :no_pool, receive_conn: false] + ) + end, + "Without conn and with pool" => fn -> + {:ok, %Tesla.Env{}} = + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], + adapter: [receive_conn: false] + ) + end, + "With reused conn and without pool" => fn -> + {:ok, %Tesla.Env{}} = + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], + adapter: [pool: :no_pool] + ) + end, + "With reused conn and with pool" => fn -> + {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500") + end + }, + parallel: 10 + ) + end end diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 24d999707..b4e8d3a0b 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -4,13 +4,13 @@ defmodule Mix.Tasks.Pleroma.Emoji do use Mix.Task + import Mix.Pleroma @shortdoc "Manages emoji packs" @moduledoc File.read!("docs/administration/CLI_tasks/emoji.md") def run(["ls-packs" | args]) do - Mix.Pleroma.start_pleroma() - Application.ensure_all_started(:hackney) + start_pleroma() {options, [], []} = parse_global_opts(args) @@ -36,8 +36,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do end def run(["get-packs" | args]) do - Mix.Pleroma.start_pleroma() - Application.ensure_all_started(:hackney) + start_pleroma() {options, pack_names, []} = parse_global_opts(args) @@ -135,7 +134,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do end def run(["gen-pack", src]) do - Application.ensure_all_started(:hackney) + start_pleroma() proposed_name = Path.basename(src) |> Path.rootname() name = String.trim(IO.gets("Pack name [#{proposed_name}]: ")) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 27758cf94..df6d3a98d 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -3,8 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Application do - import Cachex.Spec use Application + + import Cachex.Spec + + alias Pleroma.Config + require Logger @name Mix.Project.config()[:name] @@ -18,9 +22,9 @@ defmodule Pleroma.Application do def repository, do: @repository def user_agent do - case Pleroma.Config.get([:http, :user_agent], :default) do + case Config.get([:http, :user_agent], :default) do :default -> - info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>" + info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>" named_version() <> "; " <> info custom -> @@ -32,7 +36,7 @@ defmodule Pleroma.Application do # for more information on OTP Applications def start(_type, _args) do Pleroma.HTML.compile_scrubbers() - Pleroma.Config.DeprecationWarnings.warn() + Config.DeprecationWarnings.warn() Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.Repo.check_migrations_applied!() setup_instrumenters() @@ -42,17 +46,17 @@ defmodule Pleroma.Application do children = [ Pleroma.Repo, - Pleroma.Config.TransferTask, + Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, Pleroma.Plugs.RateLimiter.Supervisor ] ++ cachex_children() ++ - hackney_pool_children() ++ + http_pools_children(Config.get(:env)) ++ [ Pleroma.Stats, Pleroma.JobQueueMonitor, - {Oban, Pleroma.Config.get(Oban)} + {Oban, Config.get(Oban)} ] ++ task_children(@env) ++ streamer_child(@env) ++ @@ -62,6 +66,18 @@ defmodule Pleroma.Application do Pleroma.Gopher.Server ] + case Pleroma.OTPVersion.check_version() do + :ok -> :ok + {:error, version} -> raise " + !!!OTP VERSION WARNING!!! + You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. + " + :undefined -> raise " + !!!OTP VERSION WARNING!!! + To support correct handling of unordered certificates chains - OTP version must be > 22.2. + " + end + # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Pleroma.Supervisor] @@ -69,7 +85,7 @@ defmodule Pleroma.Application do end def load_custom_modules do - dir = Pleroma.Config.get([:modules, :runtime_dir]) + dir = Config.get([:modules, :runtime_dir]) if dir && File.exists?(dir) do dir @@ -110,20 +126,6 @@ defmodule Pleroma.Application do Pleroma.Web.Endpoint.Instrumenter.setup() end - def enabled_hackney_pools do - [:media] ++ - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - [:federation] - else - [] - end ++ - if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do - [:upload] - else - [] - end - end - defp cachex_children do [ build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), @@ -145,7 +147,7 @@ defmodule Pleroma.Application do do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) defp seconds_valid_interval, - do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])) + do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid])) defp build_cachex(type, opts), do: %{ @@ -154,7 +156,7 @@ defmodule Pleroma.Application do type: :worker } - defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) + defp chat_enabled?, do: Config.get([:chat, :enabled]) defp streamer_child(:test), do: [] @@ -168,13 +170,6 @@ defmodule Pleroma.Application do defp chat_child(_, _), do: [] - defp hackney_pool_children do - for pool <- enabled_hackney_pools() do - options = Pleroma.Config.get([:hackney_pools, pool]) - :hackney_pool.child_spec(pool, options) - end - end - defp task_children(:test) do [ %{ @@ -199,4 +194,37 @@ defmodule Pleroma.Application do } ] end + + # start hackney and gun pools in tests + defp http_pools_children(:test) do + hackney_options = Config.get([:hackney_pools, :federation]) + hackney_pool = :hackney_pool.child_spec(:federation, hackney_options) + [hackney_pool, Pleroma.Pool.Supervisor] + end + + defp http_pools_children(_) do + :tesla + |> Application.get_env(:adapter) + |> http_pools() + end + + defp http_pools(Tesla.Adapter.Hackney) do + pools = [:federation, :media] + + pools = + if Config.get([Pleroma.Upload, :proxy_remote]) do + [:upload | pools] + else + pools + end + + for pool <- pools do + options = Config.get([:hackney_pools, pool]) + :hackney_pool.child_spec(pool, options) + end + end + + defp http_pools(Tesla.Adapter.Gun), do: [Pleroma.Pool.Supervisor] + + defp http_pools(_), do: [] end diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index 119251bee..bdacefa97 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -278,8 +278,6 @@ defmodule Pleroma.ConfigDB do } end - defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]} - defp do_convert(entity) when is_tuple(entity) do value = entity @@ -323,15 +321,6 @@ defmodule Pleroma.ConfigDB do {:proxy_url, {do_transform_string(type), parse_host(host), port}} end - defp do_transform(%{"tuple" => [":partial_chain", entity]}) do - {partial_chain, []} = - entity - |> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "") - |> Code.eval_string() - - {:partial_chain, partial_chain} - end - defp do_transform(%{"tuple" => entity}) do Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end) end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 6c5ba1f95..251074aaa 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -18,7 +18,10 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, Oban}, {:pleroma, :rate_limit}, {:pleroma, :markup}, - {:plerome, :streamer} + {:pleroma, :streamer}, + {:pleroma, :pools}, + {:pleroma, :connections_pool}, + {:tesla, :adapter} ] @reboot_time_subkeys [ @@ -74,6 +77,28 @@ defmodule Pleroma.Config.TransferTask do end end + defp group_for_restart(:logger, key, _, merged_value) do + # change logger configuration in runtime, without restart + if Keyword.keyword?(merged_value) and + key not in [:compile_time_application, :backends, :compile_time_purge_matching] do + Logger.configure_backend(key, merged_value) + else + Logger.configure([{key, merged_value}]) + end + + nil + end + + defp group_for_restart(:tesla, _, _, _), do: :pleroma + + defp group_for_restart(group, _, _, _) when group != :pleroma, do: group + + defp group_for_restart(group, key, value, _) do + if pleroma_need_restart?(group, key, value) do + group + end + end + defp merge_and_update(setting) do try do key = ConfigDB.from_string(setting.key) @@ -95,21 +120,7 @@ defmodule Pleroma.Config.TransferTask do :ok = update_env(group, key, merged_value) - if group != :logger do - if group != :pleroma or pleroma_need_restart?(group, key, value) do - group - end - else - # change logger configuration in runtime, without restart - if Keyword.keyword?(merged_value) and - key not in [:compile_time_application, :backends, :compile_time_purge_matching] do - Logger.configure_backend(key, merged_value) - else - Logger.configure([{key, merged_value}]) - end - - nil - end + group_for_restart(group, key, value, merged_value) rescue error -> error_msg = diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex new file mode 100644 index 000000000..a0c3c5415 --- /dev/null +++ b/lib/pleroma/gun/api.ex @@ -0,0 +1,26 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API do + @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} + @callback info(pid()) :: map() + @callback close(pid()) :: :ok + @callback await_up(pid) :: {:ok, atom()} | {:error, atom()} + @callback connect(pid(), map()) :: reference() + @callback await(pid(), reference()) :: {:response, :fin, 200, []} + + def open(host, port, opts), do: api().open(host, port, opts) + + def info(pid), do: api().info(pid) + + def close(pid), do: api().close(pid) + + def await_up(pid), do: api().await_up(pid) + + def connect(pid, opts), do: api().connect(pid, opts) + + def await(pid, ref), do: api().await(pid, ref) + + defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun) +end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex new file mode 100644 index 000000000..0134b016e --- /dev/null +++ b/lib/pleroma/gun/api/mock.ex @@ -0,0 +1,151 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API.Mock do + @behaviour Pleroma.Gun.API + + alias Pleroma.Gun.API + + @impl API + def open('some-domain.com', 443, _) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "https", + origin_host: 'some-domain.com', + origin_port: 443 + }) + + {:ok, conn_pid} + end + + @impl API + def open(ip, port, _) + when ip in [{10_755, 10_368, 61_708, 131, 64_206, 45_068, 0, 9_694}, {127, 0, 0, 1}] and + port in [80, 443] do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + scheme = if port == 443, do: "https", else: "http" + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: scheme, + origin_host: ip, + origin_port: port + }) + + {:ok, conn_pid} + end + + @impl API + def open('localhost', 1234, %{ + protocols: [:socks], + proxy: {:socks5, 'localhost', 1234}, + socks_opts: %{host: 'proxy-socks.com', port: 80, version: 5} + }) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: 'proxy-socks.com', + origin_port: 80 + }) + + {:ok, conn_pid} + end + + @impl API + def open('localhost', 1234, %{ + protocols: [:socks], + proxy: {:socks4, 'localhost', 1234}, + socks_opts: %{ + host: 'proxy-socks.com', + port: 443, + protocols: [:http2], + tls_opts: [], + transport: :tls, + version: 4 + } + }) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "https", + origin_host: 'proxy-socks.com', + origin_port: 443 + }) + + {:ok, conn_pid} + end + + @impl API + def open('gun-not-up.com', 80, _opts), do: {:error, :timeout} + + @impl API + def open('example.com', port, _) when port in [443, 115] do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "https", + origin_host: 'example.com', + origin_port: 443 + }) + + {:ok, conn_pid} + end + + @impl API + def open(domain, 80, _) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: domain, + origin_port: 80 + }) + + {:ok, conn_pid} + end + + @impl API + def open({127, 0, 0, 1}, 8123, _) do + Task.start_link(fn -> Process.sleep(1_000) end) + end + + @impl API + def open('localhost', 9050, _) do + Task.start_link(fn -> Process.sleep(1_000) end) + end + + @impl API + def await_up(_pid), do: {:ok, :http} + + @impl API + def connect(pid, %{host: _, port: 80}) do + ref = make_ref() + Registry.register(API.Mock, ref, pid) + ref + end + + @impl API + def connect(pid, %{host: _, port: 443, protocols: [:http2], transport: :tls}) do + ref = make_ref() + Registry.register(API.Mock, ref, pid) + ref + end + + @impl API + def await(pid, ref) do + [{_, ^pid}] = Registry.lookup(API.Mock, ref) + {:response, :fin, 200, []} + end + + @impl API + def info(pid) do + [{_, info}] = Registry.lookup(API.Mock, pid) + info + end + + @impl API + def close(_pid), do: :ok +end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex new file mode 100644 index 000000000..2474829d6 --- /dev/null +++ b/lib/pleroma/gun/conn.ex @@ -0,0 +1,29 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.Conn do + @moduledoc """ + Struct for gun connection data + """ + @type gun_state :: :up | :down + @type conn_state :: :active | :idle + + @type t :: %__MODULE__{ + conn: pid(), + gun_state: gun_state(), + conn_state: conn_state(), + used_by: [pid()], + last_reference: pos_integer(), + crf: float(), + retries: pos_integer() + } + + defstruct conn: nil, + gun_state: :open, + conn_state: :init, + used_by: [], + last_reference: 0, + crf: 1, + retries: 0 +end diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex new file mode 100644 index 000000000..4a1bbc95f --- /dev/null +++ b/lib/pleroma/gun/gun.ex @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun do + @behaviour Pleroma.Gun.API + + alias Pleroma.Gun.API + + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :socks_opts, + :ws_opts + ] + + @impl API + def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys)) + + @impl API + defdelegate info(pid), to: :gun + + @impl API + defdelegate close(pid), to: :gun + + @impl API + defdelegate await_up(pid), to: :gun + + @impl API + defdelegate connect(pid, opts), to: :gun + + @impl API + defdelegate await(pid, ref), to: :gun + + @spec flush(pid() | reference()) :: :ok + defdelegate flush(pid), to: :gun +end diff --git a/lib/pleroma/http/adapter.ex b/lib/pleroma/http/adapter.ex new file mode 100644 index 000000000..6166a3eb4 --- /dev/null +++ b/lib/pleroma/http/adapter.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Adapter do + alias Pleroma.HTTP.Connection + + @type proxy :: + {Connection.host(), pos_integer()} + | {Connection.proxy_type(), pos_integer()} + @type host_type :: :domain | :ip + + @callback options(keyword(), URI.t()) :: keyword() + @callback after_request(keyword()) :: :ok + + @spec options(keyword(), URI.t()) :: keyword() + def options(opts, _uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + maybe_add_proxy(opts, format_proxy(proxy)) + end + + @spec maybe_get_conn(URI.t(), keyword()) :: keyword() + def maybe_get_conn(_uri, opts), do: opts + + @spec after_request(keyword()) :: :ok + def after_request(_opts), do: :ok + + @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil + def format_proxy(nil), do: nil + + def format_proxy(proxy_url) do + with {:ok, host, port} <- Connection.parse_proxy(proxy_url) do + {host, port} + else + {:ok, type, host, port} -> {type, host, port} + _ -> nil + end + end + + @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() + def maybe_add_proxy(opts, nil), do: opts + def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) + + @spec domain_or_fallback(String.t()) :: charlist() + def domain_or_fallback(host) do + case domain_or_ip(host) do + {:domain, domain} -> domain + {:ip, _ip} -> to_charlist(host) + end + end + + @spec domain_or_ip(String.t()) :: {host_type(), Connection.host()} + def domain_or_ip(host) do + charlist = to_charlist(host) + + case :inet.parse_address(charlist) do + {:error, :einval} -> + {:domain, :idna.encode(charlist)} + + {:ok, ip} when is_tuple(ip) and tuple_size(ip) in [4, 8] -> + {:ip, ip} + end + end +end diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex new file mode 100644 index 000000000..f25afeda7 --- /dev/null +++ b/lib/pleroma/http/adapter/gun.ex @@ -0,0 +1,123 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Adapter.Gun do + @behaviour Pleroma.HTTP.Adapter + + alias Pleroma.HTTP.Adapter + + require Logger + + alias Pleroma.Pool.Connections + + @defaults [ + connect_timeout: 20_000, + domain_lookup_timeout: 5_000, + tls_handshake_timeout: 5_000, + retry_timeout: 100, + await_up_timeout: 5_000 + ] + + @spec options(keyword(), URI.t()) :: keyword() + def options(connection_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + + @defaults + |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> add_original(uri) + |> add_scheme_opts(uri) + |> Adapter.maybe_add_proxy(Adapter.format_proxy(proxy)) + |> maybe_get_conn(uri, connection_opts) + end + + @spec after_request(keyword()) :: :ok + def after_request(opts) do + with conn when not is_nil(conn) <- opts[:conn], + body_as when body_as != :chunks <- opts[:body_as] do + Connections.checkout(conn, self(), :gun_connections) + end + + :ok + end + + defp add_original(opts, %URI{host: host, port: port}) do + formatted_host = Adapter.domain_or_fallback(host) + + Keyword.put(opts, :original, "#{formatted_host}:#{port}") + end + + defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + + defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do + adapter_opts = [ + certificates_verification: true, + tls_opts: [ + verify: :verify_peer, + cacertfile: CAStore.file_path(), + depth: 20, + reuse_sessions: false, + verify_fun: + {&:ssl_verify_hostname.verify_fun/3, [check_hostname: Adapter.domain_or_fallback(host)]} + ] + ] + + adapter_opts = + if port != 443 do + Keyword.put(adapter_opts, :transport, :tls) + else + adapter_opts + end + + Keyword.merge(opts, adapter_opts) + end + + defp maybe_get_conn(adapter_opts, uri, connection_opts) do + {receive_conn?, opts} = + adapter_opts + |> Keyword.merge(connection_opts) + |> Keyword.pop(:receive_conn, true) + + if Connections.alive?(:gun_connections) and receive_conn? do + try_to_get_conn(uri, opts) + else + opts + end + end + + defp try_to_get_conn(uri, opts) do + try do + case Connections.checkin(uri, :gun_connections) do + nil -> + Logger.info( + "Gun connections pool checkin was not succesfull. Trying to open conn for next request." + ) + + :ok = Connections.open_conn(uri, :gun_connections, opts) + opts + + conn when is_pid(conn) -> + Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri(uri)}") + + opts + |> Keyword.put(:conn, conn) + |> Keyword.put(:close_conn, false) + end + rescue + error -> + Logger.warn("Gun connections pool checkin caused error #{inspect(error)}") + opts + catch + :exit, {:timeout, _} -> + Logger.info( + "Gun connections pool checkin with timeout error #{Connections.compose_uri(uri)}" + ) + + opts + + :exit, error -> + Logger.warn("Gun pool checkin exited with error #{inspect(error)}") + opts + end + end +end diff --git a/lib/pleroma/http/adapter/hackney.ex b/lib/pleroma/http/adapter/hackney.ex new file mode 100644 index 000000000..00db30083 --- /dev/null +++ b/lib/pleroma/http/adapter/hackney.ex @@ -0,0 +1,41 @@ +defmodule Pleroma.HTTP.Adapter.Hackney do + @behaviour Pleroma.HTTP.Adapter + + @defaults [ + connect_timeout: 10_000, + recv_timeout: 20_000, + follow_redirect: true, + force_redirect: true, + pool: :federation + ] + + @spec options(keyword(), URI.t()) :: keyword() + def options(connection_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + + @defaults + |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> Keyword.merge(connection_opts) + |> add_scheme_opts(uri) + |> Pleroma.HTTP.Adapter.maybe_add_proxy(proxy) + end + + defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + + defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do + ssl_opts = [ + ssl_options: [ + # Workaround for remote server certificate chain issues + partial_chain: &:hackney_connect.partial_chain/1, + + # We don't support TLS v1.3 yet + versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"], + server_name_indication: to_charlist(host) + ] + ] + + Keyword.merge(opts, ssl_opts) + end + + def after_request(_), do: :ok +end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 7e2c6f5e8..85918341a 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -4,40 +4,99 @@ defmodule Pleroma.HTTP.Connection do @moduledoc """ - Connection for http-requests. + Configure Tesla.Client with default and customized adapter options. """ + @type ip_address :: ipv4_address() | ipv6_address() + @type ipv4_address :: {0..255, 0..255, 0..255, 0..255} + @type ipv6_address :: + {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535} + @type proxy_type() :: :socks4 | :socks5 + @type host() :: charlist() | ip_address() - @hackney_options [ - connect_timeout: 10_000, - recv_timeout: 20_000, - follow_redirect: true, - force_redirect: true, - pool: :federation - ] - @adapter Application.get_env(:tesla, :adapter) + @defaults [pool: :federation] - @doc """ - Configure a client connection + require Logger - # Returns + alias Pleroma.Config + alias Pleroma.HTTP.Adapter - Tesla.Env.client + @doc """ + Merge default connection & adapter options with received ones. """ - @spec new(Keyword.t()) :: Tesla.Env.client() - def new(opts \\ []) do - Tesla.client([], {@adapter, hackney_options(opts)}) + + @spec options(URI.t(), keyword()) :: keyword() + def options(%URI{} = uri, opts \\ []) do + @defaults + |> pool_timeout() + |> Keyword.merge(opts) + |> adapter().options(uri) + end + + defp pool_timeout(opts) do + timeout = + Config.get([:pools, opts[:pool], :timeout]) || Config.get([:pools, :default, :timeout]) + + Keyword.merge(opts, timeout: timeout) end - # fetch Hackney options - # - def hackney_options(opts) do - options = Keyword.get(opts, :adapter, []) - adapter_options = Pleroma.Config.get([:http, :adapter], []) - proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) - - @hackney_options - |> Keyword.merge(adapter_options) - |> Keyword.merge(options) - |> Keyword.merge(proxy: proxy_url) + @spec after_request(keyword()) :: :ok + def after_request(opts), do: adapter().after_request(opts) + + defp adapter do + case Application.get_env(:tesla, :adapter) do + Tesla.Adapter.Gun -> Adapter.Gun + Tesla.Adapter.Hackney -> Adapter.Hackney + _ -> Adapter + end + end + + @spec parse_proxy(String.t() | tuple() | nil) :: + {:ok, host(), pos_integer()} + | {:ok, proxy_type(), host(), pos_integer()} + | {:error, atom()} + | nil + + def parse_proxy(nil), do: nil + + def parse_proxy(proxy) when is_binary(proxy) do + with [host, port] <- String.split(proxy, ":"), + {port, ""} <- Integer.parse(port) do + {:ok, parse_host(host), port} + else + {_, _} -> + Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + {:error, :error_parsing_port_in_proxy} + + :error -> + Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + {:error, :error_parsing_port_in_proxy} + + _ -> + Logger.warn("parsing proxy fail #{inspect(proxy)}") + {:error, :error_parsing_proxy} + end + end + + def parse_proxy(proxy) when is_tuple(proxy) do + with {type, host, port} <- proxy do + {:ok, type, parse_host(host), port} + else + _ -> + Logger.warn("parsing proxy fail #{inspect(proxy)}") + {:error, :error_parsing_proxy} + end + end + + @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address() + def parse_host(host) when is_list(host), do: host + def parse_host(host) when is_atom(host), do: to_charlist(host) + + def parse_host(host) when is_binary(host) do + host = to_charlist(host) + + case :inet.parse_address(host) do + {:error, :einval} -> host + {:ok, ip} -> ip + end end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index dec24458a..ad47dc936 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -4,21 +4,47 @@ defmodule Pleroma.HTTP do @moduledoc """ - + Wrapper for `Tesla.request/2`. """ alias Pleroma.HTTP.Connection + alias Pleroma.HTTP.Request alias Pleroma.HTTP.RequestBuilder, as: Builder + alias Tesla.Client + alias Tesla.Env + + require Logger @type t :: __MODULE__ @doc """ - Builds and perform http request. + Performs GET request. + + See `Pleroma.HTTP.request/5` + """ + @spec get(Request.url() | nil, Request.headers(), keyword()) :: + nil | {:ok, Env.t()} | {:error, any()} + def get(url, headers \\ [], options \\ []) + def get(nil, _, _), do: nil + def get(url, headers, options), do: request(:get, url, "", headers, options) + + @doc """ + Performs POST request. + + See `Pleroma.HTTP.request/5` + """ + @spec post(Request.url(), String.t(), Request.headers(), keyword()) :: + {:ok, Env.t()} | {:error, any()} + def post(url, body, headers \\ [], options \\ []), + do: request(:post, url, body, headers, options) + + @doc """ + Builds and performs http request. # Arguments: `method` - :get, :post, :put, :delete - `url` - `body` + `url` - full url + `body` - request body `headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]` `options` - custom, per-request middleware or adapter options @@ -26,61 +52,97 @@ defmodule Pleroma.HTTP do `{:ok, %Tesla.Env{}}` or `{:error, error}` """ - def request(method, url, body \\ "", headers \\ [], options \\ []) do + @spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) :: + {:ok, Env.t()} | {:error, any()} + def request(method, url, body, headers, options) when is_binary(url) do + with uri <- URI.parse(url), + received_adapter_opts <- Keyword.get(options, :adapter, []), + adapter_opts <- Connection.options(uri, received_adapter_opts), + options <- put_in(options[:adapter], adapter_opts), + params <- Keyword.get(options, :params, []), + request <- build_request(method, headers, options, url, body, params), + client <- Tesla.client([Tesla.Middleware.FollowRedirects], tesla_adapter()), + pid <- Process.whereis(adapter_opts[:pool]) do + pool_alive? = + if tesla_adapter() == Tesla.Adapter.Gun do + if pid, do: Process.alive?(pid), else: false + else + false + end + + request_opts = + adapter_opts + |> Enum.into(%{}) + |> Map.put(:env, Pleroma.Config.get([:env])) + |> Map.put(:pool_alive?, pool_alive?) + + response = + request( + client, + request, + request_opts + ) + + Connection.after_request(adapter_opts) + + response + end + end + + @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()} + def request(%Client{} = client, request, %{env: :test}), do: request_try(client, request) + + def request(%Client{} = client, request, %{body_as: :chunks}) do + request_try(client, request) + end + + def request(%Client{} = client, request, %{pool_alive?: false}) do + request_try(client, request) + end + + def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do try do - options = - process_request_options(options) - |> process_sni_options(url) - - params = Keyword.get(options, :params, []) - - %{} - |> Builder.method(method) - |> Builder.headers(headers) - |> Builder.opts(options) - |> Builder.url(url) - |> Builder.add_param(:body, :body, body) - |> Builder.add_param(:query, :query, params) - |> Enum.into([]) - |> (&Tesla.request(Connection.new(options), &1)).() + :poolboy.transaction( + pool, + &Pleroma.Pool.Request.execute(&1, client, request, timeout + 500), + timeout + 1_000 + ) rescue e -> {:error, e} catch + :exit, {:timeout, _} -> + Logger.warn("Receive response from pool failed #{request[:url]}") + {:error, :recv_pool_timeout} + :exit, e -> {:error, e} end end - defp process_sni_options(options, nil), do: options - - defp process_sni_options(options, url) do - uri = URI.parse(url) - host = uri.host |> to_charlist() - - case uri.scheme do - "https" -> options ++ [ssl: [server_name_indication: host]] - _ -> options + @spec request_try(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} + def request_try(client, request) do + try do + Tesla.request(client, request) + rescue + e -> + {:error, e} + catch + :exit, e -> + {:error, e} end end - def process_request_options(options) do - Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options) + defp build_request(method, headers, options, url, body, params) do + Builder.new() + |> Builder.method(method) + |> Builder.headers(headers) + |> Builder.opts(options) + |> Builder.url(url) + |> Builder.add_param(:body, :body, body) + |> Builder.add_param(:query, :query, params) + |> Builder.convert_to_keyword() end - @doc """ - Performs GET request. - - See `Pleroma.HTTP.request/5` - """ - def get(url, headers \\ [], options \\ []), - do: request(:get, url, "", headers, options) - - @doc """ - Performs POST request. - - See `Pleroma.HTTP.request/5` - """ - def post(url, body, headers \\ [], options \\ []), - do: request(:post, url, body, headers, options) + defp tesla_adapter, do: Application.get_env(:tesla, :adapter) end diff --git a/lib/pleroma/http/request.ex b/lib/pleroma/http/request.ex new file mode 100644 index 000000000..891d88d53 --- /dev/null +++ b/lib/pleroma/http/request.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Request do + @moduledoc """ + Request struct. + """ + defstruct method: :get, url: "", query: [], headers: [], body: "", opts: [] + + @type method :: :head | :get | :delete | :trace | :options | :post | :put | :patch + @type url :: String.t() + @type headers :: [{String.t(), String.t()}] + + @type t :: %__MODULE__{ + method: method(), + url: url(), + query: keyword(), + headers: headers(), + body: String.t(), + opts: keyword() + } +end diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index e23457999..491acd0f9 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -7,77 +7,54 @@ defmodule Pleroma.HTTP.RequestBuilder do Helper functions for building Tesla requests """ - @doc """ - Specify the request method when building a request - - ## Parameters - - - request (Map) - Collected request options - - m (atom) - Request method + alias Pleroma.HTTP.Request + alias Tesla.Multipart - ## Returns - - Map + @doc """ + Creates new request """ - @spec method(map(), atom) :: map() - def method(request, m) do - Map.put_new(request, :method, m) - end + @spec new(Request.t()) :: Request.t() + def new(%Request{} = request \\ %Request{}), do: request @doc """ Specify the request method when building a request + """ + @spec method(Request.t(), Request.method()) :: Request.t() + def method(request, m), do: %{request | method: m} - ## Parameters - - - request (Map) - Collected request options - - u (String) - Request URL - - ## Returns - - Map + @doc """ + Specify the request method when building a request """ - @spec url(map(), String.t()) :: map() - def url(request, u) do - Map.put_new(request, :url, u) - end + @spec url(Request.t(), Request.url()) :: Request.t() + def url(request, u), do: %{request | url: u} @doc """ Add headers to the request """ - @spec headers(map(), list(tuple)) :: map() - def headers(request, header_list) do - header_list = + @spec headers(Request.t(), Request.headers()) :: Request.t() + def headers(request, headers) do + headers_list = if Pleroma.Config.get([:http, :send_user_agent]) do - header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + headers ++ [{"user-agent", Pleroma.Application.user_agent()}] else - header_list + headers end - Map.put_new(request, :headers, header_list) + %{request | headers: headers_list} end @doc """ Add custom, per-request middleware or adapter options to the request """ - @spec opts(map(), Keyword.t()) :: map() - def opts(request, options) do - Map.put_new(request, :opts, options) - end + @spec opts(Request.t(), keyword()) :: Request.t() + def opts(request, options), do: %{request | opts: options} + # NOTE: isn't used anywhere @doc """ Add optional parameters to the request - ## Parameters - - - request (Map) - Collected request options - - definitions (Map) - Map of parameter name to parameter location. - - options (KeywordList) - The provided optional parameters - - ## Returns - - Map """ - @spec add_optional_params(map(), %{optional(atom) => atom}, keyword()) :: map() + @spec add_optional_params(Request.t(), %{optional(atom) => atom}, keyword()) :: map() def add_optional_params(request, _, []), do: request def add_optional_params(request, definitions, [{key, value} | tail]) do @@ -94,49 +71,43 @@ defmodule Pleroma.HTTP.RequestBuilder do @doc """ Add optional parameters to the request - - ## Parameters - - - request (Map) - Collected request options - - location (atom) - Where to put the parameter - - key (atom) - The name of the parameter - - value (any) - The value of the parameter - - ## Returns - - Map """ - @spec add_param(map(), atom, atom, any()) :: map() - def add_param(request, :query, :query, values), do: Map.put(request, :query, values) + @spec add_param(Request.t(), atom(), atom(), any()) :: Request.t() + def add_param(request, :query, :query, values), do: %{request | query: values} - def add_param(request, :body, :body, value), do: Map.put(request, :body, value) + def add_param(request, :body, :body, value), do: %{request | body: value} def add_param(request, :body, key, value) do request - |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0) + |> Map.put(:body, Multipart.new()) |> Map.update!( :body, - &Tesla.Multipart.add_field( + &Multipart.add_field( &1, key, Jason.encode!(value), - headers: [{:"Content-Type", "application/json"}] + headers: [{"content-type", "application/json"}] ) ) end def add_param(request, :file, name, path) do request - |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0) - |> Map.update!(:body, &Tesla.Multipart.add_file(&1, path, name: name)) + |> Map.put(:body, Multipart.new()) + |> Map.update!(:body, &Multipart.add_file(&1, path, name: name)) end def add_param(request, :form, name, value) do - request - |> Map.update(:body, %{name => value}, &Map.put(&1, name, value)) + Map.update(request, :body, %{name => value}, &Map.put(&1, name, value)) end def add_param(request, location, key, value) do Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}])) end + + def convert_to_keyword(request) do + request + |> Map.from_struct() + |> Enum.into([]) + end end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 037c42339..5e9bf1574 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -137,7 +137,7 @@ defmodule Pleroma.Object.Fetcher do date: date }) - [{:Signature, signature}] + [{"signature", signature}] end defp sign_fetch(headers, id, date) do @@ -150,7 +150,7 @@ defmodule Pleroma.Object.Fetcher do defp maybe_date_fetch(headers, date) do if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do - headers ++ [{:Date, date}] + headers ++ [{"date", date}] else headers end @@ -162,7 +162,7 @@ defmodule Pleroma.Object.Fetcher do date = Pleroma.Signature.signed_date() headers = - [{:Accept, "application/activity+json"}] + [{"accept", "application/activity+json"}] |> maybe_date_fetch(date) |> sign_fetch(id, date) diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex new file mode 100644 index 000000000..0be189304 --- /dev/null +++ b/lib/pleroma/otp_version.ex @@ -0,0 +1,63 @@ +defmodule Pleroma.OTPVersion do + @type check_status() :: :undefined | {:error, String.t()} | :ok + + require Logger + + @spec check_version() :: check_status() + def check_version do + # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version + paths = [ + Path.join(:code.root_dir(), "OTP_VERSION"), + Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"]) + ] + + :tesla + |> Application.get_env(:adapter) + |> get_and_check_version(paths) + end + + @spec get_and_check_version(module(), [Path.t()]) :: check_status() + def get_and_check_version(Tesla.Adapter.Gun, paths) do + paths + |> check_files() + |> check_version() + end + + def get_and_check_version(_, _), do: :ok + + defp check_files([]), do: nil + + defp check_files([path | paths]) do + if File.exists?(path) do + File.read!(path) + else + check_files(paths) + end + end + + defp check_version(nil), do: :undefined + + defp check_version(version) do + try do + version = String.replace(version, ~r/\r|\n|\s/, "") + + formatted = + version + |> String.split(".") + |> Enum.map(&String.to_integer/1) + |> Enum.take(2) + + with [major, minor] when length(formatted) == 2 <- formatted, + true <- (major == 22 and minor >= 2) or major > 22 do + :ok + else + false -> {:error, version} + _ -> :undefined + end + rescue + _ -> :undefined + catch + _ -> :undefined + end + end +end diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex new file mode 100644 index 000000000..1ed16d1c1 --- /dev/null +++ b/lib/pleroma/pool/connections.ex @@ -0,0 +1,415 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool.Connections do + use GenServer + + require Logger + + @type domain :: String.t() + @type conn :: Pleroma.Gun.Conn.t() + + @type t :: %__MODULE__{ + conns: %{domain() => conn()}, + opts: keyword() + } + + defstruct conns: %{}, opts: [] + + alias Pleroma.Gun.API + alias Pleroma.Gun.Conn + + @spec start_link({atom(), keyword()}) :: {:ok, pid()} + def start_link({name, opts}) do + GenServer.start_link(__MODULE__, opts, name: name) + end + + @impl true + def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}} + + @spec checkin(String.t() | URI.t(), atom()) :: pid() | nil + def checkin(url, name) + def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) + + def checkin(%URI{} = uri, name) do + timeout = Pleroma.Config.get([:connections_pool, :receive_connection_timeout], 250) + + GenServer.call( + name, + {:checkin, uri}, + timeout + ) + end + + @spec open_conn(String.t() | URI.t(), atom(), keyword()) :: :ok + def open_conn(url, name, opts \\ []) + def open_conn(url, name, opts) when is_binary(url), do: open_conn(URI.parse(url), name, opts) + + def open_conn(%URI{} = uri, name, opts) do + pool_opts = Pleroma.Config.get([:connections_pool], []) + + opts = + opts + |> Enum.into(%{}) + |> Map.put_new(:receive, false) + |> Map.put_new(:retry, pool_opts[:retry] || 5) + |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) + |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + + GenServer.cast(name, {:open_conn, %{opts: opts, uri: uri}}) + end + + @spec alive?(atom()) :: boolean() + def alive?(name) do + pid = Process.whereis(name) + if pid, do: Process.alive?(pid), else: false + end + + @spec get_state(atom()) :: t() + def get_state(name) do + GenServer.call(name, :state) + end + + @spec checkout(pid(), pid(), atom()) :: :ok + def checkout(conn, pid, name) do + GenServer.cast(name, {:checkout, conn, pid}) + end + + @impl true + def handle_cast({:open_conn, %{opts: opts, uri: uri}}, state) do + Logger.debug("opening new #{compose_uri(uri)}") + max_connections = state.opts[:max_connections] + + key = compose_key(uri) + + if Enum.count(state.conns) < max_connections do + open_conn(key, uri, state, opts) + else + try_to_open_conn(key, uri, state, opts) + end + end + + @impl true + def handle_cast({:checkout, conn_pid, pid}, state) do + Logger.debug("checkout #{inspect(conn_pid)}") + + state = + with true <- Process.alive?(conn_pid), + {key, conn} <- find_conn(state.conns, conn_pid), + used_by <- List.keydelete(conn.used_by, pid, 0) do + conn_state = + if used_by == [] do + :idle + else + conn.conn_state + end + + put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) + else + false -> + Logger.warn("checkout for closed conn #{inspect(conn_pid)}") + state + + nil -> + Logger.info("checkout for alive conn #{inspect(conn_pid)}, but is not in state") + state + end + + {:noreply, state} + end + + @impl true + def handle_call({:checkin, uri}, from, state) do + Logger.debug("checkin #{compose_uri(uri)}") + key = compose_key(uri) + + case state.conns[key] do + %{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up -> + Logger.debug("reusing conn #{compose_uri(uri)}") + + with time <- :os.system_time(:second), + last_reference <- time - current_conn.last_reference, + current_crf <- crf(last_reference, 100, current_conn.crf), + state <- + put_in(state.conns[key], %{ + current_conn + | last_reference: time, + crf: current_crf, + conn_state: :active, + used_by: [from | current_conn.used_by] + }) do + {:reply, conn, state} + end + + %{gun_state: gun_state} when gun_state == :down -> + {:reply, nil, state} + + nil -> + {:reply, nil, state} + end + end + + @impl true + def handle_call(:state, _from, state), do: {:reply, state, state} + + @impl true + def handle_info({:gun_up, conn_pid, _protocol}, state) do + state = + with true <- Process.alive?(conn_pid), + conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), + {key, conn} <- find_conn(state.conns, conn_pid, conn_key), + time <- :os.system_time(:second), + last_reference <- time - conn.last_reference, + current_crf <- crf(last_reference, 100, conn.crf) do + put_in(state.conns[key], %{ + conn + | gun_state: :up, + last_reference: time, + crf: current_crf, + conn_state: :active, + retries: 0 + }) + else + :error_gun_info -> + Logger.warn(":gun.info caused error") + state + + false -> + Logger.warn(":gun_up message for closed conn #{inspect(conn_pid)}") + state + + nil -> + Logger.warn( + ":gun_up message for alive conn #{inspect(conn_pid)}, but deleted from state" + ) + + :ok = API.close(conn_pid) + + state + end + + {:noreply, state} + end + + @impl true + def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do + # we can't get info on this pid, because pid is dead + state = + with true <- Process.alive?(conn_pid), + {key, conn} <- find_conn(state.conns, conn_pid) do + if conn.retries == 5 do + Logger.debug("closing conn if retries is eq 5 #{inspect(conn_pid)}") + :ok = API.close(conn.conn) + + put_in( + state.conns, + Map.delete(state.conns, key) + ) + else + put_in(state.conns[key], %{ + conn + | gun_state: :down, + retries: conn.retries + 1 + }) + end + else + false -> + # gun can send gun_down for closed conn, maybe connection is not closed yet + Logger.warn(":gun_down message for closed conn #{inspect(conn_pid)}") + state + + nil -> + Logger.warn( + ":gun_down message for alive conn #{inspect(conn_pid)}, but deleted from state" + ) + + :ok = API.close(conn_pid) + + state + end + + {:noreply, state} + end + + defp compose_key(%URI{scheme: scheme, host: host, port: port}), do: "#{scheme}:#{host}:#{port}" + + defp compose_key_gun_info(pid) do + try do + # sometimes :gun.info can raise MatchError, which lead to pool terminate + %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = API.info(pid) + + host = + case :inet.ntoa(origin_host) do + {:error, :einval} -> origin_host + ip -> ip + end + + "#{scheme}:#{host}:#{port}" + rescue + _ -> :error_gun_info + end + end + + defp find_conn(conns, conn_pid) do + Enum.find(conns, fn {_key, conn} -> + conn.conn == conn_pid + end) + end + + defp find_conn(conns, conn_pid, conn_key) do + Enum.find(conns, fn {key, conn} -> + key == conn_key and conn.conn == conn_pid + end) + end + + defp open_conn(key, uri, state, %{proxy: {proxy_host, proxy_port}} = opts) do + connect_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + + with open_opts <- Map.delete(opts, :tls_opts), + {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), + {:ok, _} <- API.await_up(conn), + stream <- API.connect(conn, connect_opts), + {:response, :fin, 200, _} <- API.await(conn, stream), + state <- + put_in(state.conns[key], %Conn{ + conn: conn, + gun_state: :up, + conn_state: :active, + last_reference: :os.system_time(:second) + }) do + {:noreply, state} + else + error -> + Logger.warn( + "Received error on opening connection with http proxy #{uri.scheme}://#{ + compose_uri(uri) + }: #{inspect(error)}" + ) + + {:noreply, state} + end + end + + defp open_conn(key, uri, state, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do + version = + proxy_type + |> to_string() + |> String.last() + |> case do + "4" -> 4 + _ -> 5 + end + + socks_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + |> Map.put(:version, version) + + opts = + opts + |> Map.put(:protocols, [:socks]) + |> Map.put(:socks_opts, socks_opts) + + with {:ok, conn} <- API.open(proxy_host, proxy_port, opts), + {:ok, _} <- API.await_up(conn), + state <- + put_in(state.conns[key], %Conn{ + conn: conn, + gun_state: :up, + conn_state: :active, + last_reference: :os.system_time(:second) + }) do + {:noreply, state} + else + error -> + Logger.warn( + "Received error on opening connection with socks proxy #{uri.scheme}://#{ + compose_uri(uri) + }: #{inspect(error)}" + ) + + {:noreply, state} + end + end + + defp open_conn(key, %URI{host: host, port: port} = uri, state, opts) do + Logger.debug("opening conn #{compose_uri(uri)}") + {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + + with {:ok, conn} <- API.open(host, port, opts), + {:ok, _} <- API.await_up(conn), + state <- + put_in(state.conns[key], %Conn{ + conn: conn, + gun_state: :up, + conn_state: :active, + last_reference: :os.system_time(:second) + }) do + Logger.debug("new conn opened #{compose_uri(uri)}") + Logger.debug("replying to the call #{compose_uri(uri)}") + {:noreply, state} + else + error -> + Logger.warn( + "Received error on opening connection #{uri.scheme}://#{compose_uri(uri)}: #{ + inspect(error) + }" + ) + + {:noreply, state} + end + end + + defp destination_opts(%URI{host: host, port: port}) do + {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + %{host: host, port: port} + end + + defp add_http2_opts(opts, "https", tls_opts) do + Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts}) + end + + defp add_http2_opts(opts, _, _), do: opts + + @spec get_unused_conns(map()) :: [{domain(), conn()}] + def get_unused_conns(conns) do + conns + |> Enum.filter(fn {_k, v} -> + v.conn_state == :idle and v.used_by == [] + end) + |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> + x.crf <= y.crf and x.last_reference <= y.last_reference + end) + end + + defp try_to_open_conn(key, uri, state, opts) do + Logger.debug("try to open conn #{compose_uri(uri)}") + + with [{close_key, least_used} | _conns] <- get_unused_conns(state.conns), + :ok <- API.close(least_used.conn), + state <- + put_in( + state.conns, + Map.delete(state.conns, close_key) + ) do + Logger.debug( + "least used conn found and closed #{inspect(least_used.conn)} #{compose_uri(uri)}" + ) + + open_conn(key, uri, state, opts) + else + [] -> {:noreply, state} + end + end + + def crf(current, steps, crf) do + 1 + :math.pow(0.5, current / steps) * crf + end + + def compose_uri(%URI{} = uri), do: "#{uri.host}#{uri.path}" +end diff --git a/lib/pleroma/pool/pool.ex b/lib/pleroma/pool/pool.ex new file mode 100644 index 000000000..a7ae64ce4 --- /dev/null +++ b/lib/pleroma/pool/pool.ex @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool do + def child_spec(opts) do + poolboy_opts = + opts + |> Keyword.put(:worker_module, Pleroma.Pool.Request) + |> Keyword.put(:name, {:local, opts[:name]}) + |> Keyword.put(:size, opts[:size]) + |> Keyword.put(:max_overflow, opts[:max_overflow]) + + %{ + id: opts[:id] || {__MODULE__, make_ref()}, + start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]}, + restart: :permanent, + shutdown: 5000, + type: :worker + } + end +end diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex new file mode 100644 index 000000000..2c3574561 --- /dev/null +++ b/lib/pleroma/pool/request.ex @@ -0,0 +1,72 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool.Request do + use GenServer + + require Logger + + def start_link(args) do + GenServer.start_link(__MODULE__, args) + end + + @impl true + def init(_), do: {:ok, []} + + @spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) :: + {:ok, Tesla.Env.t()} | {:error, any()} + def execute(pid, client, request, timeout) do + GenServer.call(pid, {:execute, client, request}, timeout) + end + + @impl true + def handle_call({:execute, client, request}, _from, state) do + response = Pleroma.HTTP.request_try(client, request) + + {:reply, response, state} + end + + @impl true + def handle_info({:gun_data, _conn, stream, _, _}, state) do + # in some cases if we reuse conn and got {:error, :body_too_large} + # gun continues to send messages to this process, + # so we flush messages for this request + :ok = :gun.flush(stream) + + {:noreply, state} + end + + @impl true + def handle_info({:gun_up, _conn, _protocol}, state) do + {:noreply, state} + end + + @impl true + def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do + # don't flush messages here, because gun can reconnect + {:noreply, state} + end + + @impl true + def handle_info({:gun_error, _conn, stream, _error}, state) do + :ok = :gun.flush(stream) + {:noreply, state} + end + + @impl true + def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do + {:noreply, state} + end + + @impl true + def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do + {:noreply, state} + end + + @impl true + def handle_info(msg, state) do + Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}") + {:noreply, state} + end +end diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex new file mode 100644 index 000000000..32be2264d --- /dev/null +++ b/lib/pleroma/pool/supervisor.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool.Supervisor do + use Supervisor + + alias Pleroma.Pool + + def start_link(args) do + Supervisor.start_link(__MODULE__, args, name: __MODULE__) + end + + def init(_) do + children = + [ + %{ + id: Pool.Connections, + start: + {Pool.Connections, :start_link, + [{:gun_connections, Pleroma.Config.get([:connections_pool])}]} + } + ] ++ pools() + + Supervisor.init(children, strategy: :one_for_one) + end + + defp pools do + for {pool_name, pool_opts} <- Pleroma.Config.get([:pools]) do + pool_opts + |> Keyword.put(:id, {Pool, pool_name}) + |> Keyword.put(:name, pool_name) + |> Pool.child_spec() + end + end +end diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex index 776c4794c..63261b94c 100644 --- a/lib/pleroma/reverse_proxy/client.ex +++ b/lib/pleroma/reverse_proxy/client.ex @@ -3,19 +3,23 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client do - @callback request(atom(), String.t(), [tuple()], String.t(), list()) :: - {:ok, pos_integer(), [tuple()], reference() | map()} - | {:ok, pos_integer(), [tuple()]} + @type status :: pos_integer() + @type header_name :: String.t() + @type header_value :: String.t() + @type headers :: [{header_name(), header_value()}] + + @callback request(atom(), String.t(), headers(), String.t(), list()) :: + {:ok, status(), headers(), reference() | map()} + | {:ok, status(), headers()} | {:ok, reference()} | {:error, term()} - @callback stream_body(reference() | pid() | map()) :: - {:ok, binary()} | :done | {:error, String.t()} + @callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()} @callback close(reference() | pid() | map()) :: :ok - def request(method, url, headers, "", opts \\ []) do - client().request(method, url, headers, "", opts) + def request(method, url, headers, body \\ "", opts \\ []) do + client().request(method, url, headers, body, opts) end def stream_body(ref), do: client().stream_body(ref) @@ -23,6 +27,12 @@ defmodule Pleroma.ReverseProxy.Client do def close(ref), do: client().close(ref) defp client do - Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney) + :tesla + |> Application.get_env(:adapter) + |> client() end + + defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney + defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla + defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client) end diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex new file mode 100644 index 000000000..e41560ab0 --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReverseProxy.Client.Hackney do + @behaviour Pleroma.ReverseProxy.Client + + @impl true + def request(method, url, headers, body, opts \\ []) do + :hackney.request(method, url, headers, body, opts) + end + + @impl true + def stream_body(ref) do + case :hackney.stream_body(ref) do + :done -> :done + {:ok, data} -> {:ok, data, ref} + {:error, error} -> {:error, error} + end + end + + @impl true + def close(ref), do: :hackney.close(ref) +end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex new file mode 100644 index 000000000..55a11b4a8 --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReverseProxy.Client.Tesla do + @type headers() :: [{String.t(), String.t()}] + @type status() :: pos_integer() + + @behaviour Pleroma.ReverseProxy.Client + + @spec request(atom(), String.t(), headers(), String.t(), keyword()) :: + {:ok, status(), headers} + | {:ok, status(), headers, map()} + | {:error, atom() | String.t()} + | no_return() + + @impl true + def request(method, url, headers, body, opts \\ []) do + _adapter = check_adapter() + + with opts <- Keyword.merge(opts, body_as: :chunks, mode: :passive), + {:ok, response} <- + Pleroma.HTTP.request( + method, + url, + body, + headers, + Keyword.put(opts, :adapter, opts) + ) do + if is_map(response.body) and method != :head do + {:ok, response.status, response.headers, response.body} + else + {:ok, response.status, response.headers} + end + else + {:error, error} -> {:error, error} + end + end + + @impl true + @spec stream_body(map()) :: {:ok, binary(), map()} | {:error, atom() | String.t()} | :done + def stream_body(%{pid: pid, opts: opts, fin: true}) do + # if connection was sended and there were redirects, we need to close new conn - pid manually + if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid) + # if there were redirects we need to checkout old conn + conn = opts[:old_conn] || opts[:conn] + + if conn, do: :ok = Pleroma.Pool.Connections.checkout(conn, self(), :gun_connections) + + :done + end + + def stream_body(client) do + case read_chunk!(client) do + {:fin, body} -> + {:ok, body, Map.put(client, :fin, true)} + + {:nofin, part} -> + {:ok, part, client} + + {:error, error} -> + {:error, error} + end + end + + defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do + adapter = check_adapter() + adapter.read_chunk(pid, stream, opts) + end + + @impl true + @spec close(map) :: :ok | no_return() + def close(%{pid: pid}) do + adapter = check_adapter() + adapter.close(pid) + end + + defp check_adapter do + adapter = Application.get_env(:tesla, :adapter) + + unless adapter == Tesla.Adapter.Gun do + raise "#{adapter} doesn't support reading body in chunks" + end + + adapter + end +end diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 2ed719315..9f5710c92 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -3,8 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy do - alias Pleroma.HTTP - @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++ ~w(if-unmodified-since if-none-match if-range range) @resp_cache_headers ~w(etag date last-modified cache-control) @@ -61,10 +59,10 @@ defmodule Pleroma.ReverseProxy do * `req_headers`, `resp_headers` additional headers. - * `http`: options for [hackney](https://github.com/benoitc/hackney). + * `http`: options for [gun](https://github.com/ninenines/gun). """ - @default_hackney_options [pool: :media] + @default_options [pool: :media] @inline_content_types [ "image/gif", @@ -97,11 +95,7 @@ defmodule Pleroma.ReverseProxy do def call(_conn, _url, _opts \\ []) def call(conn = %{method: method}, url, opts) when method in @methods do - hackney_opts = - Pleroma.HTTP.Connection.hackney_options([]) - |> Keyword.merge(@default_hackney_options) - |> Keyword.merge(Keyword.get(opts, :http, [])) - |> HTTP.process_request_options() + client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, [])) req_headers = build_req_headers(conn.req_headers, opts) @@ -113,7 +107,7 @@ defmodule Pleroma.ReverseProxy do end with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url), - {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), + {:ok, code, headers, client} <- request(method, url, req_headers, client_opts), :ok <- header_length_constraint( headers, @@ -159,11 +153,11 @@ defmodule Pleroma.ReverseProxy do |> halt() end - defp request(method, url, headers, hackney_opts) do + defp request(method, url, headers, opts) do Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}") method = method |> String.downcase() |> String.to_existing_atom() - case client().request(method, url, headers, "", hackney_opts) do + case client().request(method, url, headers, "", opts) do {:ok, code, headers, client} when code in @valid_resp_codes -> {:ok, code, downcase_headers(headers), client} @@ -213,7 +207,7 @@ defmodule Pleroma.ReverseProxy do duration, Keyword.get(opts, :max_read_duration, @max_read_duration) ), - {:ok, data} <- client().stream_body(client), + {:ok, data, client} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), :ok <- diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index df774b0f7..ade87daf2 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -12,17 +12,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @hackney_options [ - pool: :media, - recv_timeout: 10_000 + @options [ + pool: :media ] def perform(:prefetch, url) do Logger.debug("Prefetching #{inspect(url)}") + opts = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.put(@options, :recv_timeout, 10_000) + else + @options + end + url |> MediaProxy.url() - |> HTTP.get([], adapter: @hackney_options) + |> HTTP.get([], adapter: opts) end def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index 16b1a53d2..0ae926375 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -3,11 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RelMe do - @hackney_options [ + @options [ pool: :media, - recv_timeout: 2_000, - max_body: 2_000_000, - with_body: true + max_body: 2_000_000 ] if Pleroma.Config.get(:env) == :test do @@ -25,8 +23,18 @@ defmodule Pleroma.Web.RelMe do def parse(_), do: {:error, "No URL provided"} defp parse_url(url) do + opts = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.merge(@options, + recv_timeout: 2_000, + with_body: true + ) + else + @options + end + with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <- - Pleroma.HTTP.get(url, [], adapter: @hackney_options), + Pleroma.HTTP.get(url, [], adapter: opts), data <- Floki.attribute(html, "link[rel~=me]", "href") ++ Floki.attribute(html, "a[rel~=me]", "href") do diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index c06b0a0f2..9deb03845 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -3,11 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser do - @hackney_options [ + @options [ pool: :media, - recv_timeout: 2_000, - max_body: 2_000_000, - with_body: true + max_body: 2_000_000 ] defp parsers do @@ -77,8 +75,18 @@ defmodule Pleroma.Web.RichMedia.Parser do end defp parse_url(url) do + opts = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.merge(@options, + recv_timeout: 2_000, + with_body: true + ) + else + @options + end + try do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: opts) html |> parse_html diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index b4cc80179..91e9e2271 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -205,7 +205,7 @@ defmodule Pleroma.Web.WebFinger do with response <- HTTP.get( address, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ), {:ok, %{status: status, body: body}} when status in 200..299 <- response do doc = XML.parse_document(body) -- cgit v1.2.3 From 7d73e7a09a72354acf526652e307149afbf5b1a3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 18 Feb 2020 09:18:09 -0600 Subject: Spelling --- lib/pleroma/http/adapter/gun.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index f25afeda7..ec6475e96 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -90,7 +90,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do case Connections.checkin(uri, :gun_connections) do nil -> Logger.info( - "Gun connections pool checkin was not succesfull. Trying to open conn for next request." + "Gun connections pool checkin was not successful. Trying to open conn for next request." ) :ok = Connections.open_conn(uri, :gun_connections, opts) -- cgit v1.2.3 From c9db0507f8d49aee9988b0b63477672f5df9c0b2 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 19 Feb 2020 12:19:03 +0300 Subject: removing retry option and changing some logger messages levels --- lib/pleroma/http/adapter/gun.ex | 28 +++++++++++++++++++++------- lib/pleroma/pool/connections.ex | 17 ++++++++--------- 2 files changed, 29 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index ec6475e96..f1018dd8d 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -15,7 +15,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do connect_timeout: 20_000, domain_lookup_timeout: 5_000, tls_handshake_timeout: 5_000, - retry_timeout: 100, + retry: 0, await_up_timeout: 5_000 ] @@ -89,7 +89,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do try do case Connections.checkin(uri, :gun_connections) do nil -> - Logger.info( + Logger.debug( "Gun connections pool checkin was not successful. Trying to open conn for next request." ) @@ -97,7 +97,9 @@ defmodule Pleroma.HTTP.Adapter.Gun do opts conn when is_pid(conn) -> - Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri(uri)}") + Logger.debug( + "received conn #{inspect(conn)} #{uri.scheme}://#{Connections.compose_uri(uri)}" + ) opts |> Keyword.put(:conn, conn) @@ -105,18 +107,30 @@ defmodule Pleroma.HTTP.Adapter.Gun do end rescue error -> - Logger.warn("Gun connections pool checkin caused error #{inspect(error)}") + Logger.warn( + "Gun connections pool checkin caused error #{uri.scheme}://#{ + Connections.compose_uri(uri) + } #{inspect(error)}" + ) + opts catch :exit, {:timeout, _} -> - Logger.info( - "Gun connections pool checkin with timeout error #{Connections.compose_uri(uri)}" + Logger.warn( + "Gun connections pool checkin with timeout error #{uri.scheme}://#{ + Connections.compose_uri(uri) + }" ) opts :exit, error -> - Logger.warn("Gun pool checkin exited with error #{inspect(error)}") + Logger.warn( + "Gun pool checkin exited with error #{uri.scheme}://#{Connections.compose_uri(uri)} #{ + inspect(error) + }" + ) + opts end end diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 1ed16d1c1..c7136e0e0 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -52,8 +52,7 @@ defmodule Pleroma.Pool.Connections do opts = opts |> Enum.into(%{}) - |> Map.put_new(:receive, false) - |> Map.put_new(:retry, pool_opts[:retry] || 5) + |> Map.put_new(:retry, pool_opts[:retry] || 0) |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) @@ -108,11 +107,11 @@ defmodule Pleroma.Pool.Connections do put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) else false -> - Logger.warn("checkout for closed conn #{inspect(conn_pid)}") + Logger.debug("checkout for closed conn #{inspect(conn_pid)}") state nil -> - Logger.info("checkout for alive conn #{inspect(conn_pid)}, but is not in state") + Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state") state end @@ -172,15 +171,15 @@ defmodule Pleroma.Pool.Connections do }) else :error_gun_info -> - Logger.warn(":gun.info caused error") + Logger.debug(":gun.info caused error") state false -> - Logger.warn(":gun_up message for closed conn #{inspect(conn_pid)}") + Logger.debug(":gun_up message for closed conn #{inspect(conn_pid)}") state nil -> - Logger.warn( + Logger.debug( ":gun_up message for alive conn #{inspect(conn_pid)}, but deleted from state" ) @@ -216,11 +215,11 @@ defmodule Pleroma.Pool.Connections do else false -> # gun can send gun_down for closed conn, maybe connection is not closed yet - Logger.warn(":gun_down message for closed conn #{inspect(conn_pid)}") + Logger.debug(":gun_down message for closed conn #{inspect(conn_pid)}") state nil -> - Logger.warn( + Logger.debug( ":gun_down message for alive conn #{inspect(conn_pid)}, but deleted from state" ) -- cgit v1.2.3 From a03c420b84d9901be70520d8c027ccb53449990d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 12:32:42 +0300 Subject: by default don't use gun retries remove conn depends on retry setting from config --- lib/pleroma/pool/connections.ex | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index c7136e0e0..d20927580 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -5,6 +5,8 @@ defmodule Pleroma.Pool.Connections do use GenServer + alias Pleroma.Config + require Logger @type domain :: String.t() @@ -33,7 +35,7 @@ defmodule Pleroma.Pool.Connections do def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) def checkin(%URI{} = uri, name) do - timeout = Pleroma.Config.get([:connections_pool, :receive_connection_timeout], 250) + timeout = Config.get([:connections_pool, :receive_connection_timeout], 250) GenServer.call( name, @@ -47,7 +49,7 @@ defmodule Pleroma.Pool.Connections do def open_conn(url, name, opts) when is_binary(url), do: open_conn(URI.parse(url), name, opts) def open_conn(%URI{} = uri, name, opts) do - pool_opts = Pleroma.Config.get([:connections_pool], []) + pool_opts = Config.get([:connections_pool], []) opts = opts @@ -193,12 +195,13 @@ defmodule Pleroma.Pool.Connections do @impl true def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do + retries = Config.get([:connections_pool, :retry], 0) # we can't get info on this pid, because pid is dead state = with true <- Process.alive?(conn_pid), {key, conn} <- find_conn(state.conns, conn_pid) do - if conn.retries == 5 do - Logger.debug("closing conn if retries is eq 5 #{inspect(conn_pid)}") + if conn.retries == retries do + Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}") :ok = API.close(conn.conn) put_in( -- cgit v1.2.3 From ad8f26c0a4a0a579e93547e78313d3e4ecef6ed5 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 12:53:40 +0300 Subject: more info in Connections.checkin timout errors --- lib/pleroma/http/adapter/gun.ex | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index f1018dd8d..fc40b324a 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -115,11 +115,16 @@ defmodule Pleroma.HTTP.Adapter.Gun do opts catch - :exit, {:timeout, _} -> + :exit, {:timeout, {_, operation, [_, {method, _}, _]}} -> + messages_len = + :gun_connections + |> Process.whereis() + |> Process.info(:message_queue_len) + Logger.warn( - "Gun connections pool checkin with timeout error #{uri.scheme}://#{ - Connections.compose_uri(uri) - }" + "Gun connections pool checkin with timeout error for #{operation} #{method} #{ + uri.scheme + }://#{Connections.compose_uri(uri)}. Messages length: #{messages_len}" ) opts -- cgit v1.2.3 From 6806df80ddb1e52aef2b89b923d9a3e2844b5aeb Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 14:28:16 +0300 Subject: don't log info ssl messages --- lib/pleroma/http/adapter/gun.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index fc40b324a..0a6872ad6 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -58,7 +58,8 @@ defmodule Pleroma.HTTP.Adapter.Gun do depth: 20, reuse_sessions: false, verify_fun: - {&:ssl_verify_hostname.verify_fun/3, [check_hostname: Adapter.domain_or_fallback(host)]} + {&:ssl_verify_hostname.verify_fun/3, [check_hostname: Adapter.domain_or_fallback(host)]}, + log_level: :warning ] ] -- cgit v1.2.3 From f604f9e47061b9d47c1bb62cc7aaf44fabdf69b3 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 14:33:55 +0300 Subject: hackney pool timeout --- lib/pleroma/http/connection.ex | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 85918341a..e2d7afbbd 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -33,8 +33,14 @@ defmodule Pleroma.HTTP.Connection do end defp pool_timeout(opts) do - timeout = - Config.get([:pools, opts[:pool], :timeout]) || Config.get([:pools, :default, :timeout]) + {config_key, default} = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + {:pools, Config.get([:pools, :default, :timeout])} + else + {:hackney_pools, 10_000} + end + + timeout = Config.get([config_key, opts[:pool], :timeout], default) Keyword.merge(opts, timeout: timeout) end -- cgit v1.2.3 From d44f9e3b6cfd5a0dae07f6194bfd05360afd6560 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 16:56:55 +0300 Subject: fix for timeout clause --- lib/pleroma/http/adapter/gun.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 0a6872ad6..7b7e38d8c 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -117,7 +117,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do opts catch :exit, {:timeout, {_, operation, [_, {method, _}, _]}} -> - messages_len = + {:message_queue_len, messages_len} = :gun_connections |> Process.whereis() |> Process.info(:message_queue_len) -- cgit v1.2.3 From 8efae966b1e87fe448a13d04eae0898c4a102c29 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 24 Feb 2020 19:56:27 +0300 Subject: open conn in separate task --- lib/mix/tasks/pleroma/benchmark.ex | 2 +- lib/pleroma/gun/api.ex | 7 +- lib/pleroma/gun/api/mock.ex | 5 +- lib/pleroma/gun/conn.ex | 146 +++++++++++++++++++ lib/pleroma/gun/gun.ex | 5 +- lib/pleroma/http/adapter/gun.ex | 21 ++- lib/pleroma/pool/connections.ex | 287 +++++++++++++------------------------ 7 files changed, 266 insertions(+), 207 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index 01e079136..7a7430289 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -79,7 +79,7 @@ defmodule Mix.Tasks.Pleroma.Benchmark do start_pleroma() :ok = - Pleroma.Pool.Connections.open_conn( + Pleroma.Gun.Conn.open( "https://httpbin.org/stream-bytes/1500", :gun_connections ) diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index a0c3c5415..f79c9f443 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -6,9 +6,10 @@ defmodule Pleroma.Gun.API do @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} @callback info(pid()) :: map() @callback close(pid()) :: :ok - @callback await_up(pid) :: {:ok, atom()} | {:error, atom()} + @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()} @callback connect(pid(), map()) :: reference() @callback await(pid(), reference()) :: {:response, :fin, 200, []} + @callback set_owner(pid(), pid()) :: :ok def open(host, port, opts), do: api().open(host, port, opts) @@ -16,11 +17,13 @@ defmodule Pleroma.Gun.API do def close(pid), do: api().close(pid) - def await_up(pid), do: api().await_up(pid) + def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout) def connect(pid, opts), do: api().connect(pid, opts) def await(pid, ref), do: api().await(pid, ref) + def set_owner(pid, owner), do: api().set_owner(pid, owner) + defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun) end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex index 0134b016e..6d24b0e69 100644 --- a/lib/pleroma/gun/api/mock.ex +++ b/lib/pleroma/gun/api/mock.ex @@ -118,7 +118,10 @@ defmodule Pleroma.Gun.API.Mock do end @impl API - def await_up(_pid), do: {:ok, :http} + def await_up(_pid, _timeout), do: {:ok, :http} + + @impl API + def set_owner(_pid, _owner), do: :ok @impl API def connect(pid, %{host: _, port: 80}) do diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 2474829d6..ddb9f30b0 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -6,6 +6,11 @@ defmodule Pleroma.Gun.Conn do @moduledoc """ Struct for gun connection data """ + alias Pleroma.Gun.API + alias Pleroma.Pool.Connections + + require Logger + @type gun_state :: :up | :down @type conn_state :: :active | :idle @@ -26,4 +31,145 @@ defmodule Pleroma.Gun.Conn do last_reference: 0, crf: 1, retries: 0 + + @spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil + def open(url, name, opts \\ []) + def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts) + + def open(%URI{} = uri, name, opts) do + pool_opts = Pleroma.Config.get([:connections_pool], []) + + opts = + opts + |> Enum.into(%{}) + |> Map.put_new(:retry, pool_opts[:retry] || 0) + |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) + |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + + key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + + Logger.debug("opening new connection #{Connections.compose_uri_log(uri)}") + + conn_pid = + if Connections.count(name) < opts[:max_connection] do + do_open(uri, opts) + else + try_do_open(name, uri, opts) + end + + if is_pid(conn_pid) do + conn = %Pleroma.Gun.Conn{ + conn: conn_pid, + gun_state: :up, + conn_state: :active, + last_reference: :os.system_time(:second) + } + + :ok = API.set_owner(conn_pid, Process.whereis(name)) + Connections.add_conn(name, key, conn) + end + end + + defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do + connect_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + + with open_opts <- Map.delete(opts, :tls_opts), + {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), + {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]), + stream <- API.connect(conn, connect_opts), + {:response, :fin, 200, _} <- API.await(conn, stream) do + conn + else + error -> + Logger.warn( + "Received error on opening connection with http proxy #{ + Connections.compose_uri_log(uri) + } #{inspect(error)}" + ) + + nil + end + end + + defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do + version = + proxy_type + |> to_string() + |> String.last() + |> case do + "4" -> 4 + _ -> 5 + end + + socks_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + |> Map.put(:version, version) + + opts = + opts + |> Map.put(:protocols, [:socks]) + |> Map.put(:socks_opts, socks_opts) + + with {:ok, conn} <- API.open(proxy_host, proxy_port, opts), + {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do + conn + else + error -> + Logger.warn( + "Received error on opening connection with socks proxy #{ + Connections.compose_uri_log(uri) + } #{inspect(error)}" + ) + + nil + end + end + + defp do_open(%URI{host: host, port: port} = uri, opts) do + {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + + with {:ok, conn} <- API.open(host, port, opts), + {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do + conn + else + error -> + Logger.warn( + "Received error on opening connection #{Connections.compose_uri_log(uri)} #{ + inspect(error) + }" + ) + + nil + end + end + + defp destination_opts(%URI{host: host, port: port}) do + {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + %{host: host, port: port} + end + + defp add_http2_opts(opts, "https", tls_opts) do + Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts}) + end + + defp add_http2_opts(opts, _, _), do: opts + + defp try_do_open(name, uri, opts) do + Logger.debug("try to open conn #{Connections.compose_uri_log(uri)}") + + with [{close_key, least_used} | _conns] <- + Connections.get_unused_conns(name), + :ok <- Pleroma.Gun.API.close(least_used.conn) do + Connections.remove_conn(name, close_key) + + do_open(uri, opts) + else + [] -> nil + end + end end diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex index 4a1bbc95f..da82983b1 100644 --- a/lib/pleroma/gun/gun.ex +++ b/lib/pleroma/gun/gun.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Gun do defdelegate close(pid), to: :gun @impl API - defdelegate await_up(pid), to: :gun + defdelegate await_up(pid, timeout \\ 5_000), to: :gun @impl API defdelegate connect(pid, opts), to: :gun @@ -42,4 +42,7 @@ defmodule Pleroma.Gun do @spec flush(pid() | reference()) :: :ok defdelegate flush(pid), to: :gun + + @impl API + defdelegate set_owner(pid, owner), to: :gun end diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 7b7e38d8c..908d71898 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -12,7 +12,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do alias Pleroma.Pool.Connections @defaults [ - connect_timeout: 20_000, + connect_timeout: 5_000, domain_lookup_timeout: 5_000, tls_handshake_timeout: 5_000, retry: 0, @@ -94,13 +94,11 @@ defmodule Pleroma.HTTP.Adapter.Gun do "Gun connections pool checkin was not successful. Trying to open conn for next request." ) - :ok = Connections.open_conn(uri, :gun_connections, opts) + Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) opts conn when is_pid(conn) -> - Logger.debug( - "received conn #{inspect(conn)} #{uri.scheme}://#{Connections.compose_uri(uri)}" - ) + Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") opts |> Keyword.put(:conn, conn) @@ -109,13 +107,14 @@ defmodule Pleroma.HTTP.Adapter.Gun do rescue error -> Logger.warn( - "Gun connections pool checkin caused error #{uri.scheme}://#{ - Connections.compose_uri(uri) - } #{inspect(error)}" + "Gun connections pool checkin caused error #{Connections.compose_uri_log(uri)} #{ + inspect(error) + }" ) opts catch + # TODO: here must be no timeouts :exit, {:timeout, {_, operation, [_, {method, _}, _]}} -> {:message_queue_len, messages_len} = :gun_connections @@ -124,15 +123,15 @@ defmodule Pleroma.HTTP.Adapter.Gun do Logger.warn( "Gun connections pool checkin with timeout error for #{operation} #{method} #{ - uri.scheme - }://#{Connections.compose_uri(uri)}. Messages length: #{messages_len}" + Connections.compose_uri_log(uri) + }. Messages length: #{messages_len}" ) opts :exit, error -> Logger.warn( - "Gun pool checkin exited with error #{uri.scheme}://#{Connections.compose_uri(uri)} #{ + "Gun pool checkin exited with error #{Connections.compose_uri_log(uri)} #{ inspect(error) }" ) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index d20927580..a444f822f 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -20,7 +20,6 @@ defmodule Pleroma.Pool.Connections do defstruct conns: %{}, opts: [] alias Pleroma.Gun.API - alias Pleroma.Gun.Conn @spec start_link({atom(), keyword()}) :: {:ok, pid()} def start_link({name, opts}) do @@ -44,23 +43,6 @@ defmodule Pleroma.Pool.Connections do ) end - @spec open_conn(String.t() | URI.t(), atom(), keyword()) :: :ok - def open_conn(url, name, opts \\ []) - def open_conn(url, name, opts) when is_binary(url), do: open_conn(URI.parse(url), name, opts) - - def open_conn(%URI{} = uri, name, opts) do - pool_opts = Config.get([:connections_pool], []) - - opts = - opts - |> Enum.into(%{}) - |> Map.put_new(:retry, pool_opts[:retry] || 0) - |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) - |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) - - GenServer.cast(name, {:open_conn, %{opts: opts, uri: uri}}) - end - @spec alive?(atom()) :: boolean() def alive?(name) do pid = Process.whereis(name) @@ -72,23 +54,37 @@ defmodule Pleroma.Pool.Connections do GenServer.call(name, :state) end + @spec count(atom()) :: pos_integer() + def count(name) do + GenServer.call(name, :count) + end + + @spec get_unused_conns(atom()) :: [{domain(), conn()}] + def get_unused_conns(name) do + GenServer.call(name, :unused_conns) + end + @spec checkout(pid(), pid(), atom()) :: :ok def checkout(conn, pid, name) do GenServer.cast(name, {:checkout, conn, pid}) end - @impl true - def handle_cast({:open_conn, %{opts: opts, uri: uri}}, state) do - Logger.debug("opening new #{compose_uri(uri)}") - max_connections = state.opts[:max_connections] + @spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok + def add_conn(name, key, conn) do + GenServer.cast(name, {:add_conn, key, conn}) + end - key = compose_key(uri) + @spec remove_conn(atom(), String.t()) :: :ok + def remove_conn(name, key) do + GenServer.cast(name, {:remove_conn, key}) + end - if Enum.count(state.conns) < max_connections do - open_conn(key, uri, state, opts) - else - try_to_open_conn(key, uri, state, opts) - end + @impl true + def handle_cast({:add_conn, key, conn}, state) do + state = put_in(state.conns[key], conn) + + Process.monitor(conn.conn) + {:noreply, state} end @impl true @@ -120,14 +116,20 @@ defmodule Pleroma.Pool.Connections do {:noreply, state} end + @impl true + def handle_cast({:remove_conn, key}, state) do + state = put_in(state.conns, Map.delete(state.conns, key)) + {:noreply, state} + end + @impl true def handle_call({:checkin, uri}, from, state) do - Logger.debug("checkin #{compose_uri(uri)}") - key = compose_key(uri) + key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + Logger.debug("checkin #{key}") case state.conns[key] do %{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up -> - Logger.debug("reusing conn #{compose_uri(uri)}") + Logger.debug("reusing conn #{key}") with time <- :os.system_time(:second), last_reference <- time - current_conn.last_reference, @@ -154,12 +156,31 @@ defmodule Pleroma.Pool.Connections do @impl true def handle_call(:state, _from, state), do: {:reply, state, state} + @impl true + def handle_call(:count, _from, state) do + {:reply, Enum.count(state.conns), state} + end + + @impl true + def handle_call(:unused_conns, _from, state) do + unused_conns = + state.conns + |> Enum.filter(fn {_k, v} -> + v.conn_state == :idle and v.used_by == [] + end) + |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> + x.crf <= y.crf and x.last_reference <= y.last_reference + end) + + {:reply, unused_conns, state} + end + @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do state = - with true <- Process.alive?(conn_pid), - conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), + with conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), {key, conn} <- find_conn(state.conns, conn_pid, conn_key), + {true, key} <- {Process.alive?(conn_pid), key}, time <- :os.system_time(:second), last_reference <- time - conn.last_reference, current_crf <- crf(last_reference, 100, conn.crf) do @@ -176,15 +197,17 @@ defmodule Pleroma.Pool.Connections do Logger.debug(":gun.info caused error") state - false -> + {false, key} -> Logger.debug(":gun_up message for closed conn #{inspect(conn_pid)}") - state - nil -> - Logger.debug( - ":gun_up message for alive conn #{inspect(conn_pid)}, but deleted from state" + put_in( + state.conns, + Map.delete(state.conns, key) ) + nil -> + Logger.debug(":gun_up message for conn which is not found in state") + :ok = API.close(conn_pid) state @@ -198,8 +221,8 @@ defmodule Pleroma.Pool.Connections do retries = Config.get([:connections_pool, :retry], 0) # we can't get info on this pid, because pid is dead state = - with true <- Process.alive?(conn_pid), - {key, conn} <- find_conn(state.conns, conn_pid) do + with {key, conn} <- find_conn(state.conns, conn_pid), + {true, key} <- {Process.alive?(conn_pid), key} do if conn.retries == retries do Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}") :ok = API.close(conn.conn) @@ -216,16 +239,18 @@ defmodule Pleroma.Pool.Connections do }) end else - false -> + {false, key} -> # gun can send gun_down for closed conn, maybe connection is not closed yet Logger.debug(":gun_down message for closed conn #{inspect(conn_pid)}") - state - nil -> - Logger.debug( - ":gun_down message for alive conn #{inspect(conn_pid)}, but deleted from state" + put_in( + state.conns, + Map.delete(state.conns, key) ) + nil -> + Logger.debug(":gun_down message for conn which is not found in state") + :ok = API.close(conn_pid) state @@ -234,7 +259,29 @@ defmodule Pleroma.Pool.Connections do {:noreply, state} end - defp compose_key(%URI{scheme: scheme, host: host, port: port}), do: "#{scheme}:#{host}:#{port}" + @impl true + def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do + Logger.debug("received DOWM message for #{inspect(conn_pid)} reason -> #{inspect(reason)}") + + state = + with {key, conn} <- find_conn(state.conns, conn_pid) do + Enum.each(conn.used_by, fn {pid, _ref} -> + Process.exit(pid, reason) + end) + + put_in( + state.conns, + Map.delete(state.conns, key) + ) + else + nil -> + Logger.debug(":DOWN message for conn which is not found in state") + + state + end + + {:noreply, state} + end defp compose_key_gun_info(pid) do try do @@ -265,153 +312,11 @@ defmodule Pleroma.Pool.Connections do end) end - defp open_conn(key, uri, state, %{proxy: {proxy_host, proxy_port}} = opts) do - connect_opts = - uri - |> destination_opts() - |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) - - with open_opts <- Map.delete(opts, :tls_opts), - {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), - {:ok, _} <- API.await_up(conn), - stream <- API.connect(conn, connect_opts), - {:response, :fin, 200, _} <- API.await(conn, stream), - state <- - put_in(state.conns[key], %Conn{ - conn: conn, - gun_state: :up, - conn_state: :active, - last_reference: :os.system_time(:second) - }) do - {:noreply, state} - else - error -> - Logger.warn( - "Received error on opening connection with http proxy #{uri.scheme}://#{ - compose_uri(uri) - }: #{inspect(error)}" - ) - - {:noreply, state} - end - end - - defp open_conn(key, uri, state, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do - version = - proxy_type - |> to_string() - |> String.last() - |> case do - "4" -> 4 - _ -> 5 - end - - socks_opts = - uri - |> destination_opts() - |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) - |> Map.put(:version, version) - - opts = - opts - |> Map.put(:protocols, [:socks]) - |> Map.put(:socks_opts, socks_opts) - - with {:ok, conn} <- API.open(proxy_host, proxy_port, opts), - {:ok, _} <- API.await_up(conn), - state <- - put_in(state.conns[key], %Conn{ - conn: conn, - gun_state: :up, - conn_state: :active, - last_reference: :os.system_time(:second) - }) do - {:noreply, state} - else - error -> - Logger.warn( - "Received error on opening connection with socks proxy #{uri.scheme}://#{ - compose_uri(uri) - }: #{inspect(error)}" - ) - - {:noreply, state} - end - end - - defp open_conn(key, %URI{host: host, port: port} = uri, state, opts) do - Logger.debug("opening conn #{compose_uri(uri)}") - {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) - - with {:ok, conn} <- API.open(host, port, opts), - {:ok, _} <- API.await_up(conn), - state <- - put_in(state.conns[key], %Conn{ - conn: conn, - gun_state: :up, - conn_state: :active, - last_reference: :os.system_time(:second) - }) do - Logger.debug("new conn opened #{compose_uri(uri)}") - Logger.debug("replying to the call #{compose_uri(uri)}") - {:noreply, state} - else - error -> - Logger.warn( - "Received error on opening connection #{uri.scheme}://#{compose_uri(uri)}: #{ - inspect(error) - }" - ) - - {:noreply, state} - end - end - - defp destination_opts(%URI{host: host, port: port}) do - {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) - %{host: host, port: port} - end - - defp add_http2_opts(opts, "https", tls_opts) do - Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts}) - end - - defp add_http2_opts(opts, _, _), do: opts - - @spec get_unused_conns(map()) :: [{domain(), conn()}] - def get_unused_conns(conns) do - conns - |> Enum.filter(fn {_k, v} -> - v.conn_state == :idle and v.used_by == [] - end) - |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> - x.crf <= y.crf and x.last_reference <= y.last_reference - end) - end - - defp try_to_open_conn(key, uri, state, opts) do - Logger.debug("try to open conn #{compose_uri(uri)}") - - with [{close_key, least_used} | _conns] <- get_unused_conns(state.conns), - :ok <- API.close(least_used.conn), - state <- - put_in( - state.conns, - Map.delete(state.conns, close_key) - ) do - Logger.debug( - "least used conn found and closed #{inspect(least_used.conn)} #{compose_uri(uri)}" - ) - - open_conn(key, uri, state, opts) - else - [] -> {:noreply, state} - end - end - def crf(current, steps, crf) do 1 + :math.pow(0.5, current / steps) * crf end - def compose_uri(%URI{} = uri), do: "#{uri.host}#{uri.path}" + def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do + "#{scheme}://#{host}#{path}" + end end -- cgit v1.2.3 From 4c8569d403f47957f7a5d698c595959007c8a95a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 12:19:29 +0300 Subject: otp_version refactor --- lib/pleroma/application.ex | 35 ++++++++++++---------- lib/pleroma/otp_version.ex | 74 ++++++++++++++++++++-------------------------- 2 files changed, 52 insertions(+), 57 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 00e33d7ac..9b228d6b9 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -66,16 +66,23 @@ defmodule Pleroma.Application do Pleroma.Gopher.Server ] - case Pleroma.OTPVersion.check_version() do - :ok -> :ok - {:error, version} -> raise " - !!!OTP VERSION WARNING!!! - You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. - " - :undefined -> raise " - !!!OTP VERSION WARNING!!! - To support correct handling of unordered certificates chains - OTP version must be > 22.2. - " + if adapter() == Tesla.Adapter.Gun do + case Pleroma.OTPVersion.check() do + :ok -> + :ok + + {:error, version} -> + raise " + !!!OTP VERSION WARNING!!! + You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. + " + + :undefined -> + raise " + !!!OTP VERSION WARNING!!! + To support correct handling of unordered certificates chains - OTP version must be > 22.2. + " + end end # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html @@ -202,11 +209,7 @@ defmodule Pleroma.Application do [hackney_pool, Pleroma.Pool.Supervisor] end - defp http_pools_children(_) do - :tesla - |> Application.get_env(:adapter) - |> http_pools() - end + defp http_pools_children(_), do: http_pools(adapter()) defp http_pools(Tesla.Adapter.Hackney) do pools = [:federation, :media] @@ -227,4 +230,6 @@ defmodule Pleroma.Application do defp http_pools(Tesla.Adapter.Gun), do: [Pleroma.Pool.Supervisor] defp http_pools(_), do: [] + + defp adapter, do: Application.get_env(:tesla, :adapter) end diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex index 0be189304..54ceaff47 100644 --- a/lib/pleroma/otp_version.ex +++ b/lib/pleroma/otp_version.ex @@ -1,63 +1,53 @@ -defmodule Pleroma.OTPVersion do - @type check_status() :: :undefined | {:error, String.t()} | :ok +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only - require Logger +defmodule Pleroma.OTPVersion do + @type check_status() :: :ok | :undefined | {:error, String.t()} - @spec check_version() :: check_status() - def check_version do + @spec check() :: check_status() + def check do # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version - paths = [ + [ Path.join(:code.root_dir(), "OTP_VERSION"), Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"]) ] - - :tesla - |> Application.get_env(:adapter) - |> get_and_check_version(paths) + |> get_version_from_files() + |> do_check() end - @spec get_and_check_version(module(), [Path.t()]) :: check_status() - def get_and_check_version(Tesla.Adapter.Gun, paths) do + @spec check([Path.t()]) :: check_status() + def check(paths) do paths - |> check_files() - |> check_version() + |> get_version_from_files() + |> do_check() end - def get_and_check_version(_, _), do: :ok - - defp check_files([]), do: nil + defp get_version_from_files([]), do: nil - defp check_files([path | paths]) do + defp get_version_from_files([path | paths]) do if File.exists?(path) do File.read!(path) else - check_files(paths) + get_version_from_files(paths) end end - defp check_version(nil), do: :undefined - - defp check_version(version) do - try do - version = String.replace(version, ~r/\r|\n|\s/, "") - - formatted = - version - |> String.split(".") - |> Enum.map(&String.to_integer/1) - |> Enum.take(2) - - with [major, minor] when length(formatted) == 2 <- formatted, - true <- (major == 22 and minor >= 2) or major > 22 do - :ok - else - false -> {:error, version} - _ -> :undefined - end - rescue - _ -> :undefined - catch - _ -> :undefined + defp do_check(nil), do: :undefined + + defp do_check(version) do + version = String.replace(version, ~r/\r|\n|\s/, "") + + [major, minor] = + version + |> String.split(".") + |> Enum.map(&String.to_integer/1) + |> Enum.take(2) + + if (major == 22 and minor >= 2) or major > 22 do + :ok + else + {:error, version} end end end -- cgit v1.2.3 From 097ad10d02598fb6b77f305c10341a13fb57ceee Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:29:51 +0000 Subject: Apply suggestion to lib/pleroma/pool/connections.ex --- lib/pleroma/pool/connections.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index a444f822f..c5098cd86 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -128,7 +128,7 @@ defmodule Pleroma.Pool.Connections do Logger.debug("checkin #{key}") case state.conns[key] do - %{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up -> + %{conn: conn, gun_state: :up} = current_conn -> Logger.debug("reusing conn #{key}") with time <- :os.system_time(:second), -- cgit v1.2.3 From 2c8d80dc0ad594cfe25ebadd9e7a187c95914b34 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:29:57 +0000 Subject: Apply suggestion to lib/pleroma/pool/connections.ex --- lib/pleroma/pool/connections.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index c5098cd86..c4c5fd66c 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -145,7 +145,7 @@ defmodule Pleroma.Pool.Connections do {:reply, conn, state} end - %{gun_state: gun_state} when gun_state == :down -> + %{gun_state: :down} -> {:reply, nil, state} nil -> -- cgit v1.2.3 From a3ad028973154dafad910d4d73d7d4d4822627c1 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:34:36 +0000 Subject: Apply suggestion to lib/pleroma/http/adapter.ex --- lib/pleroma/http/adapter.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter.ex b/lib/pleroma/http/adapter.ex index 6166a3eb4..32046b1d3 100644 --- a/lib/pleroma/http/adapter.ex +++ b/lib/pleroma/http/adapter.ex @@ -57,7 +57,7 @@ defmodule Pleroma.HTTP.Adapter do {:error, :einval} -> {:domain, :idna.encode(charlist)} - {:ok, ip} when is_tuple(ip) and tuple_size(ip) in [4, 8] -> + {:ok, ip} -> {:ip, ip} end end -- cgit v1.2.3 From d30ff35d94ff7d8bc07f0221323a75b07641ee8d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:46:53 +0000 Subject: Apply suggestion to lib/pleroma/http/request_builder.ex --- lib/pleroma/http/request_builder.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 491acd0f9..046741d99 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -35,7 +35,7 @@ defmodule Pleroma.HTTP.RequestBuilder do def headers(request, headers) do headers_list = if Pleroma.Config.get([:http, :send_user_agent]) do - headers ++ [{"user-agent", Pleroma.Application.user_agent()}] + [{"user-agent", Pleroma.Application.user_agent()} | headers] else headers end -- cgit v1.2.3 From 614e3934f9190ff199df087de34146ad5f34c660 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:50:42 +0000 Subject: Apply suggestion to lib/pleroma/http/http.ex --- lib/pleroma/http/http.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index ad47dc936..5fb468689 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -64,8 +64,8 @@ defmodule Pleroma.HTTP do client <- Tesla.client([Tesla.Middleware.FollowRedirects], tesla_adapter()), pid <- Process.whereis(adapter_opts[:pool]) do pool_alive? = - if tesla_adapter() == Tesla.Adapter.Gun do - if pid, do: Process.alive?(pid), else: false + if tesla_adapter() == Tesla.Adapter.Gun && pid do + Process.alive?(pid) else false end -- cgit v1.2.3 From a21a66972f8733de766bc538fe81f2e0ccb57925 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:52:01 +0000 Subject: Apply suggestion to lib/pleroma/http/http.ex --- lib/pleroma/http/http.ex | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 5fb468689..0235f89ea 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -76,12 +76,7 @@ defmodule Pleroma.HTTP do |> Map.put(:env, Pleroma.Config.get([:env])) |> Map.put(:pool_alive?, pool_alive?) - response = - request( - client, - request, - request_opts - ) + response = request(client, request, request_opts) Connection.after_request(adapter_opts) -- cgit v1.2.3 From 7eb65929924af50146d89192c2cf557e3bdbf07f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:53:31 +0000 Subject: Apply suggestion to lib/pleroma/pool/connections.ex --- lib/pleroma/pool/connections.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index c4c5fd66c..84617815f 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -180,10 +180,10 @@ defmodule Pleroma.Pool.Connections do state = with conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), {key, conn} <- find_conn(state.conns, conn_pid, conn_key), - {true, key} <- {Process.alive?(conn_pid), key}, - time <- :os.system_time(:second), - last_reference <- time - conn.last_reference, - current_crf <- crf(last_reference, 100, conn.crf) do + {true, key} <- {Process.alive?(conn_pid), key} do + time = :os.system_time(:second) + last_reference = time - conn.last_reference + current_crf = crf(last_reference, 100, conn.crf) put_in(state.conns[key], %{ conn | gun_state: :up, -- cgit v1.2.3 From 151dc4e387cfbb91b7cd85461ce0deb1e5f5fe30 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:53:37 +0000 Subject: Apply suggestion to lib/pleroma/reverse_proxy/client/tesla.ex --- lib/pleroma/reverse_proxy/client/tesla.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 55a11b4a8..498a905e1 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -16,7 +16,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @impl true def request(method, url, headers, body, opts \\ []) do - _adapter = check_adapter() + check_adapter() with opts <- Keyword.merge(opts, body_as: :chunks, mode: :passive), {:ok, response} <- -- cgit v1.2.3 From 28ed4b41d03c6a137d198b8c67fb081c7ebfbbc6 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 13:05:28 +0300 Subject: naming for checkin from pool timeout --- lib/pleroma/pool/connections.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 84617815f..05fa8f7ad 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -34,7 +34,7 @@ defmodule Pleroma.Pool.Connections do def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) def checkin(%URI{} = uri, name) do - timeout = Config.get([:connections_pool, :receive_connection_timeout], 250) + timeout = Config.get([:connections_pool, :checkin_timeout], 250) GenServer.call( name, @@ -184,6 +184,7 @@ defmodule Pleroma.Pool.Connections do time = :os.system_time(:second) last_reference = time - conn.last_reference current_crf = crf(last_reference, 100, conn.crf) + put_in(state.conns[key], %{ conn | gun_state: :up, -- cgit v1.2.3 From 24d1ac125c6ae719b3d119f2ec0079dcd74eadc2 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 13:24:19 +0300 Subject: hiding raise error logic to otp_version module --- lib/pleroma/application.ex | 23 ++++------------------- lib/pleroma/otp_version.ex | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9b228d6b9..d0b9c3c41 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -42,6 +42,10 @@ defmodule Pleroma.Application do setup_instrumenters() load_custom_modules() + if adapter() == Tesla.Adapter.Gun do + Pleroma.OTPVersion.check!() + end + # Define workers and child supervisors to be supervised children = [ @@ -66,25 +70,6 @@ defmodule Pleroma.Application do Pleroma.Gopher.Server ] - if adapter() == Tesla.Adapter.Gun do - case Pleroma.OTPVersion.check() do - :ok -> - :ok - - {:error, version} -> - raise " - !!!OTP VERSION WARNING!!! - You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. - " - - :undefined -> - raise " - !!!OTP VERSION WARNING!!! - To support correct handling of unordered certificates chains - OTP version must be > 22.2. - " - end - end - # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Pleroma.Supervisor] diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex index 54ceaff47..9ced2d27d 100644 --- a/lib/pleroma/otp_version.ex +++ b/lib/pleroma/otp_version.ex @@ -5,6 +5,26 @@ defmodule Pleroma.OTPVersion do @type check_status() :: :ok | :undefined | {:error, String.t()} + @spec check!() :: :ok | no_return() + def check! do + case check() do + :ok -> + :ok + + {:error, version} -> + raise " + !!!OTP VERSION WARNING!!! + You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. + " + + :undefined -> + raise " + !!!OTP VERSION WARNING!!! + To support correct handling of unordered certificates chains - OTP version must be > 22.2. + " + end + end + @spec check() :: check_status() def check do # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version -- cgit v1.2.3 From d0e4d3ca3b9d8b8ed00d58e9e1c2a05ab561326c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 14:56:49 +0300 Subject: removing unnecessary with comment in tesla client impovement --- lib/pleroma/pool/connections.ex | 40 +++++++++++++++---------------- lib/pleroma/reverse_proxy/client/tesla.ex | 8 ++++--- 2 files changed, 25 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 05fa8f7ad..bde3ffd13 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -36,17 +36,16 @@ defmodule Pleroma.Pool.Connections do def checkin(%URI{} = uri, name) do timeout = Config.get([:connections_pool, :checkin_timeout], 250) - GenServer.call( - name, - {:checkin, uri}, - timeout - ) + GenServer.call(name, {:checkin, uri}, timeout) end @spec alive?(atom()) :: boolean() def alive?(name) do - pid = Process.whereis(name) - if pid, do: Process.alive?(pid), else: false + if pid = Process.whereis(name) do + Process.alive?(pid) + else + false + end end @spec get_state(atom()) :: t() @@ -131,19 +130,20 @@ defmodule Pleroma.Pool.Connections do %{conn: conn, gun_state: :up} = current_conn -> Logger.debug("reusing conn #{key}") - with time <- :os.system_time(:second), - last_reference <- time - current_conn.last_reference, - current_crf <- crf(last_reference, 100, current_conn.crf), - state <- - put_in(state.conns[key], %{ - current_conn - | last_reference: time, - crf: current_crf, - conn_state: :active, - used_by: [from | current_conn.used_by] - }) do - {:reply, conn, state} - end + time = :os.system_time(:second) + last_reference = time - current_conn.last_reference + current_crf = crf(last_reference, 100, current_conn.crf) + + state = + put_in(state.conns[key], %{ + current_conn + | last_reference: time, + crf: current_crf, + conn_state: :active, + used_by: [from | current_conn.used_by] + }) + + {:reply, conn, state} %{gun_state: :down} -> {:reply, nil, state} diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 498a905e1..80a0c8972 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -18,8 +18,9 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do def request(method, url, headers, body, opts \\ []) do check_adapter() - with opts <- Keyword.merge(opts, body_as: :chunks, mode: :passive), - {:ok, response} <- + opts = Keyword.merge(opts, body_as: :chunks) + + with {:ok, response} <- Pleroma.HTTP.request( method, url, @@ -40,7 +41,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @impl true @spec stream_body(map()) :: {:ok, binary(), map()} | {:error, atom() | String.t()} | :done def stream_body(%{pid: pid, opts: opts, fin: true}) do - # if connection was sended and there were redirects, we need to close new conn - pid manually + # if connection was reused, but in tesla were redirects, + # tesla returns new opened connection, which must be closed manually if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid) # if there were redirects we need to checkout old conn conn = opts[:old_conn] || opts[:conn] -- cgit v1.2.3 From 05429730e46b8605544637feebd4c409a4e9ed18 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 15:11:48 +0300 Subject: unnecessary with --- lib/pleroma/http/http.ex | 59 ++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 29 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 0235f89ea..f7b0095d7 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -55,33 +55,36 @@ defmodule Pleroma.HTTP do @spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) :: {:ok, Env.t()} | {:error, any()} def request(method, url, body, headers, options) when is_binary(url) do - with uri <- URI.parse(url), - received_adapter_opts <- Keyword.get(options, :adapter, []), - adapter_opts <- Connection.options(uri, received_adapter_opts), - options <- put_in(options[:adapter], adapter_opts), - params <- Keyword.get(options, :params, []), - request <- build_request(method, headers, options, url, body, params), - client <- Tesla.client([Tesla.Middleware.FollowRedirects], tesla_adapter()), - pid <- Process.whereis(adapter_opts[:pool]) do - pool_alive? = - if tesla_adapter() == Tesla.Adapter.Gun && pid do - Process.alive?(pid) - else - false - end - - request_opts = - adapter_opts - |> Enum.into(%{}) - |> Map.put(:env, Pleroma.Config.get([:env])) - |> Map.put(:pool_alive?, pool_alive?) - - response = request(client, request, request_opts) - - Connection.after_request(adapter_opts) - - response - end + uri = URI.parse(url) + received_adapter_opts = Keyword.get(options, :adapter, []) + adapter_opts = Connection.options(uri, received_adapter_opts) + options = put_in(options[:adapter], adapter_opts) + params = Keyword.get(options, :params, []) + request = build_request(method, headers, options, url, body, params) + + adapter = Application.get_env(:tesla, :adapter) + client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter) + + pid = Process.whereis(adapter_opts[:pool]) + + pool_alive? = + if adapter == Tesla.Adapter.Gun && pid do + Process.alive?(pid) + else + false + end + + request_opts = + adapter_opts + |> Enum.into(%{}) + |> Map.put(:env, Pleroma.Config.get([:env])) + |> Map.put(:pool_alive?, pool_alive?) + + response = request(client, request, request_opts) + + Connection.after_request(adapter_opts) + + response end @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()} @@ -138,6 +141,4 @@ defmodule Pleroma.HTTP do |> Builder.add_param(:query, :query, params) |> Builder.convert_to_keyword() end - - defp tesla_adapter, do: Application.get_env(:tesla, :adapter) end -- cgit v1.2.3 From ee8071f0d5a8a53f6a9ae635d6ea57ce8576e21b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 15:12:09 +0300 Subject: removing unused method --- lib/pleroma/http/request_builder.ex | 20 -------------------- 1 file changed, 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 046741d99..5b92ce764 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -49,26 +49,6 @@ defmodule Pleroma.HTTP.RequestBuilder do @spec opts(Request.t(), keyword()) :: Request.t() def opts(request, options), do: %{request | opts: options} - # NOTE: isn't used anywhere - @doc """ - Add optional parameters to the request - - """ - @spec add_optional_params(Request.t(), %{optional(atom) => atom}, keyword()) :: map() - def add_optional_params(request, _, []), do: request - - def add_optional_params(request, definitions, [{key, value} | tail]) do - case definitions do - %{^key => location} -> - request - |> add_param(location, key, value) - |> add_optional_params(definitions, tail) - - _ -> - add_optional_params(request, definitions, tail) - end - end - @doc """ Add optional parameters to the request """ -- cgit v1.2.3 From e605e79df9761cef3d9f93c489dd4618c6b70eda Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 15:44:13 +0300 Subject: simplification of formatting host method case for format_proxy method --- lib/pleroma/gun/conn.ex | 6 +++--- lib/pleroma/http/adapter.ex | 29 +++-------------------------- lib/pleroma/http/adapter/gun.ex | 20 ++++++++++++++++---- 3 files changed, 22 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index ddb9f30b0..a33d75558 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun.Conn do @@ -131,7 +131,7 @@ defmodule Pleroma.Gun.Conn do end defp do_open(%URI{host: host, port: port} = uri, opts) do - {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + host = Pleroma.HTTP.Connection.parse_host(host) with {:ok, conn} <- API.open(host, port, opts), {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do @@ -149,7 +149,7 @@ defmodule Pleroma.Gun.Conn do end defp destination_opts(%URI{host: host, port: port}) do - {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + host = Pleroma.HTTP.Connection.parse_host(host) %{host: host, port: port} end diff --git a/lib/pleroma/http/adapter.ex b/lib/pleroma/http/adapter.ex index 32046b1d3..a3b84d8f3 100644 --- a/lib/pleroma/http/adapter.ex +++ b/lib/pleroma/http/adapter.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.Adapter do @@ -8,7 +8,6 @@ defmodule Pleroma.HTTP.Adapter do @type proxy :: {Connection.host(), pos_integer()} | {Connection.proxy_type(), pos_integer()} - @type host_type :: :domain | :ip @callback options(keyword(), URI.t()) :: keyword() @callback after_request(keyword()) :: :ok @@ -29,9 +28,8 @@ defmodule Pleroma.HTTP.Adapter do def format_proxy(nil), do: nil def format_proxy(proxy_url) do - with {:ok, host, port} <- Connection.parse_proxy(proxy_url) do - {host, port} - else + case Connection.parse_proxy(proxy_url) do + {:ok, host, port} -> {host, port} {:ok, type, host, port} -> {type, host, port} _ -> nil end @@ -40,25 +38,4 @@ defmodule Pleroma.HTTP.Adapter do @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() def maybe_add_proxy(opts, nil), do: opts def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) - - @spec domain_or_fallback(String.t()) :: charlist() - def domain_or_fallback(host) do - case domain_or_ip(host) do - {:domain, domain} -> domain - {:ip, _ip} -> to_charlist(host) - end - end - - @spec domain_or_ip(String.t()) :: {host_type(), Connection.host()} - def domain_or_ip(host) do - charlist = to_charlist(host) - - case :inet.parse_address(charlist) do - {:error, :einval} -> - {:domain, :idna.encode(charlist)} - - {:ok, ip} -> - {:ip, ip} - end - end end diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 908d71898..5e88786bd 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.Adapter.Gun do @@ -42,7 +42,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do end defp add_original(opts, %URI{host: host, port: port}) do - formatted_host = Adapter.domain_or_fallback(host) + formatted_host = format_host(host) Keyword.put(opts, :original, "#{formatted_host}:#{port}") end @@ -57,8 +57,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do cacertfile: CAStore.file_path(), depth: 20, reuse_sessions: false, - verify_fun: - {&:ssl_verify_hostname.verify_fun/3, [check_hostname: Adapter.domain_or_fallback(host)]}, + verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: format_host(host)]}, log_level: :warning ] ] @@ -139,4 +138,17 @@ defmodule Pleroma.HTTP.Adapter.Gun do opts end end + + @spec format_host(String.t()) :: charlist() + def format_host(host) do + host_charlist = to_charlist(host) + + case :inet.parse_address(host_charlist) do + {:error, :einval} -> + :idna.encode(host_charlist) + + {:ok, _ip} -> + host_charlist + end + end end -- cgit v1.2.3 From 7d68924e4f7233590457aa7e32a21f082dd0584f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 16:08:21 +0300 Subject: naming --- lib/pleroma/gun/conn.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index a33d75558..a8b8c92c1 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -54,7 +54,7 @@ defmodule Pleroma.Gun.Conn do if Connections.count(name) < opts[:max_connection] do do_open(uri, opts) else - try_do_open(name, uri, opts) + close_least_used_and_do_open(name, uri, opts) end if is_pid(conn_pid) do @@ -159,7 +159,7 @@ defmodule Pleroma.Gun.Conn do defp add_http2_opts(opts, _, _), do: opts - defp try_do_open(name, uri, opts) do + defp close_least_used_and_do_open(name, uri, opts) do Logger.debug("try to open conn #{Connections.compose_uri_log(uri)}") with [{close_key, least_used} | _conns] <- -- cgit v1.2.3 From 8fc00b7cbff86885ec99d01821c403a766202659 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 16:27:46 +0300 Subject: return error if connection failed to open --- lib/pleroma/gun/conn.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index a8b8c92c1..9ae419092 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -90,7 +90,7 @@ defmodule Pleroma.Gun.Conn do } #{inspect(error)}" ) - nil + error end end @@ -126,7 +126,7 @@ defmodule Pleroma.Gun.Conn do } #{inspect(error)}" ) - nil + error end end @@ -144,7 +144,7 @@ defmodule Pleroma.Gun.Conn do }" ) - nil + error end end @@ -169,7 +169,7 @@ defmodule Pleroma.Gun.Conn do do_open(uri, opts) else - [] -> nil + [] -> {:error, :pool_overflowed} end end end -- cgit v1.2.3 From 6ebf389d6e6ca5f3e56f9b017531f5f7e301ed3c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 16:51:49 +0300 Subject: poolboy timeout fix --- lib/pleroma/http/http.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index f7b0095d7..4b774472e 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -102,8 +102,8 @@ defmodule Pleroma.HTTP do try do :poolboy.transaction( pool, - &Pleroma.Pool.Request.execute(&1, client, request, timeout + 500), - timeout + 1_000 + &Pleroma.Pool.Request.execute(&1, client, request, timeout), + timeout ) rescue e -> -- cgit v1.2.3 From aaa879ce75a62e69a458226e65bef31b0f2ed08c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 17:27:22 +0300 Subject: proxy parsing errors --- lib/pleroma/http/connection.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index e2d7afbbd..bdd062929 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -71,15 +71,15 @@ defmodule Pleroma.HTTP.Connection do else {_, _} -> Logger.warn("parsing port in proxy fail #{inspect(proxy)}") - {:error, :error_parsing_port_in_proxy} + {:error, :invalid_proxy_port} :error -> Logger.warn("parsing port in proxy fail #{inspect(proxy)}") - {:error, :error_parsing_port_in_proxy} + {:error, :invalid_proxy_port} _ -> Logger.warn("parsing proxy fail #{inspect(proxy)}") - {:error, :error_parsing_proxy} + {:error, :invalid_proxy} end end @@ -89,7 +89,7 @@ defmodule Pleroma.HTTP.Connection do else _ -> Logger.warn("parsing proxy fail #{inspect(proxy)}") - {:error, :error_parsing_proxy} + {:error, :invalid_proxy} end end -- cgit v1.2.3 From 24bf5c4e89e6f97ed3d53157cead48c04015a51b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 17:27:56 +0300 Subject: remove try block from pool request --- lib/pleroma/http/http.ex | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 4b774472e..cc0c39400 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -99,23 +99,11 @@ defmodule Pleroma.HTTP do end def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do - try do - :poolboy.transaction( - pool, - &Pleroma.Pool.Request.execute(&1, client, request, timeout), - timeout - ) - rescue - e -> - {:error, e} - catch - :exit, {:timeout, _} -> - Logger.warn("Receive response from pool failed #{request[:url]}") - {:error, :recv_pool_timeout} - - :exit, e -> - {:error, e} - end + :poolboy.transaction( + pool, + &Pleroma.Pool.Request.execute(&1, client, request, timeout), + timeout + ) end @spec request_try(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} -- cgit v1.2.3 From 1ad34bfdbaee7d98167dc7dc7be8b65fd5e6c5f1 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 17:44:04 +0300 Subject: no try block in checkout connection --- lib/pleroma/http/adapter/gun.ex | 53 +++++++---------------------------------- 1 file changed, 9 insertions(+), 44 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 5e88786bd..30c5c3c16 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -86,56 +86,21 @@ defmodule Pleroma.HTTP.Adapter.Gun do end defp try_to_get_conn(uri, opts) do - try do - case Connections.checkin(uri, :gun_connections) do - nil -> - Logger.debug( - "Gun connections pool checkin was not successful. Trying to open conn for next request." - ) - - Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) - opts - - conn when is_pid(conn) -> - Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") - - opts - |> Keyword.put(:conn, conn) - |> Keyword.put(:close_conn, false) - end - rescue - error -> - Logger.warn( - "Gun connections pool checkin caused error #{Connections.compose_uri_log(uri)} #{ - inspect(error) - }" + case Connections.checkin(uri, :gun_connections) do + nil -> + Logger.debug( + "Gun connections pool checkin was not successful. Trying to open conn for next request." ) + Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) opts - catch - # TODO: here must be no timeouts - :exit, {:timeout, {_, operation, [_, {method, _}, _]}} -> - {:message_queue_len, messages_len} = - :gun_connections - |> Process.whereis() - |> Process.info(:message_queue_len) - - Logger.warn( - "Gun connections pool checkin with timeout error for #{operation} #{method} #{ - Connections.compose_uri_log(uri) - }. Messages length: #{messages_len}" - ) - opts - - :exit, error -> - Logger.warn( - "Gun pool checkin exited with error #{Connections.compose_uri_log(uri)} #{ - inspect(error) - }" - ) + conn when is_pid(conn) -> + Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") opts + |> Keyword.put(:conn, conn) + |> Keyword.put(:close_conn, false) end end -- cgit v1.2.3 From 8854770fc4e9079131a0897d5fb6c0ccccf98bc6 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 18:01:35 +0300 Subject: retry and retry_timeout settings default change --- lib/pleroma/gun/conn.ex | 4 ++-- lib/pleroma/http/adapter/gun.ex | 3 ++- lib/pleroma/pool/connections.ex | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 9ae419092..d73bec360 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -42,8 +42,8 @@ defmodule Pleroma.Gun.Conn do opts = opts |> Enum.into(%{}) - |> Map.put_new(:retry, pool_opts[:retry] || 0) - |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) + |> Map.put_new(:retry, pool_opts[:retry] || 1) + |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000) |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) key = "#{uri.scheme}:#{uri.host}:#{uri.port}" diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 30c5c3c16..ecf9c5b62 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -15,7 +15,8 @@ defmodule Pleroma.HTTP.Adapter.Gun do connect_timeout: 5_000, domain_lookup_timeout: 5_000, tls_handshake_timeout: 5_000, - retry: 0, + retry: 1, + retry_timeout: 1000, await_up_timeout: 5_000 ] diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index bde3ffd13..0f7a1bfd8 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -219,7 +219,7 @@ defmodule Pleroma.Pool.Connections do @impl true def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do - retries = Config.get([:connections_pool, :retry], 0) + retries = Config.get([:connections_pool, :retry], 1) # we can't get info on this pid, because pid is dead state = with {key, conn} <- find_conn(state.conns, conn_pid), -- cgit v1.2.3 From f98ee730f01de528797e38f27964b69a465662c4 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 18:53:44 +0300 Subject: adapter renaming to adapter_helper --- lib/pleroma/http/adapter.ex | 41 ---------- lib/pleroma/http/adapter/gun.ex | 120 ----------------------------- lib/pleroma/http/adapter/hackney.ex | 41 ---------- lib/pleroma/http/adapter_helper.ex | 41 ++++++++++ lib/pleroma/http/adapter_helper/gun.ex | 120 +++++++++++++++++++++++++++++ lib/pleroma/http/adapter_helper/hackney.ex | 41 ++++++++++ lib/pleroma/http/connection.ex | 8 +- 7 files changed, 206 insertions(+), 206 deletions(-) delete mode 100644 lib/pleroma/http/adapter.ex delete mode 100644 lib/pleroma/http/adapter/gun.ex delete mode 100644 lib/pleroma/http/adapter/hackney.ex create mode 100644 lib/pleroma/http/adapter_helper.ex create mode 100644 lib/pleroma/http/adapter_helper/gun.ex create mode 100644 lib/pleroma/http/adapter_helper/hackney.ex (limited to 'lib') diff --git a/lib/pleroma/http/adapter.ex b/lib/pleroma/http/adapter.ex deleted file mode 100644 index a3b84d8f3..000000000 --- a/lib/pleroma/http/adapter.ex +++ /dev/null @@ -1,41 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.Adapter do - alias Pleroma.HTTP.Connection - - @type proxy :: - {Connection.host(), pos_integer()} - | {Connection.proxy_type(), pos_integer()} - - @callback options(keyword(), URI.t()) :: keyword() - @callback after_request(keyword()) :: :ok - - @spec options(keyword(), URI.t()) :: keyword() - def options(opts, _uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) - maybe_add_proxy(opts, format_proxy(proxy)) - end - - @spec maybe_get_conn(URI.t(), keyword()) :: keyword() - def maybe_get_conn(_uri, opts), do: opts - - @spec after_request(keyword()) :: :ok - def after_request(_opts), do: :ok - - @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil - def format_proxy(nil), do: nil - - def format_proxy(proxy_url) do - case Connection.parse_proxy(proxy_url) do - {:ok, host, port} -> {host, port} - {:ok, type, host, port} -> {type, host, port} - _ -> nil - end - end - - @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() - def maybe_add_proxy(opts, nil), do: opts - def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) -end diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex deleted file mode 100644 index ecf9c5b62..000000000 --- a/lib/pleroma/http/adapter/gun.ex +++ /dev/null @@ -1,120 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.Adapter.Gun do - @behaviour Pleroma.HTTP.Adapter - - alias Pleroma.HTTP.Adapter - - require Logger - - alias Pleroma.Pool.Connections - - @defaults [ - connect_timeout: 5_000, - domain_lookup_timeout: 5_000, - tls_handshake_timeout: 5_000, - retry: 1, - retry_timeout: 1000, - await_up_timeout: 5_000 - ] - - @spec options(keyword(), URI.t()) :: keyword() - def options(connection_opts \\ [], %URI{} = uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) - - @defaults - |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) - |> add_original(uri) - |> add_scheme_opts(uri) - |> Adapter.maybe_add_proxy(Adapter.format_proxy(proxy)) - |> maybe_get_conn(uri, connection_opts) - end - - @spec after_request(keyword()) :: :ok - def after_request(opts) do - with conn when not is_nil(conn) <- opts[:conn], - body_as when body_as != :chunks <- opts[:body_as] do - Connections.checkout(conn, self(), :gun_connections) - end - - :ok - end - - defp add_original(opts, %URI{host: host, port: port}) do - formatted_host = format_host(host) - - Keyword.put(opts, :original, "#{formatted_host}:#{port}") - end - - defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts - - defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do - adapter_opts = [ - certificates_verification: true, - tls_opts: [ - verify: :verify_peer, - cacertfile: CAStore.file_path(), - depth: 20, - reuse_sessions: false, - verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: format_host(host)]}, - log_level: :warning - ] - ] - - adapter_opts = - if port != 443 do - Keyword.put(adapter_opts, :transport, :tls) - else - adapter_opts - end - - Keyword.merge(opts, adapter_opts) - end - - defp maybe_get_conn(adapter_opts, uri, connection_opts) do - {receive_conn?, opts} = - adapter_opts - |> Keyword.merge(connection_opts) - |> Keyword.pop(:receive_conn, true) - - if Connections.alive?(:gun_connections) and receive_conn? do - try_to_get_conn(uri, opts) - else - opts - end - end - - defp try_to_get_conn(uri, opts) do - case Connections.checkin(uri, :gun_connections) do - nil -> - Logger.debug( - "Gun connections pool checkin was not successful. Trying to open conn for next request." - ) - - Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) - opts - - conn when is_pid(conn) -> - Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") - - opts - |> Keyword.put(:conn, conn) - |> Keyword.put(:close_conn, false) - end - end - - @spec format_host(String.t()) :: charlist() - def format_host(host) do - host_charlist = to_charlist(host) - - case :inet.parse_address(host_charlist) do - {:error, :einval} -> - :idna.encode(host_charlist) - - {:ok, _ip} -> - host_charlist - end - end -end diff --git a/lib/pleroma/http/adapter/hackney.ex b/lib/pleroma/http/adapter/hackney.ex deleted file mode 100644 index 00db30083..000000000 --- a/lib/pleroma/http/adapter/hackney.ex +++ /dev/null @@ -1,41 +0,0 @@ -defmodule Pleroma.HTTP.Adapter.Hackney do - @behaviour Pleroma.HTTP.Adapter - - @defaults [ - connect_timeout: 10_000, - recv_timeout: 20_000, - follow_redirect: true, - force_redirect: true, - pool: :federation - ] - - @spec options(keyword(), URI.t()) :: keyword() - def options(connection_opts \\ [], %URI{} = uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) - - @defaults - |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) - |> Keyword.merge(connection_opts) - |> add_scheme_opts(uri) - |> Pleroma.HTTP.Adapter.maybe_add_proxy(proxy) - end - - defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts - - defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do - ssl_opts = [ - ssl_options: [ - # Workaround for remote server certificate chain issues - partial_chain: &:hackney_connect.partial_chain/1, - - # We don't support TLS v1.3 yet - versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"], - server_name_indication: to_charlist(host) - ] - ] - - Keyword.merge(opts, ssl_opts) - end - - def after_request(_), do: :ok -end diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex new file mode 100644 index 000000000..2c13666ec --- /dev/null +++ b/lib/pleroma/http/adapter_helper.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.AdapterHelper do + alias Pleroma.HTTP.Connection + + @type proxy :: + {Connection.host(), pos_integer()} + | {Connection.proxy_type(), pos_integer()} + + @callback options(keyword(), URI.t()) :: keyword() + @callback after_request(keyword()) :: :ok + + @spec options(keyword(), URI.t()) :: keyword() + def options(opts, _uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + maybe_add_proxy(opts, format_proxy(proxy)) + end + + @spec maybe_get_conn(URI.t(), keyword()) :: keyword() + def maybe_get_conn(_uri, opts), do: opts + + @spec after_request(keyword()) :: :ok + def after_request(_opts), do: :ok + + @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil + def format_proxy(nil), do: nil + + def format_proxy(proxy_url) do + case Connection.parse_proxy(proxy_url) do + {:ok, host, port} -> {host, port} + {:ok, type, host, port} -> {type, host, port} + _ -> nil + end + end + + @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() + def maybe_add_proxy(opts, nil), do: opts + def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) +end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex new file mode 100644 index 000000000..b3298ec7f --- /dev/null +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -0,0 +1,120 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.AdapterHelper.Gun do + @behaviour Pleroma.HTTP.AdapterHelper + + alias Pleroma.HTTP.AdapterHelper + + require Logger + + alias Pleroma.Pool.Connections + + @defaults [ + connect_timeout: 5_000, + domain_lookup_timeout: 5_000, + tls_handshake_timeout: 5_000, + retry: 1, + retry_timeout: 1000, + await_up_timeout: 5_000 + ] + + @spec options(keyword(), URI.t()) :: keyword() + def options(connection_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + + @defaults + |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> add_original(uri) + |> add_scheme_opts(uri) + |> AdapterHelper.maybe_add_proxy(AdapterHelper.format_proxy(proxy)) + |> maybe_get_conn(uri, connection_opts) + end + + @spec after_request(keyword()) :: :ok + def after_request(opts) do + with conn when not is_nil(conn) <- opts[:conn], + body_as when body_as != :chunks <- opts[:body_as] do + Connections.checkout(conn, self(), :gun_connections) + end + + :ok + end + + defp add_original(opts, %URI{host: host, port: port}) do + formatted_host = format_host(host) + + Keyword.put(opts, :original, "#{formatted_host}:#{port}") + end + + defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + + defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do + adapter_opts = [ + certificates_verification: true, + tls_opts: [ + verify: :verify_peer, + cacertfile: CAStore.file_path(), + depth: 20, + reuse_sessions: false, + verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: format_host(host)]}, + log_level: :warning + ] + ] + + adapter_opts = + if port != 443 do + Keyword.put(adapter_opts, :transport, :tls) + else + adapter_opts + end + + Keyword.merge(opts, adapter_opts) + end + + defp maybe_get_conn(adapter_opts, uri, connection_opts) do + {receive_conn?, opts} = + adapter_opts + |> Keyword.merge(connection_opts) + |> Keyword.pop(:receive_conn, true) + + if Connections.alive?(:gun_connections) and receive_conn? do + try_to_get_conn(uri, opts) + else + opts + end + end + + defp try_to_get_conn(uri, opts) do + case Connections.checkin(uri, :gun_connections) do + nil -> + Logger.debug( + "Gun connections pool checkin was not successful. Trying to open conn for next request." + ) + + Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) + opts + + conn when is_pid(conn) -> + Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") + + opts + |> Keyword.put(:conn, conn) + |> Keyword.put(:close_conn, false) + end + end + + @spec format_host(String.t()) :: charlist() + def format_host(host) do + host_charlist = to_charlist(host) + + case :inet.parse_address(host_charlist) do + {:error, :einval} -> + :idna.encode(host_charlist) + + {:ok, _ip} -> + host_charlist + end + end +end diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex new file mode 100644 index 000000000..a0e161eaa --- /dev/null +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -0,0 +1,41 @@ +defmodule Pleroma.HTTP.AdapterHelper.Hackney do + @behaviour Pleroma.HTTP.AdapterHelper + + @defaults [ + connect_timeout: 10_000, + recv_timeout: 20_000, + follow_redirect: true, + force_redirect: true, + pool: :federation + ] + + @spec options(keyword(), URI.t()) :: keyword() + def options(connection_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + + @defaults + |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> Keyword.merge(connection_opts) + |> add_scheme_opts(uri) + |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) + end + + defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + + defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do + ssl_opts = [ + ssl_options: [ + # Workaround for remote server certificate chain issues + partial_chain: &:hackney_connect.partial_chain/1, + + # We don't support TLS v1.3 yet + versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"], + server_name_indication: to_charlist(host) + ] + ] + + Keyword.merge(opts, ssl_opts) + end + + def after_request(_), do: :ok +end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index bdd062929..dc2761182 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -18,7 +18,7 @@ defmodule Pleroma.HTTP.Connection do require Logger alias Pleroma.Config - alias Pleroma.HTTP.Adapter + alias Pleroma.HTTP.AdapterHelper @doc """ Merge default connection & adapter options with received ones. @@ -50,9 +50,9 @@ defmodule Pleroma.HTTP.Connection do defp adapter do case Application.get_env(:tesla, :adapter) do - Tesla.Adapter.Gun -> Adapter.Gun - Tesla.Adapter.Hackney -> Adapter.Hackney - _ -> Adapter + Tesla.Adapter.Gun -> AdapterHelper.Gun + Tesla.Adapter.Hackney -> AdapterHelper.Hackney + _ -> AdapterHelper end end -- cgit v1.2.3 From 884d9710b209cc9981c7de61d4e95fd26cd83820 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 19:24:14 +0300 Subject: refactoring for gun api modules --- lib/pleroma/gun/api.ex | 46 ++++++++---- lib/pleroma/gun/api/mock.ex | 154 ---------------------------------------- lib/pleroma/gun/conn.ex | 22 +++--- lib/pleroma/gun/gun.ex | 49 ++++--------- lib/pleroma/pool/connections.ex | 10 +-- 5 files changed, 62 insertions(+), 219 deletions(-) delete mode 100644 lib/pleroma/gun/api/mock.ex (limited to 'lib') diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index f79c9f443..76aac5874 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -3,27 +3,43 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun.API do - @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} - @callback info(pid()) :: map() - @callback close(pid()) :: :ok - @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()} - @callback connect(pid(), map()) :: reference() - @callback await(pid(), reference()) :: {:response, :fin, 200, []} - @callback set_owner(pid(), pid()) :: :ok + @behaviour Pleroma.Gun - def open(host, port, opts), do: api().open(host, port, opts) + alias Pleroma.Gun - def info(pid), do: api().info(pid) + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :socks_opts, + :ws_opts + ] - def close(pid), do: api().close(pid) + @impl Gun + def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys)) - def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout) + @impl Gun + defdelegate info(pid), to: :gun - def connect(pid, opts), do: api().connect(pid, opts) + @impl Gun + defdelegate close(pid), to: :gun - def await(pid, ref), do: api().await(pid, ref) + @impl Gun + defdelegate await_up(pid, timeout \\ 5_000), to: :gun - def set_owner(pid, owner), do: api().set_owner(pid, owner) + @impl Gun + defdelegate connect(pid, opts), to: :gun - defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun) + @impl Gun + defdelegate await(pid, ref), to: :gun + + @impl Gun + defdelegate set_owner(pid, owner), to: :gun end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex deleted file mode 100644 index 6d24b0e69..000000000 --- a/lib/pleroma/gun/api/mock.ex +++ /dev/null @@ -1,154 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Gun.API.Mock do - @behaviour Pleroma.Gun.API - - alias Pleroma.Gun.API - - @impl API - def open('some-domain.com', 443, _) do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(API.Mock, conn_pid, %{ - origin_scheme: "https", - origin_host: 'some-domain.com', - origin_port: 443 - }) - - {:ok, conn_pid} - end - - @impl API - def open(ip, port, _) - when ip in [{10_755, 10_368, 61_708, 131, 64_206, 45_068, 0, 9_694}, {127, 0, 0, 1}] and - port in [80, 443] do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - scheme = if port == 443, do: "https", else: "http" - - Registry.register(API.Mock, conn_pid, %{ - origin_scheme: scheme, - origin_host: ip, - origin_port: port - }) - - {:ok, conn_pid} - end - - @impl API - def open('localhost', 1234, %{ - protocols: [:socks], - proxy: {:socks5, 'localhost', 1234}, - socks_opts: %{host: 'proxy-socks.com', port: 80, version: 5} - }) do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(API.Mock, conn_pid, %{ - origin_scheme: "http", - origin_host: 'proxy-socks.com', - origin_port: 80 - }) - - {:ok, conn_pid} - end - - @impl API - def open('localhost', 1234, %{ - protocols: [:socks], - proxy: {:socks4, 'localhost', 1234}, - socks_opts: %{ - host: 'proxy-socks.com', - port: 443, - protocols: [:http2], - tls_opts: [], - transport: :tls, - version: 4 - } - }) do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(API.Mock, conn_pid, %{ - origin_scheme: "https", - origin_host: 'proxy-socks.com', - origin_port: 443 - }) - - {:ok, conn_pid} - end - - @impl API - def open('gun-not-up.com', 80, _opts), do: {:error, :timeout} - - @impl API - def open('example.com', port, _) when port in [443, 115] do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(API.Mock, conn_pid, %{ - origin_scheme: "https", - origin_host: 'example.com', - origin_port: 443 - }) - - {:ok, conn_pid} - end - - @impl API - def open(domain, 80, _) do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(API.Mock, conn_pid, %{ - origin_scheme: "http", - origin_host: domain, - origin_port: 80 - }) - - {:ok, conn_pid} - end - - @impl API - def open({127, 0, 0, 1}, 8123, _) do - Task.start_link(fn -> Process.sleep(1_000) end) - end - - @impl API - def open('localhost', 9050, _) do - Task.start_link(fn -> Process.sleep(1_000) end) - end - - @impl API - def await_up(_pid, _timeout), do: {:ok, :http} - - @impl API - def set_owner(_pid, _owner), do: :ok - - @impl API - def connect(pid, %{host: _, port: 80}) do - ref = make_ref() - Registry.register(API.Mock, ref, pid) - ref - end - - @impl API - def connect(pid, %{host: _, port: 443, protocols: [:http2], transport: :tls}) do - ref = make_ref() - Registry.register(API.Mock, ref, pid) - ref - end - - @impl API - def await(pid, ref) do - [{_, ^pid}] = Registry.lookup(API.Mock, ref) - {:response, :fin, 200, []} - end - - @impl API - def info(pid) do - [{_, info}] = Registry.lookup(API.Mock, pid) - info - end - - @impl API - def close(_pid), do: :ok -end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index d73bec360..319718690 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Gun.Conn do @moduledoc """ Struct for gun connection data """ - alias Pleroma.Gun.API + alias Pleroma.Gun alias Pleroma.Pool.Connections require Logger @@ -65,7 +65,7 @@ defmodule Pleroma.Gun.Conn do last_reference: :os.system_time(:second) } - :ok = API.set_owner(conn_pid, Process.whereis(name)) + :ok = Gun.set_owner(conn_pid, Process.whereis(name)) Connections.add_conn(name, key, conn) end end @@ -77,10 +77,10 @@ defmodule Pleroma.Gun.Conn do |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) with open_opts <- Map.delete(opts, :tls_opts), - {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), - {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]), - stream <- API.connect(conn, connect_opts), - {:response, :fin, 200, _} <- API.await(conn, stream) do + {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), + {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]), + stream <- Gun.connect(conn, connect_opts), + {:response, :fin, 200, _} <- Gun.await(conn, stream) do conn else error -> @@ -115,8 +115,8 @@ defmodule Pleroma.Gun.Conn do |> Map.put(:protocols, [:socks]) |> Map.put(:socks_opts, socks_opts) - with {:ok, conn} <- API.open(proxy_host, proxy_port, opts), - {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do + with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), + {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do conn else error -> @@ -133,8 +133,8 @@ defmodule Pleroma.Gun.Conn do defp do_open(%URI{host: host, port: port} = uri, opts) do host = Pleroma.HTTP.Connection.parse_host(host) - with {:ok, conn} <- API.open(host, port, opts), - {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do + with {:ok, conn} <- Gun.open(host, port, opts), + {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do conn else error -> @@ -164,7 +164,7 @@ defmodule Pleroma.Gun.Conn do with [{close_key, least_used} | _conns] <- Connections.get_unused_conns(name), - :ok <- Pleroma.Gun.API.close(least_used.conn) do + :ok <- Gun.close(least_used.conn) do Connections.remove_conn(name, close_key) do_open(uri, opts) diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex index da82983b1..35390bb11 100644 --- a/lib/pleroma/gun/gun.ex +++ b/lib/pleroma/gun/gun.ex @@ -3,46 +3,27 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun do - @behaviour Pleroma.Gun.API + @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} + @callback info(pid()) :: map() + @callback close(pid()) :: :ok + @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()} + @callback connect(pid(), map()) :: reference() + @callback await(pid(), reference()) :: {:response, :fin, 200, []} + @callback set_owner(pid(), pid()) :: :ok - alias Pleroma.Gun.API + def open(host, port, opts), do: api().open(host, port, opts) - @gun_keys [ - :connect_timeout, - :http_opts, - :http2_opts, - :protocols, - :retry, - :retry_timeout, - :trace, - :transport, - :tls_opts, - :tcp_opts, - :socks_opts, - :ws_opts - ] + def info(pid), do: api().info(pid) - @impl API - def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys)) + def close(pid), do: api().close(pid) - @impl API - defdelegate info(pid), to: :gun + def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout) - @impl API - defdelegate close(pid), to: :gun + def connect(pid, opts), do: api().connect(pid, opts) - @impl API - defdelegate await_up(pid, timeout \\ 5_000), to: :gun + def await(pid, ref), do: api().await(pid, ref) - @impl API - defdelegate connect(pid, opts), to: :gun + def set_owner(pid, owner), do: api().set_owner(pid, owner) - @impl API - defdelegate await(pid, ref), to: :gun - - @spec flush(pid() | reference()) :: :ok - defdelegate flush(pid), to: :gun - - @impl API - defdelegate set_owner(pid, owner), to: :gun + defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API) end diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 0f7a1bfd8..92179fbfc 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Pool.Connections do defstruct conns: %{}, opts: [] - alias Pleroma.Gun.API + alias Pleroma.Gun @spec start_link({atom(), keyword()}) :: {:ok, pid()} def start_link({name, opts}) do @@ -209,7 +209,7 @@ defmodule Pleroma.Pool.Connections do nil -> Logger.debug(":gun_up message for conn which is not found in state") - :ok = API.close(conn_pid) + :ok = Gun.close(conn_pid) state end @@ -226,7 +226,7 @@ defmodule Pleroma.Pool.Connections do {true, key} <- {Process.alive?(conn_pid), key} do if conn.retries == retries do Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}") - :ok = API.close(conn.conn) + :ok = Gun.close(conn.conn) put_in( state.conns, @@ -252,7 +252,7 @@ defmodule Pleroma.Pool.Connections do nil -> Logger.debug(":gun_down message for conn which is not found in state") - :ok = API.close(conn_pid) + :ok = Gun.close(conn_pid) state end @@ -287,7 +287,7 @@ defmodule Pleroma.Pool.Connections do defp compose_key_gun_info(pid) do try do # sometimes :gun.info can raise MatchError, which lead to pool terminate - %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = API.info(pid) + %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) host = case :inet.ntoa(origin_host) do -- cgit v1.2.3 From d9c5ae7c09c7cbf3f4f66e01b7ed69a3d6388916 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 3 Mar 2020 17:16:24 -0600 Subject: Update Copyrights for gun related files --- lib/pleroma/gun/api.ex | 2 +- lib/pleroma/gun/gun.ex | 2 +- lib/pleroma/http/request.ex | 2 +- lib/pleroma/pool/connections.ex | 2 +- lib/pleroma/pool/pool.ex | 2 +- lib/pleroma/pool/request.ex | 2 +- lib/pleroma/pool/supervisor.ex | 2 +- lib/pleroma/reverse_proxy/client/hackney.ex | 2 +- lib/pleroma/reverse_proxy/client/tesla.ex | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index 76aac5874..f51cd7db8 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun.API do diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex index 35390bb11..81855e89e 100644 --- a/lib/pleroma/gun/gun.ex +++ b/lib/pleroma/gun/gun.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun do diff --git a/lib/pleroma/http/request.ex b/lib/pleroma/http/request.ex index 891d88d53..761bd6ccf 100644 --- a/lib/pleroma/http/request.ex +++ b/lib/pleroma/http/request.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.Request do diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 92179fbfc..f1fab2a24 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool.Connections do diff --git a/lib/pleroma/pool/pool.ex b/lib/pleroma/pool/pool.ex index a7ae64ce4..21a6fbbc5 100644 --- a/lib/pleroma/pool/pool.ex +++ b/lib/pleroma/pool/pool.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool do diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex index 2c3574561..cce309599 100644 --- a/lib/pleroma/pool/request.ex +++ b/lib/pleroma/pool/request.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool.Request do diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex index 32be2264d..f436849ac 100644 --- a/lib/pleroma/pool/supervisor.ex +++ b/lib/pleroma/pool/supervisor.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool.Supervisor do diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex index e41560ab0..e84118a90 100644 --- a/lib/pleroma/reverse_proxy/client/hackney.ex +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client.Hackney do diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 80a0c8972..dbc6b66a3 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client.Tesla do -- cgit v1.2.3 From 6b2fb9160cd945cdd4b1265c793d1f85d559fccb Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 4 Mar 2020 09:23:42 +0300 Subject: otp version --- lib/pleroma/application.ex | 20 ++++++++++++++- lib/pleroma/otp_version.ex | 61 ++++++---------------------------------------- 2 files changed, 27 insertions(+), 54 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d0b9c3c41..c8a0617a5 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -43,7 +43,25 @@ defmodule Pleroma.Application do load_custom_modules() if adapter() == Tesla.Adapter.Gun do - Pleroma.OTPVersion.check!() + if version = Pleroma.OTPVersion.version() do + [major, minor] = + version + |> String.split(".") + |> Enum.map(&String.to_integer/1) + |> Enum.take(2) + + if (major == 22 and minor < 2) or major < 22 do + raise " + !!!OTP VERSION WARNING!!! + You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. + " + end + else + raise " + !!!OTP VERSION WARNING!!! + To support correct handling of unordered certificates chains - OTP version must be > 22.2. + " + end end # Define workers and child supervisors to be supervised diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex index 9ced2d27d..114d0054f 100644 --- a/lib/pleroma/otp_version.ex +++ b/lib/pleroma/otp_version.ex @@ -3,71 +3,26 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.OTPVersion do - @type check_status() :: :ok | :undefined | {:error, String.t()} - - @spec check!() :: :ok | no_return() - def check! do - case check() do - :ok -> - :ok - - {:error, version} -> - raise " - !!!OTP VERSION WARNING!!! - You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. - " - - :undefined -> - raise " - !!!OTP VERSION WARNING!!! - To support correct handling of unordered certificates chains - OTP version must be > 22.2. - " - end - end - - @spec check() :: check_status() - def check do + @spec version() :: String.t() | nil + def version do # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version [ Path.join(:code.root_dir(), "OTP_VERSION"), Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"]) ] |> get_version_from_files() - |> do_check() - end - - @spec check([Path.t()]) :: check_status() - def check(paths) do - paths - |> get_version_from_files() - |> do_check() end - defp get_version_from_files([]), do: nil + @spec get_version_from_files([Path.t()]) :: String.t() | nil + def get_version_from_files([]), do: nil - defp get_version_from_files([path | paths]) do + def get_version_from_files([path | paths]) do if File.exists?(path) do - File.read!(path) + path + |> File.read!() + |> String.replace(~r/\r|\n|\s/, "") else get_version_from_files(paths) end end - - defp do_check(nil), do: :undefined - - defp do_check(version) do - version = String.replace(version, ~r/\r|\n|\s/, "") - - [major, minor] = - version - |> String.split(".") - |> Enum.map(&String.to_integer/1) - |> Enum.take(2) - - if (major == 22 and minor >= 2) or major > 22 do - :ok - else - {:error, version} - end - end end -- cgit v1.2.3 From 22d52f5691d985e7daaa955e97e0722f038f6fae Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 4 Mar 2020 09:41:23 +0300 Subject: same copyright date format --- lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index b3547ecd4..0270b96ae 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index f67f48ab6..fc3475048 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do -- cgit v1.2.3 From d6bebd4f9c8086dd87c75f3637a5d392a05f2daf Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 4 Mar 2020 18:13:24 +0300 Subject: moving some logic to tesla adapter - checking original inside gun adapter - flushing streams on max_body error --- lib/pleroma/http/adapter_helper/gun.ex | 17 ++--------------- lib/pleroma/pool/request.ex | 10 ++-------- 2 files changed, 4 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index b3298ec7f..5d5870d90 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -26,7 +26,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @defaults |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) - |> add_original(uri) |> add_scheme_opts(uri) |> AdapterHelper.maybe_add_proxy(AdapterHelper.format_proxy(proxy)) |> maybe_get_conn(uri, connection_opts) @@ -42,17 +41,12 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do :ok end - defp add_original(opts, %URI{host: host, port: port}) do - formatted_host = format_host(host) - - Keyword.put(opts, :original, "#{formatted_host}:#{port}") - end - defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts - defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do + defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do adapter_opts = [ certificates_verification: true, + transport: :tls, tls_opts: [ verify: :verify_peer, cacertfile: CAStore.file_path(), @@ -63,13 +57,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do ] ] - adapter_opts = - if port != 443 do - Keyword.put(adapter_opts, :transport, :tls) - else - adapter_opts - end - Keyword.merge(opts, adapter_opts) end diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex index cce309599..0f271b3d0 100644 --- a/lib/pleroma/pool/request.ex +++ b/lib/pleroma/pool/request.ex @@ -28,12 +28,7 @@ defmodule Pleroma.Pool.Request do end @impl true - def handle_info({:gun_data, _conn, stream, _, _}, state) do - # in some cases if we reuse conn and got {:error, :body_too_large} - # gun continues to send messages to this process, - # so we flush messages for this request - :ok = :gun.flush(stream) - + def handle_info({:gun_data, _conn, _stream, _, _}, state) do {:noreply, state} end @@ -49,8 +44,7 @@ defmodule Pleroma.Pool.Request do end @impl true - def handle_info({:gun_error, _conn, stream, _error}, state) do - :ok = :gun.flush(stream) + def handle_info({:gun_error, _conn, _stream, _error}, state) do {:noreply, state} end -- cgit v1.2.3 From eb324467d9c5c761a776ffc98347246c61ad02ae Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 5 Mar 2020 09:51:52 +0300 Subject: removing try block in getting gun info --- lib/pleroma/pool/connections.ex | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index f1fab2a24..f96c08f21 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -285,20 +285,15 @@ defmodule Pleroma.Pool.Connections do end defp compose_key_gun_info(pid) do - try do - # sometimes :gun.info can raise MatchError, which lead to pool terminate - %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) - - host = - case :inet.ntoa(origin_host) do - {:error, :einval} -> origin_host - ip -> ip - end + %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) - "#{scheme}:#{host}:#{port}" - rescue - _ -> :error_gun_info - end + host = + case :inet.ntoa(origin_host) do + {:error, :einval} -> origin_host + ip -> ip + end + + "#{scheme}:#{host}:#{port}" end defp find_conn(conns, conn_pid) do -- cgit v1.2.3 From f0753eed0fdddd30e127213c89a118dd2e087dc9 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 5 Mar 2020 17:31:06 +0300 Subject: removing try block in tesla request added mocks for tests which fail with Tesla.Mock.Error --- lib/pleroma/http/http.ex | 24 +++++------------------- lib/pleroma/pool/request.ex | 2 +- lib/pleroma/web/push/impl.ex | 2 +- lib/pleroma/web/web_finger/web_finger.ex | 3 ++- 4 files changed, 9 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 7b7c79b64..466a94adc 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -88,15 +88,11 @@ defmodule Pleroma.HTTP do end @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()} - def request(%Client{} = client, request, %{env: :test}), do: request_try(client, request) + def request(%Client{} = client, request, %{env: :test}), do: request(client, request) - def request(%Client{} = client, request, %{body_as: :chunks}) do - request_try(client, request) - end + def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request) - def request(%Client{} = client, request, %{pool_alive?: false}) do - request_try(client, request) - end + def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request) def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do :poolboy.transaction( @@ -106,18 +102,8 @@ defmodule Pleroma.HTTP do ) end - @spec request_try(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} - def request_try(client, request) do - try do - Tesla.request(client, request) - rescue - e -> - {:error, e} - catch - :exit, e -> - {:error, e} - end - end + @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} + def request(client, request), do: Tesla.request(client, request) defp build_request(method, headers, options, url, body, params) do Builder.new() diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex index 0f271b3d0..db7c10c01 100644 --- a/lib/pleroma/pool/request.ex +++ b/lib/pleroma/pool/request.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Pool.Request do @impl true def handle_call({:execute, client, request}, _from, state) do - response = Pleroma.HTTP.request_try(client, request) + response = Pleroma.HTTP.request(client, request) {:reply, response, state} end diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index afa510f08..233e55f21 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Web.Push.Impl do type = Activity.mastodon_notification_type(notif.activity) gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key) avatar_url = User.avatar_url(actor) - object = Object.normalize(activity) + object = Object.normalize(activity) || activity user = User.get_cached_by_id(user_id) direct_conversation_id = Activity.direct_conversation_id(activity, user) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index db567a02e..7ffd0e51b 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -173,7 +173,8 @@ defmodule Pleroma.Web.WebFinger do get_template_from_xml(body) else _ -> - with {:ok, %{body: body}} <- HTTP.get("https://#{domain}/.well-known/host-meta", []) do + with {:ok, %{body: body, status: status}} when status in 200..299 <- + HTTP.get("https://#{domain}/.well-known/host-meta", []) do get_template_from_xml(body) else e -> {:error, "Can't find LRDD template: #{inspect(e)}"} -- cgit v1.2.3 From 058c9b01ac063f3cca22a653032663916a16a234 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 5 Mar 2020 18:28:04 +0300 Subject: returning, not needed --- lib/pleroma/web/push/impl.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index 233e55f21..afa510f08 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Web.Push.Impl do type = Activity.mastodon_notification_type(notif.activity) gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key) avatar_url = User.avatar_url(actor) - object = Object.normalize(activity) || activity + object = Object.normalize(activity) user = User.get_cached_by_id(user_id) direct_conversation_id = Activity.direct_conversation_id(activity, user) -- cgit v1.2.3 From c93c3096d5ffb2df1493f2b8e3f0627d9a8c5910 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 6 Mar 2020 21:04:18 +0300 Subject: little refactor --- lib/pleroma/gun/gun.ex | 6 ++++-- lib/pleroma/http/adapter_helper/gun.ex | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex index 81855e89e..4043e4880 100644 --- a/lib/pleroma/gun/gun.ex +++ b/lib/pleroma/gun/gun.ex @@ -11,6 +11,10 @@ defmodule Pleroma.Gun do @callback await(pid(), reference()) :: {:response, :fin, 200, []} @callback set_owner(pid(), pid()) :: :ok + @api Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API) + + defp api, do: @api + def open(host, port, opts), do: api().open(host, port, opts) def info(pid), do: api().info(pid) @@ -24,6 +28,4 @@ defmodule Pleroma.Gun do def await(pid, ref), do: api().await(pid, ref) def set_owner(pid, owner), do: api().set_owner(pid, owner) - - defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API) end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 5d5870d90..9b03f4653 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -5,10 +5,9 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @behaviour Pleroma.HTTP.AdapterHelper - alias Pleroma.HTTP.AdapterHelper - require Logger + alias Pleroma.HTTP.AdapterHelper alias Pleroma.Pool.Connections @defaults [ @@ -22,20 +21,23 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @spec options(keyword(), URI.t()) :: keyword() def options(connection_opts \\ [], %URI{} = uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) + formatted_proxy = + Pleroma.Config.get([:http, :proxy_url], nil) + |> AdapterHelper.format_proxy() + + config_opts = Pleroma.Config.get([:http, :adapter], []) @defaults - |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> Keyword.merge(config_opts) |> add_scheme_opts(uri) - |> AdapterHelper.maybe_add_proxy(AdapterHelper.format_proxy(proxy)) + |> AdapterHelper.maybe_add_proxy(formatted_proxy) |> maybe_get_conn(uri, connection_opts) end @spec after_request(keyword()) :: :ok def after_request(opts) do - with conn when not is_nil(conn) <- opts[:conn], - body_as when body_as != :chunks <- opts[:body_as] do - Connections.checkout(conn, self(), :gun_connections) + if opts[:conn] && opts[:body_as] != :chunks do + Connections.checkout(opts[:conn], self(), :gun_connections) end :ok -- cgit v1.2.3 From 78282dc9839dbd17c4649cd3936bb8f4c8283745 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 6 Mar 2020 21:24:19 +0300 Subject: little polishing --- lib/pleroma/http/adapter_helper/gun.ex | 4 ++-- lib/pleroma/http/adapter_helper/hackney.ex | 4 +++- lib/pleroma/http/connection.ex | 15 ++++++++------- lib/pleroma/pool/connections.ex | 3 +-- 4 files changed, 14 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 9b03f4653..862e851c0 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -5,11 +5,11 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @behaviour Pleroma.HTTP.AdapterHelper - require Logger - alias Pleroma.HTTP.AdapterHelper alias Pleroma.Pool.Connections + require Logger + @defaults [ connect_timeout: 5_000, domain_lookup_timeout: 5_000, diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index a0e161eaa..d08afae0c 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -13,8 +13,10 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do def options(connection_opts \\ [], %URI{} = uri) do proxy = Pleroma.Config.get([:http, :proxy_url], nil) + config_opts = Pleroma.Config.get([:http, :adapter], []) + @defaults - |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> Keyword.merge(config_opts) |> Keyword.merge(connection_opts) |> add_scheme_opts(uri) |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 97eec88c1..777e5d4c8 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -6,6 +6,14 @@ defmodule Pleroma.HTTP.Connection do @moduledoc """ Configure Tesla.Client with default and customized adapter options. """ + + alias Pleroma.Config + alias Pleroma.HTTP.AdapterHelper + + require Logger + + @defaults [pool: :federation] + @type ip_address :: ipv4_address() | ipv6_address() @type ipv4_address :: {0..255, 0..255, 0..255, 0..255} @type ipv6_address :: @@ -13,13 +21,6 @@ defmodule Pleroma.HTTP.Connection do @type proxy_type() :: :socks4 | :socks5 @type host() :: charlist() | ip_address() - @defaults [pool: :federation] - - require Logger - - alias Pleroma.Config - alias Pleroma.HTTP.AdapterHelper - @doc """ Merge default connection & adapter options with received ones. """ diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index f96c08f21..7529e9240 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Pool.Connections do use GenServer alias Pleroma.Config + alias Pleroma.Gun require Logger @@ -19,8 +20,6 @@ defmodule Pleroma.Pool.Connections do defstruct conns: %{}, opts: [] - alias Pleroma.Gun - @spec start_link({atom(), keyword()}) :: {:ok, pid()} def start_link({name, opts}) do GenServer.start_link(__MODULE__, opts, name: name) -- cgit v1.2.3 From 5f42ecc4c74172b1b17c126106fda9da24065b11 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 7 Mar 2020 12:24:39 +0300 Subject: start gun upload pool, if proxy_remote is enabled --- lib/pleroma/pool/supervisor.ex | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex index f436849ac..8dc5b64b7 100644 --- a/lib/pleroma/pool/supervisor.ex +++ b/lib/pleroma/pool/supervisor.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Pool.Supervisor do use Supervisor + alias Pleroma.Config alias Pleroma.Pool def start_link(args) do @@ -17,8 +18,7 @@ defmodule Pleroma.Pool.Supervisor do %{ id: Pool.Connections, start: - {Pool.Connections, :start_link, - [{:gun_connections, Pleroma.Config.get([:connections_pool])}]} + {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]} } ] ++ pools() @@ -26,7 +26,16 @@ defmodule Pleroma.Pool.Supervisor do end defp pools do - for {pool_name, pool_opts} <- Pleroma.Config.get([:pools]) do + pools = Config.get(:pools) + + pools = + if Config.get([Pleroma.Upload, :proxy_remote]) == false do + Keyword.delete(pools, :upload) + else + pools + end + + for {pool_name, pool_opts} <- pools do pool_opts |> Keyword.put(:id, {Pool, pool_name}) |> Keyword.put(:name, pool_name) -- cgit v1.2.3 From 426f5ee48a09dbf321c013db08cc849c8929d86d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 10 Mar 2020 15:31:44 +0300 Subject: tesla adapter can't be changed in adminFE --- lib/pleroma/config/transfer_task.ex | 58 ++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 29 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index bf1b943d8..4a4c022f0 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -20,8 +20,7 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, :markup}, {:pleroma, :streamer}, {:pleroma, :pools}, - {:pleroma, :connections_pool}, - {:tesla, :adapter} + {:pleroma, :connections_pool} ] @reboot_time_subkeys [ @@ -35,8 +34,6 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, :gopher, [:enabled]} ] - @reject [nil, :prometheus] - def start_link(_) do load_and_update_env() if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) @@ -45,35 +42,30 @@ defmodule Pleroma.Config.TransferTask do @spec load_and_update_env([ConfigDB.t()]) :: :ok | false def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do - with {:configurable, true} <- - {:configurable, Pleroma.Config.get(:configurable_from_database)}, - true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"), - started_applications <- Application.started_applications() do + with {_, true} <- {:configurable, Pleroma.Config.get(:configurable_from_database)} do # We need to restart applications for loaded settings take effect - in_db = Repo.all(ConfigDB) with_deleted = in_db ++ deleted - reject_for_restart = if restart_pleroma?, do: @reject, else: [:pleroma | @reject] - - applications = - with_deleted - |> Enum.map(&merge_and_update(&1)) - |> Enum.uniq() - # TODO: some problem with prometheus after restart! - |> Enum.reject(&(&1 in reject_for_restart)) + # TODO: some problem with prometheus after restart! + reject = [nil, :prometheus] - # to be ensured that pleroma will be restarted last - applications = - if :pleroma in applications do - List.delete(applications, :pleroma) ++ [:pleroma] + reject_for_restart = + if restart_pleroma? do + reject else - Restarter.Pleroma.rebooted() - applications + [:pleroma | reject] end - Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env))) + started_applications = Application.started_applications() + + with_deleted + |> Enum.map(&merge_and_update(&1)) + |> Enum.uniq() + |> Enum.reject(&(&1 in reject_for_restart)) + |> maybe_set_pleroma_last() + |> Enum.each(&restart(started_applications, &1, Pleroma.Config.get(:env))) :ok else @@ -81,6 +73,18 @@ defmodule Pleroma.Config.TransferTask do end end + defp maybe_set_pleroma_last(apps) do + # to be ensured that pleroma will be restarted last + if :pleroma in apps do + apps + |> List.delete(:pleroma) + |> List.insert_at(-1, :pleroma) + else + Restarter.Pleroma.rebooted() + apps + end + end + defp group_for_restart(:logger, key, _, merged_value) do # change logger configuration in runtime, without restart if Keyword.keyword?(merged_value) and @@ -93,14 +97,10 @@ defmodule Pleroma.Config.TransferTask do nil end - defp group_for_restart(:tesla, _, _, _), do: :pleroma - defp group_for_restart(group, _, _, _) when group != :pleroma, do: group defp group_for_restart(group, key, value, _) do - if pleroma_need_restart?(group, key, value) do - group - end + if pleroma_need_restart?(group, key, value), do: group end defp merge_and_update(setting) do -- cgit v1.2.3 From f39e1b9eff859c0795911212c59304f68fca92bc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 10 Mar 2020 15:54:11 +0300 Subject: add verify tls_opts only when we open connection for other requests tesla will add tls_opts --- lib/pleroma/gun/conn.ex | 24 ++++++++++++++++++++++++ lib/pleroma/http/adapter_helper/gun.ex | 33 +++++---------------------------- lib/pleroma/http/connection.ex | 13 +++++++++++++ 3 files changed, 42 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 319718690..57a847c30 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -45,6 +45,7 @@ defmodule Pleroma.Gun.Conn do |> Map.put_new(:retry, pool_opts[:retry] || 1) |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000) |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + |> maybe_add_tls_opts(uri) key = "#{uri.scheme}:#{uri.host}:#{uri.port}" @@ -70,6 +71,29 @@ defmodule Pleroma.Gun.Conn do end end + defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts + + defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do + tls_opts = [ + verify: :verify_peer, + cacertfile: CAStore.file_path(), + depth: 20, + reuse_sessions: false, + verify_fun: + {&:ssl_verify_hostname.verify_fun/3, + [check_hostname: Pleroma.HTTP.Connection.format_host(host)]} + ] + + tls_opts = + if Keyword.keyword?(opts[:tls_opts]) do + Keyword.merge(tls_opts, opts[:tls_opts]) + else + tls_opts + end + + Map.put(opts, :tls_opts, tls_opts) + end + defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do connect_opts = uri diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 862e851c0..55c2b192a 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -45,21 +45,11 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts - defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do - adapter_opts = [ - certificates_verification: true, - transport: :tls, - tls_opts: [ - verify: :verify_peer, - cacertfile: CAStore.file_path(), - depth: 20, - reuse_sessions: false, - verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: format_host(host)]}, - log_level: :warning - ] - ] - - Keyword.merge(opts, adapter_opts) + defp add_scheme_opts(opts, %URI{scheme: "https"}) do + opts + |> Keyword.put(:certificates_verification, true) + |> Keyword.put(:transport, :tls) + |> Keyword.put(:tls_opts, log_level: :warning) end defp maybe_get_conn(adapter_opts, uri, connection_opts) do @@ -93,17 +83,4 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do |> Keyword.put(:close_conn, false) end end - - @spec format_host(String.t()) :: charlist() - def format_host(host) do - host_charlist = to_charlist(host) - - case :inet.parse_address(host_charlist) do - {:error, :einval} -> - :idna.encode(host_charlist) - - {:ok, _ip} -> - host_charlist - end - end end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 777e5d4c8..0fc88f708 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -106,4 +106,17 @@ defmodule Pleroma.HTTP.Connection do {:ok, ip} -> ip end end + + @spec format_host(String.t()) :: charlist() + def format_host(host) do + host_charlist = to_charlist(host) + + case :inet.parse_address(host_charlist) do + {:error, :einval} -> + :idna.encode(host_charlist) + + {:ok, _ip} -> + host_charlist + end + end end -- cgit v1.2.3 From 863ec33ba2a90708d199f18683ffe0c4658c710a Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 11 Mar 2020 12:21:44 +0100 Subject: Add support for funkwhale Audio activity reel2bits fixture not included as it lacks the Actor fixture for it. Closes: https://git.pleroma.social/pleroma/pleroma/issues/1624 Closes: https://git.pleroma.social/pleroma/pleroma/issues/764 --- lib/pleroma/web/activity_pub/transmogrifier.ex | 5 +++-- lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 9cd3de705..f52b065f6 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -229,7 +229,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Map.put(object, "url", url["href"]) end - def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do + def fix_url(%{"type" => object_type, "url" => url} = object) + when object_type in ["Video", "Audio"] and is_list(url) do first_element = Enum.at(url, 0) link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end) @@ -398,7 +399,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, options ) - when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer"] do + when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer", "Audio"] do actor = Containment.get_actor(data) data = diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index f7469cdff..a042075f5 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -421,7 +421,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end def render_content(%{data: %{"type" => object_type}} = object) - when object_type in ["Video", "Event"] do + when object_type in ["Video", "Event", "Audio"] do with name when not is_nil(name) and name != "" <- object.data["name"] do "

#{name}

#{object.data["content"]}" else -- cgit v1.2.3 From 1306b92997dc6e76e5d617d529dbc229d5aee200 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 12 Mar 2020 18:28:54 +0300 Subject: clean up --- lib/pleroma/application.ex | 18 +++---- lib/pleroma/config/transfer_task.ex | 42 ++++++--------- lib/pleroma/gun/conn.ex | 31 +++++------ lib/pleroma/http/adapter_helper.ex | 2 +- lib/pleroma/http/adapter_helper/gun.ex | 33 +++++------- lib/pleroma/http/connection.ex | 8 +-- lib/pleroma/http/http.ex | 5 +- lib/pleroma/pool/connections.ex | 94 +++++++++++----------------------- 8 files changed, 88 insertions(+), 145 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c8a0617a5..55b5be488 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -42,7 +42,9 @@ defmodule Pleroma.Application do setup_instrumenters() load_custom_modules() - if adapter() == Tesla.Adapter.Gun do + adapter = Application.get_env(:tesla, :adapter) + + if adapter == Tesla.Adapter.Gun do if version = Pleroma.OTPVersion.version() do [major, minor] = version @@ -74,7 +76,7 @@ defmodule Pleroma.Application do Pleroma.Plugs.RateLimiter.Supervisor ] ++ cachex_children() ++ - http_pools_children(Config.get(:env)) ++ + http_children(adapter, @env) ++ [ Pleroma.Stats, Pleroma.JobQueueMonitor, @@ -206,15 +208,13 @@ defmodule Pleroma.Application do end # start hackney and gun pools in tests - defp http_pools_children(:test) do + defp http_children(_, :test) do hackney_options = Config.get([:hackney_pools, :federation]) hackney_pool = :hackney_pool.child_spec(:federation, hackney_options) [hackney_pool, Pleroma.Pool.Supervisor] end - defp http_pools_children(_), do: http_pools(adapter()) - - defp http_pools(Tesla.Adapter.Hackney) do + defp http_children(Tesla.Adapter.Hackney, _) do pools = [:federation, :media] pools = @@ -230,9 +230,7 @@ defmodule Pleroma.Application do end end - defp http_pools(Tesla.Adapter.Gun), do: [Pleroma.Pool.Supervisor] - - defp http_pools(_), do: [] + defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor] - defp adapter, do: Application.get_env(:tesla, :adapter) + defp http_children(_, _), do: [] end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 4a4c022f0..b6d80adb7 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Config.TransferTask do use Task + alias Pleroma.Config alias Pleroma.ConfigDB alias Pleroma.Repo @@ -36,36 +37,31 @@ defmodule Pleroma.Config.TransferTask do def start_link(_) do load_and_update_env() - if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) + if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) :ignore end - @spec load_and_update_env([ConfigDB.t()]) :: :ok | false - def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do - with {_, true} <- {:configurable, Pleroma.Config.get(:configurable_from_database)} do + @spec load_and_update_env([ConfigDB.t()], boolean()) :: :ok + def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do + with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do # We need to restart applications for loaded settings take effect - in_db = Repo.all(ConfigDB) - - with_deleted = in_db ++ deleted # TODO: some problem with prometheus after restart! - reject = [nil, :prometheus] - - reject_for_restart = + reject_restart = if restart_pleroma? do - reject + [nil, :prometheus] else - [:pleroma | reject] + [:pleroma, nil, :prometheus] end started_applications = Application.started_applications() - with_deleted - |> Enum.map(&merge_and_update(&1)) + (Repo.all(ConfigDB) ++ deleted_settings) + |> Enum.map(&merge_and_update/1) |> Enum.uniq() - |> Enum.reject(&(&1 in reject_for_restart)) + |> Enum.reject(&(&1 in reject_restart)) |> maybe_set_pleroma_last() - |> Enum.each(&restart(started_applications, &1, Pleroma.Config.get(:env))) + |> Enum.each(&restart(started_applications, &1, Config.get(:env))) :ok else @@ -108,18 +104,14 @@ defmodule Pleroma.Config.TransferTask do key = ConfigDB.from_string(setting.key) group = ConfigDB.from_string(setting.group) - default = Pleroma.Config.Holder.config(group, key) + default = Config.Holder.config(group, key) value = ConfigDB.from_binary(setting.value) merged_value = - if Ecto.get_meta(setting, :state) == :deleted do - default - else - if can_be_merged?(default, value) do - ConfigDB.merge_group(group, key, default, value) - else - value - end + cond do + Ecto.get_meta(setting, :state) == :deleted -> default + can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value) + true -> value end :ok = update_env(group, key, merged_value) diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 57a847c30..20823a765 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -49,8 +49,6 @@ defmodule Pleroma.Gun.Conn do key = "#{uri.scheme}:#{uri.host}:#{uri.port}" - Logger.debug("opening new connection #{Connections.compose_uri_log(uri)}") - conn_pid = if Connections.count(name) < opts[:max_connection] do do_open(uri, opts) @@ -109,9 +107,9 @@ defmodule Pleroma.Gun.Conn do else error -> Logger.warn( - "Received error on opening connection with http proxy #{ - Connections.compose_uri_log(uri) - } #{inspect(error)}" + "Opening proxied connection to #{compose_uri_log(uri)} failed with error #{ + inspect(error) + }" ) error @@ -145,9 +143,9 @@ defmodule Pleroma.Gun.Conn do else error -> Logger.warn( - "Received error on opening connection with socks proxy #{ - Connections.compose_uri_log(uri) - } #{inspect(error)}" + "Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{ + inspect(error) + }" ) error @@ -163,9 +161,7 @@ defmodule Pleroma.Gun.Conn do else error -> Logger.warn( - "Received error on opening connection #{Connections.compose_uri_log(uri)} #{ - inspect(error) - }" + "Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" ) error @@ -184,16 +180,17 @@ defmodule Pleroma.Gun.Conn do defp add_http2_opts(opts, _, _), do: opts defp close_least_used_and_do_open(name, uri, opts) do - Logger.debug("try to open conn #{Connections.compose_uri_log(uri)}") - - with [{close_key, least_used} | _conns] <- - Connections.get_unused_conns(name), - :ok <- Gun.close(least_used.conn) do - Connections.remove_conn(name, close_key) + with [{key, conn} | _conns] <- Connections.get_unused_conns(name), + :ok <- Gun.close(conn.conn) do + Connections.remove_conn(name, key) do_open(uri, opts) else [] -> {:error, :pool_overflowed} end end + + def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do + "#{scheme}://#{host}#{path}" + end end diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 2c13666ec..510722ff9 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -7,7 +7,7 @@ defmodule Pleroma.HTTP.AdapterHelper do @type proxy :: {Connection.host(), pos_integer()} - | {Connection.proxy_type(), pos_integer()} + | {Connection.proxy_type(), Connection.host(), pos_integer()} @callback options(keyword(), URI.t()) :: keyword() @callback after_request(keyword()) :: :ok diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 55c2b192a..f14b95c19 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -20,8 +20,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do ] @spec options(keyword(), URI.t()) :: keyword() - def options(connection_opts \\ [], %URI{} = uri) do - formatted_proxy = + def options(incoming_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) |> AdapterHelper.format_proxy() @@ -30,8 +30,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @defaults |> Keyword.merge(config_opts) |> add_scheme_opts(uri) - |> AdapterHelper.maybe_add_proxy(formatted_proxy) - |> maybe_get_conn(uri, connection_opts) + |> AdapterHelper.maybe_add_proxy(proxy) + |> maybe_get_conn(uri, incoming_opts) end @spec after_request(keyword()) :: :ok @@ -43,44 +43,35 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do :ok end - defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + defp add_scheme_opts(opts, %{scheme: "http"}), do: opts - defp add_scheme_opts(opts, %URI{scheme: "https"}) do + defp add_scheme_opts(opts, %{scheme: "https"}) do opts |> Keyword.put(:certificates_verification, true) - |> Keyword.put(:transport, :tls) |> Keyword.put(:tls_opts, log_level: :warning) end - defp maybe_get_conn(adapter_opts, uri, connection_opts) do + defp maybe_get_conn(adapter_opts, uri, incoming_opts) do {receive_conn?, opts} = adapter_opts - |> Keyword.merge(connection_opts) + |> Keyword.merge(incoming_opts) |> Keyword.pop(:receive_conn, true) if Connections.alive?(:gun_connections) and receive_conn? do - try_to_get_conn(uri, opts) + checkin_conn(uri, opts) else opts end end - defp try_to_get_conn(uri, opts) do + defp checkin_conn(uri, opts) do case Connections.checkin(uri, :gun_connections) do nil -> - Logger.debug( - "Gun connections pool checkin was not successful. Trying to open conn for next request." - ) - - Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) + Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts]) opts conn when is_pid(conn) -> - Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") - - opts - |> Keyword.put(:conn, conn) - |> Keyword.put(:close_conn, false) + Keyword.merge(opts, conn: conn, close_conn: false) end end end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 0fc88f708..76de3fcfe 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -71,15 +71,15 @@ defmodule Pleroma.HTTP.Connection do {:ok, parse_host(host), port} else {_, _} -> - Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + Logger.warn("Parsing port failed #{inspect(proxy)}") {:error, :invalid_proxy_port} :error -> - Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + Logger.warn("Parsing port failed #{inspect(proxy)}") {:error, :invalid_proxy_port} _ -> - Logger.warn("parsing proxy fail #{inspect(proxy)}") + Logger.warn("Parsing proxy failed #{inspect(proxy)}") {:error, :invalid_proxy} end end @@ -89,7 +89,7 @@ defmodule Pleroma.HTTP.Connection do {:ok, type, parse_host(host), port} else _ -> - Logger.warn("parsing proxy fail #{inspect(proxy)}") + Logger.warn("Parsing proxy failed #{inspect(proxy)}") {:error, :invalid_proxy} end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 466a94adc..583b56484 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -56,10 +56,9 @@ defmodule Pleroma.HTTP do {:ok, Env.t()} | {:error, any()} def request(method, url, body, headers, options) when is_binary(url) do uri = URI.parse(url) - received_adapter_opts = Keyword.get(options, :adapter, []) - adapter_opts = Connection.options(uri, received_adapter_opts) + adapter_opts = Connection.options(uri, options[:adapter] || []) options = put_in(options[:adapter], adapter_opts) - params = Keyword.get(options, :params, []) + params = options[:params] || [] request = build_request(method, headers, options, url, body, params) adapter = Application.get_env(:tesla, :adapter) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 7529e9240..772833509 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -87,18 +87,11 @@ defmodule Pleroma.Pool.Connections do @impl true def handle_cast({:checkout, conn_pid, pid}, state) do - Logger.debug("checkout #{inspect(conn_pid)}") - state = with true <- Process.alive?(conn_pid), {key, conn} <- find_conn(state.conns, conn_pid), used_by <- List.keydelete(conn.used_by, pid, 0) do - conn_state = - if used_by == [] do - :idle - else - conn.conn_state - end + conn_state = if used_by == [], do: :idle, else: conn.conn_state put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) else @@ -123,26 +116,23 @@ defmodule Pleroma.Pool.Connections do @impl true def handle_call({:checkin, uri}, from, state) do key = "#{uri.scheme}:#{uri.host}:#{uri.port}" - Logger.debug("checkin #{key}") case state.conns[key] do - %{conn: conn, gun_state: :up} = current_conn -> - Logger.debug("reusing conn #{key}") - + %{conn: pid, gun_state: :up} = conn -> time = :os.system_time(:second) - last_reference = time - current_conn.last_reference - current_crf = crf(last_reference, 100, current_conn.crf) + last_reference = time - conn.last_reference + crf = crf(last_reference, 100, conn.crf) state = put_in(state.conns[key], %{ - current_conn + conn | last_reference: time, - crf: current_crf, + crf: crf, conn_state: :active, - used_by: [from | current_conn.used_by] + used_by: [from | conn.used_by] }) - {:reply, conn, state} + {:reply, pid, state} %{gun_state: :down} -> {:reply, nil, state} @@ -164,50 +154,48 @@ defmodule Pleroma.Pool.Connections do def handle_call(:unused_conns, _from, state) do unused_conns = state.conns - |> Enum.filter(fn {_k, v} -> - v.conn_state == :idle and v.used_by == [] - end) - |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> - x.crf <= y.crf and x.last_reference <= y.last_reference - end) + |> Enum.filter(&filter_conns/1) + |> Enum.sort(&sort_conns/2) {:reply, unused_conns, state} end + defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true + defp filter_conns(_), do: false + + defp sort_conns({_, c1}, {_, c2}) do + c1.crf <= c2.crf and c1.last_reference <= c2.last_reference + end + @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do + %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) + + host = + case :inet.ntoa(host) do + {:error, :einval} -> host + ip -> ip + end + + key = "#{scheme}:#{host}:#{port}" + state = - with conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), - {key, conn} <- find_conn(state.conns, conn_pid, conn_key), + with {_key, conn} <- find_conn(state.conns, conn_pid, key), {true, key} <- {Process.alive?(conn_pid), key} do - time = :os.system_time(:second) - last_reference = time - conn.last_reference - current_crf = crf(last_reference, 100, conn.crf) - put_in(state.conns[key], %{ conn | gun_state: :up, - last_reference: time, - crf: current_crf, conn_state: :active, retries: 0 }) else - :error_gun_info -> - Logger.debug(":gun.info caused error") - state - {false, key} -> - Logger.debug(":gun_up message for closed conn #{inspect(conn_pid)}") - put_in( state.conns, Map.delete(state.conns, key) ) nil -> - Logger.debug(":gun_up message for conn which is not found in state") - :ok = Gun.close(conn_pid) state @@ -224,7 +212,6 @@ defmodule Pleroma.Pool.Connections do with {key, conn} <- find_conn(state.conns, conn_pid), {true, key} <- {Process.alive?(conn_pid), key} do if conn.retries == retries do - Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}") :ok = Gun.close(conn.conn) put_in( @@ -240,18 +227,13 @@ defmodule Pleroma.Pool.Connections do end else {false, key} -> - # gun can send gun_down for closed conn, maybe connection is not closed yet - Logger.debug(":gun_down message for closed conn #{inspect(conn_pid)}") - put_in( state.conns, Map.delete(state.conns, key) ) nil -> - Logger.debug(":gun_down message for conn which is not found in state") - - :ok = Gun.close(conn_pid) + Logger.debug(":gun_down for conn which isn't found in state") state end @@ -275,7 +257,7 @@ defmodule Pleroma.Pool.Connections do ) else nil -> - Logger.debug(":DOWN message for conn which is not found in state") + Logger.debug(":DOWN for conn which isn't found in state") state end @@ -283,18 +265,6 @@ defmodule Pleroma.Pool.Connections do {:noreply, state} end - defp compose_key_gun_info(pid) do - %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) - - host = - case :inet.ntoa(origin_host) do - {:error, :einval} -> origin_host - ip -> ip - end - - "#{scheme}:#{host}:#{port}" - end - defp find_conn(conns, conn_pid) do Enum.find(conns, fn {_key, conn} -> conn.conn == conn_pid @@ -310,8 +280,4 @@ defmodule Pleroma.Pool.Connections do def crf(current, steps, crf) do 1 + :math.pow(0.5, current / steps) * crf end - - def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do - "#{scheme}://#{host}#{path}" - end end -- cgit v1.2.3 From 98ed0d1c4bd2db354154cc4a1d1e6530eb68f499 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 13 Mar 2020 09:37:57 +0300 Subject: more clean up --- lib/pleroma/http/adapter_helper/gun.ex | 2 +- lib/pleroma/http/adapter_helper/hackney.ex | 2 +- lib/pleroma/http/connection.ex | 12 +++++++----- lib/pleroma/pool/request.ex | 1 - lib/pleroma/pool/supervisor.ex | 17 +++++++---------- lib/pleroma/reverse_proxy/client/tesla.ex | 9 +++++---- lib/pleroma/reverse_proxy/reverse_proxy.ex | 2 +- 7 files changed, 22 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index f14b95c19..ead7cdc6b 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -22,7 +22,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @spec options(keyword(), URI.t()) :: keyword() def options(incoming_opts \\ [], %URI{} = uri) do proxy = - Pleroma.Config.get([:http, :proxy_url], nil) + Pleroma.Config.get([:http, :proxy_url]) |> AdapterHelper.format_proxy() config_opts = Pleroma.Config.get([:http, :adapter], []) diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index d08afae0c..dcb4cac71 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do @spec options(keyword(), URI.t()) :: keyword() def options(connection_opts \\ [], %URI{} = uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) + proxy = Pleroma.Config.get([:http, :proxy_url]) config_opts = Pleroma.Config.get([:http, :adapter], []) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 76de3fcfe..ebacf7902 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -30,12 +30,12 @@ defmodule Pleroma.HTTP.Connection do @defaults |> pool_timeout() |> Keyword.merge(opts) - |> adapter().options(uri) + |> adapter_helper().options(uri) end defp pool_timeout(opts) do {config_key, default} = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + if adapter() == Tesla.Adapter.Gun do {:pools, Config.get([:pools, :default, :timeout])} else {:hackney_pools, 10_000} @@ -47,10 +47,12 @@ defmodule Pleroma.HTTP.Connection do end @spec after_request(keyword()) :: :ok - def after_request(opts), do: adapter().after_request(opts) + def after_request(opts), do: adapter_helper().after_request(opts) - defp adapter do - case Application.get_env(:tesla, :adapter) do + defp adapter, do: Application.get_env(:tesla, :adapter) + + defp adapter_helper do + case adapter() do Tesla.Adapter.Gun -> AdapterHelper.Gun Tesla.Adapter.Hackney -> AdapterHelper.Hackney _ -> AdapterHelper diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex index db7c10c01..3fb930db7 100644 --- a/lib/pleroma/pool/request.ex +++ b/lib/pleroma/pool/request.ex @@ -39,7 +39,6 @@ defmodule Pleroma.Pool.Request do @impl true def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do - # don't flush messages here, because gun can reconnect {:noreply, state} end diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex index 8dc5b64b7..faf646cb2 100644 --- a/lib/pleroma/pool/supervisor.ex +++ b/lib/pleroma/pool/supervisor.ex @@ -13,16 +13,13 @@ defmodule Pleroma.Pool.Supervisor do end def init(_) do - children = - [ - %{ - id: Pool.Connections, - start: - {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]} - } - ] ++ pools() - - Supervisor.init(children, strategy: :one_for_one) + conns_child = %{ + id: Pool.Connections, + start: + {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]} + } + + Supervisor.init([conns_child | pools()], strategy: :one_for_one) end defp pools do diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index dbc6b66a3..e81ea8bde 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -3,11 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client.Tesla do + @behaviour Pleroma.ReverseProxy.Client + @type headers() :: [{String.t(), String.t()}] @type status() :: pos_integer() - @behaviour Pleroma.ReverseProxy.Client - @spec request(atom(), String.t(), headers(), String.t(), keyword()) :: {:ok, status(), headers} | {:ok, status(), headers, map()} @@ -18,7 +18,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do def request(method, url, headers, body, opts \\ []) do check_adapter() - opts = Keyword.merge(opts, body_as: :chunks) + opts = Keyword.put(opts, :body_as, :chunks) with {:ok, response} <- Pleroma.HTTP.request( @@ -39,7 +39,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do end @impl true - @spec stream_body(map()) :: {:ok, binary(), map()} | {:error, atom() | String.t()} | :done + @spec stream_body(map()) :: + {:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return() def stream_body(%{pid: pid, opts: opts, fin: true}) do # if connection was reused, but in tesla were redirects, # tesla returns new opened connection, which must be closed manually diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 8f1aa3200..35b973b56 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -59,7 +59,7 @@ defmodule Pleroma.ReverseProxy do * `req_headers`, `resp_headers` additional headers. - * `http`: options for [gun](https://github.com/ninenines/gun). + * `http`: options for [hackney](https://github.com/benoitc/hackney) or [gun](https://github.com/ninenines/gun). """ @default_options [pool: :media] -- cgit v1.2.3 From 7c8003c3fcdcab075b9722ab236bf2d1d0e0e8cd Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 15 Mar 2020 21:00:12 +0300 Subject: [#1364] Improved control over generation / sending of notifications. Fixed blocking / muting users notifications issue. Added tests. --- lib/pleroma/activity.ex | 10 ++ lib/pleroma/notification.ex | 127 ++++++++++++++++++------- lib/pleroma/thread_mute.ex | 37 +++++-- lib/pleroma/user.ex | 50 ++++++++-- lib/pleroma/user_relationship.ex | 9 +- lib/pleroma/web/activity_pub/transmogrifier.ex | 8 +- 6 files changed, 185 insertions(+), 56 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 6ca05f74e..bbaa561a7 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -95,6 +95,16 @@ defmodule Pleroma.Activity do |> preload([activity, object: object], object: object) end + def user_actor(%Activity{actor: nil}), do: nil + + def user_actor(%Activity{} = activity) do + with %User{} <- activity.user_actor do + activity.user_actor + else + _ -> User.get_cached_by_ap_id(activity.actor) + end + end + def with_joined_user_actor(query, join_type \\ :inner) do join(query, join_type, [activity], u in User, on: u.ap_id == activity.actor, diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 60dba3434..0d7a6610a 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Notification do alias Pleroma.Object alias Pleroma.Pagination alias Pleroma.Repo + alias Pleroma.ThreadMute alias Pleroma.User alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.Push @@ -17,6 +18,7 @@ defmodule Pleroma.Notification do import Ecto.Query import Ecto.Changeset + require Logger @type t :: %__MODULE__{} @@ -101,7 +103,7 @@ defmodule Pleroma.Notification do query |> where([n, a], a.actor not in ^notification_muted_ap_ids) - |> join(:left, [n, a], tm in Pleroma.ThreadMute, + |> join(:left, [n, a], tm in ThreadMute, on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data) ) |> where([n, a, o, tm], is_nil(tm.user_id)) @@ -284,58 +286,108 @@ defmodule Pleroma.Notification do def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do object = Object.normalize(activity) - unless object && object.data["type"] == "Answer" do - users = get_notified_from_activity(activity) - notifications = Enum.map(users, fn user -> create_notification(activity, user) end) - {:ok, notifications} - else + if object && object.data["type"] == "Answer" do {:ok, []} + else + do_create_notifications(activity) end end def create_notifications(%Activity{data: %{"type" => type}} = activity) when type in ["Like", "Announce", "Follow", "Move", "EmojiReact"] do + do_create_notifications(activity) + end + + def create_notifications(_), do: {:ok, []} + + defp do_create_notifications(%Activity{} = activity) do + {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity) + potential_receivers = enabled_receivers ++ disabled_receivers + notifications = - activity - |> get_notified_from_activity() - |> Enum.map(&create_notification(activity, &1)) + Enum.map(potential_receivers, fn user -> + do_send = user in enabled_receivers + create_notification(activity, user, do_send) + end) {:ok, notifications} end - def create_notifications(_), do: {:ok, []} - # TODO move to sql, too. - def create_notification(%Activity{} = activity, %User{} = user) do + def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true) do unless skip?(activity, user) do notification = %Notification{user_id: user.id, activity: activity} {:ok, notification} = Repo.insert(notification) - ["user", "user:notification"] - |> Streamer.stream(notification) + if do_send do + Streamer.stream(["user", "user:notification"], notification) + Push.send(notification) + end - Push.send(notification) notification end end + @doc """ + Returns a tuple with 2 elements: + {enabled notification receivers, currently disabled receivers (blocking / [thread] muting)} + """ def get_notified_from_activity(activity, local_only \\ true) def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only) when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do - [] - |> Utils.maybe_notify_to_recipients(activity) - |> Utils.maybe_notify_mentioned_recipients(activity) - |> Utils.maybe_notify_subscribers(activity) - |> Utils.maybe_notify_followers(activity) - |> Enum.uniq() - |> User.get_users_from_set(local_only) + potential_receiver_ap_ids = + [] + |> Utils.maybe_notify_to_recipients(activity) + |> Utils.maybe_notify_mentioned_recipients(activity) + |> Utils.maybe_notify_subscribers(activity) + |> Utils.maybe_notify_followers(activity) + |> Enum.uniq() + + notification_enabled_ap_ids = + potential_receiver_ap_ids + |> exclude_relation_restricting_ap_ids(activity) + |> exclude_thread_muter_ap_ids(activity) + + potential_receivers = + potential_receiver_ap_ids + |> Enum.uniq() + |> User.get_users_from_set(local_only) + + notification_enabled_users = + Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) + + {notification_enabled_users, potential_receivers -- notification_enabled_users} + end + + def get_notified_from_activity(_, _local_only), do: {[], []} + + @doc "Filters out AP IDs of users basing on their relationships with activity actor user" + def exclude_relation_restricting_ap_ids([], _activity), do: [] + + def exclude_relation_restricting_ap_ids(ap_ids, %Activity{} = activity) do + relation_restricted_ap_ids = + activity + |> Activity.user_actor() + |> User.incoming_relations_ungrouped_ap_ids([ + :block, + :notification_mute + ]) + + Enum.uniq(ap_ids) -- relation_restricted_ap_ids end - def get_notified_from_activity(_, _local_only), do: [] + @doc "Filters out AP IDs of users who mute activity thread" + def exclude_thread_muter_ap_ids([], _activity), do: [] + + def exclude_thread_muter_ap_ids(ap_ids, %Activity{} = activity) do + thread_muter_ap_ids = ThreadMute.muter_ap_ids(activity.data["context"]) + + Enum.uniq(ap_ids) -- thread_muter_ap_ids + end @spec skip?(Activity.t(), User.t()) :: boolean() - def skip?(activity, user) do + def skip?(%Activity{} = activity, %User{} = user) do [ :self, :followers, @@ -344,18 +396,20 @@ defmodule Pleroma.Notification do :non_follows, :recently_followed ] - |> Enum.any?(&skip?(&1, activity, user)) + |> Enum.find(&skip?(&1, activity, user)) end + def skip?(_, _), do: false + @spec skip?(atom(), Activity.t(), User.t()) :: boolean() - def skip?(:self, activity, user) do + def skip?(:self, %Activity{} = activity, %User{} = user) do activity.data["actor"] == user.ap_id end def skip?( :followers, - activity, - %{notification_settings: %{followers: false}} = user + %Activity{} = activity, + %User{notification_settings: %{followers: false}} = user ) do actor = activity.data["actor"] follower = User.get_cached_by_ap_id(actor) @@ -364,15 +418,19 @@ defmodule Pleroma.Notification do def skip?( :non_followers, - activity, - %{notification_settings: %{non_followers: false}} = user + %Activity{} = activity, + %User{notification_settings: %{non_followers: false}} = user ) do actor = activity.data["actor"] follower = User.get_cached_by_ap_id(actor) !User.following?(follower, user) end - def skip?(:follows, activity, %{notification_settings: %{follows: false}} = user) do + def skip?( + :follows, + %Activity{} = activity, + %User{notification_settings: %{follows: false}} = user + ) do actor = activity.data["actor"] followed = User.get_cached_by_ap_id(actor) User.following?(user, followed) @@ -380,15 +438,16 @@ defmodule Pleroma.Notification do def skip?( :non_follows, - activity, - %{notification_settings: %{non_follows: false}} = user + %Activity{} = activity, + %User{notification_settings: %{non_follows: false}} = user ) do actor = activity.data["actor"] followed = User.get_cached_by_ap_id(actor) !User.following?(user, followed) end - def skip?(:recently_followed, %{data: %{"type" => "Follow"}} = activity, user) do + # To do: consider defining recency in hours and checking FollowingRelationship with a single SQL + def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do actor = activity.data["actor"] Notification.for_user(user) diff --git a/lib/pleroma/thread_mute.ex b/lib/pleroma/thread_mute.ex index cc815430a..2b4cf02cf 100644 --- a/lib/pleroma/thread_mute.ex +++ b/lib/pleroma/thread_mute.ex @@ -9,7 +9,8 @@ defmodule Pleroma.ThreadMute do alias Pleroma.ThreadMute alias Pleroma.User - require Ecto.Query + import Ecto.Changeset + import Ecto.Query schema "thread_mutes" do belongs_to(:user, User, type: FlakeId.Ecto.CompatType) @@ -18,19 +19,43 @@ defmodule Pleroma.ThreadMute do def changeset(mute, params \\ %{}) do mute - |> Ecto.Changeset.cast(params, [:user_id, :context]) - |> Ecto.Changeset.foreign_key_constraint(:user_id) - |> Ecto.Changeset.unique_constraint(:user_id, name: :unique_index) + |> cast(params, [:user_id, :context]) + |> foreign_key_constraint(:user_id) + |> unique_constraint(:user_id, name: :unique_index) end def query(user_id, context) do {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id) ThreadMute - |> Ecto.Query.where(user_id: ^user_id) - |> Ecto.Query.where(context: ^context) + |> where(user_id: ^user_id) + |> where(context: ^context) end + def muters_query(context) do + ThreadMute + |> join(:inner, [tm], u in assoc(tm, :user)) + |> where([tm], tm.context == ^context) + |> select([tm, u], u.ap_id) + end + + def muter_ap_ids(context, ap_ids \\ nil) + + def muter_ap_ids(context, ap_ids) when context not in [nil, ""] do + context + |> muters_query() + |> maybe_filter_on_ap_id(ap_ids) + |> Repo.all() + end + + def muter_ap_ids(_context, _ap_ids), do: [] + + defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do + where(query, [tm, u], u.ap_id in ^ap_ids) + end + + defp maybe_filter_on_ap_id(query, _ap_ids), do: query + def add_mute(user_id, context) do %ThreadMute{} |> changeset(%{user_id: user_id, context: context}) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index db510d957..8c8ecfe35 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -149,22 +149,26 @@ defmodule Pleroma.User do {outgoing_relation, outgoing_relation_target}, {incoming_relation, incoming_relation_source} ]} <- @user_relationships_config do - # Definitions of `has_many :blocker_blocks`, `has_many :muter_mutes` etc. + # Definitions of `has_many` relations: :blocker_blocks, :muter_mutes, :reblog_muter_mutes, + # :notification_muter_mutes, :subscribee_subscriptions has_many(outgoing_relation, UserRelationship, foreign_key: :source_id, where: [relationship_type: relationship_type] ) - # Definitions of `has_many :blockee_blocks`, `has_many :mutee_mutes` etc. + # Definitions of `has_many` relations: :blockee_blocks, :mutee_mutes, :reblog_mutee_mutes, + # :notification_mutee_mutes, :subscriber_subscriptions has_many(incoming_relation, UserRelationship, foreign_key: :target_id, where: [relationship_type: relationship_type] ) - # Definitions of `has_many :blocked_users`, `has_many :muted_users` etc. + # Definitions of `has_many` relations: :blocked_users, :muted_users, :reblog_muted_users, + # :notification_muted_users, :subscriber_users has_many(outgoing_relation_target, through: [outgoing_relation, :target]) - # Definitions of `has_many :blocker_users`, `has_many :muter_users` etc. + # Definitions of `has_many` relations: :blocker_users, :muter_users, :reblog_muter_users, + # :notification_muter_users, :subscribee_users has_many(incoming_relation_source, through: [incoming_relation, :source]) end @@ -184,7 +188,9 @@ defmodule Pleroma.User do for {_relationship_type, [{_outgoing_relation, outgoing_relation_target}, _]} <- @user_relationships_config do - # Definitions of `blocked_users_relation/1`, `muted_users_relation/1`, etc. + # `def blocked_users_relation/2`, `def muted_users_relation/2`, + # `def reblog_muted_users_relation/2`, `def notification_muted_users/2`, + # `def subscriber_users/2` def unquote(:"#{outgoing_relation_target}_relation")(user, restrict_deactivated? \\ false) do target_users_query = assoc(user, unquote(outgoing_relation_target)) @@ -195,7 +201,8 @@ defmodule Pleroma.User do end end - # Definitions of `blocked_users/1`, `muted_users/1`, etc. + # `def blocked_users/2`, `def muted_users/2`, `def reblog_muted_users/2`, + # `def notification_muted_users/2`, `def subscriber_users/2` def unquote(outgoing_relation_target)(user, restrict_deactivated? \\ false) do __MODULE__ |> apply(unquote(:"#{outgoing_relation_target}_relation"), [ @@ -205,7 +212,8 @@ defmodule Pleroma.User do |> Repo.all() end - # Definitions of `blocked_users_ap_ids/1`, `muted_users_ap_ids/1`, etc. + # `def blocked_users_ap_ids/2`, `def muted_users_ap_ids/2`, `def reblog_muted_users_ap_ids/2`, + # `def notification_muted_users_ap_ids/2`, `def subscriber_users_ap_ids/2` def unquote(:"#{outgoing_relation_target}_ap_ids")(user, restrict_deactivated? \\ false) do __MODULE__ |> apply(unquote(:"#{outgoing_relation_target}_relation"), [ @@ -1217,7 +1225,9 @@ defmodule Pleroma.User do E.g. `outgoing_relations_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}` """ @spec outgoing_relations_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())} - def outgoing_relations_ap_ids(_, []), do: %{} + def outgoing_relations_ap_ids(_user, []), do: %{} + + def outgoing_relations_ap_ids(nil, _relationship_types), do: %{} def outgoing_relations_ap_ids(%User{} = user, relationship_types) when is_list(relationship_types) do @@ -1238,6 +1248,30 @@ defmodule Pleroma.User do ) end + def incoming_relations_ungrouped_ap_ids(user, relationship_types, ap_ids \\ nil) + + def incoming_relations_ungrouped_ap_ids(_user, [], _ap_ids), do: [] + + def incoming_relations_ungrouped_ap_ids(nil, _relationship_types, _ap_ids), do: [] + + def incoming_relations_ungrouped_ap_ids(%User{} = user, relationship_types, ap_ids) + when is_list(relationship_types) do + user + |> assoc(:incoming_relationships) + |> join(:inner, [user_rel], u in assoc(user_rel, :source)) + |> where([user_rel, u], user_rel.relationship_type in ^relationship_types) + |> maybe_filter_on_ap_id(ap_ids) + |> select([user_rel, u], u.ap_id) + |> distinct(true) + |> Repo.all() + end + + defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do + where(query, [user_rel, u], u.ap_id in ^ap_ids) + end + + defp maybe_filter_on_ap_id(query, _ap_ids), do: query + def deactivate_async(user, status \\ true) do BackgroundWorker.enqueue("deactivate_user", %{"user_id" => user.id, "status" => status}) end diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 393947942..01b6ace9d 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -21,15 +21,18 @@ defmodule Pleroma.UserRelationship do end for relationship_type <- Keyword.keys(UserRelationshipTypeEnum.__enum_map__()) do - # Definitions of `create_block/2`, `create_mute/2` etc. + # `def create_block/2`, `def create_mute/2`, `def create_reblog_mute/2`, + # `def create_notification_mute/2`, `def create_inverse_subscription/2` def unquote(:"create_#{relationship_type}")(source, target), do: create(unquote(relationship_type), source, target) - # Definitions of `delete_block/2`, `delete_mute/2` etc. + # `def delete_block/2`, `def delete_mute/2`, `def delete_reblog_mute/2`, + # `def delete_notification_mute/2`, `def delete_inverse_subscription/2` def unquote(:"delete_#{relationship_type}")(source, target), do: delete(unquote(relationship_type), source, target) - # Definitions of `block_exists?/2`, `mute_exists?/2` etc. + # `def block_exists?/2`, `def mute_exists?/2`, `def reblog_mute_exists?/2`, + # `def notification_mute_exists?/2`, `def inverse_subscription_exists?/2` def unquote(:"#{relationship_type}_exists?")(source, target), do: exists?(unquote(relationship_type), source, target) end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 9cd3de705..d6549a932 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1108,13 +1108,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def add_mention_tags(object) do - mentions = - object - |> Utils.get_notified_from_object() - |> Enum.map(&build_mention_tag/1) + {enabled_receivers, disabled_receivers} = Utils.get_notified_from_object(object) + potential_receivers = enabled_receivers ++ disabled_receivers + mentions = Enum.map(potential_receivers, &build_mention_tag/1) tags = object["tag"] || [] - Map.put(object, "tag", tags ++ mentions) end -- cgit v1.2.3 From 26e2076659450361b4fd4252c7a7b838099c442b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 10 Mar 2020 18:11:48 +0300 Subject: fix for feed page pagination --- lib/pleroma/web/controller_helper.ex | 7 ++++--- lib/pleroma/web/feed/tag_controller.ex | 4 ++-- lib/pleroma/web/feed/user_controller.ex | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index c9a3a2585..ad293cda9 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -87,7 +87,8 @@ defmodule Pleroma.Web.ControllerHelper do render_error(conn, :not_implemented, "Can't display this activity") end - @spec put_in_if_exist(map(), atom() | String.t(), any) :: map() - def put_in_if_exist(map, _key, nil), do: map - def put_in_if_exist(map, key, value), do: put_in(map, key, value) + @spec put_if_exist(map(), atom() | String.t(), any) :: map() + def put_if_exist(map, _key, nil), do: map + + def put_if_exist(map, key, value), do: Map.put(map, key, value) end diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex index 75c9ea17e..904047b12 100644 --- a/lib/pleroma/web/feed/tag_controller.ex +++ b/lib/pleroma/web/feed/tag_controller.ex @@ -9,14 +9,14 @@ defmodule Pleroma.Web.Feed.TagController do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.Feed.FeedView - import Pleroma.Web.ControllerHelper, only: [put_in_if_exist: 3] + import Pleroma.Web.ControllerHelper, only: [put_if_exist: 3] def feed(conn, %{"tag" => raw_tag} = params) do {format, tag} = parse_tag(raw_tag) activities = %{"type" => ["Create"], "tag" => tag} - |> put_in_if_exist("max_id", params["max_id"]) + |> put_if_exist("max_id", params["max_id"]) |> ActivityPub.fetch_public_activities() conn diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index 9ba602d9f..9ffb3b9be 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.Feed.UserController do alias Pleroma.Web.ActivityPub.ActivityPubController alias Pleroma.Web.Feed.FeedView - import Pleroma.Web.ControllerHelper, only: [put_in_if_exist: 3] + import Pleroma.Web.ControllerHelper, only: [put_if_exist: 3] plug(Pleroma.Plugs.SetFormatPlug when action in [:feed_redirect]) @@ -46,7 +46,7 @@ defmodule Pleroma.Web.Feed.UserController do "type" => ["Create"], "actor_id" => user.ap_id } - |> put_in_if_exist("max_id", params["max_id"]) + |> put_if_exist("max_id", params["max_id"]) |> ActivityPub.fetch_public_activities() conn -- cgit v1.2.3 From 91870c8995c154839d611bcce6d038f72ef0665c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 13 Mar 2020 17:41:26 +0300 Subject: adding rss for user feed --- lib/pleroma/web/feed/user_controller.ex | 13 +++++- lib/pleroma/web/router.ex | 2 +- .../web/templates/feed/feed/_activity.atom.eex | 50 ++++++++++++++++++++++ .../web/templates/feed/feed/_activity.rss.eex | 49 +++++++++++++++++++++ .../web/templates/feed/feed/_activity.xml.eex | 50 ---------------------- .../web/templates/feed/feed/_author.atom.eex | 17 ++++++++ .../web/templates/feed/feed/_author.rss.eex | 17 ++++++++ .../web/templates/feed/feed/_author.xml.eex | 17 -------- lib/pleroma/web/templates/feed/feed/user.atom.eex | 24 +++++++++++ lib/pleroma/web/templates/feed/feed/user.rss.eex | 20 +++++++++ lib/pleroma/web/templates/feed/feed/user.xml.eex | 24 ----------- 11 files changed, 189 insertions(+), 94 deletions(-) create mode 100644 lib/pleroma/web/templates/feed/feed/_activity.atom.eex create mode 100644 lib/pleroma/web/templates/feed/feed/_activity.rss.eex delete mode 100644 lib/pleroma/web/templates/feed/feed/_activity.xml.eex create mode 100644 lib/pleroma/web/templates/feed/feed/_author.atom.eex create mode 100644 lib/pleroma/web/templates/feed/feed/_author.rss.eex delete mode 100644 lib/pleroma/web/templates/feed/feed/_author.xml.eex create mode 100644 lib/pleroma/web/templates/feed/feed/user.atom.eex create mode 100644 lib/pleroma/web/templates/feed/feed/user.rss.eex delete mode 100644 lib/pleroma/web/templates/feed/feed/user.xml.eex (limited to 'lib') diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index 9ffb3b9be..e27f85929 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -40,6 +40,15 @@ defmodule Pleroma.Web.Feed.UserController do end def feed(conn, %{"nickname" => nickname} = params) do + format = get_format(conn) + + format = + if format in ["rss", "atom"] do + format + else + "atom" + end + with {_, %User{} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do activities = %{ @@ -50,9 +59,9 @@ defmodule Pleroma.Web.Feed.UserController do |> ActivityPub.fetch_public_activities() conn - |> put_resp_content_type("application/atom+xml") + |> put_resp_content_type("application/#{format}+xml") |> put_view(FeedView) - |> render("user.xml", + |> render("user.#{format}", user: user, activities: activities, feed_config: Pleroma.Config.get([:feed]) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e4e3ee704..3f36f6c1a 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -513,7 +513,7 @@ defmodule Pleroma.Web.Router do end pipeline :ostatus do - plug(:accepts, ["html", "xml", "atom", "activity+json", "json"]) + plug(:accepts, ["html", "xml", "rss", "atom", "activity+json", "json"]) plug(Pleroma.Plugs.StaticFEPlug) end diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex new file mode 100644 index 000000000..ac8a75009 --- /dev/null +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -0,0 +1,50 @@ + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <%= @data["id"] %> + <%= activity_title(@object, Keyword.get(@feed_config, :post_title, %{})) %> + <%= activity_content(@object) %> + <%= @data["published"] %> + <%= @data["published"] %> + + <%= activity_context(@activity) %> + + + + <%= if @data["summary"] do %> + <%= @data["summary"] %> + <% end %> + + <%= if @activity.local do %> + + + <% else %> + + <% end %> + + <%= for tag <- @data["tag"] || [] do %> + + <% end %> + + <%= for attachment <- @data["attachment"] || [] do %> + + <% end %> + + <%= if @data["inReplyTo"] do %> + + <% end %> + + <%= for id <- @activity.recipients do %> + <%= if id == Pleroma.Constants.as_public() do %> + + <% else %> + <%= unless Regex.match?(~r/^#{Pleroma.Web.base_url()}.+followers$/, id) do %> + + <% end %> + <% end %> + <% end %> + + <%= for {emoji, file} <- @data["emoji"] || %{} do %> + + <% end %> + diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex new file mode 100644 index 000000000..a4dbed638 --- /dev/null +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -0,0 +1,49 @@ + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <%= @data["id"] %> + <%= activity_title(@object, Keyword.get(@feed_config, :post_title, %{})) %> + <%= activity_content(@object) %> + <%= @data["published"] %> + <%= @data["published"] %> + + <%= activity_context(@activity) %> + + <%= activity_context(@activity) %> + + <%= if @data["summary"] do %> + <%= @data["summary"] %> + <% end %> + + <%= if @activity.local do %> + <%= @data["id"] %> + <% else %> + <%= @data["external_url"] %> + <% end %> + + <%= for tag <- @data["tag"] || [] do %> + + <% end %> + + <%= for attachment <- @data["attachment"] || [] do %> + <%= attachment_href(attachment) %> + <% end %> + + <%= if @data["inReplyTo"] do %> + + <% end %> + + <%= for id <- @activity.recipients do %> + <%= if id == Pleroma.Constants.as_public() do %> + http://activityschema.org/collection/public + <% else %> + <%= unless Regex.match?(~r/^#{Pleroma.Web.base_url()}.+followers$/, id) do %> + <%= id %> + <% end %> + <% end %> + <% end %> + + <%= for {emoji, file} <- @data["emoji"] || %{} do %> + <%= file %> + <% end %> + diff --git a/lib/pleroma/web/templates/feed/feed/_activity.xml.eex b/lib/pleroma/web/templates/feed/feed/_activity.xml.eex deleted file mode 100644 index ac8a75009..000000000 --- a/lib/pleroma/web/templates/feed/feed/_activity.xml.eex +++ /dev/null @@ -1,50 +0,0 @@ - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <%= @data["id"] %> - <%= activity_title(@object, Keyword.get(@feed_config, :post_title, %{})) %> - <%= activity_content(@object) %> - <%= @data["published"] %> - <%= @data["published"] %> - - <%= activity_context(@activity) %> - - - - <%= if @data["summary"] do %> - <%= @data["summary"] %> - <% end %> - - <%= if @activity.local do %> - - - <% else %> - - <% end %> - - <%= for tag <- @data["tag"] || [] do %> - - <% end %> - - <%= for attachment <- @data["attachment"] || [] do %> - - <% end %> - - <%= if @data["inReplyTo"] do %> - - <% end %> - - <%= for id <- @activity.recipients do %> - <%= if id == Pleroma.Constants.as_public() do %> - - <% else %> - <%= unless Regex.match?(~r/^#{Pleroma.Web.base_url()}.+followers$/, id) do %> - - <% end %> - <% end %> - <% end %> - - <%= for {emoji, file} <- @data["emoji"] || %{} do %> - - <% end %> - diff --git a/lib/pleroma/web/templates/feed/feed/_author.atom.eex b/lib/pleroma/web/templates/feed/feed/_author.atom.eex new file mode 100644 index 000000000..25cbffada --- /dev/null +++ b/lib/pleroma/web/templates/feed/feed/_author.atom.eex @@ -0,0 +1,17 @@ + + <%= @user.ap_id %> + http://activitystrea.ms/schema/1.0/person + <%= @user.ap_id %> + <%= @user.nickname %> + <%= @user.name %> + <%= escape(@user.bio) %> + <%= escape(@user.bio) %> + <%= @user.nickname %> + + <%= if User.banner_url(@user) do %> + + <% end %> + <%= if @user.local do %> + true + <% end %> + diff --git a/lib/pleroma/web/templates/feed/feed/_author.rss.eex b/lib/pleroma/web/templates/feed/feed/_author.rss.eex new file mode 100644 index 000000000..526aeddcf --- /dev/null +++ b/lib/pleroma/web/templates/feed/feed/_author.rss.eex @@ -0,0 +1,17 @@ + + <%= @user.ap_id %> + http://activitystrea.ms/schema/1.0/person + <%= @user.ap_id %> + <%= @user.nickname %> + <%= @user.name %> + <%= escape(@user.bio) %> + <%= escape(@user.bio) %> + <%= @user.nickname %> + <%= User.avatar_url(@user) %> + <%= if User.banner_url(@user) do %> + <%= User.banner_url(@user) %> + <% end %> + <%= if @user.local do %> + true + <% end %> + diff --git a/lib/pleroma/web/templates/feed/feed/_author.xml.eex b/lib/pleroma/web/templates/feed/feed/_author.xml.eex deleted file mode 100644 index 25cbffada..000000000 --- a/lib/pleroma/web/templates/feed/feed/_author.xml.eex +++ /dev/null @@ -1,17 +0,0 @@ - - <%= @user.ap_id %> - http://activitystrea.ms/schema/1.0/person - <%= @user.ap_id %> - <%= @user.nickname %> - <%= @user.name %> - <%= escape(@user.bio) %> - <%= escape(@user.bio) %> - <%= @user.nickname %> - - <%= if User.banner_url(@user) do %> - - <% end %> - <%= if @user.local do %> - true - <% end %> - diff --git a/lib/pleroma/web/templates/feed/feed/user.atom.eex b/lib/pleroma/web/templates/feed/feed/user.atom.eex new file mode 100644 index 000000000..c6acd848f --- /dev/null +++ b/lib/pleroma/web/templates/feed/feed/user.atom.eex @@ -0,0 +1,24 @@ + + + + <%= user_feed_url(@conn, :feed, @user.nickname) <> ".atom" %> + <%= @user.nickname <> "'s timeline" %> + <%= most_recent_update(@activities, @user) %> + <%= logo(@user) %> + + + <%= render @view_module, "_author.atom", assigns %> + + <%= if last_activity(@activities) do %> + + <% end %> + + <%= for activity <- @activities do %> + <%= render @view_module, "_activity.atom", Map.merge(assigns, prepare_activity(activity)) %> + <% end %> + diff --git a/lib/pleroma/web/templates/feed/feed/user.rss.eex b/lib/pleroma/web/templates/feed/feed/user.rss.eex new file mode 100644 index 000000000..d69120480 --- /dev/null +++ b/lib/pleroma/web/templates/feed/feed/user.rss.eex @@ -0,0 +1,20 @@ + + + + <%= user_feed_url(@conn, :feed, @user.nickname) <> ".rss" %> + <%= @user.nickname <> "'s timeline" %> + <%= most_recent_update(@activities, @user) %> + <%= logo(@user) %> + <%= '#{user_feed_url(@conn, :feed, @user.nickname)}.rss' %> + + <%= render @view_module, "_author.rss", assigns %> + + <%= if last_activity(@activities) do %> + <%= '#{user_feed_url(@conn, :feed, @user.nickname)}.rss?max_id=#{last_activity(@activities).id}' %> + <% end %> + + <%= for activity <- @activities do %> + <%= render @view_module, "_activity.rss", Map.merge(assigns, prepare_activity(activity)) %> + <% end %> + + diff --git a/lib/pleroma/web/templates/feed/feed/user.xml.eex b/lib/pleroma/web/templates/feed/feed/user.xml.eex deleted file mode 100644 index d274c08ae..000000000 --- a/lib/pleroma/web/templates/feed/feed/user.xml.eex +++ /dev/null @@ -1,24 +0,0 @@ - - - - <%= user_feed_url(@conn, :feed, @user.nickname) <> ".atom" %> - <%= @user.nickname <> "'s timeline" %> - <%= most_recent_update(@activities, @user) %> - <%= logo(@user) %> - - - <%= render @view_module, "_author.xml", assigns %> - - <%= if last_activity(@activities) do %> - - <% end %> - - <%= for activity <- @activities do %> - <%= render @view_module, "_activity.xml", Map.merge(assigns, prepare_activity(activity)) %> - <% end %> - -- cgit v1.2.3 From 89e4b3ebbd433032a2687712c9c6684902fe4ebe Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 13 Mar 2020 17:58:14 +0300 Subject: fix for content-type header for tag feed --- lib/pleroma/web/feed/tag_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex index 904047b12..8133f8480 100644 --- a/lib/pleroma/web/feed/tag_controller.ex +++ b/lib/pleroma/web/feed/tag_controller.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.Feed.TagController do |> ActivityPub.fetch_public_activities() conn - |> put_resp_content_type("application/atom+xml") + |> put_resp_content_type("application/#{format}+xml") |> put_view(FeedView) |> render("tag.#{format}", activities: activities, -- cgit v1.2.3 From 35471205f862fa069c6d87aefc1d827c9fab6e08 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 16 Mar 2020 15:47:25 +0300 Subject: temp fix for `:gun.info` MatchError --- lib/pleroma/pool/connections.ex | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 772833509..16aa80548 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -169,19 +169,26 @@ defmodule Pleroma.Pool.Connections do @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do - %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) - - host = - case :inet.ntoa(host) do - {:error, :einval} -> host - ip -> ip + # TODO: temp fix for gun MatchError https://github.com/ninenines/gun/issues/222 + # TODO: REMOVE LATER + {key, conn} = + try do + %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) + + host = + case :inet.ntoa(host) do + {:error, :einval} -> host + ip -> ip + end + + key = "#{scheme}:#{host}:#{port}" + find_conn(state.conns, conn_pid, key) + rescue + MatcheError -> find_conn(state.conns, conn_pid) end - key = "#{scheme}:#{host}:#{port}" - state = - with {_key, conn} <- find_conn(state.conns, conn_pid, key), - {true, key} <- {Process.alive?(conn_pid), key} do + with {true, key} <- {Process.alive?(conn_pid), key} do put_in(state.conns[key], %{ conn | gun_state: :up, -- cgit v1.2.3 From bf474ca3c154544b54720ea23c06191e68f32522 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 16 Mar 2020 16:23:49 +0300 Subject: fix --- lib/pleroma/pool/connections.ex | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 16aa80548..91102faf7 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -167,28 +167,30 @@ defmodule Pleroma.Pool.Connections do c1.crf <= c2.crf and c1.last_reference <= c2.last_reference end - @impl true - def handle_info({:gun_up, conn_pid, _protocol}, state) do + defp find_conn_from_gun_info(conns, pid) do # TODO: temp fix for gun MatchError https://github.com/ninenines/gun/issues/222 # TODO: REMOVE LATER - {key, conn} = - try do - %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) - - host = - case :inet.ntoa(host) do - {:error, :einval} -> host - ip -> ip - end - - key = "#{scheme}:#{host}:#{port}" - find_conn(state.conns, conn_pid, key) - rescue - MatcheError -> find_conn(state.conns, conn_pid) - end + try do + %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) + host = + case :inet.ntoa(host) do + {:error, :einval} -> host + ip -> ip + end + + key = "#{scheme}:#{host}:#{port}" + find_conn(conns, pid, key) + rescue + MatcheError -> find_conn(conns, pid) + end + end + + @impl true + def handle_info({:gun_up, conn_pid, _protocol}, state) do state = - with {true, key} <- {Process.alive?(conn_pid), key} do + with {key, conn} <- find_conn_from_gun_info(state.conns, conn_pid), + {true, key} <- {Process.alive?(conn_pid), key} do put_in(state.conns[key], %{ conn | gun_state: :up, -- cgit v1.2.3 From d198e7fa2a0c92be4e99c5a765de85096d318bfe Mon Sep 17 00:00:00 2001 From: eugenijm Date: Tue, 28 Jan 2020 09:47:59 +0300 Subject: Admin API: `PATCH /api/pleroma/admin/users/:nickname/change_password` --- lib/pleroma/moderation_log.ex | 11 ++++++++ lib/pleroma/web/admin_api/admin_api_controller.ex | 33 +++++++++++++++++++++++ lib/pleroma/web/router.ex | 1 + 3 files changed, 45 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index e32895f70..b5435a553 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -605,6 +605,17 @@ defmodule Pleroma.ModerationLog do }" end + @spec get_log_entry_message(ModerationLog) :: String.t() + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "change_password", + "subject" => subjects + } + }) do + "@#{actor_nickname} changed password for users: #{users_to_nicknames_string(subjects)}" + end + defp nicknames_to_string(nicknames) do nicknames |> Enum.map(&"@#{&1}") diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 175260bc2..2aa2c6ac2 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -658,6 +658,39 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do json_response(conn, :no_content, "") end + @doc "Changes password for a given user" + def change_password(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} = params) do + with {_, user} <- {:user, User.get_cached_by_nickname(nickname)}, + {:ok, _user} <- + User.reset_password(user, %{ + password: params["new_password"], + password_confirmation: params["new_password"] + }) do + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "change_password" + }) + + User.force_password_reset_async(user) + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "force_password_reset" + }) + + json(conn, %{status: "success"}) + else + {:error, changeset} -> + {_, {error, _}} = Enum.at(changeset.errors, 0) + json(conn, %{error: "New password #{error}."}) + + _ -> + json(conn, %{error: "Unable to change password."}) + end + end + def list_reports(conn, params) do {page, page_size} = page_params(params) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e4e3ee704..c03ad101e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -173,6 +173,7 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/password_reset", AdminAPIController, :get_password_reset) patch("/users/force_password_reset", AdminAPIController, :force_password_reset) + patch("/users/:nickname/change_password", AdminAPIController, :change_password) get("/users", AdminAPIController, :list_users) get("/users/:nickname", AdminAPIController, :user_show) -- cgit v1.2.3 From 13cce9c0debbf9a80ed5da26cb34ca563e5e1417 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Fri, 31 Jan 2020 21:07:46 +0300 Subject: Admin API: `PATCH /api/pleroma/admin/users/:nickname/credentials`, `GET /api/pleroma/admin/users/:nickname/credentials`. --- lib/pleroma/moderation_log.ex | 4 +- lib/pleroma/user.ex | 86 +++++++++++++++++++++- lib/pleroma/web/admin_api/admin_api_controller.ex | 34 ++++++--- lib/pleroma/web/admin_api/views/account_view.ex | 40 ++++++++++ .../mastodon_api/controllers/account_controller.ex | 60 +++------------ lib/pleroma/web/router.ex | 3 +- 6 files changed, 163 insertions(+), 64 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index b5435a553..7aacd9d80 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -609,11 +609,11 @@ defmodule Pleroma.ModerationLog do def get_log_entry_message(%ModerationLog{ data: %{ "actor" => %{"nickname" => actor_nickname}, - "action" => "change_password", + "action" => "updated_users", "subject" => subjects } }) do - "@#{actor_nickname} changed password for users: #{users_to_nicknames_string(subjects)}" + "@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}" end defp nicknames_to_string(nicknames) do diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 911dde6e2..44de64345 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -417,9 +417,55 @@ defmodule Pleroma.User do |> validate_format(:nickname, local_nickname_regex()) |> validate_length(:bio, max: bio_limit) |> validate_length(:name, min: 1, max: name_limit) + |> put_fields() + |> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)}) + |> put_change_if_present(:avatar, &put_upload(&1, :avatar)) + |> put_change_if_present(:banner, &put_upload(&1, :banner)) + |> put_change_if_present(:background, &put_upload(&1, :background)) + |> put_change_if_present( + :pleroma_settings_store, + &{:ok, Map.merge(struct.pleroma_settings_store, &1)} + ) |> validate_fields(false) end + defp put_fields(changeset) do + if raw_fields = get_change(changeset, :raw_fields) do + raw_fields = + raw_fields + |> Enum.filter(fn %{"name" => n} -> n != "" end) + + fields = + raw_fields + |> Enum.map(fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) + + changeset + |> put_change(:raw_fields, raw_fields) + |> put_change(:fields, fields) + else + changeset + end + end + + defp put_change_if_present(changeset, map_field, value_function) do + if value = get_change(changeset, map_field) do + with {:ok, new_value} <- value_function.(value) do + put_change(changeset, map_field, new_value) + else + _ -> changeset + end + else + changeset + end + end + + defp put_upload(value, type) do + with %Plug.Upload{} <- value, + {:ok, object} <- ActivityPub.upload(value, type: type) do + {:ok, object.data} + end + end + def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) @@ -463,6 +509,27 @@ defmodule Pleroma.User do |> validate_fields(remote?) end + def update_as_admin_changeset(struct, params) do + struct + |> update_changeset(params) + |> cast(params, [:email]) + |> delete_change(:also_known_as) + |> unique_constraint(:email) + |> validate_format(:email, @email_regex) + end + + @spec update_as_admin(%User{}, map) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} + def update_as_admin(user, params) do + params = Map.put(params, "password_confirmation", params["password"]) + changeset = update_as_admin_changeset(user, params) + + if params["password"] do + reset_password(user, changeset, params) + else + User.update_and_set_cache(changeset) + end + end + def password_update_changeset(struct, params) do struct |> cast(params, [:password, :password_confirmation]) @@ -473,10 +540,14 @@ defmodule Pleroma.User do end @spec reset_password(User.t(), map) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} - def reset_password(%User{id: user_id} = user, data) do + def reset_password(%User{} = user, params) do + reset_password(user, user, params) + end + + def reset_password(%User{id: user_id} = user, struct, params) do multi = Multi.new() - |> Multi.update(:user, password_update_changeset(user, data)) + |> Multi.update(:user, password_update_changeset(struct, params)) |> Multi.delete_all(:tokens, OAuth.Token.Query.get_by_user(user_id)) |> Multi.delete_all(:auth, OAuth.Authorization.delete_by_user_query(user)) @@ -1856,6 +1927,17 @@ defmodule Pleroma.User do def fields(%{fields: fields}), do: fields + def sanitized_fields(%User{} = user) do + user + |> User.fields() + |> Enum.map(fn %{"name" => name, "value" => value} -> + %{ + "name" => name, + "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) + } + end) + end + def validate_fields(changeset, remote? \\ false) do limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields limit = Pleroma.Config.get([:instance, limit_name], 0) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 2aa2c6ac2..0368df1e9 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do plug( OAuthScopesPlug, %{scopes: ["read:accounts"], admin: true} - when action in [:list_users, :user_show, :right_get] + when action in [:list_users, :user_show, :right_get, :show_user_credentials] ) plug( @@ -54,7 +54,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do :tag_users, :untag_users, :right_add, - :right_delete + :right_delete, + :update_user_credentials ] ) @@ -658,21 +659,34 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do json_response(conn, :no_content, "") end - @doc "Changes password for a given user" - def change_password(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} = params) do + @doc "Show a given user's credentials" + def show_user_credentials(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do + conn + |> put_view(AccountView) + |> render("credentials.json", %{user: user, for: admin}) + else + _ -> {:error, :not_found} + end + end + + @doc "Updates a given user" + def update_user_credentials( + %{assigns: %{user: admin}} = conn, + %{"nickname" => nickname} = params + ) do with {_, user} <- {:user, User.get_cached_by_nickname(nickname)}, {:ok, _user} <- - User.reset_password(user, %{ - password: params["new_password"], - password_confirmation: params["new_password"] - }) do + User.update_as_admin(user, params) do ModerationLog.insert_log(%{ actor: admin, subject: [user], - action: "change_password" + action: "updated_users" }) - User.force_password_reset_async(user) + if params["password"] do + User.force_password_reset_async(user) + end ModerationLog.insert_log(%{ actor: admin, diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index 1e03849de..a16a3ebf0 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -23,6 +23,43 @@ defmodule Pleroma.Web.AdminAPI.AccountView do } end + def render("credentials.json", %{user: user, for: for_user}) do + user = User.sanitize_html(user, User.html_filter_policy(for_user)) + avatar = User.avatar_url(user) |> MediaProxy.url() + banner = User.banner_url(user) |> MediaProxy.url() + background = image_url(user.background) |> MediaProxy.url() + + user + |> Map.take([ + :id, + :bio, + :email, + :fields, + :name, + :nickname, + :locked, + :no_rich_text, + :default_scope, + :hide_follows, + :hide_followers_count, + :hide_follows_count, + :hide_followers, + :hide_favorites, + :allow_following_move, + :show_role, + :skip_thread_containment, + :pleroma_settings_store, + :raw_fields, + :discoverable, + :actor_type + ]) + |> Map.merge(%{ + "avatar" => avatar, + "banner" => banner, + "background" => background + }) + end + def render("show.json", %{user: user}) do avatar = User.avatar_url(user) |> MediaProxy.url() display_name = Pleroma.HTML.strip_tags(user.name || user.nickname) @@ -104,4 +141,7 @@ defmodule Pleroma.Web.AdminAPI.AccountView do "" end end + + defp image_url(%{"url" => [%{"href" => href} | _]}), do: href + defp image_url(_), do: nil end diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 88c997b9f..56e6214c5 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2, truthy_param?: 1, assign_account_by_id: 2, json_response: 3] - alias Pleroma.Emoji alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.Plugs.RateLimiter alias Pleroma.User @@ -140,17 +139,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do def update_credentials(%{assigns: %{user: original_user}} = conn, params) do user = original_user - params = - if Map.has_key?(params, "fields_attributes") do - Map.update!(params, "fields_attributes", fn fields -> - fields - |> normalize_fields_attributes() - |> Enum.filter(fn %{"name" => n} -> n != "" end) - end) - else - params - end - user_params = [ :no_rich_text, @@ -169,46 +157,20 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do add_if_present(acc, params, to_string(key), key, &{:ok, truthy_param?(&1)}) end) |> add_if_present(params, "display_name", :name) - |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value, user)} end) - |> add_if_present(params, "avatar", :avatar, fn value -> - with %Plug.Upload{} <- value, - {:ok, object} <- ActivityPub.upload(value, type: :avatar) do - {:ok, object.data} - end - end) - |> add_if_present(params, "header", :banner, fn value -> - with %Plug.Upload{} <- value, - {:ok, object} <- ActivityPub.upload(value, type: :banner) do - {:ok, object.data} - end - end) - |> add_if_present(params, "pleroma_background_image", :background, fn value -> - with %Plug.Upload{} <- value, - {:ok, object} <- ActivityPub.upload(value, type: :background) do - {:ok, object.data} - end - end) - |> add_if_present(params, "fields_attributes", :fields, fn fields -> - fields = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) - - {:ok, fields} - end) - |> add_if_present(params, "fields_attributes", :raw_fields) - |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store, fn value -> - {:ok, Map.merge(user.pleroma_settings_store, value)} - end) + |> add_if_present(params, "note", :bio) + |> add_if_present(params, "avatar", :avatar) + |> add_if_present(params, "header", :banner) + |> add_if_present(params, "pleroma_background_image", :background) + |> add_if_present( + params, + "fields_attributes", + :raw_fields, + &{:ok, normalize_fields_attributes(&1)} + ) + |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store) |> add_if_present(params, "default_scope", :default_scope) |> add_if_present(params, "actor_type", :actor_type) - emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "") - - user_emojis = - user - |> Map.get(:emoji, []) - |> Enum.concat(Emoji.Formatter.get_emoji_map(emojis_text)) - |> Enum.dedup() - - user_params = Map.put(user_params, :emoji, user_emojis) changeset = User.update_changeset(user, user_params) with {:ok, user} <- User.update_and_set_cache(changeset) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index c03ad101e..2927775eb 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -173,7 +173,8 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/password_reset", AdminAPIController, :get_password_reset) patch("/users/force_password_reset", AdminAPIController, :force_password_reset) - patch("/users/:nickname/change_password", AdminAPIController, :change_password) + get("/users/:nickname/credentials", AdminAPIController, :show_user_credentials) + patch("/users/:nickname/credentials", AdminAPIController, :update_user_credentials) get("/users", AdminAPIController, :list_users) get("/users/:nickname", AdminAPIController, :user_show) -- cgit v1.2.3 From 74388336852b18d5d5f108a8305f1a038301f7a1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 16 Mar 2020 21:58:10 +0300 Subject: [#1364] Improved notification-related tests. --- lib/pleroma/notification.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 0d7a6610a..104368fd1 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -344,6 +344,7 @@ defmodule Pleroma.Notification do |> Utils.maybe_notify_followers(activity) |> Enum.uniq() + # Since even subscribers and followers can mute / thread-mute, filtering all above AP IDs notification_enabled_ap_ids = potential_receiver_ap_ids |> exclude_relation_restricting_ap_ids(activity) -- cgit v1.2.3 From 7d275970ab191af539acbc0baec3bc1d0a2558e1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 19 Mar 2020 10:08:11 -0500 Subject: Add emoji reactions to features in nodeinfo --- lib/pleroma/web/nodeinfo/nodeinfo_controller.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 18eb41333..c653a80c3 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -74,7 +74,8 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do end, if Config.get([:instance, :safe_dm_mentions]) do "safe_dm_mentions" - end + end, + "pleroma_emoji_reactions" ] |> Enum.filter(& &1) -- cgit v1.2.3 From fe15f0ba15d02809fa4c21fb646e65d06060f3bb Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 20 Mar 2020 13:04:37 +0300 Subject: restrict_unauthenticated setting --- lib/pleroma/user.ex | 13 +++++++- lib/pleroma/web/activity_pub/visibility.ex | 14 +++++++-- .../mastodon_api/controllers/account_controller.ex | 7 +++-- .../mastodon_api/controllers/status_controller.ex | 2 +- .../controllers/timeline_controller.ex | 35 +++++++++++++++------- 5 files changed, 53 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 911dde6e2..8693c0b80 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -237,7 +237,18 @@ defmodule Pleroma.User do def visible_for?(%User{invisible: true}, _), do: false - def visible_for?(%User{id: user_id}, %User{id: for_id}) when user_id == for_id, do: true + def visible_for?(%User{id: user_id}, %User{id: user_id}), do: true + + def visible_for?(%User{local: local} = user, nil) do + cfg_key = + if local, + do: :local, + else: :remote + + if Config.get([:restrict_unauthenticated, :profiles, cfg_key]), + do: false, + else: account_status(user) == :active + end def visible_for?(%User{} = user, for_user) do account_status(user) == :active || superuser?(for_user) diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index 6f226fc92..453a6842e 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -44,6 +44,7 @@ defmodule Pleroma.Web.ActivityPub.Visibility do def is_list?(%{data: %{"listMessage" => _}}), do: true def is_list?(_), do: false + @spec visible_for_user?(Activity.t(), User.t() | nil) :: boolean() def visible_for_user?(%{actor: ap_id}, %User{ap_id: ap_id}), do: true def visible_for_user?(%{data: %{"listMessage" => list_ap_id}} = activity, %User{} = user) do @@ -55,14 +56,21 @@ defmodule Pleroma.Web.ActivityPub.Visibility do def visible_for_user?(%{data: %{"listMessage" => _}}, nil), do: false - def visible_for_user?(activity, nil) do - is_public?(activity) + def visible_for_user?(%{local: local} = activity, nil) do + cfg_key = + if local, + do: :local, + else: :remote + + if Pleroma.Config.get([:restrict_unauthenticated, :activities, cfg_key]), + do: false, + else: is_public?(activity) end def visible_for_user?(activity, user) do x = [user.ap_id | User.following(user)] y = [activity.actor] ++ activity.data["to"] ++ (activity.data["cc"] || []) - visible_for_user?(activity, nil) || Enum.any?(x, &(&1 in y)) + is_public?(activity) || Enum.any?(x, &(&1 in y)) end def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 88c997b9f..6dbf11ac9 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -60,7 +60,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do plug( Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug - when action != :create + when action not in [:create, :show, :statuses] ) @relations [:follow, :unfollow] @@ -259,7 +259,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do @doc "GET /api/v1/accounts/:id/statuses" def statuses(%{assigns: %{user: reading_user}} = conn, params) do - with %User{} = user <- User.get_cached_by_nickname_or_id(params["id"], for: reading_user) do + with %User{} = user <- User.get_cached_by_nickname_or_id(params["id"], for: reading_user), + true <- User.visible_for?(user, reading_user) do params = params |> Map.put("tag", params["tagged"]) @@ -271,6 +272,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do |> add_link_headers(activities) |> put_view(StatusView) |> render("index.json", activities: activities, for: reading_user, as: :activity) + else + _e -> render_error(conn, :not_found, "Can't find user") end end diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 5c90065f6..37afe6949 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -76,7 +76,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do %{scopes: ["write:bookmarks"]} when action in [:bookmark, :unbookmark] ) - plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug) + plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug when action not in [:index, :show]) @rate_limited_status_actions ~w(reblog unreblog favourite unfavourite create delete)a diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index 09e08271b..91f41416d 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do plug(OAuthScopesPlug, %{scopes: ["read:statuses"]} when action in [:home, :direct]) plug(OAuthScopesPlug, %{scopes: ["read:lists"]} when action == :list) - plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug) + plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug when action != :public) plug(:put_view, Pleroma.Web.MastodonAPI.StatusView) @@ -75,17 +75,30 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do def public(%{assigns: %{user: user}} = conn, params) do local_only = truthy_param?(params["local"]) - activities = - params - |> Map.put("type", ["Create", "Announce"]) - |> Map.put("local_only", local_only) - |> Map.put("blocking_user", user) - |> Map.put("muting_user", user) - |> ActivityPub.fetch_public_activities() + cfg_key = + if local_only do + :local + else + :federated + end - conn - |> add_link_headers(activities, %{"local" => local_only}) - |> render("index.json", activities: activities, for: user, as: :activity) + restrict? = Pleroma.Config.get([:restrict_unauthenticated, :timelines, cfg_key]) + + if not (restrict? and is_nil(user)) do + activities = + params + |> Map.put("type", ["Create", "Announce"]) + |> Map.put("local_only", local_only) + |> Map.put("blocking_user", user) + |> Map.put("muting_user", user) + |> ActivityPub.fetch_public_activities() + + conn + |> add_link_headers(activities, %{"local" => local_only}) + |> render("index.json", activities: activities, for: user, as: :activity) + else + render_error(conn, :unauthorized, "authorization required for timeline view") + end end def hashtag_fetching(params, user, local_only) do -- cgit v1.2.3 From 981e015f1b68c7cf807b0ddbf3948809f11b7fff Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 22 Mar 2020 17:10:37 +0300 Subject: Mastodon API Account view: Remove an outdated hack The hack with caching the follow relationship was introduced when we still were storing it inside the follow activity, resulting in slow queries. Now we store follow state in `FollowRelationship` table, so this is no longer necessary. --- lib/pleroma/user.ex | 18 ------------------ lib/pleroma/web/activity_pub/activity_pub.ex | 3 +-- lib/pleroma/web/activity_pub/utils.ex | 5 +---- lib/pleroma/web/mastodon_api/views/account_view.ex | 13 +++---------- 4 files changed, 5 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 8693c0b80..12c2ad815 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -292,24 +292,6 @@ defmodule Pleroma.User do def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa def ap_following(%User{} = user), do: "#{ap_id(user)}/following" - def follow_state(%User{} = user, %User{} = target) do - case Utils.fetch_latest_follow(user, target) do - %{data: %{"state" => state}} -> state - # Ideally this would be nil, but then Cachex does not commit the value - _ -> false - end - end - - def get_cached_follow_state(user, target) do - key = "follow_state:#{user.ap_id}|#{target.ap_id}" - Cachex.fetch!(:user_cache, key, fn _ -> {:commit, follow_state(user, target)} end) - end - - @spec set_follow_state_cache(String.t(), String.t(), String.t()) :: {:ok | :error, boolean()} - def set_follow_state_cache(user_ap_id, target_ap_id, state) do - Cachex.put(:user_cache, "follow_state:#{user_ap_id}|#{target_ap_id}", state) - end - @spec restrict_deactivated(Ecto.Query.t()) :: Ecto.Query.t() def restrict_deactivated(query) do from(u in query, where: u.deactivated != ^true) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index d9f74b6a4..30e282840 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -503,8 +503,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp do_follow(follower, followed, activity_id, local) do with data <- make_follow_data(follower, followed, activity_id), {:ok, activity} <- insert(data, local), - :ok <- maybe_federate(activity), - _ <- User.set_follow_state_cache(follower.ap_id, followed.ap_id, activity.data["state"]) do + :ok <- maybe_federate(activity) do {:ok, activity} else {:error, error} -> Repo.rollback(error) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 15dd2ed45..c65bbed67 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -440,22 +440,19 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> update(set: [data: fragment("jsonb_set(data, '{state}', ?)", ^state)]) |> Repo.update_all([]) - User.set_follow_state_cache(actor, object, state) - activity = Activity.get_by_id(activity.id) {:ok, activity} end def update_follow_state( - %Activity{data: %{"actor" => actor, "object" => object}} = activity, + %Activity{} = activity, state ) do new_data = Map.put(activity.data, "state", state) changeset = Changeset.change(activity, data: new_data) with {:ok, activity} <- Repo.update(changeset) do - User.set_follow_state_cache(actor, object, state) {:ok, activity} end end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 341dc2c91..4ebce73b4 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -36,25 +36,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do end def render("relationship.json", %{user: %User{} = user, target: %User{} = target}) do - follow_state = User.get_cached_follow_state(user, target) - - requested = - if follow_state && !User.following?(user, target) do - follow_state == "pending" - else - false - end + follow_state = User.get_follow_state(user, target) %{ id: to_string(target.id), - following: User.following?(user, target), + following: follow_state == "accept", followed_by: User.following?(target, user), blocking: User.blocks_user?(user, target), blocked_by: User.blocks_user?(target, user), muting: User.mutes?(user, target), muting_notifications: User.muted_notifications?(user, target), subscribing: User.subscribed_to?(user, target), - requested: requested, + requested: follow_state == "pending", domain_blocking: User.blocks_domain?(user, target), showing_reblogs: User.showing_reblogs?(user, target), endorsed: false -- cgit v1.2.3 From 15be6ba9c200b2a4ae153d26876be1b5cbb6357e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 22 Mar 2020 16:38:12 +0100 Subject: AccountView: fix for other forms of
in bio Closes: https://git.pleroma.social/pleroma/pleroma/issues/1643 --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 4ebce73b4..2bf711386 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -115,7 +115,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do fields: user.fields, bot: bot, source: %{ - note: Pleroma.HTML.strip_tags((user.bio || "") |> String.replace("
", "\n")), + note: (user.bio || "") |> String.replace(~r(
), "\n") |> Pleroma.HTML.strip_tags(), sensitive: false, fields: user.raw_fields, pleroma: %{ -- cgit v1.2.3 From c2e415143b1dfe5d89eff06fbce6840c445aa5fa Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 22 Mar 2020 21:51:44 +0300 Subject: WIP: preloading of user relations for timeline/statuses rendering (performance improvement). --- lib/pleroma/user.ex | 6 +- lib/pleroma/user_relationship.ex | 44 ++++++++++++++ lib/pleroma/web/mastodon_api/views/account_view.ex | 69 ++++++++++++++++++---- lib/pleroma/web/mastodon_api/views/status_view.ex | 58 ++++++++++++++++-- 4 files changed, 159 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 12c2ad815..daaa6d86b 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1642,8 +1642,12 @@ defmodule Pleroma.User do |> Repo.all() end + def muting_reblogs?(%User{} = user, %User{} = target) do + UserRelationship.reblog_mute_exists?(user, target) + end + def showing_reblogs?(%User{} = user, %User{} = target) do - not UserRelationship.reblog_mute_exists?(user, target) + not muting_reblogs?(user, target) end @doc """ diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 393947942..167a3919c 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -8,6 +8,7 @@ defmodule Pleroma.UserRelationship do import Ecto.Changeset import Ecto.Query + alias FlakeId.Ecto.CompatType alias Pleroma.Repo alias Pleroma.User alias Pleroma.UserRelationship @@ -34,6 +35,10 @@ defmodule Pleroma.UserRelationship do do: exists?(unquote(relationship_type), source, target) end + def user_relationship_types, do: Keyword.keys(user_relationship_mappings()) + + def user_relationship_mappings, do: UserRelationshipTypeEnum.__enum_map__() + def changeset(%UserRelationship{} = user_relationship, params \\ %{}) do user_relationship |> cast(params, [:relationship_type, :source_id, :target_id]) @@ -72,6 +77,45 @@ defmodule Pleroma.UserRelationship do end end + def dictionary( + source_users, + target_users, + source_to_target_rel_types \\ nil, + target_to_source_rel_types \\ nil + ) + when is_list(source_users) and is_list(target_users) do + get_bin_ids = fn user -> + with {:ok, bin_id} <- CompatType.dump(user.id), do: bin_id + end + + source_user_ids = Enum.map(source_users, &get_bin_ids.(&1)) + target_user_ids = Enum.map(target_users, &get_bin_ids.(&1)) + + get_rel_type_codes = fn rel_type -> user_relationship_mappings()[rel_type] end + + source_to_target_rel_types = + Enum.map(source_to_target_rel_types || user_relationship_types(), &get_rel_type_codes.(&1)) + + target_to_source_rel_types = + Enum.map(target_to_source_rel_types || user_relationship_types(), &get_rel_type_codes.(&1)) + + __MODULE__ + |> where( + fragment( + "(source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?)) OR \ + (source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?))", + ^source_user_ids, + ^target_user_ids, + ^source_to_target_rel_types, + ^target_user_ids, + ^source_user_ids, + ^target_to_source_rel_types + ) + ) + |> select([ur], [ur.relationship_type, ur.source_id, ur.target_id]) + |> Repo.all() + end + defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do changeset |> validate_change(:target_id, fn _, target_id -> diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 4ebce73b4..15a579278 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -10,6 +10,19 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MediaProxy + def test_rel(user_relationships, rel_type, source, target, func) do + cond do + is_nil(source) or is_nil(target) -> + false + + user_relationships -> + [rel_type, source.id, target.id] in user_relationships + + true -> + func.(source, target) + end + end + def render("index.json", %{users: users} = opts) do users |> render_many(AccountView, "show.json", opts) @@ -35,21 +48,50 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do %{} end - def render("relationship.json", %{user: %User{} = user, target: %User{} = target}) do - follow_state = User.get_follow_state(user, target) + def render( + "relationship.json", + %{user: %User{} = reading_user, target: %User{} = target} = opts + ) do + user_relationships = Map.get(opts, :user_relationships) + + follow_state = User.get_follow_state(reading_user, target) + # TODO: add a note on adjusting StatusView.user_relationships_opt/1 re: preloading of user relations %{ id: to_string(target.id), following: follow_state == "accept", - followed_by: User.following?(target, user), - blocking: User.blocks_user?(user, target), - blocked_by: User.blocks_user?(target, user), - muting: User.mutes?(user, target), - muting_notifications: User.muted_notifications?(user, target), - subscribing: User.subscribed_to?(user, target), + followed_by: User.following?(target, reading_user), + blocking: + test_rel(user_relationships, :block, reading_user, target, &User.blocks_user?(&1, &2)), + blocked_by: + test_rel(user_relationships, :block, target, reading_user, &User.blocks_user?(&1, &2)), + muting: test_rel(user_relationships, :mute, reading_user, target, &User.mutes?(&1, &2)), + muting_notifications: + test_rel( + user_relationships, + :notification_mute, + reading_user, + target, + &User.muted_notifications?(&1, &2) + ), + subscribing: + test_rel( + user_relationships, + :inverse_subscription, + target, + reading_user, + &User.subscribed_to?(&2, &1) + ), requested: follow_state == "pending", - domain_blocking: User.blocks_domain?(user, target), - showing_reblogs: User.showing_reblogs?(user, target), + domain_blocking: User.blocks_domain?(reading_user, target), + showing_reblogs: + not test_rel( + user_relationships, + :reblog_mute, + reading_user, + target, + &User.muting_reblogs?(&1, &2) + ), endorsed: false } end @@ -93,7 +135,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do } end) - relationship = render("relationship.json", %{user: opts[:for], target: user}) + relationship = + render("relationship.json", %{ + user: opts[:for], + target: user, + user_relationships: opts[:user_relationships] + }) %{ id: to_string(user.id), diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index f7469cdff..e0c368ec9 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User + alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView @@ -70,11 +71,34 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do present?(user && user.ap_id in (object.data["announcements"] || [])) end + defp user_relationships_opt(opts) do + reading_user = opts[:for] + + if reading_user do + activities = opts[:activities] + actors = Enum.map(activities, fn a -> get_user(a.data["actor"]) end) + + UserRelationship.dictionary( + [reading_user], + actors, + [:block, :mute, :notification_mute, :reblog_mute], + [:block, :inverse_subscription] + ) + else + [] + end + end + def render("index.json", opts) do - replied_to_activities = get_replied_to_activities(opts.activities) - opts = Map.put(opts, :replied_to_activities, replied_to_activities) + activities = opts.activities + replied_to_activities = get_replied_to_activities(activities) + + opts = + opts + |> Map.put(:replied_to_activities, replied_to_activities) + |> Map.put(:user_relationships, user_relationships_opt(opts)) - safe_render_many(opts.activities, StatusView, "show.json", opts) + safe_render_many(activities, StatusView, "show.json", opts) end def render( @@ -107,7 +131,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do id: to_string(activity.id), uri: activity_object.data["id"], url: activity_object.data["id"], - account: AccountView.render("show.json", %{user: user, for: opts[:for]}), + account: + AccountView.render("show.json", %{ + user: user, + for: opts[:for], + user_relationships: opts[:user_relationships] + }), in_reply_to_id: nil, in_reply_to_account_id: nil, reblog: reblogged, @@ -253,11 +282,28 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do _ -> [] end + user_relationships_opt = opts[:user_relationships] + + muted = + thread_muted? || + Pleroma.Web.MastodonAPI.AccountView.test_rel( + user_relationships_opt, + :mute, + opts[:for], + user, + fn for_user, user -> User.mutes?(for_user, user) end + ) + %{ id: to_string(activity.id), uri: object.data["id"], url: url, - account: AccountView.render("show.json", %{user: user, for: opts[:for]}), + account: + AccountView.render("show.json", %{ + user: user, + for: opts[:for], + user_relationships: user_relationships_opt + }), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), reblog: nil, @@ -270,7 +316,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reblogged: reblogged?(activity, opts[:for]), favourited: present?(favorited), bookmarked: present?(bookmarked), - muted: thread_muted? || User.mutes?(opts[:for], user), + muted: muted, pinned: pinned?(activity, user), sensitive: sensitive, spoiler_text: summary, -- cgit v1.2.3 From 3c78e5f3275494b3dc4546e65f19eb3a3c97033a Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 23 Mar 2020 12:01:11 +0300 Subject: Preloading of follow relations for timeline/statuses rendering (performance improvement). Refactoring. --- lib/pleroma/following_relationship.ex | 26 ++++++++ lib/pleroma/user.ex | 7 ++ lib/pleroma/user_relationship.ex | 13 ++++ lib/pleroma/web/mastodon_api/views/account_view.ex | 75 ++++++++++++++++------ lib/pleroma/web/mastodon_api/views/status_view.ex | 46 ++++++++----- 5 files changed, 130 insertions(+), 37 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index a6d281151..dd1696136 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -129,4 +129,30 @@ defmodule Pleroma.FollowingRelationship do move_following(origin, target) end end + + def all_between_user_sets( + source_users, + target_users + ) + when is_list(source_users) and is_list(target_users) do + get_bin_ids = fn user -> + with {:ok, bin_id} <- CompatType.dump(user.id), do: bin_id + end + + source_user_ids = Enum.map(source_users, &get_bin_ids.(&1)) + target_user_ids = Enum.map(target_users, &get_bin_ids.(&1)) + + __MODULE__ + |> where( + fragment( + "(follower_id = ANY(?) AND following_id = ANY(?)) OR \ + (follower_id = ANY(?) AND following_id = ANY(?))", + ^source_user_ids, + ^target_user_ids, + ^target_user_ids, + ^source_user_ids + ) + ) + |> Repo.all() + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index daaa6d86b..eb72755a0 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -674,7 +674,14 @@ defmodule Pleroma.User do def get_follow_state(%User{} = follower, %User{} = following) do following_relationship = FollowingRelationship.get(follower, following) + get_follow_state(follower, following, following_relationship) + end + def get_follow_state( + %User{} = follower, + %User{} = following, + following_relationship + ) do case {following_relationship, following.local} do {nil, false} -> case Utils.fetch_latest_follow(follower, following) do diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 167a3919c..9423e3a42 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -116,6 +116,19 @@ defmodule Pleroma.UserRelationship do |> Repo.all() end + def exists?(dictionary, rel_type, source, target, func) do + cond do + is_nil(source) or is_nil(target) -> + false + + dictionary -> + [rel_type, source.id, target.id] in dictionary + + true -> + func.(source, target) + end + end + defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do changeset |> validate_change(:target_id, fn _, target_id -> diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 15a579278..2fe46158b 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -6,21 +6,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do use Pleroma.Web, :view alias Pleroma.User + alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MediaProxy - def test_rel(user_relationships, rel_type, source, target, func) do - cond do - is_nil(source) or is_nil(target) -> - false - - user_relationships -> - [rel_type, source.id, target.id] in user_relationships - - true -> - func.(source, target) - end + defp find_following_rel(following_relationships, follower, following) do + Enum.find(following_relationships, fn + fr -> fr.follower_id == follower.id and fr.following_id == following.id + end) end def render("index.json", %{users: users} = opts) do @@ -53,21 +47,61 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do %{user: %User{} = reading_user, target: %User{} = target} = opts ) do user_relationships = Map.get(opts, :user_relationships) + following_relationships = opts[:following_relationships] + + follow_state = + if following_relationships do + user_to_target_following_relation = + find_following_rel(following_relationships, reading_user, target) + + User.get_follow_state(reading_user, target, user_to_target_following_relation) + else + User.get_follow_state(reading_user, target) + end - follow_state = User.get_follow_state(reading_user, target) + followed_by = + if following_relationships do + with %{state: "accept"} <- + find_following_rel(following_relationships, target, reading_user) do + true + else + _ -> false + end + else + User.following?(target, reading_user) + end # TODO: add a note on adjusting StatusView.user_relationships_opt/1 re: preloading of user relations %{ id: to_string(target.id), following: follow_state == "accept", - followed_by: User.following?(target, reading_user), + followed_by: followed_by, blocking: - test_rel(user_relationships, :block, reading_user, target, &User.blocks_user?(&1, &2)), + UserRelationship.exists?( + user_relationships, + :block, + reading_user, + target, + &User.blocks_user?(&1, &2) + ), blocked_by: - test_rel(user_relationships, :block, target, reading_user, &User.blocks_user?(&1, &2)), - muting: test_rel(user_relationships, :mute, reading_user, target, &User.mutes?(&1, &2)), + UserRelationship.exists?( + user_relationships, + :block, + target, + reading_user, + &User.blocks_user?(&1, &2) + ), + muting: + UserRelationship.exists?( + user_relationships, + :mute, + reading_user, + target, + &User.mutes?(&1, &2) + ), muting_notifications: - test_rel( + UserRelationship.exists?( user_relationships, :notification_mute, reading_user, @@ -75,7 +109,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do &User.muted_notifications?(&1, &2) ), subscribing: - test_rel( + UserRelationship.exists?( user_relationships, :inverse_subscription, target, @@ -85,7 +119,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do requested: follow_state == "pending", domain_blocking: User.blocks_domain?(reading_user, target), showing_reblogs: - not test_rel( + not UserRelationship.exists?( user_relationships, :reblog_mute, reading_user, @@ -139,7 +173,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do render("relationship.json", %{ user: opts[:for], target: user, - user_relationships: opts[:user_relationships] + user_relationships: opts[:user_relationships], + following_relationships: opts[:following_relationships] }) %{ diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index e0c368ec9..55a5513f9 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Activity alias Pleroma.ActivityExpiration + alias Pleroma.FollowingRelationship alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo @@ -71,22 +72,31 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do present?(user && user.ap_id in (object.data["announcements"] || [])) end - defp user_relationships_opt(opts) do + defp relationships_opts(opts) do reading_user = opts[:for] - if reading_user do - activities = opts[:activities] - actors = Enum.map(activities, fn a -> get_user(a.data["actor"]) end) + {user_relationships, following_relationships} = + if reading_user do + activities = opts[:activities] + actors = Enum.map(activities, fn a -> get_user(a.data["actor"]) end) - UserRelationship.dictionary( - [reading_user], - actors, - [:block, :mute, :notification_mute, :reblog_mute], - [:block, :inverse_subscription] - ) - else - [] - end + user_relationships = + UserRelationship.dictionary( + [reading_user], + actors, + [:block, :mute, :notification_mute, :reblog_mute], + [:block, :inverse_subscription] + ) + + following_relationships = + FollowingRelationship.all_between_user_sets([reading_user], actors) + + {user_relationships, following_relationships} + else + {[], []} + end + + %{user_relationships: user_relationships, following_relationships: following_relationships} end def render("index.json", opts) do @@ -96,7 +106,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do opts = opts |> Map.put(:replied_to_activities, replied_to_activities) - |> Map.put(:user_relationships, user_relationships_opt(opts)) + |> Map.merge(relationships_opts(opts)) safe_render_many(activities, StatusView, "show.json", opts) end @@ -135,7 +145,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do AccountView.render("show.json", %{ user: user, for: opts[:for], - user_relationships: opts[:user_relationships] + user_relationships: opts[:user_relationships], + following_relationships: opts[:following_relationships] }), in_reply_to_id: nil, in_reply_to_account_id: nil, @@ -286,7 +297,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do muted = thread_muted? || - Pleroma.Web.MastodonAPI.AccountView.test_rel( + UserRelationship.exists?( user_relationships_opt, :mute, opts[:for], @@ -302,7 +313,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do AccountView.render("show.json", %{ user: user, for: opts[:for], - user_relationships: user_relationships_opt + user_relationships: user_relationships_opt, + following_relationships: opts[:following_relationships] }), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), -- cgit v1.2.3 From 5a34dca8eda46479a3459b60c623d6fa94fc662b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 23 Mar 2020 14:03:31 +0400 Subject: Add emoji support in statuses in staticfe --- lib/pleroma/web/static_fe/static_fe_controller.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 7f9464268..7a35238d7 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -60,7 +60,9 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do content = if data["content"] do - Pleroma.HTML.filter_tags(data["content"]) + data["content"] + |> Pleroma.HTML.filter_tags() + |> Pleroma.Emoji.Formatter.emojify(Map.get(data, "emoji", %{})) else nil end -- cgit v1.2.3 From 3bd2829e5c125f961b7508bf40ef534a21070562 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 23 Mar 2020 18:56:01 +0100 Subject: Benchmarks: Add timeline benchmark --- lib/pleroma/web/controller_helper.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index ad293cda9..b49523ec3 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -34,7 +34,12 @@ defmodule Pleroma.Web.ControllerHelper do defp param_to_integer(_, default), do: default - def add_link_headers(conn, activities, extra_params \\ %{}) do + def add_link_headers(conn, activities, extra_params \\ %{}) + + def add_link_headers(%{assigns: %{skip_link_headers: true}} = conn, _activities, _extra_params), + do: conn + + def add_link_headers(conn, activities, extra_params) do case List.last(activities) do %{id: max_id} -> params = -- cgit v1.2.3 From d1a9716a988fe9f670033ad46cc9637038fbd1e8 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 24 Mar 2020 17:38:18 +0400 Subject: Fix activity deletion --- lib/pleroma/web/activity_pub/activity_pub.ex | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 30e282840..974231925 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -583,6 +583,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + defp do_delete(%Object{data: %{"type" => "Tombstone", "id" => ap_id}}, _) do + activity = + ap_id + |> Activity.Queries.by_object_id() + |> Activity.Queries.by_type("Delete") + |> Repo.one() + + {:ok, activity} + end + @spec block(User.t(), User.t(), String.t() | nil, boolean()) :: {:ok, Activity.t()} | {:error, any()} def block(blocker, blocked, activity_id \\ nil, local \\ true) do -- cgit v1.2.3 From 13cbb9f6ada8dcb15bb7ed12be4d88a18c5db7f7 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 24 Mar 2020 22:14:26 +0300 Subject: Implemented preloading of relationships with parent activities' actors for statuses/timeline rendering. Applied preloading for notifications rendering. Fixed announces rendering issue (preloading-related). --- lib/pleroma/activity/queries.ex | 7 ++ lib/pleroma/web/mastodon_api/views/account_view.ex | 15 ++-- .../web/mastodon_api/views/notification_view.ex | 98 +++++++++++++++++----- lib/pleroma/web/mastodon_api/views/status_view.ex | 89 +++++++++++--------- 4 files changed, 140 insertions(+), 69 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index 04593b9fb..a34c20343 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -35,6 +35,13 @@ defmodule Pleroma.Activity.Queries do from(a in query, where: a.actor == ^ap_id) end + def find_by_object_ap_id(activities, object_ap_id) do + Enum.find( + activities, + &(object_ap_id in [is_map(&1.data["object"]) && &1.data["object"]["id"], &1.data["object"]]) + ) + end + @spec by_object_id(query, String.t() | [String.t()]) :: query def by_object_id(query \\ Activity, object_id) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 2fe46158b..89bea9957 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -46,8 +46,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do "relationship.json", %{user: %User{} = reading_user, target: %User{} = target} = opts ) do - user_relationships = Map.get(opts, :user_relationships) - following_relationships = opts[:following_relationships] + user_relationships = get_in(opts, [:relationships, :user_relationships]) + following_relationships = get_in(opts, [:relationships, :following_relationships]) follow_state = if following_relationships do @@ -61,17 +61,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do followed_by = if following_relationships do - with %{state: "accept"} <- - find_following_rel(following_relationships, target, reading_user) do - true - else + case find_following_rel(following_relationships, target, reading_user) do + %{state: "accept"} -> true _ -> false end else User.following?(target, reading_user) end - # TODO: add a note on adjusting StatusView.user_relationships_opt/1 re: preloading of user relations + # NOTE: adjust StatusView.relationships_opts/2 if adding new relation-related flags %{ id: to_string(target.id), following: follow_state == "accept", @@ -173,8 +171,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do render("relationship.json", %{ user: opts[:for], target: user, - user_relationships: opts[:user_relationships], - following_relationships: opts[:following_relationships] + relationships: opts[:relationships] }) %{ diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 33145c484..e9c618496 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -13,19 +13,68 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do alias Pleroma.Web.MastodonAPI.NotificationView alias Pleroma.Web.MastodonAPI.StatusView - def render("index.json", %{notifications: notifications, for: user}) do - safe_render_many(notifications, NotificationView, "show.json", %{for: user}) + def render("index.json", %{notifications: notifications, for: reading_user}) do + activities = Enum.map(notifications, & &1.activity) + + parent_activities = + activities + |> Enum.filter( + &(Activity.mastodon_notification_type(&1) in [ + "favourite", + "reblog", + "pleroma:emoji_reaction" + ]) + ) + |> Enum.map(& &1.data["object"]) + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object(:left) + |> Pleroma.Repo.all() + + move_activities_targets = + activities + |> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move")) + |> Enum.map(&User.get_cached_by_ap_id(&1.data["target"])) + + actors = + activities + |> Enum.map(fn a -> User.get_cached_by_ap_id(a.data["actor"]) end) + |> Enum.filter(& &1) + |> Kernel.++(move_activities_targets) + + opts = %{ + for: reading_user, + parent_activities: parent_activities, + relationships: StatusView.relationships_opts(reading_user, actors) + } + + safe_render_many(notifications, NotificationView, "show.json", opts) end - def render("show.json", %{ - notification: %Notification{activity: activity} = notification, - for: user - }) do + def render( + "show.json", + %{ + notification: %Notification{activity: activity} = notification, + for: reading_user + } = opts + ) do actor = User.get_cached_by_ap_id(activity.data["actor"]) - parent_activity = Activity.get_create_by_object_ap_id(activity.data["object"]) + + parent_activity_fn = fn -> + if opts[:parent_activities] do + Activity.Queries.find_by_object_ap_id(opts[:parent_activities], activity.data["object"]) + else + Activity.get_create_by_object_ap_id(activity.data["object"]) + end + end + mastodon_type = Activity.mastodon_notification_type(activity) - with %{id: _} = account <- AccountView.render("show.json", %{user: actor, for: user}) do + with %{id: _} = account <- + AccountView.render("show.json", %{ + user: actor, + for: reading_user, + relationships: opts[:relationships] + }) do response = %{ id: to_string(notification.id), type: mastodon_type, @@ -36,24 +85,28 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do } } + relationships_opts = %{relationships: opts[:relationships]} + case mastodon_type do "mention" -> - put_status(response, activity, user) + put_status(response, activity, reading_user, relationships_opts) "favourite" -> - put_status(response, parent_activity, user) + put_status(response, parent_activity_fn.(), reading_user, relationships_opts) "reblog" -> - put_status(response, parent_activity, user) + put_status(response, parent_activity_fn.(), reading_user, relationships_opts) "move" -> - put_target(response, activity, user) + put_target(response, activity, reading_user, relationships_opts) "follow" -> response "pleroma:emoji_reaction" -> - put_status(response, parent_activity, user) |> put_emoji(activity) + response + |> put_status(parent_activity_fn.(), reading_user, relationships_opts) + |> put_emoji(activity) _ -> nil @@ -64,16 +117,21 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do end defp put_emoji(response, activity) do - response - |> Map.put(:emoji, activity.data["content"]) + Map.put(response, :emoji, activity.data["content"]) end - defp put_status(response, activity, user) do - Map.put(response, :status, StatusView.render("show.json", %{activity: activity, for: user})) + defp put_status(response, activity, reading_user, opts) do + status_render_opts = Map.merge(opts, %{activity: activity, for: reading_user}) + status_render = StatusView.render("show.json", status_render_opts) + + Map.put(response, :status, status_render) end - defp put_target(response, activity, user) do - target = User.get_cached_by_ap_id(activity.data["target"]) - Map.put(response, :target, AccountView.render("show.json", %{user: target, for: user})) + defp put_target(response, activity, reading_user, opts) do + target_user = User.get_cached_by_ap_id(activity.data["target"]) + target_render_opts = Map.merge(opts, %{user: target_user, for: reading_user}) + target_render = AccountView.render("show.json", target_render_opts) + + Map.put(response, :target, target_render) end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 55a5513f9..0ef65b352 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -72,41 +72,46 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do present?(user && user.ap_id in (object.data["announcements"] || [])) end - defp relationships_opts(opts) do - reading_user = opts[:for] - - {user_relationships, following_relationships} = - if reading_user do - activities = opts[:activities] - actors = Enum.map(activities, fn a -> get_user(a.data["actor"]) end) - - user_relationships = - UserRelationship.dictionary( - [reading_user], - actors, - [:block, :mute, :notification_mute, :reblog_mute], - [:block, :inverse_subscription] - ) - - following_relationships = - FollowingRelationship.all_between_user_sets([reading_user], actors) - - {user_relationships, following_relationships} - else - {[], []} - end + def relationships_opts(_reading_user = nil, _actors) do + %{user_relationships: [], following_relationships: []} + end + + def relationships_opts(reading_user, actors) do + user_relationships = + UserRelationship.dictionary( + [reading_user], + actors, + [:block, :mute, :notification_mute, :reblog_mute], + [:block, :inverse_subscription] + ) + + following_relationships = FollowingRelationship.all_between_user_sets([reading_user], actors) %{user_relationships: user_relationships, following_relationships: following_relationships} end def render("index.json", opts) do - activities = opts.activities + # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list + activities = Enum.filter(opts.activities, & &1) replied_to_activities = get_replied_to_activities(activities) + parent_activities = + activities + |> Enum.filter(&(&1.data["type"] == "Announce" && &1.data["object"])) + |> Enum.map(&Object.normalize(&1).data["id"]) + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object(:left) + |> Activity.with_preloaded_bookmark(opts[:for]) + |> Activity.with_set_thread_muted_field(opts[:for]) + |> Repo.all() + + actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) + opts = opts |> Map.put(:replied_to_activities, replied_to_activities) - |> Map.merge(relationships_opts(opts)) + |> Map.put(:parent_activities, parent_activities) + |> Map.put(:relationships, relationships_opts(opts[:for], actors)) safe_render_many(activities, StatusView, "show.json", opts) end @@ -119,17 +124,25 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do created_at = Utils.to_masto_date(activity.data["published"]) activity_object = Object.normalize(activity) - reblogged_activity = - Activity.create_by_object_ap_id(activity_object.data["id"]) - |> Activity.with_preloaded_bookmark(opts[:for]) - |> Activity.with_set_thread_muted_field(opts[:for]) - |> Repo.one() + reblogged_parent_activity = + if opts[:parent_activities] do + Activity.Queries.find_by_object_ap_id( + opts[:parent_activities], + activity_object.data["id"] + ) + else + Activity.create_by_object_ap_id(activity_object.data["id"]) + |> Activity.with_preloaded_bookmark(opts[:for]) + |> Activity.with_set_thread_muted_field(opts[:for]) + |> Repo.one() + end - reblogged = render("show.json", Map.put(opts, :activity, reblogged_activity)) + reblog_rendering_opts = Map.put(opts, :activity, reblogged_parent_activity) + reblogged = render("show.json", reblog_rendering_opts) favorited = opts[:for] && opts[:for].ap_id in (activity_object.data["likes"] || []) - bookmarked = Activity.get_bookmark(reblogged_activity, opts[:for]) != nil + bookmarked = Activity.get_bookmark(reblogged_parent_activity, opts[:for]) != nil mentions = activity.recipients @@ -145,8 +158,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do AccountView.render("show.json", %{ user: user, for: opts[:for], - user_relationships: opts[:user_relationships], - following_relationships: opts[:following_relationships] + relationships: opts[:relationships] }), in_reply_to_id: nil, in_reply_to_account_id: nil, @@ -156,7 +168,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reblogs_count: 0, replies_count: 0, favourites_count: 0, - reblogged: reblogged?(reblogged_activity, opts[:for]), + reblogged: reblogged?(reblogged_parent_activity, opts[:for]), favourited: present?(favorited), bookmarked: present?(bookmarked), muted: false, @@ -293,12 +305,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do _ -> [] end - user_relationships_opt = opts[:user_relationships] - muted = thread_muted? || UserRelationship.exists?( - user_relationships_opt, + get_in(opts, [:relationships, :user_relationships]), :mute, opts[:for], user, @@ -313,8 +323,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do AccountView.render("show.json", %{ user: user, for: opts[:for], - user_relationships: user_relationships_opt, - following_relationships: opts[:following_relationships] + relationships: opts[:relationships] }), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), -- cgit v1.2.3 From e743c2232970e321c833604b232520587ad8e402 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 25 Mar 2020 09:04:00 +0300 Subject: Fixed incorrect usage of "relations" as a short form of "relationships". --- lib/pleroma/notification.ex | 6 +++--- lib/pleroma/user.ex | 20 ++++++++++---------- lib/pleroma/web/activity_pub/activity_pub.ex | 8 ++++---- .../mastodon_api/controllers/account_controller.ex | 10 +++++++--- lib/pleroma/web/streamer/worker.ex | 2 +- 5 files changed, 25 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 104368fd1..bc691dce3 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -39,11 +39,11 @@ defmodule Pleroma.Notification do end defp for_user_query_ap_id_opts(user, opts) do - ap_id_relations = + ap_id_relationships = [:block] ++ if opts[@include_muted_option], do: [], else: [:notification_mute] - preloaded_ap_ids = User.outgoing_relations_ap_ids(user, ap_id_relations) + preloaded_ap_ids = User.outgoing_relationships_ap_ids(user, ap_id_relationships) exclude_blocked_opts = Map.merge(%{blocked_users_ap_ids: preloaded_ap_ids[:block]}, opts) @@ -370,7 +370,7 @@ defmodule Pleroma.Notification do relation_restricted_ap_ids = activity |> Activity.user_actor() - |> User.incoming_relations_ungrouped_ap_ids([ + |> User.incoming_relationships_ungrouped_ap_ids([ :block, :notification_mute ]) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 05efc74d4..4919c8e58 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1222,15 +1222,15 @@ defmodule Pleroma.User do end @doc """ - Returns map of outgoing (blocked, muted etc.) relations' user AP IDs by relation type. - E.g. `outgoing_relations_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}` + Returns map of outgoing (blocked, muted etc.) relationships' user AP IDs by relation type. + E.g. `outgoing_relationships_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}` """ - @spec outgoing_relations_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())} - def outgoing_relations_ap_ids(_user, []), do: %{} + @spec outgoing_relationships_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())} + def outgoing_relationships_ap_ids(_user, []), do: %{} - def outgoing_relations_ap_ids(nil, _relationship_types), do: %{} + def outgoing_relationships_ap_ids(nil, _relationship_types), do: %{} - def outgoing_relations_ap_ids(%User{} = user, relationship_types) + def outgoing_relationships_ap_ids(%User{} = user, relationship_types) when is_list(relationship_types) do db_result = user @@ -1249,13 +1249,13 @@ defmodule Pleroma.User do ) end - def incoming_relations_ungrouped_ap_ids(user, relationship_types, ap_ids \\ nil) + def incoming_relationships_ungrouped_ap_ids(user, relationship_types, ap_ids \\ nil) - def incoming_relations_ungrouped_ap_ids(_user, [], _ap_ids), do: [] + def incoming_relationships_ungrouped_ap_ids(_user, [], _ap_ids), do: [] - def incoming_relations_ungrouped_ap_ids(nil, _relationship_types, _ap_ids), do: [] + def incoming_relationships_ungrouped_ap_ids(nil, _relationship_types, _ap_ids), do: [] - def incoming_relations_ungrouped_ap_ids(%User{} = user, relationship_types, ap_ids) + def incoming_relationships_ungrouped_ap_ids(%User{} = user, relationship_types, ap_ids) when is_list(relationship_types) do user |> assoc(:incoming_relationships) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index d9f74b6a4..60e74758f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1230,17 +1230,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp fetch_activities_query_ap_ids_ops(opts) do source_user = opts["muting_user"] - ap_id_relations = if source_user, do: [:mute, :reblog_mute], else: [] + ap_id_relationships = if source_user, do: [:mute, :reblog_mute], else: [] - ap_id_relations = - ap_id_relations ++ + ap_id_relationships = + ap_id_relationships ++ if opts["blocking_user"] && opts["blocking_user"] == source_user do [:block] else [] end - preloaded_ap_ids = User.outgoing_relations_ap_ids(source_user, ap_id_relations) + preloaded_ap_ids = User.outgoing_relationships_ap_ids(source_user, ap_id_relationships) restrict_blocked_opts = Map.merge(%{"blocked_users_ap_ids" => preloaded_ap_ids[:block]}, opts) restrict_muted_opts = Map.merge(%{"muted_users_ap_ids" => preloaded_ap_ids[:mute]}, opts) diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 88c997b9f..9d83a9fc1 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -63,11 +63,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do when action != :create ) - @relations [:follow, :unfollow] + @relationship_actions [:follow, :unfollow] @needs_account ~W(followers following lists follow unfollow mute unmute block unblock)a - plug(RateLimiter, [name: :relations_id_action, params: ["id", "uri"]] when action in @relations) - plug(RateLimiter, [name: :relations_actions] when action in @relations) + plug( + RateLimiter, + [name: :relation_id_action, params: ["id", "uri"]] when action in @relationship_actions + ) + + plug(RateLimiter, [name: :relations_actions] when action in @relationship_actions) plug(RateLimiter, [name: :app_account_creation] when action == :create) plug(:assign_account_by_id when action in @needs_account) diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex index 29f992a67..abfed21c8 100644 --- a/lib/pleroma/web/streamer/worker.ex +++ b/lib/pleroma/web/streamer/worker.ex @@ -130,7 +130,7 @@ defmodule Pleroma.Web.Streamer.Worker do defp should_send?(%User{} = user, %Activity{} = item) do %{block: blocked_ap_ids, mute: muted_ap_ids, reblog_mute: reblog_muted_ap_ids} = - User.outgoing_relations_ap_ids(user, [:block, :mute, :reblog_mute]) + User.outgoing_relationships_ap_ids(user, [:block, :mute, :reblog_mute]) recipient_blocks = MapSet.new(blocked_ap_ids ++ muted_ap_ids) recipients = MapSet.new(item.recipients) -- cgit v1.2.3 From 3fa3d45dbecafb06fb7eb4f0260f610d4225e0a7 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 25 Mar 2020 13:05:00 +0300 Subject: [#1364] Minor improvements / comments. Further fixes of incorrect usage of "relations" as a short form of "relationships". --- lib/pleroma/activity.ex | 1 + lib/pleroma/notification.ex | 12 +++++++----- lib/pleroma/thread_mute.ex | 7 ++++--- 3 files changed, 12 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index bbaa561a7..5a8329e69 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -95,6 +95,7 @@ defmodule Pleroma.Activity do |> preload([activity, object: object], object: object) end + # Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.) def user_actor(%Activity{actor: nil}), do: nil def user_actor(%Activity{} = activity) do diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 63e3e9be9..04ee510b9 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -322,6 +322,8 @@ defmodule Pleroma.Notification do @doc """ Returns a tuple with 2 elements: {enabled notification receivers, currently disabled receivers (blocking / [thread] muting)} + + NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1 """ def get_notified_from_activity(activity, local_only \\ true) @@ -338,7 +340,7 @@ defmodule Pleroma.Notification do # Since even subscribers and followers can mute / thread-mute, filtering all above AP IDs notification_enabled_ap_ids = potential_receiver_ap_ids - |> exclude_relation_restricting_ap_ids(activity) + |> exclude_relationship_restricted_ap_ids(activity) |> exclude_thread_muter_ap_ids(activity) potential_receivers = @@ -355,10 +357,10 @@ defmodule Pleroma.Notification do def get_notified_from_activity(_, _local_only), do: {[], []} @doc "Filters out AP IDs of users basing on their relationships with activity actor user" - def exclude_relation_restricting_ap_ids([], _activity), do: [] + def exclude_relationship_restricted_ap_ids([], _activity), do: [] - def exclude_relation_restricting_ap_ids(ap_ids, %Activity{} = activity) do - relation_restricted_ap_ids = + def exclude_relationship_restricted_ap_ids(ap_ids, %Activity{} = activity) do + relationship_restricted_ap_ids = activity |> Activity.user_actor() |> User.incoming_relationships_ungrouped_ap_ids([ @@ -366,7 +368,7 @@ defmodule Pleroma.Notification do :notification_mute ]) - Enum.uniq(ap_ids) -- relation_restricted_ap_ids + Enum.uniq(ap_ids) -- relationship_restricted_ap_ids end @doc "Filters out AP IDs of users who mute activity thread" diff --git a/lib/pleroma/thread_mute.ex b/lib/pleroma/thread_mute.ex index 2b4cf02cf..a7ea13891 100644 --- a/lib/pleroma/thread_mute.ex +++ b/lib/pleroma/thread_mute.ex @@ -41,15 +41,16 @@ defmodule Pleroma.ThreadMute do def muter_ap_ids(context, ap_ids \\ nil) - def muter_ap_ids(context, ap_ids) when context not in [nil, ""] do + # Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.) + def muter_ap_ids(context, _ap_ids) when is_nil(context), do: [] + + def muter_ap_ids(context, ap_ids) do context |> muters_query() |> maybe_filter_on_ap_id(ap_ids) |> Repo.all() end - def muter_ap_ids(_context, _ap_ids), do: [] - defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do where(query, [tm, u], u.ap_id in ^ap_ids) end -- cgit v1.2.3 From be5e2c4dbba63831ea6a0617556e686969b5080f Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 25 Mar 2020 17:01:45 +0300 Subject: Applied relationships preloading to GET /api/v1/accounts/relationships. Refactoring (User.binary_id/1). --- lib/pleroma/conversation/participation.ex | 11 ++++------- lib/pleroma/following_relationship.ex | 8 ++------ lib/pleroma/thread_mute.ex | 4 ++-- lib/pleroma/user.ex | 15 +++++++++++++++ lib/pleroma/user_relationship.ex | 9 ++------- lib/pleroma/web/mastodon_api/views/account_view.ex | 6 +++++- 6 files changed, 30 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 693825cf5..215265fc9 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -129,21 +129,18 @@ defmodule Pleroma.Conversation.Participation do end def restrict_recipients(query, user, %{"recipients" => user_ids}) do - user_ids = + user_binary_ids = [user.id | user_ids] |> Enum.uniq() - |> Enum.reduce([], fn user_id, acc -> - {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id) - [user_id | acc] - end) + |> User.binary_id() conversation_subquery = __MODULE__ |> group_by([p], p.conversation_id) |> having( [p], - count(p.user_id) == ^length(user_ids) and - fragment("array_agg(?) @> ?", p.user_id, ^user_ids) + count(p.user_id) == ^length(user_binary_ids) and + fragment("array_agg(?) @> ?", p.user_id, ^user_binary_ids) ) |> select([p], %{id: p.conversation_id}) diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index dd1696136..624bddfe4 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -135,12 +135,8 @@ defmodule Pleroma.FollowingRelationship do target_users ) when is_list(source_users) and is_list(target_users) do - get_bin_ids = fn user -> - with {:ok, bin_id} <- CompatType.dump(user.id), do: bin_id - end - - source_user_ids = Enum.map(source_users, &get_bin_ids.(&1)) - target_user_ids = Enum.map(target_users, &get_bin_ids.(&1)) + source_user_ids = User.binary_id(source_users) + target_user_ids = User.binary_id(target_users) __MODULE__ |> where( diff --git a/lib/pleroma/thread_mute.ex b/lib/pleroma/thread_mute.ex index cc815430a..f657758aa 100644 --- a/lib/pleroma/thread_mute.ex +++ b/lib/pleroma/thread_mute.ex @@ -24,10 +24,10 @@ defmodule Pleroma.ThreadMute do end def query(user_id, context) do - {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id) + user_binary_id = User.binary_id(user_id) ThreadMute - |> Ecto.Query.where(user_id: ^user_id) + |> Ecto.Query.where(user_id: ^user_binary_id) |> Ecto.Query.where(context: ^context) end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f74e43cce..699256a3b 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -218,6 +218,21 @@ defmodule Pleroma.User do end end + @doc "Dumps id to SQL-compatible format" + def binary_id(source_id) when is_binary(source_id) do + with {:ok, dumped_id} <- FlakeId.Ecto.CompatType.dump(source_id) do + dumped_id + else + _ -> source_id + end + end + + def binary_id(source_ids) when is_list(source_ids) do + Enum.map(source_ids, &binary_id/1) + end + + def binary_id(%User{} = user), do: binary_id(user.id) + @doc "Returns status account" @spec account_status(User.t()) :: account_status() def account_status(%User{deactivated: true}), do: :deactivated diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 9423e3a42..519d2998d 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -8,7 +8,6 @@ defmodule Pleroma.UserRelationship do import Ecto.Changeset import Ecto.Query - alias FlakeId.Ecto.CompatType alias Pleroma.Repo alias Pleroma.User alias Pleroma.UserRelationship @@ -84,12 +83,8 @@ defmodule Pleroma.UserRelationship do target_to_source_rel_types \\ nil ) when is_list(source_users) and is_list(target_users) do - get_bin_ids = fn user -> - with {:ok, bin_id} <- CompatType.dump(user.id), do: bin_id - end - - source_user_ids = Enum.map(source_users, &get_bin_ids.(&1)) - target_user_ids = Enum.map(target_users, &get_bin_ids.(&1)) + source_user_ids = User.binary_id(source_users) + target_user_ids = User.binary_id(target_users) get_rel_type_codes = fn rel_type -> user_relationship_mappings()[rel_type] end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 702d9e658..6b2eca1f3 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView + alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.MediaProxy defp find_following_rel(following_relationships, follower, following) do @@ -129,7 +130,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do end def render("relationships.json", %{user: user, targets: targets}) do - render_many(targets, AccountView, "relationship.json", user: user, as: :target) + relationships_opts = StatusView.relationships_opts(user, targets) + opts = %{as: :target, user: user, relationships: relationships_opts} + + render_many(targets, AccountView, "relationship.json", opts) end defp do_render("show.json", %{user: user} = opts) do -- cgit v1.2.3 From 460e41585c2cd3f137c0f80173da60167fb318bf Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 25 Mar 2020 20:33:34 +0300 Subject: Further preloading (more endpoints), refactoring, tests. --- lib/pleroma/following_relationship.ex | 6 +++ lib/pleroma/user.ex | 5 ++- lib/pleroma/user_relationship.ex | 20 ++++++++++ lib/pleroma/web/mastodon_api/views/account_view.ex | 36 +++++++++++------- .../web/mastodon_api/views/notification_view.ex | 44 +++++++++++++--------- lib/pleroma/web/mastodon_api/views/status_view.ex | 29 ++++---------- 6 files changed, 86 insertions(+), 54 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index 624bddfe4..a9538ea4e 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -151,4 +151,10 @@ defmodule Pleroma.FollowingRelationship do ) |> Repo.all() end + + def find(following_relationships, follower, following) do + Enum.find(following_relationships, fn + fr -> fr.follower_id == follower.id and fr.following_id == following.id + end) + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 699256a3b..8ccb9242d 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -218,7 +218,10 @@ defmodule Pleroma.User do end end - @doc "Dumps id to SQL-compatible format" + @doc """ + Dumps Flake Id to SQL-compatible format (16-byte UUID). + E.g. "9pQtDGXuq4p3VlcJEm" -> <<0, 0, 1, 110, 179, 218, 42, 92, 213, 41, 44, 227, 95, 213, 0, 0>> + """ def binary_id(source_id) when is_binary(source_id) do with {:ok, dumped_id} <- FlakeId.Ecto.CompatType.dump(source_id) do dumped_id diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 519d2998d..011cf6822 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -8,6 +8,7 @@ defmodule Pleroma.UserRelationship do import Ecto.Changeset import Ecto.Query + alias Pleroma.FollowingRelationship alias Pleroma.Repo alias Pleroma.User alias Pleroma.UserRelationship @@ -124,6 +125,25 @@ defmodule Pleroma.UserRelationship do end end + @doc ":relationships option for StatusView / AccountView / NotificationView" + def view_relationships_option(nil = _reading_user, _actors) do + %{user_relationships: [], following_relationships: []} + end + + def view_relationships_option(%User{} = reading_user, actors) do + user_relationships = + UserRelationship.dictionary( + [reading_user], + actors, + [:block, :mute, :notification_mute, :reblog_mute], + [:block, :inverse_subscription] + ) + + following_relationships = FollowingRelationship.all_between_user_sets([reading_user], actors) + + %{user_relationships: user_relationships, following_relationships: following_relationships} + end + defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do changeset |> validate_change(:target_id, fn _, target_id -> diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 6b2eca1f3..2cdfac7af 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -5,20 +5,23 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do use Pleroma.Web, :view + alias Pleroma.FollowingRelationship alias Pleroma.User alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView - alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.MediaProxy - defp find_following_rel(following_relationships, follower, following) do - Enum.find(following_relationships, fn - fr -> fr.follower_id == follower.id and fr.following_id == following.id - end) - end - def render("index.json", %{users: users} = opts) do + relationships_opt = + if Map.has_key?(opts, :relationships) do + opts[:relationships] + else + UserRelationship.view_relationships_option(opts[:for], users) + end + + opts = Map.put(opts, :relationships, relationships_opt) + users |> render_many(AccountView, "show.json", opts) |> Enum.filter(&Enum.any?/1) @@ -53,7 +56,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do follow_state = if following_relationships do user_to_target_following_relation = - find_following_rel(following_relationships, reading_user, target) + FollowingRelationship.find(following_relationships, reading_user, target) User.get_follow_state(reading_user, target, user_to_target_following_relation) else @@ -62,7 +65,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do followed_by = if following_relationships do - case find_following_rel(following_relationships, target, reading_user) do + case FollowingRelationship.find(following_relationships, target, reading_user) do %{state: "accept"} -> true _ -> false end @@ -70,7 +73,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do User.following?(target, reading_user) end - # NOTE: adjust StatusView.relationships_opts/2 if adding new relation-related flags + # NOTE: adjust UserRelationship.view_relationships_option/2 on new relation-related flags %{ id: to_string(target.id), following: follow_state == "accept", @@ -129,11 +132,16 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do } end - def render("relationships.json", %{user: user, targets: targets}) do - relationships_opts = StatusView.relationships_opts(user, targets) - opts = %{as: :target, user: user, relationships: relationships_opts} + def render("relationships.json", %{user: user, targets: targets} = opts) do + relationships_opt = + if Map.has_key?(opts, :relationships) do + opts[:relationships] + else + UserRelationship.view_relationships_option(user, targets) + end - render_many(targets, AccountView, "relationship.json", opts) + render_opts = %{as: :target, user: user, relationships: relationships_opt} + render_many(targets, AccountView, "relationship.json", render_opts) end defp do_render("show.json", %{user: user} = opts) do diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index e9c618496..db434271c 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -8,12 +8,13 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do alias Pleroma.Activity alias Pleroma.Notification alias Pleroma.User + alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.NotificationView alias Pleroma.Web.MastodonAPI.StatusView - def render("index.json", %{notifications: notifications, for: reading_user}) do + def render("index.json", %{notifications: notifications, for: reading_user} = opts) do activities = Enum.map(notifications, & &1.activity) parent_activities = @@ -30,21 +31,28 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do |> Activity.with_preloaded_object(:left) |> Pleroma.Repo.all() - move_activities_targets = - activities - |> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move")) - |> Enum.map(&User.get_cached_by_ap_id(&1.data["target"])) - - actors = - activities - |> Enum.map(fn a -> User.get_cached_by_ap_id(a.data["actor"]) end) - |> Enum.filter(& &1) - |> Kernel.++(move_activities_targets) + relationships_opt = + if Map.has_key?(opts, :relationships) do + opts[:relationships] + else + move_activities_targets = + activities + |> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move")) + |> Enum.map(&User.get_cached_by_ap_id(&1.data["target"])) + + actors = + activities + |> Enum.map(fn a -> User.get_cached_by_ap_id(a.data["actor"]) end) + |> Enum.filter(& &1) + |> Kernel.++(move_activities_targets) + + UserRelationship.view_relationships_option(reading_user, actors) + end opts = %{ for: reading_user, parent_activities: parent_activities, - relationships: StatusView.relationships_opts(reading_user, actors) + relationships: relationships_opt } safe_render_many(notifications, NotificationView, "show.json", opts) @@ -85,27 +93,27 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do } } - relationships_opts = %{relationships: opts[:relationships]} + relationships_opt = %{relationships: opts[:relationships]} case mastodon_type do "mention" -> - put_status(response, activity, reading_user, relationships_opts) + put_status(response, activity, reading_user, relationships_opt) "favourite" -> - put_status(response, parent_activity_fn.(), reading_user, relationships_opts) + put_status(response, parent_activity_fn.(), reading_user, relationships_opt) "reblog" -> - put_status(response, parent_activity_fn.(), reading_user, relationships_opts) + put_status(response, parent_activity_fn.(), reading_user, relationships_opt) "move" -> - put_target(response, activity, reading_user, relationships_opts) + put_target(response, activity, reading_user, relationships_opt) "follow" -> response "pleroma:emoji_reaction" -> response - |> put_status(parent_activity_fn.(), reading_user, relationships_opts) + |> put_status(parent_activity_fn.(), reading_user, relationships_opt) |> put_emoji(activity) _ -> diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 0ef65b352..7b1cb7bf8 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -9,7 +9,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Activity alias Pleroma.ActivityExpiration - alias Pleroma.FollowingRelationship alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo @@ -72,24 +71,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do present?(user && user.ap_id in (object.data["announcements"] || [])) end - def relationships_opts(_reading_user = nil, _actors) do - %{user_relationships: [], following_relationships: []} - end - - def relationships_opts(reading_user, actors) do - user_relationships = - UserRelationship.dictionary( - [reading_user], - actors, - [:block, :mute, :notification_mute, :reblog_mute], - [:block, :inverse_subscription] - ) - - following_relationships = FollowingRelationship.all_between_user_sets([reading_user], actors) - - %{user_relationships: user_relationships, following_relationships: following_relationships} - end - def render("index.json", opts) do # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list activities = Enum.filter(opts.activities, & &1) @@ -105,13 +86,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Activity.with_set_thread_muted_field(opts[:for]) |> Repo.all() - actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) + relationships_opt = + if Map.has_key?(opts, :relationships) do + opts[:relationships] + else + actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) + UserRelationship.view_relationships_option(opts[:for], actors) + end opts = opts |> Map.put(:replied_to_activities, replied_to_activities) |> Map.put(:parent_activities, parent_activities) - |> Map.put(:relationships, relationships_opts(opts[:for], actors)) + |> Map.put(:relationships, relationships_opt) safe_render_many(activities, StatusView, "show.json", opts) end -- cgit v1.2.3 From 6b793d3f8336fcba5cac596f9e76d0274633f98d Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 26 Mar 2020 21:54:01 +0300 Subject: Ensured no auxiliary computations (actors list preparation etc.) related to relationships preloading if no user is present (for statuses / accounts / relationships rendering). --- lib/pleroma/web/mastodon_api/views/account_view.ex | 26 +++++++++++----- .../web/mastodon_api/views/notification_view.ex | 35 ++++++++++++---------- lib/pleroma/web/mastodon_api/views/status_view.ex | 16 ++++++---- 3 files changed, 49 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 2cdfac7af..0efcabc01 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -14,10 +14,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do def render("index.json", %{users: users} = opts) do relationships_opt = - if Map.has_key?(opts, :relationships) do - opts[:relationships] - else - UserRelationship.view_relationships_option(opts[:for], users) + cond do + Map.has_key?(opts, :relationships) -> + opts[:relationships] + + is_nil(opts[:for]) -> + UserRelationship.view_relationships_option(nil, []) + + true -> + UserRelationship.view_relationships_option(opts[:for], users) end opts = Map.put(opts, :relationships, relationships_opt) @@ -134,10 +139,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do def render("relationships.json", %{user: user, targets: targets} = opts) do relationships_opt = - if Map.has_key?(opts, :relationships) do - opts[:relationships] - else - UserRelationship.view_relationships_option(user, targets) + cond do + Map.has_key?(opts, :relationships) -> + opts[:relationships] + + is_nil(opts[:for]) -> + UserRelationship.view_relationships_option(nil, []) + + true -> + UserRelationship.view_relationships_option(user, targets) end render_opts = %{as: :target, user: user, relationships: relationships_opt} diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index db434271c..a809080fd 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -32,21 +32,26 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do |> Pleroma.Repo.all() relationships_opt = - if Map.has_key?(opts, :relationships) do - opts[:relationships] - else - move_activities_targets = - activities - |> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move")) - |> Enum.map(&User.get_cached_by_ap_id(&1.data["target"])) - - actors = - activities - |> Enum.map(fn a -> User.get_cached_by_ap_id(a.data["actor"]) end) - |> Enum.filter(& &1) - |> Kernel.++(move_activities_targets) - - UserRelationship.view_relationships_option(reading_user, actors) + cond do + Map.has_key?(opts, :relationships) -> + opts[:relationships] + + is_nil(opts[:for]) -> + UserRelationship.view_relationships_option(nil, []) + + true -> + move_activities_targets = + activities + |> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move")) + |> Enum.map(&User.get_cached_by_ap_id(&1.data["target"])) + + actors = + activities + |> Enum.map(fn a -> User.get_cached_by_ap_id(a.data["actor"]) end) + |> Enum.filter(& &1) + |> Kernel.++(move_activities_targets) + + UserRelationship.view_relationships_option(reading_user, actors) end opts = %{ diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 7b1cb7bf8..d36b9ee5c 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -87,11 +87,17 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Repo.all() relationships_opt = - if Map.has_key?(opts, :relationships) do - opts[:relationships] - else - actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) - UserRelationship.view_relationships_option(opts[:for], actors) + cond do + Map.has_key?(opts, :relationships) -> + opts[:relationships] + + is_nil(opts[:for]) -> + UserRelationship.view_relationships_option(nil, []) + + true -> + actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) + + UserRelationship.view_relationships_option(opts[:for], actors) end opts = -- cgit v1.2.3 From dfbc05d4965a04a82d4c4c5b8842f4117757f30e Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 27 Mar 2020 08:01:03 +0300 Subject: Misc refactoring / tweaks (`ThreadMute.exists?/2`). --- lib/pleroma/thread_mute.ex | 4 ++-- lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/mastodon_api/views/notification_view.ex | 12 ++++++------ lib/pleroma/web/mastodon_api/views/status_view.ex | 7 ++++--- 4 files changed, 13 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/thread_mute.ex b/lib/pleroma/thread_mute.ex index 5768e7711..be01d541d 100644 --- a/lib/pleroma/thread_mute.ex +++ b/lib/pleroma/thread_mute.ex @@ -68,8 +68,8 @@ defmodule Pleroma.ThreadMute do |> Repo.delete_all() end - def check_muted(user_id, context) do + def exists?(user_id, context) do query(user_id, context) - |> Repo.all() + |> Repo.exists?() end end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 091011c6b..2646b9f7b 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -358,7 +358,7 @@ defmodule Pleroma.Web.CommonAPI do def thread_muted?(%{id: nil} = _user, _activity), do: false def thread_muted?(user, activity) do - ThreadMute.check_muted(user.id, activity.data["context"]) != [] + ThreadMute.exists?(user.id, activity.data["context"]) end def report(user, %{"account_id" => account_id} = data) do diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index a809080fd..89f5734ff 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -98,27 +98,27 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do } } - relationships_opt = %{relationships: opts[:relationships]} + render_opts = %{relationships: opts[:relationships]} case mastodon_type do "mention" -> - put_status(response, activity, reading_user, relationships_opt) + put_status(response, activity, reading_user, render_opts) "favourite" -> - put_status(response, parent_activity_fn.(), reading_user, relationships_opt) + put_status(response, parent_activity_fn.(), reading_user, render_opts) "reblog" -> - put_status(response, parent_activity_fn.(), reading_user, relationships_opt) + put_status(response, parent_activity_fn.(), reading_user, render_opts) "move" -> - put_target(response, activity, reading_user, relationships_opt) + put_target(response, activity, reading_user, render_opts) "follow" -> response "pleroma:emoji_reaction" -> response - |> put_status(parent_activity_fn.(), reading_user, relationships_opt) + |> put_status(parent_activity_fn.(), reading_user, render_opts) |> put_emoji(activity) _ -> diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index d36b9ee5c..440eef4ba 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -228,9 +228,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end thread_muted? = - case activity.thread_muted? do - thread_muted? when is_boolean(thread_muted?) -> thread_muted? - nil -> (opts[:for] && CommonAPI.thread_muted?(opts[:for], activity)) || false + cond do + is_nil(opts[:for]) -> false + is_boolean(activity.thread_muted?) -> activity.thread_muted? + true -> CommonAPI.thread_muted?(opts[:for], activity) end attachment_data = object.data["attachment"] || [] -- cgit v1.2.3 From eb9744cadea7191b088ddaadfbd5fa4d4fd45090 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 14 Jan 2020 14:42:30 +0300 Subject: activities generation tasks --- lib/pleroma/application.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 33f1705df..51850abb5 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -157,7 +157,7 @@ defmodule Pleroma.Application do defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) - defp streamer_child(:test), do: [] + defp streamer_child(env) when env in [:test, :benchmark], do: [] defp streamer_child(_) do [Pleroma.Web.Streamer.supervisor()] -- cgit v1.2.3 From 1f29ecdcd7ecdc4ad8d6bc8fc4c34efbc9b7fe1d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 18 Feb 2020 12:19:10 +0300 Subject: sync with develop --- lib/mix/pleroma.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 3ad6edbfb..4dfcc32e7 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -5,6 +5,7 @@ defmodule Mix.Pleroma do @doc "Common functions to be reused in mix tasks" def start_pleroma do + Mix.Task.run("app.start") Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) if Pleroma.Config.get(:env) != :test do -- cgit v1.2.3 From 1fcdcb12a717fa3dbd54a5c3778bd216df6449ad Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 30 Mar 2020 12:47:12 +0300 Subject: updating gun with bug fix https://github.com/ninenines/gun/issues/222 --- lib/pleroma/pool/connections.ex | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 91102faf7..4d4ba913c 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -167,29 +167,20 @@ defmodule Pleroma.Pool.Connections do c1.crf <= c2.crf and c1.last_reference <= c2.last_reference end - defp find_conn_from_gun_info(conns, pid) do - # TODO: temp fix for gun MatchError https://github.com/ninenines/gun/issues/222 - # TODO: REMOVE LATER - try do - %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) - - host = - case :inet.ntoa(host) do - {:error, :einval} -> host - ip -> ip - end - - key = "#{scheme}:#{host}:#{port}" - find_conn(conns, pid, key) - rescue - MatcheError -> find_conn(conns, pid) - end - end - @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do + %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) + + host = + case :inet.ntoa(host) do + {:error, :einval} -> host + ip -> ip + end + + key = "#{scheme}:#{host}:#{port}" + state = - with {key, conn} <- find_conn_from_gun_info(state.conns, conn_pid), + with {key, conn} <- find_conn(state.conns, conn_pid, key), {true, key} <- {Process.alive?(conn_pid), key} do put_in(state.conns[key], %{ conn -- cgit v1.2.3 From b607ae1a1c0ef6557094ec0fb10ba2d19d621f7f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 30 Mar 2020 13:50:00 +0300 Subject: removing grouped reports admin api endpoint --- lib/pleroma/web/activity_pub/utils.ex | 96 ----------------------- lib/pleroma/web/admin_api/admin_api_controller.ex | 8 -- lib/pleroma/web/admin_api/views/report_view.ex | 28 +------ lib/pleroma/web/router.ex | 1 - 4 files changed, 1 insertion(+), 132 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index c65bbed67..2d685ecc0 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -795,102 +795,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do ActivityPub.fetch_activities([], params, :offset) end - def parse_report_group(activity) do - reports = get_reports_by_status_id(activity["id"]) - max_date = Enum.max_by(reports, &NaiveDateTime.from_iso8601!(&1.data["published"])) - actors = Enum.map(reports, & &1.user_actor) - [%{data: %{"object" => [account_id | _]}} | _] = reports - - account = - AccountView.render("show.json", %{ - user: User.get_by_ap_id(account_id) - }) - - status = get_status_data(activity) - - %{ - date: max_date.data["published"], - account: account, - status: status, - actors: Enum.uniq(actors), - reports: reports - } - end - - defp get_status_data(status) do - case status["deleted"] do - true -> - %{ - "id" => status["id"], - "deleted" => true - } - - _ -> - Activity.get_by_ap_id(status["id"]) - end - end - - def get_reports_by_status_id(ap_id) do - from(a in Activity, - where: fragment("(?)->>'type' = 'Flag'", a.data), - where: fragment("(?)->'object' @> ?", a.data, ^[%{id: ap_id}]), - or_where: fragment("(?)->'object' @> ?", a.data, ^[ap_id]) - ) - |> Activity.with_preloaded_user_actor() - |> Repo.all() - end - - @spec get_reports_grouped_by_status([String.t()]) :: %{ - required(:groups) => [ - %{ - required(:date) => String.t(), - required(:account) => %{}, - required(:status) => %{}, - required(:actors) => [%User{}], - required(:reports) => [%Activity{}] - } - ] - } - def get_reports_grouped_by_status(activity_ids) do - parsed_groups = - activity_ids - |> Enum.map(fn id -> - id - |> build_flag_object() - |> parse_report_group() - end) - - %{ - groups: parsed_groups - } - end - - @spec get_reported_activities() :: [ - %{ - required(:activity) => String.t(), - required(:date) => String.t() - } - ] - def get_reported_activities do - reported_activities_query = - from(a in Activity, - where: fragment("(?)->>'type' = 'Flag'", a.data), - select: %{ - activity: fragment("jsonb_array_elements((? #- '{object,0}')->'object')", a.data) - }, - group_by: fragment("activity") - ) - - from(a in subquery(reported_activities_query), - distinct: true, - select: %{ - id: fragment("COALESCE(?->>'id'::text, ? #>> '{}')", a.activity, a.activity) - } - ) - |> Repo.all() - |> Enum.map(& &1.id) - end - def update_report_state(%Activity{} = activity, state) when state in @strip_status_report_states do {:ok, stripped_activity} = strip_report_status_data(activity) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 0368df1e9..ca5439920 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -715,14 +715,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> render("index.json", %{reports: reports}) end - def list_grouped_reports(conn, _params) do - statuses = Utils.get_reported_activities() - - conn - |> put_view(ReportView) - |> render("index_grouped.json", Utils.get_reports_grouped_by_status(statuses)) - end - def report_show(conn, %{"id" => id}) do with %Activity{} = report <- Activity.get_by_id(id) do conn diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index fc8733ce8..ca0bcebc7 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.AdminAPI.ReportView do use Pleroma.Web, :view - alias Pleroma.Activity + alias Pleroma.HTML alias Pleroma.User alias Pleroma.Web.AdminAPI.Report @@ -44,32 +44,6 @@ defmodule Pleroma.Web.AdminAPI.ReportView do } end - def render("index_grouped.json", %{groups: groups}) do - reports = - Enum.map(groups, fn group -> - status = - case group.status do - %Activity{} = activity -> StatusView.render("show.json", %{activity: activity}) - _ -> group.status - end - - %{ - date: group[:date], - account: group[:account], - status: Map.put_new(status, "deleted", false), - actors: Enum.map(group[:actors], &merge_account_views/1), - reports: - group[:reports] - |> Enum.map(&Report.extract_report_info(&1)) - |> Enum.map(&render(__MODULE__, "show.json", &1)) - } - end) - - %{ - reports: reports - } - end - def render("index_notes.json", %{notes: notes}) when is_list(notes) do Enum.map(notes, &render(__MODULE__, "show_note.json", &1)) end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a22f744c1..5a0902739 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -186,7 +186,6 @@ defmodule Pleroma.Web.Router do patch("/users/resend_confirmation_email", AdminAPIController, :resend_confirmation_email) get("/reports", AdminAPIController, :list_reports) - get("/grouped_reports", AdminAPIController, :list_grouped_reports) get("/reports/:id", AdminAPIController, :report_show) patch("/reports", AdminAPIController, :reports_update) post("/reports/:id/notes", AdminAPIController, :report_notes_create) -- cgit v1.2.3 From b30fb1f3bbf8fb8e49cc5276225dc09771c79477 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 29 Mar 2020 22:30:50 +0200 Subject: User: Fix use of source_data in profile_url/1 --- lib/pleroma/user.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d9aa54057..ca0bfca11 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -305,7 +305,8 @@ defmodule Pleroma.User do end end - def profile_url(%User{source_data: %{"url" => url}}), do: url + def profile_url(%User{uri: url}) when url != nil, do: url + def profile_url(%User{source_data: %{"url" => url}}) when is_binary(url), do: url def profile_url(%User{ap_id: ap_id}), do: ap_id def profile_url(_), do: nil @@ -314,7 +315,7 @@ defmodule Pleroma.User do def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers" - @spec ap_following(User.t()) :: Sring.t() + @spec ap_following(User.t()) :: String.t() def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa def ap_following(%User{} = user), do: "#{ap_id(user)}/following" -- cgit v1.2.3 From 185520d1b4d3fdf8ecde7814faec92bbb531ce59 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 30 Mar 2020 02:01:09 +0200 Subject: Provide known-good user.uri, remove User.profile_url/1 --- lib/pleroma/user.ex | 5 ----- lib/pleroma/web/activity_pub/activity_pub.ex | 13 +++++++++++++ lib/pleroma/web/mastodon_api/views/account_view.ex | 4 ++-- lib/pleroma/web/metadata/opengraph.ex | 2 +- .../web/templates/static_fe/static_fe/_user_card.html.eex | 2 +- .../web/templates/static_fe/static_fe/profile.html.eex | 2 +- 6 files changed, 18 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ca0bfca11..ff828aa17 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -305,11 +305,6 @@ defmodule Pleroma.User do end end - def profile_url(%User{uri: url}) when url != nil, do: url - def profile_url(%User{source_data: %{"url" => url}}) when is_binary(url), do: url - def profile_url(%User{ap_id: ap_id}), do: ap_id - def profile_url(_), do: nil - def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}" def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9c0f5d771..53b6ad654 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1379,6 +1379,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + @spec get_actor_url(any()) :: binary() | nil + defp get_actor_url(url) when is_binary(url), do: url + defp get_actor_url(%{"href" => href}) when is_binary(href), do: href + + defp get_actor_url(url) when is_list(url) do + url + |> List.first() + |> get_actor_url() + end + + defp get_actor_url(_url), do: nil + defp object_to_user_data(data) do avatar = data["icon"]["url"] && @@ -1408,6 +1420,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do user_data = %{ ap_id: data["id"], + uri: get_actor_url(data["url"]), ap_enabled: true, source_data: data, banner: banner, diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 0efcabc01..c482bba64 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -43,7 +43,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do id: to_string(user.id), acct: user.nickname, username: username_from_nickname(user.nickname), - url: User.profile_url(user) + url: user.uri || user.ap_id } end @@ -207,7 +207,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do following_count: following_count, statuses_count: user.note_count, note: user.bio || "", - url: User.profile_url(user), + url: user.uri || user.ap_id, avatar: image, avatar_static: image, header: header, diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 21446ac77..68c871e71 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -68,7 +68,7 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do property: "og:title", content: Utils.user_name_string(user) ], []}, - {:meta, [property: "og:url", content: User.profile_url(user)], []}, + {:meta, [property: "og:url", content: user.uri || user.ap_id], []}, {:meta, [property: "og:description", content: truncated_bio], []}, {:meta, [property: "og:type", content: "website"], []}, {:meta, [property: "og:image", content: Utils.attachment_url(User.avatar_url(user))], []}, diff --git a/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex index c7789f9ac..2a7582d45 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex @@ -1,5 +1,5 @@
- +
diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex index 94063c92d..e7d2aecad 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex @@ -8,7 +8,7 @@ <%= raw Formatter.emojify(@user.name, emoji_for_user(@user)) %> | - <%= link "@#{@user.nickname}@#{Endpoint.host()}", to: User.profile_url(@user) %> + <%= link "@#{@user.nickname}@#{Endpoint.host()}", to: (@user.uri || @user.ap_id) %>

<%= raw @user.bio %>

-- cgit v1.2.3 From 94ddbe4098e167f9537d168261a6cc76fa17508b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 1 Apr 2020 09:55:05 +0300 Subject: restrict remote users from indexing --- lib/pleroma/web/metadata.ex | 7 ++++++- lib/pleroma/web/metadata/restrict_indexing.ex | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/web/metadata/restrict_indexing.ex (limited to 'lib') diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index c9aac27dc..a9f70c43e 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -6,7 +6,12 @@ defmodule Pleroma.Web.Metadata do alias Phoenix.HTML def build_tags(params) do - Enum.reduce(Pleroma.Config.get([__MODULE__, :providers], []), "", fn parser, acc -> + providers = [ + Pleroma.Web.Metadata.Providers.RestrictIndexing + | Pleroma.Config.get([__MODULE__, :providers], []) + ] + + Enum.reduce(providers, "", fn parser, acc -> rendered_html = params |> parser.build_tags() diff --git a/lib/pleroma/web/metadata/restrict_indexing.ex b/lib/pleroma/web/metadata/restrict_indexing.ex new file mode 100644 index 000000000..f15607896 --- /dev/null +++ b/lib/pleroma/web/metadata/restrict_indexing.ex @@ -0,0 +1,25 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do + @behaviour Pleroma.Web.Metadata.Providers.Provider + + @moduledoc """ + Restricts indexing of remote users. + """ + + @impl true + def build_tags(%{user: %{local: false}}) do + [ + {:meta, + [ + name: "robots", + content: "noindex, noarchive" + ], []} + ] + end + + @impl true + def build_tags(%{user: %{local: true}}), do: [] +end -- cgit v1.2.3 From aa78325117c879ecb7ec76383c239078275adbd9 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 2 Apr 2020 19:23:30 +0300 Subject: [#2323] Fixed a typo causing /accounts/relationships to render default relationships. Improved the tests. --- lib/pleroma/web/mastodon_api/views/account_view.ex | 8 +++++--- lib/pleroma/web/mastodon_api/views/notification_view.ex | 2 +- lib/pleroma/web/mastodon_api/views/status_view.ex | 10 ++++++---- 3 files changed, 12 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index c482bba64..99e62f580 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -13,16 +13,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do alias Pleroma.Web.MediaProxy def render("index.json", %{users: users} = opts) do + reading_user = opts[:for] + relationships_opt = cond do Map.has_key?(opts, :relationships) -> opts[:relationships] - is_nil(opts[:for]) -> + is_nil(reading_user) -> UserRelationship.view_relationships_option(nil, []) true -> - UserRelationship.view_relationships_option(opts[:for], users) + UserRelationship.view_relationships_option(reading_user, users) end opts = Map.put(opts, :relationships, relationships_opt) @@ -143,7 +145,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do Map.has_key?(opts, :relationships) -> opts[:relationships] - is_nil(opts[:for]) -> + is_nil(user) -> UserRelationship.view_relationships_option(nil, []) true -> diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 89f5734ff..ae87d4701 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -36,7 +36,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do Map.has_key?(opts, :relationships) -> opts[:relationships] - is_nil(opts[:for]) -> + is_nil(reading_user) -> UserRelationship.view_relationships_option(nil, []) true -> diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 82326986c..cea76e735 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -72,6 +72,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end def render("index.json", opts) do + reading_user = opts[:for] + # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list activities = Enum.filter(opts.activities, & &1) replied_to_activities = get_replied_to_activities(activities) @@ -82,8 +84,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Enum.map(&Object.normalize(&1).data["id"]) |> Activity.create_by_object_ap_id() |> Activity.with_preloaded_object(:left) - |> Activity.with_preloaded_bookmark(opts[:for]) - |> Activity.with_set_thread_muted_field(opts[:for]) + |> Activity.with_preloaded_bookmark(reading_user) + |> Activity.with_set_thread_muted_field(reading_user) |> Repo.all() relationships_opt = @@ -91,13 +93,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do Map.has_key?(opts, :relationships) -> opts[:relationships] - is_nil(opts[:for]) -> + is_nil(reading_user) -> UserRelationship.view_relationships_option(nil, []) true -> actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) - UserRelationship.view_relationships_option(opts[:for], actors) + UserRelationship.view_relationships_option(reading_user, actors) end opts = -- cgit v1.2.3 From b59ac37b2c09d5dc80b59bd3a2aea36989bee713 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 6 Apr 2020 10:45:25 +0300 Subject: tests for emoji mix task --- lib/mix/tasks/pleroma/emoji.ex | 80 ++++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 429d763c7..cdffa88b2 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -14,8 +14,8 @@ defmodule Mix.Tasks.Pleroma.Emoji do {options, [], []} = parse_global_opts(args) - manifest = - fetch_manifest(if options[:manifest], do: options[:manifest], else: default_manifest()) + url_or_path = options[:manifest] || default_manifest() + manifest = fetch_manifest(url_or_path) Enum.each(manifest, fn {name, info} -> to_print = [ @@ -40,9 +40,9 @@ defmodule Mix.Tasks.Pleroma.Emoji do {options, pack_names, []} = parse_global_opts(args) - manifest_url = if options[:manifest], do: options[:manifest], else: default_manifest() + url_or_path = options[:manifest] || default_manifest() - manifest = fetch_manifest(manifest_url) + manifest = fetch_manifest(url_or_path) for pack_name <- pack_names do if Map.has_key?(manifest, pack_name) do @@ -75,7 +75,10 @@ defmodule Mix.Tasks.Pleroma.Emoji do end # The url specified in files should be in the same directory - files_url = Path.join(Path.dirname(manifest_url), pack["files"]) + files_url = + url_or_path + |> Path.dirname() + |> Path.join(pack["files"]) IO.puts( IO.ANSI.format([ @@ -133,38 +136,51 @@ defmodule Mix.Tasks.Pleroma.Emoji do end end - def run(["gen-pack", src]) do + def run(["gen-pack" | args]) do start_pleroma() - proposed_name = Path.basename(src) |> Path.rootname() - name = String.trim(IO.gets("Pack name [#{proposed_name}]: ")) - # If there's no name, use the default one - name = if String.length(name) > 0, do: name, else: proposed_name + {opts, [src], []} = + OptionParser.parse( + args, + strict: [ + name: :string, + license: :string, + homepage: :string, + description: :string, + files: :string, + extensions: :string + ] + ) - license = String.trim(IO.gets("License: ")) - homepage = String.trim(IO.gets("Homepage: ")) - description = String.trim(IO.gets("Description: ")) + proposed_name = Path.basename(src) |> Path.rootname() + name = get_option(opts, :name, "Pack name:", proposed_name) + license = get_option(opts, :license, "License:") + homepage = get_option(opts, :homepage, "Homepage:") + description = get_option(opts, :description, "Description:") - proposed_files_name = "#{name}.json" - files_name = String.trim(IO.gets("Save file list to [#{proposed_files_name}]: ")) - files_name = if String.length(files_name) > 0, do: files_name, else: proposed_files_name + proposed_files_name = "#{name}_files.json" + files_name = get_option(opts, :files, "Save file list to:", proposed_files_name) default_exts = [".png", ".gif"] - default_exts_str = Enum.join(default_exts, " ") - exts = - String.trim( - IO.gets("Emoji file extensions (separated with spaces) [#{default_exts_str}]: ") + custom_exts = + get_option( + opts, + :extensions, + "Emoji file extensions (separated with spaces):", + Enum.join(default_exts, " ") ) + |> String.split(" ", trim: true) exts = - if String.length(exts) > 0 do - String.split(exts, " ") - |> Enum.filter(fn e -> e |> String.trim() |> String.length() > 0 end) - else + if MapSet.equal?(MapSet.new(default_exts), MapSet.new(custom_exts)) do default_exts + else + custom_exts end + IO.puts("Using #{Enum.join(exts, " ")} extensions") + IO.puts("Downloading the pack and generating SHA256") binary_archive = Tesla.get!(client(), src).body @@ -194,14 +210,16 @@ defmodule Mix.Tasks.Pleroma.Emoji do IO.puts(""" #{files_name} has been created and contains the list of all found emojis in the pack. - Please review the files in the remove those not needed. + Please review the files in the pack and remove those not needed. """) - if File.exists?("index.json") do - existing_data = File.read!("index.json") |> Jason.decode!() + pack_file = "#{name}.json" + + if File.exists?(pack_file) do + existing_data = File.read!(pack_file) |> Jason.decode!() File.write!( - "index.json", + pack_file, Jason.encode!( Map.merge( existing_data, @@ -211,11 +229,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do ) ) - IO.puts("index.json file has been update with the #{name} pack") + IO.puts("#{pack_file} has been updated with the #{name} pack") else - File.write!("index.json", Jason.encode!(pack_json, pretty: true)) + File.write!(pack_file, Jason.encode!(pack_json, pretty: true)) - IO.puts("index.json has been created with the #{name} pack") + IO.puts("#{pack_file} has been created with the #{name} pack") end end -- cgit v1.2.3 From e67cde0ed6b55450b5f309f9ed86f7f8e2a1e73f Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 6 Apr 2020 13:46:34 +0200 Subject: Transmogrifier: Refactoring / Renaming. --- lib/pleroma/web/activity_pub/transmogrifier.ex | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index a4b385cd5..455f51fe0 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -617,9 +617,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do data |> LikeValidator.cast_data() |> Ecto.Changeset.apply_action(:insert)}, cast_data = ObjectValidator.stringify_keys(Map.from_struct(cast_data_sym)), :ok <- ObjectValidator.fetch_actor_and_object(cast_data), - {_, {:ok, cast_data}} <- {:maybe_add_context, maybe_add_context_from_object(cast_data)}, + {_, {:ok, cast_data}} <- {:ensure_context_presence, ensure_context_presence(cast_data)}, {_, {:ok, cast_data}} <- - {:maybe_add_recipients, maybe_add_recipients_from_object(cast_data)}, + {:ensure_recipients_presence, ensure_recipients_presence(cast_data)}, {_, {:ok, activity, _meta}} <- {:common_pipeline, Pipeline.common_pipeline(cast_data, local: false)} do {:ok, activity} @@ -1251,10 +1251,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def maybe_fix_user_object(data), do: maybe_fix_user_url(data) - defp maybe_add_context_from_object(%{"context" => context} = data) when is_binary(context), + defp ensure_context_presence(%{"context" => context} = data) when is_binary(context), do: {:ok, data} - defp maybe_add_context_from_object(%{"object" => object} = data) when is_binary(object) do + defp ensure_context_presence(%{"object" => object} = data) when is_binary(object) do with %{data: %{"context" => context}} when is_binary(context) <- Object.normalize(object) do {:ok, Map.put(data, "context", context)} else @@ -1263,14 +1263,14 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - defp maybe_add_context_from_object(_) do + defp ensure_context_presence(_) do {:error, :no_context} end - defp maybe_add_recipients_from_object(%{"to" => [_ | _], "cc" => [_ | _]} = data), + defp ensure_recipients_presence(%{"to" => [_ | _], "cc" => [_ | _]} = data), do: {:ok, data} - defp maybe_add_recipients_from_object(%{"object" => object} = data) do + defp ensure_recipients_presence(%{"object" => object} = data) do case Object.normalize(object) do %{data: %{"actor" => actor}} -> data = @@ -1288,7 +1288,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - defp maybe_add_recipients_from_object(_) do + defp ensure_recipients_presence(_) do {:error, :no_object} end end -- cgit v1.2.3 From 772bc258cde11b3203ad9420f69321ccd56db91a Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 6 Apr 2020 13:53:24 +0200 Subject: ObjectID Validator: Refactor. --- .../activity_pub/object_validators/types/object_id.ex | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex index 8e70effe4..ee10be0b0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex +++ b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex @@ -4,14 +4,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do def type, do: :string def cast(object) when is_binary(object) do - with %URI{ - scheme: scheme, - host: host - } - when scheme in ["https", "http"] and not is_nil(host) <- - URI.parse(object) do - {:ok, object} - else + # Host has to be present and scheme has to be an http scheme (for now) + case URI.parse(object) do + %URI{host: nil} -> + :error + + %URI{scheme: scheme} when scheme in ["https", "http"] -> + {:ok, object} + _ -> :error end -- cgit v1.2.3