From 6a3b1a526eb1a3ea49aca7914ed7ba8736c52059 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 15 Aug 2019 15:34:41 -0500 Subject: max_body_size -> max_body_length, as it should be --- lib/pleroma/reverse_proxy/reverse_proxy.ex | 4 ++-- test/reverse_proxy_test.exs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 1f98f215c..7518e4c3c 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -109,7 +109,7 @@ defmodule Pleroma.ReverseProxy do end with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), - :ok <- header_length_constraint(headers, Keyword.get(opts, :max_body_length)) do + :ok <- header_length_constraint(headers, Keyword.get(opts, :max_body_length, @max_body_length)) do response(conn, client, url, code, headers, opts) else {:ok, code, headers} -> @@ -200,7 +200,7 @@ defmodule Pleroma.ReverseProxy do {:ok, data} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), - :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)), + :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_length, @max_body_length)), {:ok, conn} <- chunk(conn, data) do chunk_reply(conn, client, opts, sent_so_far, duration) else diff --git a/test/reverse_proxy_test.exs b/test/reverse_proxy_test.exs index f4b7d6add..3a83c4c48 100644 --- a/test/reverse_proxy_test.exs +++ b/test/reverse_proxy_test.exs @@ -108,11 +108,11 @@ defmodule Pleroma.ReverseProxyTest do end end - test "max_body_size returns error if streaming body more than that option", %{conn: conn} do + test "max_body_length returns error if streaming body more than that option", %{conn: conn} do stream_mock(3, true) assert capture_log(fn -> - ReverseProxy.call(conn, "/stream-bytes/50", max_body_size: 30) + ReverseProxy.call(conn, "/stream-bytes/50", max_body_length: 30) end) =~ "[warn] Elixir.Pleroma.ReverseProxy request to /stream-bytes/50 failed while reading/chunking: :body_too_large" end -- cgit v1.2.3 From 3315a2a1c3cae3375bbb1c7112b2c75563f3a4af Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 16 Aug 2019 15:58:42 +0300 Subject: fixed User.unfollow with synchronization external user --- lib/pleroma/user.ex | 4 +++- test/user_test.exs | 64 +++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index a1040fe71..23748ef26 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -742,6 +742,7 @@ defmodule Pleroma.User do |> update_and_set_cache() end + @spec maybe_fetch_follow_information(User.t()) :: User.t() def maybe_fetch_follow_information(user) do with {:ok, user} <- fetch_follow_information(user) do user @@ -799,9 +800,10 @@ defmodule Pleroma.User do end end + @spec maybe_update_following_count(User.t()) :: User.t() def maybe_update_following_count(%User{local: false} = user) do if Pleroma.Config.get([:instance, :external_user_synchronization]) do - {:ok, maybe_fetch_follow_information(user)} + maybe_fetch_follow_information(user) else user end diff --git a/test/user_test.exs b/test/user_test.exs index b363b322c..3861cadac 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -192,24 +192,64 @@ defmodule Pleroma.UserTest do # assert websub # end - test "unfollow takes a user and another user" do - followed = insert(:user) - user = insert(:user, %{following: [User.ap_followers(followed)]}) + describe "unfollow/2" do + setup do + setting = Pleroma.Config.get([:instance, :external_user_synchronization]) - {:ok, user, _activity} = User.unfollow(user, followed) + on_exit(fn -> + Pleroma.Config.put([:instance, :external_user_synchronization], setting) + end) - user = User.get_cached_by_id(user.id) + :ok + end - assert user.following == [] - end + test "unfollow with syncronizes external user" do + Pleroma.Config.put([:instance, :external_user_synchronization], true) - test "unfollow doesn't unfollow yourself" do - user = insert(:user) + followed = + insert(:user, + nickname: "fuser1", + follower_address: "http://localhost:4001/users/fuser1/followers", + following_address: "http://localhost:4001/users/fuser1/following", + ap_id: "http://localhost:4001/users/fuser1" + ) + + user = + insert(:user, %{ + local: false, + nickname: "fuser2", + ap_id: "http://localhost:4001/users/fuser2", + follower_address: "http://localhost:4001/users/fuser2/followers", + following_address: "http://localhost:4001/users/fuser2/following", + following: [User.ap_followers(followed)] + }) - {:error, _} = User.unfollow(user, user) + {:ok, user, _activity} = User.unfollow(user, followed) - user = User.get_cached_by_id(user.id) - assert user.following == [user.ap_id] + user = User.get_cached_by_id(user.id) + + assert user.following == [] + end + + test "unfollow takes a user and another user" do + followed = insert(:user) + user = insert(:user, %{following: [User.ap_followers(followed)]}) + + {:ok, user, _activity} = User.unfollow(user, followed) + + user = User.get_cached_by_id(user.id) + + assert user.following == [] + end + + test "unfollow doesn't unfollow yourself" do + user = insert(:user) + + {:error, _} = User.unfollow(user, user) + + user = User.get_cached_by_id(user.id) + assert user.following == [user.ap_id] + end end test "test if a user is following another user" do -- cgit v1.2.3 From ef82f868d9c435d6a9498aea0271e032d3c6c1a3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 16 Aug 2019 10:00:18 -0500 Subject: Formatting --- lib/pleroma/reverse_proxy/reverse_proxy.ex | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 7518e4c3c..03efad30a 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -109,7 +109,11 @@ defmodule Pleroma.ReverseProxy do end with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), - :ok <- header_length_constraint(headers, Keyword.get(opts, :max_body_length, @max_body_length)) do + :ok <- + header_length_constraint( + headers, + Keyword.get(opts, :max_body_length, @max_body_length) + ) do response(conn, client, url, code, headers, opts) else {:ok, code, headers} -> @@ -200,7 +204,11 @@ defmodule Pleroma.ReverseProxy do {:ok, data} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), - :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_length, @max_body_length)), + :ok <- + body_size_constraint( + sent_so_far, + Keyword.get(opts, :max_body_length, @max_body_length) + ), {:ok, conn} <- chunk(conn, data) do chunk_reply(conn, client, opts, sent_so_far, duration) else -- cgit v1.2.3 From cb222b72b39ae4dc887657d1eae03d0360cbd429 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 16 Aug 2019 10:28:07 -0500 Subject: Add changelog entry too --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 358287096..d47e98cd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Pleroma.Upload base_url was not automatically whitelisted by MediaProxy. Now your custom CDN or file hosting will be accessed directly as expected. - Report email not being sent to admins when the reporter is a remote user - MRF: ensure that subdomain_match calls are case-insensitive +- Reverse Proxy limiting `max_body_length` was incorrectly defined and only checked `Content-Length` headers which may not be sufficient in some circumstances ### Added - **Breaking:** MRF describe API, which adds support for exposing configuration information about MRF policies to NodeInfo. -- cgit v1.2.3 From e652cef76b99588867315347e8dcc3d323d8a161 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 19 Aug 2019 13:31:56 +0300 Subject: removes duplicates from relay subscription list --- lib/mix/tasks/pleroma/relay.ex | 12 +++++------- test/tasks/relay_test.exs | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex index c7324fff6..a738fae75 100644 --- a/lib/mix/tasks/pleroma/relay.ex +++ b/lib/mix/tasks/pleroma/relay.ex @@ -53,13 +53,11 @@ defmodule Mix.Tasks.Pleroma.Relay do def run(["list"]) do start_pleroma() - with %User{} = user <- Relay.get_actor() do - user.following - |> Enum.each(fn entry -> - URI.parse(entry) - |> Map.get(:host) - |> shell_info() - end) + with %User{following: following} = _user <- Relay.get_actor() do + following + |> Enum.map(fn entry -> URI.parse(entry).host end) + |> Enum.uniq() + |> Enum.each(&shell_info(&1)) else e -> shell_error("Error while fetching relay subscription list: #{inspect(e)}") end diff --git a/test/tasks/relay_test.exs b/test/tasks/relay_test.exs index 9d260da3e..0d341c8d6 100644 --- a/test/tasks/relay_test.exs +++ b/test/tasks/relay_test.exs @@ -69,4 +69,27 @@ defmodule Mix.Tasks.Pleroma.RelayTest do assert undo_activity.data["object"] == cancelled_activity.data end end + + describe "mix pleroma.relay list" do + test "Prints relay subscription list" do + :ok = Mix.Tasks.Pleroma.Relay.run(["list"]) + + refute_receive {:mix_shell, :info, _} + + Pleroma.Web.ActivityPub.Relay.get_actor() + |> Ecto.Changeset.change( + following: [ + "http://test-app.com/user/test1", + "http://test-app.com/user/test1", + "http://test-app-42.com/user/test1" + ] + ) + |> Pleroma.User.update_and_set_cache() + + :ok = Mix.Tasks.Pleroma.Relay.run(["list"]) + + assert_receive {:mix_shell, :info, ["test-app.com"]} + assert_receive {:mix_shell, :info, ["test-app-42.com"]} + end + end end -- cgit v1.2.3 From a320358703db249ab20df5afd81c92fb42b8cadb Mon Sep 17 00:00:00 2001 From: Maksim Date: Mon, 19 Aug 2019 15:34:29 +0000 Subject: added test helpers to clear config after tests --- test/config/transfer_task_test.exs | 8 +--- test/conversation_test.exs | 10 +--- test/emails/mailer_test.exs | 6 +-- test/http/request_builder_test.exs | 8 ++-- test/object/fetcher_test.exs | 8 +--- .../ensure_public_or_authenticated_plug_test.exs | 17 ++----- test/plugs/http_security_plug_test.exs | 17 ++----- test/plugs/instance_static_test.exs | 10 ++-- test/support/helpers.ex | 53 ++++++++++++++++++++ test/tasks/config_test.exs | 9 ++-- test/tasks/robots_txt_test.exs | 8 ++-- test/upload/filter/anonymize_filename_test.exs | 8 +--- test/upload/filter/mogrify_test.exs | 8 +--- test/upload/filter_test.exs | 8 +--- test/upload_test.exs | 6 +-- test/uploaders/s3_test.exs | 10 +--- test/user_test.exs | 40 +++++----------- .../activity_pub/activity_pub_controller_test.exs | 15 +++--- test/web/activity_pub/mrf/mrf_test.exs | 6 +-- .../activity_pub/mrf/reject_non_public_test.exs | 7 +-- test/web/activity_pub/mrf/simple_policy_test.exs | 8 +--- .../mrf/user_allowlist_policy_test.exs | 7 +-- .../activity_pub/mrf/vocabulary_policy_test.exs | 25 ++-------- test/web/admin_api/admin_api_controller_test.exs | 56 +++++++--------------- test/web/common_api/common_api_test.exs | 9 ++-- test/web/federator_test.exs | 27 +++++------ test/web/instances/instance_test.exs | 10 +--- test/web/instances/instances_test.exs | 10 +--- .../mastodon_api/mastodon_api_controller_test.exs | 49 +++++++------------ test/web/media_proxy/media_proxy_test.exs | 7 +-- test/web/oauth/ldap_authorization_test.exs | 17 ++----- test/web/oauth/oauth_controller_test.exs | 36 +++++--------- test/web/ostatus/ostatus_controller_test.exs | 11 ++--- test/web/plugs/federating_plug_test.exs | 10 +--- test/web/rich_media/helpers_test.exs | 4 +- test/web/streamer_test.exs | 10 +--- .../twitter_api/twitter_api_controller_test.exs | 21 +++----- test/web/twitter_api/util_controller_test.exs | 28 ++--------- test/web/web_finger/web_finger_controller_test.exs | 10 ++-- test/web/websub/websub_controller_test.exs | 10 +--- 40 files changed, 210 insertions(+), 417 deletions(-) diff --git a/test/config/transfer_task_test.exs b/test/config/transfer_task_test.exs index 4455a4d47..9074f3b97 100644 --- a/test/config/transfer_task_test.exs +++ b/test/config/transfer_task_test.exs @@ -5,14 +5,8 @@ defmodule Pleroma.Config.TransferTaskTest do use Pleroma.DataCase - setup do - dynamic = Pleroma.Config.get([:instance, :dynamic_configuration]) - + clear_config([:instance, :dynamic_configuration]) do Pleroma.Config.put([:instance, :dynamic_configuration], true) - - on_exit(fn -> - Pleroma.Config.put([:instance, :dynamic_configuration], dynamic) - end) end test "transfer config values from db to env" do diff --git a/test/conversation_test.exs b/test/conversation_test.exs index aa193e0d4..4e36494f8 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -11,14 +11,8 @@ defmodule Pleroma.ConversationTest do import Pleroma.Factory - setup_all do - config_path = [:instance, :federating] - initial_setting = Pleroma.Config.get(config_path) - - Pleroma.Config.put(config_path, true) - on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) - - :ok + clear_config_all([:instance, :federating]) do + Pleroma.Config.put([:instance, :federating], true) end test "it goes through old direct conversations" do diff --git a/test/emails/mailer_test.exs b/test/emails/mailer_test.exs index 450bb09c7..ae5effb7a 100644 --- a/test/emails/mailer_test.exs +++ b/test/emails/mailer_test.exs @@ -15,11 +15,7 @@ defmodule Pleroma.Emails.MailerTest do to: [{"Test User", "user1@example.com"}] } - setup do - value = Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) - on_exit(fn -> Pleroma.Config.put([Pleroma.Emails.Mailer, :enabled], value) end) - :ok - end + clear_config([Pleroma.Emails.Mailer, :enabled]) test "not send email when mailer is disabled" do Pleroma.Config.put([Pleroma.Emails.Mailer, :enabled], false) diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index 7febe84c5..170ca916f 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -4,21 +4,19 @@ defmodule Pleroma.HTTP.RequestBuilderTest do use ExUnit.Case, async: true + use Pleroma.Tests.Helpers alias Pleroma.HTTP.RequestBuilder describe "headers/2" do + clear_config([:http, :send_user_agent]) + test "don't send pleroma user agent" do assert RequestBuilder.headers(%{}, []) == %{headers: []} end test "send pleroma user agent" do - send = Pleroma.Config.get([:http, :send_user_agent]) Pleroma.Config.put([:http, :send_user_agent], true) - on_exit(fn -> - Pleroma.Config.put([:http, :send_user_agent], send) - end) - assert RequestBuilder.headers(%{}, []) == %{ headers: [{"User-Agent", Pleroma.Application.user_agent()}] } diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 0ca87f035..895a73d2c 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -159,32 +159,28 @@ defmodule Pleroma.Object.FetcherTest do end describe "signed fetches" do + clear_config([:activitypub, :sign_object_fetches]) + test_with_mock "it signs fetches when configured to do so", Pleroma.Signature, [:passthrough], [] do - option = Pleroma.Config.get([:activitypub, :sign_object_fetches]) Pleroma.Config.put([:activitypub, :sign_object_fetches], true) Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") assert called(Pleroma.Signature.sign(:_, :_)) - - Pleroma.Config.put([:activitypub, :sign_object_fetches], option) end test_with_mock "it doesn't sign fetches when not configured to do so", Pleroma.Signature, [:passthrough], [] do - option = Pleroma.Config.get([:activitypub, :sign_object_fetches]) Pleroma.Config.put([:activitypub, :sign_object_fetches], false) Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") refute called(Pleroma.Signature.sign(:_, :_)) - - Pleroma.Config.put([:activitypub, :sign_object_fetches], option) end end end diff --git a/test/plugs/ensure_public_or_authenticated_plug_test.exs b/test/plugs/ensure_public_or_authenticated_plug_test.exs index ce5d77ff7..d45662a2a 100644 --- a/test/plugs/ensure_public_or_authenticated_plug_test.exs +++ b/test/plugs/ensure_public_or_authenticated_plug_test.exs @@ -9,8 +9,10 @@ defmodule Pleroma.Plugs.EnsurePublicOrAuthenticatedPlugTest do alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug alias Pleroma.User + clear_config([:instance, :public]) + test "it halts if not public and no user is assigned", %{conn: conn} do - set_public_to(false) + Config.put([:instance, :public], false) conn = conn @@ -21,7 +23,7 @@ defmodule Pleroma.Plugs.EnsurePublicOrAuthenticatedPlugTest do end test "it continues if public", %{conn: conn} do - set_public_to(true) + Config.put([:instance, :public], true) ret_conn = conn @@ -31,7 +33,7 @@ defmodule Pleroma.Plugs.EnsurePublicOrAuthenticatedPlugTest do end test "it continues if a user is assigned, even if not public", %{conn: conn} do - set_public_to(false) + Config.put([:instance, :public], false) conn = conn @@ -43,13 +45,4 @@ defmodule Pleroma.Plugs.EnsurePublicOrAuthenticatedPlugTest do assert ret_conn == conn end - - defp set_public_to(value) do - orig = Config.get!([:instance, :public]) - Config.put([:instance, :public], value) - - on_exit(fn -> - Config.put([:instance, :public], orig) - end) - end end diff --git a/test/plugs/http_security_plug_test.exs b/test/plugs/http_security_plug_test.exs index 7dfd50c1f..7a2835e3d 100644 --- a/test/plugs/http_security_plug_test.exs +++ b/test/plugs/http_security_plug_test.exs @@ -7,17 +7,12 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do alias Pleroma.Config alias Plug.Conn + clear_config([:http_securiy, :enabled]) + clear_config([:http_security, :sts]) + describe "http security enabled" do setup do - enabled = Config.get([:http_securiy, :enabled]) - Config.put([:http_security, :enabled], true) - - on_exit(fn -> - Config.put([:http_security, :enabled], enabled) - end) - - :ok end test "it sends CSP headers when enabled", %{conn: conn} do @@ -81,14 +76,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do end test "it does not send CSP headers when disabled", %{conn: conn} do - enabled = Config.get([:http_securiy, :enabled]) - Config.put([:http_security, :enabled], false) - on_exit(fn -> - Config.put([:http_security, :enabled], enabled) - end) - conn = get(conn, "/api/v1/instance") assert Conn.get_resp_header(conn, "x-xss-protection") == [] diff --git a/test/plugs/instance_static_test.exs b/test/plugs/instance_static_test.exs index e2dcfa3d8..6aabc45a4 100644 --- a/test/plugs/instance_static_test.exs +++ b/test/plugs/instance_static_test.exs @@ -8,14 +8,12 @@ defmodule Pleroma.Web.RuntimeStaticPlugTest do @dir "test/tmp/instance_static" setup do - static_dir = Pleroma.Config.get([:instance, :static_dir]) - Pleroma.Config.put([:instance, :static_dir], @dir) File.mkdir_p!(@dir) + on_exit(fn -> File.rm_rf(@dir) end) + end - on_exit(fn -> - Pleroma.Config.put([:instance, :static_dir], static_dir) - File.rm_rf(@dir) - end) + clear_config([:instance, :static_dir]) do + Pleroma.Config.put([:instance, :static_dir], @dir) end test "overrides index" do diff --git a/test/support/helpers.ex b/test/support/helpers.ex index 1a92be065..a601b3ec8 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -7,8 +7,52 @@ defmodule Pleroma.Tests.Helpers do Helpers for use in tests. """ + defmacro clear_config(config_path) do + quote do + clear_config(unquote(config_path)) do + end + end + end + + defmacro clear_config(config_path, do: yield) do + quote do + setup do + initial_setting = Pleroma.Config.get(unquote(config_path)) + unquote(yield) + on_exit(fn -> Pleroma.Config.put(unquote(config_path), initial_setting) end) + :ok + end + end + end + + defmacro clear_config_all(config_path) do + quote do + clear_config_all(unquote(config_path)) do + end + end + end + + defmacro clear_config_all(config_path, do: yield) do + quote do + setup_all do + initial_setting = Pleroma.Config.get(unquote(config_path)) + unquote(yield) + on_exit(fn -> Pleroma.Config.put(unquote(config_path), initial_setting) end) + :ok + end + end + end + defmacro __using__(_opts) do quote do + import Pleroma.Tests.Helpers, + only: [ + clear_config: 1, + clear_config: 2, + clear_config_all: 1, + clear_config_all: 2 + ] + def collect_ids(collection) do collection |> Enum.map(& &1.id) @@ -30,6 +74,15 @@ defmodule Pleroma.Tests.Helpers do |> Poison.encode!() |> Poison.decode!() end + + defmacro guards_config(config_path) do + quote do + initial_setting = Pleroma.Config.get(config_path) + + Pleroma.Config.put(config_path, true) + on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) + end + end end end end diff --git a/test/tasks/config_test.exs b/test/tasks/config_test.exs index a9b79eb5b..9cd47380c 100644 --- a/test/tasks/config_test.exs +++ b/test/tasks/config_test.exs @@ -11,21 +11,20 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do Mix.shell(Mix.Shell.Process) temp_file = "config/temp.exported_from_db.secret.exs" - dynamic = Pleroma.Config.get([:instance, :dynamic_configuration]) - - Pleroma.Config.put([:instance, :dynamic_configuration], true) - on_exit(fn -> Mix.shell(Mix.Shell.IO) Application.delete_env(:pleroma, :first_setting) Application.delete_env(:pleroma, :second_setting) - Pleroma.Config.put([:instance, :dynamic_configuration], dynamic) :ok = File.rm(temp_file) end) {:ok, temp_file: temp_file} end + clear_config_all([:instance, :dynamic_configuration]) do + Pleroma.Config.put([:instance, :dynamic_configuration], true) + end + test "settings are migrated to db" do assert Repo.all(Config) == [] diff --git a/test/tasks/robots_txt_test.exs b/test/tasks/robots_txt_test.exs index 78a3f17b4..917df2675 100644 --- a/test/tasks/robots_txt_test.exs +++ b/test/tasks/robots_txt_test.exs @@ -4,17 +4,17 @@ defmodule Mix.Tasks.Pleroma.RobotsTxtTest do use ExUnit.Case + use Pleroma.Tests.Helpers alias Mix.Tasks.Pleroma.RobotsTxt + clear_config([:instance, :static_dir]) + test "creates new dir" do path = "test/fixtures/new_dir/" file_path = path <> "robots.txt" - - static_dir = Pleroma.Config.get([:instance, :static_dir]) Pleroma.Config.put([:instance, :static_dir], path) on_exit(fn -> - Pleroma.Config.put([:instance, :static_dir], static_dir) {:ok, ["test/fixtures/new_dir/", "test/fixtures/new_dir/robots.txt"]} = File.rm_rf(path) end) @@ -29,11 +29,9 @@ defmodule Mix.Tasks.Pleroma.RobotsTxtTest do test "to existance folder" do path = "test/fixtures/" file_path = path <> "robots.txt" - static_dir = Pleroma.Config.get([:instance, :static_dir]) Pleroma.Config.put([:instance, :static_dir], path) on_exit(fn -> - Pleroma.Config.put([:instance, :static_dir], static_dir) :ok = File.rm(file_path) end) diff --git a/test/upload/filter/anonymize_filename_test.exs b/test/upload/filter/anonymize_filename_test.exs index a31b38ab1..6b33e7395 100644 --- a/test/upload/filter/anonymize_filename_test.exs +++ b/test/upload/filter/anonymize_filename_test.exs @@ -9,12 +9,6 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilenameTest do alias Pleroma.Upload setup do - custom_filename = Config.get([Upload.Filter.AnonymizeFilename, :text]) - - on_exit(fn -> - Config.put([Upload.Filter.AnonymizeFilename, :text], custom_filename) - end) - upload_file = %Upload{ name: "an… image.jpg", content_type: "image/jpg", @@ -24,6 +18,8 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilenameTest do %{upload_file: upload_file} end + clear_config([Pleroma.Upload.Filter.AnonymizeFilename, :text]) + test "it replaces filename on pre-defined text", %{upload_file: upload_file} do Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.png") {:ok, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file) diff --git a/test/upload/filter/mogrify_test.exs b/test/upload/filter/mogrify_test.exs index c301440fd..210320d30 100644 --- a/test/upload/filter/mogrify_test.exs +++ b/test/upload/filter/mogrify_test.exs @@ -10,13 +10,7 @@ defmodule Pleroma.Upload.Filter.MogrifyTest do alias Pleroma.Upload alias Pleroma.Upload.Filter - setup do - filter = Config.get([Filter.Mogrify, :args]) - - on_exit(fn -> - Config.put([Filter.Mogrify, :args], filter) - end) - end + clear_config([Filter.Mogrify, :args]) test "apply mogrify filter" do Config.put([Filter.Mogrify, :args], [{"tint", "40"}]) diff --git a/test/upload/filter_test.exs b/test/upload/filter_test.exs index 640cd7107..03887c06a 100644 --- a/test/upload/filter_test.exs +++ b/test/upload/filter_test.exs @@ -8,13 +8,7 @@ defmodule Pleroma.Upload.FilterTest do alias Pleroma.Config alias Pleroma.Upload.Filter - setup do - custom_filename = Config.get([Pleroma.Upload.Filter.AnonymizeFilename, :text]) - - on_exit(fn -> - Config.put([Pleroma.Upload.Filter.AnonymizeFilename, :text], custom_filename) - end) - end + clear_config([Pleroma.Upload.Filter.AnonymizeFilename, :text]) test "applies filters" do Config.put([Pleroma.Upload.Filter.AnonymizeFilename, :text], "custom-file.png") diff --git a/test/upload_test.exs b/test/upload_test.exs index 95b16078b..6721fe82e 100644 --- a/test/upload_test.exs +++ b/test/upload_test.exs @@ -250,12 +250,8 @@ defmodule Pleroma.UploadTest do end describe "Setting a custom base_url for uploaded media" do - setup do + clear_config([Pleroma.Upload, :base_url]) do Pleroma.Config.put([Pleroma.Upload, :base_url], "https://cache.pleroma.social") - - on_exit(fn -> - Pleroma.Config.put([Pleroma.Upload, :base_url], nil) - end) end test "returns a media url with configured base_url" do diff --git a/test/uploaders/s3_test.exs b/test/uploaders/s3_test.exs index a0a1cfdf0..171316340 100644 --- a/test/uploaders/s3_test.exs +++ b/test/uploaders/s3_test.exs @@ -11,19 +11,11 @@ defmodule Pleroma.Uploaders.S3Test do import Mock import ExUnit.CaptureLog - setup do - config = Config.get([Pleroma.Uploaders.S3]) - + clear_config([Pleroma.Uploaders.S3]) do Config.put([Pleroma.Uploaders.S3], bucket: "test_bucket", public_endpoint: "https://s3.amazonaws.com" ) - - on_exit(fn -> - Config.put([Pleroma.Uploaders.S3], config) - end) - - :ok end describe "get_file/1" do diff --git a/test/user_test.exs b/test/user_test.exs index 8cb6567a1..27156f036 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -21,6 +21,8 @@ defmodule Pleroma.UserTest do :ok end + clear_config([:instance, :account_activation_required]) + describe "when tags are nil" do test "tagging a user" do user = insert(:user, %{tags: nil}) @@ -287,6 +289,9 @@ defmodule Pleroma.UserTest do password_confirmation: "test", email: "email@example.com" } + clear_config([:instance, :autofollowed_nicknames]) + clear_config([:instance, :welcome_message]) + clear_config([:instance, :welcome_user_nickname]) test "it autofollows accounts that are set for it" do user = insert(:user) @@ -303,8 +308,6 @@ defmodule Pleroma.UserTest do assert User.following?(registered_user, user) refute User.following?(registered_user, remote_user) - - Pleroma.Config.put([:instance, :autofollowed_nicknames], []) end test "it sends a welcome message if it is set" do @@ -320,9 +323,6 @@ defmodule Pleroma.UserTest do assert registered_user.ap_id in activity.recipients assert Object.normalize(activity).data["content"] =~ "cool site" assert activity.actor == welcome_user.ap_id - - Pleroma.Config.put([:instance, :welcome_user_nickname], nil) - Pleroma.Config.put([:instance, :welcome_message], nil) end test "it requires an email, name, nickname and password, bio is optional" do @@ -388,15 +388,8 @@ defmodule Pleroma.UserTest do email: "email@example.com" } - setup do - setting = Pleroma.Config.get([:instance, :account_activation_required]) - - unless setting do - Pleroma.Config.put([:instance, :account_activation_required], true) - on_exit(fn -> Pleroma.Config.put([:instance, :account_activation_required], setting) end) - end - - :ok + clear_config([:instance, :account_activation_required]) do + Pleroma.Config.put([:instance, :account_activation_required], true) end test "it creates unconfirmed user" do @@ -1043,6 +1036,8 @@ defmodule Pleroma.UserTest do [user: user] end + clear_config([:instance, :federating]) + test ".delete_user_activities deletes all create activities", %{user: user} do {:ok, activity} = CommonAPI.post(user, %{"status" => "2hu"}) @@ -1093,9 +1088,7 @@ defmodule Pleroma.UserTest do Pleroma.Web.ActivityPub.Publisher, [:passthrough], [] do - config_path = [:instance, :federating] - initial_setting = Pleroma.Config.get(config_path) - Pleroma.Config.put(config_path, true) + Pleroma.Config.put([:instance, :federating], true) {:ok, follower} = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/admin") {:ok, _} = User.follow(follower, user) @@ -1107,8 +1100,6 @@ defmodule Pleroma.UserTest do inbox: "http://mastodon.example.org/inbox" }) ) - - Pleroma.Config.put(config_path, initial_setting) end end @@ -1174,8 +1165,6 @@ defmodule Pleroma.UserTest do refute User.auth_active?(local_user) assert User.auth_active?(confirmed_user) assert User.auth_active?(remote_user) - - Pleroma.Config.put([:instance, :account_activation_required], false) end describe "superuser?/1" do @@ -1220,8 +1209,6 @@ defmodule Pleroma.UserTest do other_user = insert(:user, local: true) refute User.visible_for?(user, other_user) - - Pleroma.Config.put([:instance, :account_activation_required], false) end test "returns true when the account is unauthenticated and auth is not required" do @@ -1238,8 +1225,6 @@ defmodule Pleroma.UserTest do other_user = insert(:user, local: true, info: %{is_admin: true}) assert User.visible_for?(user, other_user) - - Pleroma.Config.put([:instance, :account_activation_required], false) end end @@ -1552,10 +1537,7 @@ defmodule Pleroma.UserTest do end describe "following/followers synchronization" do - setup do - sync = Pleroma.Config.get([:instance, :external_user_synchronization]) - on_exit(fn -> Pleroma.Config.put([:instance, :external_user_synchronization], sync) end) - end + clear_config([:instance, :external_user_synchronization]) test "updates the counters normally on following/getting a follow when disabled" do Pleroma.Config.put([:instance, :external_user_synchronization], false) diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 251055ee1..77f5e39fa 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -16,17 +16,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) - - config_path = [:instance, :federating] - initial_setting = Pleroma.Config.get(config_path) - - Pleroma.Config.put(config_path, true) - on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) - :ok end + clear_config_all([:instance, :federating], + do: Pleroma.Config.put([:instance, :federating], true) + ) + describe "/relay" do + clear_config([:instance, :allow_relay]) + test "with the relay active, it returns the relay user", %{conn: conn} do res = conn @@ -43,8 +42,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> get(activity_pub_path(conn, :relay)) |> json_response(404) |> assert - - Pleroma.Config.put([:instance, :allow_relay], true) end end diff --git a/test/web/activity_pub/mrf/mrf_test.exs b/test/web/activity_pub/mrf/mrf_test.exs index 19e172939..04709df17 100644 --- a/test/web/activity_pub/mrf/mrf_test.exs +++ b/test/web/activity_pub/mrf/mrf_test.exs @@ -1,5 +1,6 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do use ExUnit.Case, async: true + use Pleroma.Tests.Helpers alias Pleroma.Web.ActivityPub.MRF test "subdomains_regex/1" do @@ -59,6 +60,8 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do end describe "describe/0" do + clear_config([:instance, :rewrite_policy]) + test "it works as expected with noop policy" do expected = %{ mrf_policies: ["NoOpPolicy"], @@ -69,7 +72,6 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do end test "it works as expected with mock policy" do - config = Pleroma.Config.get([:instance, :rewrite_policy]) Pleroma.Config.put([:instance, :rewrite_policy], [MRFModuleMock]) expected = %{ @@ -79,8 +81,6 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do } {:ok, ^expected} = MRF.describe() - - Pleroma.Config.put([:instance, :rewrite_policy], config) end end end diff --git a/test/web/activity_pub/mrf/reject_non_public_test.exs b/test/web/activity_pub/mrf/reject_non_public_test.exs index fdf6b245e..fc1d190bb 100644 --- a/test/web/activity_pub/mrf/reject_non_public_test.exs +++ b/test/web/activity_pub/mrf/reject_non_public_test.exs @@ -8,12 +8,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublicTest do alias Pleroma.Web.ActivityPub.MRF.RejectNonPublic - setup do - policy = Pleroma.Config.get([:mrf_rejectnonpublic]) - on_exit(fn -> Pleroma.Config.put([:mrf_rejectnonpublic], policy) end) - - :ok - end + clear_config([:mrf_rejectnonpublic]) describe "public message" do test "it's allowed when address is public" do diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs index 8e86d2219..7203b27da 100644 --- a/test/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -8,9 +8,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do alias Pleroma.Config alias Pleroma.Web.ActivityPub.MRF.SimplePolicy - setup do - orig = Config.get!(:mrf_simple) - + clear_config([:mrf_simple]) do Config.put(:mrf_simple, media_removal: [], media_nsfw: [], @@ -21,10 +19,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do avatar_removal: [], banner_removal: [] ) - - on_exit(fn -> - Config.put(:mrf_simple, orig) - end) end describe "when :media_removal" do diff --git a/test/web/activity_pub/mrf/user_allowlist_policy_test.exs b/test/web/activity_pub/mrf/user_allowlist_policy_test.exs index 6519e2398..72084c0fd 100644 --- a/test/web/activity_pub/mrf/user_allowlist_policy_test.exs +++ b/test/web/activity_pub/mrf/user_allowlist_policy_test.exs @@ -7,12 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicyTest do alias Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy - setup do - policy = Pleroma.Config.get([:mrf_user_allowlist]) || [] - on_exit(fn -> Pleroma.Config.put([:mrf_user_allowlist], policy) end) - - :ok - end + clear_config([:mrf_user_allowlist, :localhost]) test "pass filter if allow list is empty" do actor = insert(:user) diff --git a/test/web/activity_pub/mrf/vocabulary_policy_test.exs b/test/web/activity_pub/mrf/vocabulary_policy_test.exs index c3b11d7a1..38309f9f1 100644 --- a/test/web/activity_pub/mrf/vocabulary_policy_test.exs +++ b/test/web/activity_pub/mrf/vocabulary_policy_test.exs @@ -8,8 +8,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicyTest do alias Pleroma.Web.ActivityPub.MRF.VocabularyPolicy describe "accept" do + clear_config([:mrf_vocabulary, :accept]) + test "it accepts based on parent activity type" do - config = Pleroma.Config.get([:mrf_vocabulary, :accept]) Pleroma.Config.put([:mrf_vocabulary, :accept], ["Like"]) message = %{ @@ -18,12 +19,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicyTest do } {:ok, ^message} = VocabularyPolicy.filter(message) - - Pleroma.Config.put([:mrf_vocabulary, :accept], config) end test "it accepts based on child object type" do - config = Pleroma.Config.get([:mrf_vocabulary, :accept]) Pleroma.Config.put([:mrf_vocabulary, :accept], ["Create", "Note"]) message = %{ @@ -35,12 +33,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicyTest do } {:ok, ^message} = VocabularyPolicy.filter(message) - - Pleroma.Config.put([:mrf_vocabulary, :accept], config) end test "it does not accept disallowed child objects" do - config = Pleroma.Config.get([:mrf_vocabulary, :accept]) Pleroma.Config.put([:mrf_vocabulary, :accept], ["Create", "Note"]) message = %{ @@ -52,12 +47,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicyTest do } {:reject, nil} = VocabularyPolicy.filter(message) - - Pleroma.Config.put([:mrf_vocabulary, :accept], config) end test "it does not accept disallowed parent types" do - config = Pleroma.Config.get([:mrf_vocabulary, :accept]) Pleroma.Config.put([:mrf_vocabulary, :accept], ["Announce", "Note"]) message = %{ @@ -69,14 +61,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicyTest do } {:reject, nil} = VocabularyPolicy.filter(message) - - Pleroma.Config.put([:mrf_vocabulary, :accept], config) end end describe "reject" do + clear_config([:mrf_vocabulary, :reject]) + test "it rejects based on parent activity type" do - config = Pleroma.Config.get([:mrf_vocabulary, :reject]) Pleroma.Config.put([:mrf_vocabulary, :reject], ["Like"]) message = %{ @@ -85,12 +76,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicyTest do } {:reject, nil} = VocabularyPolicy.filter(message) - - Pleroma.Config.put([:mrf_vocabulary, :reject], config) end test "it rejects based on child object type" do - config = Pleroma.Config.get([:mrf_vocabulary, :reject]) Pleroma.Config.put([:mrf_vocabulary, :reject], ["Note"]) message = %{ @@ -102,12 +90,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicyTest do } {:reject, nil} = VocabularyPolicy.filter(message) - - Pleroma.Config.put([:mrf_vocabulary, :reject], config) end test "it passes through objects that aren't disallowed" do - config = Pleroma.Config.get([:mrf_vocabulary, :reject]) Pleroma.Config.put([:mrf_vocabulary, :reject], ["Like"]) message = %{ @@ -116,8 +101,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicyTest do } {:ok, ^message} = VocabularyPolicy.filter(message) - - Pleroma.Config.put([:mrf_vocabulary, :reject], config) end end end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index bcbc18639..844cd0732 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -294,18 +294,15 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do describe "POST /api/pleroma/admin/email_invite, with valid config" do setup do - registrations_open = Pleroma.Config.get([:instance, :registrations_open]) - invites_enabled = Pleroma.Config.get([:instance, :invites_enabled]) - Pleroma.Config.put([:instance, :registrations_open], false) - Pleroma.Config.put([:instance, :invites_enabled], true) + [user: insert(:user, info: %{is_admin: true})] + end - on_exit(fn -> - Pleroma.Config.put([:instance, :registrations_open], registrations_open) - Pleroma.Config.put([:instance, :invites_enabled], invites_enabled) - :ok - end) + clear_config([:instance, :registrations_open]) do + Pleroma.Config.put([:instance, :registrations_open], false) + end - [user: insert(:user, info: %{is_admin: true})] + clear_config([:instance, :invites_enabled]) do + Pleroma.Config.put([:instance, :invites_enabled], true) end test "sends invitation and returns 204", %{conn: conn, user: user} do @@ -360,18 +357,13 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do [user: insert(:user, info: %{is_admin: true})] end + clear_config([:instance, :registrations_open]) + clear_config([:instance, :invites_enabled]) + test "it returns 500 if `invites_enabled` is not enabled", %{conn: conn, user: user} do - registrations_open = Pleroma.Config.get([:instance, :registrations_open]) - invites_enabled = Pleroma.Config.get([:instance, :invites_enabled]) Pleroma.Config.put([:instance, :registrations_open], false) Pleroma.Config.put([:instance, :invites_enabled], false) - on_exit(fn -> - Pleroma.Config.put([:instance, :registrations_open], registrations_open) - Pleroma.Config.put([:instance, :invites_enabled], invites_enabled) - :ok - end) - conn = conn |> assign(:user, user) @@ -381,17 +373,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do end test "it returns 500 if `registrations_open` is enabled", %{conn: conn, user: user} do - registrations_open = Pleroma.Config.get([:instance, :registrations_open]) - invites_enabled = Pleroma.Config.get([:instance, :invites_enabled]) Pleroma.Config.put([:instance, :registrations_open], true) Pleroma.Config.put([:instance, :invites_enabled], true) - on_exit(fn -> - Pleroma.Config.put([:instance, :registrations_open], registrations_open) - Pleroma.Config.put([:instance, :invites_enabled], invites_enabled) - :ok - end) - conn = conn |> assign(:user, user) @@ -1402,15 +1386,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do :ok = File.rm(temp_file) end) - dynamic = Pleroma.Config.get([:instance, :dynamic_configuration]) + %{conn: assign(conn, :user, admin)} + end + clear_config([:instance, :dynamic_configuration]) do Pleroma.Config.put([:instance, :dynamic_configuration], true) - - on_exit(fn -> - Pleroma.Config.put([:instance, :dynamic_configuration], dynamic) - end) - - %{conn: assign(conn, :user, admin)} end test "create new config setting in db", %{conn: conn} do @@ -1961,15 +1941,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do :ok = File.rm(temp_file) end) - dynamic = Pleroma.Config.get([:instance, :dynamic_configuration]) + %{conn: assign(conn, :user, admin), admin: admin} + end + clear_config([:instance, :dynamic_configuration]) do Pleroma.Config.put([:instance, :dynamic_configuration], true) - - on_exit(fn -> - Pleroma.Config.put([:instance, :dynamic_configuration], dynamic) - end) - - %{conn: assign(conn, :user, admin), admin: admin} end test "transfer settings to DB and to file", %{conn: conn, admin: admin} do diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 454523349..bcbaad665 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -14,6 +14,10 @@ defmodule Pleroma.Web.CommonAPITest do import Pleroma.Factory + clear_config([:instance, :safe_dm_mentions]) + clear_config([:instance, :limit]) + clear_config([:instance, :max_pinned_statuses]) + test "when replying to a conversation / participation, it will set the correct context id even if no explicit reply_to is given" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"}) @@ -61,7 +65,6 @@ defmodule Pleroma.Web.CommonAPITest do har = insert(:user) jafnhar = insert(:user) tridi = insert(:user) - option = Pleroma.Config.get([:instance, :safe_dm_mentions]) Pleroma.Config.put([:instance, :safe_dm_mentions], true) {:ok, activity} = @@ -72,7 +75,6 @@ defmodule Pleroma.Web.CommonAPITest do refute tridi.ap_id in activity.recipients assert jafnhar.ap_id in activity.recipients - Pleroma.Config.put([:instance, :safe_dm_mentions], option) end test "it de-duplicates tags" do @@ -195,15 +197,12 @@ defmodule Pleroma.Web.CommonAPITest do end test "it returns error when character limit is exceeded" do - limit = Pleroma.Config.get([:instance, :limit]) Pleroma.Config.put([:instance, :limit], 5) user = insert(:user) assert {:error, "The status is over the character limit"} = CommonAPI.post(user, %{"status" => "foobar"}) - - Pleroma.Config.put([:instance, :limit], limit) end end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 73cfaa8f1..09e54533f 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -13,15 +13,17 @@ defmodule Pleroma.Web.FederatorTest do setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) - config_path = [:instance, :federating] - initial_setting = Pleroma.Config.get(config_path) - - Pleroma.Config.put(config_path, true) - on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) - :ok end + clear_config_all([:instance, :federating]) do + Pleroma.Config.put([:instance, :federating], true) + end + + clear_config([:instance, :allow_relay]) + clear_config([:instance, :rewrite_policy]) + clear_config([:mrf_keyword]) + describe "Publisher.perform" do test "call `perform` with unknown task" do assert { @@ -67,8 +69,6 @@ defmodule Pleroma.Web.FederatorTest do end refute_received :relay_publish - - Pleroma.Config.put([:instance, :allow_relay], true) end end @@ -231,19 +231,18 @@ defmodule Pleroma.Web.FederatorTest do end test "it does not crash if MRF rejects the post" do - policies = Pleroma.Config.get([:instance, :rewrite_policy]) - mrf_keyword_policy = Pleroma.Config.get(:mrf_keyword) Pleroma.Config.put([:mrf_keyword, :reject], ["lain"]) - Pleroma.Config.put([:instance, :rewrite_policy], Pleroma.Web.ActivityPub.MRF.KeywordPolicy) + + Pleroma.Config.put( + [:instance, :rewrite_policy], + Pleroma.Web.ActivityPub.MRF.KeywordPolicy + ) params = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() assert Federator.incoming_ap_doc(params) == :error - - Pleroma.Config.put([:instance, :rewrite_policy], policies) - Pleroma.Config.put(:mrf_keyword, mrf_keyword_policy) end end end diff --git a/test/web/instances/instance_test.exs b/test/web/instances/instance_test.exs index d28730994..3fd011fd3 100644 --- a/test/web/instances/instance_test.exs +++ b/test/web/instances/instance_test.exs @@ -10,14 +10,8 @@ defmodule Pleroma.Instances.InstanceTest do import Pleroma.Factory - setup_all do - config_path = [:instance, :federation_reachability_timeout_days] - initial_setting = Pleroma.Config.get(config_path) - - Pleroma.Config.put(config_path, 1) - on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) - - :ok + clear_config_all([:instance, :federation_reachability_timeout_days]) do + Pleroma.Config.put([:instance, :federation_reachability_timeout_days], 1) end describe "set_reachable/1" do diff --git a/test/web/instances/instances_test.exs b/test/web/instances/instances_test.exs index f0d84edea..dea8e2aea 100644 --- a/test/web/instances/instances_test.exs +++ b/test/web/instances/instances_test.exs @@ -7,14 +7,8 @@ defmodule Pleroma.InstancesTest do use Pleroma.DataCase - setup_all do - config_path = [:instance, :federation_reachability_timeout_days] - initial_setting = Pleroma.Config.get(config_path) - - Pleroma.Config.put(config_path, 1) - on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) - - :ok + clear_config_all([:instance, :federation_reachability_timeout_days]) do + Pleroma.Config.put([:instance, :federation_reachability_timeout_days], 1) end describe "reachable?/1" do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 112e272f9..77430e9c9 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -33,6 +33,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do :ok end + clear_config([:instance, :public]) + clear_config([:rich_media, :enabled]) + test "the home timeline", %{conn: conn} do user = insert(:user) following = insert(:user) @@ -86,13 +89,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end test "the public timeline when public is set to false", %{conn: conn} do - public = Config.get([:instance, :public]) Config.put([:instance, :public], false) - on_exit(fn -> - Config.put([:instance, :public], public) - end) - assert conn |> get("/api/v1/timelines/public", %{"local" => "False"}) |> json_response(403) == %{"error" => "This resource requires authentication."} @@ -261,7 +259,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert %{"id" => id, "card" => %{"title" => "The Rock"}} = json_response(conn, 200) assert Activity.get_by_id(id) - Config.put([:rich_media, :enabled], false) end test "posting a direct status", %{conn: conn} do @@ -1634,14 +1631,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do describe "media upload" do setup do - upload_config = Config.get([Pleroma.Upload]) - proxy_config = Config.get([:media_proxy]) - - on_exit(fn -> - Config.put([Pleroma.Upload], upload_config) - Config.put([:media_proxy], proxy_config) - end) - user = insert(:user) conn = @@ -1657,6 +1646,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do [conn: conn, image: image] end + clear_config([:media_proxy]) + clear_config([Pleroma.Upload]) + test "returns uploaded image", %{conn: conn, image: image} do desc = "Description of the image" @@ -2667,14 +2659,16 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do describe "pinned statuses" do setup do - Config.put([:instance, :max_pinned_statuses], 1) - user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "HI!!!"}) [user: user, activity: activity] end + clear_config([:instance, :max_pinned_statuses]) do + Config.put([:instance, :max_pinned_statuses], 1) + end + test "returns pinned statuses", %{conn: conn, user: user, activity: activity} do {:ok, _} = CommonAPI.pin(activity.id, user) @@ -2769,10 +2763,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do setup do Config.put([:rich_media, :enabled], true) - on_exit(fn -> - Config.put([:rich_media, :enabled], false) - end) - user = insert(:user) %{user: user} end @@ -3127,15 +3117,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do conn: conn, path: path } do - is_public = Config.get([:instance, :public]) Config.put([:instance, :public], false) conn = get(conn, path) assert conn.status == 302 assert redirected_to(conn) == "/web/login" - - Config.put([:instance, :public], is_public) end test "does not redirect logged in users to the login page", %{conn: conn, path: path} do @@ -3910,13 +3897,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do describe "POST /api/v1/pleroma/accounts/confirmation_resend" do setup do - setting = Config.get([:instance, :account_activation_required]) - - unless setting do - Config.put([:instance, :account_activation_required], true) - on_exit(fn -> Config.put([:instance, :account_activation_required], setting) end) - end - user = insert(:user) info_change = User.Info.confirmation_changeset(user.info, need_confirmation: true) @@ -3931,6 +3911,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do [user: user] end + clear_config([:instance, :account_activation_required]) do + Config.put([:instance, :account_activation_required], true) + end + test "resend account confirmation email", %{conn: conn, user: user} do conn |> assign(:user, user) @@ -3953,9 +3937,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do setup do user = insert(:user) other_user = insert(:user) - config = Config.get(:suggestions) - on_exit(fn -> Config.put(:suggestions, config) end) - host = Config.get([Pleroma.Web.Endpoint, :url, :host]) url500 = "http://test500?#{host}&#{user.nickname}" url200 = "http://test200?#{host}&#{user.nickname}" @@ -3977,6 +3958,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do [user: user, other_user: other_user] end + clear_config(:suggestions) + test "returns empty result when suggestions disabled", %{conn: conn, user: user} do Config.put([:suggestions, :enabled], false) diff --git a/test/web/media_proxy/media_proxy_test.exs b/test/web/media_proxy/media_proxy_test.exs index 0c94755df..79699cac5 100644 --- a/test/web/media_proxy/media_proxy_test.exs +++ b/test/web/media_proxy/media_proxy_test.exs @@ -4,14 +4,11 @@ defmodule Pleroma.Web.MediaProxyTest do use ExUnit.Case + use Pleroma.Tests.Helpers import Pleroma.Web.MediaProxy alias Pleroma.Web.MediaProxy.MediaProxyController - setup do - enabled = Pleroma.Config.get([:media_proxy, :enabled]) - on_exit(fn -> Pleroma.Config.put([:media_proxy, :enabled], enabled) end) - :ok - end + clear_config([:media_proxy, :enabled]) describe "when enabled" do setup do diff --git a/test/web/oauth/ldap_authorization_test.exs b/test/web/oauth/ldap_authorization_test.exs index 0eb191c76..1cbe133b7 100644 --- a/test/web/oauth/ldap_authorization_test.exs +++ b/test/web/oauth/ldap_authorization_test.exs @@ -12,21 +12,12 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do @skip if !Code.ensure_loaded?(:eldap), do: :skip - setup_all do - ldap_authenticator = - Pleroma.Config.get(Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator) - - ldap_enabled = Pleroma.Config.get([:ldap, :enabled]) - - on_exit(fn -> - Pleroma.Config.put(Pleroma.Web.Auth.Authenticator, ldap_authenticator) - Pleroma.Config.put([:ldap, :enabled], ldap_enabled) - end) - - Pleroma.Config.put(Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.LDAPAuthenticator) + clear_config_all([:ldap, :enabled]) do Pleroma.Config.put([:ldap, :enabled], true) + end - :ok + clear_config_all(Pleroma.Web.Auth.Authenticator) do + Pleroma.Config.put(Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.LDAPAuthenticator) end @tag @skip diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs index 92e156347..b492c7794 100644 --- a/test/web/oauth/oauth_controller_test.exs +++ b/test/web/oauth/oauth_controller_test.exs @@ -11,23 +11,15 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do alias Pleroma.Web.OAuth.OAuthController alias Pleroma.Web.OAuth.Token - @oauth_config_path [:oauth2, :issue_new_refresh_token] @session_opts [ store: :cookie, key: "_test", signing_salt: "cooldude" ] + clear_config_all([:instance, :account_activation_required]) describe "in OAuth consumer mode, " do setup do - oauth_consumer_strategies_path = [:auth, :oauth_consumer_strategies] - oauth_consumer_strategies = Pleroma.Config.get(oauth_consumer_strategies_path) - Pleroma.Config.put(oauth_consumer_strategies_path, ~w(twitter facebook)) - - on_exit(fn -> - Pleroma.Config.put(oauth_consumer_strategies_path, oauth_consumer_strategies) - end) - [ app: insert(:oauth_app), conn: @@ -37,6 +29,13 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do ] end + clear_config([:auth, :oauth_consumer_strategies]) do + Pleroma.Config.put( + [:auth, :oauth_consumer_strategies], + ~w(twitter facebook) + ) + end + test "GET /oauth/authorize renders auth forms, including OAuth consumer form", %{ app: app, conn: conn @@ -775,12 +774,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do end test "rejects token exchange for valid credentials belonging to unconfirmed user and confirmation is required" do - setting = Pleroma.Config.get([:instance, :account_activation_required]) - - unless setting do - Pleroma.Config.put([:instance, :account_activation_required], true) - on_exit(fn -> Pleroma.Config.put([:instance, :account_activation_required], setting) end) - end + Pleroma.Config.put([:instance, :account_activation_required], true) password = "testpassword" user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt(password)) @@ -857,16 +851,10 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do end describe "POST /oauth/token - refresh token" do - setup do - oauth_token_config = Pleroma.Config.get(@oauth_config_path) - - on_exit(fn -> - Pleroma.Config.get(@oauth_config_path, oauth_token_config) - end) - end + clear_config([:oauth2, :issue_new_refresh_token]) test "issues a new access token with keep fresh token" do - Pleroma.Config.put(@oauth_config_path, true) + Pleroma.Config.put([:oauth2, :issue_new_refresh_token], true) user = insert(:user) app = insert(:oauth_app, scopes: ["read", "write"]) @@ -906,7 +894,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do end test "issues a new access token with new fresh token" do - Pleroma.Config.put(@oauth_config_path, false) + Pleroma.Config.put([:oauth2, :issue_new_refresh_token], false) user = insert(:user) app = insert(:oauth_app, scopes: ["read", "write"]) diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 9f756effb..095ae7041 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -15,16 +15,13 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) - - config_path = [:instance, :federating] - initial_setting = Pleroma.Config.get(config_path) - - Pleroma.Config.put(config_path, true) - on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) - :ok end + clear_config_all([:instance, :federating]) do + Pleroma.Config.put([:instance, :federating], true) + end + describe "salmon_incoming" do test "decodes a salmon", %{conn: conn} do user = insert(:user) diff --git a/test/web/plugs/federating_plug_test.exs b/test/web/plugs/federating_plug_test.exs index c01e01124..bb2e1687a 100644 --- a/test/web/plugs/federating_plug_test.exs +++ b/test/web/plugs/federating_plug_test.exs @@ -4,15 +4,7 @@ defmodule Pleroma.Web.FederatingPlugTest do use Pleroma.Web.ConnCase - - setup_all do - config_path = [:instance, :federating] - initial_setting = Pleroma.Config.get(config_path) - - on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) - - :ok - end + clear_config_all([:instance, :federating]) test "returns and halt the conn when federating is disabled" do Pleroma.Config.put([:instance, :federating], false) diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs index 92198f3d9..48884319d 100644 --- a/test/web/rich_media/helpers_test.exs +++ b/test/web/rich_media/helpers_test.exs @@ -15,12 +15,12 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) - rich_media = Config.get([:rich_media, :enabled]) - on_exit(fn -> Config.put([:rich_media, :enabled], rich_media) end) :ok end + clear_config([:rich_media, :enabled]) + test "refuses to crawl incomplete URLs" do user = insert(:user) diff --git a/test/web/streamer_test.exs b/test/web/streamer_test.exs index 5b7fe44d4..96fa7645f 100644 --- a/test/web/streamer_test.exs +++ b/test/web/streamer_test.exs @@ -11,15 +11,7 @@ defmodule Pleroma.Web.StreamerTest do alias Pleroma.Web.Streamer import Pleroma.Factory - setup do - skip_thread_containment = Pleroma.Config.get([:instance, :skip_thread_containment]) - - on_exit(fn -> - Pleroma.Config.put([:instance, :skip_thread_containment], skip_thread_containment) - end) - - :ok - end + clear_config_all([:instance, :skip_thread_containment]) describe "user streams" do setup do diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 8bb8aa36d..8ef14b4c5 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -151,6 +151,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do describe "GET /statuses/public_timeline.json" do setup [:valid_user] + clear_config([:instance, :public]) test "returns statuses", %{conn: conn} do user = insert(:user) @@ -173,8 +174,6 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do conn |> get("/api/statuses/public_timeline.json") |> json_response(403) - - Pleroma.Config.put([:instance, :public], true) end test "returns 200 to authenticated request when the instance is not public", @@ -185,8 +184,6 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do |> with_credentials(user.nickname, "test") |> get("/api/statuses/public_timeline.json") |> json_response(200) - - Pleroma.Config.put([:instance, :public], true) end test "returns 200 to unauthenticated request when the instance is public", %{conn: conn} do @@ -220,6 +217,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do describe "GET /statuses/public_and_external_timeline.json" do setup [:valid_user] + clear_config([:instance, :public]) test "returns 403 to unauthenticated request when the instance is not public", %{conn: conn} do Pleroma.Config.put([:instance, :public], false) @@ -227,8 +225,6 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do conn |> get("/api/statuses/public_and_external_timeline.json") |> json_response(403) - - Pleroma.Config.put([:instance, :public], true) end test "returns 200 to authenticated request when the instance is not public", @@ -239,8 +235,6 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do |> with_credentials(user.nickname, "test") |> get("/api/statuses/public_and_external_timeline.json") |> json_response(200) - - Pleroma.Config.put([:instance, :public], true) end test "returns 200 to unauthenticated request when the instance is public", %{conn: conn} do @@ -1176,13 +1170,6 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do describe "POST /api/account/resend_confirmation_email" do setup do - setting = Pleroma.Config.get([:instance, :account_activation_required]) - - unless setting do - Pleroma.Config.put([:instance, :account_activation_required], true) - on_exit(fn -> Pleroma.Config.put([:instance, :account_activation_required], setting) end) - end - user = insert(:user) info_change = User.Info.confirmation_changeset(user.info, need_confirmation: true) @@ -1197,6 +1184,10 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do [user: user] end + clear_config([:instance, :account_activation_required]) do + Pleroma.Config.put([:instance, :account_activation_required], true) + end + test "it returns 204 No Content", %{conn: conn, user: user} do conn |> assign(:user, user) diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 640579c09..fe4ffdb59 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -14,20 +14,13 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do setup do Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) - - instance_config = Pleroma.Config.get([:instance]) - pleroma_fe = Pleroma.Config.get([:frontend_configurations, :pleroma_fe]) - deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked]) - - on_exit(fn -> - Pleroma.Config.put([:instance], instance_config) - Pleroma.Config.put([:frontend_configurations, :pleroma_fe], pleroma_fe) - Pleroma.Config.put([:user, :deny_follow_blocked], deny_follow_blocked) - end) - :ok end + clear_config([:instance]) + clear_config([:frontend_configurations, :pleroma_fe]) + clear_config([:user, :deny_follow_blocked]) + describe "POST /api/pleroma/follow_import" do test "it returns HTTP 200", %{conn: conn} do user1 = insert(:user) @@ -260,7 +253,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do end test "returns the state of safe_dm_mentions flag", %{conn: conn} do - option = Pleroma.Config.get([:instance, :safe_dm_mentions]) Pleroma.Config.put([:instance, :safe_dm_mentions], true) response = @@ -278,8 +270,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do |> json_response(:ok) assert response["site"]["safeDMMentionsEnabled"] == "0" - - Pleroma.Config.put([:instance, :safe_dm_mentions], option) end test "it returns the managed config", %{conn: conn} do @@ -534,15 +524,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do end describe "GET /api/pleroma/healthcheck" do - setup do - config_healthcheck = Pleroma.Config.get([:instance, :healthcheck]) - - on_exit(fn -> - Pleroma.Config.put([:instance, :healthcheck], config_healthcheck) - end) - - :ok - end + clear_config([:instance, :healthcheck]) test "returns 503 when healthcheck disabled", %{conn: conn} do Pleroma.Config.put([:instance, :healthcheck], false) diff --git a/test/web/web_finger/web_finger_controller_test.exs b/test/web/web_finger/web_finger_controller_test.exs index 7d861cbf5..e23086b2a 100644 --- a/test/web/web_finger/web_finger_controller_test.exs +++ b/test/web/web_finger/web_finger_controller_test.exs @@ -10,15 +10,13 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) - - config_path = [:instance, :federating] - initial_setting = Pleroma.Config.get(config_path) - - Pleroma.Config.put(config_path, true) - on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) :ok end + clear_config_all([:instance, :federating]) do + Pleroma.Config.put([:instance, :federating], true) + end + test "GET host-meta" do response = build_conn() diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index aa7262beb..59cacbe68 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -9,14 +9,8 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do alias Pleroma.Web.Websub alias Pleroma.Web.Websub.WebsubClientSubscription - setup_all do - config_path = [:instance, :federating] - initial_setting = Pleroma.Config.get(config_path) - - Pleroma.Config.put(config_path, true) - on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) - - :ok + clear_config_all([:instance, :federating]) do + Pleroma.Config.put([:instance, :federating], true) end test "websub subscription request", %{conn: conn} do -- cgit v1.2.3 From 75a5dd41ee4a1c196487f4cf2759a4d63bc393ef Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Mon, 19 Aug 2019 16:10:00 +0000 Subject: Add more tests for Database tasks and DigestEmailWorker --- lib/pleroma/digest_email_worker.ex | 4 +++ test/mix/tasks/pleroma.digest_test.exs | 51 ---------------------------------- test/tasks/database_test.exs | 47 +++++++++++++++++++++++++++++++ test/tasks/digest_test.exs | 51 ++++++++++++++++++++++++++++++++++ test/web/digest_email_worker_test.exs | 31 +++++++++++++++++++++ 5 files changed, 133 insertions(+), 51 deletions(-) delete mode 100644 test/mix/tasks/pleroma.digest_test.exs create mode 100644 test/tasks/digest_test.exs create mode 100644 test/web/digest_email_worker_test.exs diff --git a/lib/pleroma/digest_email_worker.ex b/lib/pleroma/digest_email_worker.ex index 18e67d39b..5644d6a67 100644 --- a/lib/pleroma/digest_email_worker.ex +++ b/lib/pleroma/digest_email_worker.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.DigestEmailWorker do import Ecto.Query diff --git a/test/mix/tasks/pleroma.digest_test.exs b/test/mix/tasks/pleroma.digest_test.exs deleted file mode 100644 index 595f64ed7..000000000 --- a/test/mix/tasks/pleroma.digest_test.exs +++ /dev/null @@ -1,51 +0,0 @@ -defmodule Mix.Tasks.Pleroma.DigestTest do - use Pleroma.DataCase - - import Pleroma.Factory - import Swoosh.TestAssertions - - alias Pleroma.Web.CommonAPI - - setup_all do - Mix.shell(Mix.Shell.Process) - - on_exit(fn -> - Mix.shell(Mix.Shell.IO) - end) - - :ok - end - - describe "pleroma.digest test" do - test "Sends digest to the given user" do - user1 = insert(:user) - user2 = insert(:user) - - Enum.each(0..10, fn i -> - {:ok, _activity} = - CommonAPI.post(user1, %{ - "status" => "hey ##{i} @#{user2.nickname}!" - }) - end) - - yesterday = - NaiveDateTime.add( - NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second), - -60 * 60 * 24, - :second - ) - - {:ok, yesterday_date} = Timex.format(yesterday, "%F", :strftime) - - :ok = Mix.Tasks.Pleroma.Digest.run(["test", user2.nickname, yesterday_date]) - - assert_receive {:mix_shell, :info, [message]} - assert message =~ "Digest email have been sent" - - assert_email_sent( - to: {user2.name, user2.email}, - html_body: ~r/new mentions:/i - ) - end - end -end diff --git a/test/tasks/database_test.exs b/test/tasks/database_test.exs index a8f25f500..a9925c361 100644 --- a/test/tasks/database_test.exs +++ b/test/tasks/database_test.exs @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.DatabaseTest do + alias Pleroma.Activity alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -22,6 +23,52 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do :ok end + describe "running remove_embedded_objects" do + test "it replaces objects with references" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "test"}) + new_data = Map.put(activity.data, "object", activity.object.data) + + {:ok, activity} = + activity + |> Activity.change(%{data: new_data}) + |> Repo.update() + + assert is_map(activity.data["object"]) + + Mix.Tasks.Pleroma.Database.run(["remove_embedded_objects"]) + + activity = Activity.get_by_id_with_object(activity.id) + assert is_binary(activity.data["object"]) + end + end + + describe "prune_objects" do + test "it prunes old objects from the database" do + insert(:note) + deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1 + + date = + Timex.now() + |> Timex.shift(days: -deadline) + |> Timex.to_naive_datetime() + |> NaiveDateTime.truncate(:second) + + %{id: id} = + :note + |> insert() + |> Ecto.Changeset.change(%{inserted_at: date}) + |> Repo.update!() + + assert length(Repo.all(Object)) == 2 + + Mix.Tasks.Pleroma.Database.run(["prune_objects"]) + + assert length(Repo.all(Object)) == 1 + refute Object.get_by_id(id) + end + end + describe "running update_users_following_followers_counts" do test "following and followers count are updated" do [user, user2] = insert_pair(:user) diff --git a/test/tasks/digest_test.exs b/test/tasks/digest_test.exs new file mode 100644 index 000000000..595f64ed7 --- /dev/null +++ b/test/tasks/digest_test.exs @@ -0,0 +1,51 @@ +defmodule Mix.Tasks.Pleroma.DigestTest do + use Pleroma.DataCase + + import Pleroma.Factory + import Swoosh.TestAssertions + + alias Pleroma.Web.CommonAPI + + setup_all do + Mix.shell(Mix.Shell.Process) + + on_exit(fn -> + Mix.shell(Mix.Shell.IO) + end) + + :ok + end + + describe "pleroma.digest test" do + test "Sends digest to the given user" do + user1 = insert(:user) + user2 = insert(:user) + + Enum.each(0..10, fn i -> + {:ok, _activity} = + CommonAPI.post(user1, %{ + "status" => "hey ##{i} @#{user2.nickname}!" + }) + end) + + yesterday = + NaiveDateTime.add( + NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second), + -60 * 60 * 24, + :second + ) + + {:ok, yesterday_date} = Timex.format(yesterday, "%F", :strftime) + + :ok = Mix.Tasks.Pleroma.Digest.run(["test", user2.nickname, yesterday_date]) + + assert_receive {:mix_shell, :info, [message]} + assert message =~ "Digest email have been sent" + + assert_email_sent( + to: {user2.name, user2.email}, + html_body: ~r/new mentions:/i + ) + end + end +end diff --git a/test/web/digest_email_worker_test.exs b/test/web/digest_email_worker_test.exs new file mode 100644 index 000000000..15002330f --- /dev/null +++ b/test/web/digest_email_worker_test.exs @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.DigestEmailWorkerTest do + use Pleroma.DataCase + import Pleroma.Factory + + alias Pleroma.DigestEmailWorker + alias Pleroma.User + alias Pleroma.Web.CommonAPI + + test "it sends digest emails" do + user = insert(:user) + + date = + Timex.now() + |> Timex.shift(days: -10) + |> Timex.to_naive_datetime() + + user2 = insert(:user, last_digest_emailed_at: date) + User.switch_email_notifications(user2, "digest", true) + CommonAPI.post(user, %{"status" => "hey @#{user2.nickname}!"}) + + DigestEmailWorker.perform() + + assert_received {:email, email} + assert email.to == [{user2.name, user2.email}] + assert email.subject == "Your digest from #{Pleroma.Config.get(:instance)[:name]}" + end +end -- cgit v1.2.3 From 4f41634ccc1bde180dadf0a6462e4e5edeb486a5 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 7 Aug 2019 18:00:43 +0300 Subject: adding gun adapter --- config/config.exs | 2 +- mix.exs | 5 ++++- mix.lock | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index 758661120..63162d594 100644 --- a/config/config.exs +++ b/config/config.exs @@ -186,7 +186,7 @@ config :mime, :types, %{ "application/ld+json" => ["activity+json"] } -config :tesla, adapter: Tesla.Adapter.Hackney +config :tesla, adapter: Tesla.Adapter.Gun # Configures http settings, upstream proxy etc. config :pleroma, :http, diff --git a/mix.exs b/mix.exs index 3170d6f2d..9aab71e90 100644 --- a/mix.exs +++ b/mix.exs @@ -111,7 +111,10 @@ defmodule Pleroma.Mixfile do {:calendar, "~> 0.17.4"}, {:cachex, "~> 3.0.2"}, {:poison, "~> 3.0", override: true}, - {:tesla, "~> 1.2"}, + {:tesla, + github: "teamon/tesla", ref: "97950754a77cde4e162ada31fb9d8cf4fd6ab822", override: true}, + {:cowlib, "~> 2.6.0", override: true}, + {:gun, "~> 1.3"}, {:jason, "~> 1.0"}, {:mogrify, "~> 0.6.1"}, {:ex_aws, "~> 2.1"}, diff --git a/mix.lock b/mix.lock index 2639e96e9..bf3d4f01a 100644 --- a/mix.lock +++ b/mix.lock @@ -13,7 +13,7 @@ "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, "cors_plug": {:hex, :cors_plug, "1.5.2", "72df63c87e4f94112f458ce9d25800900cc88608c1078f0e4faddf20933eda6e", [:mix], [{:plug, "~> 1.3 or ~> 1.4 or ~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "cowboy": {:hex, :cowboy, "2.6.3", "99aa50e94e685557cad82e704457336a453d4abcb77839ad22dbe71f311fcc06", [:rebar3], [{:cowlib, "~> 2.7.3", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, - "cowlib": {:hex, :cowlib, "2.7.3", "a7ffcd0917e6d50b4d5fb28e9e2085a0ceb3c97dea310505f7460ff5ed764ce9", [:rebar3], [], "hexpm"}, + "cowlib": {:hex, :cowlib, "2.6.0", "8aa629f81a0fc189f261dc98a42243fa842625feea3c7ec56c48f4ccdb55490f", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "crontab": {:hex, :crontab, "1.1.7", "b9219f0bdc8678b94143655a8f229716c5810c0636a4489f98c0956137e53985", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, @@ -37,6 +37,7 @@ "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, + "gun": {:hex, :gun, "1.3.0", "18e5d269649c987af95aec309f68a27ffc3930531dd227a6eaa0884d6684286e", [:rebar3], [{:cowlib, "~> 2.6.0", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm"}, "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, @@ -84,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, + "tesla": {:git, "https://github.com/teamon/tesla.git", "97950754a77cde4e162ada31fb9d8cf4fd6ab822", [ref: "97950754a77cde4e162ada31fb9d8cf4fd6ab822"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, -- cgit v1.2.3 From b233875ef5b7c1cb7b6c7fea1d30604fece2a659 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 7 Aug 2019 20:04:37 +0300 Subject: namings --- lib/pleroma/http/connection.ex | 13 ++++++++----- lib/pleroma/http/http.ex | 2 +- lib/pleroma/reverse_proxy/reverse_proxy.ex | 2 +- .../web/activity_pub/mrf/mediaproxy_warming_policy.ex | 4 ++-- lib/pleroma/web/rel_me.ex | 4 ++-- lib/pleroma/web/rich_media/parser.ex | 4 ++-- mix.exs | 4 +++- mix.lock | 2 +- 8 files changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 7e2c6f5e8..8caf989a7 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -7,8 +7,10 @@ defmodule Pleroma.HTTP.Connection do Connection for http-requests. """ - @hackney_options [ + @options [ connect_timeout: 10_000, + protocols: [:http], + timeout: 20_000, recv_timeout: 20_000, follow_redirect: true, force_redirect: true, @@ -25,17 +27,18 @@ defmodule Pleroma.HTTP.Connection do """ @spec new(Keyword.t()) :: Tesla.Env.client() def new(opts \\ []) do - Tesla.client([], {@adapter, hackney_options(opts)}) + middleware = [Tesla.Middleware.FollowRedirects] + Tesla.client(middleware, {@adapter, options(opts)}) end - # fetch Hackney options + # fetch http options # - def hackney_options(opts) do + def options(opts) do options = Keyword.get(opts, :adapter, []) adapter_options = Pleroma.Config.get([:http, :adapter], []) proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) - @hackney_options + @options |> Keyword.merge(adapter_options) |> Keyword.merge(options) |> Keyword.merge(proxy: proxy_url) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index dec24458a..6d7934841 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -65,7 +65,7 @@ defmodule Pleroma.HTTP do end def process_request_options(options) do - Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options) + Keyword.merge(Pleroma.HTTP.Connection.options([]), options) end @doc """ diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 03efad30a..3212bf90d 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -94,7 +94,7 @@ defmodule Pleroma.ReverseProxy do def call(conn = %{method: method}, url, opts) when method in @methods do hackney_opts = - Pleroma.HTTP.Connection.hackney_options([]) + Pleroma.HTTP.Connection.options([]) |> Keyword.merge(@default_hackney_options) |> Keyword.merge(Keyword.get(opts, :http, [])) |> HTTP.process_request_options() diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex index a179dd54d..52ef0167c 100644 --- a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @hackney_options [ + @options [ pool: :media, recv_timeout: 10_000 ] @@ -21,7 +21,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do url |> MediaProxy.url() - |> HTTP.get([], adapter: @hackney_options) + |> HTTP.get([], adapter: @options) end def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index d376e2069..947234aa2 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RelMe do - @hackney_options [ + @options [ pool: :media, recv_timeout: 2_000, max_body: 2_000_000, @@ -25,7 +25,7 @@ defmodule Pleroma.Web.RelMe do def parse(_), do: {:error, "No URL provided"} defp parse_url(url) do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @options) data = Floki.attribute(html, "link[rel~=me]", "href") ++ diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index f5f9e358c..ade4ac891 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser do - @hackney_options [ + @options [ pool: :media, recv_timeout: 2_000, max_body: 2_000_000, @@ -78,7 +78,7 @@ defmodule Pleroma.Web.RichMedia.Parser do defp parse_url(url) do try do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @options) html |> maybe_parse() diff --git a/mix.exs b/mix.exs index 9aab71e90..84d58f5a6 100644 --- a/mix.exs +++ b/mix.exs @@ -112,7 +112,9 @@ defmodule Pleroma.Mixfile do {:cachex, "~> 3.0.2"}, {:poison, "~> 3.0", override: true}, {:tesla, - github: "teamon/tesla", ref: "97950754a77cde4e162ada31fb9d8cf4fd6ab822", override: true}, + github: "alex-strizhakov/tesla", + ref: "9ad792fb630bdfc2266ed13b830c28b6552fb3f9", + override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, {:jason, "~> 1.0"}, diff --git a/mix.lock b/mix.lock index bf3d4f01a..d79bb9c80 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/teamon/tesla.git", "97950754a77cde4e162ada31fb9d8cf4fd6ab822", [ref: "97950754a77cde4e162ada31fb9d8cf4fd6ab822"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "9ad792fb630bdfc2266ed13b830c28b6552fb3f9", [ref: "9ad792fb630bdfc2266ed13b830c28b6552fb3f9"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, -- cgit v1.2.3 From b383d85b9b8229751dd538c66e02a18f2de45835 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 7 Aug 2019 20:06:43 +0300 Subject: headers standardisation for tesla --- lib/pleroma/http/request_builder.ex | 2 +- lib/pleroma/object/fetcher.ex | 6 +- lib/pleroma/web/ostatus/ostatus.ex | 2 +- lib/pleroma/web/web_finger/web_finger.ex | 2 +- test/http/request_builder_test.exs | 2 +- test/support/http_request_mock.ex | 84 ++++++++++++---------- test/web/twitter_api/util_controller_test.exs | 16 +++-- test/web/web_finger/web_finger_controller_test.exs | 13 ++-- 8 files changed, 72 insertions(+), 55 deletions(-) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index e23457999..4e77870bd 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -48,7 +48,7 @@ defmodule Pleroma.HTTP.RequestBuilder do def headers(request, header_list) do header_list = if Pleroma.Config.get([:http, :send_user_agent]) do - header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + header_list ++ [{"user-agent", Pleroma.Application.user_agent()}] else header_list end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 8d79ddb1f..46531a13a 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -95,7 +95,7 @@ defmodule Pleroma.Object.Fetcher do date: date }) - [{:Signature, signature}] + [{"signature", signature}] end defp sign_fetch(headers, id, date) do @@ -108,7 +108,7 @@ defmodule Pleroma.Object.Fetcher do defp maybe_date_fetch(headers, date) do if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do - headers ++ [{:Date, date}] + headers ++ [{"date", date}] else headers end @@ -122,7 +122,7 @@ defmodule Pleroma.Object.Fetcher do |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT") headers = - [{:Accept, "application/activity+json"}] + [{"accept", "application/activity+json"}] |> maybe_date_fetch(date) |> sign_fetch(id, date) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 331cbc0b7..1c400d9e4 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -373,7 +373,7 @@ defmodule Pleroma.Web.OStatus do {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get( url, - [{:Accept, "application/atom+xml"}] + [{"accept", "application/atom+xml"}] ) do Logger.debug("Got document from #{url}, handling...") handle_incoming(body, options) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index ecb39ee50..624ee5ef7 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -217,7 +217,7 @@ defmodule Pleroma.Web.WebFinger do with response <- HTTP.get( address, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ), {:ok, %{status: status, body: body}} when status in 200..299 <- response do doc = XML.parse_document(body) diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index 170ca916f..ea4dd087e 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -18,7 +18,7 @@ defmodule Pleroma.HTTP.RequestBuilderTest do Pleroma.Config.put([:http, :send_user_agent], true) assert RequestBuilder.headers(%{}, []) == %{ - headers: [{"User-Agent", Pleroma.Application.user_agent()}] + headers: [{"user-agent", Pleroma.Application.user_agent()}] } end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 3adb5ba3b..c4862eb1c 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -88,7 +88,7 @@ defmodule HttpRequestMock do "https://osada.macgirvin.com/.well-known/webfinger?resource=acct:mike@osada.macgirvin.com", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -101,7 +101,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=https://social.heldscal.la/user/29191", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -122,7 +122,7 @@ defmodule HttpRequestMock do "https://pawoo.net/.well-known/webfinger?resource=acct:https://pawoo.net/users/pekorino", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -148,7 +148,7 @@ defmodule HttpRequestMock do "https://social.stopwatchingus-heidelberg.de/.well-known/webfinger?resource=acct:https://social.stopwatchingus-heidelberg.de/user/18330", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -169,7 +169,7 @@ defmodule HttpRequestMock do "https://mamot.fr/.well-known/webfinger?resource=acct:https://mamot.fr/users/Skruyb", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -182,7 +182,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=nonexistant@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -195,7 +195,7 @@ defmodule HttpRequestMock do "https://squeet.me/xrd/?uri=lain@squeet.me", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -208,7 +208,7 @@ defmodule HttpRequestMock do "https://mst3k.interlinked.me/users/luciferMysticus", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -229,7 +229,7 @@ defmodule HttpRequestMock do "https://hubzilla.example.org/channel/kaniini", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -238,7 +238,7 @@ defmodule HttpRequestMock do }} end - def get("https://niu.moe/users/rye", _, _, Accept: "application/activity+json") do + def get("https://niu.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -246,7 +246,7 @@ defmodule HttpRequestMock do }} end - def get("https://n1u.moe/users/rye", _, _, Accept: "application/activity+json") do + def get("https://n1u.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -265,7 +265,7 @@ defmodule HttpRequestMock do }} end - def get("https://puckipedia.com/", _, _, Accept: "application/activity+json") do + def get("https://puckipedia.com/", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -321,7 +321,9 @@ defmodule HttpRequestMock do }} end - def get("http://mastodon.example.org/users/admin", _, _, Accept: "application/activity+json") do + def get("http://mastodon.example.org/users/admin", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -329,7 +331,9 @@ defmodule HttpRequestMock do }} end - def get("http://mastodon.example.org/users/gargron", _, _, Accept: "application/activity+json") do + def get("http://mastodon.example.org/users/gargron", _, _, [ + {"accept", "application/activity+json"} + ]) do {:error, :nxdomain} end @@ -337,7 +341,7 @@ defmodule HttpRequestMock do "http://mastodon.example.org/@admin/99541947525187367", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -354,7 +358,7 @@ defmodule HttpRequestMock do }} end - def get("https://mstdn.io/users/mayuutann", _, _, Accept: "application/activity+json") do + def get("https://mstdn.io/users/mayuutann", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -366,7 +370,7 @@ defmodule HttpRequestMock do "https://mstdn.io/users/mayuutann/statuses/99568293732299394", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -386,7 +390,7 @@ defmodule HttpRequestMock do }} end - def get(url, _, _, Accept: "application/xrd+xml,application/jrd+json") + def get(url, _, _, [{"accept", "application/xrd+xml,application/jrd+json"}]) when url in [ "https://pleroma.soykaf.com/.well-known/webfinger?resource=acct:https://pleroma.soykaf.com/users/lain", "https://pleroma.soykaf.com/.well-known/webfinger?resource=https://pleroma.soykaf.com/users/lain" @@ -413,7 +417,7 @@ defmodule HttpRequestMock do "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/1", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -457,7 +461,7 @@ defmodule HttpRequestMock do "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/5381", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -510,7 +514,7 @@ defmodule HttpRequestMock do "https://social.sakamoto.gq/.well-known/webfinger?resource=https://social.sakamoto.gq/users/eal", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -523,7 +527,7 @@ defmodule HttpRequestMock do "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056", _, _, - Accept: "application/atom+xml" + [{"accept", "application/atom+xml"}] ) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sakamoto.atom")}} end @@ -540,7 +544,7 @@ defmodule HttpRequestMock do "https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/lambadalambda", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -562,7 +566,7 @@ defmodule HttpRequestMock do "http://gs.example.org/.well-known/webfinger?resource=http://gs.example.org:4040/index.php/user/1", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -576,7 +580,7 @@ defmodule HttpRequestMock do "http://gs.example.org:4040/index.php/user/1", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{status: 406, body: ""}} end @@ -612,7 +616,7 @@ defmodule HttpRequestMock do "https://squeet.me/xrd?uri=lain@squeet.me", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -625,7 +629,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=shp@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -638,7 +642,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=invalid_content@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{status: 200, body: ""}} end @@ -655,7 +659,7 @@ defmodule HttpRequestMock do "http://framatube.org/main/xrd?uri=framasoft@framatube.org", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -677,7 +681,7 @@ defmodule HttpRequestMock do "http://gnusocial.de/main/xrd?uri=winterdienst@gnusocial.de", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -714,7 +718,7 @@ defmodule HttpRequestMock do "https://gerzilla.de/xrd/?uri=kaniini@gerzilla.de", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -772,7 +776,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/lambadalambda.json")}} end - def get("https://social.heldscal.la/user/23211", _, _, Accept: "application/activity+json") do + def get("https://social.heldscal.la/user/23211", _, _, [{"accept", "application/activity+json"}]) do {:ok, Tesla.Mock.json(%{"id" => "https://social.heldscal.la/user/23211"}, status: 200)} end @@ -889,7 +893,7 @@ defmodule HttpRequestMock do "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=lain@zetsubou.xn--q9jyb4c", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -902,7 +906,7 @@ defmodule HttpRequestMock do "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=https://zetsubou.xn--q9jyb4c/users/lain", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -924,7 +928,9 @@ defmodule HttpRequestMock do }} end - def get("https://info.pleroma.site/activity.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -936,7 +942,9 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 404, body: ""}} end - def get("https://info.pleroma.site/activity2.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity2.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -948,7 +956,9 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 404, body: ""}} end - def get("https://info.pleroma.site/activity3.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity3.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index fe4ffdb59..1061403f4 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do alias Pleroma.Web.CommonAPI import Pleroma.Factory import Mock + import ExUnit.CaptureLog setup do Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -367,15 +368,18 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do assert html_response(response, 200) =~ "Remote follow" end - test "show follow page with error when user cannot fecth by `acct` link", %{conn: conn} do + test "show follow page with error when user cannot fetch by `acct` link", %{conn: conn} do user = insert(:user) - response = - conn - |> assign(:user, user) - |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") + assert capture_log(fn -> + response = + conn + |> assign(:user, user) + |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") - assert html_response(response, 200) =~ "Error fetching user" + assert html_response(response, 200) =~ "Error fetching user" + end) =~ + "Could not decode user at fetch https://mastodon.social/users/not_found, {:error, \"Object has been deleted\"}" end end diff --git a/test/web/web_finger/web_finger_controller_test.exs b/test/web/web_finger/web_finger_controller_test.exs index e23086b2a..f940f95b6 100644 --- a/test/web/web_finger/web_finger_controller_test.exs +++ b/test/web/web_finger/web_finger_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do import Pleroma.Factory import Tesla.Mock + import ExUnit.CaptureLog setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -75,11 +76,13 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do test "Sends a 404 when invalid format" do user = insert(:user) - assert_raise Phoenix.NotAcceptableError, fn -> - build_conn() - |> put_req_header("accept", "text/html") - |> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost") - end + assert capture_log(fn -> + assert_raise Phoenix.NotAcceptableError, fn -> + build_conn() + |> put_req_header("accept", "text/html") + |> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost") + end + end) =~ "Internal server error:" end test "Sends a 400 when resource param is missing" do -- cgit v1.2.3 From 4e9d9209c3c598532e1eaacb35e276357906afb4 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 8 Aug 2019 18:42:50 +0300 Subject: added tesla client for reverse proxy --- lib/pleroma/http/connection.ex | 6 +--- lib/pleroma/reverse_proxy/client.ex | 17 ++++++---- lib/pleroma/reverse_proxy/client/hackney.ex | 17 ++++++++++ lib/pleroma/reverse_proxy/client/tesla.ex | 48 +++++++++++++++++++++++++++++ lib/pleroma/reverse_proxy/reverse_proxy.ex | 10 +++--- mix.exs | 2 +- mix.lock | 2 +- test/reverse_proxy_test.exs | 44 ++++++++++++++++++++++---- 8 files changed, 122 insertions(+), 24 deletions(-) create mode 100644 lib/pleroma/reverse_proxy/client/hackney.ex create mode 100644 lib/pleroma/reverse_proxy/client/tesla.ex diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 8caf989a7..4ebe16e18 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -10,11 +10,7 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, protocols: [:http], - timeout: 20_000, - recv_timeout: 20_000, - follow_redirect: true, - force_redirect: true, - pool: :federation + timeout: 20_000 ] @adapter Application.get_env(:tesla, :adapter) diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex index 776c4794c..42f2ff13b 100644 --- a/lib/pleroma/reverse_proxy/client.ex +++ b/lib/pleroma/reverse_proxy/client.ex @@ -3,9 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client do - @callback request(atom(), String.t(), [tuple()], String.t(), list()) :: - {:ok, pos_integer(), [tuple()], reference() | map()} - | {:ok, pos_integer(), [tuple()]} + @type status :: pos_integer() + @type header_name :: String.t() + @type header_value :: String.t() + @type headers :: [{header_name(), header_value()}] + + @callback request(atom(), String.t(), headers(), String.t(), list()) :: + {:ok, status(), headers(), reference() | map()} + | {:ok, status(), headers()} | {:ok, reference()} | {:error, term()} @@ -14,8 +19,8 @@ defmodule Pleroma.ReverseProxy.Client do @callback close(reference() | pid() | map()) :: :ok - def request(method, url, headers, "", opts \\ []) do - client().request(method, url, headers, "", opts) + def request(method, url, headers, body \\ "", opts \\ []) do + client().request(method, url, headers, body, opts) end def stream_body(ref), do: client().stream_body(ref) @@ -23,6 +28,6 @@ defmodule Pleroma.ReverseProxy.Client do def close(ref), do: client().close(ref) defp client do - Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney) + Pleroma.Config.get([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Hackney) end end diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex new file mode 100644 index 000000000..e6293646a --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -0,0 +1,17 @@ +defmodule Pleroma.ReverseProxy.Client.Hackney do + @behaviour Pleroma.ReverseProxy.Client + + def request(method, url, headers, body, opts \\ []) do + :hackney.request(method, url, headers, body, opts) + end + + def stream_body(ref) do + case :hackney.stream_body(ref) do + :done -> :done + {:ok, data} -> {:ok, data, ref} + {:error, error} -> {:error, error} + end + end + + def close(ref), do: :hackney.close(ref) +end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex new file mode 100644 index 000000000..d2944b9dc --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -0,0 +1,48 @@ +defmodule Pleroma.ReverseProxy.Client.Tesla do + @behaviour Pleroma.ReverseProxy.Client + + @adapters [Tesla.Adapter.Gun] + alias Pleroma.HTTP + + def request(method, url, headers, body, opts \\ []) do + adapter_opts = + Keyword.get(opts, :adapter, []) + |> Keyword.put(:chunks_response, true) + + with {:ok, response} <- + HTTP.request(method, url, body, headers, Keyword.put(opts, :adapter, adapter_opts)) do + {:ok, response.status, response.headers, response.body} + else + {:error, error} -> {:error, error} + end + end + + def stream_body(%{fin: true}), do: :done + + def stream_body(client) do + case read_chunk!(client) do + {:fin, body} -> {:ok, body, Map.put(client, :fin, true)} + {:nofin, part} -> {:ok, part, client} + end + end + + defp read_chunk!(client) do + adapter = Application.get_env(:tesla, :adapter) + + unless adapter in @adapters do + raise "#{adapter} doesn't support reading body in chunks" + end + + adapter.read_chunk(client) + end + + def close(client) do + adapter = Application.get_env(:tesla, :adapter) + + unless adapter in @adapters do + raise "#{adapter} doesn't support closing connection" + end + + adapter.close(client) + end +end diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 3212bf90d..a2cdcf393 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -61,7 +61,7 @@ defmodule Pleroma.ReverseProxy do * `http`: options for [hackney](https://github.com/benoitc/hackney). """ - @default_hackney_options [pool: :media] + @default_options [pool: :media] @inline_content_types [ "image/gif", @@ -93,9 +93,9 @@ defmodule Pleroma.ReverseProxy do def call(_conn, _url, _opts \\ []) def call(conn = %{method: method}, url, opts) when method in @methods do - hackney_opts = + client_opts = Pleroma.HTTP.Connection.options([]) - |> Keyword.merge(@default_hackney_options) + |> Keyword.merge(@default_options) |> Keyword.merge(Keyword.get(opts, :http, [])) |> HTTP.process_request_options() @@ -108,7 +108,7 @@ defmodule Pleroma.ReverseProxy do opts end - with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), + with {:ok, code, headers, client} <- request(method, url, req_headers, client_opts), :ok <- header_length_constraint( headers, @@ -201,7 +201,7 @@ defmodule Pleroma.ReverseProxy do duration, Keyword.get(opts, :max_read_duration, @max_read_duration) ), - {:ok, data} <- client().stream_body(client), + {:ok, data, client} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), :ok <- diff --git a/mix.exs b/mix.exs index 84d58f5a6..f1fbdc6b3 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do {:poison, "~> 3.0", override: true}, {:tesla, github: "alex-strizhakov/tesla", - ref: "9ad792fb630bdfc2266ed13b830c28b6552fb3f9", + ref: "beb8927358dfaa66ecd458df607befde12dd56e0", override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index d79bb9c80..635f20185 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "9ad792fb630bdfc2266ed13b830c28b6552fb3f9", [ref: "9ad792fb630bdfc2266ed13b830c28b6552fb3f9"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "beb8927358dfaa66ecd458df607befde12dd56e0", [ref: "beb8927358dfaa66ecd458df607befde12dd56e0"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/reverse_proxy_test.exs b/test/reverse_proxy_test.exs index 3a83c4c48..0644d1ed9 100644 --- a/test/reverse_proxy_test.exs +++ b/test/reverse_proxy_test.exs @@ -29,11 +29,11 @@ defmodule Pleroma.ReverseProxyTest do {"content-length", byte_size(json) |> to_string()} ], %{url: url}} end) - |> expect(:stream_body, invokes, fn %{url: url} -> + |> expect(:stream_body, invokes, fn %{url: url} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) - {:ok, json} + {:ok, json, client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) @@ -66,6 +66,38 @@ defmodule Pleroma.ReverseProxyTest do assert conn.halted end + defp stream_mock(invokes, with_close? \\ false) do + ClientMock + |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) + + {:ok, 200, [{"content-type", "application/octet-stream"}], + %{url: "/stream-bytes/" <> length}} + end) + |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} = client -> + max = String.to_integer(length) + + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do + [{_, current}] when current < max -> + Registry.update_value( + Pleroma.ReverseProxy.ClientMock, + "/stream-bytes/" <> length, + &(&1 + 10) + ) + + {:ok, "0123456789", client} + + [{_, ^max}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) + :done + end + end) + + if with_close? do + expect(ClientMock, :close, fn _ -> :ok end) + end + end + describe "max_body " do test "length returns error if content-length more than option", %{conn: conn} do user_agent_mock("hackney/1.15.1", 0) @@ -179,12 +211,12 @@ defmodule Pleroma.ReverseProxyTest do Registry.register(Pleroma.ReverseProxy.ClientMock, "/headers", 0) {:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}} end) - |> expect(:stream_body, 2, fn %{url: url, headers: headers} -> + |> expect(:stream_body, 2, fn %{url: url, headers: headers} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v} - {:ok, Jason.encode!(%{headers: headers})} + {:ok, Jason.encode!(%{headers: headers}), client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) @@ -261,11 +293,11 @@ defmodule Pleroma.ReverseProxyTest do {:ok, 200, headers, %{url: "/disposition"}} end) - |> expect(:stream_body, 2, fn %{url: "/disposition"} -> + |> expect(:stream_body, 2, fn %{url: "/disposition"} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/disposition") do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, "/disposition", &(&1 + 1)) - {:ok, ""} + {:ok, "", client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/disposition") -- cgit v1.2.3 From 80ddc2428199e4ea5638ae9f080faceacd5c15fd Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 8 Aug 2019 20:08:10 +0300 Subject: fix --- lib/pleroma/reverse_proxy/client/tesla.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index d2944b9dc..9b0be5bda 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -26,14 +26,14 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do end end - defp read_chunk!(client) do + defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do adapter = Application.get_env(:tesla, :adapter) unless adapter in @adapters do raise "#{adapter} doesn't support reading body in chunks" end - adapter.read_chunk(client) + adapter.read_chunk(pid, stream, opts) end def close(client) do -- cgit v1.2.3 From 02daf21d786c8823efcfeec37420f6b3a1790335 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 9 Aug 2019 12:24:43 +0300 Subject: possibility to set tesla adapter in runtime --- lib/pleroma/http/connection.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 4ebe16e18..ff03a3ee3 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -12,7 +12,6 @@ defmodule Pleroma.HTTP.Connection do protocols: [:http], timeout: 20_000 ] - @adapter Application.get_env(:tesla, :adapter) @doc """ Configure a client connection @@ -24,7 +23,8 @@ defmodule Pleroma.HTTP.Connection do @spec new(Keyword.t()) :: Tesla.Env.client() def new(opts \\ []) do middleware = [Tesla.Middleware.FollowRedirects] - Tesla.client(middleware, {@adapter, options(opts)}) + adapter = Application.get_env(:tesla, :adapter) + Tesla.client(middleware, {adapter, options(opts)}) end # fetch http options -- cgit v1.2.3 From c51aa48e60103307307c7c40c2d046719def7054 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 9 Aug 2019 12:25:58 +0300 Subject: reverse proxy clients integration tests tesla reverse proxy client fixes --- lib/pleroma/reverse_proxy/client/tesla.ex | 14 +- test/reverse_proxy/client/hackney_test.exs | 7 + test/reverse_proxy/client/tesla_test.exs | 9 + test/reverse_proxy/reverse_proxy_test.exs | 336 +++++++++++++++++++++++++++++ test/reverse_proxy_test.exs | 332 ---------------------------- test/support/reverse_proxy_client_case.ex | 76 +++++++ 6 files changed, 439 insertions(+), 335 deletions(-) create mode 100644 test/reverse_proxy/client/hackney_test.exs create mode 100644 test/reverse_proxy/client/tesla_test.exs create mode 100644 test/reverse_proxy/reverse_proxy_test.exs delete mode 100644 test/reverse_proxy_test.exs create mode 100644 test/support/reverse_proxy_client_case.ex diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 9b0be5bda..b1498a5a4 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -2,7 +2,6 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @behaviour Pleroma.ReverseProxy.Client @adapters [Tesla.Adapter.Gun] - alias Pleroma.HTTP def request(method, url, headers, body, opts \\ []) do adapter_opts = @@ -10,8 +9,16 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do |> Keyword.put(:chunks_response, true) with {:ok, response} <- - HTTP.request(method, url, body, headers, Keyword.put(opts, :adapter, adapter_opts)) do - {:ok, response.status, response.headers, response.body} + Pleroma.HTTP.request( + method, + url, + body, + headers, + Keyword.put(opts, :adapter, adapter_opts) + ) do + if is_map(response.body), + do: {:ok, response.status, response.headers, response.body}, + else: {:ok, response.status, response.headers} else {:error, error} -> {:error, error} end @@ -23,6 +30,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do case read_chunk!(client) do {:fin, body} -> {:ok, body, Map.put(client, :fin, true)} {:nofin, part} -> {:ok, part, client} + {:error, error} -> {:error, error} end end diff --git a/test/reverse_proxy/client/hackney_test.exs b/test/reverse_proxy/client/hackney_test.exs new file mode 100644 index 000000000..577e0b0b2 --- /dev/null +++ b/test/reverse_proxy/client/hackney_test.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.ReverseProxy.Client.HackneyTest do + use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Hackney + + defp check_ref(ref) do + assert is_reference(ref) + end +end diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs new file mode 100644 index 000000000..029a25d0f --- /dev/null +++ b/test/reverse_proxy/client/tesla_test.exs @@ -0,0 +1,9 @@ +defmodule Pleroma.ReverseProxy.Client.TeslaTest do + use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Tesla + + defp check_ref(%{pid: pid, stream: stream} = ref) do + assert is_pid(pid) + assert is_reference(stream) + assert ref[:fin] + end +end diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs new file mode 100644 index 000000000..95adae666 --- /dev/null +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -0,0 +1,336 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReverseProxyTest do + use Pleroma.Web.ConnCase, async: true + import ExUnit.CaptureLog + import Mox + alias Pleroma.ReverseProxy + alias Pleroma.ReverseProxy.ClientMock + + setup_all do + {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.ReverseProxy.ClientMock) + :ok + end + + setup :verify_on_exit! + + defp user_agent_mock(user_agent, invokes) do + json = Jason.encode!(%{"user-agent": user_agent}) + + ClientMock + |> expect(:request, fn :get, url, _, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, url, 0) + + {:ok, 200, + [ + {"content-type", "application/json"}, + {"content-length", byte_size(json) |> to_string()} + ], %{url: url}} + end) + |> expect(:stream_body, invokes, fn %{url: url} = client -> + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do + [{_, 0}] -> + Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) + {:ok, json, client} + + [{_, 1}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) + :done + end + end) + end + + describe "user-agent" do + test "don't keep", %{conn: conn} do + user_agent_mock("hackney/1.15.1", 2) + conn = ReverseProxy.call(conn, "/user-agent") + assert json_response(conn, 200) == %{"user-agent" => "hackney/1.15.1"} + end + + test "keep", %{conn: conn} do + user_agent_mock(Pleroma.Application.user_agent(), 2) + conn = ReverseProxy.call(conn, "/user-agent-keep", keep_user_agent: true) + assert json_response(conn, 200) == %{"user-agent" => Pleroma.Application.user_agent()} + end + end + + test "closed connection", %{conn: conn} do + ClientMock + |> expect(:request, fn :get, "/closed", _, _, _ -> {:ok, 200, [], %{}} end) + |> expect(:stream_body, fn _ -> {:error, :closed} end) + |> expect(:close, fn _ -> :ok end) + + conn = ReverseProxy.call(conn, "/closed") + assert conn.halted + end + + defp stream_mock(invokes, with_close? \\ false) do + ClientMock + |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) + + {:ok, 200, [{"content-type", "application/octet-stream"}], + %{url: "/stream-bytes/" <> length}} + end) + |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} = client -> + max = String.to_integer(length) + + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do + [{_, current}] when current < max -> + Registry.update_value( + Pleroma.ReverseProxy.ClientMock, + "/stream-bytes/" <> length, + &(&1 + 10) + ) + + {:ok, "0123456789", client} + + [{_, ^max}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) + :done + end + end) + + if with_close? do + expect(ClientMock, :close, fn _ -> :ok end) + end + end + + describe "max_body " do + test "length returns error if content-length more than option", %{conn: conn} do + user_agent_mock("hackney/1.15.1", 0) + + assert capture_log(fn -> + ReverseProxy.call(conn, "/user-agent", max_body_length: 4) + end) =~ + "[error] Elixir.Pleroma.ReverseProxy: request to \"/user-agent\" failed: :body_too_large" + end + + defp stream_mock(invokes, with_close? \\ false) do + ClientMock + |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) + + {:ok, 200, [{"content-type", "application/octet-stream"}], + %{url: "/stream-bytes/" <> length}} + end) + |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} -> + max = String.to_integer(length) + + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do + [{_, current}] when current < max -> + Registry.update_value( + Pleroma.ReverseProxy.ClientMock, + "/stream-bytes/" <> length, + &(&1 + 10) + ) + + {:ok, "0123456789"} + + [{_, ^max}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) + :done + end + end) + + if with_close? do + expect(ClientMock, :close, fn _ -> :ok end) + end + end + + test "max_body_length returns error if streaming body more than that option", %{conn: conn} do + stream_mock(3, true) + + assert capture_log(fn -> + ReverseProxy.call(conn, "/stream-bytes/50", max_body_length: 30) + end) =~ + "[warn] Elixir.Pleroma.ReverseProxy request to /stream-bytes/50 failed while reading/chunking: :body_too_large" + end + end + + describe "HEAD requests" do + test "common", %{conn: conn} do + ClientMock + |> expect(:request, fn :head, "/head", _, _, _ -> + {:ok, 200, [{"content-type", "text/html; charset=utf-8"}]} + end) + + conn = ReverseProxy.call(Map.put(conn, :method, "HEAD"), "/head") + assert html_response(conn, 200) == "" + end + end + + defp error_mock(status) when is_integer(status) do + ClientMock + |> expect(:request, fn :get, "/status/" <> _, _, _, _ -> + {:error, status} + end) + end + + describe "returns error on" do + test "500", %{conn: conn} do + error_mock(500) + + capture_log(fn -> ReverseProxy.call(conn, "/status/500") end) =~ + "[error] Elixir.Pleroma.ReverseProxy: request to /status/500 failed with HTTP status 500" + end + + test "400", %{conn: conn} do + error_mock(400) + + capture_log(fn -> ReverseProxy.call(conn, "/status/400") end) =~ + "[error] Elixir.Pleroma.ReverseProxy: request to /status/400 failed with HTTP status 400" + end + + test "204", %{conn: conn} do + ClientMock + |> expect(:request, fn :get, "/status/204", _, _, _ -> {:ok, 204, [], %{}} end) + + capture_log(fn -> + conn = ReverseProxy.call(conn, "/status/204") + assert conn.resp_body == "Request failed: No Content" + assert conn.halted + end) =~ + "[error] Elixir.Pleroma.ReverseProxy: request to \"/status/204\" failed with HTTP status 204" + end + end + + test "streaming", %{conn: conn} do + stream_mock(21) + conn = ReverseProxy.call(conn, "/stream-bytes/200") + assert conn.state == :chunked + assert byte_size(conn.resp_body) == 200 + assert Plug.Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"] + end + + defp headers_mock(_) do + ClientMock + |> expect(:request, fn :get, "/headers", headers, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, "/headers", 0) + {:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}} + end) + |> expect(:stream_body, 2, fn %{url: url, headers: headers} = client -> + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do + [{_, 0}] -> + Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) + headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v} + {:ok, Jason.encode!(%{headers: headers}), client} + + [{_, 1}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) + :done + end + end) + + :ok + end + + describe "keep request headers" do + setup [:headers_mock] + + test "header passes", %{conn: conn} do + conn = + Plug.Conn.put_req_header( + conn, + "accept", + "text/html" + ) + |> ReverseProxy.call("/headers") + + %{"headers" => headers} = json_response(conn, 200) + assert headers["Accept"] == "text/html" + end + + test "header is filtered", %{conn: conn} do + conn = + Plug.Conn.put_req_header( + conn, + "accept-language", + "en-US" + ) + |> ReverseProxy.call("/headers") + + %{"headers" => headers} = json_response(conn, 200) + refute headers["Accept-Language"] + end + end + + test "returns 400 on non GET, HEAD requests", %{conn: conn} do + conn = ReverseProxy.call(Map.put(conn, :method, "POST"), "/ip") + assert conn.status == 400 + end + + describe "cache resp headers" do + test "returns headers", %{conn: conn} do + ClientMock + |> expect(:request, fn :get, "/cache/" <> ttl, _, _, _ -> + {:ok, 200, [{"cache-control", "public, max-age=" <> ttl}], %{}} + end) + |> expect(:stream_body, fn _ -> :done end) + + conn = ReverseProxy.call(conn, "/cache/10") + assert {"cache-control", "public, max-age=10"} in conn.resp_headers + end + + test "add cache-control", %{conn: conn} do + ClientMock + |> expect(:request, fn :get, "/cache", _, _, _ -> + {:ok, 200, [{"ETag", "some ETag"}], %{}} + end) + |> expect(:stream_body, fn _ -> :done end) + + conn = ReverseProxy.call(conn, "/cache") + assert {"cache-control", "public"} in conn.resp_headers + end + end + + defp disposition_headers_mock(headers) do + ClientMock + |> expect(:request, fn :get, "/disposition", _, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, "/disposition", 0) + + {:ok, 200, headers, %{url: "/disposition"}} + end) + |> expect(:stream_body, 2, fn %{url: "/disposition"} = client -> + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/disposition") do + [{_, 0}] -> + Registry.update_value(Pleroma.ReverseProxy.ClientMock, "/disposition", &(&1 + 1)) + {:ok, "", client} + + [{_, 1}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/disposition") + :done + end + end) + end + + describe "response content disposition header" do + test "not atachment", %{conn: conn} do + disposition_headers_mock([ + {"content-type", "image/gif"}, + {"content-length", 0} + ]) + + conn = ReverseProxy.call(conn, "/disposition") + + assert {"content-type", "image/gif"} in conn.resp_headers + end + + test "with content-disposition header", %{conn: conn} do + disposition_headers_mock([ + {"content-disposition", "attachment; filename=\"filename.jpg\""}, + {"content-length", 0} + ]) + + conn = ReverseProxy.call(conn, "/disposition") + + assert {"content-disposition", "attachment; filename=\"filename.jpg\""} in conn.resp_headers + end + end + + describe "integration tests" do + @describetag :integration + end +end diff --git a/test/reverse_proxy_test.exs b/test/reverse_proxy_test.exs deleted file mode 100644 index 0644d1ed9..000000000 --- a/test/reverse_proxy_test.exs +++ /dev/null @@ -1,332 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ReverseProxyTest do - use Pleroma.Web.ConnCase, async: true - import ExUnit.CaptureLog - import Mox - alias Pleroma.ReverseProxy - alias Pleroma.ReverseProxy.ClientMock - - setup_all do - {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.ReverseProxy.ClientMock) - :ok - end - - setup :verify_on_exit! - - defp user_agent_mock(user_agent, invokes) do - json = Jason.encode!(%{"user-agent": user_agent}) - - ClientMock - |> expect(:request, fn :get, url, _, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, url, 0) - - {:ok, 200, - [ - {"content-type", "application/json"}, - {"content-length", byte_size(json) |> to_string()} - ], %{url: url}} - end) - |> expect(:stream_body, invokes, fn %{url: url} = client -> - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do - [{_, 0}] -> - Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) - {:ok, json, client} - - [{_, 1}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) - :done - end - end) - end - - describe "user-agent" do - test "don't keep", %{conn: conn} do - user_agent_mock("hackney/1.15.1", 2) - conn = ReverseProxy.call(conn, "/user-agent") - assert json_response(conn, 200) == %{"user-agent" => "hackney/1.15.1"} - end - - test "keep", %{conn: conn} do - user_agent_mock(Pleroma.Application.user_agent(), 2) - conn = ReverseProxy.call(conn, "/user-agent-keep", keep_user_agent: true) - assert json_response(conn, 200) == %{"user-agent" => Pleroma.Application.user_agent()} - end - end - - test "closed connection", %{conn: conn} do - ClientMock - |> expect(:request, fn :get, "/closed", _, _, _ -> {:ok, 200, [], %{}} end) - |> expect(:stream_body, fn _ -> {:error, :closed} end) - |> expect(:close, fn _ -> :ok end) - - conn = ReverseProxy.call(conn, "/closed") - assert conn.halted - end - - defp stream_mock(invokes, with_close? \\ false) do - ClientMock - |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) - - {:ok, 200, [{"content-type", "application/octet-stream"}], - %{url: "/stream-bytes/" <> length}} - end) - |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} = client -> - max = String.to_integer(length) - - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do - [{_, current}] when current < max -> - Registry.update_value( - Pleroma.ReverseProxy.ClientMock, - "/stream-bytes/" <> length, - &(&1 + 10) - ) - - {:ok, "0123456789", client} - - [{_, ^max}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) - :done - end - end) - - if with_close? do - expect(ClientMock, :close, fn _ -> :ok end) - end - end - - describe "max_body " do - test "length returns error if content-length more than option", %{conn: conn} do - user_agent_mock("hackney/1.15.1", 0) - - assert capture_log(fn -> - ReverseProxy.call(conn, "/user-agent", max_body_length: 4) - end) =~ - "[error] Elixir.Pleroma.ReverseProxy: request to \"/user-agent\" failed: :body_too_large" - end - - defp stream_mock(invokes, with_close? \\ false) do - ClientMock - |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) - - {:ok, 200, [{"content-type", "application/octet-stream"}], - %{url: "/stream-bytes/" <> length}} - end) - |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} -> - max = String.to_integer(length) - - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do - [{_, current}] when current < max -> - Registry.update_value( - Pleroma.ReverseProxy.ClientMock, - "/stream-bytes/" <> length, - &(&1 + 10) - ) - - {:ok, "0123456789"} - - [{_, ^max}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) - :done - end - end) - - if with_close? do - expect(ClientMock, :close, fn _ -> :ok end) - end - end - - test "max_body_length returns error if streaming body more than that option", %{conn: conn} do - stream_mock(3, true) - - assert capture_log(fn -> - ReverseProxy.call(conn, "/stream-bytes/50", max_body_length: 30) - end) =~ - "[warn] Elixir.Pleroma.ReverseProxy request to /stream-bytes/50 failed while reading/chunking: :body_too_large" - end - end - - describe "HEAD requests" do - test "common", %{conn: conn} do - ClientMock - |> expect(:request, fn :head, "/head", _, _, _ -> - {:ok, 200, [{"content-type", "text/html; charset=utf-8"}]} - end) - - conn = ReverseProxy.call(Map.put(conn, :method, "HEAD"), "/head") - assert html_response(conn, 200) == "" - end - end - - defp error_mock(status) when is_integer(status) do - ClientMock - |> expect(:request, fn :get, "/status/" <> _, _, _, _ -> - {:error, status} - end) - end - - describe "returns error on" do - test "500", %{conn: conn} do - error_mock(500) - - capture_log(fn -> ReverseProxy.call(conn, "/status/500") end) =~ - "[error] Elixir.Pleroma.ReverseProxy: request to /status/500 failed with HTTP status 500" - end - - test "400", %{conn: conn} do - error_mock(400) - - capture_log(fn -> ReverseProxy.call(conn, "/status/400") end) =~ - "[error] Elixir.Pleroma.ReverseProxy: request to /status/400 failed with HTTP status 400" - end - - test "204", %{conn: conn} do - ClientMock - |> expect(:request, fn :get, "/status/204", _, _, _ -> {:ok, 204, [], %{}} end) - - capture_log(fn -> - conn = ReverseProxy.call(conn, "/status/204") - assert conn.resp_body == "Request failed: No Content" - assert conn.halted - end) =~ - "[error] Elixir.Pleroma.ReverseProxy: request to \"/status/204\" failed with HTTP status 204" - end - end - - test "streaming", %{conn: conn} do - stream_mock(21) - conn = ReverseProxy.call(conn, "/stream-bytes/200") - assert conn.state == :chunked - assert byte_size(conn.resp_body) == 200 - assert Plug.Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"] - end - - defp headers_mock(_) do - ClientMock - |> expect(:request, fn :get, "/headers", headers, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, "/headers", 0) - {:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}} - end) - |> expect(:stream_body, 2, fn %{url: url, headers: headers} = client -> - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do - [{_, 0}] -> - Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) - headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v} - {:ok, Jason.encode!(%{headers: headers}), client} - - [{_, 1}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) - :done - end - end) - - :ok - end - - describe "keep request headers" do - setup [:headers_mock] - - test "header passes", %{conn: conn} do - conn = - Plug.Conn.put_req_header( - conn, - "accept", - "text/html" - ) - |> ReverseProxy.call("/headers") - - %{"headers" => headers} = json_response(conn, 200) - assert headers["Accept"] == "text/html" - end - - test "header is filtered", %{conn: conn} do - conn = - Plug.Conn.put_req_header( - conn, - "accept-language", - "en-US" - ) - |> ReverseProxy.call("/headers") - - %{"headers" => headers} = json_response(conn, 200) - refute headers["Accept-Language"] - end - end - - test "returns 400 on non GET, HEAD requests", %{conn: conn} do - conn = ReverseProxy.call(Map.put(conn, :method, "POST"), "/ip") - assert conn.status == 400 - end - - describe "cache resp headers" do - test "returns headers", %{conn: conn} do - ClientMock - |> expect(:request, fn :get, "/cache/" <> ttl, _, _, _ -> - {:ok, 200, [{"cache-control", "public, max-age=" <> ttl}], %{}} - end) - |> expect(:stream_body, fn _ -> :done end) - - conn = ReverseProxy.call(conn, "/cache/10") - assert {"cache-control", "public, max-age=10"} in conn.resp_headers - end - - test "add cache-control", %{conn: conn} do - ClientMock - |> expect(:request, fn :get, "/cache", _, _, _ -> - {:ok, 200, [{"ETag", "some ETag"}], %{}} - end) - |> expect(:stream_body, fn _ -> :done end) - - conn = ReverseProxy.call(conn, "/cache") - assert {"cache-control", "public"} in conn.resp_headers - end - end - - defp disposition_headers_mock(headers) do - ClientMock - |> expect(:request, fn :get, "/disposition", _, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, "/disposition", 0) - - {:ok, 200, headers, %{url: "/disposition"}} - end) - |> expect(:stream_body, 2, fn %{url: "/disposition"} = client -> - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/disposition") do - [{_, 0}] -> - Registry.update_value(Pleroma.ReverseProxy.ClientMock, "/disposition", &(&1 + 1)) - {:ok, "", client} - - [{_, 1}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/disposition") - :done - end - end) - end - - describe "response content disposition header" do - test "not atachment", %{conn: conn} do - disposition_headers_mock([ - {"content-type", "image/gif"}, - {"content-length", 0} - ]) - - conn = ReverseProxy.call(conn, "/disposition") - - assert {"content-type", "image/gif"} in conn.resp_headers - end - - test "with content-disposition header", %{conn: conn} do - disposition_headers_mock([ - {"content-disposition", "attachment; filename=\"filename.jpg\""}, - {"content-length", 0} - ]) - - conn = ReverseProxy.call(conn, "/disposition") - - assert {"content-disposition", "attachment; filename=\"filename.jpg\""} in conn.resp_headers - end - end -end diff --git a/test/support/reverse_proxy_client_case.ex b/test/support/reverse_proxy_client_case.ex new file mode 100644 index 000000000..40cd59ea2 --- /dev/null +++ b/test/support/reverse_proxy_client_case.ex @@ -0,0 +1,76 @@ +defmodule Pleroma.ReverseProxyClientCase do + defmacro __using__(client: client) do + quote do + use ExUnit.Case + @moduletag :integration + @client unquote(client) + + setup do + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + on_exit(fn -> Application.put_env(:tesla, :adapter, Tesla.Mock) end) + end + + test "get response body stream" do + {:ok, status, headers, ref} = + @client.request( + :get, + "http://httpbin.org/stream-bytes/10", + [{"accept", "application/octet-stream"}], + "", + [] + ) + + assert status == 200 + assert headers != [] + + {:ok, response, ref} = @client.stream_body(ref) + check_ref(ref) + assert is_binary(response) + assert byte_size(response) == 10 + + assert :done == @client.stream_body(ref) + end + + test "head response" do + {:ok, status, headers} = @client.request(:head, "http://httpbin.org/get", [], "", []) + + assert status == 200 + assert headers != [] + end + + test "get error response" do + case @client.request( + :get, + "http://httpbin.org/status/500", + [], + "", + [] + ) do + {:ok, status, headers, ref} -> + assert status == 500 + assert headers != [] + check_ref(ref) + + assert :ok = @client.close(ref) + + {:ok, status, headers} -> + assert headers != [] + end + end + + test "head error response" do + {:ok, status, headers} = + @client.request( + :head, + "http://httpbin.org/status/500", + [], + "", + [] + ) + + assert status == 500 + assert headers != [] + end + end + end +end -- cgit v1.2.3 From 2caf9ad95424b0a4f47c1e22ce9d57a29fcf9fbb Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 13 Aug 2019 14:37:19 +0300 Subject: added gun connections genserver --- config/test.exs | 2 + lib/pleroma/gun/api/api.ex | 15 +++ lib/pleroma/gun/api/mock.ex | 40 +++++++ lib/pleroma/gun/conn.ex | 17 +++ lib/pleroma/gun/connections.ex | 104 ++++++++++++++++++ lib/pleroma/http/connection.ex | 1 - lib/pleroma/reverse_proxy/client.ex | 2 +- lib/pleroma/reverse_proxy/client/tesla.ex | 10 +- mix.exs | 2 +- mix.lock | 2 +- test/gun/connections_test.exs | 172 ++++++++++++++++++++++++++++++ test/reverse_proxy/reverse_proxy_test.exs | 33 ++++++ 12 files changed, 393 insertions(+), 7 deletions(-) create mode 100644 lib/pleroma/gun/api/api.ex create mode 100644 lib/pleroma/gun/api/mock.ex create mode 100644 lib/pleroma/gun/conn.ex create mode 100644 lib/pleroma/gun/connections.ex create mode 100644 test/gun/connections_test.exs diff --git a/config/test.exs b/config/test.exs index 6f75f39b5..ca916d59e 100644 --- a/config/test.exs +++ b/config/test.exs @@ -85,6 +85,8 @@ config :joken, default_signer: "yU8uHKq+yyAkZ11Hx//jcdacWc8yQ1bxAAGrplzB0Zwwjkp3 config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock +config :pleroma, Pleroma.Gun.API, Pleroma.Gun.API.Mock + try do import_config "test.secret.exs" rescue diff --git a/lib/pleroma/gun/api/api.ex b/lib/pleroma/gun/api/api.ex new file mode 100644 index 000000000..c69ab890e --- /dev/null +++ b/lib/pleroma/gun/api/api.ex @@ -0,0 +1,15 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API do + @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} + + def open(host, port, opts) do + api().open(host, port, opts) + end + + defp api do + Pleroma.Config.get([Pleroma.Gun.API], :gun) + end +end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex new file mode 100644 index 000000000..3348715c4 --- /dev/null +++ b/lib/pleroma/gun/api/mock.ex @@ -0,0 +1,40 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API.Mock do + @behaviour Pleroma.Gun.API + @impl Pleroma.Gun.API + def open('some-domain.com', 80, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + send(genserver_pid, {:gun_up, conn_pid, :http}) + {:ok, conn_pid} + end + + def open('some-domain.com', 443, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + send(genserver_pid, {:gun_up, conn_pid, :https}) + {:ok, conn_pid} + end + + @impl Pleroma.Gun.API + def open('gun_down.com', _port, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil}) + {:ok, conn_pid} + end + + @impl Pleroma.Gun.API + def open('gun_down_and_up.com', _port, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil}) + + {:ok, _} = + Task.start_link(fn -> + Process.sleep(500) + send(genserver_pid, {:gun_up, conn_pid, :http}) + end) + + {:ok, conn_pid} + end +end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex new file mode 100644 index 000000000..62ef146a1 --- /dev/null +++ b/lib/pleroma/gun/conn.ex @@ -0,0 +1,17 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.Conn do + @moduledoc """ + Struct for gun connection data + """ + @type t :: %__MODULE__{ + conn: pid(), + state: atom(), + waiting_pids: [pid()], + protocol: atom() + } + + defstruct conn: nil, state: :open, waiting_pids: [], protocol: :http +end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex new file mode 100644 index 000000000..60ec68d89 --- /dev/null +++ b/lib/pleroma/gun/connections.ex @@ -0,0 +1,104 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.Connections do + use GenServer + + @type domain :: String.t() + @type conn :: Gun.Conn.t() + @type t :: %__MODULE__{ + conns: %{domain() => conn()} + } + + defstruct conns: %{} + + def start_link(name \\ __MODULE__) do + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + GenServer.start_link(__MODULE__, [], name: name) + else + :ignore + end + end + + @impl true + def init(_) do + {:ok, %__MODULE__{conns: %{}}} + end + + @spec get_conn(atom(), String.t(), keyword()) :: pid() + def get_conn(name \\ __MODULE__, url, opts \\ []) do + opts = Enum.into(opts, %{}) + uri = URI.parse(url) + + opts = if uri.scheme == "https", do: Map.put(opts, :transport, :tls), else: opts + + GenServer.call( + name, + {:conn, %{opts: opts, uri: uri}} + ) + end + + @spec get_state(atom()) :: t() + def get_state(name \\ __MODULE__) do + GenServer.call(name, {:state}) + end + + @impl true + def handle_call({:conn, %{opts: opts, uri: uri}}, from, state) do + key = compose_key(uri) + + case state.conns[key] do + %{conn: conn, state: conn_state} when conn_state == :up -> + {:reply, conn, state} + + %{state: conn_state, waiting_pids: pids} when conn_state in [:open, :down] -> + state = put_in(state.conns[key].waiting_pids, [from | pids]) + {:noreply, state} + + nil -> + {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) + + state = + put_in(state.conns[key], %Pleroma.Gun.Conn{ + conn: conn, + waiting_pids: [from], + protocol: String.to_atom(uri.scheme) + }) + + {:noreply, state} + end + end + + @impl true + def handle_call({:state}, _from, state), do: {:reply, state, state} + + @impl true + def handle_info({:gun_up, conn_pid, protocol}, state) do + {key, conn} = find_conn(state.conns, conn_pid, protocol) + + # Send to all waiting processes connection pid + Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end) + + # Update state of the current connection and set waiting_pids to empty list + state = put_in(state.conns[key], %{conn | state: :up, waiting_pids: []}) + {:noreply, state} + end + + @impl true + # Do we need to do something with killed & unprocessed references? + def handle_info({:gun_down, conn_pid, protocol, _reason, _killed, _unprocessed}, state) do + {key, conn} = find_conn(state.conns, conn_pid, protocol) + + # We don't want to block requests to GenServer if gun send down message, return nil, so we can make some retries, while connection is not up + Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) + + state = put_in(state.conns[key].state, :down) + {:noreply, state} + end + + defp compose_key(uri), do: uri.host <> ":" <> to_string(uri.port) + + defp find_conn(conns, conn_pid, protocol), + do: Enum.find(conns, fn {_, conn} -> conn.conn == conn_pid and conn.protocol == protocol end) +end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index ff03a3ee3..6cb26c0fe 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -9,7 +9,6 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, - protocols: [:http], timeout: 20_000 ] diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex index 42f2ff13b..71c2b2911 100644 --- a/lib/pleroma/reverse_proxy/client.ex +++ b/lib/pleroma/reverse_proxy/client.ex @@ -28,6 +28,6 @@ defmodule Pleroma.ReverseProxy.Client do def close(ref), do: client().close(ref) defp client do - Pleroma.Config.get([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Hackney) + Pleroma.Config.get([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) end end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index b1498a5a4..fad577ec1 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-onl + defmodule Pleroma.ReverseProxy.Client.Tesla do @behaviour Pleroma.ReverseProxy.Client @@ -6,7 +10,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do def request(method, url, headers, body, opts \\ []) do adapter_opts = Keyword.get(opts, :adapter, []) - |> Keyword.put(:chunks_response, true) + |> Keyword.put(:body_as, :chunks) with {:ok, response} <- Pleroma.HTTP.request( @@ -44,13 +48,13 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do adapter.read_chunk(pid, stream, opts) end - def close(client) do + def close(pid) do adapter = Application.get_env(:tesla, :adapter) unless adapter in @adapters do raise "#{adapter} doesn't support closing connection" end - adapter.close(client) + adapter.close(pid) end end diff --git a/mix.exs b/mix.exs index f1fbdc6b3..9d04b5771 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do {:poison, "~> 3.0", override: true}, {:tesla, github: "alex-strizhakov/tesla", - ref: "beb8927358dfaa66ecd458df607befde12dd56e0", + ref: "c29a7fd030fa6decbf7091152f563fe322e2b589", override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index 635f20185..ef5ebda6c 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "beb8927358dfaa66ecd458df607befde12dd56e0", [ref: "beb8927358dfaa66ecd458df607befde12dd56e0"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "c29a7fd030fa6decbf7091152f563fe322e2b589", [ref: "c29a7fd030fa6decbf7091152f563fe322e2b589"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs new file mode 100644 index 000000000..2ec8f3993 --- /dev/null +++ b/test/gun/connections_test.exs @@ -0,0 +1,172 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Gun.ConnectionsTest do + use ExUnit.Case, async: true + alias Pleroma.Gun.{Connections, Conn, API} + + setup do + name = :test_gun_connections + {:ok, pid} = Connections.start_link(name) + + {:ok, name: name, pid: pid} + end + + test "opens connection and reuse it on next request", %{name: name, pid: pid} do + conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "some-domain.com:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + + test "reuses connection based on protocol", %{name: name, pid: pid} do + conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + assert is_pid(conn) + assert Process.alive?(conn) + + https_conn = Connections.get_conn(name, "https://some-domain.com", genserver_pid: pid) + + refute conn == https_conn + + reused_https = Connections.get_conn(name, "https://some-domain.com", genserver_pid: pid) + + refute conn == reused_https + + assert reused_https == https_conn + + %Connections{ + conns: %{ + "some-domain.com:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + }, + "some-domain.com:443" => %Conn{ + conn: ^https_conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + + test "process gun_down message", %{name: name, pid: pid} do + conn = Connections.get_conn(name, "http://gun_down.com", genserver_pid: pid) + + refute conn + + %Connections{ + conns: %{ + "gun_down.com:80" => %Conn{ + conn: _, + state: :down, + waiting_pids: _ + } + } + } = Connections.get_state(name) + end + + test "process gun_down message and then gun_up", %{name: name, pid: pid} do + conn = Connections.get_conn(name, "http://gun_down_and_up.com", genserver_pid: pid) + + refute conn + + %Connections{ + conns: %{ + "gun_down_and_up.com:80" => %Conn{ + conn: _, + state: :down, + waiting_pids: _ + } + } + } = Connections.get_state(name) + + conn = Connections.get_conn(name, "http://gun_down_and_up.com", genserver_pid: pid) + + assert is_pid(conn) + assert Process.alive?(conn) + + %Connections{ + conns: %{ + "gun_down_and_up.com:80" => %Conn{ + conn: _, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + + test "async processes get same conn for same domain", %{name: name, pid: pid} do + tasks = + for _ <- 1..5 do + Task.async(fn -> + Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + end) + end + + tasks_with_results = Task.yield_many(tasks) + + results = + Enum.map(tasks_with_results, fn {task, res} -> + res || Task.shutdown(task, :brutal_kill) + end) + + conns = for {:ok, value} <- results, do: value + + %Connections{ + conns: %{ + "some-domain.com:80" => %Conn{ + conn: conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + + assert Enum.all?(conns, fn res -> res == conn end) + end + + describe "integration test" do + @describetag :integration + + test "opens connection and reuse it on next request", %{name: name} do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], :gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + conn = Connections.get_conn(name, "http://httpbin.org") + + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.get_conn(name, "http://httpbin.org") + + assert conn == reused_conn + + %Connections{ + conns: %{ + "httpbin.org:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + end +end diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 95adae666..56ac8672b 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -332,5 +332,38 @@ defmodule Pleroma.ReverseProxyTest do describe "integration tests" do @describetag :integration + + test "with hackney client", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Hackney) + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + end) + + conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + end + + test "with tesla client with gun adapter", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + Application.put_env(:tesla, :adapter, adapter) + end) + end end end -- cgit v1.2.3 From aee44f3f4bfeb9591e4c469fbfeb1ab0857cf89d Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 13:37:27 +0300 Subject: added gun connections holder genserver --- lib/pleroma/gun/connections.ex | 25 ++++++++++++-- lib/pleroma/http/http.ex | 23 +++++++++++++ test/gun/connections_test.exs | 57 ++++++++++++++++++++++++------- test/reverse_proxy/reverse_proxy_test.exs | 7 +++- 4 files changed, 97 insertions(+), 15 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 60ec68d89..d5a9d8607 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -26,8 +26,8 @@ defmodule Pleroma.Gun.Connections do {:ok, %__MODULE__{conns: %{}}} end - @spec get_conn(atom(), String.t(), keyword()) :: pid() - def get_conn(name \\ __MODULE__, url, opts \\ []) do + @spec get_conn(String.t(), keyword(), atom()) :: pid() + def get_conn(url, opts \\ [], name \\ __MODULE__) do opts = Enum.into(opts, %{}) uri = URI.parse(url) @@ -39,6 +39,27 @@ defmodule Pleroma.Gun.Connections do ) end + # TODO: only for testing, add this parameter to the config + @spec try_to_get_gun_conn(String.t(), keyword(), atom()) :: nil | pid() + def try_to_get_gun_conn(url, opts \\ [], name \\ __MODULE__), + do: try_to_get_gun_conn(url, opts, name, 0) + + @spec try_to_get_gun_conn(String.t(), keyword(), atom(), pos_integer()) :: nil | pid() + def try_to_get_gun_conn(_url, _, _, 3), do: nil + + def try_to_get_gun_conn(url, opts, name, acc) do + case Pleroma.Gun.Connections.get_conn(url, opts, name) do + nil -> try_to_get_gun_conn(url, acc + 1) + conn -> conn + end + end + + @spec alive?(atom()) :: boolean() + def alive?(name \\ __MODULE__) do + pid = Process.whereis(name) + if pid, do: Process.alive?(pid), else: false + end + @spec get_state(atom()) :: t() def get_state(name \\ __MODULE__) do GenServer.call(name, {:state}) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 6d7934841..1846749c0 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -32,6 +32,15 @@ defmodule Pleroma.HTTP do process_request_options(options) |> process_sni_options(url) + adapter_gun? = Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun + + options = + if adapter_gun? and Pleroma.Gun.Connections.alive?() do + get_conn_for_gun(url, options) + else + options + end + params = Keyword.get(options, :params, []) %{} @@ -52,6 +61,20 @@ defmodule Pleroma.HTTP do end end + defp get_conn_for_gun(url, options) do + case Pleroma.Gun.Connections.try_to_get_gun_conn(url) do + nil -> + options + + conn -> + adapter_opts = + Keyword.get(options, :adapter, []) + |> Keyword.put(:conn, conn) + + Keyword.put(options, :adapter, adapter_opts) + end + end + defp process_sni_options(options, nil), do: options defp process_sni_options(options, url) do diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index 2ec8f3993..42354d330 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -3,23 +3,56 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Gun.ConnectionsTest do - use ExUnit.Case, async: true + use ExUnit.Case alias Pleroma.Gun.{Connections, Conn, API} setup do name = :test_gun_connections + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) {:ok, pid} = Connections.start_link(name) {:ok, name: name, pid: pid} end + describe "alive?/2" do + test "is alive", %{name: name} do + assert Connections.alive?(name) + end + + test "returns false if not started" do + refute Connections.alive?(:some_random_name) + end + end + + test "try_to_get_gun_conn/1 returns conn", %{name: name, pid: pid} do + conn = Connections.try_to_get_gun_conn("http://some-domain.com", [genserver_pid: pid], name) + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "some-domain.com:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + test "opens connection and reuse it on next request", %{name: name, pid: pid} do - conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) assert is_pid(conn) assert Process.alive?(conn) - reused_conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + reused_conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) assert conn == reused_conn @@ -35,15 +68,15 @@ defmodule Gun.ConnectionsTest do end test "reuses connection based on protocol", %{name: name, pid: pid} do - conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) assert is_pid(conn) assert Process.alive?(conn) - https_conn = Connections.get_conn(name, "https://some-domain.com", genserver_pid: pid) + https_conn = Connections.get_conn("https://some-domain.com", [genserver_pid: pid], name) refute conn == https_conn - reused_https = Connections.get_conn(name, "https://some-domain.com", genserver_pid: pid) + reused_https = Connections.get_conn("https://some-domain.com", [genserver_pid: pid], name) refute conn == reused_https @@ -66,7 +99,7 @@ defmodule Gun.ConnectionsTest do end test "process gun_down message", %{name: name, pid: pid} do - conn = Connections.get_conn(name, "http://gun_down.com", genserver_pid: pid) + conn = Connections.get_conn("http://gun_down.com", [genserver_pid: pid], name) refute conn @@ -82,7 +115,7 @@ defmodule Gun.ConnectionsTest do end test "process gun_down message and then gun_up", %{name: name, pid: pid} do - conn = Connections.get_conn(name, "http://gun_down_and_up.com", genserver_pid: pid) + conn = Connections.get_conn("http://gun_down_and_up.com", [genserver_pid: pid], name) refute conn @@ -96,7 +129,7 @@ defmodule Gun.ConnectionsTest do } } = Connections.get_state(name) - conn = Connections.get_conn(name, "http://gun_down_and_up.com", genserver_pid: pid) + conn = Connections.get_conn("http://gun_down_and_up.com", [genserver_pid: pid], name) assert is_pid(conn) assert Process.alive?(conn) @@ -116,7 +149,7 @@ defmodule Gun.ConnectionsTest do tasks = for _ <- 1..5 do Task.async(fn -> - Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) end) end @@ -149,12 +182,12 @@ defmodule Gun.ConnectionsTest do api = Pleroma.Config.get([API]) Pleroma.Config.put([API], :gun) on_exit(fn -> Pleroma.Config.put([API], api) end) - conn = Connections.get_conn(name, "http://httpbin.org") + conn = Connections.get_conn("http://httpbin.org", [], name) assert is_pid(conn) assert Process.alive?(conn) - reused_conn = Connections.get_conn(name, "http://httpbin.org") + reused_conn = Connections.get_conn("http://httpbin.org", [], name) assert conn == reused_conn diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 56ac8672b..2092b0dc3 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxyTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase import ExUnit.CaptureLog import Mox alias Pleroma.ReverseProxy @@ -354,6 +354,10 @@ defmodule Pleroma.ReverseProxyTest do adapter = Application.get_env(:tesla, :adapter) Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], :gun) + {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") assert byte_size(conn.resp_body) == 10 @@ -363,6 +367,7 @@ defmodule Pleroma.ReverseProxyTest do on_exit(fn -> Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) end) end end -- cgit v1.2.3 From 9241df642c257bcfc5b697d50f0deed426370694 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:08:10 +0300 Subject: some debug --- lib/pleroma/gun/connections.ex | 5 +++++ lib/pleroma/http/http.ex | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index d5a9d8607..82579604a 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -69,6 +69,10 @@ defmodule Pleroma.Gun.Connections do def handle_call({:conn, %{opts: opts, uri: uri}}, from, state) do key = compose_key(uri) + IO.inspect(state) + + IO.inspect(key) + case state.conns[key] do %{conn: conn, state: conn_state} when conn_state == :up -> {:reply, conn, state} @@ -87,6 +91,7 @@ defmodule Pleroma.Gun.Connections do protocol: String.to_atom(uri.scheme) }) + IO.inspect(state) {:noreply, state} end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 1846749c0..3ad891d11 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -62,7 +62,7 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.try_to_get_gun_conn(url) do + case Pleroma.Gun.Connections.get_conn(url) do nil -> options -- cgit v1.2.3 From 27d5aa7546a3263856581e972e7d79389007840f Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:27:55 +0300 Subject: ssl fixes --- lib/pleroma/gun/api/mock.ex | 2 +- lib/pleroma/gun/connections.ex | 16 ++++++++++------ test/gun/connections_test.exs | 24 ++++++++++++++++++++++++ test/reverse_proxy/reverse_proxy_test.exs | 23 +++++++++++++++++++++++ 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex index 3348715c4..ff9e13a74 100644 --- a/lib/pleroma/gun/api/mock.ex +++ b/lib/pleroma/gun/api/mock.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Gun.API.Mock do def open('some-domain.com', 443, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - send(genserver_pid, {:gun_up, conn_pid, :https}) + send(genserver_pid, {:gun_up, conn_pid, :http2}) {:ok, conn_pid} end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 82579604a..6fcf4332a 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -100,8 +100,8 @@ defmodule Pleroma.Gun.Connections do def handle_call({:state}, _from, state), do: {:reply, state, state} @impl true - def handle_info({:gun_up, conn_pid, protocol}, state) do - {key, conn} = find_conn(state.conns, conn_pid, protocol) + def handle_info({:gun_up, conn_pid, _protocol}, state) do + {key, conn} = find_conn(state.conns, conn_pid) # Send to all waiting processes connection pid Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end) @@ -113,8 +113,8 @@ defmodule Pleroma.Gun.Connections do @impl true # Do we need to do something with killed & unprocessed references? - def handle_info({:gun_down, conn_pid, protocol, _reason, _killed, _unprocessed}, state) do - {key, conn} = find_conn(state.conns, conn_pid, protocol) + def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed, _unprocessed}, state) do + {key, conn} = find_conn(state.conns, conn_pid) # We don't want to block requests to GenServer if gun send down message, return nil, so we can make some retries, while connection is not up Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) @@ -125,6 +125,10 @@ defmodule Pleroma.Gun.Connections do defp compose_key(uri), do: uri.host <> ":" <> to_string(uri.port) - defp find_conn(conns, conn_pid, protocol), - do: Enum.find(conns, fn {_, conn} -> conn.conn == conn_pid and conn.protocol == protocol end) + defp find_conn(conns, conn_pid) do + Enum.find(conns, fn {key, conn} -> + protocol = if String.ends_with?(key, ":443"), do: :https, else: :http + conn.conn == conn_pid and conn.protocol == protocol + end) + end end diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index 42354d330..aad70a644 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -201,5 +201,29 @@ defmodule Gun.ConnectionsTest do } } = Connections.get_state(name) end + + test "opens ssl connection and reuse it on next request", %{name: name} do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], :gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + conn = Connections.get_conn("https://httpbin.org", [], name) + + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.get_conn("https://httpbin.org", [], name) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "httpbin.org:443" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end end end diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 2092b0dc3..c5563f955 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -370,5 +370,28 @@ defmodule Pleroma.ReverseProxyTest do Pleroma.Config.put([Pleroma.Gun.API], api) end) end + + test "with tesla client with gun adapter with ssl", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], :gun) + {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + + conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) + end) + end end end -- cgit v1.2.3 From c0916b66f8f4d9e2917e11d7d24fd5931507a35c Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:34:49 +0300 Subject: removing debug --- lib/pleroma/gun/connections.ex | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 6fcf4332a..e06cad276 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -69,10 +69,6 @@ defmodule Pleroma.Gun.Connections do def handle_call({:conn, %{opts: opts, uri: uri}}, from, state) do key = compose_key(uri) - IO.inspect(state) - - IO.inspect(key) - case state.conns[key] do %{conn: conn, state: conn_state} when conn_state == :up -> {:reply, conn, state} @@ -91,7 +87,6 @@ defmodule Pleroma.Gun.Connections do protocol: String.to_atom(uri.scheme) }) - IO.inspect(state) {:noreply, state} end end -- cgit v1.2.3 From 888ac63df42075f771161a607be1de873ca7d666 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:38:13 +0300 Subject: some debug --- lib/pleroma/http/http.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 3ad891d11..a6c325017 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -41,6 +41,8 @@ defmodule Pleroma.HTTP do options end + IO.inspect(options) + params = Keyword.get(options, :params, []) %{} @@ -62,7 +64,7 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.get_conn(url) do + case Pleroma.Gun.Connections.try_to_get_gun_conn(url) do nil -> options -- cgit v1.2.3 From db848b2a13b4d47870dcff1c6d8e26f0300ca0a9 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:41:08 +0300 Subject: tesla adapter options debug --- lib/pleroma/http/http.ex | 2 -- lib/pleroma/reverse_proxy/client/tesla.ex | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index a6c325017..1846749c0 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -41,8 +41,6 @@ defmodule Pleroma.HTTP do options end - IO.inspect(options) - params = Keyword.get(options, :params, []) %{} diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index fad577ec1..282f9dcb7 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -12,6 +12,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do Keyword.get(opts, :adapter, []) |> Keyword.put(:body_as, :chunks) + IO.inspect(adapter_opts) + with {:ok, response} <- Pleroma.HTTP.request( method, -- cgit v1.2.3 From d28051e2847420031873c31278590567d80e4cd1 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:45:21 +0300 Subject: another debug --- lib/pleroma/http/connection.ex | 4 +++- lib/pleroma/reverse_proxy/client/tesla.ex | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 6cb26c0fe..20c83c572 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -23,7 +23,9 @@ defmodule Pleroma.HTTP.Connection do def new(opts \\ []) do middleware = [Tesla.Middleware.FollowRedirects] adapter = Application.get_env(:tesla, :adapter) - Tesla.client(middleware, {adapter, options(opts)}) + options = options(opts) + IO.inspect(options) + Tesla.client(middleware, {adapter, options}) end # fetch http options diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 282f9dcb7..fad577ec1 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -12,8 +12,6 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do Keyword.get(opts, :adapter, []) |> Keyword.put(:body_as, :chunks) - IO.inspect(adapter_opts) - with {:ok, response} <- Pleroma.HTTP.request( method, -- cgit v1.2.3 From 43e5307d65874503125584a1110764ce9f2e7e51 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 15:54:25 +0300 Subject: like this --- lib/pleroma/http/connection.ex | 1 - lib/pleroma/http/http.ex | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 20c83c572..222a8b6aa 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -24,7 +24,6 @@ defmodule Pleroma.HTTP.Connection do middleware = [Tesla.Middleware.FollowRedirects] adapter = Application.get_env(:tesla, :adapter) options = options(opts) - IO.inspect(options) Tesla.client(middleware, {adapter, options}) end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 1846749c0..3ad891d11 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -62,7 +62,7 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.try_to_get_gun_conn(url) do + case Pleroma.Gun.Connections.get_conn(url) do nil -> options -- cgit v1.2.3 From e0b52ca086b3e3a7c9a4ed5ac3c812fd3a56c45b Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 16:31:13 +0300 Subject: only http --- lib/pleroma/http/connection.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 222a8b6aa..ef1c1cabd 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -9,7 +9,8 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, - timeout: 20_000 + timeout: 20_000, + protocols: [:http] ] @doc """ -- cgit v1.2.3 From 55a5523accf5095e12e2f60b06f19ace49d0a3b8 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 17:48:22 +0300 Subject: don't close connection in gun adapter on reusing --- lib/pleroma/http/http.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 3ad891d11..d493a8b4a 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -70,6 +70,7 @@ defmodule Pleroma.HTTP do adapter_opts = Keyword.get(options, :adapter, []) |> Keyword.put(:conn, conn) + |> Keyword.put(:close_conn, false) Keyword.put(options, :adapter, adapter_opts) end -- cgit v1.2.3 From 59336f14482c369c6c3a7bbc621d0a2828fd577f Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 17:56:16 +0300 Subject: tesla update --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 9d04b5771..f5200d15d 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do {:poison, "~> 3.0", override: true}, {:tesla, github: "alex-strizhakov/tesla", - ref: "c29a7fd030fa6decbf7091152f563fe322e2b589", + ref: "99856638d0b8f382eb22e704b76bc30b4c4c379d", override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index ef5ebda6c..f09b7fece 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "c29a7fd030fa6decbf7091152f563fe322e2b589", [ref: "c29a7fd030fa6decbf7091152f563fe322e2b589"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "99856638d0b8f382eb22e704b76bc30b4c4c379d", [ref: "99856638d0b8f382eb22e704b76bc30b4c4c379d"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, -- cgit v1.2.3 From 916aeb934f4a961948ba41f9ac9c0f52d986600f Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 20:08:15 +0300 Subject: adding host header for gun adapter --- lib/pleroma/http/connection.ex | 6 ++---- lib/pleroma/http/http.ex | 2 +- lib/pleroma/http/request_builder.ex | 9 +++++++++ test/http/request_builder_test.exs | 13 ++++++++++++- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index ef1c1cabd..6cb26c0fe 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -9,8 +9,7 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, - timeout: 20_000, - protocols: [:http] + timeout: 20_000 ] @doc """ @@ -24,8 +23,7 @@ defmodule Pleroma.HTTP.Connection do def new(opts \\ []) do middleware = [Tesla.Middleware.FollowRedirects] adapter = Application.get_env(:tesla, :adapter) - options = options(opts) - Tesla.client(middleware, {adapter, options}) + Tesla.client(middleware, {adapter, options(opts)}) end # fetch http options diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index d493a8b4a..21c057f4b 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -45,9 +45,9 @@ defmodule Pleroma.HTTP do %{} |> Builder.method(method) + |> Builder.url(url) |> Builder.headers(headers) |> Builder.opts(options) - |> Builder.url(url) |> Builder.add_param(:body, :body, body) |> Builder.add_param(:query, :query, params) |> Enum.into([]) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 4e77870bd..6edf15bbe 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -53,6 +53,15 @@ defmodule Pleroma.HTTP.RequestBuilder do header_list end + # TODO: maybe we need this header for all adapters, so we won't fail on adapter change + header_list = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + %{host: host} = URI.parse(request.url) + header_list ++ [{"host", host}] + else + header_list + end + Map.put_new(request, :headers, header_list) end diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index ea4dd087e..631730f65 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.RequestBuilderTest do - use ExUnit.Case, async: true + use ExUnit.Case use Pleroma.Tests.Helpers alias Pleroma.HTTP.RequestBuilder @@ -21,6 +21,17 @@ defmodule Pleroma.HTTP.RequestBuilderTest do headers: [{"user-agent", Pleroma.Application.user_agent()}] } end + + test "it adds host header for gun adapter" do + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) + + assert RequestBuilder.headers(%{url: "https://example.com"}, []) == %{ + headers: [{"host", "example.com"}], + url: "https://example.com" + } + end end describe "add_optional_params/3" do -- cgit v1.2.3 From ff5a0a1d416391756c6c5d6935a1d41bbb162fa7 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 15 Aug 2019 15:49:32 +0300 Subject: receive adapter messages in adapter --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index f5200d15d..7c10ab75a 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do {:poison, "~> 3.0", override: true}, {:tesla, github: "alex-strizhakov/tesla", - ref: "99856638d0b8f382eb22e704b76bc30b4c4c379d", + ref: "82dd3ace650b93d6699cf379b1a238d891f45cfb", override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index f09b7fece..26be6d663 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "99856638d0b8f382eb22e704b76bc30b4c4c379d", [ref: "99856638d0b8f382eb22e704b76bc30b4c4c379d"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "82dd3ace650b93d6699cf379b1a238d891f45cfb", [ref: "82dd3ace650b93d6699cf379b1a238d891f45cfb"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, -- cgit v1.2.3 From 66a1573f81021c04063672382ef83716dfc4471d Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 15 Aug 2019 20:43:15 +0300 Subject: updating tesla --- lib/pleroma/http/http.ex | 3 +++ mix.exs | 10 ++++++---- mix.lock | 2 +- test/reverse_proxy/reverse_proxy_test.exs | 23 +++++++++++++++++++++++ 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 21c057f4b..6a09b8260 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -67,10 +67,13 @@ defmodule Pleroma.HTTP do options conn -> + %{host: host, port: port} = URI.parse(url) + adapter_opts = Keyword.get(options, :adapter, []) |> Keyword.put(:conn, conn) |> Keyword.put(:close_conn, false) + |> Keyword.put(:original, "#{host}:#{port}") Keyword.put(options, :adapter, adapter_opts) end diff --git a/mix.exs b/mix.exs index 7c10ab75a..d156b9e36 100644 --- a/mix.exs +++ b/mix.exs @@ -111,10 +111,12 @@ defmodule Pleroma.Mixfile do {:calendar, "~> 0.17.4"}, {:cachex, "~> 3.0.2"}, {:poison, "~> 3.0", override: true}, - {:tesla, - github: "alex-strizhakov/tesla", - ref: "82dd3ace650b93d6699cf379b1a238d891f45cfb", - override: true}, + { + :tesla, + github: "alex-strizhakov/tesla", + ref: "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8", + override: true + }, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, {:jason, "~> 1.0"}, diff --git a/mix.lock b/mix.lock index 26be6d663..606dbfa61 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "82dd3ace650b93d6699cf379b1a238d891f45cfb", [ref: "82dd3ace650b93d6699cf379b1a238d891f45cfb"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8", [ref: "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index c5563f955..0a982c56c 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -393,5 +393,28 @@ defmodule Pleroma.ReverseProxyTest do Pleroma.Config.put([Pleroma.Gun.API], api) end) end + + test "tesla client with gun client follow redirects", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], :gun) + {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + + conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/7") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) + end) + end end end -- cgit v1.2.3 From 7f431996c5e2ad64b4641385e2358636aa37e92e Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 09:35:47 +0300 Subject: updated deps --- lib/pleroma/http/request_builder.ex | 9 --------- mix.exs | 7 ++++--- mix.lock | 6 +++--- test/http/request_builder_test.exs | 11 ----------- test/reverse_proxy/reverse_proxy_test.exs | 3 +-- 5 files changed, 8 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 6edf15bbe..4e77870bd 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -53,15 +53,6 @@ defmodule Pleroma.HTTP.RequestBuilder do header_list end - # TODO: maybe we need this header for all adapters, so we won't fail on adapter change - header_list = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do - %{host: host} = URI.parse(request.url) - header_list ++ [{"host", host}] - else - header_list - end - Map.put_new(request, :headers, header_list) end diff --git a/mix.exs b/mix.exs index d156b9e36..29f1c90c5 100644 --- a/mix.exs +++ b/mix.exs @@ -114,11 +114,12 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8", + ref: "929d68446a9b9d08149bd92d5b51c7ae9f87cfee", override: true }, - {:cowlib, "~> 2.6.0", override: true}, - {:gun, "~> 1.3"}, + {:cowlib, "~> 2.7.3", override: true}, + {:gun, + github: "ninenines/gun", ref: "491ddf58c0e14824a741852fdc522b390b306ae2", override: true}, {:jason, "~> 1.0"}, {:mogrify, "~> 0.6.1"}, {:ex_aws, "~> 2.1"}, diff --git a/mix.lock b/mix.lock index 606dbfa61..1d4532fed 100644 --- a/mix.lock +++ b/mix.lock @@ -13,7 +13,7 @@ "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, "cors_plug": {:hex, :cors_plug, "1.5.2", "72df63c87e4f94112f458ce9d25800900cc88608c1078f0e4faddf20933eda6e", [:mix], [{:plug, "~> 1.3 or ~> 1.4 or ~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "cowboy": {:hex, :cowboy, "2.6.3", "99aa50e94e685557cad82e704457336a453d4abcb77839ad22dbe71f311fcc06", [:rebar3], [{:cowlib, "~> 2.7.3", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, - "cowlib": {:hex, :cowlib, "2.6.0", "8aa629f81a0fc189f261dc98a42243fa842625feea3c7ec56c48f4ccdb55490f", [:rebar3], [], "hexpm"}, + "cowlib": {:hex, :cowlib, "2.7.3", "a7ffcd0917e6d50b4d5fb28e9e2085a0ceb3c97dea310505f7460ff5ed764ce9", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "crontab": {:hex, :crontab, "1.1.7", "b9219f0bdc8678b94143655a8f229716c5810c0636a4489f98c0956137e53985", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, @@ -37,7 +37,7 @@ "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, - "gun": {:hex, :gun, "1.3.0", "18e5d269649c987af95aec309f68a27ffc3930531dd227a6eaa0884d6684286e", [:rebar3], [{:cowlib, "~> 2.6.0", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm"}, + "gun": {:git, "https://github.com/ninenines/gun.git", "491ddf58c0e14824a741852fdc522b390b306ae2", [ref: "491ddf58c0e14824a741852fdc522b390b306ae2"]}, "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8", [ref: "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "929d68446a9b9d08149bd92d5b51c7ae9f87cfee", [ref: "929d68446a9b9d08149bd92d5b51c7ae9f87cfee"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index 631730f65..77a1e870a 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -21,17 +21,6 @@ defmodule Pleroma.HTTP.RequestBuilderTest do headers: [{"user-agent", Pleroma.Application.user_agent()}] } end - - test "it adds host header for gun adapter" do - adapter = Application.get_env(:tesla, :adapter) - Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) - on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) - - assert RequestBuilder.headers(%{url: "https://example.com"}, []) == %{ - headers: [{"host", "example.com"}], - url: "https://example.com" - } - end end describe "add_optional_params/3" do diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 0a982c56c..cd65163d1 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -404,9 +404,8 @@ defmodule Pleroma.ReverseProxyTest do Pleroma.Config.put([Pleroma.Gun.API], :gun) {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) - conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/7") + conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5") - assert byte_size(conn.resp_body) == 10 assert conn.state == :chunked assert conn.status == 200 -- cgit v1.2.3 From d052dca152b3f1cbd66873af6f0399a3d97debcc Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 09:44:00 +0300 Subject: fix --- lib/pleroma/gun/connections.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index e06cad276..a3f1b0351 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -31,7 +31,10 @@ defmodule Pleroma.Gun.Connections do opts = Enum.into(opts, %{}) uri = URI.parse(url) - opts = if uri.scheme == "https", do: Map.put(opts, :transport, :tls), else: opts + opts = + if uri.scheme == "https" and uri.port != 443, + do: Map.put(opts, :transport, :tls), + else: opts GenServer.call( name, -- cgit v1.2.3 From 7354d8958b66b93fcedb111ee929bc5046cc464f Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 09:51:10 +0300 Subject: debug for media proxy --- lib/pleroma/web/media_proxy/media_proxy_controller.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 8403850ff..7d285a549 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -13,6 +13,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), + IO.inspect(sig64), + IO.inspect(url64), + IO.inspect(url), :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else -- cgit v1.2.3 From c97979da776c6da83655b31e8ed20a0ece59de6f Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 16:23:48 +0300 Subject: passing options for gun --- lib/pleroma/gun/connections.ex | 17 ++++++++++++++++- lib/pleroma/http/http.ex | 13 ++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index a3f1b0351..b8cf2f9b5 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -11,6 +11,20 @@ defmodule Pleroma.Gun.Connections do conns: %{domain() => conn()} } + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :ws_opts + ] + defstruct conns: %{} def start_link(name \\ __MODULE__) do @@ -81,7 +95,8 @@ defmodule Pleroma.Gun.Connections do {:noreply, state} nil -> - {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) + {:ok, conn} = + Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, Map.take(opts, @gun_keys)) state = put_in(state.conns[key], %Pleroma.Gun.Conn{ diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 6a09b8260..a7f42d0c0 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -62,7 +62,7 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.get_conn(url) do + case Pleroma.Gun.Connections.get_conn(url, options) do nil -> options @@ -86,8 +86,15 @@ defmodule Pleroma.HTTP do host = uri.host |> to_charlist() case uri.scheme do - "https" -> options ++ [ssl: [server_name_indication: host]] - _ -> options + "https" -> + tls_opts = + Keyword.get(options, :tls_opts, []) + |> Keyword.put(:server_name_indication, host) + + Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] + + _ -> + options end end -- cgit v1.2.3 From ac0d158c24549bd22b6308c86a20a99ee338c7f7 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 16:33:23 +0300 Subject: test fixes --- lib/pleroma/gun/api/api.ex | 2 +- lib/pleroma/gun/api/gun.ex | 22 ++++++++++++++++++++++ lib/pleroma/gun/connections.ex | 17 +---------------- mix.exs | 2 +- mix.lock | 2 +- 5 files changed, 26 insertions(+), 19 deletions(-) create mode 100644 lib/pleroma/gun/api/gun.ex diff --git a/lib/pleroma/gun/api/api.ex b/lib/pleroma/gun/api/api.ex index c69ab890e..19adc1bf0 100644 --- a/lib/pleroma/gun/api/api.ex +++ b/lib/pleroma/gun/api/api.ex @@ -10,6 +10,6 @@ defmodule Pleroma.Gun.API do end defp api do - Pleroma.Config.get([Pleroma.Gun.API], :gun) + Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun.API.Gun) end end diff --git a/lib/pleroma/gun/api/gun.ex b/lib/pleroma/gun/api/gun.ex new file mode 100644 index 000000000..14a4b7275 --- /dev/null +++ b/lib/pleroma/gun/api/gun.ex @@ -0,0 +1,22 @@ +defmodule Pleroma.Gun.API.Gun do + @behaviour Pleroma.Gun.API + + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :ws_opts + ] + + @impl Pleroma.Gun.API + def open(host, port, opts) do + :gun.open(host, port, Map.take(opts, @gun_keys)) + end +end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index b8cf2f9b5..a3f1b0351 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -11,20 +11,6 @@ defmodule Pleroma.Gun.Connections do conns: %{domain() => conn()} } - @gun_keys [ - :connect_timeout, - :http_opts, - :http2_opts, - :protocols, - :retry, - :retry_timeout, - :trace, - :transport, - :tls_opts, - :tcp_opts, - :ws_opts - ] - defstruct conns: %{} def start_link(name \\ __MODULE__) do @@ -95,8 +81,7 @@ defmodule Pleroma.Gun.Connections do {:noreply, state} nil -> - {:ok, conn} = - Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, Map.take(opts, @gun_keys)) + {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) state = put_in(state.conns[key], %Pleroma.Gun.Conn{ diff --git a/mix.exs b/mix.exs index 29f1c90c5..64c56c4e3 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "929d68446a9b9d08149bd92d5b51c7ae9f87cfee", + ref: "28b06f772632fe82d9fb4cc39b0477709b1f3d6f", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index 1d4532fed..554b81fa8 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "929d68446a9b9d08149bd92d5b51c7ae9f87cfee", [ref: "929d68446a9b9d08149bd92d5b51c7ae9f87cfee"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "28b06f772632fe82d9fb4cc39b0477709b1f3d6f", [ref: "28b06f772632fe82d9fb4cc39b0477709b1f3d6f"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, -- cgit v1.2.3 From fb0d06432892bd178baeaadc5b4bdc5e6e21b89e Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 16:45:50 +0300 Subject: more fixes --- lib/pleroma/web/media_proxy/media_proxy_controller.ex | 3 --- test/gun/connections_test.exs | 4 ++-- test/reverse_proxy/reverse_proxy_test.exs | 6 +++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 7d285a549..8403850ff 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -13,9 +13,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), - IO.inspect(sig64), - IO.inspect(url64), - IO.inspect(url), :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index aad70a644..a63c8eaf9 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -180,7 +180,7 @@ defmodule Gun.ConnectionsTest do test "opens connection and reuse it on next request", %{name: name} do api = Pleroma.Config.get([API]) - Pleroma.Config.put([API], :gun) + Pleroma.Config.put([API], API.Gun) on_exit(fn -> Pleroma.Config.put([API], api) end) conn = Connections.get_conn("http://httpbin.org", [], name) @@ -204,7 +204,7 @@ defmodule Gun.ConnectionsTest do test "opens ssl connection and reuse it on next request", %{name: name} do api = Pleroma.Config.get([API]) - Pleroma.Config.put([API], :gun) + Pleroma.Config.put([API], API.Gun) on_exit(fn -> Pleroma.Config.put([API], api) end) conn = Connections.get_conn("https://httpbin.org", [], name) diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index cd65163d1..476cff643 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -355,7 +355,7 @@ defmodule Pleroma.ReverseProxyTest do Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) api = Pleroma.Config.get([Pleroma.Gun.API]) - Pleroma.Config.put([Pleroma.Gun.API], :gun) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") @@ -378,7 +378,7 @@ defmodule Pleroma.ReverseProxyTest do Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) api = Pleroma.Config.get([Pleroma.Gun.API]) - Pleroma.Config.put([Pleroma.Gun.API], :gun) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10") @@ -401,7 +401,7 @@ defmodule Pleroma.ReverseProxyTest do Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) api = Pleroma.Config.get([Pleroma.Gun.API]) - Pleroma.Config.put([Pleroma.Gun.API], :gun) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5") -- cgit v1.2.3 From 5e309605d2087c534e0902cbaff68f0163155929 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 19:21:02 +0300 Subject: tesla update --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 64c56c4e3..d51b1488d 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "28b06f772632fe82d9fb4cc39b0477709b1f3d6f", + ref: "81433bd4011217323a1f443661252fdab23f5e59", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index 554b81fa8..daec58b4f 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "28b06f772632fe82d9fb4cc39b0477709b1f3d6f", [ref: "28b06f772632fe82d9fb4cc39b0477709b1f3d6f"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "81433bd4011217323a1f443661252fdab23f5e59", [ref: "81433bd4011217323a1f443661252fdab23f5e59"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, -- cgit v1.2.3 From 6e1408d05ac5d28123b3d62ff555df7fd118f61f Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 20:04:13 +0300 Subject: tls versions for gun --- lib/pleroma/http/http.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index a7f42d0c0..9339532af 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,6 +90,7 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) + |> Keyword.put(:versions, ['tlsv1.2', 'tlsv1.1', 'tlsv1']) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] -- cgit v1.2.3 From 4a4a61f4ceac98fde293130f0ac4db164ccd8251 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 20:07:08 +0300 Subject: like this --- lib/pleroma/http/http.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 9339532af..b6e338aab 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,7 +90,7 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) - |> Keyword.put(:versions, ['tlsv1.2', 'tlsv1.1', 'tlsv1']) + |> Keyword.put(:versions, [:tlsv1, :"tlsv1.1", :"tlsv1.2"]) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] -- cgit v1.2.3 From 102ec7bb0ca4b53d59f722f44ad168ca2d79b6da Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 20:47:24 +0300 Subject: like thiis --- lib/pleroma/http/http.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index b6e338aab..40b3e1a29 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,7 +90,7 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) - |> Keyword.put(:versions, [:tlsv1, :"tlsv1.1", :"tlsv1.2"]) + |> Keyword.put(:versions, ['tlsv1.2', 'tlsv1.1', :tlsv1]) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] -- cgit v1.2.3 From 699fc4f6eb5fa170d0ae56f5949ead7618c264ab Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 20:52:00 +0300 Subject: order --- lib/pleroma/http/http.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 40b3e1a29..3e29b7b6c 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,7 +90,7 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) - |> Keyword.put(:versions, ['tlsv1.2', 'tlsv1.1', :tlsv1]) + |> Keyword.put(:versions, [:"tlsv1.2", :"tlsv1.1", :tlsv1]) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] -- cgit v1.2.3 From 210e116dc0ee48183bf7b7a363c6d053f88d30fe Mon Sep 17 00:00:00 2001 From: Alex S Date: Sat, 17 Aug 2019 21:50:50 +0300 Subject: no hardcode of versions --- lib/pleroma/http/http.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 3e29b7b6c..a7f42d0c0 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,7 +90,6 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) - |> Keyword.put(:versions, [:"tlsv1.2", :"tlsv1.1", :tlsv1]) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] -- cgit v1.2.3 From 814159e668d48ace95d8feac8847e1d68d07810b Mon Sep 17 00:00:00 2001 From: Alex S Date: Mon, 19 Aug 2019 15:40:06 +0300 Subject: expanding gun connections closing least frequently used separate pools with settings --- config/config.exs | 18 ++++++++ lib/pleroma/application.ex | 14 +++++++ lib/pleroma/gun/api/api.ex | 10 +++++ lib/pleroma/gun/api/gun.ex | 6 +++ lib/pleroma/gun/api/mock.ex | 45 ++++++++++++++++++-- lib/pleroma/gun/conn.ex | 4 +- lib/pleroma/gun/connections.ex | 89 +++++++++++++++++++++++++++------------ test/gun/connections_test.exs | 94 ++++++++++++++++++++++++++++++++++-------- 8 files changed, 230 insertions(+), 50 deletions(-) diff --git a/config/config.exs b/config/config.exs index 63162d594..e56b9730d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -550,6 +550,24 @@ config :pleroma, :rate_limit, password_reset: {1_800_000, 5}, account_confirmation_resend: {8_640_000, 5} +config :pleroma, :gun_pools, + federation: [ + max_connections: 50, + timeout: 150_000 + ], + media: [ + max_connections: 50, + timeout: 150_000 + ], + upload: [ + max_connections: 25, + timeout: 300_000 + ], + default: [ + max_connections: 10, + timout: 20_000 + ] + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 25e56b9e2..06d1a187e 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -39,6 +39,7 @@ defmodule Pleroma.Application do ] ++ cachex_children() ++ hackney_pool_children() ++ + gun_pools() ++ [ Pleroma.Web.Federator.RetryQueue, Pleroma.Stats, @@ -163,6 +164,19 @@ defmodule Pleroma.Application do end end + defp gun_pools do + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + for {pool_name, opts} <- Pleroma.Config.get([:gun_pools]) do + %{ + id: :"gun_pool_#{pool_name}", + start: {Pleroma.Gun.Connections, :start_link, [{pool_name, opts}]} + } + end + else + [] + end + end + defp after_supervisor_start do with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], true <- digest_config[:active] do diff --git a/lib/pleroma/gun/api/api.ex b/lib/pleroma/gun/api/api.ex index 19adc1bf0..7e6d2f929 100644 --- a/lib/pleroma/gun/api/api.ex +++ b/lib/pleroma/gun/api/api.ex @@ -4,11 +4,21 @@ defmodule Pleroma.Gun.API do @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} + @callback info(pid()) :: map() + @callback close(pid()) :: :ok def open(host, port, opts) do api().open(host, port, opts) end + def info(pid) do + api().info(pid) + end + + def close(pid) do + api().close(pid) + end + defp api do Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun.API.Gun) end diff --git a/lib/pleroma/gun/api/gun.ex b/lib/pleroma/gun/api/gun.ex index 14a4b7275..33e7985a1 100644 --- a/lib/pleroma/gun/api/gun.ex +++ b/lib/pleroma/gun/api/gun.ex @@ -19,4 +19,10 @@ defmodule Pleroma.Gun.API.Gun do def open(host, port, opts) do :gun.open(host, port, Map.take(opts, @gun_keys)) end + + @impl Pleroma.Gun.API + def info(pid), do: :gun.info(pid) + + @impl Pleroma.Gun.API + def close(pid), do: :gun.close(pid) end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex index ff9e13a74..a80559f0b 100644 --- a/lib/pleroma/gun/api/mock.ex +++ b/lib/pleroma/gun/api/mock.ex @@ -5,36 +5,75 @@ defmodule Pleroma.Gun.API.Mock do @behaviour Pleroma.Gun.API @impl Pleroma.Gun.API - def open('some-domain.com', 80, %{genserver_pid: genserver_pid}) do + def open(domain, 80, %{genserver_pid: genserver_pid}) + when domain in ['another-domain.com', 'some-domain.com'] do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: domain, + origin_port: 80 + }) + send(genserver_pid, {:gun_up, conn_pid, :http}) {:ok, conn_pid} end def open('some-domain.com', 443, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + origin_scheme: "https", + origin_host: 'some-domain.com', + origin_port: 443 + }) + send(genserver_pid, {:gun_up, conn_pid, :http2}) {:ok, conn_pid} end @impl Pleroma.Gun.API - def open('gun_down.com', _port, %{genserver_pid: genserver_pid}) do + def open('gun_down.com', 80, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: 'gun_down.com', + origin_port: 80 + }) + send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil}) {:ok, conn_pid} end @impl Pleroma.Gun.API - def open('gun_down_and_up.com', _port, %{genserver_pid: genserver_pid}) do + def open('gun_down_and_up.com', 80, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: 'gun_down_and_up.com', + origin_port: 80 + }) + send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil}) {:ok, _} = Task.start_link(fn -> Process.sleep(500) + send(genserver_pid, {:gun_up, conn_pid, :http}) end) {:ok, conn_pid} end + + @impl Pleroma.Gun.API + def info(pid) do + [{_, info}] = Registry.lookup(Pleroma.Gun.API.Mock, pid) + info + end + + @impl Pleroma.Gun.API + def close(_pid), do: :ok end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 62ef146a1..20ddec64c 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -10,8 +10,8 @@ defmodule Pleroma.Gun.Conn do conn: pid(), state: atom(), waiting_pids: [pid()], - protocol: atom() + used: pos_integer() } - defstruct conn: nil, state: :open, waiting_pids: [], protocol: :http + defstruct conn: nil, state: :open, waiting_pids: [], used: 0 end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index a3f1b0351..cec3de2ca 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -6,29 +6,31 @@ defmodule Pleroma.Gun.Connections do use GenServer @type domain :: String.t() - @type conn :: Gun.Conn.t() + @type conn :: Pleroma.Gun.Conn.t() + @type t :: %__MODULE__{ - conns: %{domain() => conn()} + conns: %{domain() => conn()}, + opts: keyword() } - defstruct conns: %{} + defstruct conns: %{}, opts: [] - def start_link(name \\ __MODULE__) do + @spec start_link({atom(), keyword()}) :: {:ok, pid()} | :ignore + def start_link({name, opts}) do if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do - GenServer.start_link(__MODULE__, [], name: name) + GenServer.start_link(__MODULE__, opts, name: name) else :ignore end end @impl true - def init(_) do - {:ok, %__MODULE__{conns: %{}}} - end + def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}} @spec get_conn(String.t(), keyword(), atom()) :: pid() - def get_conn(url, opts \\ [], name \\ __MODULE__) do + def get_conn(url, opts \\ [], name \\ :default) do opts = Enum.into(opts, %{}) + uri = URI.parse(url) opts = @@ -58,13 +60,13 @@ defmodule Pleroma.Gun.Connections do end @spec alive?(atom()) :: boolean() - def alive?(name \\ __MODULE__) do + def alive?(name \\ :default) do pid = Process.whereis(name) if pid, do: Process.alive?(pid), else: false end @spec get_state(atom()) :: t() - def get_state(name \\ __MODULE__) do + def get_state(name \\ :default) do GenServer.call(name, {:state}) end @@ -73,7 +75,8 @@ defmodule Pleroma.Gun.Connections do key = compose_key(uri) case state.conns[key] do - %{conn: conn, state: conn_state} when conn_state == :up -> + %{conn: conn, state: conn_state, used: used} when conn_state == :up -> + state = put_in(state.conns[key].used, used + 1) {:reply, conn, state} %{state: conn_state, waiting_pids: pids} when conn_state in [:open, :down] -> @@ -81,16 +84,23 @@ defmodule Pleroma.Gun.Connections do {:noreply, state} nil -> - {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) + max_connections = state.opts[:max_connections] - state = - put_in(state.conns[key], %Pleroma.Gun.Conn{ - conn: conn, - waiting_pids: [from], - protocol: String.to_atom(uri.scheme) - }) + if Enum.count(state.conns) < max_connections do + open_conn(key, uri, from, state, opts) + else + [{close_key, least_used} | _conns] = Enum.sort_by(state.conns, fn {_k, v} -> v.used end) - {:noreply, state} + :ok = Pleroma.Gun.API.close(least_used.conn) + + state = + put_in( + state.conns, + Map.delete(state.conns, close_key) + ) + + open_conn(key, uri, from, state, opts) + end end end @@ -99,20 +109,29 @@ defmodule Pleroma.Gun.Connections do @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do - {key, conn} = find_conn(state.conns, conn_pid) + conn_key = compose_key_gun_info(conn_pid) + {key, conn} = find_conn(state.conns, conn_pid, conn_key) # Send to all waiting processes connection pid Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end) # Update state of the current connection and set waiting_pids to empty list - state = put_in(state.conns[key], %{conn | state: :up, waiting_pids: []}) + state = + put_in(state.conns[key], %{ + conn + | state: :up, + waiting_pids: [], + used: conn.used + length(conn.waiting_pids) + }) + {:noreply, state} end @impl true # Do we need to do something with killed & unprocessed references? def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed, _unprocessed}, state) do - {key, conn} = find_conn(state.conns, conn_pid) + conn_key = compose_key_gun_info(conn_pid) + {key, conn} = find_conn(state.conns, conn_pid, conn_key) # We don't want to block requests to GenServer if gun send down message, return nil, so we can make some retries, while connection is not up Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) @@ -121,12 +140,28 @@ defmodule Pleroma.Gun.Connections do {:noreply, state} end - defp compose_key(uri), do: uri.host <> ":" <> to_string(uri.port) + defp compose_key(uri), do: "#{uri.scheme}:#{uri.host}:#{uri.port}" - defp find_conn(conns, conn_pid) do + defp compose_key_gun_info(pid) do + info = Pleroma.Gun.API.info(pid) + "#{info.origin_scheme}:#{info.origin_host}:#{info.origin_port}" + end + + defp find_conn(conns, conn_pid, conn_key) do Enum.find(conns, fn {key, conn} -> - protocol = if String.ends_with?(key, ":443"), do: :https, else: :http - conn.conn == conn_pid and conn.protocol == protocol + key == conn_key and conn.conn == conn_pid end) end + + defp open_conn(key, uri, from, state, opts) do + {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) + + state = + put_in(state.conns[key], %Pleroma.Gun.Conn{ + conn: conn, + waiting_pids: [from] + }) + + {:noreply, state} + end end diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index a63c8eaf9..8308b5f9f 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -6,12 +6,17 @@ defmodule Gun.ConnectionsTest do use ExUnit.Case alias Pleroma.Gun.{Connections, Conn, API} + setup_all do + {:ok, _} = Registry.start_link(keys: :unique, name: API.Mock) + :ok + end + setup do name = :test_gun_connections adapter = Application.get_env(:tesla, :adapter) Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) - {:ok, pid} = Connections.start_link(name) + {:ok, pid} = Connections.start_link({name, [max_connections: 2, timeout: 10]}) {:ok, name: name, pid: pid} end @@ -37,10 +42,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "some-domain.com:80" => %Conn{ + "http:some-domain.com:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [] + waiting_pids: [], + used: 2 } } } = Connections.get_state(name) @@ -58,10 +64,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "some-domain.com:80" => %Conn{ + "http:some-domain.com:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [] + waiting_pids: [], + used: 2 } } } = Connections.get_state(name) @@ -84,12 +91,12 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "some-domain.com:80" => %Conn{ + "http:some-domain.com:80" => %Conn{ conn: ^conn, state: :up, waiting_pids: [] }, - "some-domain.com:443" => %Conn{ + "https:some-domain.com:443" => %Conn{ conn: ^https_conn, state: :up, waiting_pids: [] @@ -105,7 +112,7 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "gun_down.com:80" => %Conn{ + "http:gun_down.com:80" => %Conn{ conn: _, state: :down, waiting_pids: _ @@ -121,10 +128,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "gun_down_and_up.com:80" => %Conn{ + "http:gun_down_and_up.com:80" => %Conn{ conn: _, state: :down, - waiting_pids: _ + waiting_pids: _, + used: 0 } } } = Connections.get_state(name) @@ -136,10 +144,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "gun_down_and_up.com:80" => %Conn{ + "http:gun_down_and_up.com:80" => %Conn{ conn: _, state: :up, - waiting_pids: [] + waiting_pids: [], + used: 2 } } } = Connections.get_state(name) @@ -164,10 +173,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "some-domain.com:80" => %Conn{ + "http:some-domain.com:80" => %Conn{ conn: conn, state: :up, - waiting_pids: [] + waiting_pids: [], + used: 5 } } } = Connections.get_state(name) @@ -175,6 +185,52 @@ defmodule Gun.ConnectionsTest do assert Enum.all?(conns, fn res -> res == conn end) end + test "remove frequently used", %{name: name, pid: pid} do + Connections.get_conn("https://some-domain.com", [genserver_pid: pid], name) + + for _ <- 1..4 do + Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) + end + + %Connections{ + conns: %{ + "http:some-domain.com:80" => %Conn{ + conn: _, + state: :up, + waiting_pids: [], + used: 4 + }, + "https:some-domain.com:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [], + used: 1 + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + conn = Connections.get_conn("http://another-domain.com", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "http:another-domain.com:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [], + used: 1 + }, + "http:some-domain.com:80" => %Conn{ + conn: _, + state: :up, + waiting_pids: [], + used: 4 + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + end + describe "integration test" do @describetag :integration @@ -193,10 +249,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "httpbin.org:80" => %Conn{ + "http:httpbin.org:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [] + waiting_pids: [], + used: 2 } } } = Connections.get_state(name) @@ -217,10 +274,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "httpbin.org:443" => %Conn{ + "https:httpbin.org:443" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [] + waiting_pids: [], + used: 2 } } } = Connections.get_state(name) -- cgit v1.2.3 From 05ca6a7d2c27509b52a985810c0e22f0b1cb5112 Mon Sep 17 00:00:00 2001 From: Alex S Date: Mon, 19 Aug 2019 16:27:59 +0300 Subject: test fixes --- lib/pleroma/gun/connections.ex | 2 +- lib/pleroma/http/http.ex | 4 +++- test/reverse_proxy/reverse_proxy_test.exs | 12 +++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index cec3de2ca..b6845e52d 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -46,7 +46,7 @@ defmodule Pleroma.Gun.Connections do # TODO: only for testing, add this parameter to the config @spec try_to_get_gun_conn(String.t(), keyword(), atom()) :: nil | pid() - def try_to_get_gun_conn(url, opts \\ [], name \\ __MODULE__), + def try_to_get_gun_conn(url, opts \\ [], name \\ :default), do: try_to_get_gun_conn(url, opts, name, 0) @spec try_to_get_gun_conn(String.t(), keyword(), atom(), pos_integer()) :: nil | pid() diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index a7f42d0c0..26ab0bb03 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -62,7 +62,9 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.get_conn(url, options) do + pool = if options[:adapter][:pool], do: options[:adapter][:pool], else: :default + + case Pleroma.Gun.Connections.try_to_get_gun_conn(url, options, pool) do nil -> options diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 476cff643..8235fcc86 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -356,7 +356,9 @@ defmodule Pleroma.ReverseProxyTest do api = Pleroma.Config.get([Pleroma.Gun.API]) Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) - {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + + {:ok, _} = + Pleroma.Gun.Connections.start_link({:media, [max_connections: 5, timeout: 5_000]}) conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") @@ -379,7 +381,9 @@ defmodule Pleroma.ReverseProxyTest do api = Pleroma.Config.get([Pleroma.Gun.API]) Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) - {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + + {:ok, _} = + Pleroma.Gun.Connections.start_link({:media, [max_connections: 5, timeout: 5_000]}) conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10") @@ -402,7 +406,9 @@ defmodule Pleroma.ReverseProxyTest do api = Pleroma.Config.get([Pleroma.Gun.API]) Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) - {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + + {:ok, _} = + Pleroma.Gun.Connections.start_link({:media, [max_connections: 5, timeout: 5_000]}) conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5") -- cgit v1.2.3 From 3bc3e61dc5e7b6a10eee4c212dd113e6b2ca788a Mon Sep 17 00:00:00 2001 From: Alex S Date: Mon, 19 Aug 2019 17:43:06 +0300 Subject: some clean up and pleroma header --- config/config.exs | 4 ---- lib/pleroma/gun/api/gun.ex | 12 +++++++++--- lib/pleroma/gun/api/mock.ex | 24 ++++++++++++++---------- lib/pleroma/gun/connections.ex | 25 +++++-------------------- lib/pleroma/http/connection.ex | 3 ++- lib/pleroma/http/http.ex | 12 ++++++------ lib/pleroma/reverse_proxy/client/hackney.ex | 4 ++++ test/gun/connections_test.exs | 21 --------------------- test/reverse_proxy/client/hackney_test.exs | 4 ++++ test/reverse_proxy/client/tesla_test.exs | 4 ++++ test/support/reverse_proxy_client_case.ex | 4 ++++ 11 files changed, 52 insertions(+), 65 deletions(-) diff --git a/config/config.exs b/config/config.exs index e56b9730d..237c61ac9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -562,10 +562,6 @@ config :pleroma, :gun_pools, upload: [ max_connections: 25, timeout: 300_000 - ], - default: [ - max_connections: 10, - timout: 20_000 ] # Import environment specific config. This must remain at the bottom diff --git a/lib/pleroma/gun/api/gun.ex b/lib/pleroma/gun/api/gun.ex index 33e7985a1..d97f5a7c9 100644 --- a/lib/pleroma/gun/api/gun.ex +++ b/lib/pleroma/gun/api/gun.ex @@ -1,6 +1,12 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Gun.API.Gun do @behaviour Pleroma.Gun.API + alias Pleroma.Gun.API + @gun_keys [ :connect_timeout, :http_opts, @@ -15,14 +21,14 @@ defmodule Pleroma.Gun.API.Gun do :ws_opts ] - @impl Pleroma.Gun.API + @impl API def open(host, port, opts) do :gun.open(host, port, Map.take(opts, @gun_keys)) end - @impl Pleroma.Gun.API + @impl API def info(pid), do: :gun.info(pid) - @impl Pleroma.Gun.API + @impl API def close(pid), do: :gun.close(pid) end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex index a80559f0b..b1a30a73c 100644 --- a/lib/pleroma/gun/api/mock.ex +++ b/lib/pleroma/gun/api/mock.ex @@ -4,12 +4,15 @@ defmodule Pleroma.Gun.API.Mock do @behaviour Pleroma.Gun.API - @impl Pleroma.Gun.API + + alias Pleroma.Gun.API + + @impl API def open(domain, 80, %{genserver_pid: genserver_pid}) when domain in ['another-domain.com', 'some-domain.com'] do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + Registry.register(API.Mock, conn_pid, %{ origin_scheme: "http", origin_host: domain, origin_port: 80 @@ -19,10 +22,11 @@ defmodule Pleroma.Gun.API.Mock do {:ok, conn_pid} end + @impl API def open('some-domain.com', 443, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + Registry.register(API.Mock, conn_pid, %{ origin_scheme: "https", origin_host: 'some-domain.com', origin_port: 443 @@ -32,11 +36,11 @@ defmodule Pleroma.Gun.API.Mock do {:ok, conn_pid} end - @impl Pleroma.Gun.API + @impl API def open('gun_down.com', 80, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + Registry.register(API.Mock, conn_pid, %{ origin_scheme: "http", origin_host: 'gun_down.com', origin_port: 80 @@ -46,11 +50,11 @@ defmodule Pleroma.Gun.API.Mock do {:ok, conn_pid} end - @impl Pleroma.Gun.API + @impl API def open('gun_down_and_up.com', 80, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + Registry.register(API.Mock, conn_pid, %{ origin_scheme: "http", origin_host: 'gun_down_and_up.com', origin_port: 80 @@ -68,12 +72,12 @@ defmodule Pleroma.Gun.API.Mock do {:ok, conn_pid} end - @impl Pleroma.Gun.API + @impl API def info(pid) do - [{_, info}] = Registry.lookup(Pleroma.Gun.API.Mock, pid) + [{_, info}] = Registry.lookup(API.Mock, pid) info end - @impl Pleroma.Gun.API + @impl API def close(_pid), do: :ok end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index b6845e52d..361e8aaee 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -15,6 +15,8 @@ defmodule Pleroma.Gun.Connections do defstruct conns: %{}, opts: [] + alias Pleroma.Gun.API + @spec start_link({atom(), keyword()}) :: {:ok, pid()} | :ignore def start_link({name, opts}) do if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do @@ -44,21 +46,6 @@ defmodule Pleroma.Gun.Connections do ) end - # TODO: only for testing, add this parameter to the config - @spec try_to_get_gun_conn(String.t(), keyword(), atom()) :: nil | pid() - def try_to_get_gun_conn(url, opts \\ [], name \\ :default), - do: try_to_get_gun_conn(url, opts, name, 0) - - @spec try_to_get_gun_conn(String.t(), keyword(), atom(), pos_integer()) :: nil | pid() - def try_to_get_gun_conn(_url, _, _, 3), do: nil - - def try_to_get_gun_conn(url, opts, name, acc) do - case Pleroma.Gun.Connections.get_conn(url, opts, name) do - nil -> try_to_get_gun_conn(url, acc + 1) - conn -> conn - end - end - @spec alive?(atom()) :: boolean() def alive?(name \\ :default) do pid = Process.whereis(name) @@ -91,7 +78,7 @@ defmodule Pleroma.Gun.Connections do else [{close_key, least_used} | _conns] = Enum.sort_by(state.conns, fn {_k, v} -> v.used end) - :ok = Pleroma.Gun.API.close(least_used.conn) + :ok = API.close(least_used.conn) state = put_in( @@ -128,12 +115,10 @@ defmodule Pleroma.Gun.Connections do end @impl true - # Do we need to do something with killed & unprocessed references? def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed, _unprocessed}, state) do conn_key = compose_key_gun_info(conn_pid) {key, conn} = find_conn(state.conns, conn_pid, conn_key) - # We don't want to block requests to GenServer if gun send down message, return nil, so we can make some retries, while connection is not up Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) state = put_in(state.conns[key].state, :down) @@ -143,7 +128,7 @@ defmodule Pleroma.Gun.Connections do defp compose_key(uri), do: "#{uri.scheme}:#{uri.host}:#{uri.port}" defp compose_key_gun_info(pid) do - info = Pleroma.Gun.API.info(pid) + info = API.info(pid) "#{info.origin_scheme}:#{info.origin_host}:#{info.origin_port}" end @@ -154,7 +139,7 @@ defmodule Pleroma.Gun.Connections do end defp open_conn(key, uri, from, state, opts) do - {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) + {:ok, conn} = API.open(to_charlist(uri.host), uri.port, opts) state = put_in(state.conns[key], %Pleroma.Gun.Conn{ diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 6cb26c0fe..93e661e1b 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -9,7 +9,8 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, - timeout: 20_000 + timeout: 20_000, + pool: :federation ] @doc """ diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 26ab0bb03..ab0fd55de 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -34,9 +34,11 @@ defmodule Pleroma.HTTP do adapter_gun? = Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun + pool = options[:adapter][:pool] + options = - if adapter_gun? and Pleroma.Gun.Connections.alive?() do - get_conn_for_gun(url, options) + if adapter_gun? and not is_nil(pool) and Pleroma.Gun.Connections.alive?(pool) do + get_conn_for_gun(url, options, pool) else options end @@ -61,10 +63,8 @@ defmodule Pleroma.HTTP do end end - defp get_conn_for_gun(url, options) do - pool = if options[:adapter][:pool], do: options[:adapter][:pool], else: :default - - case Pleroma.Gun.Connections.try_to_get_gun_conn(url, options, pool) do + defp get_conn_for_gun(url, options, pool) do + case Pleroma.Gun.Connections.get_conn(url, options, pool) do nil -> options diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex index e6293646a..402c183af 100644 --- a/lib/pleroma/reverse_proxy/client/hackney.ex +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.ReverseProxy.Client.Hackney do @behaviour Pleroma.ReverseProxy.Client diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index 8308b5f9f..4da42d854 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -31,27 +31,6 @@ defmodule Gun.ConnectionsTest do end end - test "try_to_get_gun_conn/1 returns conn", %{name: name, pid: pid} do - conn = Connections.try_to_get_gun_conn("http://some-domain.com", [genserver_pid: pid], name) - assert is_pid(conn) - assert Process.alive?(conn) - - reused_conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) - - assert conn == reused_conn - - %Connections{ - conns: %{ - "http:some-domain.com:80" => %Conn{ - conn: ^conn, - state: :up, - waiting_pids: [], - used: 2 - } - } - } = Connections.get_state(name) - end - test "opens connection and reuse it on next request", %{name: name, pid: pid} do conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) diff --git a/test/reverse_proxy/client/hackney_test.exs b/test/reverse_proxy/client/hackney_test.exs index 577e0b0b2..cf9de912a 100644 --- a/test/reverse_proxy/client/hackney_test.exs +++ b/test/reverse_proxy/client/hackney_test.exs @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.ReverseProxy.Client.HackneyTest do use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Hackney diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs index 029a25d0f..a18d609d2 100644 --- a/test/reverse_proxy/client/tesla_test.exs +++ b/test/reverse_proxy/client/tesla_test.exs @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.ReverseProxy.Client.TeslaTest do use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Tesla diff --git a/test/support/reverse_proxy_client_case.ex b/test/support/reverse_proxy_client_case.ex index 40cd59ea2..16bc2803b 100644 --- a/test/support/reverse_proxy_client_case.ex +++ b/test/support/reverse_proxy_client_case.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.ReverseProxyClientCase do defmacro __using__(client: client) do quote do -- cgit v1.2.3 From af12d08da7d928f1855df882418ba88870e52047 Mon Sep 17 00:00:00 2001 From: Alex S Date: Mon, 19 Aug 2019 18:05:57 +0300 Subject: no need in this --- lib/pleroma/gun/connections.ex | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 361e8aaee..d3c44017d 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -19,11 +19,7 @@ defmodule Pleroma.Gun.Connections do @spec start_link({atom(), keyword()}) :: {:ok, pid()} | :ignore def start_link({name, opts}) do - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do - GenServer.start_link(__MODULE__, opts, name: name) - else - :ignore - end + GenServer.start_link(__MODULE__, opts, name: name) end @impl true -- cgit v1.2.3 From 2a2d11f2b305f415fb7eec7a799fd4c17d6e3446 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 10:51:56 +0300 Subject: some fixes and tests --- lib/pleroma/application.ex | 2 +- lib/pleroma/http/http.ex | 8 +++-- test/gun/connections_test.exs | 54 +++++++++++++++++++++++++++++- test/http_test.exs | 24 +++++++++++++ test/reverse_proxy/client/hackney_test.exs | 4 +++ test/reverse_proxy/client/tesla_test.exs | 4 +++ test/reverse_proxy/reverse_proxy_test.exs | 9 ----- test/support/reverse_proxy_client_case.ex | 2 +- 8 files changed, 93 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 06d1a187e..347f520bb 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -165,7 +165,7 @@ defmodule Pleroma.Application do end defp gun_pools do - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun || Mix.env() == :test do for {pool_name, opts} <- Pleroma.Config.get([:gun_pools]) do %{ id: :"gun_pool_#{pool_name}", diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index ab0fd55de..b18ce2803 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -89,11 +89,15 @@ defmodule Pleroma.HTTP do case uri.scheme do "https" -> + adapter_opts = Keyword.get(options, :adapter, []) + tls_opts = - Keyword.get(options, :tls_opts, []) + Keyword.get(adapter_opts, :tls_opts, []) |> Keyword.put(:server_name_indication, host) - Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] + adapter_opts = Keyword.put(adapter_opts, :tls_opts, tls_opts) + + Keyword.put(options, :adapter, adapter_opts) ++ [ssl: [server_name_indication: host]] _ -> options diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index 4da42d854..1e41e771b 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -4,7 +4,9 @@ defmodule Gun.ConnectionsTest do use ExUnit.Case - alias Pleroma.Gun.{Connections, Conn, API} + alias Pleroma.Gun.API + alias Pleroma.Gun.Conn + alias Pleroma.Gun.Connections setup_all do {:ok, _} = Registry.start_link(keys: :unique, name: API.Mock) @@ -262,5 +264,55 @@ defmodule Gun.ConnectionsTest do } } = Connections.get_state(name) end + + test "remove frequently used", %{name: name, pid: pid} do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], API.Gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + + Connections.get_conn("https://www.google.com", [genserver_pid: pid], name) + + for _ <- 1..4 do + Connections.get_conn("https://httpbin.org", [genserver_pid: pid], name) + end + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [], + used: 4 + }, + "https:www.google.com:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [], + used: 1 + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + conn = Connections.get_conn("http://httpbin.org", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "http:httpbin.org:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [], + used: 1 + }, + "https:httpbin.org:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [], + used: 4 + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + end end end diff --git a/test/http_test.exs b/test/http_test.exs index 5f9522cf0..5ffd351e4 100644 --- a/test/http_test.exs +++ b/test/http_test.exs @@ -56,4 +56,28 @@ defmodule Pleroma.HTTPTest do } end end + + @tag :integration + test "get_conn_for_gun/3" do + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) + + on_exit(fn -> + Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) + end) + + options = [adapter: [pool: :federation]] + + assert {:ok, resp} = + Pleroma.HTTP.request(:get, "https://httpbin.org/user-agent", "", [], options) + + adapter_opts = resp.opts[:adapter] + + assert adapter_opts[:original] == "httpbin.org:443" + refute adapter_opts[:close_conn] + assert adapter_opts[:pool] == :federation + end end diff --git a/test/reverse_proxy/client/hackney_test.exs b/test/reverse_proxy/client/hackney_test.exs index cf9de912a..3c552dc83 100644 --- a/test/reverse_proxy/client/hackney_test.exs +++ b/test/reverse_proxy/client/hackney_test.exs @@ -8,4 +8,8 @@ defmodule Pleroma.ReverseProxy.Client.HackneyTest do defp check_ref(ref) do assert is_reference(ref) end + + defp close(ref) do + Pleroma.ReverseProxy.Client.Hackney.close(ref) + end end diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs index a18d609d2..8adefbe59 100644 --- a/test/reverse_proxy/client/tesla_test.exs +++ b/test/reverse_proxy/client/tesla_test.exs @@ -10,4 +10,8 @@ defmodule Pleroma.ReverseProxy.Client.TeslaTest do assert is_reference(stream) assert ref[:fin] end + + defp close(%{pid: pid}) do + Pleroma.ReverseProxy.Client.Tesla.close(pid) + end end diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 8235fcc86..1d3e554b3 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -357,9 +357,6 @@ defmodule Pleroma.ReverseProxyTest do api = Pleroma.Config.get([Pleroma.Gun.API]) Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) - {:ok, _} = - Pleroma.Gun.Connections.start_link({:media, [max_connections: 5, timeout: 5_000]}) - conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") assert byte_size(conn.resp_body) == 10 @@ -382,9 +379,6 @@ defmodule Pleroma.ReverseProxyTest do api = Pleroma.Config.get([Pleroma.Gun.API]) Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) - {:ok, _} = - Pleroma.Gun.Connections.start_link({:media, [max_connections: 5, timeout: 5_000]}) - conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10") assert byte_size(conn.resp_body) == 10 @@ -407,9 +401,6 @@ defmodule Pleroma.ReverseProxyTest do api = Pleroma.Config.get([Pleroma.Gun.API]) Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) - {:ok, _} = - Pleroma.Gun.Connections.start_link({:media, [max_connections: 5, timeout: 5_000]}) - conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5") assert conn.state == :chunked diff --git a/test/support/reverse_proxy_client_case.ex b/test/support/reverse_proxy_client_case.ex index 16bc2803b..36df1ed95 100644 --- a/test/support/reverse_proxy_client_case.ex +++ b/test/support/reverse_proxy_client_case.ex @@ -55,7 +55,7 @@ defmodule Pleroma.ReverseProxyClientCase do assert headers != [] check_ref(ref) - assert :ok = @client.close(ref) + assert :ok == close(ref) {:ok, status, headers} -> assert headers != [] -- cgit v1.2.3 From 5209abf09b3b0ec4c2cf40ac2d7c5eb38472ee0f Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 12:11:54 +0300 Subject: testing without sni --- lib/pleroma/http/http.ex | 5 ++--- mix.exs | 2 +- mix.lock | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index b18ce2803..54ea04764 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -91,9 +91,8 @@ defmodule Pleroma.HTTP do "https" -> adapter_opts = Keyword.get(options, :adapter, []) - tls_opts = - Keyword.get(adapter_opts, :tls_opts, []) - |> Keyword.put(:server_name_indication, host) + tls_opts = Keyword.get(adapter_opts, :tls_opts, []) + # |> Keyword.put(:server_name_indication, host) adapter_opts = Keyword.put(adapter_opts, :tls_opts, tls_opts) diff --git a/mix.exs b/mix.exs index d51b1488d..8d9a63362 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "81433bd4011217323a1f443661252fdab23f5e59", + ref: "a64f34a83b579d46cc4ec7bac2ef561cc8b03f1d", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index daec58b4f..f0633f8a9 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "81433bd4011217323a1f443661252fdab23f5e59", [ref: "81433bd4011217323a1f443661252fdab23f5e59"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "a64f34a83b579d46cc4ec7bac2ef561cc8b03f1d", [ref: "a64f34a83b579d46cc4ec7bac2ef561cc8b03f1d"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, -- cgit v1.2.3 From 6c3990b1eaa57f4a633b3e136089ebaa9edc2d38 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 12:25:58 +0300 Subject: with sni --- lib/pleroma/http/http.ex | 2 +- mix.exs | 2 +- mix.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 54ea04764..aca378c9d 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -92,7 +92,7 @@ defmodule Pleroma.HTTP do adapter_opts = Keyword.get(options, :adapter, []) tls_opts = Keyword.get(adapter_opts, :tls_opts, []) - # |> Keyword.put(:server_name_indication, host) + |> Keyword.put(:server_name_indication, host) adapter_opts = Keyword.put(adapter_opts, :tls_opts, tls_opts) diff --git a/mix.exs b/mix.exs index 8d9a63362..8b1a898bf 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "a64f34a83b579d46cc4ec7bac2ef561cc8b03f1d", + ref: "245f75502052e53d45cdb44f6afb481f4bd0f576", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index f0633f8a9..484a3a1a9 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "a64f34a83b579d46cc4ec7bac2ef561cc8b03f1d", [ref: "a64f34a83b579d46cc4ec7bac2ef561cc8b03f1d"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "245f75502052e53d45cdb44f6afb481f4bd0f576", [ref: "245f75502052e53d45cdb44f6afb481f4bd0f576"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, -- cgit v1.2.3 From b8ee02579275b2c2fd7b511b357ae039b9c82fb7 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 12:47:16 +0300 Subject: rebase fixes --- test/reverse_proxy/reverse_proxy_test.exs | 32 ------------------------------- 1 file changed, 32 deletions(-) diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 1d3e554b3..a1adaedf6 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -108,38 +108,6 @@ defmodule Pleroma.ReverseProxyTest do "[error] Elixir.Pleroma.ReverseProxy: request to \"/user-agent\" failed: :body_too_large" end - defp stream_mock(invokes, with_close? \\ false) do - ClientMock - |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) - - {:ok, 200, [{"content-type", "application/octet-stream"}], - %{url: "/stream-bytes/" <> length}} - end) - |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} -> - max = String.to_integer(length) - - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do - [{_, current}] when current < max -> - Registry.update_value( - Pleroma.ReverseProxy.ClientMock, - "/stream-bytes/" <> length, - &(&1 + 10) - ) - - {:ok, "0123456789"} - - [{_, ^max}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) - :done - end - end) - - if with_close? do - expect(ClientMock, :close, fn _ -> :ok end) - end - end - test "max_body_length returns error if streaming body more than that option", %{conn: conn} do stream_mock(3, true) -- cgit v1.2.3 From 70c4f003497ca6a324014c112a4276666e2e8ed0 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 13:10:28 +0300 Subject: formatting and pool fix --- lib/pleroma/gun/connections.ex | 10 ++++++++-- lib/pleroma/http/http.ex | 5 +++-- test/web/activity_pub/activity_pub_test.exs | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index d3c44017d..5b0605026 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -112,8 +112,8 @@ defmodule Pleroma.Gun.Connections do @impl true def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed, _unprocessed}, state) do - conn_key = compose_key_gun_info(conn_pid) - {key, conn} = find_conn(state.conns, conn_pid, conn_key) + # we can't get info on this pid, because pid is dead + {key, conn} = find_conn(state.conns, conn_pid) Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) @@ -128,6 +128,12 @@ defmodule Pleroma.Gun.Connections do "#{info.origin_scheme}:#{info.origin_host}:#{info.origin_port}" end + defp find_conn(conns, conn_pid) do + Enum.find(conns, fn {_key, conn} -> + conn.conn == conn_pid + end) + end + defp find_conn(conns, conn_pid, conn_key) do Enum.find(conns, fn {key, conn} -> key == conn_key and conn.conn == conn_pid diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index aca378c9d..b18ce2803 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -91,8 +91,9 @@ defmodule Pleroma.HTTP do "https" -> adapter_opts = Keyword.get(options, :adapter, []) - tls_opts = Keyword.get(adapter_opts, :tls_opts, []) - |> Keyword.put(:server_name_indication, host) + tls_opts = + Keyword.get(adapter_opts, :tls_opts, []) + |> Keyword.put(:server_name_indication, host) adapter_opts = Keyword.put(adapter_opts, :tls_opts, tls_opts) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index f20cd2840..1515f4eb6 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -555,7 +555,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do note_two = insert(:note, data: %{"context" => "suya.."}) activity_two = insert(:note_activity, note: note_two) - {:ok, activity_two} = CommonAPI.add_mute(user, activity_two) + {:ok, _activity_two} = CommonAPI.add_mute(user, activity_two) assert [_activity_two, _activity_one] = ActivityPub.fetch_activities([], %{"muting_user" => user, "with_muted" => true}) -- cgit v1.2.3 From 50bc6c5ea98494ceeda7cdb18d75defe485d9a64 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 16:09:50 +0300 Subject: tesla gun adapter supports sni --- lib/pleroma/http/connection.ex | 3 ++- lib/pleroma/http/http.ex | 10 +--------- mix.exs | 2 +- mix.lock | 2 +- 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 93e661e1b..69589bc5e 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -10,7 +10,8 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, timeout: 20_000, - pool: :federation + pool: :federation, + version: :master ] @doc """ diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index b18ce2803..89afba720 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -89,15 +89,7 @@ defmodule Pleroma.HTTP do case uri.scheme do "https" -> - adapter_opts = Keyword.get(options, :adapter, []) - - tls_opts = - Keyword.get(adapter_opts, :tls_opts, []) - |> Keyword.put(:server_name_indication, host) - - adapter_opts = Keyword.put(adapter_opts, :tls_opts, tls_opts) - - Keyword.put(options, :adapter, adapter_opts) ++ [ssl: [server_name_indication: host]] + options ++ [ssl: [server_name_indication: host]] _ -> options diff --git a/mix.exs b/mix.exs index 8b1a898bf..9926a0836 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "245f75502052e53d45cdb44f6afb481f4bd0f576", + ref: "9f56f950a88a9ab0c21ad351e2cfd47b98594310", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index 484a3a1a9..a10677c49 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "245f75502052e53d45cdb44f6afb481f4bd0f576", [ref: "245f75502052e53d45cdb44f6afb481f4bd0f576"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "9f56f950a88a9ab0c21ad351e2cfd47b98594310", [ref: "9f56f950a88a9ab0c21ad351e2cfd47b98594310"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, -- cgit v1.2.3 From 6f08e772693d3b45da3b548404a56be1c88ec2da Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 18:06:08 +0300 Subject: sni option if we open connection in gun pool --- lib/pleroma/gun/connections.ex | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 5b0605026..3716d9f74 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -36,6 +36,19 @@ defmodule Pleroma.Gun.Connections do do: Map.put(opts, :transport, :tls), else: opts + opts = + if uri.scheme == "https" do + host = uri.host |> to_charlist() + + tls_opts = + Map.get(opts, :tls_opts, []) + |> Keyword.put(:server_name_indication, host) + + Map.put(opts, :tls_opts, tls_opts) + else + opts + end + GenServer.call( name, {:conn, %{opts: opts, uri: uri}} -- cgit v1.2.3 From cee945bead8d6e95e05ccb661330cd5561caf7a3 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 18:37:22 +0300 Subject: set adapter opts in connection --- lib/pleroma/http/connection.ex | 32 ++++++++++++++++++++++++++++---- lib/pleroma/http/http.ex | 28 ++++++---------------------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 69589bc5e..ef2ee918d 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -35,9 +35,33 @@ defmodule Pleroma.HTTP.Connection do adapter_options = Pleroma.Config.get([:http, :adapter], []) proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) - @options - |> Keyword.merge(adapter_options) - |> Keyword.merge(options) - |> Keyword.merge(proxy: proxy_url) + options = + @options + |> Keyword.merge(adapter_options) + |> Keyword.merge(options) + |> Keyword.merge(proxy: proxy_url) + + pool = options[:pool] + url = options[:url] + + if not is_nil(url) and not is_nil(pool) and Pleroma.Gun.Connections.alive?(pool) do + get_conn_for_gun(url, options, pool) + else + options + end + end + + defp get_conn_for_gun(url, options, pool) do + case Pleroma.Gun.Connections.get_conn(url, options, pool) do + nil -> + options + + conn -> + %{host: host, port: port} = URI.parse(url) + + Keyword.put(options, :conn, conn) + |> Keyword.put(:close_conn, false) + |> Keyword.put(:original, "#{host}:#{port}") + end end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 89afba720..da86ebcc7 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -34,11 +34,13 @@ defmodule Pleroma.HTTP do adapter_gun? = Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun - pool = options[:adapter][:pool] - options = - if adapter_gun? and not is_nil(pool) and Pleroma.Gun.Connections.alive?(pool) do - get_conn_for_gun(url, options, pool) + if adapter_gun? do + adapter_opts = + Keyword.get(options, :adapter, []) + |> Keyword.put(:url, url) + + Keyword.put(options, :adapter, adapter_opts) else options end @@ -63,24 +65,6 @@ defmodule Pleroma.HTTP do end end - defp get_conn_for_gun(url, options, pool) do - case Pleroma.Gun.Connections.get_conn(url, options, pool) do - nil -> - options - - conn -> - %{host: host, port: port} = URI.parse(url) - - adapter_opts = - Keyword.get(options, :adapter, []) - |> Keyword.put(:conn, conn) - |> Keyword.put(:close_conn, false) - |> Keyword.put(:original, "#{host}:#{port}") - - Keyword.put(options, :adapter, adapter_opts) - end - end - defp process_sni_options(options, nil), do: options defp process_sni_options(options, url) do -- cgit v1.2.3 From 4930bc3ca100f4ee299a30377545dce66c7c88c8 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 18:56:15 +0300 Subject: test fix --- test/http_test.exs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/http_test.exs b/test/http_test.exs index 5ffd351e4..460f8f7c2 100644 --- a/test/http_test.exs +++ b/test/http_test.exs @@ -76,8 +76,6 @@ defmodule Pleroma.HTTPTest do adapter_opts = resp.opts[:adapter] - assert adapter_opts[:original] == "httpbin.org:443" - refute adapter_opts[:close_conn] - assert adapter_opts[:pool] == :federation + assert adapter_opts[:url] == "https://httpbin.org/user-agent" end end -- cgit v1.2.3