Merge branch 'release/2.1.0' into 'stable'v2.1.0

Release/2.1.0

See merge request pleroma/pleroma!2927

4 files changed, 175 insertions, 40 deletions

diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex
index 26d14fabd..0d13ff174 100644
--- a/lib/pleroma/reverse_proxy/client.ex
+++ b/lib/pleroma/reverse_proxy/client.ex
@@ -3,19 +3,23 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy.Client do
-  @callback request(atom(), String.t(), [tuple()], String.t(), list()) ::
-              {:ok, pos_integer(), [tuple()], reference() | map()}
-              | {:ok, pos_integer(), [tuple()]}
+  @type status :: pos_integer()
+  @type header_name :: String.t()
+  @type header_value :: String.t()
+  @type headers :: [{header_name(), header_value()}]
+
+  @callback request(atom(), String.t(), headers(), String.t(), list()) ::
+              {:ok, status(), headers(), reference() | map()}
+              | {:ok, status(), headers()}
               | {:ok, reference()}
               | {:error, term()}
 
-  @callback stream_body(reference() | pid() | map()) ::
-              {:ok, binary()} | :done | {:error, String.t()}
+  @callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()}
 
   @callback close(reference() | pid() | map()) :: :ok
 
-  def request(method, url, headers, "", opts \\ []) do
-    client().request(method, url, headers, "", opts)
+  def request(method, url, headers, body \\ "", opts \\ []) do
+    client().request(method, url, headers, body, opts)
   end
 
   def stream_body(ref), do: client().stream_body(ref)
@@ -23,6 +27,12 @@ defmodule Pleroma.ReverseProxy.Client do
   def close(ref), do: client().close(ref)
 
   defp client do
-    Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney)
+    :tesla
+    |> Application.get_env(:adapter)
+    |> client()
   end
+
+  defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney
+  defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla
+  defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client)
 end
diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex
new file mode 100644
index 000000000..e84118a90
--- /dev/null
+++ b/lib/pleroma/reverse_proxy/client/hackney.ex
@@ -0,0 +1,24 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ReverseProxy.Client.Hackney do
+  @behaviour Pleroma.ReverseProxy.Client
+
+  @impl true
+  def request(method, url, headers, body, opts \\ []) do
+    :hackney.request(method, url, headers, body, opts)
+  end
+
+  @impl true
+  def stream_body(ref) do
+    case :hackney.stream_body(ref) do
+      :done -> :done
+      {:ok, data} -> {:ok, data, ref}
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  @impl true
+  def close(ref), do: :hackney.close(ref)
+end
diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex
new file mode 100644
index 000000000..d5a339681
--- /dev/null
+++ b/lib/pleroma/reverse_proxy/client/tesla.ex
@@ -0,0 +1,86 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ReverseProxy.Client.Tesla do
+  @behaviour Pleroma.ReverseProxy.Client
+
+  alias Pleroma.Gun.ConnectionPool
+
+  @type headers() :: [{String.t(), String.t()}]
+  @type status() :: pos_integer()
+
+  @spec request(atom(), String.t(), headers(), String.t(), keyword()) ::
+          {:ok, status(), headers}
+          | {:ok, status(), headers, map()}
+          | {:error, atom() | String.t()}
+          | no_return()
+
+  @impl true
+  def request(method, url, headers, body, opts \\ []) do
+    check_adapter()
+
+    opts = Keyword.put(opts, :body_as, :chunks)
+
+    with {:ok, response} <-
+           Pleroma.HTTP.request(
+             method,
+             url,
+             body,
+             headers,
+             Keyword.put(opts, :adapter, opts)
+           ) do
+      if is_map(response.body) and method != :head do
+        {:ok, response.status, response.headers, response.body}
+      else
+        conn_pid = response.opts[:adapter][:conn]
+        ConnectionPool.release_conn(conn_pid)
+        {:ok, response.status, response.headers}
+      end
+    else
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  @impl true
+  @spec stream_body(map()) ::
+          {:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
+  def stream_body(%{pid: pid, fin: true}) do
+    ConnectionPool.release_conn(pid)
+    :done
+  end
+
+  def stream_body(client) do
+    case read_chunk!(client) do
+      {:fin, body} ->
+        {:ok, body, Map.put(client, :fin, true)}
+
+      {:nofin, part} ->
+        {:ok, part, client}
+
+      {:error, error} ->
+        {:error, error}
+    end
+  end
+
+  defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do
+    adapter = check_adapter()
+    adapter.read_chunk(pid, stream, opts)
+  end
+
+  @impl true
+  @spec close(map) :: :ok | no_return()
+  def close(%{pid: pid}) do
+    ConnectionPool.release_conn(pid)
+  end
+
+  defp check_adapter do
+    adapter = Application.get_env(:tesla, :adapter)
+
+    unless adapter == Tesla.Adapter.Gun do
+      raise "#{adapter} doesn't support reading body in chunks"
+    end
+
+    adapter
+  end
+end
diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex
index 8b713b8f4..0de4e2309 100644
--- a/lib/pleroma/reverse_proxy/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex
@@ -3,14 +3,13 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy do
-  alias Pleroma.HTTP
-
+  @range_headers ~w(range if-range)
   @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++
-                      ~w(if-unmodified-since if-none-match if-range range)
+                      ~w(if-unmodified-since if-none-match) ++ @range_headers
   @resp_cache_headers ~w(etag date last-modified)
   @keep_resp_headers @resp_cache_headers ++
-                       ~w(content-type content-disposition content-encoding content-range) ++
-                       ~w(accept-ranges vary)
+                       ~w(content-length content-type content-disposition content-encoding) ++
+                       ~w(content-range accept-ranges vary)
   @default_cache_control_header "public, max-age=1209600"
   @valid_resp_codes [200, 206, 304]
   @max_read_duration :timer.seconds(30)
@@ -58,10 +57,10 @@ defmodule Pleroma.ReverseProxy do
 
   * `req_headers`, `resp_headers` additional headers.
 
-  * `http`: options for [hackney](https://github.com/benoitc/hackney).
+  * `http`: options for [hackney](https://github.com/benoitc/hackney) or [gun](https://github.com/ninenines/gun).
 
   """
-  @default_hackney_options [pool: :media]
+  @default_options [pool: :media]
 
   @inline_content_types [
     "image/gif",
@@ -94,11 +93,7 @@ defmodule Pleroma.ReverseProxy do
   def call(_conn, _url, _opts \\ [])
 
   def call(conn = %{method: method}, url, opts) when method in @methods do
-    hackney_opts =
-      Pleroma.HTTP.Connection.hackney_options([])
-      |> Keyword.merge(@default_hackney_options)
-      |> Keyword.merge(Keyword.get(opts, :http, []))
-      |> HTTP.process_request_options()
+    client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, []))
 
     req_headers = build_req_headers(conn.req_headers, opts)
 
@@ -110,7 +105,7 @@ defmodule Pleroma.ReverseProxy do
       end
 
     with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url),
-         {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts),
+         {:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
          :ok <-
            header_length_constraint(
              headers,
@@ -156,11 +151,11 @@ defmodule Pleroma.ReverseProxy do
     |> halt()
   end
 
-  defp request(method, url, headers, hackney_opts) do
+  defp request(method, url, headers, opts) do
     Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
     method = method |> String.downcase() |> String.to_existing_atom()
 
-    case client().request(method, url, headers, "", hackney_opts) do
+    case client().request(method, url, headers, "", opts) do
       {:ok, code, headers, client} when code in @valid_resp_codes ->
         {:ok, code, downcase_headers(headers), client}
 
@@ -170,12 +165,17 @@ defmodule Pleroma.ReverseProxy do
       {:ok, code, _, _} ->
         {:error, {:invalid_http_response, code}}
 
+      {:ok, code, _} ->
+        {:error, {:invalid_http_response, code}}
+
       {:error, error} ->
         {:error, error}
     end
   end
 
   defp response(conn, client, url, status, headers, opts) do
+    Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}")
+
     result =
       conn
       |> put_resp_headers(build_resp_headers(headers, opts))
@@ -210,7 +210,7 @@ defmodule Pleroma.ReverseProxy do
              duration,
              Keyword.get(opts, :max_read_duration, @max_read_duration)
            ),
-         {:ok, data} <- client().stream_body(client),
+         {:ok, data, client} <- client().stream_body(client),
          {:ok, duration} <- increase_read_duration(duration),
          sent_so_far = sent_so_far + byte_size(data),
          :ok <-
@@ -226,7 +226,9 @@ defmodule Pleroma.ReverseProxy do
     end
   end
 
-  defp head_response(conn, _url, code, headers, opts) do
+  defp head_response(conn, url, code, headers, opts) do
+    Logger.debug("#{__MODULE__} #{code} #{url} #{inspect(headers)}")
+
     conn
     |> put_resp_headers(build_resp_headers(headers, opts))
     |> send_resp(code, "")
@@ -268,20 +270,33 @@ defmodule Pleroma.ReverseProxy do
     headers
     |> downcase_headers()
     |> Enum.filter(fn {k, _} -> k in @keep_req_headers end)
-    |> (fn headers ->
-          headers = headers ++ Keyword.get(opts, :req_headers, [])
-
-          if Keyword.get(opts, :keep_user_agent, false) do
-            List.keystore(
-              headers,
-              "user-agent",
-              0,
-              {"user-agent", Pleroma.Application.user_agent()}
-            )
-          else
-            headers
-          end
-        end).()
+    |> build_req_range_or_encoding_header(opts)
+    |> build_req_user_agent_header(opts)
+    |> Keyword.merge(Keyword.get(opts, :req_headers, []))
+  end
+
+  # Disable content-encoding if any @range_headers are requested (see #1823).
+  defp build_req_range_or_encoding_header(headers, _opts) do
+    range? = Enum.any?(headers, fn {header, _} -> Enum.member?(@range_headers, header) end)
+
+    if range? && List.keymember?(headers, "accept-encoding", 0) do
+      List.keydelete(headers, "accept-encoding", 0)
+    else
+      headers
+    end
+  end
+
+  defp build_req_user_agent_header(headers, opts) do
+    if Keyword.get(opts, :keep_user_agent, false) do
+      List.keystore(
+        headers,
+        "user-agent",
+        0,
+        {"user-agent", Pleroma.Application.user_agent()}
+      )
+    else
+      headers
+    end
   end
 
   defp build_resp_headers(headers, opts) do
@@ -289,7 +304,7 @@ defmodule Pleroma.ReverseProxy do
     |> Enum.filter(fn {k, _} -> k in @keep_resp_headers end)
     |> build_resp_cache_headers(opts)
     |> build_resp_content_disposition_header(opts)
-    |> (fn headers -> headers ++ Keyword.get(opts, :resp_headers, []) end).()
+    |> Keyword.merge(Keyword.get(opts, :resp_headers, []))
   end
 
   defp build_resp_cache_headers(headers, _opts) do