Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into validate-user-info

7 files changed, 46 insertions, 162 deletions

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 5cb350c8d..7e207c620 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -572,10 +572,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Enum.reverse()
   end
 
-  def upload(file, size_limit \\ nil) do
-    with data <-
-           Upload.store(file, Application.get_env(:pleroma, :instance)[:dedupe_media], size_limit),
-         false <- is_nil(data) do
+  def upload(file, opts \\ []) do
+    with {:ok, data} <- Upload.store(file, opts) do
       Repo.insert(%Object{data: data})
     end
   end
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 8728c908b..c5f9d51d9 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -12,7 +12,7 @@ defmodule Pleroma.Web.Endpoint do
   plug(CORSPlug)
   plug(Pleroma.Plugs.HTTPSecurityPlug)
 
-  plug(Plug.Static, at: "/media", from: Pleroma.Uploaders.Local.upload_path(), gzip: false)
+  plug(Pleroma.Plugs.UploadedMedia)
 
   plug(
     Plug.Static,
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 3a49a96a1..f209bb43c 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -35,14 +35,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   def update_credentials(%{assigns: %{user: user}} = conn, params) do
     original_user = user
 
-    avatar_upload_limit =
-      Application.get_env(:pleroma, :instance)
-      |> Keyword.fetch(:avatar_upload_limit)
-
-    banner_upload_limit =
-      Application.get_env(:pleroma, :instance)
-      |> Keyword.fetch(:banner_upload_limit)
-
     params =
       if bio = params["note"] do
         Map.put(params, "bio", bio)
@@ -60,7 +52,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     user =
       if avatar = params["avatar"] do
         with %Plug.Upload{} <- avatar,
-             {:ok, object} <- ActivityPub.upload(avatar, avatar_upload_limit),
+             {:ok, object} <- ActivityPub.upload(avatar, type: :avatar),
              change = Ecto.Changeset.change(user, %{avatar: object.data}),
              {:ok, user} = User.update_and_set_cache(change) do
           user
@@ -74,7 +66,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     # user =
     #   if banner = params["header"] do
     #     with %Plug.Upload{} <- banner,
-    #          {:ok, object} <- ActivityPub.upload(banner, banner_upload_limit),
+    #          {:ok, object} <- ActivityPub.upload(banner, type: :banner),
     #          new_info <- Map.put(user.info, "banner", object.data),
     #          change <- User.info_changeset(user, %{info: new_info}),
     #          {:ok, user} <- User.update_and_set_cache(change) do
@@ -471,19 +463,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def upload(%{assigns: %{user: _}} = conn, %{"file" => file} = data) do
-    with {:ok, object} <- ActivityPub.upload(file) do
-      objdata =
-        if Map.has_key?(data, "description") do
-          Map.put(object.data, "name", data["description"])
-        else
-          object.data
-        end
-
-      change = Object.change(object, %{data: objdata})
+    with {:ok, object} <- ActivityPub.upload(file, description: Map.get(data, "description")) do
+      change = Object.change(object, %{data: object.data})
       {:ok, object} = Repo.update(change)
 
       objdata =
-        objdata
+        object.data
         |> Map.put("id", object.id)
 
       render(conn, StatusView, "attachment.json", %{attachment: objdata})
diff --git a/lib/pleroma/web/media_proxy/controller.ex b/lib/pleroma/web/media_proxy/controller.ex
index bb257c262..81ea5d510 100644
--- a/lib/pleroma/web/media_proxy/controller.ex
+++ b/lib/pleroma/web/media_proxy/controller.ex
@@ -1,135 +1,34 @@
 defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   use Pleroma.Web, :controller
-  require Logger
+  alias Pleroma.{Web.MediaProxy, ReverseProxy}
 
-  @httpoison Application.get_env(:pleroma, :httpoison)
+  @default_proxy_opts [max_body_length: 25 * 1_048_576]
 
-  @max_body_length 25 * 1_048_576
-
-  @cache_control %{
-    default: "public, max-age=1209600",
-    error: "public, must-revalidate, max-age=160"
-  }
-
-  # Content-types that will not be returned as content-disposition attachments
-  # Override with :media_proxy, :safe_content_types in the configuration
-  @safe_content_types [
-    "image/gif",
-    "image/jpeg",
-    "image/jpg",
-    "image/png",
-    "image/svg+xml",
-    "audio/mpeg",
-    "audio/mp3",
-    "video/webm",
-    "video/mp4"
-  ]
-
-  def remote(conn, params = %{"sig" => sig, "url" => url}) do
-    config = Application.get_env(:pleroma, :media_proxy, [])
-
-    with true <- Keyword.get(config, :enabled, false),
-         {:ok, url} <- Pleroma.Web.MediaProxy.decode_url(sig, url),
+  def remote(conn, params = %{"sig" => sig64, "url" => url64}) do
+    with config <- Pleroma.Config.get([:media_proxy]),
+         true <- Keyword.get(config, :enabled, false),
+         {:ok, url} <- MediaProxy.decode_url(sig64, url64),
          filename <- Path.basename(URI.parse(url).path),
-         true <-
-           if(Map.get(params, "filename"),
-             do: filename == Path.basename(conn.request_path),
-             else: true
-           ),
-         {:ok, content_type, body} <- proxy_request(url),
-         safe_content_type <-
-           Enum.member?(
-             Keyword.get(config, :safe_content_types, @safe_content_types),
-             content_type
-           ) do
-      conn
-      |> put_resp_content_type(content_type)
-      |> set_cache_header(:default)
-      |> put_resp_header(
-        "content-security-policy",
-        "default-src 'none'; style-src 'unsafe-inline'; media-src data:; img-src 'self' data:"
-      )
-      |> put_resp_header("x-xss-protection", "1; mode=block")
-      |> put_resp_header("x-content-type-options", "nosniff")
-      |> put_attachement_header(safe_content_type, filename)
-      |> send_resp(200, body)
+         :ok <- filename_matches(Map.has_key?(params, "filename"), conn.request_path, url) do
+      ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_length))
     else
       false ->
-        send_error(conn, 404)
+        send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
 
       {:error, :invalid_signature} ->
-        send_error(conn, 403)
+        send_resp(conn, 403, Plug.Conn.Status.reason_phrase(403))
 
-      {:error, {:http, _, url}} ->
-        redirect_or_error(conn, url, Keyword.get(config, :redirect_on_failure, true))
+      {:wrong_filename, filename} ->
+        redirect(conn, external: MediaProxy.build_url(sig64, url64, filename))
     end
   end
 
-  defp proxy_request(link) do
-    headers = [
-      {"user-agent",
-       "Pleroma/MediaProxy; #{Pleroma.Web.base_url()} <#{
-         Application.get_env(:pleroma, :instance)[:email]
-       }>"}
-    ]
-
-    options =
-      @httpoison.process_request_options([:insecure, {:follow_redirect, true}]) ++
-        [{:pool, :default}]
-
-    with {:ok, 200, headers, client} <- :hackney.request(:get, link, headers, "", options),
-         headers = Enum.into(headers, Map.new()),
-         {:ok, body} <- proxy_request_body(client),
-         content_type <- proxy_request_content_type(headers, body) do
-      {:ok, content_type, body}
-    else
-      {:ok, status, _, _} ->
-        Logger.warn("MediaProxy: request failed, status #{status}, link: #{link}")
-        {:error, {:http, :bad_status, link}}
+  def filename_matches(has_filename, path, url) do
+    filename = MediaProxy.filename(url)
 
-      {:error, error} ->
-        Logger.warn("MediaProxy: request failed, error #{inspect(error)}, link: #{link}")
-        {:error, {:http, error, link}}
+    cond do
+      has_filename && filename && Path.basename(path) != filename -> {:wrong_filename, filename}
+      true -> :ok
     end
   end
-
-  defp set_cache_header(conn, key) do
-    Plug.Conn.put_resp_header(conn, "cache-control", @cache_control[key])
-  end
-
-  defp redirect_or_error(conn, url, true), do: redirect(conn, external: url)
-  defp redirect_or_error(conn, url, _), do: send_error(conn, 502, "Media proxy error: " <> url)
-
-  defp send_error(conn, code, body \\ "") do
-    conn
-    |> set_cache_header(:error)
-    |> send_resp(code, body)
-  end
-
-  defp proxy_request_body(client), do: proxy_request_body(client, <<>>)
-
-  defp proxy_request_body(client, body) when byte_size(body) < @max_body_length do
-    case :hackney.stream_body(client) do
-      {:ok, data} -> proxy_request_body(client, <<body::binary, data::binary>>)
-      :done -> {:ok, body}
-      {:error, reason} -> {:error, reason}
-    end
-  end
-
-  defp proxy_request_body(client, _) do
-    :hackney.close(client)
-    {:error, :body_too_large}
-  end
-
-  # TODO: the body is passed here as well because some hosts do not provide a content-type.
-  # At some point we may want to use magic numbers to discover the content-type and reply a proper one.
-  defp proxy_request_content_type(headers, _body) do
-    headers["Content-Type"] || headers["content-type"] || "application/octet-stream"
-  end
-
-  defp put_attachement_header(conn, true, _), do: conn
-
-  defp put_attachement_header(conn, false, filename) do
-    put_resp_header(conn, "content-disposition", "attachment; filename='#{filename}'")
-  end
 end
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index 0fc0a07b2..28aacb0b1 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -17,10 +17,8 @@ defmodule Pleroma.Web.MediaProxy do
       base64 = Base.url_encode64(url, @base64_opts)
       sig = :crypto.hmac(:sha, secret, base64)
       sig64 = sig |> Base.url_encode64(@base64_opts)
-      filename = if path = URI.parse(url).path, do: "/" <> Path.basename(path), else: ""
 
-      Keyword.get(config, :base_url, Pleroma.Web.base_url()) <>
-        "/proxy/#{sig64}/#{base64}#{filename}"
+      build_url(sig64, base64, filename(url))
     end
   end
 
@@ -35,4 +33,20 @@ defmodule Pleroma.Web.MediaProxy do
       {:error, :invalid_signature}
     end
   end
+
+  def filename(url_or_path) do
+    if path = URI.parse(url_or_path).path, do: Path.basename(path)
+  end
+
+  def build_url(sig_base64, url_base64, filename \\ nil) do
+    [
+      Pleroma.Config.get([:media_proxy, :base_url], Pleroma.Web.base_url()),
+      "proxy",
+      sig_base64,
+      url_base64,
+      filename
+    ]
+    |> Enum.filter(fn value -> value end)
+    |> Path.join()
+  end
 end
diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex
index d98deeb03..39a2974bb 100644
--- a/lib/pleroma/web/twitter_api/twitter_api.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api.ex
@@ -97,7 +97,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
     {:ok, object} = ActivityPub.upload(file)
 
     url = List.first(object.data["url"])
-    href = url["href"] |> MediaProxy.url()
+    href = url["href"]
     type = url["mediaType"]
 
     case format do
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index b0bd98dbc..8f9d6c905 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -290,11 +290,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   end
 
   def update_avatar(%{assigns: %{user: user}} = conn, params) do
-    upload_limit =
-      Application.get_env(:pleroma, :instance)
-      |> Keyword.fetch(:avatar_upload_limit)
-
-    {:ok, object} = ActivityPub.upload(params, upload_limit)
+    {:ok, object} = ActivityPub.upload(params, type: :avatar)
     change = Changeset.change(user, %{avatar: object.data})
     {:ok, user} = User.update_and_set_cache(change)
     CommonAPI.update(user)
@@ -303,11 +299,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   end
 
   def update_banner(%{assigns: %{user: user}} = conn, params) do
-    upload_limit =
-      Application.get_env(:pleroma, :instance)
-      |> Keyword.fetch(:banner_upload_limit)
-
-    with {:ok, object} <- ActivityPub.upload(%{"img" => params["banner"]}, upload_limit),
+    with {:ok, object} <- ActivityPub.upload(%{"img" => params["banner"]}, type: :banner),
          new_info <- Map.put(user.info, "banner", object.data),
          change <- User.info_changeset(user, %{info: new_info}),
          {:ok, user} <- User.update_and_set_cache(change) do
@@ -321,11 +313,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   end
 
   def update_background(%{assigns: %{user: user}} = conn, params) do
-    upload_limit =
-      Application.get_env(:pleroma, :instance)
-      |> Keyword.fetch(:background_upload_limit)
-
-    with {:ok, object} <- ActivityPub.upload(params, upload_limit),
+    with {:ok, object} <- ActivityPub.upload(params, type: :background),
          new_info <- Map.put(user.info, "background", object.data),
          change <- User.info_changeset(user, %{info: new_info}),
          {:ok, _user} <- User.update_and_set_cache(change) do