diff options
| -rw-r--r-- | config/config.exs | 7 | ||||
| -rwxr-xr-x | installation/nginx-cache-purge.sh.example | 5 | ||||
| -rw-r--r-- | lib/pleroma/application.ex | 3 | ||||
| -rw-r--r-- | lib/pleroma/plugs/uploaded_media.ex | 16 | ||||
| -rw-r--r-- | lib/pleroma/web/activity_pub/activity_pub.ex | 24 | ||||
| -rw-r--r-- | lib/pleroma/web/media_proxy/invalidation.ex | 29 | ||||
| -rw-r--r-- | lib/pleroma/web/media_proxy/invalidations/http.ex | 8 | ||||
| -rw-r--r-- | lib/pleroma/web/media_proxy/invalidations/script.ex | 36 | ||||
| -rw-r--r-- | lib/pleroma/web/media_proxy/media_proxy.ex | 38 | ||||
| -rw-r--r-- | lib/pleroma/web/media_proxy/media_proxy_controller.ex | 3 | ||||
| -rw-r--r-- | lib/pleroma/workers/attachments_cleanup_worker.ex | 125 | ||||
| -rw-r--r-- | test/object_test.exs | 36 | ||||
| -rw-r--r-- | test/support/invalidation_mock.ex | 10 | ||||
| -rw-r--r-- | test/web/media_proxy/invalidation_test.exs | 65 | ||||
| -rw-r--r-- | test/web/media_proxy/invalidations/http_test.exs | 13 | ||||
| -rw-r--r-- | test/web/media_proxy/invalidations/script_test.exs | 21 | ||||
| -rw-r--r-- | test/web/media_proxy/media_proxy_controller_test.exs | 17 |
17 files changed, 340 insertions, 116 deletions
diff --git a/config/config.exs b/config/config.exs index 9508ae077..e299fb8dd 100644 --- a/config/config.exs +++ b/config/config.exs @@ -406,6 +406,13 @@ config :pleroma, :media_proxy, ], whitelist: [] +config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http, + method: :purge, + headers: [], + options: [] + +config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil + config :pleroma, :chat, enabled: true config :phoenix, :format_encoders, json: Jason diff --git a/installation/nginx-cache-purge.sh.example b/installation/nginx-cache-purge.sh.example index b2915321c..83a3a6df2 100755 --- a/installation/nginx-cache-purge.sh.example +++ b/installation/nginx-cache-purge.sh.example @@ -13,7 +13,7 @@ CACHE_DIRECTORY="/tmp/pleroma-media-cache" ## $3 - (optional) the number of parallel processes to run for grep. get_cache_files() { local max_parallel=${3-16} - find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E Rl "^KEY:.*$1" | sort -u + find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E -Rl "^KEY:.*$1" | sort -u } ## Removes an item from the given cache zone. @@ -33,8 +33,9 @@ purge() { do echo "$SCRIPTNAME delete \`$url\` from cache ($CACHE_DIRECTORY)" purge_item $url $CACHE_DIRECTORY + echo "\n" done } -purge $1 +purge $@ diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9d3d92b38..adebebc7a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -148,7 +148,8 @@ defmodule Pleroma.Application do build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), - build_cachex("failed_proxy_url", limit: 2500) + build_cachex("failed_proxy_url", limit: 2500), + build_cachex("deleted_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) ] end diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex index 94147e0c4..2f3fde002 100644 --- a/lib/pleroma/plugs/uploaded_media.ex +++ b/lib/pleroma/plugs/uploaded_media.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Plugs.UploadedMedia do import Pleroma.Web.Gettext require Logger + alias Pleroma.Web.MediaProxy + @behaviour Plug # no slashes @path "media" @@ -35,8 +37,7 @@ defmodule Pleroma.Plugs.UploadedMedia do %{query_params: %{"name" => name}} = conn -> name = String.replace(name, "\"", "\\\"") - conn - |> put_resp_header("content-disposition", "filename=\"#{name}\"") + put_resp_header(conn, "content-disposition", "filename=\"#{name}\"") conn -> conn @@ -47,7 +48,8 @@ defmodule Pleroma.Plugs.UploadedMedia do with uploader <- Keyword.fetch!(config, :uploader), proxy_remote = Keyword.get(config, :proxy_remote, false), - {:ok, get_method} <- uploader.get_file(file) do + {:ok, get_method} <- uploader.get_file(file), + false <- media_is_deleted(conn, get_method) do get_media(conn, get_method, proxy_remote, opts) else _ -> @@ -59,6 +61,14 @@ defmodule Pleroma.Plugs.UploadedMedia do def call(conn, _opts), do: conn + defp media_is_deleted(%{request_path: path} = _conn, {:static_dir, _}) do + MediaProxy.in_deleted_urls(Pleroma.Web.base_url() <> path) + end + + defp media_is_deleted(_, {:url, url}), do: MediaProxy.in_deleted_urls(url) + + defp media_is_deleted(_, _), do: false + defp get_media(conn, {:static_dir, directory}, _, opts) do static_opts = Map.get(opts, :static_plug_opts) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 958f3e5af..9c376e07d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -20,6 +20,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.MediaProxy alias Pleroma.Web.Streamer alias Pleroma.Web.WebFinger alias Pleroma.Workers.BackgroundWorker @@ -87,6 +88,22 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp check_remote_limit(_), do: true + @spec maybe_remove_mediaproxy_invalidation(boolean, map()) :: :ok + def maybe_remove_mediaproxy_invalidation(true, %{ + "object" => %{"attachment" => [_ | _] = attachments} + }) do + attachments + |> Enum.flat_map(fn + %{"url" => urls} -> Enum.map(urls, & &1["href"]) + _ -> [] + end) + |> MediaProxy.remove_from_deleted_urls() + + :ok + end + + def maybe_remove_mediaproxy_invalidation(_, _), do: :ok + def increase_note_count_if_public(actor, object) do if is_public?(object), do: User.increase_note_count(actor), else: {:ok, actor} end @@ -275,16 +292,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:fake, false, activity} <- {:fake, fake, activity}, _ <- increase_replies_count_if_reply(create_data), _ <- increase_poll_votes_if_vote(create_data), + _ <- + maybe_remove_mediaproxy_invalidation(MediaProxy.Invalidation.enabled(), create_data), {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity}, {:ok, _actor} <- increase_note_count_if_public(actor, activity), _ <- notify_and_stream(activity), :ok <- maybe_federate(activity) do {:ok, activity} else - {:quick_insert, true, activity} -> - {:ok, activity} - - {:fake, true, activity} -> + {error, true, activity} when error in [:fake, :quick_insert] -> {:ok, activity} {:error, message} -> diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index c037ff13e..537cb76d6 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -5,22 +5,33 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do @moduledoc false - @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()} + @callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()} alias Pleroma.Config + alias Pleroma.Web.MediaProxy - @spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()} + def enabled, do: Config.get([:media_proxy, :invalidation, :enabled]) + + @spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()} def purge(urls) do - [:media_proxy, :invalidation, :enabled] - |> Config.get() - |> do_purge(urls) + prepared_urls = prepare_urls(urls) + + if enabled() do + do_purge(prepared_urls) + else + {:ok, prepared_urls} + end end - defp do_purge(true, urls) do + defp do_purge(urls) do provider = Config.get([:media_proxy, :invalidation, :provider]) - options = Config.get(provider) - provider.purge(urls, options) + provider.purge(urls, Config.get(provider)) end - defp do_purge(_, _), do: :ok + def prepare_urls(urls) do + urls + |> List.wrap() + |> Enum.filter(&MediaProxy.is_url_proxiable?(&1)) + |> Enum.map(&MediaProxy.url(&1)) + end end diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex index 07248df6e..3694b56e8 100644 --- a/lib/pleroma/web/media_proxy/invalidations/http.ex +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -10,9 +10,9 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do @impl Pleroma.Web.MediaProxy.Invalidation def purge(urls, opts) do - method = Map.get(opts, :method, :purge) - headers = Map.get(opts, :headers, []) - options = Map.get(opts, :options, []) + method = Keyword.get(opts, :method, :purge) + headers = Keyword.get(opts, :headers, []) + options = Keyword.get(opts, :options, []) Logger.debug("Running cache purge: #{inspect(urls)}") @@ -22,7 +22,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do end end) - {:ok, "success"} + {:ok, urls} end defp do_purge(method, url, headers, options) do diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index 6be782132..d41d647bb 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -10,32 +10,34 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, %{script_path: script_path} = _options) do + def purge(urls, opts) do args = urls |> List.wrap() |> Enum.uniq() |> Enum.join(" ") - path = Path.expand(script_path) - - Logger.debug("Running cache purge: #{inspect(urls)}, #{path}") - - case do_purge(path, [args]) do - {result, exit_status} when exit_status > 0 -> - Logger.error("Error while cache purge: #{inspect(result)}") - {:error, inspect(result)} - - _ -> - {:ok, "success"} - end + opts + |> Keyword.get(:script_path, nil) + |> do_purge([args]) + |> handle_result(urls) end - def purge(_, _), do: {:error, "not found script path"} - - defp do_purge(path, args) do + defp do_purge(script_path, args) when is_binary(script_path) do + path = Path.expand(script_path) + Logger.debug("Running cache purge: #{inspect(args)}, #{inspect(path)}") System.cmd(path, args) rescue - error -> {inspect(error), 1} + error -> error + end + + defp do_purge(_, _), do: {:error, "not found script path"} + + defp handle_result({_result, 0}, urls), do: {:ok, urls} + defp handle_result({:error, error}, urls), do: handle_result(error, urls) + + defp handle_result(error, _) do + Logger.error("Error while cache purge: #{inspect(error)}") + {:error, inspect(error)} end end diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index b2b524524..e5f1d242e 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -6,20 +6,56 @@ defmodule Pleroma.Web.MediaProxy do alias Pleroma.Config alias Pleroma.Upload alias Pleroma.Web + alias Pleroma.Web.MediaProxy.Invalidation @base64_opts [padding: false] + @spec in_deleted_urls(String.t()) :: boolean() + def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url(url)), 1) + + def remove_from_deleted_urls(urls) when is_list(urls) do + Cachex.execute!(:deleted_urls_cache, fn cache -> + Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1)) + end) + end + + def remove_from_deleted_urls(url) when is_binary(url) do + Cachex.del(:deleted_urls_cache, url(url)) + end + + def put_in_deleted_urls(urls) when is_list(urls) do + Cachex.execute!(:deleted_urls_cache, fn cache -> + Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true)) + end) + end + + def put_in_deleted_urls(url) when is_binary(url) do + if is_url_proxiable?(url) do + Cachex.put(:deleted_urls_cache, url(url), true) + else + true + end + end + def url(url) when is_nil(url) or url == "", do: nil def url("/" <> _ = url), do: url def url(url) do - if disabled?() or local?(url) or whitelisted?(url) do + if disabled?() or not is_url_proxiable?(url) do url else encode_url(url) end end + def is_url_proxiable?(url) do + if local?(url) or whitelisted?(url) do + false + else + true + end + end + defp disabled?, do: !Config.get([:media_proxy, :enabled], false) defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url()) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 4657a4383..ff0158d83 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -14,10 +14,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), + {_, false} <- {:in_deleted_urls, MediaProxy.in_deleted_urls(url)}, :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else - false -> + error when error in [false, {:in_deleted_urls, true}] -> send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404)) {:error, :invalid_signature} -> diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 49352db2a..170bca976 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Web.MediaProxy use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup" @@ -23,8 +24,6 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do Enum.map(attachment["url"], & &1["href"]) end) - names = Enum.map(attachments, & &1["name"]) - uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) prefix = @@ -40,67 +39,87 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do ) # find all objects for copies of the attachments, name and actor doesn't matter here - object_ids_and_hrefs = - from(o in Object, - where: - fragment( - "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", - o.data, - o.data, - ^hrefs - ) - ) - # The query above can be time consumptive on large instances until we - # refactor how uploads are stored - |> Repo.all(timeout: :infinity) - # we should delete 1 object for any given attachment, but don't delete - # files if there are more than 1 object for it - |> Enum.reduce(%{}, fn %{ - id: id, - data: %{ - "url" => [%{"href" => href}], - "actor" => obj_actor, - "name" => name - } - }, - acc -> - Map.update(acc, href, %{id: id, count: 1}, fn val -> - case obj_actor == actor and name in names do - true -> - # set id of the actor's object that will be deleted - %{val | id: id, count: val.count + 1} - - false -> - # another actor's object, just increase count to not delete file - %{val | count: val.count + 1} - end - end) - end) - |> Enum.map(fn {href, %{id: id, count: count}} -> - # only delete files that have single instance + {object_ids, attachment_urls, exclude_urls} = + hrefs + |> fetch_objects + |> prepare_objects(actor, Enum.map(attachments, & &1["name"])) + |> Enum.reduce({[], [], []}, fn {href, %{id: id, count: count}}, + {ids, hrefs, exclude_urls} -> with 1 <- count do - href - |> String.trim_leading("#{base_url}/#{prefix}") - |> uploader.delete_file() - - {id, href} + {ids ++ [id], hrefs ++ [href], exclude_urls} else - _ -> {id, nil} + _ -> {ids ++ [id], hrefs, exclude_urls ++ [href]} end end) - object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end) + lock_attachments(MediaProxy.Invalidation.enabled(), hrefs -- exclude_urls) - from(o in Object, where: o.id in ^object_ids) - |> Repo.delete_all() + Enum.each(attachment_urls, fn href -> + href + |> String.trim_leading("#{base_url}/#{prefix}") + |> uploader.delete_file() + end) - object_ids_and_hrefs - |> Enum.filter(fn {_, href} -> not is_nil(href) end) - |> Enum.map(&elem(&1, 1)) - |> Pleroma.Web.MediaProxy.Invalidation.purge() + delete_objects(object_ids) + + cache_purge(MediaProxy.Invalidation.enabled(), hrefs -- exclude_urls) {:ok, :success} end def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} + + defp delete_objects([_ | _] = object_ids) do + Repo.delete_all(from(o in Object, where: o.id in ^object_ids)) + end + + defp delete_objects(_), do: :ok + + defp cache_purge(true, [_ | _] = urls), do: MediaProxy.Invalidation.purge(urls) + defp cache_purge(_, _), do: :ok + + defp lock_attachments(true, urls), do: MediaProxy.put_in_deleted_urls(urls) + defp lock_attachments(_, _), do: :ok + + # we should delete 1 object for any given attachment, but don't delete + # files if there are more than 1 object for it + def prepare_objects(objects, actor, names) do + objects + |> Enum.reduce(%{}, fn %{ + id: id, + data: %{ + "url" => [%{"href" => href}], + "actor" => obj_actor, + "name" => name + } + }, + acc -> + Map.update(acc, href, %{id: id, count: 1}, fn val -> + case obj_actor == actor and name in names do + true -> + # set id of the actor's object that will be deleted + %{val | id: id, count: val.count + 1} + + false -> + # another actor's object, just increase count to not delete file + %{val | count: val.count + 1} + end + end) + end) + end + + def fetch_objects(hrefs) do + from(o in Object, + where: + fragment( + "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", + o.data, + o.data, + ^hrefs + ) + ) + # The query above can be time consumptive on large instances until we + # refactor how uploads are stored + |> Repo.all(timeout: :infinity) + end end diff --git a/test/object_test.exs b/test/object_test.exs index 198d3b1cf..5cd2d2c80 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -9,10 +9,12 @@ defmodule Pleroma.ObjectTest do import Pleroma.Factory import Tesla.Mock alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.Object alias Pleroma.Repo alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI + alias Pleroma.Web.MediaProxy.Invalidation setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -76,6 +78,7 @@ defmodule Pleroma.ObjectTest do describe "delete attachments" do setup do: clear_config([Pleroma.Upload]) setup do: clear_config([:instance, :cleanup_attachments]) + setup do: clear_config([:media_proxy]) test "Disabled via config" do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) @@ -109,11 +112,14 @@ defmodule Pleroma.ObjectTest do refute Object.get_by_id(attachment.id) == nil assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}") + refute Pleroma.Web.MediaProxy.in_deleted_urls(href) end test "in subdirectories" do - Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) - Pleroma.Config.put([:instance, :cleanup_attachments], true) + Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + Config.put([:instance, :cleanup_attachments], true) + Config.put([:media_proxy, :invalidation, :enabled], true) + Config.put([:media_proxy, :invalidation, :provider], Invalidation.Mock) file = %Plug.Upload{ content_type: "image/jpg", @@ -143,12 +149,15 @@ defmodule Pleroma.ObjectTest do assert Object.get_by_id(attachment.id) == nil assert {:ok, []} == File.ls("#{uploads_dir}/#{path}") + refute Pleroma.Web.MediaProxy.in_deleted_urls(href) end test "with dedupe enabled" do - Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) - Pleroma.Config.put([Pleroma.Upload, :filters], [Pleroma.Upload.Filter.Dedupe]) - Pleroma.Config.put([:instance, :cleanup_attachments], true) + Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + Config.put([Pleroma.Upload, :filters], [Pleroma.Upload.Filter.Dedupe]) + Config.put([:instance, :cleanup_attachments], true) + Config.put([:media_proxy, :invalidation, :enabled], true) + Config.put([:media_proxy, :invalidation, :provider], Invalidation.Mock) uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads]) @@ -181,11 +190,14 @@ defmodule Pleroma.ObjectTest do assert Object.get_by_id(attachment.id) == nil assert {:ok, files} = File.ls(uploads_dir) refute filename in files + refute Pleroma.Web.MediaProxy.in_deleted_urls(href) end test "with objects that have legacy data.url attribute" do - Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) - Pleroma.Config.put([:instance, :cleanup_attachments], true) + Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + Config.put([:instance, :cleanup_attachments], true) + Config.put([:media_proxy, :invalidation, :enabled], true) + Config.put([:media_proxy, :invalidation, :provider], Invalidation.Mock) file = %Plug.Upload{ content_type: "image/jpg", @@ -217,12 +229,15 @@ defmodule Pleroma.ObjectTest do assert Object.get_by_id(attachment.id) == nil assert {:ok, []} == File.ls("#{uploads_dir}/#{path}") + refute Pleroma.Web.MediaProxy.in_deleted_urls(href) end test "With custom base_url" do - Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) - Pleroma.Config.put([Pleroma.Upload, :base_url], "https://sub.domain.tld/dir/") - Pleroma.Config.put([:instance, :cleanup_attachments], true) + Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + Config.put([Pleroma.Upload, :base_url], "https://sub.domain.tld/dir/") + Config.put([:instance, :cleanup_attachments], true) + Config.put([:media_proxy, :invalidation, :enabled], true) + Config.put([:media_proxy, :invalidation, :provider], Invalidation.Mock) file = %Plug.Upload{ content_type: "image/jpg", @@ -252,6 +267,7 @@ defmodule Pleroma.ObjectTest do assert Object.get_by_id(attachment.id) == nil assert {:ok, []} == File.ls("#{uploads_dir}/#{path}") + refute Pleroma.Web.MediaProxy.in_deleted_urls(href) end end diff --git a/test/support/invalidation_mock.ex b/test/support/invalidation_mock.ex new file mode 100644 index 000000000..3cba697dc --- /dev/null +++ b/test/support/invalidation_mock.ex @@ -0,0 +1,10 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation.Mock do + @moduledoc false + + @behaviour Pleroma.Web.MediaProxy.Invalidation + + @impl Pleroma.Web.MediaProxy.Invalidation + def purge(urls, _opts) do + {:ok, urls} + end +end diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs new file mode 100644 index 000000000..3a9fa8c88 --- /dev/null +++ b/test/web/media_proxy/invalidation_test.exs @@ -0,0 +1,65 @@ +defmodule Pleroma.Web.MediaProxy.InvalidationTest do + use ExUnit.Case + use Pleroma.Tests.Helpers + + alias Pleroma.Config + alias Pleroma.Web.MediaProxy.Invalidation + + import ExUnit.CaptureLog + import Mock + import Tesla.Mock + + setup do: clear_config([:media_proxy]) + + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end + + describe "Invalidation.Http" do + test "perform request to clear cache" do + Config.put([:media_proxy, :enabled], false) + Config.put([:media_proxy, :invalidation, :enabled], true) + Config.put([:media_proxy, :invalidation, :provider], Invalidation.Http) + + Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}]) + image_url = "http://example.com/media/example.jpg" + Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url) + + mock(fn + %{ + method: :purge, + url: "http://example.com/media/example.jpg", + headers: [{"x-refresh", 1}] + } -> + %Tesla.Env{status: 200} + end) + + assert capture_log(fn -> + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Invalidation.purge([image_url]) == {:ok, [image_url]} + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + end) =~ "Running cache purge: [\"#{image_url}\"]" + end + end + + describe "Invalidation.Script" do + test "run script to clear cache" do + Config.put([:media_proxy, :enabled], false) + Config.put([:media_proxy, :invalidation, :enabled], true) + Config.put([:media_proxy, :invalidation, :provider], Invalidation.Script) + Config.put([Invalidation.Script], script_path: "purge-nginx") + + image_url = "http://example.com/media/example.jpg" + Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url) + + with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do + assert capture_log(fn -> + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Invalidation.purge([image_url]) == {:ok, [image_url]} + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + end) =~ "Running cache purge: [\"#{image_url}\"]" + end + end + end +end diff --git a/test/web/media_proxy/invalidations/http_test.exs b/test/web/media_proxy/invalidations/http_test.exs index 8a3b4141c..09e7ca0fb 100644 --- a/test/web/media_proxy/invalidations/http_test.exs +++ b/test/web/media_proxy/invalidations/http_test.exs @@ -5,6 +5,11 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do import ExUnit.CaptureLog import Tesla.Mock + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end + test "logs hasn't error message when request is valid" do mock(fn %{method: :purge, url: "http://example.com/media/example.jpg"} -> @@ -14,8 +19,8 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do refute capture_log(fn -> assert Invalidation.Http.purge( ["http://example.com/media/example.jpg"], - %{} - ) == {:ok, "success"} + [] + ) == {:ok, ["http://example.com/media/example.jpg"]} end) =~ "Error while cache purge" end @@ -28,8 +33,8 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do assert capture_log(fn -> assert Invalidation.Http.purge( ["http://example.com/media/example1.jpg"], - %{} - ) == {:ok, "success"} + [] + ) == {:ok, ["http://example.com/media/example1.jpg"]} end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg" end end diff --git a/test/web/media_proxy/invalidations/script_test.exs b/test/web/media_proxy/invalidations/script_test.exs index 1358963ab..c69cec07a 100644 --- a/test/web/media_proxy/invalidations/script_test.exs +++ b/test/web/media_proxy/invalidations/script_test.exs @@ -4,17 +4,24 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do import ExUnit.CaptureLog + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end + test "it logger error when script not found" do assert capture_log(fn -> assert Invalidation.Script.purge( ["http://example.com/media/example.jpg"], - %{script_path: "./example"} - ) == {:error, "\"%ErlangError{original: :enoent}\""} - end) =~ "Error while cache purge: \"%ErlangError{original: :enoent}\"" + script_path: "./example" + ) == {:error, "%ErlangError{original: :enoent}"} + end) =~ "Error while cache purge: %ErlangError{original: :enoent}" - assert Invalidation.Script.purge( - ["http://example.com/media/example.jpg"], - %{} - ) == {:error, "not found script path"} + capture_log(fn -> + assert Invalidation.Script.purge( + ["http://example.com/media/example.jpg"], + [] + ) == {:error, "\"not found script path\""} + end) end end diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs index da79d38a5..2b6b25221 100644 --- a/test/web/media_proxy/media_proxy_controller_test.exs +++ b/test/web/media_proxy/media_proxy_controller_test.exs @@ -10,6 +10,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do setup do: clear_config(:media_proxy) setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base]) + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end + test "it returns 404 when MediaProxy disabled", %{conn: conn} do Config.put([:media_proxy, :enabled], false) @@ -66,4 +71,16 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do assert %Plug.Conn{status: :success} = get(conn, url) end end + + test "it returns 404 when url contains in deleted_urls cache", %{conn: conn} do + Config.put([:media_proxy, :enabled], true) + Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000") + url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png") + Pleroma.Web.MediaProxy.put_in_deleted_urls("https://google.fn/test.png") + + with_mock Pleroma.ReverseProxy, + call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do + assert %Plug.Conn{status: 404, resp_body: "Not Found"} = get(conn, url) + end + end end |