added `deleted_urls_cache`

9 files changed, 72 insertions, 25 deletions

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 9d3d92b38..7ddb03529 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -148,7 +148,8 @@ defmodule Pleroma.Application do
       build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
       build_cachex("web_resp", limit: 2500),
       build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
-      build_cachex("failed_proxy_url", limit: 2500)
+      build_cachex("failed_proxy_url", limit: 2500),
+      build_cachex("deleted_urls", default_ttl: :timer.minutes(60), limit: 2500)
     ]
   end
 
diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex
index bd6618f0a..324f8a7ee 100644
--- a/lib/pleroma/web/media_proxy/invalidation.ex
+++ b/lib/pleroma/web/media_proxy/invalidation.ex
@@ -5,27 +5,32 @@
 defmodule Pleroma.Web.MediaProxy.Invalidation do
   @moduledoc false
 
-  @callback purge(list(String.t()), Keyword.t()) :: {:ok, String.t()} | {:error, String.t()}
+  @callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()}
 
   alias Pleroma.Config
   alias Pleroma.Web.MediaProxy
 
-  @spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()}
+  @spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()}
   def purge(urls) do
-    [:media_proxy, :invalidation, :enabled]
-    |> Config.get()
-    |> do_purge(urls)
+    prepared_urls = prepare_urls(urls)
+
+    if Config.get([:media_proxy, :invalidation, :enabled]) do
+      result = do_purge(prepared_urls)
+      MediaProxy.remove_from_deleted_urls(prepared_urls)
+      result
+    else
+      {:ok, prepared_urls}
+    end
   end
 
-  defp do_purge(true, urls) do
+  defp do_purge(urls) do
     provider = Config.get([:media_proxy, :invalidation, :provider])
-    options = Config.get(provider)
+    provider.purge(urls, Config.get(provider))
+  end
 
+  defp prepare_urls(urls) do
     urls
     |> List.wrap()
     |> Enum.map(&MediaProxy.url(&1))
-    |> provider.purge(options)
   end
-
-  defp do_purge(_, _), do: :ok
 end
diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex
index 725fddc04..3694b56e8 100644
--- a/lib/pleroma/web/media_proxy/invalidations/http.ex
+++ b/lib/pleroma/web/media_proxy/invalidations/http.ex
@@ -22,7 +22,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do
       end
     end)
 
-    {:ok, "success"}
+    {:ok, urls}
   end
 
   defp do_purge(method, url, headers, options) do
diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex
index fbb0a2e52..d41d647bb 100644
--- a/lib/pleroma/web/media_proxy/invalidations/script.ex
+++ b/lib/pleroma/web/media_proxy/invalidations/script.ex
@@ -20,7 +20,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
     opts
     |> Keyword.get(:script_path, nil)
     |> do_purge([args])
-    |> handle_error
+    |> handle_result(urls)
   end
 
   defp do_purge(script_path, args) when is_binary(script_path) do
@@ -33,10 +33,10 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
 
   defp do_purge(_, _), do: {:error, "not found script path"}
 
-  defp handle_error({_result, 0}), do: {:ok, "success"}
-  defp handle_error({:error, error}), do: handle_error(error)
+  defp handle_result({_result, 0}, urls), do: {:ok, urls}
+  defp handle_result({:error, error}, urls), do: handle_result(error, urls)
 
-  defp handle_error(error) do
+  defp handle_result(error, _) do
     Logger.error("Error while cache purge: #{inspect(error)}")
     {:error, inspect(error)}
   end
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index b2b524524..25eabca1e 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -9,6 +9,25 @@ defmodule Pleroma.Web.MediaProxy do
 
   @base64_opts [padding: false]
 
+  @spec in_deleted_urls(String.t()) :: boolean()
+  def in_deleted_urls(url), do: !!Cachex.get!(:deleted_urls_cache, url)
+
+  def remove_from_deleted_urls(urls) when is_list(urls) do
+    Enum.each(urls, &remove_from_deleted_urls(&1))
+  end
+
+  def remove_from_deleted_urls(url) when is_binary(url) do
+    Cachex.del(:deleted_urls_cache, url)
+  end
+
+  def put_in_deleted_urls(urls) when is_list(urls) do
+    Enum.each(urls, &put_in_deleted_urls(&1))
+  end
+
+  def put_in_deleted_urls(url) when is_binary(url) do
+    Cachex.put(:deleted_urls_cache, url, true)
+  end
+
   def url(url) when is_nil(url) or url == "", do: nil
   def url("/" <> _ = url), do: url
 
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 4657a4383..ff0158d83 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -14,10 +14,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     with config <- Pleroma.Config.get([:media_proxy], []),
          true <- Keyword.get(config, :enabled, false),
          {:ok, url} <- MediaProxy.decode_url(sig64, url64),
+         {_, false} <- {:in_deleted_urls, MediaProxy.in_deleted_urls(url)},
          :ok <- filename_matches(params, conn.request_path, url) do
       ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
     else
-      false ->
+      error when error in [false, {:in_deleted_urls, true}] ->
         send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
 
       {:error, :invalid_signature} ->
diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs
index e97cf9fc1..8348663f3 100644
--- a/test/web/media_proxy/invalidation_test.exs
+++ b/test/web/media_proxy/invalidation_test.exs
@@ -13,10 +13,13 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do
 
   describe "Invalidation.Http" do
     test "perform request to clear cache" do
+      Config.put([:media_proxy, :enabled], false)
       Config.put([:media_proxy, :invalidation, :enabled], true)
       Config.put([:media_proxy, :invalidation, :provider], Invalidation.Http)
 
       Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}])
+      image_url = "http://example.com/media/example.jpg"
+      Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url)
 
       mock(fn
         %{
@@ -28,23 +31,29 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do
       end)
 
       assert capture_log(fn ->
-               assert Invalidation.purge(["http://example.com/media/example.jpg"]) ==
-                        {:ok, "success"}
-             end) =~ "Running cache purge: [\"http://example.com/media/example.jpg\"]"
+               assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+               assert Invalidation.purge([image_url]) == {:ok, [image_url]}
+               refute Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+             end) =~ "Running cache purge: [\"#{image_url}\"]"
     end
   end
 
   describe "Invalidation.Script" do
     test "run script to clear cache" do
+      Config.put([:media_proxy, :enabled], false)
       Config.put([:media_proxy, :invalidation, :enabled], true)
       Config.put([:media_proxy, :invalidation, :provider], Invalidation.Script)
       Config.put([Invalidation.Script], script_path: "purge-nginx")
 
+      image_url = "http://example.com/media/example.jpg"
+      Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url)
+
       with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do
         assert capture_log(fn ->
-                 assert Invalidation.purge(["http://example.com/media/example.jpg"]) ==
-                          {:ok, "success"}
-               end) =~ "Running cache purge: [\"http://example.com/media/example.jpg\"]"
+                 assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+                 assert Invalidation.purge([image_url]) == {:ok, [image_url]}
+                 refute Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+               end) =~ "Running cache purge: [\"#{image_url}\"]"
       end
     end
   end
diff --git a/test/web/media_proxy/invalidations/http_test.exs b/test/web/media_proxy/invalidations/http_test.exs
index 553b86b76..062dd0b87 100644
--- a/test/web/media_proxy/invalidations/http_test.exs
+++ b/test/web/media_proxy/invalidations/http_test.exs
@@ -15,7 +15,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
              assert Invalidation.Http.purge(
                       ["http://example.com/media/example.jpg"],
                       []
-                    ) == {:ok, "success"}
+                    ) == {:ok, ["http://example.com/media/example.jpg"]}
            end) =~ "Error while cache purge"
   end
 
@@ -29,7 +29,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
              assert Invalidation.Http.purge(
                       ["http://example.com/media/example1.jpg"],
                       []
-                    ) == {:ok, "success"}
+                    ) == {:ok, ["http://example.com/media/example1.jpg"]}
            end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg"
   end
 end
diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs
index da79d38a5..306d53bf8 100644
--- a/test/web/media_proxy/media_proxy_controller_test.exs
+++ b/test/web/media_proxy/media_proxy_controller_test.exs
@@ -66,4 +66,16 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
       assert %Plug.Conn{status: :success} = get(conn, url)
     end
   end
+
+  test "it returns 404 when url contains in deleted_urls cache", %{conn: conn} do
+    Config.put([:media_proxy, :enabled], true)
+    Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
+    url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png")
+    Pleroma.Web.MediaProxy.put_in_deleted_urls("https://google.fn/test.png")
+
+    with_mock Pleroma.ReverseProxy,
+      call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do
+      assert %Plug.Conn{status: 404, resp_body: "Not Found"} = get(conn, url)
+    end
+  end
 end