diff options
| -rw-r--r-- | config/config.exs | 4 | ||||
| -rw-r--r-- | docs/config.md | 4 | ||||
| -rw-r--r-- | lib/pleroma/user.ex | 47 | ||||
| -rw-r--r-- | lib/pleroma/web/activity_pub/utils.ex | 39 | ||||
| -rw-r--r-- | test/web/activity_pub/utils_test.exs | 65 |
5 files changed, 149 insertions, 10 deletions
diff --git a/config/config.exs b/config/config.exs index a867dd0bc..2b9aabf80 100644 --- a/config/config.exs +++ b/config/config.exs @@ -348,6 +348,10 @@ config :pleroma, Pleroma.Jobs, federator_outgoing: [max_jobs: 50], mailer: [max_jobs: 10] +config :pleroma, :fetch_initial_posts, + enabled: false, + pages: 5 + config :auto_linker, opts: [ scheme: true, diff --git a/docs/config.md b/docs/config.md index 465bc1d2b..a09ea95f3 100644 --- a/docs/config.md +++ b/docs/config.md @@ -285,6 +285,10 @@ This config contains two queues: `federator_incoming` and `federator_outgoing`. ## :rich_media * `enabled`: if enabled the instance will parse metadata from attached links to generate link previews +## :fetch_initial_posts +* `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts +* `pages`: the amount of pages to fetch + ## :hackney_pools Advanced. Tweaks Hackney (http client) connections pools. diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3878e4efa..f49ede149 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -532,6 +532,10 @@ defmodule Pleroma.User do _e -> with [_nick, _domain] <- String.split(nickname, "@"), {:ok, user} <- fetch_by_nickname(nickname) do + if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do + {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user]) + end + user else _e -> nil @@ -539,6 +543,17 @@ defmodule Pleroma.User do end end + @doc "Fetch some posts when the user has just been federated with" + def fetch_initial_posts(user) do + pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) + + Enum.each( + # Insert all the posts in reverse order, so they're in the right order on the timeline + Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), + &Pleroma.Web.Federator.incoming_ap_doc/1 + ) + end + def get_followers_query(%User{id: id, follower_address: follower_address}, nil) do from( u in User, @@ -1121,24 +1136,36 @@ defmodule Pleroma.User do def html_filter_policy(_), do: @default_scrubbers + def fetch_by_ap_id(ap_id) do + ap_try = ActivityPub.make_user_from_ap_id(ap_id) + + case ap_try do + {:ok, user} -> + user + + _ -> + case OStatus.make_user(ap_id) do + {:ok, user} -> user + _ -> {:error, "Could not fetch by AP id"} + end + end + end + def get_or_fetch_by_ap_id(ap_id) do user = get_by_ap_id(ap_id) if !is_nil(user) and !User.needs_update?(user) do user else - ap_try = ActivityPub.make_user_from_ap_id(ap_id) - - case ap_try do - {:ok, user} -> - user + user = fetch_by_ap_id(ap_id) - _ -> - case OStatus.make_user(ap_id) do - {:ok, user} -> user - _ -> {:error, "Could not fetch by AP id"} - end + if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do + with %User{} = user do + {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user]) + end end + + user end end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 9e50789db..629c39315 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -633,4 +633,43 @@ defmodule Pleroma.Web.ActivityPub.Utils do } |> Map.merge(additional) end + + @doc """ + Fetches the OrderedCollection/OrderedCollectionPage from `from`, limiting the amount of pages fetched after + the first one to `pages_left` pages. + If the amount of pages is higher than the collection has, it returns whatever was there. + """ + def fetch_ordered_collection(from, pages_left, acc \\ []) do + with {:ok, response} <- Tesla.get(from), + {:ok, collection} <- Poison.decode(response.body) do + case collection["type"] do + "OrderedCollection" -> + # If we've encountered the OrderedCollection and not the page, + # just call the same function on the page address + fetch_ordered_collection(collection["first"], pages_left) + + "OrderedCollectionPage" -> + if pages_left > 0 do + # There are still more pages + if Map.has_key?(collection, "next") do + # There are still more pages, go deeper saving what we have into the accumulator + fetch_ordered_collection( + collection["next"], + pages_left - 1, + acc ++ collection["orderedItems"] + ) + else + # No more pages left, just return whatever we already have + acc ++ collection["orderedItems"] + end + else + # Got the amount of pages needed, add them all to the accumulator + acc ++ collection["orderedItems"] + end + + _ -> + {:error, "Not an OrderedCollection or OrderedCollectionPage"} + end + end + end end diff --git a/test/web/activity_pub/utils_test.exs b/test/web/activity_pub/utils_test.exs index 2e5e95795..1300039aa 100644 --- a/test/web/activity_pub/utils_test.exs +++ b/test/web/activity_pub/utils_test.exs @@ -104,4 +104,69 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do assert Enum.sort(cc) == expected_cc end end + + describe "fetch_ordered_collection" do + import Tesla.Mock + + test "fetches the first OrderedCollectionPage when an OrderedCollection is encountered" do + mock(fn + %{method: :get, url: "http://mastodon.com/outbox"} -> + json(%{"type" => "OrderedCollection", "first" => "http://mastodon.com/outbox?page=true"}) + + %{method: :get, url: "http://mastodon.com/outbox?page=true"} -> + json(%{"type" => "OrderedCollectionPage", "orderedItems" => ["ok"]}) + end) + + assert Utils.fetch_ordered_collection("http://mastodon.com/outbox", 1) == ["ok"] + end + + test "fetches several pages in the right order one after another, but only the specified amount" do + mock(fn + %{method: :get, url: "http://example.com/outbox"} -> + json(%{ + "type" => "OrderedCollectionPage", + "orderedItems" => [0], + "next" => "http://example.com/outbox?page=1" + }) + + %{method: :get, url: "http://example.com/outbox?page=1"} -> + json(%{ + "type" => "OrderedCollectionPage", + "orderedItems" => [1], + "next" => "http://example.com/outbox?page=2" + }) + + %{method: :get, url: "http://example.com/outbox?page=2"} -> + json(%{"type" => "OrderedCollectionPage", "orderedItems" => [2]}) + end) + + assert Utils.fetch_ordered_collection("http://example.com/outbox", 0) == [0] + assert Utils.fetch_ordered_collection("http://example.com/outbox", 1) == [0, 1] + end + + test "returns an error if the url doesn't have an OrderedCollection/Page" do + mock(fn + %{method: :get, url: "http://example.com/not-an-outbox"} -> + json(%{"type" => "NotAnOutbox"}) + end) + + assert {:error, _} = Utils.fetch_ordered_collection("http://example.com/not-an-outbox", 1) + end + + test "returns the what was collected if there are less pages than specified" do + mock(fn + %{method: :get, url: "http://example.com/outbox"} -> + json(%{ + "type" => "OrderedCollectionPage", + "orderedItems" => [0], + "next" => "http://example.com/outbox?page=1" + }) + + %{method: :get, url: "http://example.com/outbox?page=1"} -> + json(%{"type" => "OrderedCollectionPage", "orderedItems" => [1]}) + end) + + assert Utils.fetch_ordered_collection("http://example.com/outbox", 5) == [0, 1] + end + end end |