diff options
31 files changed, 624 insertions, 80 deletions
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index a973beaa9..41a36851e 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -33,7 +33,7 @@ defmodule Mix.Tasks.Pleroma.Database do Logger.info("Removing embedded objects") Repo.query!( - "update activities set data = safe_jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;", + "update objects set data = safe_jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;", [], timeout: :infinity ) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 4106feef6..30d8f7092 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -26,7 +26,7 @@ defmodule Pleroma.Activity do @cachex Pleroma.Config.get([:cachex, :provider], Cachex) - schema "activities" do + schema "objects" do field(:data, :map) field(:local, :boolean, default: true) field(:actor, :string) diff --git a/lib/pleroma/chat/message_reference.ex b/lib/pleroma/chat/message_reference.ex index 89537d155..06fdb3401 100644 --- a/lib/pleroma/chat/message_reference.ex +++ b/lib/pleroma/chat/message_reference.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Chat.MessageReference do @primary_key {:id, FlakeId.Ecto.Type, autogenerate: true} schema "chat_message_references" do - belongs_to(:object, Object) + belongs_to(:object, Object, type: FlakeId.Ecto.CompatType) belongs_to(:chat, Chat, type: FlakeId.Ecto.CompatType) field(:unread, :boolean, default: true) diff --git a/lib/pleroma/data_migration_failed_id.ex b/lib/pleroma/data_migration_failed_id.ex new file mode 100644 index 000000000..117795d44 --- /dev/null +++ b/lib/pleroma/data_migration_failed_id.ex @@ -0,0 +1,13 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.DataMigrationFailedId do + use Ecto.Schema + alias Pleroma.DataMigration + + schema "data_migration_failed_ids" do + belongs_to(:data_migration, DataMigration) + field(:record_id, FlakeId.Ecto.CompatType) + end +end diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex index 511d5cf58..b1ed6a8fd 100644 --- a/lib/pleroma/delivery.ex +++ b/lib/pleroma/delivery.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Delivery do schema "deliveries" do belongs_to(:user, User, type: FlakeId.Ecto.CompatType) - belongs_to(:object, Object) + belongs_to(:object, Object, type: FlakeId.Ecto.CompatType) end def changeset(delivery, params \\ %{}) do diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index 53e2e9c89..046c67943 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -10,13 +10,14 @@ defmodule Pleroma.Hashtag do alias Ecto.Multi alias Pleroma.Hashtag + alias Pleroma.HashtagObject alias Pleroma.Object alias Pleroma.Repo schema "hashtags" do field(:name, :string) - many_to_many(:objects, Object, join_through: "hashtags_objects", on_replace: :delete) + many_to_many(:objects, Object, join_through: HashtagObject, on_replace: :delete) timestamps() end @@ -80,7 +81,7 @@ defmodule Pleroma.Hashtag do def unlink(%Object{id: object_id}) do with {_, hashtag_ids} <- - from(hto in "hashtags_objects", + from(hto in HashtagObject, where: hto.object_id == ^object_id, select: hto.hashtag_id ) diff --git a/lib/pleroma/hashtag_object.ex b/lib/pleroma/hashtag_object.ex new file mode 100644 index 000000000..12b570715 --- /dev/null +++ b/lib/pleroma/hashtag_object.ex @@ -0,0 +1,17 @@ +defmodule Pleroma.HashtagObject do + @moduledoc """ + Through table relationship between hashtags and objects. + https://hexdocs.pm/ecto/polymorphic-associations-with-many-to-many.html + """ + use Ecto.Schema + + alias Pleroma.Hashtag + alias Pleroma.Object + + @primary_key false + + schema "hashtags_objects" do + belongs_to(:hashtag, Hashtag) + belongs_to(:object, Object, type: FlakeId.Ecto.CompatType) + end +end diff --git a/lib/pleroma/migration_helper/legacy_activity.ex b/lib/pleroma/migration_helper/legacy_activity.ex new file mode 100644 index 000000000..30a139d4e --- /dev/null +++ b/lib/pleroma/migration_helper/legacy_activity.ex @@ -0,0 +1,103 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.MigrationHelper.LegacyActivity do + @moduledoc """ + Legacy "activities" schema needed for old migrations. + """ + use Ecto.Schema + + alias Pleroma.Activity.Queries + alias Pleroma.Bookmark + alias Pleroma.MigrationHelper.LegacyActivity, as: Activity + alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.Repo + alias Pleroma.ReportNote + alias Pleroma.User + + import Ecto.Query + + @type t :: %__MODULE__{} + @type actor :: String.t() + + @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true} + + schema "activities" do + field(:data, :map) + field(:local, :boolean, default: true) + field(:actor, :string) + field(:recipients, {:array, :string}, default: []) + field(:thread_muted?, :boolean, virtual: true) + + # A field that can be used if you need to join some kind of other + # id to order / paginate this field by + field(:pagination_id, :string, virtual: true) + + # This is a fake relation, + # do not use outside of with_preloaded_user_actor/with_joined_user_actor + has_one(:user_actor, User, on_delete: :nothing, foreign_key: :id) + # This is a fake relation, do not use outside of with_preloaded_bookmark/get_bookmark + has_one(:bookmark, Bookmark, foreign_key: :activity_id) + # This is a fake relation, do not use outside of with_preloaded_report_notes + has_many(:report_notes, ReportNote, foreign_key: :activity_id) + has_many(:notifications, Notification, on_delete: :delete_all, foreign_key: :activity_id) + + # Attention: this is a fake relation, don't try to preload it blindly and expect it to work! + # The foreign key is embedded in a jsonb field. + # + # To use it, you probably want to do an inner join and a preload: + # + # ``` + # |> join(:inner, [activity], o in Object, + # on: fragment("(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')", + # o.data, activity.data, activity.data)) + # |> preload([activity, object], [object: object]) + # ``` + # + # As a convenience, Activity.with_preloaded_object() sets up an inner join and preload for the + # typical case. + has_one(:object, Object, on_delete: :nothing, foreign_key: :id) + + timestamps() + end + + def with_joined_object(query, join_type \\ :inner) do + join(query, join_type, [activity], o in Object, + on: + fragment( + "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", + o.data, + activity.data, + activity.data + ), + as: :object + ) + end + + def with_preloaded_object(query, join_type \\ :inner) do + query + |> has_named_binding?(:object) + |> if(do: query, else: with_joined_object(query, join_type)) + |> preload([activity, object: object], object: object) + end + + def all_by_ids_with_object(ids) do + Activity + |> where([a], a.id in ^ids) + |> with_preloaded_object() + |> Repo.all() + end + + @doc """ + Accepts `ap_id` or list of `ap_id`. + Returns a query. + """ + @spec create_by_object_ap_id(String.t() | [String.t()]) :: Ecto.Queryable.t() + def create_by_object_ap_id(ap_id) do + Activity + |> Queries.by_object_id(ap_id) + |> Queries.by_type("Create") + end +end diff --git a/lib/pleroma/migration_helper/object_id.ex b/lib/pleroma/migration_helper/object_id.ex new file mode 100644 index 000000000..4c8acffde --- /dev/null +++ b/lib/pleroma/migration_helper/object_id.ex @@ -0,0 +1,72 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.MigrationHelper.ObjectId do + @moduledoc """ + Functions for migrating Object IDs. + """ + alias Pleroma.Chat.MessageReference + alias Pleroma.DataMigrationFailedId + alias Pleroma.Delivery + alias Pleroma.HashtagObject + alias Pleroma.Object + alias Pleroma.Repo + + import Ecto.Changeset + import Ecto.Query + + @doc "Change an object's ID including all references." + def change_id(%Object{id: old_id} = object, new_id) do + Repo.transaction(fn -> + with {:ok, object} <- Repo.update(change(object, id: new_id)), + {:ok, _} <- update_object_fk(MessageReference, old_id, new_id), + {:ok, _} <- update_object_fk(Delivery, old_id, new_id), + {:ok, _} <- update_object_fk(HashtagObject, old_id, new_id), + {:ok, _} <- update_object_fk(DataMigrationFailedId, old_id, new_id, :record_id) do + {:ok, object} + end + end) + end + + defp update_object_fk(schema, old_id, new_id, field \\ :object_id) do + binding = [{field, old_id}] + + schema + |> where(^binding) + |> Repo.update_all(set: [{field, new_id}]) + end + + @doc "Shift a FlakeId by N places." + def shift_id(flake_id, n) when is_integer(n) do + flake_id + |> FlakeId.from_string() + |> FlakeId.to_integer() + |> Kernel.+(n) + |> FlakeId.from_integer() + |> FlakeId.to_string() + end + + @doc "Generate a FlakeId from a datetime." + @spec flake_from_time(NaiveDateTime.t()) :: flake_id :: String.t() + def flake_from_time(%NaiveDateTime{} = dt) do + dt + |> build_worker() + |> FlakeId.Worker.gen_flake() + |> FlakeId.to_string() + end + + # Build a one-off FlakeId worker. + defp build_worker(%NaiveDateTime{} = dt) do + %FlakeId.Worker{ + node: FlakeId.Worker.worker_id(), + time: get_timestamp(dt, :millisecond) + } + end + + # Convert a NaiveDateTime into a Unix timestamp. + @epoch ~N[1970-01-01 00:00:00] + defp get_timestamp(%NaiveDateTime{} = dt, unit) do + NaiveDateTime.diff(dt, @epoch, unit) + end +end diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index b84058e11..b5add9659 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -12,10 +12,14 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do use Pleroma.Migrators.Support.BaseMigrator + alias Pleroma.DataMigrationFailedId alias Pleroma.Hashtag + alias Pleroma.HashtagObject alias Pleroma.Migrators.Support.BaseMigrator alias Pleroma.Object + import Ecto.Query + @impl BaseMigrator def feature_config_path, do: [:features, :improved_hashtag_timeline] @@ -50,19 +54,20 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do for failed_id <- failed_ids do _ = - Repo.query( - "INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <> - "VALUES ($1, $2) ON CONFLICT DO NOTHING;", - [data_migration_id, failed_id] - ) + %DataMigrationFailedId{ + data_migration_id: data_migration_id, + record_id: failed_id + } + |> Repo.insert(on_conflict: :nothing) end + record_ids = object_ids -- failed_ids + _ = - Repo.query( - "DELETE FROM data_migration_failed_ids " <> - "WHERE data_migration_id = $1 AND record_id = ANY($2)", - [data_migration_id, object_ids -- failed_ids] - ) + DataMigrationFailedId + |> where(data_migration_id: ^data_migration_id) + |> where([dmf], dmf.record_id in ^record_ids) + |> Repo.delete_all() max_object_id = Enum.at(object_ids, -1) @@ -120,7 +125,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do try do with {rows_count, _} when is_integer(rows_count) <- - Repo.insert_all("hashtags_objects", maps, on_conflict: :nothing) do + Repo.insert_all(HashtagObject, maps, on_conflict: :nothing) do object.id else e -> @@ -147,14 +152,13 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do failed_objects_query() |> Repo.chunk_stream(100, :one) - |> Stream.each(fn object -> + |> Stream.each(fn %{id: object_id} = object -> with {res, _} when res != :error <- transfer_object_hashtags(object) do _ = - Repo.query( - "DELETE FROM data_migration_failed_ids " <> - "WHERE data_migration_id = $1 AND record_id = $2", - [data_migration_id, object.id] - ) + DataMigrationFailedId + |> where(data_migration_id: ^data_migration_id) + |> where(record_id: ^object_id) + |> Repo.delete_all() end end) |> Stream.run() @@ -167,9 +171,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do defp failed_objects_query do from(o in Object) - |> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"), - on: dmf.record_id == o.id - ) + |> join(:inner, [o], dmf in DataMigrationFailedId, on: dmf.record_id == o.id) |> where([_o, dmf], dmf.data_migration_id == ^data_migration_id()) |> order_by([o], asc: o.id) end diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index c3ea1b98b..8569fed8c 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Object do alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Hashtag + alias Pleroma.HashtagObject alias Pleroma.Object alias Pleroma.Object.Fetcher alias Pleroma.ObjectTombstone @@ -22,6 +23,8 @@ defmodule Pleroma.Object do @type t() :: %__MODULE__{} + @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true} + @derive {Jason.Encoder, only: [:data]} @cachex Pleroma.Config.get([:cachex, :provider], Cachex) @@ -29,7 +32,7 @@ defmodule Pleroma.Object do schema "objects" do field(:data, :map) - many_to_many(:hashtags, Hashtag, join_through: "hashtags_objects", on_replace: :delete) + many_to_many(:hashtags, Hashtag, join_through: HashtagObject, on_replace: :delete) timestamps() end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 756096952..e97f9c138 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Conversation.Participation alias Pleroma.Filter alias Pleroma.Hashtag + alias Pleroma.HashtagObject alias Pleroma.Maps alias Pleroma.Notification alias Pleroma.Object @@ -780,8 +781,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_embedded_tag_reject_any(query, _), do: query defp object_ids_query_for_tags(tags) do - from(hto in "hashtags_objects") - |> join(:inner, [hto], ht in Pleroma.Hashtag, on: hto.hashtag_id == ht.id) + from(hto in HashtagObject) + |> join(:inner, [hto], ht in Hashtag, on: hto.hashtag_id == ht.id) |> where([hto, ht], ht.name in ^tags) |> select([hto], hto.object_id) |> distinct([hto], true) @@ -830,7 +831,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # Note: NO extra ordering should be done on "activities.id desc nulls last" for optimal plan from( [_activity, object] in query, - join: hto in "hashtags_objects", + join: hto in HashtagObject, on: hto.object_id == object.id, where: hto.hashtag_id in ^hashtag_ids, distinct: [desc: object.id], @@ -1165,7 +1166,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_instance(query, %{instance: instance}) when is_binary(instance) do from( activity in query, - where: fragment("split_part(actor::text, '/'::text, 3) = ?", ^instance) + where: fragment("split_part(?::text, '/'::text, 3) = ?", activity.actor, ^instance) ) end diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex index 872f80ec3..cb1f4e144 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex @@ -52,7 +52,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do field(:context, :string) # short identifier for PleromaFE to group statuses by context - field(:context_id, :integer) + field(:context_id, :string) field(:sensitive, :boolean, default: false) field(:replies_count, :integer, default: 0) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index c1f6b2b49..d5f0a3245 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -712,9 +712,10 @@ defmodule Pleroma.Web.ActivityPub.Utils do defp build_flag_object(%{statuses: statuses}) do Enum.map(statuses || [], &build_flag_object/1) + |> Enum.reject(&is_nil/1) end - defp build_flag_object(%Activity{data: %{"id" => id}, object: %{data: data}}) do + defp build_flag_object(%Activity{data: %{"id" => id, "type" => "Create"}, object: %{data: data}}) do activity_actor = User.get_by_ap_id(data["actor"]) %{ @@ -730,28 +731,26 @@ defmodule Pleroma.Web.ActivityPub.Utils do } end - defp build_flag_object(act) when is_map(act) or is_binary(act) do - id = - case act do - %Activity{} = act -> act.data["id"] - act when is_map(act) -> act["id"] - act when is_binary(act) -> act - end + defp build_flag_object(%{data: %{"id" => id}}), do: build_flag_object(id) + defp build_flag_object(%{"id" => id}), do: build_flag_object(id) - case Activity.get_by_ap_id_with_object(id) do - %Activity{} = activity -> + defp build_flag_object(ap_id) when is_binary(ap_id) do + case Activity.get_by_ap_id_with_object(ap_id) do + %Activity{data: %{"type" => "Create"}} = activity -> build_flag_object(activity) - nil -> - if activity = Activity.get_by_object_ap_id_with_object(id) do - build_flag_object(activity) - else - %{"id" => id, "deleted" => true} + _ -> + case Activity.get_by_object_ap_id_with_object(ap_id) do + %Activity{data: %{"type" => "Create"}} = activity -> + build_flag_object(activity) + + _ -> + %{"id" => ap_id, "deleted" => true} end end end - defp build_flag_object(_), do: [] + defp build_flag_object(_), do: nil #### Report-related helpers def get_reports(params, page, page_size) do diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 3caab0f00..2129e351a 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -143,7 +143,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do "A map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`" }, conversation_id: %Schema{ - type: :integer, + type: :string, description: "The ID of the AP context the status is associated with (if any)" }, direct_conversation_id: %Schema{ @@ -319,7 +319,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do "pinned" => false, "pleroma" => %{ "content" => %{"text/plain" => "foobar"}, - "conversation_id" => 345_972, + "conversation_id" => "AEXFhY7X4zd8hZK8oK", "direct_conversation_id" => nil, "emoji_reactions" => [], "expires_at" => nil, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 463f34198..ce350ad23 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -57,11 +57,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end) end - defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id), + defp get_context_id(%{data: %{"context_id" => context_id}}) when is_binary(context_id), do: context_id - defp get_context_id(%{data: %{"context" => context}}) when is_binary(context), - do: Utils.context_to_conversation_id(context) + defp get_context_id(%{data: %{"context_id" => context_id}}) when is_integer(context_id), + do: to_string(context_id) + + defp get_context_id(%{data: %{"context" => context}}) when is_binary(context) do + case Utils.context_to_conversation_id(context) do + id when is_binary(id) -> id + id when is_integer(id) -> to_string(id) + _ -> nil + end + end defp get_context_id(_), do: nil diff --git a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs index b6f0ac66b..cbc9884a5 100644 --- a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs +++ b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs @@ -1,8 +1,8 @@ defmodule Pleroma.Repo.Migrations.MigrateOldBookmarks do use Ecto.Migration import Ecto.Query - alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.MigrationHelper.LegacyActivity alias Pleroma.Repo def up do @@ -18,7 +18,7 @@ defmodule Pleroma.Repo.Migrations.MigrateOldBookmarks do Enum.each(bookmarks, fn ap_id -> activity = ap_id - |> Activity.create_by_object_ap_id() + |> LegacyActivity.create_by_object_ap_id() |> Repo.one() unless is_nil(activity), do: {:ok, _} = Bookmark.create(user_id, activity.id) diff --git a/priv/repo/migrations/20200527163635_delete_notifications_from_invisible_users.exs b/priv/repo/migrations/20200527163635_delete_notifications_from_invisible_users.exs index 9e95a8111..a2010e188 100644 --- a/priv/repo/migrations/20200527163635_delete_notifications_from_invisible_users.exs +++ b/priv/repo/migrations/20200527163635_delete_notifications_from_invisible_users.exs @@ -6,7 +6,7 @@ defmodule Pleroma.Repo.Migrations.DeleteNotificationsFromInvisibleUsers do def up do Pleroma.Notification - |> join(:inner, [n], activity in assoc(n, :activity)) + |> join(:inner, [n], activity in "activities") |> where( [n, a], fragment("? in (SELECT ap_id FROM users WHERE invisible = true)", a.actor) diff --git a/priv/repo/migrations/20200914105638_delete_notification_without_activity.exs b/priv/repo/migrations/20200914105638_delete_notification_without_activity.exs index 9333fc5a1..252eb0716 100644 --- a/priv/repo/migrations/20200914105638_delete_notification_without_activity.exs +++ b/priv/repo/migrations/20200914105638_delete_notification_without_activity.exs @@ -7,7 +7,7 @@ defmodule Pleroma.Repo.Migrations.DeleteNotificationWithoutActivity do def up do from( q in Pleroma.Notification, - left_join: c in assoc(q, :activity), + left_join: c in "activities", select: %{id: type(q.id, :integer)}, where: is_nil(c.id) ) diff --git a/priv/repo/migrations/20210205145000_move_pinned_activities_into_pinned_objects.exs b/priv/repo/migrations/20210205145000_move_pinned_activities_into_pinned_objects.exs index 9aee545e3..a6dd886bf 100644 --- a/priv/repo/migrations/20210205145000_move_pinned_activities_into_pinned_objects.exs +++ b/priv/repo/migrations/20210205145000_move_pinned_activities_into_pinned_objects.exs @@ -3,6 +3,7 @@ defmodule Pleroma.Repo.Migrations.MovePinnedActivitiesIntoPinnedObjects do import Ecto.Query + alias Pleroma.MigrationHelper.LegacyActivity alias Pleroma.Repo alias Pleroma.User @@ -11,7 +12,7 @@ defmodule Pleroma.Repo.Migrations.MovePinnedActivitiesIntoPinnedObjects do |> select([u], {u.id, fragment("?.pinned_activities", u)}) |> Repo.stream() |> Stream.each(fn {user_id, pinned_activities_ids} -> - pinned_activities = Pleroma.Activity.all_by_ids_with_object(pinned_activities_ids) + pinned_activities = LegacyActivity.all_by_ids_with_object(pinned_activities_ids) pins = Map.new(pinned_activities, fn %{object: %{data: %{"id" => object_id}}} -> diff --git a/priv/repo/migrations/20211218181632_change_object_id_to_flake.exs b/priv/repo/migrations/20211218181632_change_object_id_to_flake.exs new file mode 100644 index 000000000..a4533b2ab --- /dev/null +++ b/priv/repo/migrations/20211218181632_change_object_id_to_flake.exs @@ -0,0 +1,51 @@ +defmodule Pleroma.Repo.Migrations.ChangeObjectIdToFlake do + @moduledoc """ + Convert object IDs to FlakeIds. + Fortunately only a few tables have a foreign key to objects. Update them. + """ + use Ecto.Migration + + def up do + # Switch object IDs to FlakeIds + execute(""" + alter table objects + drop constraint objects_pkey cascade, + alter column id drop default, + alter column id set data type uuid using cast( lpad( to_hex(id), 32, '0') as uuid), + add primary key (id) + """) + + # Update data_migration_failed_ids + execute(""" + alter table data_migration_failed_ids + drop constraint data_migration_failed_ids_pkey cascade, + alter column record_id set data type uuid using cast( lpad( to_hex(record_id), 32, '0') as uuid), + add primary key (data_migration_id, record_id) + """) + + # Update chat message foreign key + execute(""" + alter table chat_message_references + alter column object_id set data type uuid using cast( lpad( to_hex(object_id), 32, '0') as uuid), + add constraint chat_message_references_object_id_fkey foreign key (object_id) references objects(id) on delete cascade + """) + + # Update delivery foreign key + execute(""" + alter table deliveries + alter column object_id set data type uuid using cast( lpad( to_hex(object_id), 32, '0') as uuid), + add constraint deliveries_object_id_fkey foreign key (object_id) references objects(id) + """) + + # Update hashtag many-to-many foreign key + execute(""" + alter table hashtags_objects + alter column object_id set data type uuid using cast( lpad( to_hex(object_id), 32, '0') as uuid), + add constraint hashtags_objects_object_id_fkey foreign key (object_id) references objects(id) on delete cascade + """) + end + + def down do + :ok + end +end diff --git a/priv/repo/migrations/20211218181640_resolve_activity_object_conflicts.exs b/priv/repo/migrations/20211218181640_resolve_activity_object_conflicts.exs new file mode 100644 index 000000000..2c24c32f3 --- /dev/null +++ b/priv/repo/migrations/20211218181640_resolve_activity_object_conflicts.exs @@ -0,0 +1,35 @@ +defmodule Pleroma.Repo.Migrations.ResolveActivityObjectConflicts do + @moduledoc """ + Find objects with a conflicting activity ID, and update them. + This should only happen on servers that existed before "20181218172826_users_and_activities_flake_id". + """ + use Ecto.Migration + + alias Pleroma.Object + alias Pleroma.MigrationHelper.ObjectId + alias Pleroma.Repo + + import Ecto.Query + + def up do + activity_conflict_query() + |> Repo.stream() + |> Stream.each(&update_object/1) + |> Stream.run() + end + + # Get only objects with a conflicting activity ID. + defp activity_conflict_query() do + join(Object, :inner, [o], a in "activities", on: a.id == o.id) + end + + # Update the object and its relations with a newly-generated ID. + defp update_object(object) do + new_id = ObjectId.flake_from_time(object.inserted_at) + ObjectId.change_id(object, new_id) + end + + def down do + :ok + end +end diff --git a/priv/repo/migrations/20211218181647_combine_activities_and_objects.exs b/priv/repo/migrations/20211218181647_combine_activities_and_objects.exs new file mode 100644 index 000000000..bc692d99d --- /dev/null +++ b/priv/repo/migrations/20211218181647_combine_activities_and_objects.exs @@ -0,0 +1,157 @@ +defmodule Pleroma.Repo.Migrations.CombineActivitiesAndObjects do + use Ecto.Migration + + @function_name "update_status_visibility_counter_cache" + @trigger_name "status_visibility_counter_cache_trigger" + + def up do + # Lock both tables to avoid a running server meddling with our transaction + execute("LOCK TABLE activities") + execute("LOCK TABLE users") + + # Add missing fields to objects table + alter table(:objects) do + add(:local, :boolean, null: false, default: true) + add(:actor, :string) + add(:recipients, {:array, :string}, default: []) + end + + # Add missing indexes to objects + create_if_not_exists(index(:objects, [:local])) + create_if_not_exists(index(:objects, [:actor, "id DESC NULLS LAST"])) + create_if_not_exists(index(:objects, [:recipients], using: :gin)) + + # Intentionally omit these. According to LiveDashboard they're not used: + # + # create_if_not_exists( + # index(:objects, ["(data->'to')"], name: :objects_to_index, using: :gin) + # ) + # + # create_if_not_exists( + # index(:objects, ["(data->'cc')"], name: :objects_cc_index, using: :gin) + # ) + + create_if_not_exists( + index(:objects, ["(data->>'actor')", "inserted_at desc"], name: :objects_actor_index) + ) + + # Some obscure Fediverse backends (WordPress, Juick) send a Create and a Note + # with the exact same ActivityPub ID. This violates the spec and doesn't + # work in the new system. WordPress devs were notified. + execute( + "DELETE FROM activities USING objects WHERE activities.data->>'id' = objects.data->>'id'" + ) + + # Copy all activities into the newly formatted objects table + execute("INSERT INTO objects (SELECT * FROM activities)") + + # Update notifications foreign key + execute("alter table notifications drop constraint notifications_activity_id_fkey") + + execute( + "alter table notifications add constraint notifications_object_id_fkey foreign key (activity_id) references objects(id) on delete cascade" + ) + + # Update bookmarks foreign key + execute("alter table bookmarks drop constraint bookmarks_activity_id_fkey") + + execute( + "alter table bookmarks add constraint bookmarks_object_id_fkey foreign key (activity_id) references objects(id) on delete cascade" + ) + + # Update report notes foreign key + execute("alter table report_notes drop constraint report_notes_activity_id_fkey") + + execute( + "alter table report_notes add constraint report_notes_object_id_fkey foreign key (activity_id) references objects(id)" + ) + + # Nuke the old activities table + execute("drop table activities") + + # Update triggers + """ + CREATE TRIGGER #{@trigger_name} + BEFORE + INSERT + OR UPDATE of recipients, data + OR DELETE + ON objects + FOR EACH ROW + EXECUTE PROCEDURE #{@function_name}(); + """ + |> execute() + + execute("drop function if exists thread_visibility(actor varchar, activity_id varchar)") + execute(update_thread_visibility()) + end + + def down do + raise "Lol, there's no going back from this." + end + + # It acts upon objects instead of activities now + def update_thread_visibility do + """ + CREATE OR REPLACE FUNCTION thread_visibility(actor varchar, object_id varchar) RETURNS boolean AS $$ + DECLARE + public varchar := 'https://www.w3.org/ns/activitystreams#Public'; + child objects%ROWTYPE; + object objects%ROWTYPE; + author_fa varchar; + valid_recipients varchar[]; + actor_user_following varchar[]; + BEGIN + --- Fetch actor following + SELECT array_agg(following.follower_address) INTO actor_user_following FROM following_relationships + JOIN users ON users.id = following_relationships.follower_id + JOIN users AS following ON following.id = following_relationships.following_id + WHERE users.ap_id = actor; + + --- Fetch our initial object. + SELECT * INTO object FROM objects WHERE objects.data->>'id' = object_id; + + LOOP + --- Ensure that we have an object before continuing. + --- If we don't, the thread is not satisfiable. + IF object IS NULL THEN + RETURN false; + END IF; + + --- We only care about Create objects. + IF object.data->>'type' != 'Create' THEN + RETURN true; + END IF; + + --- Normalize the child object into child. + SELECT * INTO child FROM objects + WHERE COALESCE(object.data->'object'->>'id', object.data->>'object') = objects.data->>'id'; + + --- Fetch the author's AS2 following collection. + SELECT COALESCE(users.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = object.actor; + + --- Prepare valid recipients array. + valid_recipients := ARRAY[actor, public]; + IF ARRAY[author_fa] && actor_user_following THEN + valid_recipients := valid_recipients || author_fa; + END IF; + + --- Check visibility. + IF NOT valid_recipients && object.recipients THEN + --- object not visible, break out of the loop + RETURN false; + END IF; + + --- If there's a parent, load it and do this all over again. + IF (child.data->'inReplyTo' IS NOT NULL) AND (child.data->'inReplyTo' != 'null'::jsonb) THEN + SELECT * INTO object FROM objects + WHERE child.data->>'inReplyTo' = objects.data->>'id'; + ELSE + RETURN true; + END IF; + END LOOP; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """ + end +end diff --git a/priv/repo/migrations/20211231013155_add_object_concurrent_indexes.exs b/priv/repo/migrations/20211231013155_add_object_concurrent_indexes.exs new file mode 100644 index 000000000..3d112d172 --- /dev/null +++ b/priv/repo/migrations/20211231013155_add_object_concurrent_indexes.exs @@ -0,0 +1,40 @@ +defmodule Pleroma.Repo.Migrations.AddObjectConcurrentIndexes do + use Ecto.Migration + @disable_migration_lock true + @disable_ddl_transaction true + + def change do + create_if_not_exists(index(:objects, [:actor, "id DESC NULLS LAST"], concurrently: true)) + + create_if_not_exists( + index(:objects, ["(data->>'type')", "(data->>'context')"], + name: :objects_context_index, + concurrently: true + ) + ) + + create_if_not_exists( + index(:objects, ["(split_part(actor, '/', 3))"], + concurrently: true, + name: :objects_hosts + ) + ) + + create_if_not_exists(index(:objects, ["id desc nulls last", "local"], concurrently: true)) + + create_if_not_exists( + index(:objects, ["activity_visibility(actor, recipients, data)", "id DESC NULLS LAST"], + name: :objects_visibility_index, + concurrently: true, + where: "data->>'type' = 'Create'" + ) + ) + + create_if_not_exists( + index(:objects, ["(coalesce(data->'object'->>'id', data->>'object'))"], + name: :objects_create_objects_index, + concurrently: true + ) + ) + end +end diff --git a/test/pleroma/activity_test.exs b/test/pleroma/activity_test.exs index 4f9144f91..7d0e427fa 100644 --- a/test/pleroma/activity_test.exs +++ b/test/pleroma/activity_test.exs @@ -51,7 +51,8 @@ defmodule Pleroma.ActivityTest do {:ok, bookmark3} = Bookmark.create(user3.id, activity.id) queried_activity = - Ecto.Query.from(Pleroma.Activity) + Activity + |> Ecto.Query.where(id: ^activity.id) |> Activity.with_preloaded_bookmark(user3) |> Repo.one() @@ -64,17 +65,19 @@ defmodule Pleroma.ActivityTest do annoyed_user = insert(:user) {:ok, _} = ThreadMute.add_mute(annoyed_user.id, activity.data["context"]) + query = Ecto.Query.where(Activity, id: ^activity.id) + activity_with_unset_thread_muted_field = - Ecto.Query.from(Activity) + query |> Repo.one() activity_for_user = - Ecto.Query.from(Activity) + query |> Activity.with_set_thread_muted_field(user) |> Repo.one() activity_for_annoyed_user = - Ecto.Query.from(Activity) + query |> Activity.with_set_thread_muted_field(annoyed_user) |> Repo.one() @@ -90,7 +93,7 @@ defmodule Pleroma.ActivityTest do {:ok, bookmark} = Bookmark.create(user.id, activity.id) queried_activity = - Ecto.Query.from(Pleroma.Activity) + Ecto.Query.where(Activity, id: ^activity.id) |> Activity.with_preloaded_bookmark(user) |> Repo.one() @@ -103,7 +106,7 @@ defmodule Pleroma.ActivityTest do {:ok, bookmark} = Bookmark.create(user.id, activity.id) queried_activity = - Ecto.Query.from(Pleroma.Activity) + Ecto.Query.where(Activity, id: ^activity.id) |> Repo.one() assert Activity.get_bookmark(queried_activity, user) == bookmark @@ -266,7 +269,11 @@ defmodule Pleroma.ActivityTest do insert(:add_activity, user: user, note: note) insert(:add_activity, user: user) - assert Repo.aggregate(Activity, :count, :id) == 4 + activities_query = + Activity + |> Ecto.Query.where(fragment("data->>'type' IN ('Create', 'Add')")) + + assert Repo.aggregate(activities_query, :count, :id) == 4 add_query = Activity.add_by_params_query(note.data["object"], user.ap_id, user.featured_address) @@ -276,6 +283,6 @@ defmodule Pleroma.ActivityTest do Repo.delete_all(add_query) assert Repo.aggregate(add_query, :count, :id) == 0 - assert Repo.aggregate(Activity, :count, :id) == 2 + assert Repo.aggregate(activities_query, :count, :id) == 2 end end diff --git a/test/pleroma/migration_helper/object_id_test.exs b/test/pleroma/migration_helper/object_id_test.exs new file mode 100644 index 000000000..45567f54a --- /dev/null +++ b/test/pleroma/migration_helper/object_id_test.exs @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.MigrationHelper.ObjectIdTest do + use Pleroma.DataCase, async: true + alias Pleroma.MigrationHelper.ObjectId + + test "shift_id/2" do + id = "AEma8DXGjGtUDO6Qeu" + assert ObjectId.shift_id(id, 1) == "AEma8DXGjGtUDO6Qev" + assert ObjectId.shift_id(id, -1) == "AEma8DXGjGtUDO6Qet" + end + + test "flake_from_time/1" do + now = NaiveDateTime.utc_now() + id = ObjectId.flake_from_time(now) + + assert FlakeId.flake_id?(id) + end +end diff --git a/test/pleroma/pagination_test.exs b/test/pleroma/pagination_test.exs index bc26c8b46..6e5c767e6 100644 --- a/test/pleroma/pagination_test.exs +++ b/test/pleroma/pagination_test.exs @@ -18,7 +18,7 @@ defmodule Pleroma.PaginationTest do end test "paginates by min_id", %{notes: notes} do - id = Enum.at(notes, 2).id |> Integer.to_string() + id = Enum.at(notes, 2).id %{total: total, items: paginated} = Pagination.fetch_paginated(Object, %{min_id: id, total: true}) @@ -28,7 +28,7 @@ defmodule Pleroma.PaginationTest do end test "paginates by since_id", %{notes: notes} do - id = Enum.at(notes, 2).id |> Integer.to_string() + id = Enum.at(notes, 2).id %{total: total, items: paginated} = Pagination.fetch_paginated(Object, %{since_id: id, total: true}) @@ -38,7 +38,7 @@ defmodule Pleroma.PaginationTest do end test "paginates by max_id", %{notes: notes} do - id = Enum.at(notes, 1).id |> Integer.to_string() + id = Enum.at(notes, 1).id %{total: total, items: paginated} = Pagination.fetch_paginated(Object, %{max_id: id, total: true}) @@ -48,7 +48,7 @@ defmodule Pleroma.PaginationTest do end test "paginates by min_id & limit", %{notes: notes} do - id = Enum.at(notes, 2).id |> Integer.to_string() + id = Enum.at(notes, 2).id paginated = Pagination.fetch_paginated(Object, %{min_id: id, limit: 1}) @@ -56,7 +56,7 @@ defmodule Pleroma.PaginationTest do end test "handles id gracefully", %{notes: notes} do - id = Enum.at(notes, 1).id |> Integer.to_string() + id = Enum.at(notes, 1).id paginated = Pagination.fetch_paginated(Object, %{ diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 6cd93c34c..607d81470 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -19,6 +19,8 @@ defmodule Pleroma.UserTest do import ExUnit.CaptureLog import Swoosh.TestAssertions + require Ecto.Query + setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) :ok @@ -436,7 +438,11 @@ defmodule Pleroma.UserTest do {:ok, registered_user} = User.register(cng) ObanHelpers.perform_all() - activity = Repo.one(Pleroma.Activity) + activity = + Activity + |> Ecto.Query.where(fragment("data->>'type' = 'Create'")) + |> Repo.one() + assert registered_user.ap_id in activity.recipients assert Object.normalize(activity, fetch: false).data["content"] =~ "direct message" assert activity.actor == welcome_user.ap_id @@ -452,7 +458,11 @@ defmodule Pleroma.UserTest do {:ok, registered_user} = User.register(cng) ObanHelpers.perform_all() - activity = Repo.one(Pleroma.Activity) + activity = + Activity + |> Ecto.Query.where(fragment("data->>'type' = 'Create'")) + |> Repo.one() + assert registered_user.ap_id in activity.recipients assert Object.normalize(activity, fetch: false).data["content"] =~ "chat message" assert activity.actor == welcome_user.ap_id @@ -491,7 +501,11 @@ defmodule Pleroma.UserTest do {:ok, registered_user} = User.register(cng) ObanHelpers.perform_all() - activity = Repo.one(Pleroma.Activity) + activity = + Activity + |> Ecto.Query.where(fragment("data->>'type' = 'Create'")) + |> Repo.one() + assert registered_user.ap_id in activity.recipients assert Object.normalize(activity, fetch: false).data["content"] =~ "chat message" assert activity.actor == welcome_user.ap_id diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 50315e21f..30ea6be50 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -1143,7 +1143,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) - assert Pleroma.Repo.aggregate(Activity, :count, :id) == 2 + assert Pleroma.Repo.aggregate(Object, :count, :id) == 4 ObanHelpers.perform_all() diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index 574ef0d71..7313ed42e 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -1394,8 +1394,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do ActivityPub.fetch_activities([user1.ap_id | User.following(user1)], %{user: user1}) |> Enum.map(fn a -> a.id end) - assert [public_activity.id, private_activity_1.id] == activities - assert length(activities) == 2 + assert [public_activity.id, private_activity_1.id, private_activity_3.id] == activities + assert length(activities) == 3 end end @@ -1514,8 +1514,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do content: content }) - assert Repo.aggregate(Activity, :count, :id) == 1 - assert Repo.aggregate(Object, :count, :id) == 2 + assert Repo.aggregate(Object, :count, :id) == 3 assert Repo.aggregate(Notification, :count, :id) == 0 end end diff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs index 99cc7bbd0..e480a0118 100644 --- a/test/pleroma/web/admin_api/controllers/report_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/report_controller_test.exs @@ -363,7 +363,7 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do [note, _] = notes assert note["user"]["nickname"] == admin.nickname - assert note["content"] == "this is disgusting!" + assert note["content"] =~ "this is disgusting" assert note["created_at"] assert response["total"] == 1 end |