Merge branch 'refactor/following-relationships' into 'develop'

Move following relationships to a separate table

Closes #1233

See merge request pleroma/pleroma!1814

3 files changed, 254 insertions, 0 deletions

diff --git a/priv/repo/migrations/20191007073319_create_following_relationships.exs b/priv/repo/migrations/20191007073319_create_following_relationships.exs
new file mode 100644
index 000000000..d49e24ee4
--- /dev/null
+++ b/priv/repo/migrations/20191007073319_create_following_relationships.exs
@@ -0,0 +1,149 @@
+defmodule Pleroma.Repo.Migrations.CreateFollowingRelationships do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists table(:following_relationships) do
+      add(:follower_id, references(:users, type: :uuid, on_delete: :delete_all), null: false)
+      add(:following_id, references(:users, type: :uuid, on_delete: :delete_all), null: false)
+      add(:state, :string, null: false)
+
+      timestamps()
+    end
+
+    create_if_not_exists(index(:following_relationships, :follower_id))
+    create_if_not_exists(unique_index(:following_relationships, [:follower_id, :following_id]))
+
+    execute(update_thread_visibility(), restore_thread_visibility())
+  end
+
+  # The only difference between the original version: `actor_user` replaced with `actor_user_following`
+  def update_thread_visibility do
+    """
+    CREATE OR REPLACE FUNCTION thread_visibility(actor varchar, activity_id varchar) RETURNS boolean AS $$
+    DECLARE
+      public varchar := 'https://www.w3.org/ns/activitystreams#Public';
+      child objects%ROWTYPE;
+      activity activities%ROWTYPE;
+      author_fa varchar;
+      valid_recipients varchar[];
+      actor_user_following varchar[];
+    BEGIN
+      --- Fetch actor following
+      SELECT array_agg(following.follower_address) INTO actor_user_following FROM following_relationships
+      JOIN users ON users.id = following_relationships.follower_id
+      JOIN users AS following ON following.id = following_relationships.following_id
+      WHERE users.ap_id = actor;
+
+      --- Fetch our initial activity.
+      SELECT * INTO activity FROM activities WHERE activities.data->>'id' = activity_id;
+
+      LOOP
+        --- Ensure that we have an activity before continuing.
+        --- If we don't, the thread is not satisfiable.
+        IF activity IS NULL THEN
+          RETURN false;
+        END IF;
+
+        --- We only care about Create activities.
+        IF activity.data->>'type' != 'Create' THEN
+          RETURN true;
+        END IF;
+
+        --- Normalize the child object into child.
+        SELECT * INTO child FROM objects
+        INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id'
+        WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id';
+
+        --- Fetch the author's AS2 following collection.
+        SELECT COALESCE(users.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor;
+
+        --- Prepare valid recipients array.
+        valid_recipients := ARRAY[actor, public];
+        IF ARRAY[author_fa] && actor_user_following THEN
+          valid_recipients := valid_recipients || author_fa;
+        END IF;
+
+        --- Check visibility.
+        IF NOT valid_recipients && activity.recipients THEN
+          --- activity not visible, break out of the loop
+          RETURN false;
+        END IF;
+
+        --- If there's a parent, load it and do this all over again.
+        IF (child.data->'inReplyTo' IS NOT NULL) AND (child.data->'inReplyTo' != 'null'::jsonb) THEN
+          SELECT * INTO activity FROM activities
+          INNER JOIN objects ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id'
+          WHERE child.data->>'inReplyTo' = objects.data->>'id';
+        ELSE
+          RETURN true;
+        END IF;
+      END LOOP;
+    END;
+    $$ LANGUAGE plpgsql IMMUTABLE;
+    """
+  end
+
+  # priv/repo/migrations/20190515222404_add_thread_visibility_function.exs
+  def restore_thread_visibility do
+    """
+    CREATE OR REPLACE FUNCTION thread_visibility(actor varchar, activity_id varchar) RETURNS boolean AS $$
+    DECLARE
+      public varchar := 'https://www.w3.org/ns/activitystreams#Public';
+      child objects%ROWTYPE;
+      activity activities%ROWTYPE;
+      actor_user users%ROWTYPE;
+      author_fa varchar;
+      valid_recipients varchar[];
+    BEGIN
+      --- Fetch our actor.
+      SELECT * INTO actor_user FROM users WHERE users.ap_id = actor;
+
+      --- Fetch our initial activity.
+      SELECT * INTO activity FROM activities WHERE activities.data->>'id' = activity_id;
+
+      LOOP
+        --- Ensure that we have an activity before continuing.
+        --- If we don't, the thread is not satisfiable.
+        IF activity IS NULL THEN
+          RETURN false;
+        END IF;
+
+        --- We only care about Create activities.
+        IF activity.data->>'type' != 'Create' THEN
+          RETURN true;
+        END IF;
+
+        --- Normalize the child object into child.
+        SELECT * INTO child FROM objects
+        INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id'
+        WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id';
+
+        --- Fetch the author's AS2 following collection.
+        SELECT COALESCE(users.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor;
+
+        --- Prepare valid recipients array.
+        valid_recipients := ARRAY[actor, public];
+        IF ARRAY[author_fa] && actor_user.following THEN
+          valid_recipients := valid_recipients || author_fa;
+        END IF;
+
+        --- Check visibility.
+        IF NOT valid_recipients && activity.recipients THEN
+          --- activity not visible, break out of the loop
+          RETURN false;
+        END IF;
+
+        --- If there's a parent, load it and do this all over again.
+        IF (child.data->'inReplyTo' IS NOT NULL) AND (child.data->'inReplyTo' != 'null'::jsonb) THEN
+          SELECT * INTO activity FROM activities
+          INNER JOIN objects ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id'
+          WHERE child.data->>'inReplyTo' = objects.data->>'id';
+        ELSE
+          RETURN true;
+        END IF;
+      END LOOP;
+    END;
+    $$ LANGUAGE plpgsql IMMUTABLE;
+    """
+  end
+end
diff --git a/priv/repo/migrations/20191008132217_migrate_following_relationships.exs b/priv/repo/migrations/20191008132217_migrate_following_relationships.exs
new file mode 100644
index 000000000..9d5c2648f
--- /dev/null
+++ b/priv/repo/migrations/20191008132217_migrate_following_relationships.exs
@@ -0,0 +1,89 @@
+defmodule Pleroma.Repo.Migrations.MigrateFollowingRelationships do
+  use Ecto.Migration
+
+  def change do
+    execute(import_following_from_users(), "")
+    execute(import_following_from_activities(), restore_following_column())
+  end
+
+  defp import_following_from_users do
+    """
+    INSERT INTO following_relationships (follower_id, following_id, state, inserted_at, updated_at)
+    SELECT
+        relations.follower_id,
+        following.id,
+        'accept',
+        now(),
+        now()
+    FROM (
+        SELECT
+            users.id AS follower_id,
+            unnest(users.following) AS following_ap_id
+        FROM
+            users
+        WHERE
+            users.following != '{}'
+            AND users.local = false OR users.local = true AND users.email IS NOT NULL -- Exclude `internal/fetch` and `relay`
+    ) AS relations
+        JOIN users AS "following" ON "following".follower_address = relations.following_ap_id
+
+        WHERE relations.follower_id != following.id
+    ON CONFLICT DO NOTHING
+    """
+  end
+
+  defp import_following_from_activities do
+    """
+    INSERT INTO
+        following_relationships (
+            follower_id,
+            following_id,
+            state,
+            inserted_at,
+            updated_at
+        )
+    SELECT
+        followers.id,
+        following.id,
+        activities.data ->> 'state',
+        (activities.data ->> 'published') :: timestamp,
+        now()
+    FROM
+        activities
+        JOIN users AS followers ON (activities.actor = followers.ap_id)
+        JOIN users AS following ON (activities.data ->> 'object' = following.ap_id)
+    WHERE
+        activities.data ->> 'type' = 'Follow'
+        AND activities.data ->> 'state' IN ('accept', 'pending', 'reject')
+    ORDER BY activities.updated_at DESC
+    ON CONFLICT DO NOTHING
+    """
+  end
+
+  defp restore_following_column do
+    """
+    UPDATE
+        users
+    SET
+        following = following_query.following_array,
+        updated_at = now()
+    FROM (
+        SELECT
+            follower.id AS follower_id,
+            CASE follower.local
+            WHEN TRUE THEN
+                array_prepend(follower.follower_address, array_agg(following.follower_address))
+            ELSE
+                array_agg(following.follower_address)
+            END AS following_array
+        FROM
+            following_relationships
+            JOIN users AS follower ON follower.id = following_relationships.follower_id
+            JOIN users AS following ON following.id = following_relationships.following_id
+        GROUP BY
+            follower.id) AS following_query
+    WHERE
+        following_query.follower_id = users.id
+    """
+  end
+end
diff --git a/priv/repo/migrations/20191008132427_drop_users_following.exs b/priv/repo/migrations/20191008132427_drop_users_following.exs
new file mode 100644
index 000000000..21c0af9f4
--- /dev/null
+++ b/priv/repo/migrations/20191008132427_drop_users_following.exs
@@ -0,0 +1,16 @@
+defmodule Pleroma.Repo.Migrations.DropUsersFollowing do
+  use Ecto.Migration
+
+  # had to disable these to be able to restore `following` index concurrently
+  # https://hexdocs.pm/ecto_sql/Ecto.Migration.html#index/3-adding-dropping-indexes-concurrently
+  @disable_ddl_transaction true
+  @disable_migration_lock true
+
+  def change do
+    drop(index(:users, [:following], concurrently: true, using: :gin))
+
+    alter table(:users) do
+      remove(:following, {:array, :string}, default: [])
+    end
+  end
+end