Pipeline Ingestion: Note

10 files changed, 81 insertions, 64 deletions

diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex
index b76547e75..a03471462 100644
--- a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex
+++ b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex
@@ -13,20 +13,23 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do
     cast([object])
   end
 
+  def cast(object) when is_map(object) do
+    case ObjectID.cast(object) do
+      {:ok, data} -> {:ok, data}
+      _ -> :error
+    end
+  end
+
   def cast(data) when is_list(data) do
     data
-    |> Enum.reduce_while({:ok, []}, fn
-      nil, {:ok, list} ->
-        {:cont, {:ok, list}}
-
-      element, {:ok, list} ->
-        case ObjectID.cast(element) do
-          {:ok, id} ->
-            {:cont, {:ok, [id | list]}}
-
-          _ ->
-            {:halt, {:error, element}}
-        end
+    |> Enum.reduce_while({:ok, []}, fn element, {:ok, list} ->
+      case ObjectID.cast(element) do
+        {:ok, id} ->
+          {:cont, {:ok, [id | list]}}
+
+        _ ->
+          {:cont, {:ok, list}}
+      end
     end)
   end
 
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index efbf92c70..b74af3f3b 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -88,7 +88,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  @object_types ~w[ChatMessage Question Answer Audio Video Event Article]
+  @object_types ~w[ChatMessage Question Answer Audio Video Event Article Note]
   @impl true
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index 70d9a35a9..e5b35cdd4 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -101,7 +101,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
         meta
       )
-      when objtype in ~w[Question Answer Audio Video Event Article] do
+      when objtype in ~w[Question Answer Audio Video Event Article Note] do
     with {:ok, object_data} <- cast_and_apply(object),
          meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
          {:ok, create_activity} <-
@@ -114,7 +114,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   end
 
   def validate(%{"type" => type} = object, meta)
-      when type in ~w[Event Question Audio Video Article] do
+      when type in ~w[Event Question Audio Video Article Note] do
     validator =
       case type do
         "Event" -> EventValidator
@@ -122,6 +122,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
         "Audio" -> AudioVideoValidator
         "Video" -> AudioVideoValidator
         "Article" -> ArticleNoteValidator
+        "Note" -> ArticleNoteValidator
       end
 
     with {:ok, object} <-
@@ -183,7 +184,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     EventValidator.cast_and_apply(object)
   end
 
-  def cast_and_apply(%{"type" => "Article"} = object) do
+  def cast_and_apply(%{"type" => type} = object) when type in ~w[Article Note] do
     ArticleNoteValidator.cast_and_apply(object)
   end
 
diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex
index d2026b5ea..193f85f49 100644
--- a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex
@@ -50,6 +50,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
 
     field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
     field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
+
+    field(:replies, {:array, ObjectValidators.ObjectID}, default: [])
   end
 
   def cast_and_apply(data) do
@@ -65,24 +67,39 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
   end
 
   def cast_data(data) do
-    data = fix(data)
-
     %__MODULE__{}
     |> changeset(data)
   end
 
-  defp fix_url(%{"url" => url} = data) when is_map(url) do
-    Map.put(data, "url", url["href"])
-  end
-
+  defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data
+  defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"])
   defp fix_url(data), do: data
 
+  defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data
+  defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag])
+  defp fix_tag(data), do: Map.drop(data, ["tag"])
+
+  defp fix_replies(%{"replies" => %{"first" => %{"items" => replies}}} = data)
+       when is_list(replies),
+       do: Map.put(data, "replies", replies)
+
+  defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies),
+    do: Map.put(data, "replies", replies)
+
+  defp fix_replies(%{"replies" => replies} = data) when is_bitstring(replies),
+    do: Map.drop(data, ["replies"])
+
+  defp fix_replies(data), do: data
+
   defp fix(data) do
     data
     |> CommonFixes.fix_actor()
     |> CommonFixes.fix_object_defaults()
     |> fix_url()
+    |> fix_tag()
+    |> fix_replies()
     |> Transmogrifier.fix_emoji()
+    |> Transmogrifier.fix_content_map()
   end
 
   def changeset(struct, data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
index 950eb1494..7309f6af2 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
@@ -26,14 +26,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
     |> Transmogrifier.fix_implicit_addressing(follower_collection)
   end
 
-  def fix_activity_defaults(data, meta) do
+  defp fix_activity_recipients(activity, field, object) do
+    {:ok, data} = ObjectValidators.Recipients.cast(activity[field] || object[field])
+
+    Map.put(activity, field, data)
+  end
+
+  def fix_activity_defaults(activity, meta) do
     object = meta[:object_data] || %{}
 
-    data
-    |> Map.put_new("to", object["to"] || [])
-    |> Map.put_new("cc", object["cc"] || [])
-    |> Map.put_new("bto", object["bto"] || [])
-    |> Map.put_new("bcc", object["bcc"] || [])
+    activity
+    |> fix_activity_recipients("to", object)
+    |> fix_activity_recipients("cc", object)
+    |> fix_activity_recipients("bto", object)
+    |> fix_activity_recipients("bcc", object)
   end
 
   def fix_actor(data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
index 093549a45..85ac07044 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
@@ -14,6 +14,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
       fields
       |> Enum.map(fn field -> get_field(cng, field) end)
       |> Enum.any?(fn
+        nil -> false
         [] -> false
         _ -> true
       end)
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex
deleted file mode 100644
index a85a0298c..000000000
--- a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex
+++ /dev/null
@@ -1,29 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do
-  use Ecto.Schema
-
-  alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator
-
-  import Ecto.Changeset
-
-  @primary_key false
-
-  embedded_schema do
-    field(:id, ObjectValidators.ObjectID, primary_key: true)
-    field(:actor, ObjectValidators.ObjectID)
-    field(:type, :string)
-    field(:to, ObjectValidators.Recipients, default: [])
-    field(:cc, ObjectValidators.Recipients, default: [])
-    field(:bto, ObjectValidators.Recipients, default: [])
-    field(:bcc, ObjectValidators.Recipients, default: [])
-    embeds_one(:object, NoteValidator)
-  end
-
-  def cast_data(data) do
-    cast(%__MODULE__{}, data, __schema__(:fields))
-  end
-end
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 0b9a9f0c5..3234b9e43 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -203,6 +203,19 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
         Object.increase_replies_count(in_reply_to)
       end
 
+      reply_depth = (meta[:depth] || 0) + 1
+
+      # FIXME: Force inReplyTo to replies
+      if Pleroma.Web.Federator.allowed_thread_distance?(reply_depth) and
+           object.data["replies"] != nil do
+        for reply_id <- object.data["replies"] do
+          Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{
+            "id" => reply_id,
+            "depth" => reply_depth
+          })
+        end
+      end
+
       ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
         Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
       end)
@@ -366,7 +379,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   end
 
   def handle_object_creation(%{"type" => objtype} = object, meta)
-      when objtype in ~w[Audio Video Question Event Article] do
+      when objtype in ~w[Audio Video Question Event Article Note] do
     with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
       {:ok, object, meta}
     end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 047f23918..28bc25363 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -404,10 +404,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   # - tags
   # - emoji
   def handle_incoming(
-        %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
+        %{"type" => "Create", "object" => %{"type" => "Page"} = object} = data,
         options
-      )
-      when objtype in ~w{Note Page} do
+      ) do
     actor = Containment.get_actor(data)
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
@@ -499,14 +498,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def handle_incoming(
         %{"type" => "Create", "object" => %{"type" => objtype, "id" => obj_id}} = data,
-        _options
+        options
       )
-      when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do
+      when objtype in ~w{Question Answer ChatMessage Audio Video Event Article Note} do
     data = Map.put(data, "object", strip_internal_fields(data["object"]))
+    options = Keyword.put(options, :local, false)
 
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
          nil <- Activity.get_create_by_object_ap_id(obj_id),
-         {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
+         {:ok, activity, _} <- Pipeline.common_pipeline(data, options) do
       {:ok, activity}
     else
       %Activity{} = activity -> {:ok, activity}
diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex
index f5ef76d32..69cfc2d52 100644
--- a/lib/pleroma/web/federator.ex
+++ b/lib/pleroma/web/federator.ex
@@ -96,6 +96,11 @@ defmodule Pleroma.Web.Federator do
         Logger.debug("Unhandled actor #{actor}, #{inspect(e)}")
         {:error, e}
 
+      {:error, {:validate_object, _}} = e ->
+        Logger.error("Incoming AP doc validation error: #{inspect(e)}")
+        Logger.debug(Jason.encode!(params, pretty: true))
+        e
+
       e ->
         # Just drop those for now
         Logger.debug(fn -> "Unhandled activity\n" <> Jason.encode!(params, pretty: true) end)