From e9e17e5df34051bce60232890ea042582af31f8c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 00:27:51 -0500 Subject: Upgrade Earmark to v1.4.10 --- lib/pleroma/earmark_renderer.ex | 256 --------------------- lib/pleroma/formatter.ex | 8 + .../object_validators/audio_video_validator.ex | 3 +- lib/pleroma/web/common_api/utils.ex | 3 +- 4 files changed, 11 insertions(+), 259 deletions(-) delete mode 100644 lib/pleroma/earmark_renderer.ex (limited to 'lib') diff --git a/lib/pleroma/earmark_renderer.ex b/lib/pleroma/earmark_renderer.ex deleted file mode 100644 index 6211a3b4a..000000000 --- a/lib/pleroma/earmark_renderer.ex +++ /dev/null @@ -1,256 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only -# -# This file is derived from Earmark, under the following copyright: -# Copyright © 2014 Dave Thomas, The Pragmatic Programmers -# SPDX-License-Identifier: Apache-2.0 -# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex -defmodule Pleroma.EarmarkRenderer do - @moduledoc false - - alias Earmark.Block - alias Earmark.Context - alias Earmark.HtmlRenderer - alias Earmark.Options - - import Earmark.Inline, only: [convert: 3] - import Earmark.Helpers.HtmlHelpers - import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2] - import Earmark.Context, only: [append: 2, set_value: 2] - import Earmark.Options, only: [get_mapper: 1] - - @doc false - def render(blocks, %Context{options: %Options{}} = context) do - messages = get_messages(context) - - {contexts, html} = - get_mapper(context.options).( - blocks, - &render_block(&1, put_in(context.options.messages, [])) - ) - |> Enum.unzip() - - all_messages = - contexts - |> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end) - - {put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()} - end - - ############# - # Paragraph # - ############# - defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do - lines = convert(lines, lnb, context) - add_attrs(lines, "

#{lines.value}

", attrs, [], lnb) - end - - ######## - # Html # - ######## - defp render_block(%Block.Html{html: html}, context) do - {context, html} - end - - defp render_block(%Block.HtmlComment{lines: lines}, context) do - {context, lines} - end - - defp render_block(%Block.HtmlOneline{html: html}, context) do - {context, html} - end - - ######### - # Ruler # - ######### - defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do - add_attrs(context, "
", attrs, [], lnb) - end - - ########### - # Heading # - ########### - defp render_block( - %Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs}, - context - ) do - converted = convert(content, lnb, context) - html = "#{converted.value}" - add_attrs(converted, html, attrs, [], lnb) - end - - ############## - # Blockquote # - ############## - - defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do - {context1, body} = render(blocks, context) - html = "
#{body}
" - add_attrs(context1, html, attrs, [], lnb) - end - - ######### - # Table # - ######### - - defp render_block( - %Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs}, - context - ) do - {context1, html} = add_attrs(context, "", attrs, [], lnb) - context2 = set_value(context1, html) - - context3 = - if header do - append(add_trs(append(context2, ""), [header], "th", aligns, lnb), "") - else - # Maybe an error, needed append(context, html) - context2 - end - - context4 = append(add_trs(append(context3, ""), rows, "td", aligns, lnb), "") - - {context4, [context4.value, "
"]} - end - - ######## - # Code # - ######## - - defp render_block( - %Block.Code{lnb: lnb, language: language, attrs: attrs} = block, - %Context{options: options} = context - ) do - class = - if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: "" - - tag = ~s[
]
-    lines = options.render_code.(block)
-    html = ~s[#{tag}#{lines}
] - add_attrs(context, html, attrs, [], lnb) - end - - ######### - # Lists # - ######### - - defp render_block( - %Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start}, - context - ) do - {context1, content} = render(items, context) - html = "<#{type}#{start}>#{content}" - add_attrs(context1, html, attrs, [], lnb) - end - - # format a single paragraph list item, and remove the para tags - defp render_block( - %Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs}, - context - ) - when length(blocks) == 1 do - {context1, content} = render(blocks, context) - content = Regex.replace(~r{}, content, "") - html = "
  • #{content}
    • " - add_attrs(context1, html, attrs, [], lnb) - end - - # format a spaced list item - defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do - {context1, content} = render(blocks, context) - html = "
  • #{content}
    • " - add_attrs(context1, html, attrs, [], lnb) - end - - ################## - # Footnote Block # - ################## - - defp render_block(%Block.FnList{blocks: footnotes}, context) do - items = - Enum.map(footnotes, fn note -> - blocks = append_footnote_link(note) - %Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks} - end) - - {context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context) - {context1, Enum.join([~s[
    ], "
    ", html, "
    "])} - end - - ####################################### - # Isolated IALs are rendered as paras # - ####################################### - - defp render_block(%Block.Ial{verbatim: verbatim}, context) do - {context, "

    {:#{verbatim}}

    "} - end - - #################### - # IDDef is ignored # - #################### - - defp render_block(%Block.IdDef{}, context), do: {context, ""} - - ##################################### - # And here are the inline renderers # - ##################################### - - defdelegate br, to: HtmlRenderer - defdelegate codespan(text), to: HtmlRenderer - defdelegate em(text), to: HtmlRenderer - defdelegate strong(text), to: HtmlRenderer - defdelegate strikethrough(text), to: HtmlRenderer - - defdelegate link(url, text), to: HtmlRenderer - defdelegate link(url, text, title), to: HtmlRenderer - - defdelegate image(path, alt, title), to: HtmlRenderer - - defdelegate footnote_link(ref, backref, number), to: HtmlRenderer - - # Table rows - defp add_trs(context, rows, tag, aligns, lnb) do - numbered_rows = - rows - |> Enum.zip(Stream.iterate(lnb, &(&1 + 1))) - - numbered_rows - |> Enum.reduce(context, fn {row, lnb}, ctx -> - append(add_tds(append(ctx, ""), row, tag, aligns, lnb), "") - end) - end - - defp add_tds(context, row, tag, aligns, lnb) do - Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb)) - end - - defp add_td_fn(row, tag, aligns, lnb) do - fn n, ctx -> - style = - case Enum.at(aligns, n - 1, :default) do - :default -> "" - align -> " style=\"text-align: #{align}\"" - end - - col = Enum.at(row, n - 1) - converted = convert(col, lnb, set_messages(ctx, [])) - append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}") - end - end - - ############################### - # Append Footnote Return Link # - ############################### - - defdelegate append_footnote_link(note), to: HtmlRenderer - defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer - - defdelegate render_code(lines), to: HtmlRenderer - - defp code_classes(language, prefix) do - ["" | String.split(prefix || "")] - |> Enum.map(fn pfx -> "#{pfx}#{language}" end) - |> Enum.join(" ") - end -end diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 0c450eae4..b0e4a84ae 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -138,6 +138,14 @@ defmodule Pleroma.Formatter do |> Enum.join("") end + def minify({text, mentions, hashtags}, type) do + {minify(text, type), mentions, hashtags} + end + + def minify(text, "text/html") do + String.replace(text, "\n", "") + end + def truncate(text, max_length \\ 200, omission \\ "...") do # Remove trailing whitespace text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}") diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index 16973e5db..eaf94797a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do use Ecto.Schema - alias Pleroma.EarmarkRenderer alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes @@ -96,7 +95,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do when is_binary(content) do content = content - |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer}) + |> Earmark.as_html!() |> Pleroma.HTML.filter_tags() Map.put(data, "content", content) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1c74ea787..b434a069e 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -294,8 +294,9 @@ defmodule Pleroma.Web.CommonAPI.Utils do def format_input(text, "text/markdown", options) do text |> Formatter.mentions_escape(options) - |> Earmark.as_html!(%Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + |> Earmark.as_html!() |> Formatter.linkify(options) + |> Formatter.minify("text/html") |> Formatter.html_escape("text/html") end -- cgit v1.2.3 From ba71bbf6101847292346ba3b1fbe78ce4c385919 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 01:53:25 -0500 Subject: Improve Formatter.minify/2 --- lib/pleroma/formatter.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index b0e4a84ae..61906dda6 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -143,7 +143,10 @@ defmodule Pleroma.Formatter do end def minify(text, "text/html") do - String.replace(text, "\n", "") + text + |> String.replace(">\n", ">") + |> String.replace("> ", ">") + |> String.replace(" <", "<") end def truncate(text, max_length \\ 200, omission \\ "...") do -- cgit v1.2.3 From f8c93246d69a193ead81248879ba260e98673b3d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 14:27:50 -0500 Subject: Refactor Earmark code, fix tests --- lib/pleroma/formatter.ex | 4 ++++ .../web/activity_pub/object_validators/audio_video_validator.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 61906dda6..1be12055f 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -121,6 +121,10 @@ defmodule Pleroma.Formatter do end end + def markdown_to_html(text) do + Earmark.as_html!(text) + end + def html_escape({text, mentions, hashtags}, type) do {html_escape(text, type), mentions, hashtags} end diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index eaf94797a..9b38aa4c2 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -95,7 +95,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do when is_binary(content) do content = content - |> Earmark.as_html!() + |> Pleroma.Formatter.markdown_to_html() |> Pleroma.HTML.filter_tags() Map.put(data, "content", content) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index b434a069e..be86009af 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -294,7 +294,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do def format_input(text, "text/markdown", options) do text |> Formatter.mentions_escape(options) - |> Earmark.as_html!() + |> Formatter.markdown_to_html() |> Formatter.linkify(options) |> Formatter.minify("text/html") |> Formatter.html_escape("text/html") -- cgit v1.2.3 From 642729b49fca41fb142c6121fedf35c96c03b018 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 19:16:57 -0500 Subject: Fix AudioVideoValidator markdown --- lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index 9b38aa4c2..fa3e2c026 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -96,6 +96,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do content = content |> Pleroma.Formatter.markdown_to_html() + |> Pleroma.Formatter.minify("text/html") |> Pleroma.HTML.filter_tags() Map.put(data, "content", content) -- cgit v1.2.3 From 6520599b7deac56780e1496c969cc45ff2e9f5da Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 11 Dec 2020 13:43:40 -0600 Subject: Update Earmark to 1.4.13, use the new compact_output mode --- lib/pleroma/formatter.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 1be12055f..2aa236ca9 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -122,7 +122,7 @@ defmodule Pleroma.Formatter do end def markdown_to_html(text) do - Earmark.as_html!(text) + Earmark.as_html!(text, %Earmark.Options{compact_output: true}) end def html_escape({text, mentions, hashtags}, type) do -- cgit v1.2.3 From ee221277b05d2f682c340c1e1b81fbce4931735a Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 21 Dec 2020 22:54:26 +0300 Subject: Encapsulation of tags / hashtags fetching from objects. --- lib/pleroma/activity/ir/topics.ex | 10 ++--- lib/pleroma/object.ex | 45 +++++++++++++++++++--- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 8 ++-- lib/pleroma/web/activity_pub/transmogrifier.ex | 29 +++++++------- lib/pleroma/web/feed/feed_view.ex | 1 + lib/pleroma/web/mastodon_api/views/status_view.ex | 6 ++- .../web/templates/feed/feed/_activity.atom.eex | 2 +- .../web/templates/feed/feed/_activity.rss.eex | 2 +- .../web/templates/feed/feed/_tag_activity.atom.eex | 2 +- 9 files changed, 71 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex index fe2e8cb5c..2c74ac2bf 100644 --- a/lib/pleroma/activity/ir/topics.ex +++ b/lib/pleroma/activity/ir/topics.ex @@ -48,14 +48,12 @@ defmodule Pleroma.Activity.Ir.Topics do tags end - defp hashtags_to_topics(%{data: %{"tag" => tags}}) do - tags - |> Enum.filter(&is_bitstring(&1)) - |> Enum.map(fn tag -> "hashtag:" <> tag end) + defp hashtags_to_topics(object) do + object + |> Object.hashtags() + |> Enum.map(fn hashtag -> "hashtag:" <> hashtag end) end - defp hashtags_to_topics(_), do: [] - defp remote_topics(%{local: true}), do: [] defp remote_topics(%{actor: actor}) when is_binary(actor), diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 052ad413b..2088c7656 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -47,17 +47,33 @@ defmodule Pleroma.Object do end def create(data) do - Object.change(%Object{}, %{data: data}) + %Object{} + |> Object.change(%{data: data}) |> Repo.insert() end def change(struct, params \\ %{}) do - struct - |> cast(params, [:data]) - |> validate_required([:data]) - |> unique_constraint(:ap_id, name: :objects_unique_apid_index) + changeset = + struct + |> cast(params, [:data]) + |> validate_required([:data]) + |> unique_constraint(:ap_id, name: :objects_unique_apid_index) + + if hashtags_changed?(struct, get_change(changeset, :data)) do + # TODO: modify assoc once it's introduced + changeset + else + changeset + end + end + + defp hashtags_changed?(%Object{} = struct, %{"tag" => _} = data) do + Enum.sort(embedded_hashtags(struct)) != + Enum.sort(object_data_hashtags(data)) end + defp hashtags_changed?(_, _), do: false + def get_by_id(nil), do: nil def get_by_id(id), do: Repo.get(Object, id) @@ -344,4 +360,23 @@ defmodule Pleroma.Object do def self_replies(object, opts \\ []), do: replies(object, Keyword.put(opts, :self_only, true)) + + def tags(%Object{data: %{"tag" => tags}}) when is_list(tags), do: tags + + def tags(_), do: [] + + def hashtags(object), do: embedded_hashtags(object) + + defp embedded_hashtags(%Object{data: data}) do + object_data_hashtags(data) + end + + defp embedded_hashtags(_), do: [] + + defp object_data_hashtags(%{"tag" => tags}) when is_list(tags) do + # Note: AS2 map-type elements are ignored + Enum.filter(tags, &is_bitstring(&1)) + end + + defp object_data_hashtags(_), do: [] end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 6cd91826d..e92091d66 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -74,9 +74,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do object = if MRF.subdomain_match?(media_nsfw, actor_host) do - tags = (child_object["tag"] || []) ++ ["nsfw"] - child_object = Map.put(child_object, "tag", tags) - child_object = Map.put(child_object, "sensitive", true) + child_object = + child_object + |> Map.put("tag", (child_object["tag"] || []) ++ ["nsfw"]) + |> Map.put("sensitive", true) + Map.put(object, "object", child_object) else object diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 565d32433..fd17793d0 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -32,18 +32,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do """ def fix_object(object, options \\ []) do object - |> strip_internal_fields - |> fix_actor - |> fix_url - |> fix_attachments - |> fix_context + |> strip_internal_fields() + |> fix_actor() + |> fix_url() + |> fix_attachments() + |> fix_context() |> fix_in_reply_to(options) - |> fix_emoji - |> fix_tag - |> set_sensitive - |> fix_content_map - |> fix_addressing - |> fix_summary + |> fix_emoji() + |> fix_tag() + |> set_sensitive() + |> fix_content_map() + |> fix_addressing() + |> fix_summary() |> fix_type(options) end @@ -315,10 +315,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do tags = tag |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end) - |> Enum.map(fn %{"name" => name} -> - name - |> String.slice(1..-1) - |> String.downcase() + |> Enum.map(fn + %{"name" => "#" <> hashtag} -> String.downcase(hashtag) + %{"name" => hashtag} -> String.downcase(hashtag) end) Map.put(object, "tag", tag ++ tags) diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index 30e0a2a55..1155c6a39 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -32,6 +32,7 @@ defmodule Pleroma.Web.Feed.FeedView do %{ activity: activity, + object: object, data: Map.get(object, :data), actor: actor } diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 2301e21cf..bd08aa203 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -201,8 +201,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do like_count = object.data["like_count"] || 0 announcement_count = object.data["announcement_count"] || 0 - tags = object.data["tag"] || [] - sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw") + hashtags = Object.hashtags(object) + sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw") + + tags = Object.tags(object) tag_mentions = tags diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index 3fd150c4e..6688830ba 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -22,7 +22,7 @@ <% end %> - <%= for tag <- @data["tag"] || [] do %> + <%= for tag <- Pleroma.Object.hashtags(@object) do %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index 42960de7d..fc6d74b42 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -21,7 +21,7 @@ <%= @data["external_url"] %> <% end %> - <%= for tag <- @data["tag"] || [] do %> + <%= for tag <- Pleroma.Object.hashtags(@object) do %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index cf5874a91..c2de28fe4 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -41,7 +41,7 @@ <% end %> <% end %> - <%= for tag <- @data["tag"] || [] do %> + <%= for tag <- Pleroma.Object.hashtags(@object) do %> <% end %> -- cgit v1.2.3 From e369b1306b2f8b9732c21333b9957f7e4e408f90 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 22 Dec 2020 22:04:33 +0300 Subject: Added Hashtag entity and objects-hashtags association with auto-sync with `data.tag` on Object update. --- lib/pleroma/hashtag.ex | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ lib/pleroma/object.ex | 37 +++++++++++++++++++++++--------- 2 files changed, 85 insertions(+), 10 deletions(-) create mode 100644 lib/pleroma/hashtag.ex (limited to 'lib') diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex new file mode 100644 index 000000000..b05927563 --- /dev/null +++ b/lib/pleroma/hashtag.ex @@ -0,0 +1,58 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Hashtag do + use Ecto.Schema + + import Ecto.Changeset + + alias Pleroma.Hashtag + alias Pleroma.Repo + + @derive {Jason.Encoder, only: [:data]} + + schema "hashtags" do + field(:name, :string) + field(:data, :map, default: %{}) + + many_to_many(:objects, Pleroma.Object, join_through: "hashtags_objects", on_replace: :delete) + + timestamps() + end + + def get_by_name(name) do + Repo.get_by(Hashtag, name: name) + end + + def get_or_create_by_name(name) when is_bitstring(name) do + with %Hashtag{} = hashtag <- get_by_name(name) do + {:ok, hashtag} + else + _ -> + %Hashtag{} + |> changeset(%{name: name}) + |> Repo.insert() + end + end + + def get_or_create_by_names(names) when is_list(names) do + Enum.reduce_while(names, {:ok, []}, fn name, {:ok, list} -> + case get_or_create_by_name(name) do + {:ok, %Hashtag{} = hashtag} -> + {:cont, {:ok, list ++ [hashtag]}} + + error -> + {:halt, error} + end + end) + end + + def changeset(%Hashtag{} = struct, params) do + struct + |> cast(params, [:name, :data]) + |> update_change(:name, &String.downcase/1) + |> validate_required([:name]) + |> unique_constraint(:name) + end +end diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 2088c7656..357a3b504 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Object do alias Pleroma.Activity alias Pleroma.Config + alias Pleroma.Hashtag alias Pleroma.Object alias Pleroma.Object.Fetcher alias Pleroma.ObjectTombstone @@ -26,6 +27,8 @@ defmodule Pleroma.Object do schema "objects" do field(:data, :map) + many_to_many(:hashtags, Hashtag, join_through: "hashtags_objects", on_replace: :delete) + timestamps() end @@ -53,17 +56,31 @@ defmodule Pleroma.Object do end def change(struct, params \\ %{}) do - changeset = - struct - |> cast(params, [:data]) - |> validate_required([:data]) - |> unique_constraint(:ap_id, name: :objects_unique_apid_index) - - if hashtags_changed?(struct, get_change(changeset, :data)) do - # TODO: modify assoc once it's introduced - changeset + struct + |> cast(params, [:data]) + |> validate_required([:data]) + |> unique_constraint(:ap_id, name: :objects_unique_apid_index) + |> maybe_handle_hashtags_change(struct) + end + + defp maybe_handle_hashtags_change(changeset, struct) do + with data_hashtags_change = get_change(changeset, :data), + true <- hashtags_changed?(struct, data_hashtags_change), + {:ok, hashtag_records} <- + data_hashtags_change + |> object_data_hashtags() + |> Hashtag.get_or_create_by_names() do + put_assoc(changeset, :hashtags, hashtag_records) else - changeset + false -> + changeset + + {:error, hashtag_changeset} -> + failed_hashtag = get_field(hashtag_changeset, :name) + + validate_change(changeset, :data, fn _, _ -> + [data: "error referencing hashtag: #{failed_hashtag}"] + end) end end -- cgit v1.2.3 From cbb19d0e1882f5ce641f30b51d7156336f81aba9 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 26 Dec 2020 22:20:55 +0300 Subject: [#3213] Hashtag-filtering functions in ActivityPub. Mix task for migrating hashtags to `hashtags` table. --- lib/mix/tasks/pleroma/database.ex | 64 ++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 167 ++++++++++++++++++++------- 2 files changed, 190 insertions(+), 41 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 22151ce08..093c7dd30 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -4,14 +4,18 @@ defmodule Mix.Tasks.Pleroma.Database do alias Pleroma.Conversation + alias Pleroma.Hashtag alias Pleroma.Maintenance alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User + require Logger require Pleroma.Constants + import Ecto.Query import Mix.Pleroma + use Mix.Task @shortdoc "A collection of database related tasks" @@ -128,6 +132,66 @@ defmodule Mix.Tasks.Pleroma.Database do |> Stream.run() end + def run(["transfer_hashtags"]) do + import Ecto.Query + + start_pleroma() + + from( + object in Object, + left_join: hashtag in assoc(object, :hashtags), + where: is_nil(hashtag.id), + where: fragment("(?)->>'tag' != '[]'", object.data), + select: %{ + id: object.id, + inserted_at: object.inserted_at, + tag: fragment("(?)->>'tag'", object.data) + }, + order_by: [desc: object.id] + ) + |> Pleroma.Repo.chunk_stream(100, :batches) + |> Stream.each(fn objects -> + chunk_start = List.first(objects) + chunk_end = List.last(objects) + + Logger.info( + "transfer_hashtags: " <> + "#{chunk_start.id} (#{chunk_start.inserted_at}) -- " <> + "#{chunk_end.id} (#{chunk_end.inserted_at})" + ) + + Enum.map( + objects, + fn object -> + hashtags = + object.tag + |> Jason.decode!() + |> Enum.filter(&is_bitstring(&1)) + + with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do + Repo.transaction(fn -> + for hashtag_record <- hashtag_records do + with {:error, _} <- + Ecto.Adapters.SQL.query( + Repo, + "insert into hashtags_objects(hashtag_id, object_id) values " <> + "(#{hashtag_record.id}, #{object.id});" + ) do + Logger.warn( + "ERROR: could not link object #{object.id} and hashtag #{hashtag_record.id}" + ) + end + end + end) + else + e -> Logger.warn("ERROR: could not process object #{object.id}: #{inspect(e)}") + end + end + ) + end) + |> Stream.run() + end + def run(["vacuum", args]) do start_pleroma() diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1c91bc074..2e25412c6 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -660,33 +660,41 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_since(query, _), do: query defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do - raise "Can't use the child object without preloading!" + raise_on_missing_preload() end - defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do + defp restrict_tag_reject(query, %{tag_reject: tag_reject}) when is_list(tag_reject) do from( [_activity, object] in query, where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject) ) end + defp restrict_tag_reject(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do + restrict_tag_reject(query, %{tag_reject: [tag_reject]}) + end + defp restrict_tag_reject(query, _), do: query defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do - raise "Can't use the child object without preloading!" + raise_on_missing_preload() end - defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do + defp restrict_tag_all(query, %{tag_all: tag_all}) when is_list(tag_all) do from( [_activity, object] in query, where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) ) end + defp restrict_tag_all(query, %{tag_all: tag}) when is_binary(tag) do + restrict_tag(query, %{tag: tag}) + end + defp restrict_tag_all(query, _), do: query defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do - raise "Can't use the child object without preloading!" + raise_on_missing_preload() end defp restrict_tag(query, %{tag: tag}) when is_list(tag) do @@ -697,13 +705,79 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do + restrict_tag(query, %{tag: [tag]}) + end + + defp restrict_tag(query, _), do: query + + defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do + raise_on_missing_preload() + end + + defp restrict_hashtag_reject_any(query, %{tag_reject: tags_reject}) when is_list(tags_reject) do + if has_named_binding?(query, :thread_mute) do + from( + [activity, object, thread_mute] in query, + group_by: [activity.id, object.id, thread_mute.id] + ) + else + from( + [activity, object] in query, + group_by: [activity.id, object.id] + ) + end + |> join(:left, [_activity, object], hashtag in assoc(object, :hashtags), as: :hashtag) + |> having( + [hashtag: hashtag], + fragment("not(array_agg(?) && (?))", hashtag.name, ^tags_reject) + ) + end + + defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do + restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]}) + end + + defp restrict_hashtag_reject_any(query, _), do: query + + defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do + raise_on_missing_preload() + end + + defp restrict_hashtag_all(query, %{tag_all: tags}) when is_list(tags) do + Enum.reduce( + tags, + query, + fn tag, acc -> restrict_hashtag_any(acc, %{tag: tag}) end + ) + end + + defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do + restrict_hashtag_any(query, %{tag: tag}) + end + + defp restrict_hashtag_all(query, _), do: query + + defp restrict_hashtag_any(_query, %{tag: _tag, skip_preload: true}) do + raise_on_missing_preload() + end + + defp restrict_hashtag_any(query, %{tag: tags}) when is_list(tags) do from( [_activity, object] in query, - where: fragment("(?)->'tag' \\? (?)", object.data, ^tag) + join: hashtag in assoc(object, :hashtags), + where: hashtag.name in ^tags ) end - defp restrict_tag(query, _), do: query + defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do + restrict_hashtag_any(query, %{tag: [tag]}) + end + + defp restrict_hashtag_any(query, _), do: query + + defp raise_on_missing_preload do + raise "Can't use the child object without preloading!" + end defp restrict_recipients(query, [], _user), do: query @@ -1088,40 +1162,51 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do skip_thread_containment: Config.get([:instance, :skip_thread_containment]) } - Activity - |> maybe_preload_objects(opts) - |> maybe_preload_bookmarks(opts) - |> maybe_preload_report_notes(opts) - |> maybe_set_thread_muted_field(opts) - |> maybe_order(opts) - |> restrict_recipients(recipients, opts[:user]) - |> restrict_replies(opts) - |> restrict_tag(opts) - |> restrict_tag_reject(opts) - |> restrict_tag_all(opts) - |> restrict_since(opts) - |> restrict_local(opts) - |> restrict_actor(opts) - |> restrict_type(opts) - |> restrict_state(opts) - |> restrict_favorited_by(opts) - |> restrict_blocked(restrict_blocked_opts) - |> restrict_muted(restrict_muted_opts) - |> restrict_filtered(opts) - |> restrict_media(opts) - |> restrict_visibility(opts) - |> restrict_thread_visibility(opts, config) - |> restrict_reblogs(opts) - |> restrict_pinned(opts) - |> restrict_muted_reblogs(restrict_muted_reblogs_opts) - |> restrict_instance(opts) - |> restrict_announce_object_actor(opts) - |> restrict_filtered(opts) - |> Activity.restrict_deactivated_users() - |> exclude_poll_votes(opts) - |> exclude_chat_messages(opts) - |> exclude_invisible_actors(opts) - |> exclude_visibility(opts) + query = + Activity + |> distinct([a], true) + |> maybe_preload_objects(opts) + |> maybe_preload_bookmarks(opts) + |> maybe_preload_report_notes(opts) + |> maybe_set_thread_muted_field(opts) + |> maybe_order(opts) + |> restrict_recipients(recipients, opts[:user]) + |> restrict_replies(opts) + |> restrict_since(opts) + |> restrict_local(opts) + |> restrict_actor(opts) + |> restrict_type(opts) + |> restrict_state(opts) + |> restrict_favorited_by(opts) + |> restrict_blocked(restrict_blocked_opts) + |> restrict_muted(restrict_muted_opts) + |> restrict_filtered(opts) + |> restrict_media(opts) + |> restrict_visibility(opts) + |> restrict_thread_visibility(opts, config) + |> restrict_reblogs(opts) + |> restrict_pinned(opts) + |> restrict_muted_reblogs(restrict_muted_reblogs_opts) + |> restrict_instance(opts) + |> restrict_announce_object_actor(opts) + |> restrict_filtered(opts) + |> Activity.restrict_deactivated_users() + |> exclude_poll_votes(opts) + |> exclude_chat_messages(opts) + |> exclude_invisible_actors(opts) + |> exclude_visibility(opts) + + if Config.get([:instance, :improved_hashtag_timeline]) do + query + |> restrict_hashtag_any(opts) + |> restrict_hashtag_all(opts) + |> restrict_hashtag_reject_any(opts) + else + query + |> restrict_tag(opts) + |> restrict_tag_reject(opts) + |> restrict_tag_all(opts) + end end def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do -- cgit v1.2.3 From 14fae94c0e4b04123c7af148260d0a4a51042570 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 28 Dec 2020 00:08:09 +0300 Subject: [#3213] Made Object.hashtags/1 work with :hashtags assoc. Adjusted tests. --- lib/pleroma/config.ex | 2 ++ lib/pleroma/object.ex | 14 +++++++++++++- lib/pleroma/web/activity_pub/activity_pub.ex | 12 ++++++------ 3 files changed, 21 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index 86d4f6b72..ee0167f4e 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -96,6 +96,8 @@ defmodule Pleroma.Config do end end + def object_embedded_hashtags?, do: !get([:instance, :improved_hashtag_timeline]) + def oauth_consumer_strategies, do: get([:auth, :oauth_consumer_strategies], []) def oauth_consumer_enabled?, do: oauth_consumer_strategies() != [] diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 1d756bcd1..08114d4f2 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -384,7 +384,19 @@ defmodule Pleroma.Object do def tags(_), do: [] - def hashtags(object), do: embedded_hashtags(object) + def hashtags(%Object{} = object) do + cond do + Config.object_embedded_hashtags?() -> + embedded_hashtags(object) + + object.id == "pleroma:fake_object_id" -> + [] + + true -> + hashtag_records = Repo.preload(object, :hashtags).hashtags + Enum.map(hashtag_records, & &1.name) + end + end defp embedded_hashtags(%Object{data: data}) do object_data_hashtags(data) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 54d1a2350..626cad336 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1199,16 +1199,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> exclude_invisible_actors(opts) |> exclude_visibility(opts) - if Config.get([:instance, :improved_hashtag_timeline]) do - query - |> restrict_hashtag_any(opts) - |> restrict_hashtag_all(opts) - |> restrict_hashtag_reject_any(opts) - else + if Config.object_embedded_hashtags?() do query |> restrict_tag(opts) |> restrict_tag_reject(opts) |> restrict_tag_all(opts) + else + query + |> restrict_hashtag_any(opts) + |> restrict_hashtag_all(opts) + |> restrict_hashtag_reject_any(opts) end end -- cgit v1.2.3 From a25c1e8ec0b6f4ef2e9f68c4ad5e48e18f5f01a7 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 30 Dec 2020 14:35:19 +0300 Subject: [#3213] Improved `database.transfer_hashtags` mix task: proper rollback, speedup. --- lib/mix/tasks/pleroma/database.ex | 46 +++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 093c7dd30..d44bd3478 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -137,6 +137,8 @@ defmodule Mix.Tasks.Pleroma.Database do start_pleroma() + Logger.info("Starting transferring object embedded hashtags to `hashtags` table...") + from( object in Object, left_join: hashtag in assoc(object, :hashtags), @@ -144,21 +146,12 @@ defmodule Mix.Tasks.Pleroma.Database do where: fragment("(?)->>'tag' != '[]'", object.data), select: %{ id: object.id, - inserted_at: object.inserted_at, tag: fragment("(?)->>'tag'", object.data) - }, - order_by: [desc: object.id] + } ) |> Pleroma.Repo.chunk_stream(100, :batches) |> Stream.each(fn objects -> - chunk_start = List.first(objects) - chunk_end = List.last(objects) - - Logger.info( - "transfer_hashtags: " <> - "#{chunk_start.id} (#{chunk_start.inserted_at}) -- " <> - "#{chunk_end.id} (#{chunk_end.inserted_at})" - ) + Logger.info("Processing #{length(objects)} objects...") Enum.map( objects, @@ -168,28 +161,39 @@ defmodule Mix.Tasks.Pleroma.Database do |> Jason.decode!() |> Enum.filter(&is_bitstring(&1)) - with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do - Repo.transaction(fn -> + Repo.transaction(fn -> + with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do for hashtag_record <- hashtag_records do - with {:error, _} <- + with {:ok, _} <- Ecto.Adapters.SQL.query( Repo, "insert into hashtags_objects(hashtag_id, object_id) values " <> "(#{hashtag_record.id}, #{object.id});" ) do - Logger.warn( - "ERROR: could not link object #{object.id} and hashtag #{hashtag_record.id}" - ) + :noop + else + {:error, e} -> + error = + "ERROR: could not link object #{object.id} and hashtag " <> + "#{hashtag_record.id}: #{inspect(e)}" + + Logger.error(error) + Repo.rollback(error) end end - end) - else - e -> Logger.warn("ERROR: could not process object #{object.id}: #{inspect(e)}") - end + else + e -> + error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}" + Logger.error(error) + Repo.rollback(error) + end + end) end ) end) |> Stream.run() + + Logger.info("Done transferring hashtags. Please check logs to ensure no errors.") end def run(["vacuum", args]) do -- cgit v1.2.3 From e0b5edb6d5a423bfd247e0774d2f5bc642b2fb80 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 30 Dec 2020 14:42:35 +0300 Subject: [#3213] Fixed Object.object_data_hashtags/1 to process only AS2 elements of `data.tag` (basing on #2984). --- lib/pleroma/object.ex | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 08114d4f2..dad572f2b 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -405,8 +405,16 @@ defmodule Pleroma.Object do defp embedded_hashtags(_), do: [] defp object_data_hashtags(%{"tag" => tags}) when is_list(tags) do - # Note: AS2 map-type elements are ignored - Enum.filter(tags, &is_bitstring(&1)) + # Note: Old format with copy of hashtags as strings is ignored, using AS2 + tags + |> Enum.filter(fn + %{"type" => "Hashtag"} = data -> Map.has_key?(data, "name") + _ -> false + end) + |> Enum.map(fn + %{"name" => "#" <> hashtag} -> String.downcase(hashtag) + %{"name" => hashtag} -> String.downcase(hashtag) + end) end defp object_data_hashtags(_), do: [] -- cgit v1.2.3 From 8d1a0c1afd46f8683e9022523cecffb9b60c9f8c Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 30 Dec 2020 15:22:49 +0300 Subject: [#3213] Made Object.object_data_hashtags/1 handle both AS2 and plain text hashtags. --- lib/pleroma/object.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index dad572f2b..7e79e15ee 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -405,16 +405,18 @@ defmodule Pleroma.Object do defp embedded_hashtags(_), do: [] defp object_data_hashtags(%{"tag" => tags}) when is_list(tags) do - # Note: Old format with copy of hashtags as strings is ignored, using AS2 tags |> Enum.filter(fn %{"type" => "Hashtag"} = data -> Map.has_key?(data, "name") + plain_text when is_bitstring(plain_text) -> true _ -> false end) |> Enum.map(fn %{"name" => "#" <> hashtag} -> String.downcase(hashtag) %{"name" => hashtag} -> String.downcase(hashtag) + hashtag when is_bitstring(hashtag) -> String.downcase(hashtag) end) + |> Enum.uniq() end defp object_data_hashtags(_), do: [] -- cgit v1.2.3 From 367f0c31c3c15f75aed1d3ba66914e4197c19596 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 31 Dec 2020 09:36:26 +0300 Subject: [#3213] Added query options support for Repo.chunk_stream/4. Used infinite timeout in transfer_hashtags select query. --- lib/mix/tasks/pleroma/database.ex | 11 +++++------ lib/pleroma/repo.ex | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index d44bd3478..f903cf75b 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -149,9 +149,9 @@ defmodule Mix.Tasks.Pleroma.Database do tag: fragment("(?)->>'tag'", object.data) } ) - |> Pleroma.Repo.chunk_stream(100, :batches) + |> Repo.chunk_stream(100, :batches, timeout: :infinity) |> Stream.each(fn objects -> - Logger.info("Processing #{length(objects)} objects...") + Logger.info("Processing #{length(objects)} objects starting from id #{hd(objects).id}...") Enum.map( objects, @@ -165,10 +165,9 @@ defmodule Mix.Tasks.Pleroma.Database do with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do for hashtag_record <- hashtag_records do with {:ok, _} <- - Ecto.Adapters.SQL.query( - Repo, - "insert into hashtags_objects(hashtag_id, object_id) values " <> - "(#{hashtag_record.id}, #{object.id});" + Repo.query( + "insert into hashtags_objects(hashtag_id, object_id) values ($1, $2);", + [hashtag_record.id, object.id] ) do :noop else diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index 4524bd5e2..78711e6ac 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -63,8 +63,8 @@ defmodule Pleroma.Repo do iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches) """ @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t() - def chunk_stream(query, chunk_size, returns_as \\ :one) do - # We don't actually need start and end funcitons of resource streaming, + def chunk_stream(query, chunk_size, returns_as \\ :one, query_options \\ []) do + # We don't actually need start and end functions of resource streaming, # but it seems to be the only way to not fetch records one-by-one and # have individual records be the elements of the stream, instead of # lists of records @@ -76,7 +76,7 @@ defmodule Pleroma.Repo do |> order_by(asc: :id) |> where([r], r.id > ^last_id) |> limit(^chunk_size) - |> all() + |> all(query_options) |> case do [] -> {:halt, last_id} -- cgit v1.2.3 From 303055456f19152821ec5ec1df88d60c03f60905 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 31 Dec 2020 12:45:23 +0300 Subject: Alternative implementation of hashtag-filtering queries in ActivityPub. Fixed GROUP BY clause for aggregation on hashtags. --- lib/pleroma/activity.ex | 2 + lib/pleroma/web/activity_pub/activity_pub.ex | 120 ++++++++++++++++++++++----- 2 files changed, 100 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 9d970a808..df216e4de 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -113,6 +113,7 @@ defmodule Pleroma.Activity do from([a] in query, left_join: b in Bookmark, on: b.user_id == ^user.id and b.activity_id == a.id, + as: :bookmark, preload: [bookmark: b] ) end @@ -123,6 +124,7 @@ defmodule Pleroma.Activity do from([a] in query, left_join: r in ReportNote, on: a.id == r.activity_id, + as: :report_note, preload: [report_notes: r] ) end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 626cad336..339843330 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -713,22 +713,92 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_tag(query, _), do: query + defp restrict_hashtag(query, opts) do + [tag_any, tag_all, tag_reject] = + [:tag, :tag_all, :tag_reject] + |> Enum.map(&opts[&1]) + |> Enum.map(&List.wrap(&1)) + + has_conditions = Enum.any?([tag_any, tag_all, tag_reject], &Enum.any?(&1)) + + cond do + !has_conditions -> + query + + opts[:skip_preload] -> + raise_on_missing_preload() + + true -> + query + |> group_by_all_bindings() + |> join(:left, [_activity, object], hashtag in assoc(object, :hashtags), as: :hashtag) + |> maybe_restrict_hashtag_any(tag_any) + |> maybe_restrict_hashtag_all(tag_all) + |> maybe_restrict_hashtag_reject_any(tag_reject) + end + end + + # Groups by all bindings to allow aggregation on hashtags + defp group_by_all_bindings(query) do + # Expecting named bindings: :object, :bookmark, :thread_mute, :report_note + cond do + Enum.count(query.aliases) == 4 -> + from([a, o, b3, b4, b5] in query, group_by: [a.id, o.id, b3.id, b4.id, b5.id]) + + Enum.count(query.aliases) == 3 -> + from([a, o, b3, b4] in query, group_by: [a.id, o.id, b3.id, b4.id]) + + Enum.count(query.aliases) == 2 -> + from([a, o, b3] in query, group_by: [a.id, o.id, b3.id]) + + true -> + from([a, o] in query, group_by: [a.id, o.id]) + end + end + + defp maybe_restrict_hashtag_any(query, []) do + query + end + + defp maybe_restrict_hashtag_any(query, tags) do + having( + query, + [hashtag: hashtag], + fragment("array_agg(?) && (?)", hashtag.name, ^tags) + ) + end + + defp maybe_restrict_hashtag_all(query, []) do + query + end + + defp maybe_restrict_hashtag_all(query, tags) do + having( + query, + [hashtag: hashtag], + fragment("array_agg(?) @> (?)", hashtag.name, ^tags) + ) + end + + defp maybe_restrict_hashtag_reject_any(query, []) do + query + end + + defp maybe_restrict_hashtag_reject_any(query, tags) do + having( + query, + [hashtag: hashtag], + fragment("not(array_agg(?) && (?))", hashtag.name, ^tags) + ) + end + defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do raise_on_missing_preload() end defp restrict_hashtag_reject_any(query, %{tag_reject: tags_reject}) when is_list(tags_reject) do - if has_named_binding?(query, :thread_mute) do - from( - [activity, object, thread_mute] in query, - group_by: [activity.id, object.id, thread_mute.id] - ) - else - from( - [activity, object] in query, - group_by: [activity.id, object.id] - ) - end + query + |> group_by_all_bindings() |> join(:left, [_activity, object], hashtag in assoc(object, :hashtags), as: :hashtag) |> having( [hashtag: hashtag], @@ -1167,7 +1237,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do query = Activity - |> distinct([a], true) |> maybe_preload_objects(opts) |> maybe_preload_bookmarks(opts) |> maybe_preload_report_notes(opts) @@ -1199,16 +1268,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> exclude_invisible_actors(opts) |> exclude_visibility(opts) - if Config.object_embedded_hashtags?() do - query - |> restrict_tag(opts) - |> restrict_tag_reject(opts) - |> restrict_tag_all(opts) - else - query - |> restrict_hashtag_any(opts) - |> restrict_hashtag_all(opts) - |> restrict_hashtag_reject_any(opts) + cond do + Config.object_embedded_hashtags?() -> + query + |> restrict_tag(opts) + |> restrict_tag_reject(opts) + |> restrict_tag_all(opts) + + # TODO: benchmark (initial approach preferring non-aggregate ops when possible) + Config.get([:instance, :improved_hashtag_timeline]) == :join -> + query + |> distinct([activity], true) + |> restrict_hashtag_any(opts) + |> restrict_hashtag_all(opts) + |> restrict_hashtag_reject_any(opts) + + true -> + restrict_hashtag(query, opts) end end -- cgit v1.2.3 From 0d521022fe6157ce9a346c6915ce38292e653bb3 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 7 Jan 2021 12:20:29 +0300 Subject: [#3213] Removed PK from hashtags_objects table. Improved hashtags_transfer mix task (logging of failed ids). --- lib/mix/tasks/pleroma/database.ex | 29 +++++++++++++++++------------ lib/pleroma/object.ex | 4 ++-- 2 files changed, 19 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index f903cf75b..918752dc2 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -139,6 +139,7 @@ defmodule Mix.Tasks.Pleroma.Database do Logger.info("Starting transferring object embedded hashtags to `hashtags` table...") + # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out) from( object in Object, left_join: hashtag in assoc(object, :hashtags), @@ -153,13 +154,10 @@ defmodule Mix.Tasks.Pleroma.Database do |> Stream.each(fn objects -> Logger.info("Processing #{length(objects)} objects starting from id #{hd(objects).id}...") - Enum.map( - objects, - fn object -> - hashtags = - object.tag - |> Jason.decode!() - |> Enum.filter(&is_bitstring(&1)) + failed_ids = + objects + |> Enum.map(fn object -> + hashtags = Object.object_data_hashtags(%{"tag" => Jason.decode!(object.tag)}) Repo.transaction(fn -> with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do @@ -169,7 +167,7 @@ defmodule Mix.Tasks.Pleroma.Database do "insert into hashtags_objects(hashtag_id, object_id) values ($1, $2);", [hashtag_record.id, object.id] ) do - :noop + nil else {:error, e} -> error = @@ -177,18 +175,25 @@ defmodule Mix.Tasks.Pleroma.Database do "#{hashtag_record.id}: #{inspect(e)}" Logger.error(error) - Repo.rollback(error) + Repo.rollback(object.id) end end + + object.id else e -> error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}" Logger.error(error) - Repo.rollback(error) + Repo.rollback(object.id) end end) - end - ) + end) + |> Enum.filter(&(elem(&1, 0) == :error)) + |> Enum.map(&elem(&1, 1)) + + if Enum.any?(failed_ids) do + Logger.error("ERROR: transfer_hashtags iteration failed for ids: #{inspect(failed_ids)}") + end end) |> Stream.run() diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 7e79e15ee..61f2ffa19 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -404,7 +404,7 @@ defmodule Pleroma.Object do defp embedded_hashtags(_), do: [] - defp object_data_hashtags(%{"tag" => tags}) when is_list(tags) do + def object_data_hashtags(%{"tag" => tags}) when is_list(tags) do tags |> Enum.filter(fn %{"type" => "Hashtag"} = data -> Map.has_key?(data, "name") @@ -419,5 +419,5 @@ defmodule Pleroma.Object do |> Enum.uniq() end - defp object_data_hashtags(_), do: [] + def object_data_hashtags(_), do: [] end -- cgit v1.2.3 From 8c972de0457199098c5f3378313d08a9dd2d64ce Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 10 Jan 2021 11:44:39 +0300 Subject: [#3213] transfer_hashtags mix task refactoring. --- lib/mix/tasks/pleroma/database.ex | 127 ++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 68 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 918752dc2..e9686fc1b 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -132,74 +132,6 @@ defmodule Mix.Tasks.Pleroma.Database do |> Stream.run() end - def run(["transfer_hashtags"]) do - import Ecto.Query - - start_pleroma() - - Logger.info("Starting transferring object embedded hashtags to `hashtags` table...") - - # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out) - from( - object in Object, - left_join: hashtag in assoc(object, :hashtags), - where: is_nil(hashtag.id), - where: fragment("(?)->>'tag' != '[]'", object.data), - select: %{ - id: object.id, - tag: fragment("(?)->>'tag'", object.data) - } - ) - |> Repo.chunk_stream(100, :batches, timeout: :infinity) - |> Stream.each(fn objects -> - Logger.info("Processing #{length(objects)} objects starting from id #{hd(objects).id}...") - - failed_ids = - objects - |> Enum.map(fn object -> - hashtags = Object.object_data_hashtags(%{"tag" => Jason.decode!(object.tag)}) - - Repo.transaction(fn -> - with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do - for hashtag_record <- hashtag_records do - with {:ok, _} <- - Repo.query( - "insert into hashtags_objects(hashtag_id, object_id) values ($1, $2);", - [hashtag_record.id, object.id] - ) do - nil - else - {:error, e} -> - error = - "ERROR: could not link object #{object.id} and hashtag " <> - "#{hashtag_record.id}: #{inspect(e)}" - - Logger.error(error) - Repo.rollback(object.id) - end - end - - object.id - else - e -> - error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}" - Logger.error(error) - Repo.rollback(object.id) - end - end) - end) - |> Enum.filter(&(elem(&1, 0) == :error)) - |> Enum.map(&elem(&1, 1)) - - if Enum.any?(failed_ids) do - Logger.error("ERROR: transfer_hashtags iteration failed for ids: #{inspect(failed_ids)}") - end - end) - |> Stream.run() - - Logger.info("Done transferring hashtags. Please check logs to ensure no errors.") - end - def run(["vacuum", args]) do start_pleroma() @@ -239,4 +171,63 @@ defmodule Mix.Tasks.Pleroma.Database do end) |> Stream.run() end + + def run(["transfer_hashtags"]) do + import Ecto.Query + + start_pleroma() + + Logger.info("Starting transferring object embedded hashtags to `hashtags` table...") + + # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out) + from( + object in Object, + left_join: hashtag in assoc(object, :hashtags), + where: is_nil(hashtag.id), + where: + fragment("(?)->'tag' IS NOT NULL AND (?)->'tag' != '[]'::jsonb", object.data, object.data), + select: %{ + id: object.id, + tag: fragment("(?)->'tag'", object.data) + } + ) + |> Repo.chunk_stream(100, :one, timeout: :infinity) + |> Stream.each(&transfer_object_hashtags(&1)) + |> Stream.run() + + Logger.info("Done transferring hashtags. Please check logs to ensure no errors.") + end + + defp transfer_object_hashtags(object) do + hashtags = Object.object_data_hashtags(%{"tag" => object.tag}) + + Repo.transaction(fn -> + with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do + for hashtag_record <- hashtag_records do + with {:ok, _} <- + Repo.query( + "insert into hashtags_objects(hashtag_id, object_id) values ($1, $2);", + [hashtag_record.id, object.id] + ) do + nil + else + {:error, e} -> + error = + "ERROR: could not link object #{object.id} and hashtag " <> + "#{hashtag_record.id}: #{inspect(e)}" + + Logger.error(error) + Repo.rollback(object.id) + end + end + + object.id + else + e -> + error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}" + Logger.error(error) + Repo.rollback(object.id) + end + end) + end end -- cgit v1.2.3 From 3e4d84729a4ca8d9779d439a9aa2c8c23b3acd1d Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 13 Jan 2021 22:07:38 +0300 Subject: [#3213] Prototype of data migrations functionality / HashtagsTableMigrator. --- lib/mix/tasks/pleroma/database.ex | 60 ------- lib/pleroma/application.ex | 3 +- lib/pleroma/config.ex | 4 +- lib/pleroma/data_migration.ex | 46 +++++ lib/pleroma/delivery.ex | 1 - lib/pleroma/ecto_enums.ex | 8 + lib/pleroma/migrators/hashtags_table_migrator.ex | 211 +++++++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 8 files changed, 271 insertions(+), 64 deletions(-) create mode 100644 lib/pleroma/data_migration.ex create mode 100644 lib/pleroma/migrators/hashtags_table_migrator.ex (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index e9686fc1b..08ede9eef 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -4,7 +4,6 @@ defmodule Mix.Tasks.Pleroma.Database do alias Pleroma.Conversation - alias Pleroma.Hashtag alias Pleroma.Maintenance alias Pleroma.Object alias Pleroma.Repo @@ -171,63 +170,4 @@ defmodule Mix.Tasks.Pleroma.Database do end) |> Stream.run() end - - def run(["transfer_hashtags"]) do - import Ecto.Query - - start_pleroma() - - Logger.info("Starting transferring object embedded hashtags to `hashtags` table...") - - # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out) - from( - object in Object, - left_join: hashtag in assoc(object, :hashtags), - where: is_nil(hashtag.id), - where: - fragment("(?)->'tag' IS NOT NULL AND (?)->'tag' != '[]'::jsonb", object.data, object.data), - select: %{ - id: object.id, - tag: fragment("(?)->'tag'", object.data) - } - ) - |> Repo.chunk_stream(100, :one, timeout: :infinity) - |> Stream.each(&transfer_object_hashtags(&1)) - |> Stream.run() - - Logger.info("Done transferring hashtags. Please check logs to ensure no errors.") - end - - defp transfer_object_hashtags(object) do - hashtags = Object.object_data_hashtags(%{"tag" => object.tag}) - - Repo.transaction(fn -> - with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do - for hashtag_record <- hashtag_records do - with {:ok, _} <- - Repo.query( - "insert into hashtags_objects(hashtag_id, object_id) values ($1, $2);", - [hashtag_record.id, object.id] - ) do - nil - else - {:error, e} -> - error = - "ERROR: could not link object #{object.id} and hashtag " <> - "#{hashtag_record.id}: #{inspect(e)}" - - Logger.error(error) - Repo.rollback(object.id) - end - end - - object.id - else - e -> - error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}" - Logger.error(error) - Repo.rollback(object.id) - end - end) - end end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index bd568d858..962529dfd 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -104,7 +104,8 @@ defmodule Pleroma.Application do chat_child(chat_enabled?()) ++ [ Pleroma.Web.Endpoint, - Pleroma.Gopher.Server + Pleroma.Gopher.Server, + Pleroma.Migrators.HashtagsTableMigrator ] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index ee0167f4e..dbfb114d6 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -96,7 +96,9 @@ defmodule Pleroma.Config do end end - def object_embedded_hashtags?, do: !get([:instance, :improved_hashtag_timeline]) + def improved_hashtag_timeline_path, do: [:instance, :improved_hashtag_timeline] + def improved_hashtag_timeline, do: get(improved_hashtag_timeline_path()) + def object_embedded_hashtags?, do: !improved_hashtag_timeline() def oauth_consumer_strategies, do: get([:auth, :oauth_consumer_strategies], []) diff --git a/lib/pleroma/data_migration.ex b/lib/pleroma/data_migration.ex new file mode 100644 index 000000000..64fa155ff --- /dev/null +++ b/lib/pleroma/data_migration.ex @@ -0,0 +1,46 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.DataMigration do + use Ecto.Schema + + alias Pleroma.DataMigration + alias Pleroma.DataMigration.State + alias Pleroma.Repo + + import Ecto.Changeset + + schema "data_migrations" do + field(:name, :string) + field(:state, State, default: :pending) + field(:feature_lock, :boolean, default: false) + field(:params, :map, default: %{}) + field(:data, :map, default: %{}) + + timestamps() + end + + def changeset(data_migration, params \\ %{}) do + data_migration + |> cast(params, [:name, :state, :feature_lock, :params, :data]) + |> validate_required([:name]) + |> unique_constraint(:name) + end + + def update(data_migration, params \\ %{}) do + data_migration + |> changeset(params) + |> Repo.update() + end + + def update_state(data_migration, new_state) do + update(data_migration, %{state: new_state}) + end + + def get_by_name(name) do + Repo.get_by(DataMigration, name: name) + end + + def populate_hashtags_table, do: get_by_name("populate_hashtags_table") +end diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex index 0ded2855c..baf79dda7 100644 --- a/lib/pleroma/delivery.ex +++ b/lib/pleroma/delivery.ex @@ -9,7 +9,6 @@ defmodule Pleroma.Delivery do alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User - alias Pleroma.User import Ecto.Changeset import Ecto.Query diff --git a/lib/pleroma/ecto_enums.ex b/lib/pleroma/ecto_enums.ex index 6fc47620c..f0ae658a4 100644 --- a/lib/pleroma/ecto_enums.ex +++ b/lib/pleroma/ecto_enums.ex @@ -17,3 +17,11 @@ defenum(Pleroma.FollowingRelationship.State, follow_accept: 2, follow_reject: 3 ) + +defenum(Pleroma.DataMigration.State, + pending: 1, + running: 2, + complete: 3, + failed: 4, + manual: 5 +) diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex new file mode 100644 index 000000000..a7e3de542 --- /dev/null +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -0,0 +1,211 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Migrators.HashtagsTableMigrator do + defmodule State do + use Agent + + @init_state %{} + + def start_link(_) do + Agent.start_link(fn -> @init_state end, name: __MODULE__) + end + + def get do + Agent.get(__MODULE__, & &1) + end + + def put(key, value) do + Agent.update(__MODULE__, fn state -> + Map.put(state, key, value) + end) + end + + def increment(key, increment \\ 1) do + Agent.update(__MODULE__, fn state -> + updated_value = (state[key] || 0) + increment + Map.put(state, key, updated_value) + end) + end + end + + use GenServer + + require Logger + + import Ecto.Query + + alias Pleroma.Config + alias Pleroma.DataMigration + alias Pleroma.Hashtag + alias Pleroma.Object + alias Pleroma.Repo + + defdelegate state(), to: State, as: :get + defdelegate put_state(key, value), to: State, as: :put + defdelegate increment_state(key, increment), to: State, as: :increment + + defdelegate data_migration(), to: DataMigration, as: :populate_hashtags_table + + def start_link(_) do + GenServer.start_link(__MODULE__, nil, name: __MODULE__) + end + + @impl true + def init(_) do + {:ok, nil, {:continue, :init_state}} + end + + @impl true + def handle_continue(:init_state, _state) do + {:ok, _} = State.start_link(nil) + + put_state(:status, :init) + + dm = data_migration() + + cond do + Config.get(:env) == :test -> + put_state(:status, :noop) + + is_nil(dm) -> + put_state(:status, :halt) + put_state(:message, "Data migration does not exist.") + + dm.state == :manual -> + put_state(:status, :noop) + put_state(:message, "Data migration is in manual execution state.") + + dm.state == :complete -> + handle_success() + + true -> + send(self(), :migrate_hashtags) + end + + {:noreply, nil} + end + + @impl true + def handle_info(:migrate_hashtags, state) do + data_migration = data_migration() + + {:ok, data_migration} = DataMigration.update_state(data_migration, :running) + put_state(:status, :running) + + Logger.info("Starting transferring object embedded hashtags to `hashtags` table...") + + max_processed_id = data_migration.data["max_processed_id"] || 0 + + # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out) + from( + object in Object, + left_join: hashtag in assoc(object, :hashtags), + where: object.id > ^max_processed_id, + where: is_nil(hashtag.id), + where: + fragment("(?)->'tag' IS NOT NULL AND (?)->'tag' != '[]'::jsonb", object.data, object.data), + select: %{ + id: object.id, + tag: fragment("(?)->'tag'", object.data) + } + ) + |> Repo.chunk_stream(100, :batches, timeout: :infinity) + |> Stream.each(fn objects -> + object_ids = Enum.map(objects, & &1.id) + + failed_ids = + objects + |> Enum.map(&transfer_object_hashtags(&1)) + |> Enum.filter(&(elem(&1, 0) == :error)) + |> Enum.map(&elem(&1, 1)) + + for failed_id <- failed_ids do + _ = + Repo.query( + "INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <> + "VALUES ($1, $2) ON CONFLICT DO NOTHING;", + [data_migration.id, failed_id] + ) + end + + _ = + Repo.query( + "DELETE FROM data_migration_failed_ids WHERE id = ANY($1)", + [object_ids -- failed_ids] + ) + + max_object_id = Enum.at(object_ids, -1) + _ = DataMigration.update(data_migration, %{data: %{"max_processed_id" => max_object_id}}) + + increment_state(:processed_count, length(object_ids)) + increment_state(:failed_count, length(failed_ids)) + + # A quick and dirty approach to controlling the load this background migration imposes + sleep_interval = Config.get([:populate_hashtags_table, :sleep_interval_ms], 0) + Process.sleep(sleep_interval) + end) + |> Stream.run() + + with {:ok, %{rows: [[0]]}} <- + Repo.query( + "SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;", + [data_migration.id] + ) do + put_state(:status, :complete) + _ = DataMigration.update_state(data_migration, :complete) + + handle_success() + else + _ -> + put_state(:status, :failed) + put_state(:message, "Please check data_migration_failed_ids records.") + end + + {:noreply, state} + end + + defp transfer_object_hashtags(object) do + hashtags = Object.object_data_hashtags(%{"tag" => object.tag}) + + Repo.transaction(fn -> + with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do + for hashtag_record <- hashtag_records do + with {:ok, _} <- + Repo.query( + "insert into hashtags_objects(hashtag_id, object_id) values ($1, $2);", + [hashtag_record.id, object.id] + ) do + nil + else + {:error, e} -> + error = + "ERROR: could not link object #{object.id} and hashtag " <> + "#{hashtag_record.id}: #{inspect(e)}" + + Logger.error(error) + Repo.rollback(object.id) + end + end + + object.id + else + e -> + error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}" + Logger.error(error) + Repo.rollback(object.id) + end + end) + end + + defp handle_success do + put_state(:status, :complete) + + unless Config.improved_hashtag_timeline() do + Config.put(Config.improved_hashtag_timeline_path(), true) + end + + :ok + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 339843330..6131ae85b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1276,7 +1276,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_tag_all(opts) # TODO: benchmark (initial approach preferring non-aggregate ops when possible) - Config.get([:instance, :improved_hashtag_timeline]) == :join -> + Config.improved_hashtag_timeline() == :join -> query |> distinct([activity], true) |> restrict_hashtag_any(opts) -- cgit v1.2.3 From f5f267fa764f53ef617bc9504c7ecb68b5d3d7ab Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 14 Jan 2021 22:41:27 +0300 Subject: [#3213] Refactoring of HashtagsTableMigrator. --- lib/pleroma/migrators/hashtags_table_migrator.ex | 98 ++++++++++++---------- .../migrators/hashtags_table_migrator/state.ex | 26 ++++++ 2 files changed, 78 insertions(+), 46 deletions(-) create mode 100644 lib/pleroma/migrators/hashtags_table_migrator/state.ex (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index a7e3de542..9f1a00f9c 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -3,39 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Migrators.HashtagsTableMigrator do - defmodule State do - use Agent - - @init_state %{} - - def start_link(_) do - Agent.start_link(fn -> @init_state end, name: __MODULE__) - end - - def get do - Agent.get(__MODULE__, & &1) - end - - def put(key, value) do - Agent.update(__MODULE__, fn state -> - Map.put(state, key, value) - end) - end - - def increment(key, increment \\ 1) do - Agent.update(__MODULE__, fn state -> - updated_value = (state[key] || 0) + increment - Map.put(state, key, updated_value) - end) - end - end - use GenServer require Logger import Ecto.Query + alias __MODULE__.State alias Pleroma.Config alias Pleroma.DataMigration alias Pleroma.Hashtag @@ -43,13 +17,23 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do alias Pleroma.Repo defdelegate state(), to: State, as: :get - defdelegate put_state(key, value), to: State, as: :put - defdelegate increment_state(key, increment), to: State, as: :increment + defdelegate put_stat(key, value), to: State, as: :put + defdelegate increment_stat(key, increment), to: State, as: :increment defdelegate data_migration(), to: DataMigration, as: :populate_hashtags_table + @reg_name {:global, __MODULE__} + + def whereis, do: GenServer.whereis(@reg_name) + def start_link(_) do - GenServer.start_link(__MODULE__, nil, name: __MODULE__) + case whereis() do + nil -> + GenServer.start_link(__MODULE__, nil, name: @reg_name) + + pid -> + {:ok, pid} + end end @impl true @@ -61,21 +45,22 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do def handle_continue(:init_state, _state) do {:ok, _} = State.start_link(nil) - put_state(:status, :init) + put_stat(:status, :init) dm = data_migration() + manual_migrations = Config.get([:instance, :manual_data_migrations], []) cond do Config.get(:env) == :test -> - put_state(:status, :noop) + put_stat(:status, :noop) is_nil(dm) -> - put_state(:status, :halt) - put_state(:message, "Data migration does not exist.") + put_stat(:status, :halt) + put_stat(:message, "Data migration does not exist.") - dm.state == :manual -> - put_state(:status, :noop) - put_state(:message, "Data migration is in manual execution state.") + dm.state == :manual or dm.name in manual_migrations -> + put_stat(:status, :noop) + put_stat(:message, "Data migration is in manual execution state.") dm.state == :complete -> handle_success() @@ -91,8 +76,12 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do def handle_info(:migrate_hashtags, state) do data_migration = data_migration() - {:ok, data_migration} = DataMigration.update_state(data_migration, :running) - put_state(:status, :running) + persistent_data = Map.take(data_migration.data, ["max_processed_id"]) + + {:ok, data_migration} = + DataMigration.update(data_migration, %{state: :running, data: persistent_data}) + + put_stat(:status, :running) Logger.info("Starting transferring object embedded hashtags to `hashtags` table...") @@ -137,10 +126,12 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do ) max_object_id = Enum.at(object_ids, -1) - _ = DataMigration.update(data_migration, %{data: %{"max_processed_id" => max_object_id}}) - increment_state(:processed_count, length(object_ids)) - increment_state(:failed_count, length(failed_ids)) + put_stat(:max_processed_id, max_object_id) + increment_stat(:processed_count, length(object_ids)) + increment_stat(:failed_count, length(failed_ids)) + + persist_stats(data_migration) # A quick and dirty approach to controlling the load this background migration imposes sleep_interval = Config.get([:populate_hashtags_table, :sleep_interval_ms], 0) @@ -153,14 +144,15 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do "SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;", [data_migration.id] ) do - put_state(:status, :complete) _ = DataMigration.update_state(data_migration, :complete) handle_success() else _ -> - put_state(:status, :failed) - put_state(:message, "Please check data_migration_failed_ids records.") + _ = DataMigration.update_state(data_migration, :failed) + + put_stat(:status, :failed) + put_stat(:message, "Please check data_migration_failed_ids records.") end {:noreply, state} @@ -199,8 +191,13 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do end) end + defp persist_stats(data_migration) do + runner_state = Map.drop(state(), [:status]) + _ = DataMigration.update(data_migration, %{data: runner_state}) + end + defp handle_success do - put_state(:status, :complete) + put_stat(:status, :complete) unless Config.improved_hashtag_timeline() do Config.put(Config.improved_hashtag_timeline_path(), true) @@ -208,4 +205,13 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do :ok end + + def force_continue do + send(whereis(), :migrate_hashtags) + end + + def force_restart do + {:ok, _} = DataMigration.update(data_migration(), %{state: :pending, data: %{}}) + force_continue() + end end diff --git a/lib/pleroma/migrators/hashtags_table_migrator/state.ex b/lib/pleroma/migrators/hashtags_table_migrator/state.ex new file mode 100644 index 000000000..79926892c --- /dev/null +++ b/lib/pleroma/migrators/hashtags_table_migrator/state.ex @@ -0,0 +1,26 @@ +defmodule Pleroma.Migrators.HashtagsTableMigrator.State do + use Agent + + @init_state %{} + + def start_link(_) do + Agent.start_link(fn -> @init_state end, name: __MODULE__) + end + + def get do + Agent.get(__MODULE__, & &1) + end + + def put(key, value) do + Agent.update(__MODULE__, fn state -> + Map.put(state, key, value) + end) + end + + def increment(key, increment \\ 1) do + Agent.update(__MODULE__, fn state -> + updated_value = (state[key] || 0) + increment + Map.put(state, key, updated_value) + end) + end +end -- cgit v1.2.3 From 48b399cedb7d46ea0f08181cfbe4df222861f65b Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 16 Jan 2021 20:22:14 +0300 Subject: [#3213] Refactoring of HashtagsTableMigrator. Hashtag timeline performance optimization (auto switch to non-aggregate join strategy when efficient). --- lib/pleroma/migrators/hashtags_table_migrator.ex | 47 +++++++++------ .../migrators/hashtags_table_migrator/state.ex | 9 +-- lib/pleroma/web/activity_pub/activity_pub.ex | 68 +++++++++++++--------- 3 files changed, 75 insertions(+), 49 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index 9f1a00f9c..b40578d50 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -45,25 +45,23 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do def handle_continue(:init_state, _state) do {:ok, _} = State.start_link(nil) - put_stat(:status, :init) + update_status(:init) - dm = data_migration() + data_migration = data_migration() manual_migrations = Config.get([:instance, :manual_data_migrations], []) cond do Config.get(:env) == :test -> - put_stat(:status, :noop) + update_status(:noop) - is_nil(dm) -> - put_stat(:status, :halt) - put_stat(:message, "Data migration does not exist.") + is_nil(data_migration) -> + update_status(:halt, "Data migration does not exist.") - dm.state == :manual or dm.name in manual_migrations -> - put_stat(:status, :noop) - put_stat(:message, "Data migration is in manual execution state.") + data_migration.state == :manual or data_migration.name in manual_migrations -> + update_status(:noop, "Data migration is in manual execution state.") - dm.state == :complete -> - handle_success() + data_migration.state == :complete -> + handle_success(data_migration) true -> send(self(), :migrate_hashtags) @@ -81,7 +79,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do {:ok, data_migration} = DataMigration.update(data_migration, %{state: :running, data: persistent_data}) - put_stat(:status, :running) + update_status(:running) Logger.info("Starting transferring object embedded hashtags to `hashtags` table...") @@ -146,13 +144,12 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do ) do _ = DataMigration.update_state(data_migration, :complete) - handle_success() + handle_success(data_migration) else _ -> _ = DataMigration.update_state(data_migration, :failed) - put_stat(:status, :failed) - put_stat(:message, "Please check data_migration_failed_ids records.") + update_status(:failed, "Please check data_migration_failed_ids records.") end {:noreply, state} @@ -196,16 +193,25 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do _ = DataMigration.update(data_migration, %{data: runner_state}) end - defp handle_success do - put_stat(:status, :complete) + defp handle_success(data_migration) do + update_status(:complete) - unless Config.improved_hashtag_timeline() do + unless data_migration.feature_lock || Config.improved_hashtag_timeline() do Config.put(Config.improved_hashtag_timeline_path(), true) end :ok end + def failed_objects_query do + from(o in Object) + |> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"), + on: dmf.record_id == o.id + ) + |> where([_o, dmf], dmf.data_migration_id == ^data_migration().id) + |> order_by([o], asc: o.id) + end + def force_continue do send(whereis(), :migrate_hashtags) end @@ -214,4 +220,9 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do {:ok, _} = DataMigration.update(data_migration(), %{state: :pending, data: %{}}) force_continue() end + + defp update_status(status, message \\ nil) do + put_stat(:status, status) + put_stat(:message, message) + end end diff --git a/lib/pleroma/migrators/hashtags_table_migrator/state.ex b/lib/pleroma/migrators/hashtags_table_migrator/state.ex index 79926892c..c1a2709fc 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator/state.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator/state.ex @@ -2,23 +2,24 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator.State do use Agent @init_state %{} + @reg_name {:global, __MODULE__} def start_link(_) do - Agent.start_link(fn -> @init_state end, name: __MODULE__) + Agent.start_link(fn -> @init_state end, name: @reg_name) end def get do - Agent.get(__MODULE__, & &1) + Agent.get(@reg_name, & &1) end def put(key, value) do - Agent.update(__MODULE__, fn state -> + Agent.update(@reg_name, fn state -> Map.put(state, key, value) end) end def increment(key, increment \\ 1) do - Agent.update(__MODULE__, fn state -> + Agent.update(@reg_name, fn state -> updated_value = (state[key] || 0) + increment Map.put(state, key, updated_value) end) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f5563b0fd..0609827ec 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -669,63 +669,66 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_since(query, _), do: query - defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do + defp restrict_embedded_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do raise_on_missing_preload() end - defp restrict_tag_reject(query, %{tag_reject: tag_reject}) when is_list(tag_reject) do + defp restrict_embedded_tag_reject(query, %{tag_reject: tag_reject}) when is_list(tag_reject) do from( [_activity, object] in query, where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject) ) end - defp restrict_tag_reject(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do - restrict_tag_reject(query, %{tag_reject: [tag_reject]}) + defp restrict_embedded_tag_reject(query, %{tag_reject: tag_reject}) + when is_binary(tag_reject) do + restrict_embedded_tag_reject(query, %{tag_reject: [tag_reject]}) end - defp restrict_tag_reject(query, _), do: query + defp restrict_embedded_tag_reject(query, _), do: query - defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do + defp restrict_embedded_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do raise_on_missing_preload() end - defp restrict_tag_all(query, %{tag_all: tag_all}) when is_list(tag_all) do + defp restrict_embedded_tag_all(query, %{tag_all: tag_all}) when is_list(tag_all) do from( [_activity, object] in query, where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) ) end - defp restrict_tag_all(query, %{tag_all: tag}) when is_binary(tag) do - restrict_tag(query, %{tag: tag}) + defp restrict_embedded_tag_all(query, %{tag_all: tag}) when is_binary(tag) do + restrict_embedded_tag(query, %{tag: tag}) end - defp restrict_tag_all(query, _), do: query + defp restrict_embedded_tag_all(query, _), do: query - defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do + defp restrict_embedded_tag(_query, %{tag: _tag, skip_preload: true}) do raise_on_missing_preload() end - defp restrict_tag(query, %{tag: tag}) when is_list(tag) do + defp restrict_embedded_tag(query, %{tag: tag}) when is_list(tag) do from( [_activity, object] in query, where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag) ) end - defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do - restrict_tag(query, %{tag: [tag]}) + defp restrict_embedded_tag(query, %{tag: tag}) when is_binary(tag) do + restrict_embedded_tag(query, %{tag: [tag]}) end - defp restrict_tag(query, _), do: query + defp restrict_embedded_tag(query, _), do: query - defp restrict_hashtag(query, opts) do - [tag_any, tag_all, tag_reject] = - [:tag, :tag_all, :tag_reject] - |> Enum.map(&opts[&1]) - |> Enum.map(&List.wrap(&1)) + defp hashtag_conditions(opts) do + [:tag, :tag_all, :tag_reject] + |> Enum.map(&opts[&1]) + |> Enum.map(&List.wrap(&1)) + end + defp restrict_hashtag_agg(query, opts) do + [tag_any, tag_all, tag_reject] = hashtag_conditions(opts) has_conditions = Enum.any?([tag_any, tag_all, tag_reject], &Enum.any?(&1)) cond do @@ -1275,15 +1278,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> exclude_invisible_actors(opts) |> exclude_visibility(opts) + hashtag_timeline_strategy = Config.improved_hashtag_timeline() + cond do - Config.object_embedded_hashtags?() -> + !hashtag_timeline_strategy -> query - |> restrict_tag(opts) - |> restrict_tag_reject(opts) - |> restrict_tag_all(opts) + |> restrict_embedded_tag(opts) + |> restrict_embedded_tag_reject(opts) + |> restrict_embedded_tag_all(opts) + + hashtag_timeline_strategy == :prefer_aggregation -> + restrict_hashtag_agg(query, opts) - # TODO: benchmark (initial approach preferring non-aggregate ops when possible) - Config.improved_hashtag_timeline() == :join -> + hashtag_timeline_strategy == :avoid_aggregation or avoid_hashtags_aggregation?(opts) -> query |> distinct([activity], true) |> restrict_hashtag_any(opts) @@ -1291,10 +1298,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_hashtag_reject_any(opts) true -> - restrict_hashtag(query, opts) + restrict_hashtag_agg(query, opts) end end + defp avoid_hashtags_aggregation?(opts) do + [tag_any, tag_all, tag_reject] = hashtag_conditions(opts) + + joins_count = length(tag_all) + if Enum.any?(tag_any), do: 1, else: 0 + Enum.empty?(tag_reject) and joins_count <= 2 + end + def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do list_memberships = Pleroma.List.memberships(opts[:user]) -- cgit v1.2.3 From 85f7ef4d13adea9d64d279d1395d17c6ebc20678 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 17 Jan 2021 10:57:06 +0300 Subject: [#3213] Feature lock adjustment for HashtagsTableMigrator. --- lib/pleroma/migrators/hashtags_table_migrator.ex | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index b40578d50..47de5e134 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -196,11 +196,17 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do defp handle_success(data_migration) do update_status(:complete) - unless data_migration.feature_lock || Config.improved_hashtag_timeline() do - Config.put(Config.improved_hashtag_timeline_path(), true) - end + cond do + data_migration.feature_lock -> + :noop + + not is_nil(Config.improved_hashtag_timeline()) -> + :noop - :ok + true -> + Config.put(Config.improved_hashtag_timeline_path(), true) + :ok + end end def failed_objects_query do -- cgit v1.2.3 From 9d28a7ebfbc7bd8fb893cf1e2ad555ed71f4c812 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 17 Jan 2021 21:58:15 +0300 Subject: [#3213] Missing copyright header for HashtagsTableMigrator.State. --- lib/pleroma/migrators/hashtags_table_migrator/state.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator/state.ex b/lib/pleroma/migrators/hashtags_table_migrator/state.ex index c1a2709fc..43f7270e2 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator/state.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator/state.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Migrators.HashtagsTableMigrator.State do use Agent -- cgit v1.2.3 From 7f07909a7b56eb368b3f8aab4752def1551c12fe Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 19 Jan 2021 21:13:32 +0300 Subject: [#3213] Added `HashtagsTableMigrator.count/1`. --- lib/pleroma/migrators/hashtags_table_migrator.ex | 42 ++++++++++++++++-------- 1 file changed, 29 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index 47de5e134..048f3c8ee 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -85,19 +85,8 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do max_processed_id = data_migration.data["max_processed_id"] || 0 - # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out) - from( - object in Object, - left_join: hashtag in assoc(object, :hashtags), - where: object.id > ^max_processed_id, - where: is_nil(hashtag.id), - where: - fragment("(?)->'tag' IS NOT NULL AND (?)->'tag' != '[]'::jsonb", object.data, object.data), - select: %{ - id: object.id, - tag: fragment("(?)->'tag'", object.data) - } - ) + query() + |> where([object], object.id > ^max_processed_id) |> Repo.chunk_stream(100, :batches, timeout: :infinity) |> Stream.each(fn objects -> object_ids = Enum.map(objects, & &1.id) @@ -155,6 +144,21 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do {:noreply, state} end + defp query do + # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out) + from( + object in Object, + left_join: hashtag in assoc(object, :hashtags), + where: is_nil(hashtag.id), + where: + fragment("(?)->'tag' IS NOT NULL AND (?)->'tag' != '[]'::jsonb", object.data, object.data), + select: %{ + id: object.id, + tag: fragment("(?)->'tag'", object.data) + } + ) + end + defp transfer_object_hashtags(object) do hashtags = Object.object_data_hashtags(%{"tag" => object.tag}) @@ -188,6 +192,18 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do end) end + def count(force \\ false) do + stored_count = state()[:count] + + if stored_count && !force do + stored_count + else + count = Repo.aggregate(query(), :count, :id) + put_stat(:count, count) + count + end + end + defp persist_stats(data_migration) do runner_state = Map.drop(state(), [:status]) _ = DataMigration.update(data_migration, %{data: runner_state}) -- cgit v1.2.3 From f0f0f2af00e8b73a7013c1308289795961b23f4b Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 19 Jan 2021 21:17:06 +0300 Subject: [#3213] `timeout` option for `HashtagsTableMigrator.count/_`. --- lib/pleroma/migrators/hashtags_table_migrator.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index 048f3c8ee..6a6a7b5b8 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -192,13 +192,13 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do end) end - def count(force \\ false) do + def count(force \\ false, timeout \\ :infinity) do stored_count = state()[:count] if stored_count && !force do stored_count else - count = Repo.aggregate(query(), :count, :id) + count = Repo.aggregate(query(), :count, :id, timeout: timeout) put_stat(:count, count) count end -- cgit v1.2.3 From b830605577f369d6b1a8730a5b3476ceea4fef5a Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 19 Jan 2021 22:03:25 +0300 Subject: [#3213] Performance-related stat in HashtagsTableMigrator. Reworked `count/_` to indicate approximate total count for current iteration. --- lib/pleroma/migrators/hashtags_table_migrator.ex | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index 6a6a7b5b8..e9dd9b70c 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -80,6 +80,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do DataMigration.update(data_migration, %{state: :running, data: persistent_data}) update_status(:running) + put_stat(:started_at, NaiveDateTime.utc_now()) Logger.info("Starting transferring object embedded hashtags to `hashtags` table...") @@ -118,6 +119,12 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do increment_stat(:processed_count, length(object_ids)) increment_stat(:failed_count, length(failed_ids)) + put_stat( + :records_per_second, + state()[:processed_count] / + Enum.max([NaiveDateTime.diff(NaiveDateTime.utc_now(), state()[:started_at]), 1]) + ) + persist_stats(data_migration) # A quick and dirty approach to controlling the load this background migration imposes @@ -192,13 +199,18 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do end) end + @doc "Approximate count for current iteration (including processed records count)" def count(force \\ false, timeout \\ :infinity) do stored_count = state()[:count] if stored_count && !force do stored_count else - count = Repo.aggregate(query(), :count, :id, timeout: timeout) + processed_count = state()[:processed_count] || 0 + max_processed_id = data_migration().data["max_processed_id"] || 0 + query = where(query(), [object], object.id > ^max_processed_id) + + count = Repo.aggregate(query, :count, :id, timeout: timeout) + processed_count put_stat(:count, count) count end -- cgit v1.2.3 From c041e9c6300726a40a00146bba04d3ec752219d9 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 21 Jan 2021 20:19:09 +0300 Subject: [#3213] HashtagsTableMigrator: failures handling fix, retry function. Changed default hashtags filtering strategy to non-aggregate approach. --- lib/pleroma/migrators/hashtags_table_migrator.ex | 52 ++++++++++++++++++++---- lib/pleroma/web/activity_pub/activity_pub.ex | 13 +----- 2 files changed, 45 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index e9dd9b70c..8ad2c8c73 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -109,8 +109,9 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do _ = Repo.query( - "DELETE FROM data_migration_failed_ids WHERE id = ANY($1)", - [object_ids -- failed_ids] + "DELETE FROM data_migration_failed_ids " <> + "WHERE data_migration_id = $1 AND record_id = ANY($2)", + [data_migration.id, object_ids -- failed_ids] ) max_object_id = Enum.at(object_ids, -1) @@ -133,12 +134,8 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do end) |> Stream.run() - with {:ok, %{rows: [[0]]}} <- - Repo.query( - "SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;", - [data_migration.id] - ) do - _ = DataMigration.update_state(data_migration, :complete) + with 0 <- failures_count(data_migration.id) do + {:ok, data_migration} = DataMigration.update_state(data_migration, :complete) handle_success(data_migration) else @@ -167,7 +164,8 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do end defp transfer_object_hashtags(object) do - hashtags = Object.object_data_hashtags(%{"tag" => object.tag}) + embedded_tags = (Map.has_key?(object, :tag) && object.tag) || object.data["tag"] + hashtags = Object.object_data_hashtags(%{"tag" => embedded_tags}) Repo.transaction(fn -> with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do @@ -246,6 +244,36 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do |> order_by([o], asc: o.id) end + def failures_count(data_migration_id \\ nil) do + data_migration_id = data_migration_id || data_migration().id + + with {:ok, %{rows: [[count]]}} <- + Repo.query( + "SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;", + [data_migration_id] + ) do + count + end + end + + def retry_failed do + data_migration = data_migration() + + failed_objects_query() + |> Repo.chunk_stream(100, :one) + |> Stream.each(fn object -> + with {:ok, _} <- transfer_object_hashtags(object) do + _ = + Repo.query( + "DELETE FROM data_migration_failed_ids " <> + "WHERE data_migration_id = $1 AND record_id = $2", + [data_migration.id, object.id] + ) + end + end) + |> Stream.run() + end + def force_continue do send(whereis(), :migrate_hashtags) end @@ -255,6 +283,12 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do force_continue() end + def force_complete do + {:ok, data_migration} = DataMigration.update_state(data_migration(), :complete) + + handle_success(data_migration) + end + defp update_status(status, message \\ nil) do put_stat(:status, status) put_stat(:message, message) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 0609827ec..dbfd3839d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -727,6 +727,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Enum.map(&List.wrap(&1)) end + # Note: times out on larger instances (with default timeout), intended for complex queries defp restrict_hashtag_agg(query, opts) do [tag_any, tag_all, tag_reject] = hashtag_conditions(opts) has_conditions = Enum.any?([tag_any, tag_all, tag_reject], &Enum.any?(&1)) @@ -1290,25 +1291,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do hashtag_timeline_strategy == :prefer_aggregation -> restrict_hashtag_agg(query, opts) - hashtag_timeline_strategy == :avoid_aggregation or avoid_hashtags_aggregation?(opts) -> + true -> query |> distinct([activity], true) |> restrict_hashtag_any(opts) |> restrict_hashtag_all(opts) |> restrict_hashtag_reject_any(opts) - - true -> - restrict_hashtag_agg(query, opts) end end - defp avoid_hashtags_aggregation?(opts) do - [tag_any, tag_all, tag_reject] = hashtag_conditions(opts) - - joins_count = length(tag_all) + if Enum.any?(tag_any), do: 1, else: 0 - Enum.empty?(tag_reject) and joins_count <= 2 - end - def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do list_memberships = Pleroma.List.memberships(opts[:user]) -- cgit v1.2.3 From ca7f24064304945587fc232325dce4b834ff6c94 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 21 Jan 2021 20:50:06 +0300 Subject: [#3213] Ignoring of blank elements from objects.data->tag. --- lib/pleroma/object.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 5102be1de..9b5c1bec1 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -420,6 +420,8 @@ defmodule Pleroma.Object do hashtag when is_bitstring(hashtag) -> String.downcase(hashtag) end) |> Enum.uniq() + # Note: "" elements (plain text) might occur in `data.tag` for incoming objects + |> Enum.filter(&(&1 not in [nil, ""])) end def object_data_hashtags(_), do: [] -- cgit v1.2.3 From f264d930cc00c463d0f506a94f6f6b494aab7022 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 24 Jan 2021 23:27:02 +0300 Subject: [#3213] Speedup of HashtagsTableMigrator (query optimization). State handling fix. --- lib/pleroma/migrators/hashtags_table_migrator.ex | 18 +++++++++++++++--- lib/pleroma/migrators/hashtags_table_migrator/state.ex | 4 ++++ 2 files changed, 19 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index 8ad2c8c73..6a1c9592c 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -72,6 +72,8 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do @impl true def handle_info(:migrate_hashtags, state) do + State.clear() + data_migration = data_migration() persistent_data = Map.take(data_migration.data, ["max_processed_id"]) @@ -152,8 +154,6 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out) from( object in Object, - left_join: hashtag in assoc(object, :hashtags), - where: is_nil(hashtag.id), where: fragment("(?)->'tag' IS NOT NULL AND (?)->'tag' != '[]'::jsonb", object.data, object.data), select: %{ @@ -161,12 +161,24 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do tag: fragment("(?)->'tag'", object.data) } ) + |> join(:left, [o], hashtags_objects in fragment("SELECT object_id FROM hashtags_objects"), + on: hashtags_objects.object_id == o.id + ) + |> where([_o, hashtags_objects], is_nil(hashtags_objects.object_id)) end defp transfer_object_hashtags(object) do - embedded_tags = (Map.has_key?(object, :tag) && object.tag) || object.data["tag"] + embedded_tags = if Map.has_key?(object, :tag), do: object.tag, else: object.data["tag"] hashtags = Object.object_data_hashtags(%{"tag" => embedded_tags}) + if Enum.any?(hashtags) do + transfer_object_hashtags(object, hashtags) + else + {:ok, object.id} + end + end + + defp transfer_object_hashtags(object, hashtags) do Repo.transaction(fn -> with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do for hashtag_record <- hashtag_records do diff --git a/lib/pleroma/migrators/hashtags_table_migrator/state.ex b/lib/pleroma/migrators/hashtags_table_migrator/state.ex index 43f7270e2..901563426 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator/state.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator/state.ex @@ -12,6 +12,10 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator.State do Agent.start_link(fn -> @init_state end, name: @reg_name) end + def clear do + Agent.update(@reg_name, fn _state -> @init_state end) + end + def get do Agent.get(@reg_name, & &1) end -- cgit v1.2.3 From ea4785213a449f3bcd68bcb4ecb3bb6d794736b1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 25 Jan 2021 20:12:09 +0300 Subject: [#3213] Switched to using embedded hashtags in Object.hashtags/1 (to avoid extra joins / preload in timeline queries). --- lib/pleroma/config.ex | 1 - lib/pleroma/object.ex | 18 +++++------------- 2 files changed, 5 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index ceb8c8b5a..0a6ac0ad0 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -98,7 +98,6 @@ defmodule Pleroma.Config do def improved_hashtag_timeline_path, do: [:instance, :improved_hashtag_timeline] def improved_hashtag_timeline, do: get(improved_hashtag_timeline_path()) - def object_embedded_hashtags?, do: !improved_hashtag_timeline() def oauth_consumer_strategies, do: get([:auth, :oauth_consumer_strategies], []) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 9b5c1bec1..9edf43e04 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -388,24 +388,16 @@ defmodule Pleroma.Object do def tags(_), do: [] def hashtags(%Object{} = object) do - cond do - Config.object_embedded_hashtags?() -> - embedded_hashtags(object) - - object.id == "pleroma:fake_object_id" -> - [] - - true -> - hashtag_records = Repo.preload(object, :hashtags).hashtags - Enum.map(hashtag_records, & &1.name) - end + # Note: always using embedded hashtags regardless whether they are migrated to hashtags table + # (embedded hashtags stay in sync anyways, and we avoid extra joins and preload hassle) + embedded_hashtags(object) end - defp embedded_hashtags(%Object{data: data}) do + def embedded_hashtags(%Object{data: data}) do object_data_hashtags(data) end - defp embedded_hashtags(_), do: [] + def embedded_hashtags(_), do: [] def object_data_hashtags(%{"tag" => tags}) when is_list(tags) do tags -- cgit v1.2.3 From e7864a32d7c9930e5f6c62bd77cef64c68f1eb21 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 25 Jan 2021 22:31:23 +0300 Subject: [#3213] Removed DISTINCT clause from ActivityPub.fetch_activities_query/2. --- lib/pleroma/web/activity_pub/activity_pub.ex | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index fbda89a25..be81e0833 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1293,7 +1293,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do true -> query - |> distinct([activity], true) |> restrict_hashtag_any(opts) |> restrict_hashtag_all(opts) |> restrict_hashtag_reject_any(opts) -- cgit v1.2.3 From 380d0cce6b802baf4d13031a4a39f169dd65bffd Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 29 Jan 2021 00:17:33 +0300 Subject: [#3213] Reinstated DISTINCT clause for hashtag "any" filtering with 2+ terms. Added test. --- lib/pleroma/web/activity_pub/activity_pub.ex | 17 ++++++++++++----- .../web/mastodon_api/controllers/timeline_controller.ex | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index be81e0833..0a21ac2f2 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -846,11 +846,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_hashtag_any(query, %{tag: tags}) when is_list(tags) do - from( - [_activity, object] in query, - join: hashtag in assoc(object, :hashtags), - where: hashtag.name in ^tags - ) + query = + from( + [_activity, object] in query, + join: hashtag in assoc(object, :hashtags), + where: hashtag.name in ^tags + ) + + if length(tags) > 1 do + distinct(query, [activity], true) + else + query + end end defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index 08e6f23b9..1fb954a9b 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -134,9 +134,9 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do tags = [params[:tag], params[:any]] |> List.flatten() - |> Enum.uniq() |> Enum.reject(&is_nil/1) |> Enum.map(&String.downcase/1) + |> Enum.uniq() tag_all = params -- cgit v1.2.3 From 9948ff3356f9e9e214584207a53eba614c73383c Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 31 Jan 2021 18:24:19 +0300 Subject: [#3213] Added HashtagsCleanupWorker periodic job. --- lib/pleroma/migrators/hashtags_table_migrator.ex | 1 + lib/pleroma/object.ex | 1 + .../workers/cron/hashtags_cleanup_worker.ex | 57 ++++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 lib/pleroma/workers/cron/hashtags_cleanup_worker.ex (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index 6a1c9592c..07b42a7f4 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -152,6 +152,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do defp query do # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out) + # Note: not checking activity type; HashtagsCleanupWorker should clean up unused records later from( object in Object, where: diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 9edf43e04..52b77e41c 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -65,6 +65,7 @@ defmodule Pleroma.Object do |> maybe_handle_hashtags_change(struct) end + # Note: not checking activity type; HashtagsCleanupWorker should clean up unused records later defp maybe_handle_hashtags_change(changeset, struct) do with data_hashtags_change = get_change(changeset, :data), true <- hashtags_changed?(struct, data_hashtags_change), diff --git a/lib/pleroma/workers/cron/hashtags_cleanup_worker.ex b/lib/pleroma/workers/cron/hashtags_cleanup_worker.ex new file mode 100644 index 000000000..b319067ca --- /dev/null +++ b/lib/pleroma/workers/cron/hashtags_cleanup_worker.ex @@ -0,0 +1,57 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.HashtagsCleanupWorker do + @moduledoc """ + The worker to clean up unused hashtags_objects and hashtags. + """ + + use Oban.Worker, queue: "hashtags_cleanup" + + alias Pleroma.Repo + + require Logger + + @hashtags_objects_query """ + DELETE FROM hashtags_objects WHERE object_id IN + (SELECT DISTINCT objects.id FROM objects + JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities + ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = + (objects.data->>'id') + AND activities.data->>'type' = 'Create' + WHERE activities.id IS NULL); + """ + + @hashtags_query """ + DELETE FROM hashtags WHERE id IN + (SELECT hashtags.id FROM hashtags + LEFT OUTER JOIN hashtags_objects + ON hashtags_objects.hashtag_id = hashtags.id + WHERE hashtags_objects.hashtag_id IS NULL AND hashtags.inserted_at < $1); + """ + + @impl Oban.Worker + def perform(_job) do + Logger.info("Cleaning up unused `hashtags_objects` records...") + + {:ok, %{num_rows: hashtags_objects_count}} = + Repo.query(@hashtags_objects_query, [], timeout: :infinity) + + Logger.info("Deleted #{hashtags_objects_count} unused `hashtags_objects` records.") + + Logger.info("Cleaning up unused `hashtags` records...") + + # Note: ignoring recently created hashtags since references are added after hashtag is created + {:ok, %{num_rows: hashtags_count}} = + Repo.query(@hashtags_query, [NaiveDateTime.add(NaiveDateTime.utc_now(), -3600 * 24)], + timeout: :infinity + ) + + Logger.info("Deleted #{hashtags_count} unused `hashtags` records.") + + Logger.info("HashtagsCleanupWorker complete.") + + :ok + end +end -- cgit v1.2.3 From 6fd4163ab60be07b1a20ac8911e105ddca8e2095 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 31 Jan 2021 20:37:33 +0300 Subject: [#3213] ActivityPub: implemented subqueries-based hashtags filtering, removed aggregation-based hashtags filtering. --- lib/pleroma/web/activity_pub/activity_pub.ex | 228 +++++++++------------------ 1 file changed, 78 insertions(+), 150 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 0a21ac2f2..cda8d3f54 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -669,24 +669,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_since(query, _), do: query - defp restrict_embedded_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do - raise_on_missing_preload() - end - - defp restrict_embedded_tag_reject(query, %{tag_reject: tag_reject}) when is_list(tag_reject) do - from( - [_activity, object] in query, - where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject) - ) - end - - defp restrict_embedded_tag_reject(query, %{tag_reject: tag_reject}) - when is_binary(tag_reject) do - restrict_embedded_tag_reject(query, %{tag_reject: [tag_reject]}) - end - - defp restrict_embedded_tag_reject(query, _), do: query - defp restrict_embedded_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do raise_on_missing_preload() end @@ -699,139 +681,65 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_embedded_tag_all(query, %{tag_all: tag}) when is_binary(tag) do - restrict_embedded_tag(query, %{tag: tag}) + restrict_embedded_tag_any(query, %{tag: tag}) end defp restrict_embedded_tag_all(query, _), do: query - defp restrict_embedded_tag(_query, %{tag: _tag, skip_preload: true}) do + defp restrict_embedded_tag_any(_query, %{tag: _tag, skip_preload: true}) do raise_on_missing_preload() end - defp restrict_embedded_tag(query, %{tag: tag}) when is_list(tag) do + defp restrict_embedded_tag_any(query, %{tag: tag}) when is_list(tag) do from( [_activity, object] in query, where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag) ) end - defp restrict_embedded_tag(query, %{tag: tag}) when is_binary(tag) do - restrict_embedded_tag(query, %{tag: [tag]}) - end - - defp restrict_embedded_tag(query, _), do: query - - defp hashtag_conditions(opts) do - [:tag, :tag_all, :tag_reject] - |> Enum.map(&opts[&1]) - |> Enum.map(&List.wrap(&1)) - end - - # Note: times out on larger instances (with default timeout), intended for complex queries - defp restrict_hashtag_agg(query, opts) do - [tag_any, tag_all, tag_reject] = hashtag_conditions(opts) - has_conditions = Enum.any?([tag_any, tag_all, tag_reject], &Enum.any?(&1)) - - cond do - !has_conditions -> - query - - opts[:skip_preload] -> - raise_on_missing_preload() - - true -> - query - |> group_by_all_bindings() - |> join(:left, [_activity, object], hashtag in assoc(object, :hashtags), as: :hashtag) - |> maybe_restrict_hashtag_any(tag_any) - |> maybe_restrict_hashtag_all(tag_all) - |> maybe_restrict_hashtag_reject_any(tag_reject) - end - end - - # Groups by all bindings to allow aggregation on hashtags - defp group_by_all_bindings(query) do - # Expecting named bindings: :object, :bookmark, :thread_mute, :report_note - cond do - Enum.count(query.aliases) == 4 -> - from([a, o, b3, b4, b5] in query, group_by: [a.id, o.id, b3.id, b4.id, b5.id]) - - Enum.count(query.aliases) == 3 -> - from([a, o, b3, b4] in query, group_by: [a.id, o.id, b3.id, b4.id]) - - Enum.count(query.aliases) == 2 -> - from([a, o, b3] in query, group_by: [a.id, o.id, b3.id]) - - true -> - from([a, o] in query, group_by: [a.id, o.id]) - end - end - - defp maybe_restrict_hashtag_any(query, []) do - query - end - - defp maybe_restrict_hashtag_any(query, tags) do - having( - query, - [hashtag: hashtag], - fragment("array_agg(?) && (?)", hashtag.name, ^tags) - ) - end - - defp maybe_restrict_hashtag_all(query, []) do - query + defp restrict_embedded_tag_any(query, %{tag: tag}) when is_binary(tag) do + restrict_embedded_tag_any(query, %{tag: [tag]}) end - defp maybe_restrict_hashtag_all(query, tags) do - having( - query, - [hashtag: hashtag], - fragment("array_agg(?) @> (?)", hashtag.name, ^tags) - ) - end + defp restrict_embedded_tag_any(query, _), do: query - defp maybe_restrict_hashtag_reject_any(query, []) do - query - end - - defp maybe_restrict_hashtag_reject_any(query, tags) do - having( - query, - [hashtag: hashtag], - fragment("not(array_agg(?) && (?))", hashtag.name, ^tags) - ) - end - - defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do + defp restrict_embedded_tag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do raise_on_missing_preload() end - defp restrict_hashtag_reject_any(query, %{tag_reject: tags_reject}) when is_list(tags_reject) do - query - |> group_by_all_bindings() - |> join(:left, [_activity, object], hashtag in assoc(object, :hashtags), as: :hashtag) - |> having( - [hashtag: hashtag], - fragment("not(array_agg(?) && (?))", hashtag.name, ^tags_reject) + defp restrict_embedded_tag_reject_any(query, %{tag_reject: tag_reject}) + when is_list(tag_reject) do + from( + [_activity, object] in query, + where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject) ) end - defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do - restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]}) + defp restrict_embedded_tag_reject_any(query, %{tag_reject: tag_reject}) + when is_binary(tag_reject) do + restrict_embedded_tag_reject_any(query, %{tag_reject: [tag_reject]}) end - defp restrict_hashtag_reject_any(query, _), do: query + defp restrict_embedded_tag_reject_any(query, _), do: query defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do raise_on_missing_preload() end defp restrict_hashtag_all(query, %{tag_all: tags}) when is_list(tags) do - Enum.reduce( - tags, - query, - fn tag, acc -> restrict_hashtag_any(acc, %{tag: tag}) end + from( + [_activity, object] in query, + where: + fragment( + """ + (SELECT array_agg(hashtags.name) FROM hashtags JOIN hashtags_objects + ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) + AND hashtags_objects.object_id = ?) @> ? + """, + ^tags, + object.id, + ^tags + ) ) end @@ -846,18 +754,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_hashtag_any(query, %{tag: tags}) when is_list(tags) do - query = - from( - [_activity, object] in query, - join: hashtag in assoc(object, :hashtags), - where: hashtag.name in ^tags - ) - - if length(tags) > 1 do - distinct(query, [activity], true) - else - query - end + from( + [_activity, object] in query, + where: + fragment( + """ + EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects + ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) + AND hashtags_objects.object_id = ? LIMIT 1) + """, + ^tags, + object.id + ) + ) end defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do @@ -866,6 +775,32 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_hashtag_any(query, _), do: query + defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do + raise_on_missing_preload() + end + + defp restrict_hashtag_reject_any(query, %{tag_reject: tags_reject}) when is_list(tags_reject) do + from( + [_activity, object] in query, + where: + fragment( + """ + NOT EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects + ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) + AND hashtags_objects.object_id = ? LIMIT 1) + """, + ^tags_reject, + object.id + ) + ) + end + + defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do + restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]}) + end + + defp restrict_hashtag_reject_any(query, _), do: query + defp raise_on_missing_preload do raise "Can't use the child object without preloading!" end @@ -1286,23 +1221,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> exclude_invisible_actors(opts) |> exclude_visibility(opts) - hashtag_timeline_strategy = Config.improved_hashtag_timeline() - - cond do - !hashtag_timeline_strategy -> - query - |> restrict_embedded_tag(opts) - |> restrict_embedded_tag_reject(opts) - |> restrict_embedded_tag_all(opts) - - hashtag_timeline_strategy == :prefer_aggregation -> - restrict_hashtag_agg(query, opts) - - true -> - query - |> restrict_hashtag_any(opts) - |> restrict_hashtag_all(opts) - |> restrict_hashtag_reject_any(opts) + if Config.improved_hashtag_timeline() do + query + |> restrict_hashtag_any(opts) + |> restrict_hashtag_all(opts) + |> restrict_hashtag_reject_any(opts) + else + query + |> restrict_embedded_tag_any(opts) + |> restrict_embedded_tag_all(opts) + |> restrict_embedded_tag_reject_any(opts) end end -- cgit v1.2.3 From 108e90b18edcfb57b9839e7c5d6d444a63ae2069 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 31 Jan 2021 22:03:59 +0300 Subject: [#3213] Explicitly defined PKs in hashtags_objects and data_migration_failed_ids. Added "pleroma.database rollback" task to revert a single migration. --- lib/mix/tasks/pleroma/database.ex | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 4ddace9c9..30c0d2bf1 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -20,6 +20,30 @@ defmodule Mix.Tasks.Pleroma.Database do @shortdoc "A collection of database related tasks" @moduledoc File.read!("docs/administration/CLI_tasks/database.md") + # Rolls back a specific migration (leaving subsequent migrations applied) + # Based on https://stackoverflow.com/a/53825840 + def run(["rollback", version]) do + start_pleroma() + + version = String.to_integer(version) + re = ~r/^#{version}_.*\.exs/ + path = Application.app_dir(:pleroma, Path.join(["priv", "repo", "migrations"])) + + result = + with {:find, "" <> file} <- {:find, Enum.find(File.ls!(path), &String.match?(&1, re))}, + {:compile, [{mod, _} | _]} <- {:compile, Code.compile_file(Path.join(path, file))}, + {:rollback, :ok} <- {:rollback, Ecto.Migrator.down(Repo, version, mod)} do + {:ok, "Reversed migration: #{file}"} + else + {:find, _} -> {:error, "No migration found with version prefix: #{version}"} + {:compile, e} -> {:error, "Problem compiling migration module: #{inspect(e)}"} + {:rollback, e} -> {:error, "Problem reversing migration: #{inspect(e)}"} + e -> {:error, "Something unexpected happened: #{inspect(e)}"} + end + + IO.inspect(result) + end + def run(["remove_embedded_objects" | args]) do {options, [], []} = OptionParser.parse( -- cgit v1.2.3 From 10207f840ce3515dddfde36288575f203c52840f Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 31 Jan 2021 22:36:46 +0300 Subject: [#3213] ActivityPub: temporarily reverted to previous hashtags filtering implementation due to blank results issue. --- lib/pleroma/web/activity_pub/activity_pub.ex | 106 ++++++++++++++------------- 1 file changed, 54 insertions(+), 52 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index cda8d3f54..fd0144aad 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -722,24 +722,53 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_embedded_tag_reject_any(query, _), do: query + # Groups by all bindings to allow aggregation on hashtags + defp group_by_all_bindings(query) do + # Expecting named bindings: :object, :bookmark, :thread_mute, :report_note + cond do + Enum.count(query.aliases) == 4 -> + from([a, o, b3, b4, b5] in query, group_by: [a.id, o.id, b3.id, b4.id, b5.id]) + + Enum.count(query.aliases) == 3 -> + from([a, o, b3, b4] in query, group_by: [a.id, o.id, b3.id, b4.id]) + + Enum.count(query.aliases) == 2 -> + from([a, o, b3] in query, group_by: [a.id, o.id, b3.id]) + + true -> + from([a, o] in query, group_by: [a.id, o.id]) + end + end + + defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do + raise_on_missing_preload() + end + + defp restrict_hashtag_reject_any(query, %{tag_reject: tags_reject}) when is_list(tags_reject) do + query + |> group_by_all_bindings() + |> join(:left, [_activity, object], hashtag in assoc(object, :hashtags), as: :hashtag) + |> having( + [hashtag: hashtag], + fragment("not(array_agg(?) && (?))", hashtag.name, ^tags_reject) + ) + end + + defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do + restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]}) + end + + defp restrict_hashtag_reject_any(query, _), do: query + defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do raise_on_missing_preload() end defp restrict_hashtag_all(query, %{tag_all: tags}) when is_list(tags) do - from( - [_activity, object] in query, - where: - fragment( - """ - (SELECT array_agg(hashtags.name) FROM hashtags JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) - AND hashtags_objects.object_id = ?) @> ? - """, - ^tags, - object.id, - ^tags - ) + Enum.reduce( + tags, + query, + fn tag, acc -> restrict_hashtag_any(acc, %{tag: tag}) end ) end @@ -754,19 +783,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_hashtag_any(query, %{tag: tags}) when is_list(tags) do - from( - [_activity, object] in query, - where: - fragment( - """ - EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) - AND hashtags_objects.object_id = ? LIMIT 1) - """, - ^tags, - object.id - ) - ) + query = + from( + [_activity, object] in query, + join: hashtag in assoc(object, :hashtags), + where: hashtag.name in ^tags + ) + + if length(tags) > 1 do + distinct(query, [activity], true) + else + query + end end defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do @@ -775,32 +803,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_hashtag_any(query, _), do: query - defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do - raise_on_missing_preload() - end - - defp restrict_hashtag_reject_any(query, %{tag_reject: tags_reject}) when is_list(tags_reject) do - from( - [_activity, object] in query, - where: - fragment( - """ - NOT EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) - AND hashtags_objects.object_id = ? LIMIT 1) - """, - ^tags_reject, - object.id - ) - ) - end - - defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do - restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]}) - end - - defp restrict_hashtag_reject_any(query, _), do: query - defp raise_on_missing_preload do raise "Can't use the child object without preloading!" end -- cgit v1.2.3 From cf4765af4098098fa4d6996193432bd19c439a75 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 31 Jan 2021 23:06:38 +0300 Subject: [#3213] ActivityPub: fixed subquery-based hashtags filtering implementation (addressed empty list options issue). Added regression test. --- lib/pleroma/web/activity_pub/activity_pub.ex | 115 +++++++++++++-------------- 1 file changed, 56 insertions(+), 59 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index fd0144aad..6cf4093fb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -673,7 +673,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do raise_on_missing_preload() end - defp restrict_embedded_tag_all(query, %{tag_all: tag_all}) when is_list(tag_all) do + defp restrict_embedded_tag_all(query, %{tag_all: [_ | _] = tag_all}) do from( [_activity, object] in query, where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) @@ -690,7 +690,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do raise_on_missing_preload() end - defp restrict_embedded_tag_any(query, %{tag: tag}) when is_list(tag) do + defp restrict_embedded_tag_any(query, %{tag: [_ | _] = tag}) do from( [_activity, object] in query, where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag) @@ -707,8 +707,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do raise_on_missing_preload() end - defp restrict_embedded_tag_reject_any(query, %{tag_reject: tag_reject}) - when is_list(tag_reject) do + defp restrict_embedded_tag_reject_any(query, %{tag_reject: [_ | _] = tag_reject}) do from( [_activity, object] in query, where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject) @@ -722,86 +721,84 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_embedded_tag_reject_any(query, _), do: query - # Groups by all bindings to allow aggregation on hashtags - defp group_by_all_bindings(query) do - # Expecting named bindings: :object, :bookmark, :thread_mute, :report_note - cond do - Enum.count(query.aliases) == 4 -> - from([a, o, b3, b4, b5] in query, group_by: [a.id, o.id, b3.id, b4.id, b5.id]) - - Enum.count(query.aliases) == 3 -> - from([a, o, b3, b4] in query, group_by: [a.id, o.id, b3.id, b4.id]) - - Enum.count(query.aliases) == 2 -> - from([a, o, b3] in query, group_by: [a.id, o.id, b3.id]) - - true -> - from([a, o] in query, group_by: [a.id, o.id]) - end - end - - defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do + defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do raise_on_missing_preload() end - defp restrict_hashtag_reject_any(query, %{tag_reject: tags_reject}) when is_list(tags_reject) do - query - |> group_by_all_bindings() - |> join(:left, [_activity, object], hashtag in assoc(object, :hashtags), as: :hashtag) - |> having( - [hashtag: hashtag], - fragment("not(array_agg(?) && (?))", hashtag.name, ^tags_reject) + defp restrict_hashtag_all(query, %{tag_all: [_ | _] = tags}) do + from( + [_activity, object] in query, + where: + fragment( + """ + (SELECT array_agg(hashtags.name) FROM hashtags JOIN hashtags_objects + ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) + AND hashtags_objects.object_id = ?) @> ? + """, + ^tags, + object.id, + ^tags + ) ) end - defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do - restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]}) + defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do + restrict_hashtag_any(query, %{tag: tag}) end - defp restrict_hashtag_reject_any(query, _), do: query + defp restrict_hashtag_all(query, _), do: query - defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do + defp restrict_hashtag_any(_query, %{tag: _tag, skip_preload: true}) do raise_on_missing_preload() end - defp restrict_hashtag_all(query, %{tag_all: tags}) when is_list(tags) do - Enum.reduce( - tags, - query, - fn tag, acc -> restrict_hashtag_any(acc, %{tag: tag}) end + defp restrict_hashtag_any(query, %{tag: [_ | _] = tags}) do + from( + [_activity, object] in query, + where: + fragment( + """ + EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects + ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) + AND hashtags_objects.object_id = ? LIMIT 1) + """, + ^tags, + object.id + ) ) end - defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do - restrict_hashtag_any(query, %{tag: tag}) + defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do + restrict_hashtag_any(query, %{tag: [tag]}) end - defp restrict_hashtag_all(query, _), do: query + defp restrict_hashtag_any(query, _), do: query - defp restrict_hashtag_any(_query, %{tag: _tag, skip_preload: true}) do + defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do raise_on_missing_preload() end - defp restrict_hashtag_any(query, %{tag: tags}) when is_list(tags) do - query = - from( - [_activity, object] in query, - join: hashtag in assoc(object, :hashtags), - where: hashtag.name in ^tags - ) - - if length(tags) > 1 do - distinct(query, [activity], true) - else - query - end + defp restrict_hashtag_reject_any(query, %{tag_reject: [_ | _] = tags_reject}) do + from( + [_activity, object] in query, + where: + fragment( + """ + NOT EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects + ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) + AND hashtags_objects.object_id = ? LIMIT 1) + """, + ^tags_reject, + object.id + ) + ) end - defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do - restrict_hashtag_any(query, %{tag: [tag]}) + defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do + restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]}) end - defp restrict_hashtag_any(query, _), do: query + defp restrict_hashtag_reject_any(query, _), do: query defp raise_on_missing_preload do raise "Can't use the child object without preloading!" -- cgit v1.2.3 From d1c6dd97aa503ca7c897d67d98fe8c924e113a61 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 7 Feb 2021 22:24:12 +0300 Subject: [#3213] Partially addressed code review points. migration rollback task changes, hashtags-related config handling tweaks, `hashtags.data` deletion (unused). --- lib/mix/tasks/pleroma/database.ex | 53 +++++++++++++----------- lib/pleroma/config.ex | 3 -- lib/pleroma/hashtag.ex | 5 +-- lib/pleroma/migrators/hashtags_table_migrator.ex | 4 +- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 5 files changed, 33 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 30c0d2bf1..7c4f54141 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -20,30 +20,6 @@ defmodule Mix.Tasks.Pleroma.Database do @shortdoc "A collection of database related tasks" @moduledoc File.read!("docs/administration/CLI_tasks/database.md") - # Rolls back a specific migration (leaving subsequent migrations applied) - # Based on https://stackoverflow.com/a/53825840 - def run(["rollback", version]) do - start_pleroma() - - version = String.to_integer(version) - re = ~r/^#{version}_.*\.exs/ - path = Application.app_dir(:pleroma, Path.join(["priv", "repo", "migrations"])) - - result = - with {:find, "" <> file} <- {:find, Enum.find(File.ls!(path), &String.match?(&1, re))}, - {:compile, [{mod, _} | _]} <- {:compile, Code.compile_file(Path.join(path, file))}, - {:rollback, :ok} <- {:rollback, Ecto.Migrator.down(Repo, version, mod)} do - {:ok, "Reversed migration: #{file}"} - else - {:find, _} -> {:error, "No migration found with version prefix: #{version}"} - {:compile, e} -> {:error, "Problem compiling migration module: #{inspect(e)}"} - {:rollback, e} -> {:error, "Problem reversing migration: #{inspect(e)}"} - e -> {:error, "Something unexpected happened: #{inspect(e)}"} - end - - IO.inspect(result) - end - def run(["remove_embedded_objects" | args]) do {options, [], []} = OptionParser.parse( @@ -194,4 +170,33 @@ defmodule Mix.Tasks.Pleroma.Database do end) |> Stream.run() end + + # Rolls back a specific migration (leaving subsequent migrations applied). + # WARNING: imposes a risk of unrecoverable data loss — proceed at your own responsibility. + # Based on https://stackoverflow.com/a/53825840 + def run(["rollback", version]) do + prompt = "SEVERE WARNING: this operation may result in unrecoverable data loss. Continue?" + + if shell_prompt(prompt, "n") in ~w(Yn Y y) do + {_, result, _} = + Ecto.Migrator.with_repo(Pleroma.Repo, fn repo -> + version = String.to_integer(version) + re = ~r/^#{version}_.*\.exs/ + path = Ecto.Migrator.migrations_path(repo) + + with {:find, "" <> file} <- {:find, Enum.find(File.ls!(path), &String.match?(&1, re))}, + {:compile, [{mod, _} | _]} <- {:compile, Code.compile_file(Path.join(path, file))}, + {:rollback, :ok} <- {:rollback, Ecto.Migrator.down(repo, version, mod)} do + {:ok, "Reversed migration: #{file}"} + else + {:find, _} -> {:error, "No migration found with version prefix: #{version}"} + {:compile, e} -> {:error, "Problem compiling migration module: #{inspect(e)}"} + {:rollback, e} -> {:error, "Problem reversing migration: #{inspect(e)}"} + e -> {:error, "Something unexpected happened: #{inspect(e)}"} + end + end) + + IO.inspect(result) + end + end end diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index 0a6ac0ad0..f17e14128 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -96,9 +96,6 @@ defmodule Pleroma.Config do end end - def improved_hashtag_timeline_path, do: [:instance, :improved_hashtag_timeline] - def improved_hashtag_timeline, do: get(improved_hashtag_timeline_path()) - def oauth_consumer_strategies, do: get([:auth, :oauth_consumer_strategies], []) def oauth_consumer_enabled?, do: oauth_consumer_strategies() != [] diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index b05927563..9e4c6c894 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -10,11 +10,8 @@ defmodule Pleroma.Hashtag do alias Pleroma.Hashtag alias Pleroma.Repo - @derive {Jason.Encoder, only: [:data]} - schema "hashtags" do field(:name, :string) - field(:data, :map, default: %{}) many_to_many(:objects, Pleroma.Object, join_through: "hashtags_objects", on_replace: :delete) @@ -50,7 +47,7 @@ defmodule Pleroma.Hashtag do def changeset(%Hashtag{} = struct, params) do struct - |> cast(params, [:name, :data]) + |> cast(params, [:name]) |> update_change(:name, &String.downcase/1) |> validate_required([:name]) |> unique_constraint(:name) diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index 07b42a7f4..9a036e0b2 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -239,11 +239,11 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do data_migration.feature_lock -> :noop - not is_nil(Config.improved_hashtag_timeline()) -> + not is_nil(Config.get([:database, :improved_hashtag_timeline])) -> :noop true -> - Config.put(Config.improved_hashtag_timeline_path(), true) + Config.put([:database, :improved_hashtag_timeline], true) :ok end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 573b4243c..7ac18e5c5 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1227,7 +1227,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> exclude_invisible_actors(opts) |> exclude_visibility(opts) - if Config.improved_hashtag_timeline() do + if Config.get([:database, :improved_hashtag_timeline]) do query |> restrict_hashtag_any(opts) |> restrict_hashtag_all(opts) -- cgit v1.2.3 From a996ab46a54acbfa7a19da3eae12c78ed6466a1a Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 11 Feb 2021 19:30:21 +0300 Subject: [#3213] Reorganized hashtags cleanup. Transaction-wrapped Hashtag.get_or_create_by_names/1. Misc. improvements. --- lib/pleroma/hashtag.ex | 60 +++++++++++++++---- lib/pleroma/migrators/hashtags_table_migrator.ex | 70 +++++++++++++++------- lib/pleroma/object.ex | 27 +++++---- .../workers/cron/hashtags_cleanup_worker.ex | 57 ------------------ 4 files changed, 114 insertions(+), 100 deletions(-) delete mode 100644 lib/pleroma/workers/cron/hashtags_cleanup_worker.ex (limited to 'lib') diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index 9e4c6c894..de52c4dae 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -6,14 +6,17 @@ defmodule Pleroma.Hashtag do use Ecto.Schema import Ecto.Changeset + import Ecto.Query + alias Ecto.Multi alias Pleroma.Hashtag + alias Pleroma.Object alias Pleroma.Repo schema "hashtags" do field(:name, :string) - many_to_many(:objects, Pleroma.Object, join_through: "hashtags_objects", on_replace: :delete) + many_to_many(:objects, Object, join_through: "hashtags_objects", on_replace: :delete) timestamps() end @@ -34,15 +37,27 @@ defmodule Pleroma.Hashtag do end def get_or_create_by_names(names) when is_list(names) do - Enum.reduce_while(names, {:ok, []}, fn name, {:ok, list} -> - case get_or_create_by_name(name) do - {:ok, %Hashtag{} = hashtag} -> - {:cont, {:ok, list ++ [hashtag]}} - - error -> - {:halt, error} - end - end) + timestamp = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + + structs = + Enum.map(names, fn name -> + %Hashtag{} + |> changeset(%{name: name}) + |> Map.get(:changes) + |> Map.merge(%{inserted_at: timestamp, updated_at: timestamp}) + end) + + with {:ok, %{query_op: hashtags}} <- + Multi.new() + |> Multi.insert_all(:insert_all_op, Hashtag, structs, on_conflict: :nothing) + |> Multi.run(:query_op, fn _repo, _changes -> + {:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))} + end) + |> Repo.transaction() do + {:ok, hashtags} + else + {:error, _name, value, _changes_so_far} -> {:error, value} + end end def changeset(%Hashtag{} = struct, params) do @@ -52,4 +67,29 @@ defmodule Pleroma.Hashtag do |> validate_required([:name]) |> unique_constraint(:name) end + + def unlink(%Object{id: object_id}) do + with {_, hashtag_ids} <- + from(hto in "hashtags_objects", + where: hto.object_id == ^object_id, + select: hto.hashtag_id + ) + |> Repo.delete_all() do + delete_unreferenced(hashtag_ids) + end + end + + @delete_unreferenced_query """ + DELETE FROM hashtags WHERE id IN + (SELECT hashtags.id FROM hashtags + LEFT OUTER JOIN hashtags_objects + ON hashtags_objects.hashtag_id = hashtags.id + WHERE hashtags_objects.hashtag_id IS NULL AND hashtags.id = ANY($1)); + """ + + def delete_unreferenced(ids) do + with {:ok, %{num_rows: deleted_count}} <- Repo.query(@delete_unreferenced_query, [ids]) do + {:ok, deleted_count} + end + end end diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index 9a036e0b2..c53f6be12 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -74,16 +74,15 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do def handle_info(:migrate_hashtags, state) do State.clear() - data_migration = data_migration() + update_status(:running) + put_stat(:started_at, NaiveDateTime.utc_now()) + data_migration = data_migration() persistent_data = Map.take(data_migration.data, ["max_processed_id"]) {:ok, data_migration} = DataMigration.update(data_migration, %{state: :running, data: persistent_data}) - update_status(:running) - put_stat(:started_at, NaiveDateTime.utc_now()) - Logger.info("Starting transferring object embedded hashtags to `hashtags` table...") max_processed_id = data_migration.data["max_processed_id"] || 0 @@ -137,6 +136,8 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do |> Stream.run() with 0 <- failures_count(data_migration.id) do + _ = delete_non_create_activities_hashtags() + {:ok, data_migration} = DataMigration.update_state(data_migration, :complete) handle_success(data_migration) @@ -150,9 +151,37 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do {:noreply, state} end + @hashtags_objects_cleanup_query """ + DELETE FROM hashtags_objects WHERE object_id IN + (SELECT DISTINCT objects.id FROM objects + JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities + ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = + (objects.data->>'id') + AND activities.data->>'type' = 'Create' + WHERE activities.id IS NULL); + """ + + @hashtags_cleanup_query """ + DELETE FROM hashtags WHERE id IN + (SELECT hashtags.id FROM hashtags + LEFT OUTER JOIN hashtags_objects + ON hashtags_objects.hashtag_id = hashtags.id + WHERE hashtags_objects.hashtag_id IS NULL); + """ + + def delete_non_create_activities_hashtags do + {:ok, %{num_rows: hashtags_objects_count}} = + Repo.query(@hashtags_objects_cleanup_query, [], timeout: :infinity) + + {:ok, %{num_rows: hashtags_count}} = + Repo.query(@hashtags_cleanup_query, [], timeout: :infinity) + + {:ok, hashtags_objects_count, hashtags_count} + end + defp query do # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out) - # Note: not checking activity type; HashtagsCleanupWorker should clean up unused records later + # Note: not checking activity type, expecting remove_non_create_objects_hashtags/_ to clean up from( object in Object, where: @@ -182,25 +211,20 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do defp transfer_object_hashtags(object, hashtags) do Repo.transaction(fn -> with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do - for hashtag_record <- hashtag_records do - with {:ok, _} <- - Repo.query( - "insert into hashtags_objects(hashtag_id, object_id) values ($1, $2);", - [hashtag_record.id, object.id] - ) do - nil - else - {:error, e} -> - error = - "ERROR: could not link object #{object.id} and hashtag " <> - "#{hashtag_record.id}: #{inspect(e)}" - - Logger.error(error) - Repo.rollback(object.id) - end + maps = Enum.map(hashtag_records, &%{hashtag_id: &1.id, object_id: object.id}) + expected_rows = length(hashtag_records) + + with {^expected_rows, _} <- Repo.insert_all("hashtags_objects", maps) do + object.id + else + e -> + error = + "ERROR when inserting #{expected_rows} hashtags_objects " <> + "for object #{object.id}: #{inspect(e)}" + + Logger.error(error) + Repo.rollback(object.id) end - - object.id else e -> error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}" diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 52b77e41c..3ba749d1a 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -62,27 +62,30 @@ defmodule Pleroma.Object do |> cast(params, [:data]) |> validate_required([:data]) |> unique_constraint(:ap_id, name: :objects_unique_apid_index) + # Expecting `maybe_handle_hashtags_change/1` to run last: |> maybe_handle_hashtags_change(struct) end - # Note: not checking activity type; HashtagsCleanupWorker should clean up unused records later + # Note: not checking activity type (assuming non-legacy objects are associated with Create act.) defp maybe_handle_hashtags_change(changeset, struct) do - with data_hashtags_change = get_change(changeset, :data), - true <- hashtags_changed?(struct, data_hashtags_change), + with %Ecto.Changeset{valid?: true} <- changeset, + data_hashtags_change = get_change(changeset, :data), + {_, true} <- {:changed, hashtags_changed?(struct, data_hashtags_change)}, {:ok, hashtag_records} <- data_hashtags_change |> object_data_hashtags() |> Hashtag.get_or_create_by_names() do put_assoc(changeset, :hashtags, hashtag_records) else - false -> + %{valid?: false} -> changeset - {:error, hashtag_changeset} -> - failed_hashtag = get_field(hashtag_changeset, :name) + {:changed, false} -> + changeset + {:error, _} -> validate_change(changeset, :data, fn _, _ -> - [data: "error referencing hashtag: #{failed_hashtag}"] + [data: "error referencing hashtags"] end) end end @@ -221,9 +224,13 @@ defmodule Pleroma.Object do def swap_object_with_tombstone(object) do tombstone = make_tombstone(object) - object - |> Object.change(%{data: tombstone}) - |> Repo.update() + with {:ok, object} <- + object + |> Object.change(%{data: tombstone}) + |> Repo.update() do + Hashtag.unlink(object) + {:ok, object} + end end def delete(%Object{data: %{"id" => id}} = object) do diff --git a/lib/pleroma/workers/cron/hashtags_cleanup_worker.ex b/lib/pleroma/workers/cron/hashtags_cleanup_worker.ex deleted file mode 100644 index b319067ca..000000000 --- a/lib/pleroma/workers/cron/hashtags_cleanup_worker.ex +++ /dev/null @@ -1,57 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.HashtagsCleanupWorker do - @moduledoc """ - The worker to clean up unused hashtags_objects and hashtags. - """ - - use Oban.Worker, queue: "hashtags_cleanup" - - alias Pleroma.Repo - - require Logger - - @hashtags_objects_query """ - DELETE FROM hashtags_objects WHERE object_id IN - (SELECT DISTINCT objects.id FROM objects - JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities - ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = - (objects.data->>'id') - AND activities.data->>'type' = 'Create' - WHERE activities.id IS NULL); - """ - - @hashtags_query """ - DELETE FROM hashtags WHERE id IN - (SELECT hashtags.id FROM hashtags - LEFT OUTER JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id - WHERE hashtags_objects.hashtag_id IS NULL AND hashtags.inserted_at < $1); - """ - - @impl Oban.Worker - def perform(_job) do - Logger.info("Cleaning up unused `hashtags_objects` records...") - - {:ok, %{num_rows: hashtags_objects_count}} = - Repo.query(@hashtags_objects_query, [], timeout: :infinity) - - Logger.info("Deleted #{hashtags_objects_count} unused `hashtags_objects` records.") - - Logger.info("Cleaning up unused `hashtags` records...") - - # Note: ignoring recently created hashtags since references are added after hashtag is created - {:ok, %{num_rows: hashtags_count}} = - Repo.query(@hashtags_query, [NaiveDateTime.add(NaiveDateTime.utc_now(), -3600 * 24)], - timeout: :infinity - ) - - Logger.info("Deleted #{hashtags_count} unused `hashtags` records.") - - Logger.info("HashtagsCleanupWorker complete.") - - :ok - end -end -- cgit v1.2.3 From 349b8b0f4fb1c2b86f913e1840f15c052ff43c24 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 13 Feb 2021 22:01:11 +0300 Subject: [#3213] `rescue` around potentially-raising `Repo.insert_all/_` calls. Misc. improvements (docs etc.). --- lib/pleroma/hashtag.ex | 29 ++++++++++++++---------- lib/pleroma/migrators/hashtags_table_migrator.ex | 21 ++++++++++------- 2 files changed, 30 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index de52c4dae..0d6a4d09e 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -47,16 +47,20 @@ defmodule Pleroma.Hashtag do |> Map.merge(%{inserted_at: timestamp, updated_at: timestamp}) end) - with {:ok, %{query_op: hashtags}} <- - Multi.new() - |> Multi.insert_all(:insert_all_op, Hashtag, structs, on_conflict: :nothing) - |> Multi.run(:query_op, fn _repo, _changes -> - {:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))} - end) - |> Repo.transaction() do - {:ok, hashtags} - else - {:error, _name, value, _changes_so_far} -> {:error, value} + try do + with {:ok, %{query_op: hashtags}} <- + Multi.new() + |> Multi.insert_all(:insert_all_op, Hashtag, structs, on_conflict: :nothing) + |> Multi.run(:query_op, fn _repo, _changes -> + {:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))} + end) + |> Repo.transaction() do + {:ok, hashtags} + else + {:error, _name, value, _changes_so_far} -> {:error, value} + end + rescue + e -> {:error, e} end end @@ -74,8 +78,9 @@ defmodule Pleroma.Hashtag do where: hto.object_id == ^object_id, select: hto.hashtag_id ) - |> Repo.delete_all() do - delete_unreferenced(hashtag_ids) + |> Repo.delete_all(), + {:ok, unreferenced_count} <- delete_unreferenced(hashtag_ids) do + {:ok, length(hashtag_ids), unreferenced_count} end end diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index c53f6be12..432c3401a 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -214,15 +214,20 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do maps = Enum.map(hashtag_records, &%{hashtag_id: &1.id, object_id: object.id}) expected_rows = length(hashtag_records) - with {^expected_rows, _} <- Repo.insert_all("hashtags_objects", maps) do - object.id - else + base_error = + "ERROR when inserting #{expected_rows} hashtags_objects for obj. #{object.id}" + + try do + with {^expected_rows, _} <- Repo.insert_all("hashtags_objects", maps) do + object.id + else + e -> + Logger.error("#{base_error}: #{inspect(e)}") + Repo.rollback(object.id) + end + rescue e -> - error = - "ERROR when inserting #{expected_rows} hashtags_objects " <> - "for object #{object.id}: #{inspect(e)}" - - Logger.error(error) + Logger.error("#{base_error}: #{inspect(e)}") Repo.rollback(object.id) end else -- cgit v1.2.3 From 1dac7d14623f36744953a523650211540d90d1fc Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 15 Feb 2021 21:13:14 +0300 Subject: [#3213] Fixed `hashtags.name` lookup (must use `citext` type to do index scan). Fixed embedded hashtags lookup (lowercasing), adjusted tests. --- lib/pleroma/hashtag.ex | 8 ++++++-- lib/pleroma/web/activity_pub/activity_pub.ex | 22 ++++++++++++++++------ 2 files changed, 22 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index 0d6a4d09e..a6d033816 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -22,7 +22,9 @@ defmodule Pleroma.Hashtag do end def get_by_name(name) do - Repo.get_by(Hashtag, name: name) + from(h in Hashtag) + |> where([h], fragment("name = ?::citext", ^String.downcase(name))) + |> Repo.one() end def get_or_create_by_name(name) when is_bitstring(name) do @@ -37,6 +39,7 @@ defmodule Pleroma.Hashtag do end def get_or_create_by_names(names) when is_list(names) do + names = Enum.map(names, &String.downcase/1) timestamp = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) structs = @@ -52,7 +55,8 @@ defmodule Pleroma.Hashtag do Multi.new() |> Multi.insert_all(:insert_all_op, Hashtag, structs, on_conflict: :nothing) |> Multi.run(:query_op, fn _repo, _changes -> - {:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))} + {:ok, + Repo.all(from(ht in Hashtag, where: ht.name in fragment("?::citext[]", ^names)))} end) |> Repo.transaction() do {:ok, hashtags} diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9623e635a..e012f2779 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -698,6 +698,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_embedded_tag_all(query, %{tag_all: [_ | _] = tag_all}) do + tag_all = Enum.map(tag_all, &String.downcase/1) + from( [_activity, object] in query, where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) @@ -714,10 +716,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do raise_on_missing_preload() end - defp restrict_embedded_tag_any(query, %{tag: [_ | _] = tag}) do + defp restrict_embedded_tag_any(query, %{tag: [_ | _] = tag_any}) do + tag_any = Enum.map(tag_any, &String.downcase/1) + from( [_activity, object] in query, - where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag) + where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag_any) ) end @@ -732,6 +736,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_embedded_tag_reject_any(query, %{tag_reject: [_ | _] = tag_reject}) do + tag_reject = Enum.map(tag_reject, &String.downcase/1) + from( [_activity, object] in query, where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject) @@ -749,6 +755,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do raise_on_missing_preload() end + defp restrict_hashtag_all(query, %{tag_all: [single_tag]}) do + restrict_hashtag_any(query, %{tag: single_tag}) + end + defp restrict_hashtag_all(query, %{tag_all: [_ | _] = tags}) do from( [_activity, object] in query, @@ -756,7 +766,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do fragment( """ (SELECT array_agg(hashtags.name) FROM hashtags JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) + ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?::citext[]) AND hashtags_objects.object_id = ?) @> ? """, ^tags, @@ -767,7 +777,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do - restrict_hashtag_any(query, %{tag: tag}) + restrict_hashtag_all(query, %{tag_all: [tag]}) end defp restrict_hashtag_all(query, _), do: query @@ -783,7 +793,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do fragment( """ EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) + ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?::citext[]) AND hashtags_objects.object_id = ? LIMIT 1) """, ^tags, @@ -809,7 +819,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do fragment( """ NOT EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) + ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?::citext[]) AND hashtags_objects.object_id = ? LIMIT 1) """, ^tags_reject, -- cgit v1.2.3 From 938823c73040f6b55896581daf5baf732f859f02 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 16 Feb 2021 23:14:15 +0300 Subject: [#3213] HashtagsTableMigrator state management refactoring & improvements (proper stats serialization etc.). --- lib/pleroma/data_migration.ex | 15 ++-- lib/pleroma/migrators/hashtags_table_migrator.ex | 87 ++++++++++------------ .../migrators/hashtags_table_migrator/state.ex | 87 +++++++++++++++++++--- 3 files changed, 122 insertions(+), 67 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/data_migration.ex b/lib/pleroma/data_migration.ex index 64fa155ff..1377af16e 100644 --- a/lib/pleroma/data_migration.ex +++ b/lib/pleroma/data_migration.ex @@ -10,6 +10,7 @@ defmodule Pleroma.DataMigration do alias Pleroma.Repo import Ecto.Changeset + import Ecto.Query schema "data_migrations" do field(:name, :string) @@ -28,14 +29,12 @@ defmodule Pleroma.DataMigration do |> unique_constraint(:name) end - def update(data_migration, params \\ %{}) do - data_migration - |> changeset(params) - |> Repo.update() - end - - def update_state(data_migration, new_state) do - update(data_migration, %{state: new_state}) + def update_one_by_id(id, params \\ %{}) do + with {1, _} <- + from(dm in DataMigration, where: dm.id == ^id) + |> Repo.update_all(set: params) do + :ok + end end def get_by_name(name) do diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index 432c3401a..a226d9d29 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -11,16 +11,16 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do alias __MODULE__.State alias Pleroma.Config - alias Pleroma.DataMigration alias Pleroma.Hashtag alias Pleroma.Object alias Pleroma.Repo - defdelegate state(), to: State, as: :get - defdelegate put_stat(key, value), to: State, as: :put - defdelegate increment_stat(key, increment), to: State, as: :increment + defdelegate data_migration(), to: State - defdelegate data_migration(), to: DataMigration, as: :populate_hashtags_table + defdelegate state(), to: State + defdelegate get_stat(key, value), to: State, as: :get_data_key + defdelegate put_stat(key, value), to: State, as: :put_data_key + defdelegate increment_stat(key, increment), to: State, as: :increment_data_key @reg_name {:global, __MODULE__} @@ -45,7 +45,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do def handle_continue(:init_state, _state) do {:ok, _} = State.start_link(nil) - update_status(:init) + update_status(:pending) data_migration = data_migration() manual_migrations = Config.get([:instance, :manual_data_migrations], []) @@ -55,13 +55,13 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do update_status(:noop) is_nil(data_migration) -> - update_status(:halt, "Data migration does not exist.") + update_status(:failed, "Data migration does not exist.") data_migration.state == :manual or data_migration.name in manual_migrations -> - update_status(:noop, "Data migration is in manual execution state.") + update_status(:manual, "Data migration is in manual execution state.") data_migration.state == :complete -> - handle_success(data_migration) + on_complete(data_migration) true -> send(self(), :migrate_hashtags) @@ -72,20 +72,15 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do @impl true def handle_info(:migrate_hashtags, state) do - State.clear() + State.reinit() update_status(:running) put_stat(:started_at, NaiveDateTime.utc_now()) - data_migration = data_migration() - persistent_data = Map.take(data_migration.data, ["max_processed_id"]) - - {:ok, data_migration} = - DataMigration.update(data_migration, %{state: :running, data: persistent_data}) - - Logger.info("Starting transferring object embedded hashtags to `hashtags` table...") + %{id: data_migration_id} = data_migration() + max_processed_id = get_stat(:max_processed_id, 0) - max_processed_id = data_migration.data["max_processed_id"] || 0 + Logger.info("Transferring embedded hashtags to `hashtags` (from oid: #{max_processed_id})...") query() |> where([object], object.id > ^max_processed_id) @@ -104,7 +99,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do Repo.query( "INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <> "VALUES ($1, $2) ON CONFLICT DO NOTHING;", - [data_migration.id, failed_id] + [data_migration_id, failed_id] ) end @@ -112,7 +107,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do Repo.query( "DELETE FROM data_migration_failed_ids " <> "WHERE data_migration_id = $1 AND record_id = ANY($2)", - [data_migration.id, object_ids -- failed_ids] + [data_migration_id, object_ids -- failed_ids] ) max_object_id = Enum.at(object_ids, -1) @@ -120,14 +115,8 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do put_stat(:max_processed_id, max_object_id) increment_stat(:processed_count, length(object_ids)) increment_stat(:failed_count, length(failed_ids)) - - put_stat( - :records_per_second, - state()[:processed_count] / - Enum.max([NaiveDateTime.diff(NaiveDateTime.utc_now(), state()[:started_at]), 1]) - ) - - persist_stats(data_migration) + put_stat(:records_per_second, records_per_second()) + _ = State.persist_to_db() # A quick and dirty approach to controlling the load this background migration imposes sleep_interval = Config.get([:populate_hashtags_table, :sleep_interval_ms], 0) @@ -135,22 +124,25 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do end) |> Stream.run() - with 0 <- failures_count(data_migration.id) do + with 0 <- failures_count(data_migration_id) do _ = delete_non_create_activities_hashtags() - - {:ok, data_migration} = DataMigration.update_state(data_migration, :complete) - - handle_success(data_migration) + set_complete() else _ -> - _ = DataMigration.update_state(data_migration, :failed) - update_status(:failed, "Please check data_migration_failed_ids records.") end {:noreply, state} end + defp records_per_second do + get_stat(:processed_count, 0) / Enum.max([running_time(), 1]) + end + + defp running_time do + NaiveDateTime.diff(NaiveDateTime.utc_now(), get_stat(:started_at, NaiveDateTime.utc_now())) + end + @hashtags_objects_cleanup_query """ DELETE FROM hashtags_objects WHERE object_id IN (SELECT DISTINCT objects.id FROM objects @@ -169,6 +161,10 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do WHERE hashtags_objects.hashtag_id IS NULL); """ + @doc """ + Deletes `hashtags_objects` for legacy objects not asoociated with Create activity. + Also deletes unreferenced `hashtags` records (might occur after deletion of `hashtags_objects`). + """ def delete_non_create_activities_hashtags do {:ok, %{num_rows: hashtags_objects_count}} = Repo.query(@hashtags_objects_cleanup_query, [], timeout: :infinity) @@ -256,14 +252,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do end end - defp persist_stats(data_migration) do - runner_state = Map.drop(state(), [:status]) - _ = DataMigration.update(data_migration, %{data: runner_state}) - end - - defp handle_success(data_migration) do - update_status(:complete) - + defp on_complete(data_migration) do cond do data_migration.feature_lock -> :noop @@ -321,18 +310,18 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do end def force_restart do - {:ok, _} = DataMigration.update(data_migration(), %{state: :pending, data: %{}}) + :ok = State.reset() force_continue() end - def force_complete do - {:ok, data_migration} = DataMigration.update_state(data_migration(), :complete) - - handle_success(data_migration) + def set_complete do + update_status(:complete) + _ = State.persist_to_db() + on_complete(data_migration()) end defp update_status(status, message \\ nil) do - put_stat(:status, status) + put_stat(:state, status) put_stat(:message, message) end end diff --git a/lib/pleroma/migrators/hashtags_table_migrator/state.ex b/lib/pleroma/migrators/hashtags_table_migrator/state.ex index 901563426..ed9848824 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator/state.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator/state.ex @@ -5,31 +5,98 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator.State do use Agent - @init_state %{} + alias Pleroma.DataMigration + + defdelegate data_migration(), to: DataMigration, as: :populate_hashtags_table + @reg_name {:global, __MODULE__} def start_link(_) do - Agent.start_link(fn -> @init_state end, name: @reg_name) + Agent.start_link(fn -> load_state_from_db() end, name: @reg_name) + end + + defp load_state_from_db do + data_migration = data_migration() + + data = + if data_migration do + Map.new(data_migration.data, fn {k, v} -> {String.to_atom(k), v} end) + else + %{} + end + + %{ + data_migration_id: data_migration && data_migration.id, + data: data + } end - def clear do - Agent.update(@reg_name, fn _state -> @init_state end) + def persist_to_db do + %{data_migration_id: data_migration_id, data: data} = state() + + if data_migration_id do + DataMigration.update_one_by_id(data_migration_id, data: data) + else + {:error, :nil_data_migration_id} + end + end + + def reset do + %{data_migration_id: data_migration_id} = state() + + with false <- is_nil(data_migration_id), + :ok <- + DataMigration.update_one_by_id(data_migration_id, + state: :pending, + data: %{} + ) do + reinit() + else + true -> {:error, :nil_data_migration_id} + e -> e + end end - def get do + def reinit do + Agent.update(@reg_name, fn _state -> load_state_from_db() end) + end + + def state do Agent.get(@reg_name, & &1) end - def put(key, value) do + def get_data_key(key, default \\ nil) do + get_in(state(), [:data, key]) || default + end + + def put_data_key(key, value) do + _ = persist_non_data_change(key, value) + Agent.update(@reg_name, fn state -> - Map.put(state, key, value) + put_in(state, [:data, key], value) end) end - def increment(key, increment \\ 1) do + def increment_data_key(key, increment \\ 1) do Agent.update(@reg_name, fn state -> - updated_value = (state[key] || 0) + increment - Map.put(state, key, updated_value) + initial_value = get_in(state, [:data, key]) || 0 + updated_value = initial_value + increment + put_in(state, [:data, key], updated_value) end) end + + defp persist_non_data_change(:state, value) do + with true <- get_data_key(:state) != value, + true <- value in Pleroma.DataMigration.State.__valid_values__(), + %{data_migration_id: data_migration_id} when not is_nil(data_migration_id) <- state() do + DataMigration.update_one_by_id(data_migration_id, state: value) + else + false -> :ok + _ -> {:error, :nil_data_migration_id} + end + end + + defp persist_non_data_change(_, _) do + nil + end end -- cgit v1.2.3 From 854ea1aefb5ff4e03e9e9af6e8dd50f66c61c913 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 17 Feb 2021 09:23:35 +0300 Subject: [#3213] Fixed `HashtagsTableMigrator.count/1`. --- lib/pleroma/migrators/hashtags_table_migrator.ex | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index a226d9d29..ac17f91cc 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -18,7 +18,8 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do defdelegate data_migration(), to: State defdelegate state(), to: State - defdelegate get_stat(key, value), to: State, as: :get_data_key + defdelegate persist_state(), to: State, as: :persist_to_db + defdelegate get_stat(key, value \\ nil), to: State, as: :get_data_key defdelegate put_stat(key, value), to: State, as: :put_data_key defdelegate increment_stat(key, increment), to: State, as: :increment_data_key @@ -116,7 +117,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do increment_stat(:processed_count, length(object_ids)) increment_stat(:failed_count, length(failed_ids)) put_stat(:records_per_second, records_per_second()) - _ = State.persist_to_db() + persist_state() # A quick and dirty approach to controlling the load this background migration imposes sleep_interval = Config.get([:populate_hashtags_table, :sleep_interval_ms], 0) @@ -237,17 +238,19 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do @doc "Approximate count for current iteration (including processed records count)" def count(force \\ false, timeout \\ :infinity) do - stored_count = state()[:count] + stored_count = get_stat(:count) if stored_count && !force do stored_count else - processed_count = state()[:processed_count] || 0 - max_processed_id = data_migration().data["max_processed_id"] || 0 + processed_count = get_stat(:processed_count, 0) + max_processed_id = get_stat(:max_processed_id, 0) query = where(query(), [object], object.id > ^max_processed_id) count = Repo.aggregate(query, :count, :id, timeout: timeout) + processed_count put_stat(:count, count) + persist_state() + count end end @@ -316,7 +319,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do def set_complete do update_status(:complete) - _ = State.persist_to_db() + persist_state() on_complete(data_migration()) end -- cgit v1.2.3 From b981edad8a7d8f27b231bc6164fc0546efbdb646 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 18 Feb 2021 20:40:10 +0300 Subject: [#3213] HashtagsTableMigrator: fault rate allowance to enable the feature (defaults to 1%), counting of affected objects, misc. tweaks. --- lib/pleroma/migrators/hashtags_table_migrator.ex | 101 +++++++++++++++------ .../migrators/hashtags_table_migrator/state.ex | 4 +- 2 files changed, 74 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index ac17f91cc..45dab8470 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -15,7 +15,8 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do alias Pleroma.Object alias Pleroma.Repo - defdelegate data_migration(), to: State + defdelegate data_migration(), to: Pleroma.DataMigration, as: :populate_hashtags_table + defdelegate data_migration_id(), to: State defdelegate state(), to: State defdelegate persist_state(), to: State, as: :persist_to_db @@ -23,10 +24,13 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do defdelegate put_stat(key, value), to: State, as: :put_data_key defdelegate increment_stat(key, increment), to: State, as: :increment_data_key + @feature_config_path [:database, :improved_hashtag_timeline] @reg_name {:global, __MODULE__} def whereis, do: GenServer.whereis(@reg_name) + def feature_state, do: Config.get(@feature_config_path) + def start_link(_) do case whereis() do nil -> @@ -46,8 +50,6 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do def handle_continue(:init_state, _state) do {:ok, _} = State.start_link(nil) - update_status(:pending) - data_migration = data_migration() manual_migrations = Config.get([:instance, :manual_data_migrations], []) @@ -56,10 +58,14 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do update_status(:noop) is_nil(data_migration) -> - update_status(:failed, "Data migration does not exist.") + message = "Data migration does not exist." + update_status(:failed, message) + Logger.error("#{__MODULE__}: #{message}") data_migration.state == :manual or data_migration.name in manual_migrations -> - update_status(:manual, "Data migration is in manual execution state.") + message = "Data migration is in manual execution or manual fix mode." + update_status(:manual, message) + Logger.warn("#{__MODULE__}: #{message}") data_migration.state == :complete -> on_complete(data_migration) @@ -78,7 +84,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do update_status(:running) put_stat(:started_at, NaiveDateTime.utc_now()) - %{id: data_migration_id} = data_migration() + data_migration_id = data_migration_id() max_processed_id = get_stat(:max_processed_id, 0) Logger.info("Transferring embedded hashtags to `hashtags` (from oid: #{max_processed_id})...") @@ -89,12 +95,19 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do |> Stream.each(fn objects -> object_ids = Enum.map(objects, & &1.id) + results = Enum.map(objects, &transfer_object_hashtags(&1)) + failed_ids = - objects - |> Enum.map(&transfer_object_hashtags(&1)) + results |> Enum.filter(&(elem(&1, 0) == :error)) |> Enum.map(&elem(&1, 1)) + # Count of objects with hashtags (`{:noop, id}` is returned for objects having other AS2 tags) + chunk_affected_count = + results + |> Enum.filter(&(elem(&1, 0) == :ok)) + |> length() + for failed_id <- failed_ids do _ = Repo.query( @@ -116,6 +129,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do put_stat(:max_processed_id, max_object_id) increment_stat(:processed_count, length(object_ids)) increment_stat(:failed_count, length(failed_ids)) + increment_stat(:affected_count, chunk_affected_count) put_stat(:records_per_second, records_per_second()) persist_state() @@ -125,17 +139,42 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do end) |> Stream.run() - with 0 <- failures_count(data_migration_id) do - _ = delete_non_create_activities_hashtags() - set_complete() - else - _ -> - update_status(:failed, "Please check data_migration_failed_ids records.") + fault_rate = fault_rate() + put_stat(:fault_rate, fault_rate) + fault_rate_allowance = Config.get([:populate_hashtags_table, :fault_rate_allowance], 0) + + cond do + fault_rate == 0 -> + set_complete() + + is_float(fault_rate) and fault_rate <= fault_rate_allowance -> + message = """ + Done with fault rate of #{fault_rate} which doesn't exceed #{fault_rate_allowance}. + Putting data migration to manual fix mode. Check `retry_failed/0`. + """ + + Logger.warn("#{__MODULE__}: #{message}") + update_status(:manual, message) + on_complete(data_migration()) + + true -> + message = "Too many failures. Check data_migration_failed_ids records / `retry_failed/0`." + Logger.error("#{__MODULE__}: #{message}") + update_status(:failed, message) end + persist_state() {:noreply, state} end + def fault_rate do + with failures_count when is_integer(failures_count) <- failures_count() do + failures_count / Enum.max([get_stat(:affected_count, 0), 1]) + else + _ -> :error + end + end + defp records_per_second do get_stat(:processed_count, 0) / Enum.max([running_time(), 1]) end @@ -194,6 +233,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do |> where([_o, hashtags_objects], is_nil(hashtags_objects.object_id)) end + @spec transfer_object_hashtags(Map.t()) :: {:noop | :ok | :error, integer()} defp transfer_object_hashtags(object) do embedded_tags = if Map.has_key?(object, :tag), do: object.tag, else: object.data["tag"] hashtags = Object.object_data_hashtags(%{"tag" => embedded_tags}) @@ -201,7 +241,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do if Enum.any?(hashtags) do transfer_object_hashtags(object, hashtags) else - {:ok, object.id} + {:noop, object.id} end end @@ -209,13 +249,11 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do Repo.transaction(fn -> with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do maps = Enum.map(hashtag_records, &%{hashtag_id: &1.id, object_id: object.id}) - expected_rows = length(hashtag_records) - - base_error = - "ERROR when inserting #{expected_rows} hashtags_objects for obj. #{object.id}" + base_error = "ERROR when inserting hashtags_objects for object with id #{object.id}" try do - with {^expected_rows, _} <- Repo.insert_all("hashtags_objects", maps) do + with {rows_count, _} when is_integer(rows_count) <- + Repo.insert_all("hashtags_objects", maps, on_conflict: :nothing) do object.id else e -> @@ -260,11 +298,11 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do data_migration.feature_lock -> :noop - not is_nil(Config.get([:database, :improved_hashtag_timeline])) -> + not is_nil(feature_state()) -> :noop true -> - Config.put([:database, :improved_hashtag_timeline], true) + Config.put(@feature_config_path, true) :ok end end @@ -274,38 +312,41 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do |> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"), on: dmf.record_id == o.id ) - |> where([_o, dmf], dmf.data_migration_id == ^data_migration().id) + |> where([_o, dmf], dmf.data_migration_id == ^data_migration_id()) |> order_by([o], asc: o.id) end - def failures_count(data_migration_id \\ nil) do - data_migration_id = data_migration_id || data_migration().id - + def failures_count do with {:ok, %{rows: [[count]]}} <- Repo.query( "SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;", - [data_migration_id] + [data_migration_id()] ) do count end end def retry_failed do - data_migration = data_migration() + data_migration_id = data_migration_id() failed_objects_query() |> Repo.chunk_stream(100, :one) |> Stream.each(fn object -> - with {:ok, _} <- transfer_object_hashtags(object) do + with {res, _} when res != :error <- transfer_object_hashtags(object) do _ = Repo.query( "DELETE FROM data_migration_failed_ids " <> "WHERE data_migration_id = $1 AND record_id = $2", - [data_migration.id, object.id] + [data_migration_id, object.id] ) end end) |> Stream.run() + + put_stat(:failed_count, failures_count()) + persist_state() + + force_continue() end def force_continue do diff --git a/lib/pleroma/migrators/hashtags_table_migrator/state.ex b/lib/pleroma/migrators/hashtags_table_migrator/state.ex index ed9848824..ee0009b2e 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator/state.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator/state.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator.State do alias Pleroma.DataMigration - defdelegate data_migration(), to: DataMigration, as: :populate_hashtags_table + defdelegate data_migration(), to: Pleroma.Migrators.HashtagsTableMigrator @reg_name {:global, __MODULE__} @@ -99,4 +99,6 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator.State do defp persist_non_data_change(_, _) do nil end + + def data_migration_id, do: Map.get(state(), :data_migration_id) end -- cgit v1.2.3 From 998437d4a4111055e019f28dd84a8af1f9a27047 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 18 Feb 2021 21:03:06 +0300 Subject: [#3213] Experimental / debug feature: `database: [improved_hashtag_timeline: :preselect_hashtag_ids]`. --- lib/pleroma/web/activity_pub/activity_pub.ex | 47 +++++++++++++++++++++------- 1 file changed, 35 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index e012f2779..5392ce7c9 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -787,19 +787,42 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_hashtag_any(query, %{tag: [_ | _] = tags}) do - from( - [_activity, object] in query, - where: - fragment( - """ - EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?::citext[]) - AND hashtags_objects.object_id = ? LIMIT 1) - """, - ^tags, - object.id + # TODO: refactor: debug / experimental feature + if Config.get([:database, :improved_hashtag_timeline]) == :preselect_hashtag_ids do + hashtag_ids = + from(ht in Pleroma.Hashtag, + where: fragment("name = ANY(?::citext[])", ^tags), + select: ht.id ) - ) + |> Repo.all() + + from( + [_activity, object] in query, + where: + fragment( + """ + EXISTS ( + SELECT 1 FROM hashtags_objects WHERE hashtag_id = ANY(?) AND object_id = ? LIMIT 1) + """, + ^hashtag_ids, + object.id + ) + ) + else + from( + [_activity, object] in query, + where: + fragment( + """ + EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects + ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?::citext[]) + AND hashtags_objects.object_id = ? LIMIT 1) + """, + ^tags, + object.id + ) + ) + end end defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do -- cgit v1.2.3 From 6531eddf361fa52db3906ab011a4e33c7a5f9552 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 22 Feb 2021 23:26:07 +0300 Subject: [#3213] `hashtags`: altered `name` type to `text`. `hashtags_objects`: removed unused index. HashtagsTableMigrator: records_per_second calculation fix. ActivityPub: hashtags-related options normalization. --- lib/pleroma/hashtag.ex | 22 ++++--- lib/pleroma/migrators/hashtags_table_migrator.ex | 4 +- lib/pleroma/web/activity_pub/activity_pub.ex | 75 ++++++++++-------------- 3 files changed, 49 insertions(+), 52 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index a6d033816..e9d143fb1 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -21,10 +21,14 @@ defmodule Pleroma.Hashtag do timestamps() end + def normalize_name(name) do + name + |> String.downcase() + |> String.trim() + end + def get_by_name(name) do - from(h in Hashtag) - |> where([h], fragment("name = ?::citext", ^String.downcase(name))) - |> Repo.one() + Repo.get_by(Hashtag, name: normalize_name(name)) end def get_or_create_by_name(name) when is_bitstring(name) do @@ -39,7 +43,7 @@ defmodule Pleroma.Hashtag do end def get_or_create_by_names(names) when is_list(names) do - names = Enum.map(names, &String.downcase/1) + names = Enum.map(names, &normalize_name/1) timestamp = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) structs = @@ -53,10 +57,12 @@ defmodule Pleroma.Hashtag do try do with {:ok, %{query_op: hashtags}} <- Multi.new() - |> Multi.insert_all(:insert_all_op, Hashtag, structs, on_conflict: :nothing) + |> Multi.insert_all(:insert_all_op, Hashtag, structs, + on_conflict: :nothing, + conflict_target: :name + ) |> Multi.run(:query_op, fn _repo, _changes -> - {:ok, - Repo.all(from(ht in Hashtag, where: ht.name in fragment("?::citext[]", ^names)))} + {:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))} end) |> Repo.transaction() do {:ok, hashtags} @@ -71,7 +77,7 @@ defmodule Pleroma.Hashtag do def changeset(%Hashtag{} = struct, params) do struct |> cast(params, [:name]) - |> update_change(:name, &String.downcase/1) + |> update_change(:name, &normalize_name/1) |> validate_required([:name]) |> unique_constraint(:name) end diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index 45dab8470..07bb9aeb2 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -82,6 +82,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do State.reinit() update_status(:running) + put_stat(:iteration_processed_count, 0) put_stat(:started_at, NaiveDateTime.utc_now()) data_migration_id = data_migration_id() @@ -127,6 +128,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do max_object_id = Enum.at(object_ids, -1) put_stat(:max_processed_id, max_object_id) + increment_stat(:iteration_processed_count, length(object_ids)) increment_stat(:processed_count, length(object_ids)) increment_stat(:failed_count, length(failed_ids)) increment_stat(:affected_count, chunk_affected_count) @@ -176,7 +178,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do end defp records_per_second do - get_stat(:processed_count, 0) / Enum.max([running_time(), 1]) + get_stat(:iteration_processed_count, 0) / Enum.max([running_time(), 1]) end defp running_time do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5392ce7c9..8182bc205 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Conversation alias Pleroma.Conversation.Participation alias Pleroma.Filter + alias Pleroma.Hashtag alias Pleroma.Maps alias Pleroma.Notification alias Pleroma.Object @@ -698,8 +699,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_embedded_tag_all(query, %{tag_all: [_ | _] = tag_all}) do - tag_all = Enum.map(tag_all, &String.downcase/1) - from( [_activity, object] in query, where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) @@ -717,8 +716,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_embedded_tag_any(query, %{tag: [_ | _] = tag_any}) do - tag_any = Enum.map(tag_any, &String.downcase/1) - from( [_activity, object] in query, where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag_any) @@ -736,8 +733,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_embedded_tag_reject_any(query, %{tag_reject: [_ | _] = tag_reject}) do - tag_reject = Enum.map(tag_reject, &String.downcase/1) - from( [_activity, object] in query, where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject) @@ -766,7 +761,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do fragment( """ (SELECT array_agg(hashtags.name) FROM hashtags JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?::citext[]) + ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) AND hashtags_objects.object_id = ?) @> ? """, ^tags, @@ -787,42 +782,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_hashtag_any(query, %{tag: [_ | _] = tags}) do - # TODO: refactor: debug / experimental feature - if Config.get([:database, :improved_hashtag_timeline]) == :preselect_hashtag_ids do - hashtag_ids = - from(ht in Pleroma.Hashtag, - where: fragment("name = ANY(?::citext[])", ^tags), - select: ht.id + from( + [_activity, object] in query, + where: + fragment( + """ + EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects + ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) + AND hashtags_objects.object_id = ? LIMIT 1) + """, + ^tags, + object.id ) - |> Repo.all() - - from( - [_activity, object] in query, - where: - fragment( - """ - EXISTS ( - SELECT 1 FROM hashtags_objects WHERE hashtag_id = ANY(?) AND object_id = ? LIMIT 1) - """, - ^hashtag_ids, - object.id - ) - ) - else - from( - [_activity, object] in query, - where: - fragment( - """ - EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?::citext[]) - AND hashtags_objects.object_id = ? LIMIT 1) - """, - ^tags, - object.id - ) - ) - end + ) end defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do @@ -842,7 +814,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do fragment( """ NOT EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?::citext[]) + ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) AND hashtags_objects.object_id = ? LIMIT 1) """, ^tags_reject, @@ -1220,6 +1192,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp maybe_order(query, _), do: query + defp normalize_fetch_activities_query_opts(opts) do + Enum.reduce([:tag, :tag_all, :tag_reject], opts, fn key, opts -> + case opts[key] do + value when is_bitstring(value) -> + Map.put(opts, key, Hashtag.normalize_name(value)) + + value when is_list(value) -> + Map.put(opts, key, Enum.map(value, &Hashtag.normalize_name/1)) + + _ -> + opts + end + end) + end + defp fetch_activities_query_ap_ids_ops(opts) do source_user = opts[:muting_user] ap_id_relationships = if source_user, do: [:mute, :reblog_mute], else: [] @@ -1243,6 +1230,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def fetch_activities_query(recipients, opts \\ %{}) do + opts = normalize_fetch_activities_query_opts(opts) + {restrict_blocked_opts, restrict_muted_opts, restrict_muted_reblogs_opts} = fetch_activities_query_ap_ids_ops(opts) -- cgit v1.2.3 From 77f3da035894e2add911101466bfe41b99ee481e Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 23 Feb 2021 13:52:28 +0300 Subject: [#3213] Misc. tweaks: proper upsert in Hashtag, better feature toggle management. --- lib/pleroma/config.ex | 4 ++++ lib/pleroma/hashtag.ex | 22 +++++++++------------- lib/pleroma/migrators/hashtags_table_migrator.ex | 18 +++++++----------- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 4 files changed, 21 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index f17e14128..e057d8c02 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -111,4 +111,8 @@ defmodule Pleroma.Config do end ) end + + def feature_enabled?(feature_name) do + get([:features, feature_name]) not in [nil, false, :disabled, :auto] + end end diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index e9d143fb1..53e2e9c89 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -27,19 +27,15 @@ defmodule Pleroma.Hashtag do |> String.trim() end - def get_by_name(name) do - Repo.get_by(Hashtag, name: normalize_name(name)) - end - - def get_or_create_by_name(name) when is_bitstring(name) do - with %Hashtag{} = hashtag <- get_by_name(name) do - {:ok, hashtag} - else - _ -> - %Hashtag{} - |> changeset(%{name: name}) - |> Repo.insert() - end + def get_or_create_by_name(name) do + changeset = changeset(%Hashtag{}, %{name: name}) + + Repo.insert( + changeset, + on_conflict: [set: [name: get_field(changeset, :name)]], + conflict_target: :name, + returning: true + ) end def get_or_create_by_names(names) when is_list(names) do diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index 07bb9aeb2..6123c88e0 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do defdelegate put_stat(key, value), to: State, as: :put_data_key defdelegate increment_stat(key, increment), to: State, as: :increment_data_key - @feature_config_path [:database, :improved_hashtag_timeline] + @feature_config_path [:features, :improved_hashtag_timeline] @reg_name {:global, __MODULE__} def whereis, do: GenServer.whereis(@reg_name) @@ -296,16 +296,12 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do end defp on_complete(data_migration) do - cond do - data_migration.feature_lock -> - :noop - - not is_nil(feature_state()) -> - :noop - - true -> - Config.put(@feature_config_path, true) - :ok + if data_migration.feature_lock || feature_state() == :disabled do + Logger.warn("#{__MODULE__}: migration complete but feature is locked; consider enabling.") + :noop + else + Config.put(@feature_config_path, :enabled) + :ok end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 8182bc205..9d557c2cd 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1273,7 +1273,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> exclude_invisible_actors(opts) |> exclude_visibility(opts) - if Config.get([:database, :improved_hashtag_timeline]) do + if Config.feature_enabled?(:improved_hashtag_timeline) do query |> restrict_hashtag_any(opts) |> restrict_hashtag_all(opts) -- cgit v1.2.3 From 40d4362261abaf0856a1b4397a4bff6344137120 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 23 Feb 2021 18:11:25 +0300 Subject: [#3213] `mix pleroma.database rollback` tweaks. --- lib/mix/tasks/pleroma/database.ex | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 2136ddb02..e7f4b67a4 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -231,19 +231,18 @@ defmodule Mix.Tasks.Pleroma.Database do re = ~r/^#{version}_.*\.exs/ path = Ecto.Migrator.migrations_path(repo) - with {:find, "" <> file} <- {:find, Enum.find(File.ls!(path), &String.match?(&1, re))}, - {:compile, [{mod, _} | _]} <- {:compile, Code.compile_file(Path.join(path, file))}, - {:rollback, :ok} <- {:rollback, Ecto.Migrator.down(repo, version, mod)} do + with {_, "" <> file} <- {:find, Enum.find(File.ls!(path), &String.match?(&1, re))}, + {_, [{mod, _} | _]} <- {:compile, Code.compile_file(Path.join(path, file))}, + {_, :ok} <- {:rollback, Ecto.Migrator.down(repo, version, mod)} do {:ok, "Reversed migration: #{file}"} else {:find, _} -> {:error, "No migration found with version prefix: #{version}"} {:compile, e} -> {:error, "Problem compiling migration module: #{inspect(e)}"} {:rollback, e} -> {:error, "Problem reversing migration: #{inspect(e)}"} - e -> {:error, "Something unexpected happened: #{inspect(e)}"} end end) - IO.inspect(result) + shell_info(inspect(result)) end end end -- cgit v1.2.3 From 3bc7d122712b5cc35ba509542bde63ca130d6a40 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 28 Dec 2020 23:21:53 +0100 Subject: Remove sensitive-property setting #nsfw, create HashtagPolicy --- lib/pleroma/web/activity_pub/mrf.ex | 4 +- lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex | 116 +++++++++++++++++++++ lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 12 +-- lib/pleroma/web/activity_pub/mrf/tag_policy.ex | 13 +-- lib/pleroma/web/activity_pub/transmogrifier.ex | 11 -- lib/pleroma/web/common_api/activity_draft.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 8 -- 7 files changed, 125 insertions(+), 41 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index ef5a09a93..f2fec3ff6 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -92,7 +92,9 @@ defmodule Pleroma.Web.ActivityPub.MRF do end def get_policies do - Pleroma.Config.get([:mrf, :policies], []) |> get_policies() + Pleroma.Config.get([:mrf, :policies], []) + |> get_policies() + |> Enum.concat([Pleroma.Web.ActivityPub.MRF.HashtagPolicy]) end defp get_policies(policy) when is_atom(policy), do: [policy] diff --git a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex new file mode 100644 index 000000000..def0c437c --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex @@ -0,0 +1,116 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do + require Pleroma.Constants + + alias Pleroma.Config + alias Pleroma.Object + + @moduledoc """ + Reject, TWKN-remove or Set-Sensitive messsages with specific hashtags (without the leading #) + + Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists. + """ + + @behaviour Pleroma.Web.ActivityPub.MRF + + defp check_reject(message, hashtags) do + if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do + {:reject, "[HashtagPolicy] Matches with rejected keyword"} + else + {:ok, message} + end + end + + defp check_ftl_removal(%{"to" => to} = message, hashtags) do + if Pleroma.Constants.as_public() in to and + Enum.any?(Config.get([:mrf_hashtag, :federated_timeline_removal]), fn match -> + match in hashtags + end) do + to = List.delete(to, Pleroma.Constants.as_public()) + cc = [Pleroma.Constants.as_public() | message["cc"] || []] + + message = + message + |> Map.put("to", to) + |> Map.put("cc", cc) + |> Kernel.put_in(["object", "to"], to) + |> Kernel.put_in(["object", "cc"], cc) + + {:ok, message} + else + {:ok, message} + end + end + + defp check_ftl_removal(message, _hashtags), do: {:ok, message} + + defp check_sensitive(message, hashtags) do + if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do + {:ok, Kernel.put_in(message, ["object", "sensitive"], true)} + else + {:ok, message} + end + end + + @impl true + def filter(%{"type" => "Create", "object" => object} = message) do + hashtags = Object.hashtags(%Object{data: object}) + + if hashtags != [] do + with {:ok, message} <- check_reject(message, hashtags), + {:ok, message} <- check_ftl_removal(message, hashtags), + {:ok, message} <- check_sensitive(message, hashtags) do + {:ok, message} + end + else + {:ok, message} + end + end + + @impl true + def filter(message), do: {:ok, message} + + @impl true + def describe do + mrf_hashtag = + Config.get(:mrf_hashtag) + |> Enum.into(%{}) + + {:ok, %{mrf_hashtag: mrf_hashtag}} + end + + @impl true + def config_description do + %{ + key: :mrf_hashtag, + related_policy: "Pleroma.Web.ActivityPub.MRF.HashtagPolicy", + label: "MRF Hashtag", + description: @moduledoc, + children: [ + %{ + key: :reject, + type: {:list, :string}, + description: "A list of hashtags which result in message being rejected.", + suggestions: ["foo"] + }, + %{ + key: :federated_timeline_removal, + type: {:list, :string}, + description: + "A list of hashtags which result in message being removed from federated timelines (a.k.a unlisted).", + suggestions: ["foo"] + }, + %{ + key: :sensitive, + type: {:list, :string}, + description: + "A list of hashtags which result in message being set as sensitive (a.k.a NSFW/R-18)", + suggestions: ["nsfw", "r18"] + } + ] + } + end +end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 0b1be8c51..62024c58c 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -64,22 +64,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do %{host: actor_host} = _actor_info, %{ "type" => "Create", - "object" => child_object + "object" => %{} = _child_object } = object - ) - when is_map(child_object) do + ) do media_nsfw = Config.get([:mrf_simple, :media_nsfw]) |> MRF.subdomains_regex() object = if MRF.subdomain_match?(media_nsfw, actor_host) do - child_object = - child_object - |> Map.put("tag", (child_object["tag"] || []) ++ ["nsfw"]) - |> Map.put("sensitive", true) - - Map.put(object, "object", child_object) + Kernel.put_in(object, ["object", "sensitive"], true) else object end diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index 5739cee63..528093ac0 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -28,20 +28,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do "mrf_tag:media-force-nsfw", %{ "type" => "Create", - "object" => %{"attachment" => child_attachment} = object + "object" => %{"attachment" => child_attachment} } = message ) when length(child_attachment) > 0 do - tags = (object["tag"] || []) ++ ["nsfw"] - - object = - object - |> Map.put("tag", tags) - |> Map.put("sensitive", true) - - message = Map.put(message, "object", object) - - {:ok, message} + {:ok, Kernel.put_in(message, ["object", "sensitive"], true)} end defp process_tag( diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 0a701334f..8c7d6a747 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -40,7 +40,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> fix_in_reply_to(options) |> fix_emoji() |> fix_tag() - |> set_sensitive() |> fix_content_map() |> fix_addressing() |> fix_summary() @@ -741,7 +740,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # Prepares the object of an outgoing create activity. def prepare_object(object) do object - |> set_sensitive |> add_hashtags |> add_mention_tags |> add_emoji_tags @@ -932,15 +930,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Map.put(object, "conversation", object["context"]) end - def set_sensitive(%{"sensitive" => _} = object) do - object - end - - def set_sensitive(object) do - tags = object["tag"] || [] - Map.put(object, "sensitive", "nsfw" in tags) - end - def set_type(%{"type" => "Answer"} = object) do Map.put(object, "type", "Note") end diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index fb059c27c..da726a690 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -179,7 +179,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end defp sensitive(draft) do - sensitive = draft.params[:sensitive] || Enum.member?(draft.tags, {"#nsfw", "nsfw"}) + sensitive = draft.params[:sensitive] %__MODULE__{draft | sensitive: sensitive} end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 9587dfa25..4e6a3feb0 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -217,7 +217,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do draft.status |> format_input(content_type, options) |> maybe_add_attachments(draft.attachments, attachment_links) - |> maybe_add_nsfw_tag(draft.params) end defp get_content_type(content_type) do @@ -228,13 +227,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end - defp maybe_add_nsfw_tag({text, mentions, tags}, %{"sensitive" => sensitive}) - when sensitive in [true, "True", "true", "1"] do - {text, mentions, [{"#nsfw", "nsfw"} | tags]} - end - - defp maybe_add_nsfw_tag(data, _), do: data - def make_context(_, %Participation{} = participation) do Repo.preload(participation, :conversation).conversation.ap_id end -- cgit v1.2.3 From 3aae5231b2c8f669eadba9228cece254349dd2aa Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 2 Mar 2021 20:49:17 +0400 Subject: Add OpenAPI spec for AdminAPI.UserController --- lib/pleroma/user.ex | 7 - .../web/admin_api/controllers/user_controller.ex | 128 +++---- lib/pleroma/web/admin_api/views/account_view.ex | 19 +- .../api_spec/operations/admin/user_operation.ex | 389 +++++++++++++++++++++ lib/pleroma/web/router.ex | 2 +- 5 files changed, 472 insertions(+), 73 deletions(-) create mode 100644 lib/pleroma/web/api_spec/operations/admin/user_operation.ex (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9942617d8..c1aa0f716 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2255,13 +2255,6 @@ defmodule Pleroma.User do |> update_and_set_cache() end - def roles(%{is_moderator: is_moderator, is_admin: is_admin}) do - %{ - admin: is_admin, - moderator: is_moderator - } - end - def validate_fields(changeset, remote? \\ false) do limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields limit = Config.get([:instance, limit_name], 0) diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex index 65bc63cb9..d3e4c18a3 100644 --- a/lib/pleroma/web/admin_api/controllers/user_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -13,16 +13,17 @@ defmodule Pleroma.Web.AdminAPI.UserController do alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.AdminAPI - alias Pleroma.Web.AdminAPI.AccountView alias Pleroma.Web.AdminAPI.Search alias Pleroma.Web.Plugs.OAuthScopesPlug @users_page_size 50 + plug(Pleroma.Web.ApiSpec.CastAndValidate) + plug( OAuthScopesPlug, %{scopes: ["admin:read:accounts"]} - when action in [:list, :show] + when action in [:index, :show] ) plug( @@ -44,13 +45,19 @@ defmodule Pleroma.Web.AdminAPI.UserController do when action in [:follow, :unfollow] ) + plug(:put_view, Pleroma.Web.AdminAPI.AccountView) + action_fallback(AdminAPI.FallbackController) - def delete(conn, %{"nickname" => nickname}) do - delete(conn, %{"nicknames" => [nickname]}) + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.UserOperation + + def delete(conn, %{nickname: nickname}) do + conn + |> Map.put(:body_params, %{nicknames: [nickname]}) + |> delete(%{}) end - def delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + def delete(%{assigns: %{user: admin}, body_params: %{nicknames: nicknames}} = conn, _) do users = Enum.map(nicknames, &User.get_cached_by_nickname/1) Enum.each(users, fn user -> @@ -67,10 +74,16 @@ defmodule Pleroma.Web.AdminAPI.UserController do json(conn, nicknames) end - def follow(%{assigns: %{user: admin}} = conn, %{ - "follower" => follower_nick, - "followed" => followed_nick - }) do + def follow( + %{ + assigns: %{user: admin}, + body_params: %{ + follower: follower_nick, + followed: followed_nick + } + } = conn, + _ + ) do with %User{} = follower <- User.get_cached_by_nickname(follower_nick), %User{} = followed <- User.get_cached_by_nickname(followed_nick) do User.follow(follower, followed) @@ -86,10 +99,16 @@ defmodule Pleroma.Web.AdminAPI.UserController do json(conn, "ok") end - def unfollow(%{assigns: %{user: admin}} = conn, %{ - "follower" => follower_nick, - "followed" => followed_nick - }) do + def unfollow( + %{ + assigns: %{user: admin}, + body_params: %{ + follower: follower_nick, + followed: followed_nick + } + } = conn, + _ + ) do with %User{} = follower <- User.get_cached_by_nickname(follower_nick), %User{} = followed <- User.get_cached_by_nickname(followed_nick) do User.unfollow(follower, followed) @@ -105,9 +124,10 @@ defmodule Pleroma.Web.AdminAPI.UserController do json(conn, "ok") end - def create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do + def create(%{assigns: %{user: admin}, body_params: %{users: users}} = conn, _) do changesets = - Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} -> + users + |> Enum.map(fn %{nickname: nickname, email: email, password: password} -> user_data = %{ nickname: nickname, name: nickname, @@ -124,52 +144,49 @@ defmodule Pleroma.Web.AdminAPI.UserController do end) case Pleroma.Repo.transaction(changesets) do - {:ok, users} -> - res = - users + {:ok, users_map} -> + users = + users_map |> Map.values() |> Enum.map(fn user -> {:ok, user} = User.post_register_action(user) user end) - |> Enum.map(&AccountView.render("created.json", %{user: &1})) ModerationLog.insert_log(%{ actor: admin, - subjects: Map.values(users), + subjects: users, action: "create" }) - json(conn, res) + render(conn, "created_many.json", users: users) {:error, id, changeset, _} -> - res = + changesets = Enum.map(changesets.operations, fn - {current_id, {:changeset, _current_changeset, _}} when current_id == id -> - AccountView.render("create-error.json", %{changeset: changeset}) + {^id, {:changeset, _current_changeset, _}} -> + changeset {_, {:changeset, current_changeset, _}} -> - AccountView.render("create-error.json", %{changeset: current_changeset}) + current_changeset end) conn |> put_status(:conflict) - |> json(res) + |> render("create_errors.json", changesets: changesets) end end - def show(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + def show(%{assigns: %{user: admin}} = conn, %{nickname: nickname}) do with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do - conn - |> put_view(AccountView) - |> render("show.json", %{user: user}) + render(conn, "show.json", %{user: user}) else _ -> {:error, :not_found} end end - def toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + def toggle_activation(%{assigns: %{user: admin}} = conn, %{nickname: nickname}) do user = User.get_cached_by_nickname(nickname) {:ok, updated_user} = User.set_activation(user, !user.is_active) @@ -182,12 +199,10 @@ defmodule Pleroma.Web.AdminAPI.UserController do action: action }) - conn - |> put_view(AccountView) - |> render("show.json", %{user: updated_user}) + render(conn, "show.json", user: updated_user) end - def activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + def activate(%{assigns: %{user: admin}, body_params: %{nicknames: nicknames}} = conn, _) do users = Enum.map(nicknames, &User.get_cached_by_nickname/1) {:ok, updated_users} = User.set_activation(users, true) @@ -197,12 +212,10 @@ defmodule Pleroma.Web.AdminAPI.UserController do action: "activate" }) - conn - |> put_view(AccountView) - |> render("index.json", %{users: Keyword.values(updated_users)}) + render(conn, "index.json", users: Keyword.values(updated_users)) end - def deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + def deactivate(%{assigns: %{user: admin}, body_params: %{nicknames: nicknames}} = conn, _) do users = Enum.map(nicknames, &User.get_cached_by_nickname/1) {:ok, updated_users} = User.set_activation(users, false) @@ -212,12 +225,10 @@ defmodule Pleroma.Web.AdminAPI.UserController do action: "deactivate" }) - conn - |> put_view(AccountView) - |> render("index.json", %{users: Keyword.values(updated_users)}) + render(conn, "index.json", users: Keyword.values(updated_users)) end - def approve(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + def approve(%{assigns: %{user: admin}, body_params: %{nicknames: nicknames}} = conn, _) do users = Enum.map(nicknames, &User.get_cached_by_nickname/1) {:ok, updated_users} = User.approve(users) @@ -227,36 +238,27 @@ defmodule Pleroma.Web.AdminAPI.UserController do action: "approve" }) - conn - |> put_view(AccountView) - |> render("index.json", %{users: updated_users}) + render(conn, "index.json", users: updated_users) end - def list(conn, params) do + def index(conn, params) do {page, page_size} = page_params(params) - filters = maybe_parse_filters(params["filters"]) + filters = maybe_parse_filters(params[:filters]) search_params = %{ - query: params["query"], + query: params[:query], page: page, page_size: page_size, - tags: params["tags"], - name: params["name"], - email: params["email"], - actor_types: params["actor_types"] + tags: params[:tags], + name: params[:name], + email: params[:email], + actor_types: params[:actor_types] } |> Map.merge(filters) with {:ok, users, count} <- Search.user(search_params) do - json( - conn, - AccountView.render("index.json", - users: users, - count: count, - page_size: page_size - ) - ) + render(conn, "index.json", users: users, count: count, page_size: page_size) end end @@ -274,8 +276,8 @@ defmodule Pleroma.Web.AdminAPI.UserController do defp page_params(params) do { - fetch_integer_param(params, "page", 1), - fetch_integer_param(params, "page_size", @users_page_size) + fetch_integer_param(params, :page, 1), + fetch_integer_param(params, :page_size, @users_page_size) } end end diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index d7c63d385..e053a9b67 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -75,7 +75,7 @@ defmodule Pleroma.Web.AdminAPI.AccountView do "display_name" => display_name, "is_active" => user.is_active, "local" => user.local, - "roles" => User.roles(user), + "roles" => roles(user), "tags" => user.tags || [], "is_confirmed" => user.is_confirmed, "is_approved" => user.is_approved, @@ -85,6 +85,10 @@ defmodule Pleroma.Web.AdminAPI.AccountView do } end + def render("created_many.json", %{users: users}) do + render_many(users, AccountView, "created.json", as: :user) + end + def render("created.json", %{user: user}) do %{ type: "success", @@ -96,7 +100,11 @@ defmodule Pleroma.Web.AdminAPI.AccountView do } end - def render("create-error.json", %{changeset: %Ecto.Changeset{changes: changes, errors: errors}}) do + def render("create_errors.json", %{changesets: changesets}) do + render_many(changesets, AccountView, "create_error.json", as: :changeset) + end + + def render("create_error.json", %{changeset: %Ecto.Changeset{changes: changes, errors: errors}}) do %{ type: "error", code: 409, @@ -140,4 +148,11 @@ defmodule Pleroma.Web.AdminAPI.AccountView do defp image_url(%{"url" => [%{"href" => href} | _]}), do: href defp image_url(_), do: nil + + defp roles(%{is_moderator: is_moderator, is_admin: is_admin}) do + %{ + admin: is_admin, + moderator: is_moderator + } + end end diff --git a/lib/pleroma/web/api_spec/operations/admin/user_operation.ex b/lib/pleroma/web/api_spec/operations/admin/user_operation.ex new file mode 100644 index 000000000..183c61236 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/admin/user_operation.ex @@ -0,0 +1,389 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ActorType + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Users"], + summary: "List users", + operationId: "AdminAPI.UserController.index", + security: [%{"oAuth" => ["admin:read:accounts"]}], + parameters: [ + Operation.parameter(:filters, :query, :string, "Comma separated list of filters"), + Operation.parameter(:query, :query, :string, "Search users query"), + Operation.parameter(:name, :query, :string, "Search by display name"), + Operation.parameter(:email, :query, :string, "Search by email"), + Operation.parameter(:page, :query, :integer, "Page Number"), + Operation.parameter(:page_size, :query, :integer, "Number of users to return per page"), + Operation.parameter( + :actor_types, + :query, + %Schema{type: :array, items: ActorType}, + "Filter by actor type" + ), + Operation.parameter( + :tags, + :query, + %Schema{type: :array, items: %Schema{type: :string}}, + "Filter by tags" + ) + | admin_api_params() + ], + responses: %{ + 200 => + Operation.response( + "Response", + "application/json", + %Schema{ + type: :object, + properties: %{ + users: %Schema{type: :array, items: user()}, + count: %Schema{type: :integer}, + page_size: %Schema{type: :integer} + } + } + ), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def create_operation do + %Operation{ + tags: ["Users"], + summary: "Create a single or multiple users", + operationId: "AdminAPI.UserController.create", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: admin_api_params(), + requestBody: + request_body( + "Parameters", + %Schema{ + description: "POST body for creating users", + type: :object, + properties: %{ + users: %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + nickname: %Schema{type: :string}, + email: %Schema{type: :string}, + password: %Schema{type: :string} + } + } + } + } + } + ), + responses: %{ + 200 => + Operation.response("Response", "application/json", %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + code: %Schema{type: :integer}, + type: %Schema{type: :string}, + data: %Schema{ + type: :object, + properties: %{ + email: %Schema{type: :string, format: :email}, + nickname: %Schema{type: :string} + } + } + } + } + }), + 403 => Operation.response("Forbidden", "application/json", ApiError), + 409 => + Operation.response("Conflict", "application/json", %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + code: %Schema{type: :integer}, + error: %Schema{type: :string}, + type: %Schema{type: :string}, + data: %Schema{ + type: :object, + properties: %{ + email: %Schema{type: :string, format: :email}, + nickname: %Schema{type: :string} + } + } + } + } + }) + } + } + end + + def show_operation do + %Operation{ + tags: ["Users"], + summary: "Show user", + operationId: "AdminAPI.UserController.show", + security: [%{"oAuth" => ["admin:read:accounts"]}], + parameters: [ + Operation.parameter( + :nickname, + :path, + :string, + "User nickname or ID" + ) + | admin_api_params() + ], + responses: %{ + 200 => Operation.response("Response", "application/json", user()), + 403 => Operation.response("Forbidden", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } + end + + def follow_operation do + %Operation{ + tags: ["Users"], + summary: "Follow", + operationId: "AdminAPI.UserController.follow", + security: [%{"oAuth" => ["admin:write:follows"]}], + parameters: admin_api_params(), + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + properties: %{ + follower: %Schema{type: :string, description: "Follower nickname"}, + followed: %Schema{type: :string, description: "Followed nickname"} + } + } + ), + responses: %{ + 200 => Operation.response("Response", "application/json", %Schema{type: :string}), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def unfollow_operation do + %Operation{ + tags: ["Users"], + summary: "Unfollow", + operationId: "AdminAPI.UserController.unfollow", + security: [%{"oAuth" => ["admin:write:follows"]}], + parameters: admin_api_params(), + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + properties: %{ + follower: %Schema{type: :string, description: "Follower nickname"}, + followed: %Schema{type: :string, description: "Followed nickname"} + } + } + ), + responses: %{ + 200 => Operation.response("Response", "application/json", %Schema{type: :string}), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def approve_operation do + %Operation{ + tags: ["Users"], + summary: "Approve multiple users", + operationId: "AdminAPI.UserController.approve", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: admin_api_params(), + requestBody: + request_body( + "Parameters", + %Schema{ + description: "POST body for deleting multiple users", + type: :object, + properties: %{ + nicknames: %Schema{ + type: :array, + items: %Schema{type: :string} + } + } + } + ), + responses: %{ + 200 => + Operation.response("Response", "application/json", %Schema{ + type: :object, + properties: %{user: %Schema{type: :array, items: user()}} + }), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def toggle_activation_operation do + %Operation{ + tags: ["Users"], + summary: "Toggle user activation", + operationId: "AdminAPI.UserController.toggle_activation", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: [ + Operation.parameter(:nickname, :path, :string, "User nickname") + | admin_api_params() + ], + responses: %{ + 200 => Operation.response("Response", "application/json", user()), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def activate_operation do + %Operation{ + tags: ["Users"], + summary: "Activate multiple users", + operationId: "AdminAPI.UserController.activate", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: admin_api_params(), + requestBody: + request_body( + "Parameters", + %Schema{ + description: "POST body for deleting multiple users", + type: :object, + properties: %{ + nicknames: %Schema{ + type: :array, + items: %Schema{type: :string} + } + } + } + ), + responses: %{ + 200 => + Operation.response("Response", "application/json", %Schema{ + type: :object, + properties: %{user: %Schema{type: :array, items: user()}} + }), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def deactivate_operation do + %Operation{ + tags: ["Users"], + summary: "Deactivates multiple users", + operationId: "AdminAPI.UserController.deactivate", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: admin_api_params(), + requestBody: + request_body( + "Parameters", + %Schema{ + description: "POST body for deleting multiple users", + type: :object, + properties: %{ + nicknames: %Schema{ + type: :array, + items: %Schema{type: :string} + } + } + } + ), + responses: %{ + 200 => + Operation.response("Response", "application/json", %Schema{ + type: :object, + properties: %{user: %Schema{type: :array, items: user()}} + }), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def delete_operation do + %Operation{ + tags: ["Users"], + summary: "Removes a single or multiple users", + operationId: "AdminAPI.UserController.delete", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: [ + Operation.parameter( + :nickname, + :query, + :string, + "User nickname" + ) + | admin_api_params() + ], + requestBody: + request_body( + "Parameters", + %Schema{ + description: "POST body for deleting multiple users", + type: :object, + properties: %{ + nicknames: %Schema{ + type: :array, + items: %Schema{type: :string} + } + } + } + ), + responses: %{ + 200 => + Operation.response("Response", "application/json", %Schema{ + description: "Array of nicknames", + type: :array, + items: %Schema{type: :string} + }), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + defp user do + %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :string}, + email: %Schema{type: :string, format: :email}, + avatar: %Schema{type: :string, format: :uri}, + nickname: %Schema{type: :string}, + display_name: %Schema{type: :string}, + is_active: %Schema{type: :boolean}, + local: %Schema{type: :boolean}, + roles: %Schema{ + type: :object, + properties: %{ + admin: %Schema{type: :boolean}, + moderator: %Schema{type: :boolean} + } + }, + tags: %Schema{type: :array, items: %Schema{type: :string}}, + is_confirmed: %Schema{type: :boolean}, + is_approved: %Schema{type: :boolean}, + url: %Schema{type: :string, format: :uri}, + registration_reason: %Schema{type: :string, nullable: true}, + actor_type: %Schema{type: :string} + } + } + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 72ad14f05..de0bd27d7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -204,7 +204,7 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/credentials", AdminAPIController, :show_user_credentials) patch("/users/:nickname/credentials", AdminAPIController, :update_user_credentials) - get("/users", UserController, :list) + get("/users", UserController, :index) get("/users/:nickname", UserController, :show) get("/users/:nickname/statuses", AdminAPIController, :list_user_statuses) get("/users/:nickname/chats", AdminAPIController, :list_user_chats) -- cgit v1.2.3 From 85b2387f665045a303486d10e6879a46a7ab922e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 2 Mar 2021 11:37:37 -0600 Subject: Fix build_application/1 match --- lib/pleroma/web/mastodon_api/views/status_view.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index bac897a57..a7e762ac1 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -536,6 +536,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end @spec build_application(map() | nil) :: map() | nil - defp build_application(%{type: _type, name: name, url: url}), do: %{name: name, website: url} + defp build_application(%{"type" => _type, "name" => name, "url" => url}), + do: %{name: name, website: url} + defp build_application(_), do: nil end -- cgit v1.2.3 From 8d601d3b234cfe2a6a942dd156712cc400af8500 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 2 Mar 2021 14:14:38 -0600 Subject: Make the object reference in both render("show.json", _) functions consistently named --- lib/pleroma/web/mastodon_api/views/status_view.ex | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index a7e762ac1..f3f54e03d 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -124,16 +124,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do ) do user = CommonAPI.get_user(activity.data["actor"]) created_at = Utils.to_masto_date(activity.data["published"]) - activity_object = Object.normalize(activity, fetch: false) + object = Object.normalize(activity, fetch: false) reblogged_parent_activity = if opts[:parent_activities] do Activity.Queries.find_by_object_ap_id( opts[:parent_activities], - activity_object.data["id"] + object.data["id"] ) else - Activity.create_by_object_ap_id(activity_object.data["id"]) + Activity.create_by_object_ap_id(object.data["id"]) |> Activity.with_preloaded_bookmark(opts[:for]) |> Activity.with_set_thread_muted_field(opts[:for]) |> Repo.one() @@ -142,7 +142,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reblog_rendering_opts = Map.put(opts, :activity, reblogged_parent_activity) reblogged = render("show.json", reblog_rendering_opts) - favorited = opts[:for] && opts[:for].ap_id in (activity_object.data["likes"] || []) + favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) bookmarked = Activity.get_bookmark(reblogged_parent_activity, opts[:for]) != nil @@ -154,8 +154,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do %{ id: to_string(activity.id), - uri: activity_object.data["id"], - url: activity_object.data["id"], + uri: object.data["id"], + url: object.data["id"], account: AccountView.render("show.json", %{ user: user, @@ -180,7 +180,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do media_attachments: reblogged[:media_attachments] || [], mentions: mentions, tags: reblogged[:tags] || [], - application: build_application(activity_object.data["generator"]), + application: build_application(object.data["generator"]), language: nil, emojis: [], pleroma: %{ -- cgit v1.2.3 From 2e296c079f0666a8239a0d3ce5b5fba6baf45a29 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 3 Mar 2021 15:33:06 +0100 Subject: Revert "StatusController: Deactivate application support for now." This reverts commit 024c11c18d289d4acd65d749f939ad3684f31905. --- .../mastodon_api/controllers/status_controller.ex | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index d1a58d5e1..b051fca74 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.ScheduledActivityView - # alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.OAuth.Token alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.Web.Plugs.RateLimiter @@ -420,16 +420,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do ) end - # Deactivated for 2.3.0 - # defp put_application(params, - # %{assigns: %{token: %Token{user: %User{} = user} = token}} = _conn) do - # if user.disclose_client do - # %{client_name: client_name, website: website} = Repo.preload(token, :app).app - # Map.put(params, :generator, %{type: "Application", name: client_name, url: website}) - # else - # Map.put(params, :generator, nil) - # end - # end + defp put_application(params, %{assigns: %{token: %Token{user: %User{} = user} = token}} = _conn) do + if user.disclose_client do + %{client_name: client_name, website: website} = Repo.preload(token, :app).app + Map.put(params, :generator, %{type: "Application", name: client_name, url: website}) + else + Map.put(params, :generator, nil) + end + end defp put_application(params, _), do: Map.put(params, :generator, nil) end -- cgit v1.2.3 From 5856f51717c12f4c6b0b89e480ff689c8480393d Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 3 Mar 2021 23:09:30 +0300 Subject: [#3213] ActivityPub hashtags filtering refactoring. Test fix. --- lib/pleroma/repo.ex | 2 ++ lib/pleroma/web/activity_pub/activity_pub.ex | 29 +++++++++------------------- 2 files changed, 11 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index 61b64ed3e..b8ea06e33 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Repo do adapter: Ecto.Adapters.Postgres, migration_timestamps: [type: :naive_datetime_usec] + use Ecto.Explain + import Ecto.Query require Logger diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9d557c2cd..a4b48ec9b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -746,6 +746,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_embedded_tag_reject_any(query, _), do: query + defp object_ids_query_for_tags(tags) do + from(hto in "hashtags_objects") + |> join(:inner, [hto], ht in Pleroma.Hashtag, on: hto.hashtag_id == ht.id) + |> where([hto, ht], ht.name in ^tags) + |> select([hto], hto.object_id) + end + defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do raise_on_missing_preload() end @@ -784,16 +791,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_hashtag_any(query, %{tag: [_ | _] = tags}) do from( [_activity, object] in query, - where: - fragment( - """ - EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) - AND hashtags_objects.object_id = ? LIMIT 1) - """, - ^tags, - object.id - ) + where: object.id in subquery(object_ids_query_for_tags(tags)) ) end @@ -810,16 +808,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_hashtag_reject_any(query, %{tag_reject: [_ | _] = tags_reject}) do from( [_activity, object] in query, - where: - fragment( - """ - NOT EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) - AND hashtags_objects.object_id = ? LIMIT 1) - """, - ^tags_reject, - object.id - ) + where: object.id not in subquery(object_ids_query_for_tags(tags_reject)) ) end -- cgit v1.2.3 From 9876fa8e902e66a77193ebeef674a9f0e9f37657 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 4 Mar 2021 21:13:53 +0400 Subject: Add UserOperation to Redoc --- lib/pleroma/web/api_spec.ex | 5 +++-- .../web/api_spec/operations/admin/user_operation.ex | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index adc8762dc..528cd9cf4 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -92,9 +92,10 @@ defmodule Pleroma.Web.ApiSpec do "Invites", "MediaProxy cache", "OAuth application managment", - "Report managment", "Relays", - "Status administration" + "Report managment", + "Status administration", + "User administration" ] }, %{"name" => "Applications", "tags" => ["Applications", "Push subscriptions"]}, diff --git a/lib/pleroma/web/api_spec/operations/admin/user_operation.ex b/lib/pleroma/web/api_spec/operations/admin/user_operation.ex index 183c61236..c9d0bfd7c 100644 --- a/lib/pleroma/web/api_spec/operations/admin/user_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/user_operation.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do def index_operation do %Operation{ - tags: ["Users"], + tags: ["User administration"], summary: "List users", operationId: "AdminAPI.UserController.index", security: [%{"oAuth" => ["admin:read:accounts"]}], @@ -63,7 +63,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do def create_operation do %Operation{ - tags: ["Users"], + tags: ["User administration"], summary: "Create a single or multiple users", operationId: "AdminAPI.UserController.create", security: [%{"oAuth" => ["admin:write:accounts"]}], @@ -134,7 +134,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do def show_operation do %Operation{ - tags: ["Users"], + tags: ["User administration"], summary: "Show user", operationId: "AdminAPI.UserController.show", security: [%{"oAuth" => ["admin:read:accounts"]}], @@ -157,7 +157,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do def follow_operation do %Operation{ - tags: ["Users"], + tags: ["User administration"], summary: "Follow", operationId: "AdminAPI.UserController.follow", security: [%{"oAuth" => ["admin:write:follows"]}], @@ -182,7 +182,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do def unfollow_operation do %Operation{ - tags: ["Users"], + tags: ["User administration"], summary: "Unfollow", operationId: "AdminAPI.UserController.unfollow", security: [%{"oAuth" => ["admin:write:follows"]}], @@ -207,7 +207,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do def approve_operation do %Operation{ - tags: ["Users"], + tags: ["User administration"], summary: "Approve multiple users", operationId: "AdminAPI.UserController.approve", security: [%{"oAuth" => ["admin:write:accounts"]}], @@ -239,7 +239,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do def toggle_activation_operation do %Operation{ - tags: ["Users"], + tags: ["User administration"], summary: "Toggle user activation", operationId: "AdminAPI.UserController.toggle_activation", security: [%{"oAuth" => ["admin:write:accounts"]}], @@ -256,7 +256,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do def activate_operation do %Operation{ - tags: ["Users"], + tags: ["User administration"], summary: "Activate multiple users", operationId: "AdminAPI.UserController.activate", security: [%{"oAuth" => ["admin:write:accounts"]}], @@ -288,7 +288,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do def deactivate_operation do %Operation{ - tags: ["Users"], + tags: ["User administration"], summary: "Deactivates multiple users", operationId: "AdminAPI.UserController.deactivate", security: [%{"oAuth" => ["admin:write:accounts"]}], @@ -320,7 +320,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do def delete_operation do %Operation{ - tags: ["Users"], + tags: ["User administration"], summary: "Removes a single or multiple users", operationId: "AdminAPI.UserController.delete", security: [%{"oAuth" => ["admin:write:accounts"]}], -- cgit v1.2.3 From 92ab72dbbb4f56a0e0c3d0882ce29d54739437c1 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 5 Mar 2021 15:51:29 +0400 Subject: Update OpenApiSpex dependency --- lib/pleroma/web/api_spec/cast_and_validate.ex | 31 +++++++++------------- .../web/api_spec/operations/status_operation.ex | 2 +- lib/pleroma/web/api_spec/schemas/boolean_like.ex | 10 ++++--- .../controllers/instance_controller.ex | 2 +- .../pleroma_api/controllers/backup_controller.ex | 2 +- .../web/pleroma_api/controllers/chat_controller.ex | 2 +- .../controllers/user_import_controller.ex | 2 +- 7 files changed, 25 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/cast_and_validate.ex b/lib/pleroma/web/api_spec/cast_and_validate.ex index a3da856ff..d23a7dcb6 100644 --- a/lib/pleroma/web/api_spec/cast_and_validate.ex +++ b/lib/pleroma/web/api_spec/cast_and_validate.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do @behaviour Plug + alias OpenApiSpex.Plug.PutApiSpec alias Plug.Conn @impl Plug @@ -25,12 +26,10 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do end @impl Plug - def call(%{private: %{open_api_spex: private_data}} = conn, %{ - operation_id: operation_id, - render_error: render_error - }) do - spec = private_data.spec - operation = private_data.operation_lookup[operation_id] + + def call(conn, %{operation_id: operation_id, render_error: render_error}) do + {spec, operation_lookup} = PutApiSpec.get_spec_and_operation_lookup(conn) + operation = operation_lookup[operation_id] content_type = case Conn.get_req_header(conn, "content-type") do @@ -43,8 +42,7 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do "application/json" end - private_data = Map.put(private_data, :operation_id, operation_id) - conn = Conn.put_private(conn, :open_api_spex, private_data) + conn = Conn.put_private(conn, :operation_id, operation_id) case cast_and_validate(spec, operation, conn, content_type, strict?()) do {:ok, conn} -> @@ -64,25 +62,22 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do private: %{ phoenix_controller: controller, phoenix_action: action, - open_api_spex: private_data + open_api_spex: %{spec_module: spec_module} } } = conn, opts ) do + {spec, operation_lookup} = PutApiSpec.get_spec_and_operation_lookup(conn) + operation = - case private_data.operation_lookup[{controller, action}] do + case operation_lookup[{controller, action}] do nil -> operation_id = controller.open_api_operation(action).operationId - operation = private_data.operation_lookup[operation_id] + operation = operation_lookup[operation_id] - operation_lookup = - private_data.operation_lookup - |> Map.put({controller, action}, operation) + operation_lookup = Map.put(operation_lookup, {controller, action}, operation) - OpenApiSpex.Plug.Cache.adapter().put( - private_data.spec_module, - {private_data.spec, operation_lookup} - ) + OpenApiSpex.Plug.Cache.adapter().put(spec_module, {spec, operation_lookup}) operation diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 40edc747d..4bdb8e281 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -59,7 +59,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do Operation.response( "Status. When `scheduled_at` is present, ScheduledStatus is returned instead", "application/json", - %Schema{oneOf: [Status, ScheduledStatus]} + %Schema{anyOf: [Status, ScheduledStatus]} ), 422 => Operation.response("Bad Request / MRF Rejection", "application/json", ApiError) } diff --git a/lib/pleroma/web/api_spec/schemas/boolean_like.ex b/lib/pleroma/web/api_spec/schemas/boolean_like.ex index eb001c5bb..778158f66 100644 --- a/lib/pleroma/web/api_spec/schemas/boolean_like.ex +++ b/lib/pleroma/web/api_spec/schemas/boolean_like.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ApiSpec.Schemas.BooleanLike do + alias OpenApiSpex.Cast alias OpenApiSpex.Schema require OpenApiSpex @@ -27,10 +28,13 @@ defmodule Pleroma.Web.ApiSpec.Schemas.BooleanLike do %Schema{type: :boolean}, %Schema{type: :string}, %Schema{type: :integer} - ] + ], + "x-validate": __MODULE__ }) - def after_cast(value, _schmea) do - {:ok, Pleroma.Web.ControllerHelper.truthy_param?(value)} + def cast(%Cast{value: value} = context) do + context + |> Map.put(:value, Pleroma.Web.ControllerHelper.truthy_param?(value)) + |> Cast.ok() end end diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex index 267d0f03b..c7a5267d4 100644 --- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do use Pleroma.Web, :controller - plug(OpenApiSpex.Plug.CastAndValidate) + plug(Pleroma.Web.ApiSpec.CastAndValidate) plug( :skip_plug, diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex index 315657e9c..fc5d16771 100644 --- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do action_fallback(Pleroma.Web.MastodonAPI.FallbackController) plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) - plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError) + plug(Pleroma.Web.ApiSpec.CastAndValidate) defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex index 4adc685fe..dcd54b1af 100644 --- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do %{scopes: ["read:chats"]} when action in [:messages, :index, :index2, :show] ) - plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError) + plug(Pleroma.Web.ApiSpec.CastAndValidate) defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.ChatOperation diff --git a/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex b/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex index 6d9a11fb6..078d470d9 100644 --- a/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Web.PleromaAPI.UserImportController do plug(OAuthScopesPlug, %{scopes: ["follow", "write:blocks"]} when action == :blocks) plug(OAuthScopesPlug, %{scopes: ["follow", "write:mutes"]} when action == :mutes) - plug(OpenApiSpex.Plug.CastAndValidate) + plug(Pleroma.Web.ApiSpec.CastAndValidate) defdelegate open_api_operation(action), to: ApiSpec.UserImportOperation def follow(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do -- cgit v1.2.3 From e97b34f65d71b3dd11aab151fe7ce6def315635a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Mar 2021 13:18:37 -0600 Subject: Add simple way to decode fully qualified mediaproxy URLs --- lib/pleroma/web/media_proxy.ex | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/media_proxy.ex b/lib/pleroma/web/media_proxy.ex index 27f337138..d0d4bb4b3 100644 --- a/lib/pleroma/web/media_proxy.ex +++ b/lib/pleroma/web/media_proxy.ex @@ -121,6 +121,11 @@ defmodule Pleroma.Web.MediaProxy do end end + def decode_url(encoded) do + [_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/") + decode_url(sig, base64) + end + defp signed_url(url) do :crypto.hmac(:sha, Config.get([Web.Endpoint, :secret_key_base]), url) end -- cgit v1.2.3 From 7f8785fd9be11fbb09283c2dbd32aeb7903a4f58 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 7 Mar 2021 11:33:21 +0300 Subject: [#3213] Performance optimization of filtering by hashtags ("any" condition). --- lib/pleroma/pagination.ex | 3 ++ lib/pleroma/web/activity_pub/activity_pub.ex | 47 ++++++++++++++++------ .../controllers/timeline_controller.ex | 43 ++++++++------------ 3 files changed, 55 insertions(+), 38 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pagination.ex b/lib/pleroma/pagination.ex index 0d24e1010..33e45a0eb 100644 --- a/lib/pleroma/pagination.ex +++ b/lib/pleroma/pagination.ex @@ -93,6 +93,7 @@ defmodule Pleroma.Pagination do max_id: :string, offset: :integer, limit: :integer, + skip_extra_order: :boolean, skip_order: :boolean } @@ -114,6 +115,8 @@ defmodule Pleroma.Pagination do defp restrict(query, :order, %{skip_order: true}, _), do: query + defp restrict(%{order_bys: [_ | _]} = query, :order, %{skip_extra_order: true}, _), do: query + defp restrict(query, :order, %{min_id: _}, table_binding) do order_by( query, diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a4b48ec9b..230faf024 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -466,6 +466,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Repo.one() end + defp fetch_paginated_optimized(query, opts, pagination) do + # Note: tag-filtering funcs may apply "ORDER BY objects.id DESC", + # and extra sorting on "activities.id DESC NULLS LAST" would worse the query plan + opts = Map.put(opts, :skip_extra_order, true) + + Pagination.fetch_paginated(query, opts, pagination) + end + + def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do + list_memberships = Pleroma.List.memberships(opts[:user]) + + fetch_activities_query(recipients ++ list_memberships, opts) + |> fetch_paginated_optimized(opts, pagination) + |> Enum.reverse() + |> maybe_update_cc(list_memberships, opts[:user]) + end + @spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()] def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do opts = Map.delete(opts, :user) @@ -473,7 +490,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do [Constants.as_public()] |> fetch_activities_query(opts) |> restrict_unlisted(opts) - |> Pagination.fetch_paginated(opts, pagination) + |> fetch_paginated_optimized(opts, pagination) end @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()] @@ -751,6 +768,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> join(:inner, [hto], ht in Pleroma.Hashtag, on: hto.hashtag_id == ht.id) |> where([hto, ht], ht.name in ^tags) |> select([hto], hto.object_id) + |> distinct([hto], true) end defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do @@ -789,9 +807,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_hashtag_any(query, %{tag: [_ | _] = tags}) do + hashtag_ids = + from(ht in Hashtag, where: ht.name in ^tags, select: ht.id) + |> Repo.all() + + # Note: NO extra ordering should be done on "activities.id desc nulls last" for optimal plan from( [_activity, object] in query, - where: object.id in subquery(object_ids_query_for_tags(tags)) + join: hto in "hashtags_objects", + on: hto.object_id == object.id, + where: hto.hashtag_id in ^hashtag_ids, + distinct: [desc: object.id], + order_by: [desc: object.id] ) end @@ -1188,7 +1215,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Map.put(opts, key, Hashtag.normalize_name(value)) value when is_list(value) -> - Map.put(opts, key, Enum.map(value, &Hashtag.normalize_name/1)) + normalized_value = + value + |> Enum.map(&Hashtag.normalize_name/1) + |> Enum.uniq() + + Map.put(opts, key, normalized_value) _ -> opts @@ -1275,15 +1307,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do - list_memberships = Pleroma.List.memberships(opts[:user]) - - fetch_activities_query(recipients ++ list_memberships, opts) - |> Pagination.fetch_paginated(opts, pagination) - |> Enum.reverse() - |> maybe_update_cc(list_memberships, opts[:user]) - end - @doc """ Fetch favorites activities of user with order by sort adds to favorites """ diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index 87effa00b..c611958be 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -133,34 +133,25 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do end defp hashtag_fetching(params, user, local_only) do - tags = + # Note: not sanitizing tag options at this stage (may be mix-cased, have duplicates etc.) + tags_any = [params[:tag], params[:any]] |> List.flatten() - |> Enum.reject(&is_nil/1) - |> Enum.map(&String.downcase/1) - |> Enum.uniq() - - tag_all = - params - |> Map.get(:all, []) - |> Enum.map(&String.downcase/1) - - tag_reject = - params - |> Map.get(:none, []) - |> Enum.map(&String.downcase/1) - - _activities = - params - |> Map.put(:type, "Create") - |> Map.put(:local_only, local_only) - |> Map.put(:blocking_user, user) - |> Map.put(:muting_user, user) - |> Map.put(:user, user) - |> Map.put(:tag, tags) - |> Map.put(:tag_all, tag_all) - |> Map.put(:tag_reject, tag_reject) - |> ActivityPub.fetch_public_activities() + |> Enum.filter(& &1) + + tag_all = Map.get(params, :all, []) + tag_reject = Map.get(params, :none, []) + + params + |> Map.put(:type, "Create") + |> Map.put(:local_only, local_only) + |> Map.put(:blocking_user, user) + |> Map.put(:muting_user, user) + |> Map.put(:user, user) + |> Map.put(:tag, tags_any) + |> Map.put(:tag_all, tag_all) + |> Map.put(:tag_reject, tag_reject) + |> ActivityPub.fetch_public_activities() end # GET /api/v1/timelines/tag/:tag -- cgit v1.2.3 From 8246db2a968943a0cab615b8b5c1439aa4cb2547 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Mar 2021 12:02:32 -0600 Subject: Workaround for URI.merge/2 bug https://github.com/elixir-lang/elixir/issues/10771 If we avoid URI.merge unless we know we need it we reduce the edge cases we could encounter. The site would need to both have "//" in the %URI{:path} and the image needs to be a relative URL. --- lib/pleroma/web/mastodon_api/views/status_view.ex | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index f3f54e03d..cf8037abb 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -380,9 +380,15 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do page_url = page_url_data |> to_string image_url = - if is_binary(rich_media["image"]) do - URI.merge(page_url_data, URI.parse(rich_media["image"])) - |> to_string + cond do + !is_binary(rich_media["image"]) -> + nil + + String.starts_with?(rich_media["image"], "http") -> + rich_media["image"] + + true -> + URI.merge(page_url_data, URI.parse(rich_media["image"])) |> to_string end %{ -- cgit v1.2.3 From 029ff6538972b59c6259dd7345ad9c4465fb3f73 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 11 Mar 2021 09:20:29 -0600 Subject: Leverage function pattern matching instead --- lib/pleroma/web/mastodon_api/views/status_view.ex | 36 ++++++++++++++++------- 1 file changed, 26 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index cf8037abb..581b4e952 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -379,18 +379,15 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do page_url = page_url_data |> to_string - image_url = - cond do - !is_binary(rich_media["image"]) -> - nil - - String.starts_with?(rich_media["image"], "http") -> - rich_media["image"] - - true -> - URI.merge(page_url_data, URI.parse(rich_media["image"])) |> to_string + image_url_data = + if is_binary(rich_media["image"]) do + URI.parse(rich_media["image"]) + else + nil end + image_url = get_image_url(image_url_data, page_url_data) + %{ type: "link", provider_name: page_url_data.host, @@ -546,4 +543,23 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do do: %{name: name, website: url} defp build_application(_), do: nil + + # Workaround for Elixir issue #10771 + # Avoid applying URI.merge unless necessary + # TODO: revert to always attempting URI.merge(image_url_data, page_url_data) + # when Elixir 1.12 is the minimum supported version + @spec get_image_url(struct() | nil, struct()) :: String.t() | nil + defp get_image_url( + %URI{scheme: image_scheme, host: image_host} = image_url_data, + %URI{} = _page_url_data + ) + when not is_nil(image_scheme) and not is_nil(image_host) do + image_url_data |> to_string + end + + defp get_image_url(%URI{} = image_url_data, %URI{} = page_url_data) do + URI.merge(page_url_data, image_url_data) |> to_string + end + + defp get_image_url(_, _), do: nil end -- cgit v1.2.3 From 3edf45021eb6c3fba06bc083b346f7db54cd073f Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 12 Mar 2021 12:18:11 +0300 Subject: [#3213] Background migration infrastructure refactoring. Extracted BaseMigrator and BaseMigratorState. --- lib/pleroma/application.ex | 11 +- lib/pleroma/migrators/hashtags_table_migrator.ex | 265 ++++----------------- .../migrators/hashtags_table_migrator/state.ex | 104 -------- lib/pleroma/migrators/support/base_migrator.ex | 210 ++++++++++++++++ .../migrators/support/base_migrator_state.ex | 116 +++++++++ 5 files changed, 385 insertions(+), 321 deletions(-) delete mode 100644 lib/pleroma/migrators/hashtags_table_migrator/state.ex create mode 100644 lib/pleroma/migrators/support/base_migrator.ex create mode 100644 lib/pleroma/migrators/support/base_migrator_state.ex (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 2ff7562e2..06d399b2e 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -103,10 +103,7 @@ defmodule Pleroma.Application do task_children(@mix_env) ++ dont_run_in_test(@mix_env) ++ chat_child(chat_enabled?()) ++ - [ - Pleroma.Migrators.HashtagsTableMigrator, - Pleroma.Gopher.Server - ] + [Pleroma.Gopher.Server] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options @@ -231,6 +228,12 @@ defmodule Pleroma.Application do keys: :duplicate, partitions: System.schedulers_online() ]} + ] ++ background_migrators() + end + + defp background_migrators do + [ + Pleroma.Migrators.HashtagsTableMigrator ] end diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index 6123c88e0..b84058e11 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -3,88 +3,27 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Migrators.HashtagsTableMigrator do - use GenServer + defmodule State do + use Pleroma.Migrators.Support.BaseMigratorState - require Logger + @impl Pleroma.Migrators.Support.BaseMigratorState + defdelegate data_migration(), to: Pleroma.DataMigration, as: :populate_hashtags_table + end - import Ecto.Query + use Pleroma.Migrators.Support.BaseMigrator - alias __MODULE__.State - alias Pleroma.Config alias Pleroma.Hashtag + alias Pleroma.Migrators.Support.BaseMigrator alias Pleroma.Object - alias Pleroma.Repo - - defdelegate data_migration(), to: Pleroma.DataMigration, as: :populate_hashtags_table - defdelegate data_migration_id(), to: State - - defdelegate state(), to: State - defdelegate persist_state(), to: State, as: :persist_to_db - defdelegate get_stat(key, value \\ nil), to: State, as: :get_data_key - defdelegate put_stat(key, value), to: State, as: :put_data_key - defdelegate increment_stat(key, increment), to: State, as: :increment_data_key - - @feature_config_path [:features, :improved_hashtag_timeline] - @reg_name {:global, __MODULE__} - - def whereis, do: GenServer.whereis(@reg_name) - - def feature_state, do: Config.get(@feature_config_path) - - def start_link(_) do - case whereis() do - nil -> - GenServer.start_link(__MODULE__, nil, name: @reg_name) - - pid -> - {:ok, pid} - end - end - - @impl true - def init(_) do - {:ok, nil, {:continue, :init_state}} - end - @impl true - def handle_continue(:init_state, _state) do - {:ok, _} = State.start_link(nil) + @impl BaseMigrator + def feature_config_path, do: [:features, :improved_hashtag_timeline] - data_migration = data_migration() - manual_migrations = Config.get([:instance, :manual_data_migrations], []) - - cond do - Config.get(:env) == :test -> - update_status(:noop) - - is_nil(data_migration) -> - message = "Data migration does not exist." - update_status(:failed, message) - Logger.error("#{__MODULE__}: #{message}") - - data_migration.state == :manual or data_migration.name in manual_migrations -> - message = "Data migration is in manual execution or manual fix mode." - update_status(:manual, message) - Logger.warn("#{__MODULE__}: #{message}") - - data_migration.state == :complete -> - on_complete(data_migration) - - true -> - send(self(), :migrate_hashtags) - end - - {:noreply, nil} - end - - @impl true - def handle_info(:migrate_hashtags, state) do - State.reinit() - - update_status(:running) - put_stat(:iteration_processed_count, 0) - put_stat(:started_at, NaiveDateTime.utc_now()) + @impl BaseMigrator + def fault_rate_allowance, do: Config.get([:populate_hashtags_table, :fault_rate_allowance], 0) + @impl BaseMigrator + def perform do data_migration_id = data_migration_id() max_processed_id = get_stat(:max_processed_id, 0) @@ -103,7 +42,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do |> Enum.filter(&(elem(&1, 0) == :error)) |> Enum.map(&elem(&1, 1)) - # Count of objects with hashtags (`{:noop, id}` is returned for objects having other AS2 tags) + # Count of objects with hashtags: `{:noop, id}` is returned for objects having other AS2 tags chunk_affected_count = results |> Enum.filter(&(elem(&1, 0) == :ok)) @@ -140,84 +79,10 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do Process.sleep(sleep_interval) end) |> Stream.run() - - fault_rate = fault_rate() - put_stat(:fault_rate, fault_rate) - fault_rate_allowance = Config.get([:populate_hashtags_table, :fault_rate_allowance], 0) - - cond do - fault_rate == 0 -> - set_complete() - - is_float(fault_rate) and fault_rate <= fault_rate_allowance -> - message = """ - Done with fault rate of #{fault_rate} which doesn't exceed #{fault_rate_allowance}. - Putting data migration to manual fix mode. Check `retry_failed/0`. - """ - - Logger.warn("#{__MODULE__}: #{message}") - update_status(:manual, message) - on_complete(data_migration()) - - true -> - message = "Too many failures. Check data_migration_failed_ids records / `retry_failed/0`." - Logger.error("#{__MODULE__}: #{message}") - update_status(:failed, message) - end - - persist_state() - {:noreply, state} - end - - def fault_rate do - with failures_count when is_integer(failures_count) <- failures_count() do - failures_count / Enum.max([get_stat(:affected_count, 0), 1]) - else - _ -> :error - end - end - - defp records_per_second do - get_stat(:iteration_processed_count, 0) / Enum.max([running_time(), 1]) - end - - defp running_time do - NaiveDateTime.diff(NaiveDateTime.utc_now(), get_stat(:started_at, NaiveDateTime.utc_now())) - end - - @hashtags_objects_cleanup_query """ - DELETE FROM hashtags_objects WHERE object_id IN - (SELECT DISTINCT objects.id FROM objects - JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities - ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = - (objects.data->>'id') - AND activities.data->>'type' = 'Create' - WHERE activities.id IS NULL); - """ - - @hashtags_cleanup_query """ - DELETE FROM hashtags WHERE id IN - (SELECT hashtags.id FROM hashtags - LEFT OUTER JOIN hashtags_objects - ON hashtags_objects.hashtag_id = hashtags.id - WHERE hashtags_objects.hashtag_id IS NULL); - """ - - @doc """ - Deletes `hashtags_objects` for legacy objects not asoociated with Create activity. - Also deletes unreferenced `hashtags` records (might occur after deletion of `hashtags_objects`). - """ - def delete_non_create_activities_hashtags do - {:ok, %{num_rows: hashtags_objects_count}} = - Repo.query(@hashtags_objects_cleanup_query, [], timeout: :infinity) - - {:ok, %{num_rows: hashtags_count}} = - Repo.query(@hashtags_cleanup_query, [], timeout: :infinity) - - {:ok, hashtags_objects_count, hashtags_count} end - defp query do + @impl BaseMigrator + def query do # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out) # Note: not checking activity type, expecting remove_non_create_objects_hashtags/_ to clean up from( @@ -276,54 +141,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do end) end - @doc "Approximate count for current iteration (including processed records count)" - def count(force \\ false, timeout \\ :infinity) do - stored_count = get_stat(:count) - - if stored_count && !force do - stored_count - else - processed_count = get_stat(:processed_count, 0) - max_processed_id = get_stat(:max_processed_id, 0) - query = where(query(), [object], object.id > ^max_processed_id) - - count = Repo.aggregate(query, :count, :id, timeout: timeout) + processed_count - put_stat(:count, count) - persist_state() - - count - end - end - - defp on_complete(data_migration) do - if data_migration.feature_lock || feature_state() == :disabled do - Logger.warn("#{__MODULE__}: migration complete but feature is locked; consider enabling.") - :noop - else - Config.put(@feature_config_path, :enabled) - :ok - end - end - - def failed_objects_query do - from(o in Object) - |> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"), - on: dmf.record_id == o.id - ) - |> where([_o, dmf], dmf.data_migration_id == ^data_migration_id()) - |> order_by([o], asc: o.id) - end - - def failures_count do - with {:ok, %{rows: [[count]]}} <- - Repo.query( - "SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;", - [data_migration_id()] - ) do - count - end - end - + @impl BaseMigrator def retry_failed do data_migration_id = data_migration_id() @@ -347,23 +165,44 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do force_continue() end - def force_continue do - send(whereis(), :migrate_hashtags) + defp failed_objects_query do + from(o in Object) + |> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"), + on: dmf.record_id == o.id + ) + |> where([_o, dmf], dmf.data_migration_id == ^data_migration_id()) + |> order_by([o], asc: o.id) end - def force_restart do - :ok = State.reset() - force_continue() - end + @doc """ + Service func to delete `hashtags_objects` for legacy objects not associated with Create activity. + Also deletes unreferenced `hashtags` records (might occur after deletion of `hashtags_objects`). + """ + def delete_non_create_activities_hashtags do + hashtags_objects_cleanup_query = """ + DELETE FROM hashtags_objects WHERE object_id IN + (SELECT DISTINCT objects.id FROM objects + JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities + ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = + (objects.data->>'id') + AND activities.data->>'type' = 'Create' + WHERE activities.id IS NULL); + """ + + hashtags_cleanup_query = """ + DELETE FROM hashtags WHERE id IN + (SELECT hashtags.id FROM hashtags + LEFT OUTER JOIN hashtags_objects + ON hashtags_objects.hashtag_id = hashtags.id + WHERE hashtags_objects.hashtag_id IS NULL); + """ - def set_complete do - update_status(:complete) - persist_state() - on_complete(data_migration()) - end + {:ok, %{num_rows: hashtags_objects_count}} = + Repo.query(hashtags_objects_cleanup_query, [], timeout: :infinity) - defp update_status(status, message \\ nil) do - put_stat(:state, status) - put_stat(:message, message) + {:ok, %{num_rows: hashtags_count}} = + Repo.query(hashtags_cleanup_query, [], timeout: :infinity) + + {:ok, hashtags_objects_count, hashtags_count} end end diff --git a/lib/pleroma/migrators/hashtags_table_migrator/state.ex b/lib/pleroma/migrators/hashtags_table_migrator/state.ex deleted file mode 100644 index ee0009b2e..000000000 --- a/lib/pleroma/migrators/hashtags_table_migrator/state.ex +++ /dev/null @@ -1,104 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Migrators.HashtagsTableMigrator.State do - use Agent - - alias Pleroma.DataMigration - - defdelegate data_migration(), to: Pleroma.Migrators.HashtagsTableMigrator - - @reg_name {:global, __MODULE__} - - def start_link(_) do - Agent.start_link(fn -> load_state_from_db() end, name: @reg_name) - end - - defp load_state_from_db do - data_migration = data_migration() - - data = - if data_migration do - Map.new(data_migration.data, fn {k, v} -> {String.to_atom(k), v} end) - else - %{} - end - - %{ - data_migration_id: data_migration && data_migration.id, - data: data - } - end - - def persist_to_db do - %{data_migration_id: data_migration_id, data: data} = state() - - if data_migration_id do - DataMigration.update_one_by_id(data_migration_id, data: data) - else - {:error, :nil_data_migration_id} - end - end - - def reset do - %{data_migration_id: data_migration_id} = state() - - with false <- is_nil(data_migration_id), - :ok <- - DataMigration.update_one_by_id(data_migration_id, - state: :pending, - data: %{} - ) do - reinit() - else - true -> {:error, :nil_data_migration_id} - e -> e - end - end - - def reinit do - Agent.update(@reg_name, fn _state -> load_state_from_db() end) - end - - def state do - Agent.get(@reg_name, & &1) - end - - def get_data_key(key, default \\ nil) do - get_in(state(), [:data, key]) || default - end - - def put_data_key(key, value) do - _ = persist_non_data_change(key, value) - - Agent.update(@reg_name, fn state -> - put_in(state, [:data, key], value) - end) - end - - def increment_data_key(key, increment \\ 1) do - Agent.update(@reg_name, fn state -> - initial_value = get_in(state, [:data, key]) || 0 - updated_value = initial_value + increment - put_in(state, [:data, key], updated_value) - end) - end - - defp persist_non_data_change(:state, value) do - with true <- get_data_key(:state) != value, - true <- value in Pleroma.DataMigration.State.__valid_values__(), - %{data_migration_id: data_migration_id} when not is_nil(data_migration_id) <- state() do - DataMigration.update_one_by_id(data_migration_id, state: value) - else - false -> :ok - _ -> {:error, :nil_data_migration_id} - end - end - - defp persist_non_data_change(_, _) do - nil - end - - def data_migration_id, do: Map.get(state(), :data_migration_id) -end diff --git a/lib/pleroma/migrators/support/base_migrator.ex b/lib/pleroma/migrators/support/base_migrator.ex new file mode 100644 index 000000000..1f8a5402b --- /dev/null +++ b/lib/pleroma/migrators/support/base_migrator.ex @@ -0,0 +1,210 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Migrators.Support.BaseMigrator do + @moduledoc """ + Base background migrator functionality. + """ + + @callback perform() :: any() + @callback retry_failed() :: any() + @callback feature_config_path() :: list(atom()) + @callback query() :: Ecto.Query.t() + @callback fault_rate_allowance() :: integer() | float() + + defmacro __using__(_opts) do + quote do + use GenServer + + require Logger + + import Ecto.Query + + alias __MODULE__.State + alias Pleroma.Config + alias Pleroma.Repo + + @behaviour Pleroma.Migrators.Support.BaseMigrator + + defdelegate data_migration(), to: State + defdelegate data_migration_id(), to: State + defdelegate state(), to: State + defdelegate persist_state(), to: State, as: :persist_to_db + defdelegate get_stat(key, value \\ nil), to: State, as: :get_data_key + defdelegate put_stat(key, value), to: State, as: :put_data_key + defdelegate increment_stat(key, increment), to: State, as: :increment_data_key + + @reg_name {:global, __MODULE__} + + def whereis, do: GenServer.whereis(@reg_name) + + def start_link(_) do + case whereis() do + nil -> + GenServer.start_link(__MODULE__, nil, name: @reg_name) + + pid -> + {:ok, pid} + end + end + + @impl true + def init(_) do + {:ok, nil, {:continue, :init_state}} + end + + @impl true + def handle_continue(:init_state, _state) do + {:ok, _} = State.start_link(nil) + + data_migration = data_migration() + manual_migrations = Config.get([:instance, :manual_data_migrations], []) + + cond do + Config.get(:env) == :test -> + update_status(:noop) + + is_nil(data_migration) -> + message = "Data migration does not exist." + update_status(:failed, message) + Logger.error("#{__MODULE__}: #{message}") + + data_migration.state == :manual or data_migration.name in manual_migrations -> + message = "Data migration is in manual execution or manual fix mode." + update_status(:manual, message) + Logger.warn("#{__MODULE__}: #{message}") + + data_migration.state == :complete -> + on_complete(data_migration) + + true -> + send(self(), :perform) + end + + {:noreply, nil} + end + + @impl true + def handle_info(:perform, state) do + State.reinit() + + update_status(:running) + put_stat(:iteration_processed_count, 0) + put_stat(:started_at, NaiveDateTime.utc_now()) + + perform() + + fault_rate = fault_rate() + put_stat(:fault_rate, fault_rate) + fault_rate_allowance = fault_rate_allowance() + + cond do + fault_rate == 0 -> + set_complete() + + is_float(fault_rate) and fault_rate <= fault_rate_allowance -> + message = """ + Done with fault rate of #{fault_rate} which doesn't exceed #{fault_rate_allowance}. + Putting data migration to manual fix mode. Try running `#{__MODULE__}.retry_failed/0`. + """ + + Logger.warn("#{__MODULE__}: #{message}") + update_status(:manual, message) + on_complete(data_migration()) + + true -> + message = "Too many failures. Try running `#{__MODULE__}.retry_failed/0`." + Logger.error("#{__MODULE__}: #{message}") + update_status(:failed, message) + end + + persist_state() + {:noreply, state} + end + + defp on_complete(data_migration) do + if data_migration.feature_lock || feature_state() == :disabled do + Logger.warn( + "#{__MODULE__}: migration complete but feature is locked; consider enabling." + ) + + :noop + else + Config.put(feature_config_path(), :enabled) + :ok + end + end + + @doc "Approximate count for current iteration (including processed records count)" + def count(force \\ false, timeout \\ :infinity) do + stored_count = get_stat(:count) + + if stored_count && !force do + stored_count + else + processed_count = get_stat(:processed_count, 0) + max_processed_id = get_stat(:max_processed_id, 0) + query = where(query(), [entity], entity.id > ^max_processed_id) + + count = Repo.aggregate(query, :count, :id, timeout: timeout) + processed_count + put_stat(:count, count) + persist_state() + + count + end + end + + def failures_count do + with {:ok, %{rows: [[count]]}} <- + Repo.query( + "SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;", + [data_migration_id()] + ) do + count + end + end + + def feature_state, do: Config.get(feature_config_path()) + + def force_continue do + send(whereis(), :perform) + end + + def force_restart do + :ok = State.reset() + force_continue() + end + + def set_complete do + update_status(:complete) + persist_state() + on_complete(data_migration()) + end + + defp update_status(status, message \\ nil) do + put_stat(:state, status) + put_stat(:message, message) + end + + defp fault_rate do + with failures_count when is_integer(failures_count) <- failures_count() do + failures_count / Enum.max([get_stat(:affected_count, 0), 1]) + else + _ -> :error + end + end + + defp records_per_second do + get_stat(:iteration_processed_count, 0) / Enum.max([running_time(), 1]) + end + + defp running_time do + NaiveDateTime.diff( + NaiveDateTime.utc_now(), + get_stat(:started_at, NaiveDateTime.utc_now()) + ) + end + end + end +end diff --git a/lib/pleroma/migrators/support/base_migrator_state.ex b/lib/pleroma/migrators/support/base_migrator_state.ex new file mode 100644 index 000000000..69724ae79 --- /dev/null +++ b/lib/pleroma/migrators/support/base_migrator_state.ex @@ -0,0 +1,116 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Migrators.Support.BaseMigratorState do + @moduledoc """ + Base background migrator state functionality. + """ + + @callback data_migration() :: Pleroma.DataMigration.t() + + defmacro __using__(_opts) do + quote do + use Agent + + alias Pleroma.DataMigration + + @behaviour Pleroma.Migrators.Support.BaseMigratorState + @reg_name {:global, __MODULE__} + + def start_link(_) do + Agent.start_link(fn -> load_state_from_db() end, name: @reg_name) + end + + def data_migration, do: raise("data_migration/0 is not implemented") + defoverridable data_migration: 0 + + defp load_state_from_db do + data_migration = data_migration() + + data = + if data_migration do + Map.new(data_migration.data, fn {k, v} -> {String.to_atom(k), v} end) + else + %{} + end + + %{ + data_migration_id: data_migration && data_migration.id, + data: data + } + end + + def persist_to_db do + %{data_migration_id: data_migration_id, data: data} = state() + + if data_migration_id do + DataMigration.update_one_by_id(data_migration_id, data: data) + else + {:error, :nil_data_migration_id} + end + end + + def reset do + %{data_migration_id: data_migration_id} = state() + + with false <- is_nil(data_migration_id), + :ok <- + DataMigration.update_one_by_id(data_migration_id, + state: :pending, + data: %{} + ) do + reinit() + else + true -> {:error, :nil_data_migration_id} + e -> e + end + end + + def reinit do + Agent.update(@reg_name, fn _state -> load_state_from_db() end) + end + + def state do + Agent.get(@reg_name, & &1) + end + + def get_data_key(key, default \\ nil) do + get_in(state(), [:data, key]) || default + end + + def put_data_key(key, value) do + _ = persist_non_data_change(key, value) + + Agent.update(@reg_name, fn state -> + put_in(state, [:data, key], value) + end) + end + + def increment_data_key(key, increment \\ 1) do + Agent.update(@reg_name, fn state -> + initial_value = get_in(state, [:data, key]) || 0 + updated_value = initial_value + increment + put_in(state, [:data, key], updated_value) + end) + end + + defp persist_non_data_change(:state, value) do + with true <- get_data_key(:state) != value, + true <- value in Pleroma.DataMigration.State.__valid_values__(), + %{data_migration_id: data_migration_id} when not is_nil(data_migration_id) <- state() do + DataMigration.update_one_by_id(data_migration_id, state: value) + else + false -> :ok + _ -> {:error, :nil_data_migration_id} + end + end + + defp persist_non_data_change(_, _) do + nil + end + + def data_migration_id, do: Map.get(state(), :data_migration_id) + end + end +end -- cgit v1.2.3 From cb734566093f406fc3db12de2408fc166486f417 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 12 Mar 2021 12:25:18 +0300 Subject: [#3213] Code formatting fix. --- lib/pleroma/migrators/support/base_migrator_state.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/migrators/support/base_migrator_state.ex b/lib/pleroma/migrators/support/base_migrator_state.ex index 69724ae79..b698587f2 100644 --- a/lib/pleroma/migrators/support/base_migrator_state.ex +++ b/lib/pleroma/migrators/support/base_migrator_state.ex @@ -98,7 +98,8 @@ defmodule Pleroma.Migrators.Support.BaseMigratorState do defp persist_non_data_change(:state, value) do with true <- get_data_key(:state) != value, true <- value in Pleroma.DataMigration.State.__valid_values__(), - %{data_migration_id: data_migration_id} when not is_nil(data_migration_id) <- state() do + %{data_migration_id: data_migration_id} when not is_nil(data_migration_id) <- + state() do DataMigration.update_one_by_id(data_migration_id, state: value) else false -> :ok -- cgit v1.2.3 From b80f868c6b41d1407cf6e4f2df8913bdf7a954c0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 13 Mar 2021 12:27:15 -0600 Subject: Prefer naming this function build_image_url/2 --- lib/pleroma/web/mastodon_api/views/status_view.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 581b4e952..71f659ba0 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -386,7 +386,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do nil end - image_url = get_image_url(image_url_data, page_url_data) + image_url = build_image_url(image_url_data, page_url_data) %{ type: "link", @@ -548,8 +548,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # Avoid applying URI.merge unless necessary # TODO: revert to always attempting URI.merge(image_url_data, page_url_data) # when Elixir 1.12 is the minimum supported version - @spec get_image_url(struct() | nil, struct()) :: String.t() | nil - defp get_image_url( + @spec build_image_url(struct() | nil, struct()) :: String.t() | nil + defp build_image_url( %URI{scheme: image_scheme, host: image_host} = image_url_data, %URI{} = _page_url_data ) @@ -557,9 +557,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do image_url_data |> to_string end - defp get_image_url(%URI{} = image_url_data, %URI{} = page_url_data) do + defp build_image_url(%URI{} = image_url_data, %URI{} = page_url_data) do URI.merge(page_url_data, image_url_data) |> to_string end - defp get_image_url(_, _), do: nil + defp build_image_url(_, _), do: nil end -- cgit v1.2.3 From b1d4b2b81ec97143c41d16ac3f5bc2825b836f4b Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 15 Mar 2021 06:43:12 +0100 Subject: Add support for actor icon being a list (Bridgy) --- lib/pleroma/web/activity_pub/activity_pub.ex | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5b45e2ca1..ff478f44c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1250,21 +1250,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp get_actor_url(_url), do: nil - defp object_to_user_data(data) do - avatar = - data["icon"]["url"] && - %{ - "type" => "Image", - "url" => [%{"href" => data["icon"]["url"]}] - } + defp normalize_image(%{"url" => url}) do + %{ + "type" => "Image", + "url" => [%{"href" => url}] + } + end - banner = - data["image"]["url"] && - %{ - "type" => "Image", - "url" => [%{"href" => data["image"]["url"]}] - } + defp normalize_image(urls) when is_list(urls), do: urls |> List.first() |> normalize_image() + defp normalize_image(_), do: nil + defp object_to_user_data(data) do fields = data |> Map.get("attachment", []) @@ -1308,13 +1304,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do ap_id: data["id"], uri: get_actor_url(data["url"]), ap_enabled: true, - banner: banner, + banner: normalize_image(data["image"]), fields: fields, emoji: emojis, is_locked: is_locked, is_discoverable: is_discoverable, invisible: invisible, - avatar: avatar, + avatar: normalize_image(data["icon"]), name: data["name"], follower_address: data["followers"], following_address: data["following"], -- cgit v1.2.3 From 7eecc3b61d6da64e0bfdc5b155cba0dae07b84d5 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 16 Feb 2021 23:23:35 +0100 Subject: OpenAPI: MastodonAPI Timeline Controller --- lib/pleroma/web/api_spec/operations/timeline_operation.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex index cae18c758..24d792916 100644 --- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex +++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex @@ -115,7 +115,8 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do ], operationId: "TimelineController.hashtag", responses: %{ - 200 => Operation.response("Array of Status", "application/json", array_of_statuses()) + 200 => Operation.response("Array of Status", "application/json", array_of_statuses()), + 401 => Operation.response("Error", "application/json", ApiError) } } end -- cgit v1.2.3 From 3123ecdd6e7a189f815624ee78be4f62487aa3db Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 16 Feb 2021 23:37:16 +0100 Subject: OpenAPI: MastodonAPI Media Controller --- lib/pleroma/web/api_spec/operations/media_operation.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/media_operation.ex b/lib/pleroma/web/api_spec/operations/media_operation.ex index 85aa14869..1e245b291 100644 --- a/lib/pleroma/web/api_spec/operations/media_operation.ex +++ b/lib/pleroma/web/api_spec/operations/media_operation.ex @@ -105,6 +105,7 @@ defmodule Pleroma.Web.ApiSpec.MediaOperation do responses: %{ 200 => Operation.response("Media", "application/json", Attachment), 401 => Operation.response("Media", "application/json", ApiError), + 403 => Operation.response("Media", "application/json", ApiError), 422 => Operation.response("Media", "application/json", ApiError) } } -- cgit v1.2.3 From e4743847a18cb7cbb9e607232f25eb1cf63a4551 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 17 Feb 2021 01:07:56 +0100 Subject: OpenAPI: PleromaAPI UserImport Controller --- lib/pleroma/web/api_spec/operations/user_import_operation.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/user_import_operation.ex b/lib/pleroma/web/api_spec/operations/user_import_operation.ex index 6292e2004..8df19f1fc 100644 --- a/lib/pleroma/web/api_spec/operations/user_import_operation.ex +++ b/lib/pleroma/web/api_spec/operations/user_import_operation.ex @@ -23,6 +23,7 @@ defmodule Pleroma.Web.ApiSpec.UserImportOperation do requestBody: request_body("Parameters", import_request(), required: true), responses: %{ 200 => ok_response(), + 403 => Operation.response("Error", "application/json", ApiError), 500 => Operation.response("Error", "application/json", ApiError) }, security: [%{"oAuth" => ["write:follow"]}] -- cgit v1.2.3 From a22c53810b36c5382c805e1c5ed7e1cf3d747ebc Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 17 Feb 2021 01:19:25 +0100 Subject: Remove deprecated /api/qvitter/statuses/notifications/read --- lib/pleroma/web/router.ex | 6 ------ lib/pleroma/web/twitter_api/controller.ex | 33 ------------------------------- 2 files changed, 39 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index de0bd27d7..ce2d701d7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -620,12 +620,6 @@ defmodule Pleroma.Web.Router do get("/oauth_tokens", TwitterAPI.Controller, :oauth_tokens) delete("/oauth_tokens/:id", TwitterAPI.Controller, :revoke_token) - - post( - "/qvitter/statuses/notifications/read", - TwitterAPI.Controller, - :mark_notifications_as_read - ) end scope "/", Pleroma.Web do diff --git a/lib/pleroma/web/twitter_api/controller.ex b/lib/pleroma/web/twitter_api/controller.ex index 077bfa70d..e32713311 100644 --- a/lib/pleroma/web/twitter_api/controller.ex +++ b/lib/pleroma/web/twitter_api/controller.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do use Pleroma.Web, :controller - alias Pleroma.Notification alias Pleroma.User alias Pleroma.Web.OAuth.Token alias Pleroma.Web.Plugs.EnsurePublicOrAuthenticatedPlug @@ -14,11 +13,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do require Logger - plug( - OAuthScopesPlug, - %{scopes: ["write:notifications"]} when action == :mark_notifications_as_read - ) - plug( :skip_plug, [OAuthScopesPlug, EnsurePublicOrAuthenticatedPlug] when action == :confirm_email @@ -67,31 +61,4 @@ defmodule Pleroma.Web.TwitterAPI.Controller do |> put_resp_content_type("application/json") |> send_resp(status, json) end - - def mark_notifications_as_read( - %{assigns: %{user: user}} = conn, - %{"latest_id" => latest_id} = params - ) do - Notification.set_read_up_to(user, latest_id) - - notifications = Notification.for_user(user, params) - - conn - # XXX: This is a hack because pleroma-fe still uses that API. - |> put_view(Pleroma.Web.MastodonAPI.NotificationView) - |> render("index.json", %{notifications: notifications, for: user}) - end - - def mark_notifications_as_read(%{assigns: %{user: _user}} = conn, _) do - bad_request_reply(conn, "You need to specify latest_id") - end - - defp bad_request_reply(conn, error_message) do - json = error_json(conn, error_message) - json_reply(conn, 400, json) - end - - defp error_json(conn, error_message) do - %{"error" => error_message, "request" => conn.request_path} |> Jason.encode!() - end end -- cgit v1.2.3 From 65cd9cb6384676c1660aa7f4da0f98ff7f43b999 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 17 Feb 2021 09:41:40 +0100 Subject: TwitterAPI: Remove unused read notification function --- lib/pleroma/web/twitter_api/controllers/util_controller.ex | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 940a645bb..60266aaab 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -10,7 +10,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Config alias Pleroma.Emoji alias Pleroma.Healthcheck - alias Pleroma.Notification alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.Plugs.OAuthScopesPlug @@ -30,7 +29,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do ] ) - plug(OAuthScopesPlug, %{scopes: ["write:notifications"]} when action == :notifications_read) def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do with %User{} = user <- User.get_cached_by_nickname(nick), @@ -62,17 +60,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end - def notifications_read(%{assigns: %{user: user}} = conn, %{"id" => notification_id}) do - with {:ok, _} <- Notification.read_one(user, notification_id) do - json(conn, %{status: "success"}) - else - {:error, message} -> - conn - |> put_resp_content_type("application/json") - |> send_resp(403, Jason.encode!(%{"error" => message})) - end - end - def frontend_configurations(conn, _params) do render(conn, "frontend_configurations.json") end -- cgit v1.2.3 From 55bdfb075c1cc5226948e3ff9d39fdae27aa9257 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 24 Feb 2021 23:40:33 +0100 Subject: OpenAPI: TwitterAPI Util Controller --- .../api_spec/operations/twitter_util_operation.ex | 219 +++++++++++++++++++++ .../web/twitter_api/controllers/util_controller.ex | 24 ++- 2 files changed, 234 insertions(+), 9 deletions(-) create mode 100644 lib/pleroma/web/api_spec/operations/twitter_util_operation.ex (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex new file mode 100644 index 000000000..62c9826f6 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -0,0 +1,219 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.BooleanLike + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def emoji_operation do + %Operation{ + tags: ["Emojis"], + summary: "List all custom emojis", + operationId: "UtilController.emoji", + parameters: [], + responses: %{ + 200 => + Operation.response("List", "application/json", %Schema{ + type: :object, + additionalProperties: %Schema{ + type: :object, + properties: %{ + image_url: %Schema{type: :string}, + tags: %Schema{type: :array, items: %Schema{type: :string}} + } + }, + example: %{ + "firefox" => %{ + "image_url" => "/emoji/firefox.png", + "tag" => ["Fun"] + } + } + }) + } + } + end + + def frontend_configurations_operation do + %Operation{ + tags: ["Configuration"], + summary: "Dump frontend configurations", + operationId: "UtilController.frontend_configurations", + parameters: [], + responses: %{ + 200 => + Operation.response("List", "application/json", %Schema{ + type: :object, + additionalProperties: %Schema{type: :object} + }) + } + } + end + + def change_password_operation do + %Operation{ + tags: ["Accounts"], + summary: "Change account password", + security: [%{"oAuth" => ["write:accounts"]}], + operationId: "UtilController.change_password", + parameters: [ + Operation.parameter(:password, :query, :string, "Current password", required: true), + Operation.parameter(:new_password, :query, :string, "New password", required: true), + Operation.parameter( + :new_password_confirmation, + :query, + :string, + "New password, confirmation", + required: true + ) + ], + responses: %{ + 200 => + Operation.response("Success", "application/json", %Schema{ + type: :object, + properties: %{status: %Schema{type: :string, example: "success"}} + }), + 400 => Operation.response("Error", "application/json", ApiError), + 403 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def change_email_operation do + %Operation{ + tags: ["Accounts"], + summary: "Change account email", + security: [%{"oAuth" => ["write:accounts"]}], + operationId: "UtilController.change_email", + parameters: [ + Operation.parameter(:password, :query, :string, "Current password", required: true), + Operation.parameter(:email, :query, :string, "New email", required: true) + ], + requestBody: nil, + responses: %{ + 200 => + Operation.response("Success", "application/json", %Schema{ + type: :object, + properties: %{status: %Schema{type: :string, example: "success"}} + }), + 400 => Operation.response("Error", "application/json", ApiError), + 403 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def update_notificaton_settings_operation do + %Operation{ + tags: ["Accounts"], + summary: "Update Notification Settings", + security: [%{"oAuth" => ["write:accounts"]}], + operationId: "UtilController.update_notificaton_settings", + parameters: [ + Operation.parameter( + :block_from_strangers, + :query, + BooleanLike, + "blocks notifications from accounts you do not follow" + ), + Operation.parameter( + :hide_notification_contents, + :query, + BooleanLike, + "removes the contents of a message from the push notification" + ) + ], + requestBody: nil, + responses: %{ + 200 => + Operation.response("Success", "application/json", %Schema{ + type: :object, + properties: %{status: %Schema{type: :string, example: "success"}} + }), + 400 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def disable_account_operation do + %Operation{ + tags: ["Accounts"], + summary: "Disable Account", + security: [%{"oAuth" => ["write:accounts"]}], + operationId: "UtilController.disable_account", + parameters: [ + Operation.parameter(:password, :query, :string, "Password") + ], + responses: %{ + 200 => + Operation.response("Success", "application/json", %Schema{ + type: :object, + properties: %{status: %Schema{type: :string, example: "success"}} + }), + 403 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def delete_account_operation do + %Operation{ + tags: ["Accounts"], + summary: "Delete Account", + security: [%{"oAuth" => ["write:accounts"]}], + operationId: "UtilController.delete_account", + parameters: [ + Operation.parameter(:password, :query, :string, "Password") + ], + responses: %{ + 200 => + Operation.response("Success", "application/json", %Schema{ + type: :object, + properties: %{status: %Schema{type: :string, example: "success"}} + }), + 403 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def captcha_operation do + %Operation{ + summary: "Get a captcha", + operationId: "UtilController.captcha", + parameters: [], + responses: %{ + 200 => Operation.response("Success", "application/json", %Schema{type: :object}) + } + } + end + + def healthcheck_operation do + %Operation{ + tags: ["Accounts"], + summary: "Disable Account", + security: [%{"oAuth" => ["write:accounts"]}], + operationId: "UtilController.healthcheck", + parameters: [], + responses: %{ + 200 => Operation.response("Healthy", "application/json", %Schema{type: :object}), + 503 => + Operation.response("Disabled or Unhealthy", "application/json", %Schema{type: :object}) + } + } + end + + def remote_subscribe_operation do + %Operation{ + tags: ["Accounts"], + summary: "Remote Subscribe", + operationId: "UtilController.remote_subscribe", + parameters: [], + responses: %{200 => Operation.response("Web Page", "test/html", %Schema{type: :string})} + } + end +end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 60266aaab..a2e69666e 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.Web.WebFinger + plug(Pleroma.Web.ApiSpec.CastAndValidate when action != :remote_subscribe) plug(Pleroma.Web.Plugs.FederatingPlug when action == :remote_subscribe) plug( @@ -29,6 +30,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do ] ) + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.TwitterUtilOperation def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do with %User{} = user <- User.get_cached_by_nickname(nick), @@ -79,13 +81,17 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end - def change_password(%{assigns: %{user: user}} = conn, params) do - case CommonAPI.Utils.confirm_current_password(user, params["password"]) do + def change_password(%{assigns: %{user: user}} = conn, %{ + password: password, + new_password: new_password, + new_password_confirmation: new_password_confirmation + }) do + case CommonAPI.Utils.confirm_current_password(user, password) do {:ok, user} -> with {:ok, _user} <- User.reset_password(user, %{ - password: params["new_password"], - password_confirmation: params["new_password_confirmation"] + password: new_password, + password_confirmation: new_password_confirmation }) do json(conn, %{status: "success"}) else @@ -102,10 +108,10 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end - def change_email(%{assigns: %{user: user}} = conn, params) do - case CommonAPI.Utils.confirm_current_password(user, params["password"]) do + def change_email(%{assigns: %{user: user}} = conn, %{password: password, email: email}) do + case CommonAPI.Utils.confirm_current_password(user, password) do {:ok, user} -> - with {:ok, _user} <- User.change_email(user, params["email"]) do + with {:ok, _user} <- User.change_email(user, email) do json(conn, %{status: "success"}) else {:error, changeset} -> @@ -122,7 +128,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end def delete_account(%{assigns: %{user: user}} = conn, params) do - password = params["password"] || "" + password = params[:password] || "" case CommonAPI.Utils.confirm_current_password(user, password) do {:ok, user} -> @@ -135,7 +141,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end def disable_account(%{assigns: %{user: user}} = conn, params) do - case CommonAPI.Utils.confirm_current_password(user, params["password"]) do + case CommonAPI.Utils.confirm_current_password(user, params[:password]) do {:ok, user} -> User.set_activation_async(user, false) json(conn, %{status: "success"}) -- cgit v1.2.3 From d7e51206a251b9da0180a4df3c879531ac302e1a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 18 Mar 2021 13:49:03 +0300 Subject: respect content-type header in finger request --- lib/pleroma/web/web_finger.ex | 98 ++++++++++++++++++++++++------------------- lib/pleroma/web/xml.ex | 2 +- 2 files changed, 56 insertions(+), 44 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index 15002b29f..21b10e654 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -94,52 +94,56 @@ defmodule Pleroma.Web.WebFinger do |> XmlBuilder.to_doc() end - defp webfinger_from_xml(doc) do - subject = XML.string_from_xpath("//Subject", doc) - - subscribe_address = - ~s{//Link[@rel="http://ostatus.org/schema/1.0/subscribe"]/@template} - |> XML.string_from_xpath(doc) - - ap_id = - ~s{//Link[@rel="self" and @type="application/activity+json"]/@href} - |> XML.string_from_xpath(doc) - - data = %{ - "subject" => subject, - "subscribe_address" => subscribe_address, - "ap_id" => ap_id - } + defp webfinger_from_xml(body) do + with {:ok, doc} <- XML.parse_document(body) do + subject = XML.string_from_xpath("//Subject", doc) + + subscribe_address = + ~s{//Link[@rel="http://ostatus.org/schema/1.0/subscribe"]/@template} + |> XML.string_from_xpath(doc) + + ap_id = + ~s{//Link[@rel="self" and @type="application/activity+json"]/@href} + |> XML.string_from_xpath(doc) + + data = %{ + "subject" => subject, + "subscribe_address" => subscribe_address, + "ap_id" => ap_id + } - {:ok, data} + {:ok, data} + end end - defp webfinger_from_json(doc) do - data = - Enum.reduce(doc["links"], %{"subject" => doc["subject"]}, fn link, data -> - case {link["type"], link["rel"]} do - {"application/activity+json", "self"} -> - Map.put(data, "ap_id", link["href"]) + defp webfinger_from_json(body) do + with {:ok, doc} <- Jason.decode(body) do + data = + Enum.reduce(doc["links"], %{"subject" => doc["subject"]}, fn link, data -> + case {link["type"], link["rel"]} do + {"application/activity+json", "self"} -> + Map.put(data, "ap_id", link["href"]) - {"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", "self"} -> - Map.put(data, "ap_id", link["href"]) + {"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", "self"} -> + Map.put(data, "ap_id", link["href"]) - {nil, "http://ostatus.org/schema/1.0/subscribe"} -> - Map.put(data, "subscribe_address", link["template"]) + {nil, "http://ostatus.org/schema/1.0/subscribe"} -> + Map.put(data, "subscribe_address", link["template"]) - _ -> - Logger.debug("Unhandled type: #{inspect(link["type"])}") - data - end - end) + _ -> + Logger.debug("Unhandled type: #{inspect(link["type"])}") + data + end + end) - {:ok, data} + {:ok, data} + end end def get_template_from_xml(body) do xpath = "//Link[@rel='lrdd']/@template" - with doc when doc != :error <- XML.parse_document(body), + with {:ok, doc} <- XML.parse_document(body), template when template != nil <- XML.string_from_xpath(xpath, doc) do {:ok, template} end @@ -192,15 +196,23 @@ defmodule Pleroma.Web.WebFinger do address, [{"accept", "application/xrd+xml,application/jrd+json"}] ), - {:ok, %{status: status, body: body}} when status in 200..299 <- response do - doc = XML.parse_document(body) - - if doc != :error do - webfinger_from_xml(doc) - else - with {:ok, doc} <- Jason.decode(body) do - webfinger_from_json(doc) - end + {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <- + response do + case List.keyfind(headers, "content-type", 0) do + {_, content_type} -> + case Plug.Conn.Utils.media_type(content_type) do + {:ok, "application", subtype, _} when subtype in ~w(xrd+xml xml) -> + webfinger_from_xml(body) + + {:ok, "application", subtype, _} when subtype in ~w(jrd+json json) -> + webfinger_from_json(body) + + _ -> + {:error, {:content_type, content_type}} + end + + _ -> + {:error, {:content_type, nil}} end else e -> diff --git a/lib/pleroma/web/xml.ex b/lib/pleroma/web/xml.ex index 2b34611ac..0ab6e9d32 100644 --- a/lib/pleroma/web/xml.ex +++ b/lib/pleroma/web/xml.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Web.XML do |> :binary.bin_to_list() |> :xmerl_scan.string(quiet: true) - doc + {:ok, doc} rescue _e -> Logger.debug("Couldn't parse XML: #{inspect(text)}") -- cgit v1.2.3 From d3660b24d37862bb58cf309c582cfe7432fd7bb6 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 22 Mar 2021 20:07:07 +0300 Subject: Copy emoji in the subject from parent post Sometimes people put emoji in the subject, which results in the subject looking broken if someone replies to it from a server that does not have the said emoji under the same shortcode. This patch solves the problem by extending the emoji set available in the summary to that of the parent post. --- lib/pleroma/web/common_api/activity_draft.ex | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 8668b600e..80a9fa7bb 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do alias Pleroma.Activity alias Pleroma.Conversation.Participation + alias Pleroma.Object alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils @@ -186,6 +187,32 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp object(draft) do emoji = Map.merge(Pleroma.Emoji.Formatter.get_emoji_map(draft.full_payload), draft.emoji) + # Sometimes people create posts with subject containing emoji, + # since subjects are usually copied this will result in a broken + # subject when someone replies from an instance that does not have + # the emoji or has it under different shortcode. This is an attempt + # to mitigate this by copying emoji from inReplyTo if they are present + # in the subject. + summary_emoji = + with %Activity{} <- draft.in_reply_to, + %Object{data: %{"tag" => [_ | _] = tag}} <- Object.normalize(draft.in_reply_to) do + Enum.reduce(tag, %{}, fn + %{"type" => "Emoji", "name" => name, "icon" => %{"url" => url}}, acc -> + if String.contains?(draft.summary, name) do + Map.put(acc, name, url) + else + acc + end + + _, acc -> + acc + end) + else + _ -> %{} + end + + emoji = Map.merge(emoji, summary_emoji) + object = Utils.make_note_data(draft) |> Map.put("emoji", emoji) -- cgit v1.2.3 From 03843a53868860c0b6b2bebcf262bde746482f7e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 23 Mar 2021 14:23:37 +0300 Subject: migrating config to tmp folder --- lib/mix/tasks/pleroma/config.ex | 49 +++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index 1962154b9..ac89702ae 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -27,7 +27,7 @@ defmodule Mix.Tasks.Pleroma.Config do {opts, _} = OptionParser.parse!(options, - strict: [env: :string, delete: :boolean], + strict: [env: :string, delete: :boolean, path: :string], aliases: [d: :delete] ) @@ -259,18 +259,43 @@ defmodule Mix.Tasks.Pleroma.Config do defp migrate_from_db(opts) do env = opts[:env] || Pleroma.Config.get(:env) + filename = "#{env}.exported_from_db.secret.exs" + config_path = - if Pleroma.Config.get(:release) do - :config_path - |> Pleroma.Config.get() - |> Path.dirname() - else - "config" + cond do + opts[:path] -> + opts[:path] + + Pleroma.Config.get(:release) -> + :config_path + |> Pleroma.Config.get() + |> Path.dirname() + + true -> + "config" end - |> Path.join("#{env}.exported_from_db.secret.exs") + |> Path.join(filename) - file = File.open!(config_path, [:write, :utf8]) + with {:ok, file} <- File.open(config_path, [:write, :utf8]) do + write_config(file, config_path, opts) + shell_info("Database configuration settings have been exported to #{config_path}") + else + _ -> + shell_error("Impossible to save settings to this directory #{Path.dirname(config_path)}") + tmp_config_path = Path.join("/tmp", filename) + file = File.open!(tmp_config_path) + + shell_info( + "Saving database configuration settings to #{tmp_config_path}. Copy it to the #{ + Path.dirname(config_path) + } manually." + ) + write_config(file, tmp_config_path, opts) + end + end + + defp write_config(file, path, opts) do IO.write(file, config_header()) ConfigDB @@ -278,11 +303,7 @@ defmodule Mix.Tasks.Pleroma.Config do |> Enum.each(&write_and_delete(&1, file, opts[:delete])) :ok = File.close(file) - System.cmd("mix", ["format", config_path]) - - shell_info( - "Database configuration settings have been exported to config/#{env}.exported_from_db.secret.exs" - ) + System.cmd("mix", ["format", path]) end if Code.ensure_loaded?(Config.Reader) do -- cgit v1.2.3 From 4cd34d019764fdd68829ebd4282118abc4534133 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 23 Mar 2021 17:27:02 +0300 Subject: suggestion --- lib/mix/tasks/pleroma/config.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index ac89702ae..22502a522 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -282,7 +282,7 @@ defmodule Mix.Tasks.Pleroma.Config do else _ -> shell_error("Impossible to save settings to this directory #{Path.dirname(config_path)}") - tmp_config_path = Path.join("/tmp", filename) + tmp_config_path = Path.join(System.tmp_dir!(), filename) file = File.open!(tmp_config_path) shell_info( -- cgit v1.2.3 From b6a69b5efda5f75ad716252c69ae658a4e885b0a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 24 Mar 2021 12:50:05 -0500 Subject: Return token's primary key with POST /oauth/token --- lib/pleroma/web/o_auth/o_auth_view.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/o_auth/o_auth_view.ex b/lib/pleroma/web/o_auth/o_auth_view.ex index 281bbcc3c..1419c96a2 100644 --- a/lib/pleroma/web/o_auth/o_auth_view.ex +++ b/lib/pleroma/web/o_auth/o_auth_view.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.OAuth.OAuthView do def render("token.json", %{token: token} = opts) do response = %{ + id: token.id, token_type: "Bearer", access_token: token.token, refresh_token: token.refresh_token, -- cgit v1.2.3 From 3ec1dbd9223aa44205e90967175f07cc532501ab Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 3 Feb 2021 16:09:28 +0300 Subject: Let pins federate - save object ids on pin, instead of activity ids - pins federation - removed pinned_activities field from the users table - activityPub endpoint for user pins - pulling remote users pins --- lib/pleroma/activity.ex | 81 ++++++++++++---------- lib/pleroma/activity/queries.ex | 5 ++ lib/pleroma/user.ex | 77 ++++++++++---------- lib/pleroma/web/activity_pub/activity_pub.ex | 60 ++++++++++++++-- .../web/activity_pub/activity_pub_controller.ex | 8 +++ lib/pleroma/web/activity_pub/builder.ex | 32 +++++++++ lib/pleroma/web/activity_pub/object_validator.ex | 11 +++ .../object_validators/pin_validator.ex | 42 +++++++++++ lib/pleroma/web/activity_pub/side_effects.ex | 56 ++++++++++++++- lib/pleroma/web/activity_pub/transmogrifier.ex | 9 +++ lib/pleroma/web/activity_pub/views/user_view.ex | 21 ++++++ .../web/api_spec/operations/status_operation.ex | 46 +++++++++++- lib/pleroma/web/api_spec/schemas/status.ex | 7 ++ lib/pleroma/web/common_api.ex | 57 ++++++++++----- .../controllers/fallback_controller.ex | 6 ++ .../mastodon_api/controllers/status_controller.ex | 12 ++++ lib/pleroma/web/mastodon_api/views/status_view.ex | 23 ++++-- lib/pleroma/web/router.ex | 1 + 18 files changed, 451 insertions(+), 103 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/object_validators/pin_validator.ex (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index d59403884..a4cfca4c5 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -184,40 +184,48 @@ defmodule Pleroma.Activity do |> Repo.one() end - @spec get_by_id(String.t()) :: Activity.t() | nil - def get_by_id(id) do - case FlakeId.flake_id?(id) do - true -> - Activity - |> where([a], a.id == ^id) - |> restrict_deactivated_users() - |> Repo.one() - - _ -> - nil - end - end - - def get_by_id_with_user_actor(id) do - case FlakeId.flake_id?(id) do - true -> - Activity - |> where([a], a.id == ^id) - |> with_preloaded_user_actor() - |> Repo.one() - - _ -> - nil + @doc """ + Gets activity by ID, doesn't load activities from deactivated actors by default. + """ + @spec get_by_id(String.t(), keyword()) :: t() | nil + def get_by_id(id, opts \\ [filter: [:restrict_deactivated]]), do: get_by_id_with_opts(id, opts) + + @spec get_by_id_with_user_actor(String.t()) :: t() | nil + def get_by_id_with_user_actor(id), do: get_by_id_with_opts(id, preload: [:user_actor]) + + @spec get_by_id_with_object(String.t()) :: t() | nil + def get_by_id_with_object(id), do: get_by_id_with_opts(id, preload: [:object]) + + defp get_by_id_with_opts(id, opts) do + if FlakeId.flake_id?(id) do + query = Queries.by_id(id) + + with_filters_query = + if is_list(opts[:filter]) do + Enum.reduce(opts[:filter], query, fn + {:type, type}, acc -> Queries.by_type(acc, type) + :restrict_deactivated, acc -> restrict_deactivated_users(acc) + _, acc -> acc + end) + else + query + end + + with_preloads_query = + if is_list(opts[:preload]) do + Enum.reduce(opts[:preload], with_filters_query, fn + :user_actor, acc -> with_preloaded_user_actor(acc) + :object, acc -> with_preloaded_object(acc) + _, acc -> acc + end) + else + with_filters_query + end + + Repo.one(with_preloads_query) end end - def get_by_id_with_object(id) do - Activity - |> where(id: ^id) - |> with_preloaded_object() - |> Repo.one() - end - def all_by_ids_with_object(ids) do Activity |> where([a], a.id in ^ids) @@ -269,6 +277,11 @@ defmodule Pleroma.Activity do def get_create_by_object_ap_id_with_object(_), do: nil + @spec create_by_id_with_object(String.t()) :: t() | nil + def create_by_id_with_object(id) do + get_by_id_with_opts(id, preload: [:object], filter: [type: "Create"]) + end + defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do get_create_by_object_ap_id_with_object(ap_id) end @@ -368,12 +381,6 @@ defmodule Pleroma.Activity do end end - @spec pinned_by_actor?(Activity.t()) :: boolean() - def pinned_by_actor?(%Activity{} = activity) do - actor = user_actor(activity) - activity.id in actor.pinned_activities - end - @spec get_by_object_ap_id_with_object(String.t()) :: t() | nil def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do ap_id diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index a6b02a889..4632651b0 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -14,6 +14,11 @@ defmodule Pleroma.Activity.Queries do alias Pleroma.Activity alias Pleroma.User + @spec by_id(query(), String.t()) :: query() + def by_id(query \\ Activity, id) do + from(a in query, where: a.id == ^id) + end + @spec by_ap_id(query, String.t()) :: query def by_ap_id(query \\ Activity, ap_id) do from( diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c1aa0f716..b78777141 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -99,6 +99,7 @@ defmodule Pleroma.User do field(:local, :boolean, default: true) field(:follower_address, :string) field(:following_address, :string) + field(:featured_address, :string) field(:search_rank, :float, virtual: true) field(:search_type, :integer, virtual: true) field(:tags, {:array, :string}, default: []) @@ -130,7 +131,6 @@ defmodule Pleroma.User do field(:hide_followers, :boolean, default: false) field(:hide_follows, :boolean, default: false) field(:hide_favorites, :boolean, default: true) - field(:pinned_activities, {:array, :string}, default: []) field(:email_notifications, :map, default: %{"digest" => false}) field(:mascot, :map, default: nil) field(:emoji, :map, default: %{}) @@ -148,6 +148,7 @@ defmodule Pleroma.User do field(:accepts_chat_messages, :boolean, default: nil) field(:last_active_at, :naive_datetime) field(:disclose_client, :boolean, default: true) + field(:pinned_objects, :map, default: %{}) embeds_one( :notification_settings, @@ -372,8 +373,10 @@ defmodule Pleroma.User do end # Should probably be renamed or removed + @spec ap_id(User.t()) :: String.t() def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}" + @spec ap_followers(User.t()) :: String.t() def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers" @@ -381,6 +384,11 @@ defmodule Pleroma.User do def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa def ap_following(%User{} = user), do: "#{ap_id(user)}/following" + @spec ap_featured_collection(User.t()) :: String.t() + def ap_featured_collection(%User{featured_address: fa}) when is_binary(fa), do: fa + + def ap_featured_collection(%User{} = user), do: "#{ap_id(user)}/collections/featured" + defp truncate_fields_param(params) do if Map.has_key?(params, :fields) do Map.put(params, :fields, Enum.map(params[:fields], &truncate_field/1)) @@ -443,6 +451,7 @@ defmodule Pleroma.User do :uri, :follower_address, :following_address, + :featured_address, :hide_followers, :hide_follows, :hide_followers_count, @@ -454,7 +463,8 @@ defmodule Pleroma.User do :invisible, :actor_type, :also_known_as, - :accepts_chat_messages + :accepts_chat_messages, + :pinned_objects ] ) |> cast(params, [:name], empty_values: []) @@ -686,7 +696,7 @@ defmodule Pleroma.User do |> validate_format(:nickname, local_nickname_regex()) |> put_ap_id() |> unique_constraint(:ap_id) - |> put_following_and_follower_address() + |> put_following_and_follower_and_featured_address() end def register_changeset(struct, params \\ %{}, opts \\ []) do @@ -747,7 +757,7 @@ defmodule Pleroma.User do |> put_password_hash |> put_ap_id() |> unique_constraint(:ap_id) - |> put_following_and_follower_address() + |> put_following_and_follower_and_featured_address() end def maybe_validate_required_email(changeset, true), do: changeset @@ -765,11 +775,16 @@ defmodule Pleroma.User do put_change(changeset, :ap_id, ap_id) end - defp put_following_and_follower_address(changeset) do - followers = ap_followers(%User{nickname: get_field(changeset, :nickname)}) + defp put_following_and_follower_and_featured_address(changeset) do + user = %User{nickname: get_field(changeset, :nickname)} + followers = ap_followers(user) + following = ap_following(user) + featured = ap_featured_collection(user) changeset |> put_change(:follower_address, followers) + |> put_change(:following_address, following) + |> put_change(:featured_address, featured) end defp autofollow_users(user) do @@ -2343,45 +2358,35 @@ defmodule Pleroma.User do cast(user, %{is_approved: approved?}, [:is_approved]) end - def add_pinnned_activity(user, %Pleroma.Activity{id: id}) do - if id not in user.pinned_activities do - max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0) - params = %{pinned_activities: user.pinned_activities ++ [id]} - - # if pinned activity was scheduled for deletion, we remove job - if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(id) do - Oban.cancel_job(expiration.id) - end + @spec add_pinned_object_id(User.t(), String.t()) :: {:ok, User.t()} | {:error, term()} + def add_pinned_object_id(%User{} = user, object_id) do + if !user.pinned_objects[object_id] do + params = %{pinned_objects: Map.put(user.pinned_objects, object_id, NaiveDateTime.utc_now())} user - |> cast(params, [:pinned_activities]) - |> validate_length(:pinned_activities, - max: max_pinned_statuses, - message: "You have already pinned the maximum number of statuses" - ) + |> cast(params, [:pinned_objects]) + |> validate_change(:pinned_objects, fn :pinned_objects, pinned_objects -> + max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0) + + if Enum.count(pinned_objects) <= max_pinned_statuses do + [] + else + [pinned_objects: "You have already pinned the maximum number of statuses"] + end + end) else change(user) end |> update_and_set_cache() end - def remove_pinnned_activity(user, %Pleroma.Activity{id: id, data: data}) do - params = %{pinned_activities: List.delete(user.pinned_activities, id)} - - # if pinned activity was scheduled for deletion, we reschedule it for deletion - if data["expires_at"] do - # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation - {:ok, expires_at} = - data["expires_at"] |> Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast() - - Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ - activity_id: id, - expires_at: expires_at - }) - end - + @spec remove_pinned_object_id(User.t(), String.t()) :: {:ok, t()} | {:error, term()} + def remove_pinned_object_id(%User{} = user, object_id) do user - |> cast(params, [:pinned_activities]) + |> cast( + %{pinned_objects: Map.delete(user.pinned_objects, object_id)}, + [:pinned_objects] + ) |> update_and_set_cache() end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index efbf92c70..d0051d1cb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -630,7 +630,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Map.put(:type, ["Create", "Announce"]) |> Map.put(:user, reading_user) |> Map.put(:actor_id, user.ap_id) - |> Map.put(:pinned_activity_ids, user.pinned_activities) + |> Map.put(:pinned_object_ids, Map.keys(user.pinned_objects)) params = if User.blocks?(reading_user, user) do @@ -1075,8 +1075,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_unlisted(query, _), do: query - defp restrict_pinned(query, %{pinned: true, pinned_activity_ids: ids}) do - from(activity in query, where: activity.id in ^ids) + defp restrict_pinned(query, %{pinned: true, pinned_object_ids: ids}) do + from( + [activity, object: o] in query, + where: + fragment( + "(?)->>'type' = 'Create' and coalesce((?)->'object'->>'id', (?)->>'object') = any (?)", + activity.data, + activity.data, + activity.data, + ^ids + ) + ) end defp restrict_pinned(query, _), do: query @@ -1419,6 +1429,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do invisible = data["invisible"] || false actor_type = data["type"] || "Person" + featured_address = data["featured"] + {:ok, pinned_objects} = fetch_and_prepare_featured_from_ap_id(featured_address) + public_key = if is_map(data["publicKey"]) && is_binary(data["publicKey"]["publicKeyPem"]) do data["publicKey"]["publicKeyPem"] @@ -1447,13 +1460,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do name: data["name"], follower_address: data["followers"], following_address: data["following"], + featured_address: featured_address, bio: data["summary"] || "", actor_type: actor_type, also_known_as: Map.get(data, "alsoKnownAs", []), public_key: public_key, inbox: data["inbox"], shared_inbox: shared_inbox, - accepts_chat_messages: accepts_chat_messages + accepts_chat_messages: accepts_chat_messages, + pinned_objects: pinned_objects } # nickname can be nil because of virtual actors @@ -1591,6 +1606,41 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + def pin_data_from_featured_collection(%{ + "type" => type, + "orderedItems" => objects + }) + when type in ["OrderedCollection", "Collection"] do + Map.new(objects, fn %{"id" => object_ap_id} -> {object_ap_id, NaiveDateTime.utc_now()} end) + end + + def fetch_and_prepare_featured_from_ap_id(nil) do + {:ok, %{}} + end + + def fetch_and_prepare_featured_from_ap_id(ap_id) do + with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do + {:ok, pin_data_from_featured_collection(data)} + else + e -> + Logger.error("Could not decode featured collection at fetch #{ap_id}, #{inspect(e)}") + {:ok, %{}} + end + end + + def pinned_fetch_task(nil), do: nil + + def pinned_fetch_task(%{pinned_objects: pins}) do + if Enum.all?(pins, fn {ap_id, _} -> + Object.get_cached_by_ap_id(ap_id) || + match?({:ok, _object}, Fetcher.fetch_object_from_id(ap_id)) + end) do + :ok + else + :error + end + end + def make_user_from_ap_id(ap_id) do user = User.get_cached_by_ap_id(ap_id) @@ -1598,6 +1648,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Transmogrifier.upgrade_user_from_ap_id(ap_id) else with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do + {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end) + if user do user |> User.remote_user_changeset(data) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 9d3dcc7f9..5aa3b281a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -543,4 +543,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> json(object.data) end end + + def pinned(conn, %{"nickname" => nickname}) do + with %User{} = user <- User.get_cached_by_nickname(nickname) do + conn + |> put_resp_header("content-type", "application/activity+json") + |> json(UserView.render("featured.json", %{user: user})) + end + end end diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index f56bfc600..91a45836f 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -273,4 +273,36 @@ defmodule Pleroma.Web.ActivityPub.Builder do "context" => object.data["context"] }, []} end + + @spec pin(User.t(), Object.t()) :: {:ok, map(), keyword()} + def pin(%User{} = user, object) do + {:ok, + %{ + "id" => Utils.generate_activity_id(), + "target" => pinned_url(user.nickname), + "object" => object.data["id"], + "actor" => user.ap_id, + "type" => "Add", + "to" => [Pleroma.Constants.as_public()], + "cc" => [user.follower_address] + }, []} + end + + @spec unpin(User.t(), Object.t()) :: {:ok, map, keyword()} + def unpin(%User{} = user, object) do + {:ok, + %{ + "id" => Utils.generate_activity_id(), + "target" => pinned_url(user.nickname), + "object" => object.data["id"], + "actor" => user.ap_id, + "type" => "Remove", + "to" => [Pleroma.Constants.as_public()], + "cc" => [user.follower_address] + }, []} + end + + defp pinned_url(nickname) when is_binary(nickname) do + Pleroma.Web.Router.Helpers.activity_pub_url(Pleroma.Web.Endpoint, :pinned, nickname) + end end diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 297c19cc0..11432ef38 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -30,6 +30,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.EventValidator alias Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.PinValidator alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator alias Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator @@ -234,6 +235,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end end + def validate(%{"type" => type} = object, meta) when type in ~w(Add Remove) do + with {:ok, object} <- + object + |> PinValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + def cast_and_apply(%{"type" => "ChatMessage"} = object) do ChatMessageValidator.cast_and_apply(object) end diff --git a/lib/pleroma/web/activity_pub/object_validators/pin_validator.ex b/lib/pleroma/web/activity_pub/object_validators/pin_validator.ex new file mode 100644 index 000000000..dca8cba6f --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/pin_validator.ex @@ -0,0 +1,42 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.PinValidator do + use Ecto.Schema + + import Ecto.Changeset + import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + + @primary_key false + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:target) + field(:object, ObjectValidators.ObjectID) + field(:actor, ObjectValidators.ObjectID) + field(:type) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + defp cast_data(data) do + cast(%__MODULE__{}, data, __schema__(:fields)) + end + + defp validate_data(changeset) do + changeset + |> validate_required([:id, :target, :object, :actor, :type, :to, :cc]) + |> validate_inclusion(:type, ~w(Add Remove)) + |> validate_actor_presence() + |> validate_object_presence() + end +end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 0b9a9f0c5..9d22f9d3c 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -276,10 +276,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do result = case deleted_object do %Object{} -> - with {:ok, deleted_object, activity} <- Object.delete(deleted_object), + with {:ok, deleted_object, _activity} <- Object.delete(deleted_object), {_, actor} when is_binary(actor) <- {:actor, deleted_object.data["actor"]}, %User{} = user <- User.get_cached_by_ap_id(actor) do - User.remove_pinnned_activity(user, activity) + User.remove_pinned_object_id(user, deleted_object.data["id"]) {:ok, user} = ActivityPub.decrease_note_count_if_public(user, deleted_object) @@ -312,6 +312,58 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end end + # Tasks this handles: + # - adds pin to user + # - removes expiration job for pinned activity, if was set for expiration + @impl true + def handle(%{data: %{"type" => "Add"} = data} = object, meta) do + with %User{} = user <- User.get_cached_by_ap_id(data["actor"]), + {:ok, _user} <- User.add_pinned_object_id(user, data["object"]) do + # if pinned activity was scheduled for deletion, we remove job + if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(meta[:activity_id]) do + Oban.cancel_job(expiration.id) + end + + {:ok, object, meta} + else + nil -> + {:error, :user_not_found} + + {:error, changeset} -> + if changeset.errors[:pinned_objects] do + {:error, :pinned_statuses_limit_reached} + else + changeset.errors + end + end + end + + # Tasks this handles: + # - removes pin from user + # - if activity had expiration, recreates activity expiration job + @impl true + def handle(%{data: %{"type" => "Remove"} = data} = object, meta) do + with %User{} = user <- User.get_cached_by_ap_id(data["actor"]), + {:ok, _user} <- User.remove_pinned_object_id(user, data["object"]) do + # if pinned activity was scheduled for deletion, we reschedule it for deletion + if meta[:expires_at] do + # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation + {:ok, expires_at} = + Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast(meta[:expires_at]) + + Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ + activity_id: meta[:activity_id], + expires_at: expires_at + }) + end + + {:ok, object, meta} + else + nil -> {:error, :user_not_found} + error -> error + end + end + # Nothing to do @impl true def handle(object, meta) do diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 8c7d6a747..270cea6dc 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -556,6 +556,14 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end + def handle_incoming(%{"type" => type} = data, _options) when type in ~w(Add Remove) do + with :ok <- ObjectValidator.fetch_actor_and_object(data), + %Object{} <- Object.normalize(data["object"], fetch: true), + {:ok, activity, _meta} <- Pipeline.common_pipeline(data, local: false) do + {:ok, activity} + end + end + def handle_incoming( %{"type" => "Delete"} = data, _options @@ -1000,6 +1008,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id), {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id), {:ok, user} <- update_user(user, data) do + {:ok, _pid} = Task.start(fn -> ActivityPub.pinned_fetch_task(user) end) TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id}) {:ok, user} else diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 8adc9878a..462f3b4a7 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -6,8 +6,10 @@ defmodule Pleroma.Web.ActivityPub.UserView do use Pleroma.Web, :view alias Pleroma.Keys + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Endpoint @@ -97,6 +99,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "followers" => "#{user.ap_id}/followers", "inbox" => "#{user.ap_id}/inbox", "outbox" => "#{user.ap_id}/outbox", + "featured" => "#{user.ap_id}/collections/featured", "preferredUsername" => user.nickname, "name" => user.name, "summary" => user.bio, @@ -245,6 +248,24 @@ defmodule Pleroma.Web.ActivityPub.UserView do |> Map.merge(pagination) end + def render("featured.json", %{ + user: %{featured_address: featured_address, pinned_objects: pinned_objects} + }) do + objects = + pinned_objects + |> Enum.sort_by(fn {_, pinned_at} -> pinned_at end, &>=/2) + |> Enum.map(fn {id, _} -> + ObjectView.render("object.json", %{object: Object.get_cached_by_ap_id(id)}) + end) + + %{ + "id" => featured_address, + "type" => "OrderedCollection", + "orderedItems" => objects + } + |> Map.merge(Utils.make_json_ld_header()) + end + defp maybe_put_total_items(map, false, _total), do: map defp maybe_put_total_items(map, true, total) do diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 4bdb8e281..802fbef3e 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -182,7 +182,34 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do parameters: [id_param()], responses: %{ 200 => status_response(), - 400 => Operation.response("Error", "application/json", ApiError) + 400 => + Operation.response("Bad Request", "application/json", %Schema{ + allOf: [ApiError], + title: "Unprocessable Entity", + example: %{ + "error" => "You have already pinned the maximum number of statuses" + } + }), + 404 => + Operation.response("Not found", "application/json", %Schema{ + allOf: [ApiError], + title: "Unprocessable Entity", + example: %{ + "error" => "Record not found" + } + }), + 422 => + Operation.response( + "Unprocessable Entity", + "application/json", + %Schema{ + allOf: [ApiError], + title: "Unprocessable Entity", + example: %{ + "error" => "Someone else's status cannot be pinned" + } + } + ) } } end @@ -197,7 +224,22 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do parameters: [id_param()], responses: %{ 200 => status_response(), - 400 => Operation.response("Error", "application/json", ApiError) + 400 => + Operation.response("Bad Request", "application/json", %Schema{ + allOf: [ApiError], + title: "Unprocessable Entity", + example: %{ + "error" => "You have already pinned the maximum number of statuses" + } + }), + 404 => + Operation.response("Not found", "application/json", %Schema{ + allOf: [ApiError], + title: "Unprocessable Entity", + example: %{ + "error" => "Record not found" + } + }) } } end diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 42fa98718..3d042dc19 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -194,6 +194,13 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do parent_visible: %Schema{ type: :boolean, description: "`true` if the parent post is visible to the user" + }, + pinned_at: %Schema{ + type: :string, + format: "date-time", + nullable: true, + description: + "A datetime (ISO 8601) that states when the post was pinned or `null` if the post is not pinned" } } }, diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index b003e30c7..d35a0f219 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -411,29 +411,54 @@ defmodule Pleroma.Web.CommonAPI do end end - def pin(id, %{ap_id: user_ap_id} = user) do - with %Activity{ - actor: ^user_ap_id, - data: %{"type" => "Create"}, - object: %Object{data: %{"type" => object_type}} - } = activity <- Activity.get_by_id_with_object(id), - true <- object_type in ["Note", "Article", "Question"], - true <- Visibility.is_public?(activity), - {:ok, _user} <- User.add_pinnned_activity(user, activity) do + @spec pin(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, term()} + def pin(id, %User{ap_id: actor} = user) do + with %Activity{} = activity <- create_activity_by_id(id), + true <- activity_belongs_to_actor(activity, actor), + true <- object_type_is_allowed_for_pin(activity.object), + true <- activity_is_public(activity), + {:ok, pin_data, _} <- Builder.pin(user, activity.object), + {:ok, _pin, _} <- + Pipeline.common_pipeline(pin_data, local: true, activity_id: id) do {:ok, activity} else - {:error, %{errors: [pinned_activities: {err, _}]}} -> {:error, err} - _ -> {:error, dgettext("errors", "Could not pin")} + {:error, {:execute_side_effects, error}} -> error + error -> error end end + defp create_activity_by_id(id) do + with nil <- Activity.create_by_id_with_object(id) do + {:error, :not_found} + end + end + + defp activity_belongs_to_actor(%{actor: actor}, actor), do: true + defp activity_belongs_to_actor(_, _), do: {:error, :ownership_error} + + defp object_type_is_allowed_for_pin(%{data: %{"type" => type}}) do + with false <- type in ["Note", "Article", "Question"] do + {:error, :not_allowed} + end + end + + defp activity_is_public(activity) do + with false <- Visibility.is_public?(activity) do + {:error, :visibility_error} + end + end + + @spec unpin(String.t(), User.t()) :: {:ok, User.t()} | {:error, term()} def unpin(id, user) do - with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id), - {:ok, _user} <- User.remove_pinnned_activity(user, activity) do + with %Activity{} = activity <- create_activity_by_id(id), + {:ok, unpin_data, _} <- Builder.unpin(user, activity.object), + {:ok, _unpin, _} <- + Pipeline.common_pipeline(unpin_data, + local: true, + activity_id: activity.id, + expires_at: activity.data["expires_at"] + ) do {:ok, activity} - else - {:error, %{errors: [pinned_activities: {err, _}]}} -> {:error, err} - _ -> {:error, dgettext("errors", "Could not unpin")} end end diff --git a/lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex b/lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex index d25f84837..84621500e 100644 --- a/lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex @@ -30,6 +30,12 @@ defmodule Pleroma.Web.MastodonAPI.FallbackController do |> json(%{error: error_message}) end + def call(conn, {:error, status, message}) do + conn + |> put_status(status) + |> json(%{error: message}) + end + def call(conn, _) do conn |> put_status(:internal_server_error) diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index b051fca74..724dc5c5d 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -260,6 +260,18 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do def pin(%{assigns: %{user: user}} = conn, %{id: ap_id_or_id}) do with {:ok, activity} <- CommonAPI.pin(ap_id_or_id, user) do try_render(conn, "show.json", activity: activity, for: user, as: :activity) + else + {:error, :pinned_statuses_limit_reached} -> + {:error, "You have already pinned the maximum number of statuses"} + + {:error, :ownership_error} -> + {:error, :unprocessable_entity, "Someone else's status cannot be pinned"} + + {:error, :visibility_error} -> + {:error, :unprocessable_entity, "Non-public status cannot be pinned"} + + error -> + error end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 3753588f2..d0247fa4a 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -152,6 +152,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Enum.filter(& &1) |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end) + {pinned?, pinned_at} = pin_data(activity_object, user) + %{ id: to_string(activity.id), uri: object.data["id"], @@ -173,7 +175,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do favourited: present?(favorited), bookmarked: present?(bookmarked), muted: false, - pinned: pinned?(activity, user), + pinned: pinned?, sensitive: false, spoiler_text: "", visibility: get_visibility(activity), @@ -184,7 +186,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do language: nil, emojis: [], pleroma: %{ - local: activity.local + local: activity.local, + pinned_at: pinned_at } } end @@ -316,6 +319,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do fn for_user, user -> User.mutes?(for_user, user) end ) + {pinned?, pinned_at} = pin_data(object, user) + %{ id: to_string(activity.id), uri: object.data["id"], @@ -339,7 +344,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do favourited: present?(favorited), bookmarked: present?(bookmarked), muted: muted, - pinned: pinned?(activity, user), + pinned: pinned?, sensitive: sensitive, spoiler_text: summary, visibility: get_visibility(object), @@ -360,7 +365,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do direct_conversation_id: direct_conversation_id, thread_muted: thread_muted?, emoji_reactions: emoji_reactions, - parent_visible: visible_for_user?(reply_to, opts[:for]) + parent_visible: visible_for_user?(reply_to, opts[:for]), + pinned_at: pinned_at } } end @@ -529,8 +535,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do defp present?(false), do: false defp present?(_), do: true - defp pinned?(%Activity{id: id}, %User{pinned_activities: pinned_activities}), - do: id in pinned_activities + defp pin_data(%Object{data: %{"id" => object_id}}, %User{pinned_objects: pinned_objects}) do + if pinned_at = pinned_objects[object_id] do + {true, Utils.to_masto_date(pinned_at)} + else + {false, nil} + end + end defp build_emoji_map(emoji, users, current_user) do %{ diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index de0bd27d7..ccf2ef796 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -704,6 +704,7 @@ defmodule Pleroma.Web.Router do # The following two are S2S as well, see `ActivityPub.fetch_follow_information_for_user/1`: get("/users/:nickname/followers", ActivityPubController, :followers) get("/users/:nickname/following", ActivityPubController, :following) + get("/users/:nickname/collections/featured", ActivityPubController, :pinned) end scope "/", Pleroma.Web.ActivityPub do -- cgit v1.2.3 From 17f28c0507e3c34ce75e63747eed9abb66713e6e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 25 Feb 2021 14:00:44 +0300 Subject: mastodon pins --- lib/pleroma/object/containment.ex | 8 ++++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 17 +++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index fb0398f92..040537acf 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -71,6 +71,14 @@ defmodule Pleroma.Object.Containment do compare_uris(id_uri, other_uri) end + # Mastodon pin activities don't have an id, so we check the object field, which will be pinned. + def contain_origin_from_id(id, %{"object" => object}) when is_binary(object) do + id_uri = URI.parse(id) + object_uri = URI.parse(object) + + compare_uris(id_uri, object_uri) + end + def contain_origin_from_id(_id, _data), do: :error def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}), diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 270cea6dc..b662f5379 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -557,10 +557,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming(%{"type" => type} = data, _options) when type in ~w(Add Remove) do - with :ok <- ObjectValidator.fetch_actor_and_object(data), - %Object{} <- Object.normalize(data["object"], fetch: true), - {:ok, activity, _meta} <- Pipeline.common_pipeline(data, local: false) do - {:ok, activity} + with {:ok, user} <- ObjectValidator.fetch_actor(data), + %Object{} <- Object.normalize(data["object"], fetch: true) do + # Mastodon sends pin/unpin objects without id, to, cc fields + data = + data + |> Map.put_new("id", Utils.generate_activity_id()) + |> Map.put_new("to", [Pleroma.Constants.as_public()]) + |> Map.put_new("cc", [user.follower_address]) + + case Pipeline.common_pipeline(data, local: false) do + {:ok, activity, _meta} -> {:ok, activity} + error -> error + end end end -- cgit v1.2.3 From ff612750b1bae5223bca76b34a39e7d2bd05770c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 2 Mar 2021 17:24:06 +0300 Subject: validator renaming & add validation for target --- lib/pleroma/web/activity_pub/object_validator.ex | 4 +- .../object_validators/add_remove_validator.ex | 53 ++++++++++++++++++++++ .../object_validators/common_validations.ex | 8 ++++ .../object_validators/pin_validator.ex | 42 ----------------- 4 files changed, 63 insertions(+), 44 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex delete mode 100644 lib/pleroma/web/activity_pub/object_validators/pin_validator.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 11432ef38..14c3e8531 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -17,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Object.Containment alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator @@ -30,7 +31,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.EventValidator alias Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.PinValidator alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator alias Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator @@ -238,7 +238,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def validate(%{"type" => type} = object, meta) when type in ~w(Add Remove) do with {:ok, object} <- object - |> PinValidator.cast_and_validate() + |> AddRemoveValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do object = stringify_keys(object) {:ok, object, meta} diff --git a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex new file mode 100644 index 000000000..73d1c03f0 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex @@ -0,0 +1,53 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do + use Ecto.Schema + + import Ecto.Changeset + import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + + @primary_key false + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:target) + field(:object, ObjectValidators.ObjectID) + field(:actor, ObjectValidators.ObjectID) + field(:type) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + defp cast_data(data) do + cast(%__MODULE__{}, data, __schema__(:fields)) + end + + defp validate_data(changeset) do + changeset + |> validate_required([:id, :target, :object, :actor, :type, :to, :cc]) + |> validate_inclusion(:type, ~w(Add Remove)) + |> validate_actor_presence() + |> validate_collection_belongs_to_actor() + |> validate_object_presence() + end + + defp validate_collection_belongs_to_actor(changeset) do + validate_change(changeset, :target, fn :target, target -> + if String.starts_with?(target, changeset.changes[:actor]) do + [] + else + [target: "collection doesn't belong to actor"] + end + end) + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index 093549a45..940430588 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do alias Pleroma.Object alias Pleroma.User + @spec validate_any_presence(Ecto.Changeset.t(), [atom()]) :: Ecto.Changeset.t() def validate_any_presence(cng, fields) do non_empty = fields @@ -29,6 +30,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do end end + @spec validate_actor_presence(Ecto.Changeset.t(), keyword()) :: Ecto.Changeset.t() def validate_actor_presence(cng, options \\ []) do field_name = Keyword.get(options, :field_name, :actor) @@ -47,6 +49,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do end) end + @spec validate_object_presence(Ecto.Changeset.t(), keyword()) :: Ecto.Changeset.t() def validate_object_presence(cng, options \\ []) do field_name = Keyword.get(options, :field_name, :object) allowed_types = Keyword.get(options, :allowed_types, false) @@ -68,6 +71,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do end) end + @spec validate_object_or_user_presence(Ecto.Changeset.t(), keyword()) :: Ecto.Changeset.t() def validate_object_or_user_presence(cng, options \\ []) do field_name = Keyword.get(options, :field_name, :object) options = Keyword.put(options, :field_name, field_name) @@ -83,6 +87,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do if actor_cng.valid?, do: actor_cng, else: object_cng end + @spec validate_host_match(Ecto.Changeset.t(), [atom()]) :: Ecto.Changeset.t() def validate_host_match(cng, fields \\ [:id, :actor]) do if same_domain?(cng, fields) do cng @@ -95,6 +100,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do end end + @spec validate_fields_match(Ecto.Changeset.t(), [atom()]) :: Ecto.Changeset.t() def validate_fields_match(cng, fields) do if map_unique?(cng, fields) do cng @@ -122,12 +128,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do end) end + @spec same_domain?(Ecto.Changeset.t(), [atom()]) :: boolean() def same_domain?(cng, fields \\ [:actor, :object]) do map_unique?(cng, fields, fn value -> URI.parse(value).host end) end # This figures out if a user is able to create, delete or modify something # based on the domain and superuser status + @spec validate_modification_rights(Ecto.Changeset.t()) :: Ecto.Changeset.t() def validate_modification_rights(cng) do actor = User.get_cached_by_ap_id(get_field(cng, :actor)) diff --git a/lib/pleroma/web/activity_pub/object_validators/pin_validator.ex b/lib/pleroma/web/activity_pub/object_validators/pin_validator.ex deleted file mode 100644 index dca8cba6f..000000000 --- a/lib/pleroma/web/activity_pub/object_validators/pin_validator.ex +++ /dev/null @@ -1,42 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ActivityPub.ObjectValidators.PinValidator do - use Ecto.Schema - - import Ecto.Changeset - import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations - - alias Pleroma.EctoType.ActivityPub.ObjectValidators - - @primary_key false - - embedded_schema do - field(:id, ObjectValidators.ObjectID, primary_key: true) - field(:target) - field(:object, ObjectValidators.ObjectID) - field(:actor, ObjectValidators.ObjectID) - field(:type) - field(:to, ObjectValidators.Recipients, default: []) - field(:cc, ObjectValidators.Recipients, default: []) - end - - def cast_and_validate(data) do - data - |> cast_data() - |> validate_data() - end - - defp cast_data(data) do - cast(%__MODULE__{}, data, __schema__(:fields)) - end - - defp validate_data(changeset) do - changeset - |> validate_required([:id, :target, :object, :actor, :type, :to, :cc]) - |> validate_inclusion(:type, ~w(Add Remove)) - |> validate_actor_presence() - |> validate_object_presence() - end -end -- cgit v1.2.3 From d1d2744ee3e6015064cf50ac5725bfe45b682466 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 3 Mar 2021 15:41:05 +0300 Subject: featured_address valition in AddRemoveValidator --- lib/pleroma/web/activity_pub/object_validator.ex | 2 +- .../activity_pub/object_validators/add_remove_validator.ex | 12 ++++++------ lib/pleroma/web/activity_pub/transmogrifier.ex | 7 +++++-- lib/pleroma/web/common_api.ex | 13 +++++++++---- 4 files changed, 21 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 14c3e8531..3ca9136aa 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -238,7 +238,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def validate(%{"type" => type} = object, meta) when type in ~w(Add Remove) do with {:ok, object} <- object - |> AddRemoveValidator.cast_and_validate() + |> AddRemoveValidator.cast_and_validate(meta) |> Ecto.Changeset.apply_action(:insert) do object = stringify_keys(object) {:ok, object, meta} diff --git a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex index 73d1c03f0..885282f32 100644 --- a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex @@ -22,28 +22,28 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do field(:cc, ObjectValidators.Recipients, default: []) end - def cast_and_validate(data) do + def cast_and_validate(data, meta) do data |> cast_data() - |> validate_data() + |> validate_data(meta) end defp cast_data(data) do cast(%__MODULE__{}, data, __schema__(:fields)) end - defp validate_data(changeset) do + defp validate_data(changeset, meta) do changeset |> validate_required([:id, :target, :object, :actor, :type, :to, :cc]) |> validate_inclusion(:type, ~w(Add Remove)) |> validate_actor_presence() - |> validate_collection_belongs_to_actor() + |> validate_collection_belongs_to_actor(meta) |> validate_object_presence() end - defp validate_collection_belongs_to_actor(changeset) do + defp validate_collection_belongs_to_actor(changeset, meta) do validate_change(changeset, :target, fn :target, target -> - if String.starts_with?(target, changeset.changes[:actor]) do + if target == meta[:featured_address] do [] else [target: "collection doesn't belong to actor"] diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b662f5379..fa62e0db2 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -557,7 +557,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming(%{"type" => type} = data, _options) when type in ~w(Add Remove) do - with {:ok, user} <- ObjectValidator.fetch_actor(data), + with {:ok, %User{} = user} <- ObjectValidator.fetch_actor(data), %Object{} <- Object.normalize(data["object"], fetch: true) do # Mastodon sends pin/unpin objects without id, to, cc fields data = @@ -566,7 +566,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put_new("to", [Pleroma.Constants.as_public()]) |> Map.put_new("cc", [user.follower_address]) - case Pipeline.common_pipeline(data, local: false) do + case Pipeline.common_pipeline(data, + local: false, + featured_address: user.featured_address + ) do {:ok, activity, _meta} -> {:ok, activity} error -> error end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index d35a0f219..175d690cc 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -412,14 +412,18 @@ defmodule Pleroma.Web.CommonAPI do end @spec pin(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, term()} - def pin(id, %User{ap_id: actor} = user) do + def pin(id, %User{} = user) do with %Activity{} = activity <- create_activity_by_id(id), - true <- activity_belongs_to_actor(activity, actor), + true <- activity_belongs_to_actor(activity, user.ap_id), true <- object_type_is_allowed_for_pin(activity.object), true <- activity_is_public(activity), {:ok, pin_data, _} <- Builder.pin(user, activity.object), {:ok, _pin, _} <- - Pipeline.common_pipeline(pin_data, local: true, activity_id: id) do + Pipeline.common_pipeline(pin_data, + local: true, + activity_id: id, + featured_address: user.featured_address + ) do {:ok, activity} else {:error, {:execute_side_effects, error}} -> error @@ -456,7 +460,8 @@ defmodule Pleroma.Web.CommonAPI do Pipeline.common_pipeline(unpin_data, local: true, activity_id: activity.id, - expires_at: activity.data["expires_at"] + expires_at: activity.data["expires_at"], + featured_address: user.featured_address ) do {:ok, activity} end -- cgit v1.2.3 From 3adb43cc20751540ea590645b31b985807684202 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 3 Mar 2021 18:04:06 +0300 Subject: refetch user on incoming add/remove activity if featured_address is nil --- lib/pleroma/web/activity_pub/transmogrifier.ex | 8 ++++++++ lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index fa62e0db2..c4b11a655 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -558,6 +558,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming(%{"type" => type} = data, _options) when type in ~w(Add Remove) do with {:ok, %User{} = user} <- ObjectValidator.fetch_actor(data), + # maybe locally user doesn't have featured_address + {:ok, user} <- maybe_refetch_user(user), %Object{} <- Object.normalize(data["object"], fetch: true) do # Mastodon sends pin/unpin objects without id, to, cc fields data = @@ -669,6 +671,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming(_, _), do: :error + defp maybe_refetch_user(%User{featured_address: address} = user) when is_binary(address) do + {:ok, user} + end + + defp maybe_refetch_user(%User{ap_id: ap_id}), do: upgrade_user_from_ap_id(ap_id) + @spec get_obj_helper(String.t(), Keyword.t()) :: {:ok, Object.t()} | nil def get_obj_helper(id, options \\ []) do options = Keyword.put(options, :fetch, true) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index d0247fa4a..814b3d142 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -152,7 +152,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Enum.filter(& &1) |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end) - {pinned?, pinned_at} = pin_data(activity_object, user) + {pinned?, pinned_at} = pin_data(object, user) %{ id: to_string(activity.id), -- cgit v1.2.3 From 8857242c952dcac0bc5363e1c80160efaf7a1638 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 9 Mar 2021 11:57:20 +0300 Subject: removeing corresponding add activity --- lib/pleroma/activity.ex | 9 +++++++++ lib/pleroma/web/activity_pub/side_effects.ex | 5 +++++ 2 files changed, 14 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index a4cfca4c5..53beca5e6 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -391,4 +391,13 @@ defmodule Pleroma.Activity do end def get_by_object_ap_id_with_object(_), do: nil + + @spec add_by_params_query(String.t(), String.t(), String.t()) :: Ecto.Query.t() + def add_by_params_query(object_id, actor, target) do + object_id + |> Queries.by_object_id() + |> Queries.by_type("Add") + |> Queries.by_actor(actor) + |> where([a], fragment("?->>'target' = ?", a.data, ^target)) + end end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 9d22f9d3c..5fe143c2b 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -340,11 +340,16 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # Tasks this handles: # - removes pin from user + # - removes corresponding Add activity # - if activity had expiration, recreates activity expiration job @impl true def handle(%{data: %{"type" => "Remove"} = data} = object, meta) do with %User{} = user <- User.get_cached_by_ap_id(data["actor"]), {:ok, _user} <- User.remove_pinned_object_id(user, data["object"]) do + data["object"] + |> Activity.add_by_params_query(user.ap_id, user.featured_address) + |> Repo.delete_all() + # if pinned activity was scheduled for deletion, we reschedule it for deletion if meta[:expires_at] do # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation -- cgit v1.2.3 From 2a520ba008f432e7e1fa297954966e0181245f01 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 19 Mar 2021 17:25:12 +0300 Subject: expanding AddRemoveValidator --- lib/pleroma/web/activity_pub/object_validator.ex | 2 +- .../object_validators/add_remove_validator.ex | 26 +++++++++++++++++----- lib/pleroma/web/activity_pub/transmogrifier.ex | 22 +++++------------- lib/pleroma/web/common_api.ex | 3 +-- 4 files changed, 27 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 3ca9136aa..14c3e8531 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -238,7 +238,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def validate(%{"type" => type} = object, meta) when type in ~w(Add Remove) do with {:ok, object} <- object - |> AddRemoveValidator.cast_and_validate(meta) + |> AddRemoveValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do object = stringify_keys(object) {:ok, object, meta} diff --git a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex index 885282f32..c38f86a0e 100644 --- a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + require Pleroma.Constants + alias Pleroma.EctoType.ActivityPub.ObjectValidators @primary_key false @@ -22,28 +24,40 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do field(:cc, ObjectValidators.Recipients, default: []) end - def cast_and_validate(data, meta) do + def cast_and_validate(data) do data + |> maybe_fix_data_for_mastodon() |> cast_data() - |> validate_data(meta) + |> validate_data() + end + + defp maybe_fix_data_for_mastodon(data) do + {:ok, actor} = Pleroma.User.get_or_fetch_by_ap_id(data["actor"]) + # Mastodon sends pin/unpin objects without id, to, cc fields + data + |> Map.put_new("id", Pleroma.Web.ActivityPub.Utils.generate_activity_id()) + |> Map.put_new("to", [Pleroma.Constants.as_public()]) + |> Map.put_new("cc", [actor.follower_address]) end defp cast_data(data) do cast(%__MODULE__{}, data, __schema__(:fields)) end - defp validate_data(changeset, meta) do + defp validate_data(changeset) do changeset |> validate_required([:id, :target, :object, :actor, :type, :to, :cc]) |> validate_inclusion(:type, ~w(Add Remove)) |> validate_actor_presence() - |> validate_collection_belongs_to_actor(meta) + |> validate_collection_belongs_to_actor() |> validate_object_presence() end - defp validate_collection_belongs_to_actor(changeset, meta) do + defp validate_collection_belongs_to_actor(changeset) do + {:ok, actor} = Pleroma.User.get_or_fetch_by_ap_id(changeset.changes[:actor]) + validate_change(changeset, :target, fn :target, target -> - if target == meta[:featured_address] do + if target == actor.featured_address do [] else [target: "collection doesn't belong to actor"] diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index c4b11a655..2172e7736 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -557,24 +557,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming(%{"type" => type} = data, _options) when type in ~w(Add Remove) do - with {:ok, %User{} = user} <- ObjectValidator.fetch_actor(data), + with :ok <- ObjectValidator.fetch_actor_and_object(data), + {:ok, actor} <- Pleroma.User.get_or_fetch_by_ap_id(data["actor"]), # maybe locally user doesn't have featured_address - {:ok, user} <- maybe_refetch_user(user), - %Object{} <- Object.normalize(data["object"], fetch: true) do - # Mastodon sends pin/unpin objects without id, to, cc fields - data = - data - |> Map.put_new("id", Utils.generate_activity_id()) - |> Map.put_new("to", [Pleroma.Constants.as_public()]) - |> Map.put_new("cc", [user.follower_address]) - - case Pipeline.common_pipeline(data, - local: false, - featured_address: user.featured_address - ) do - {:ok, activity, _meta} -> {:ok, activity} - error -> error - end + {:ok, _} <- maybe_refetch_user(actor), + {:ok, activity, _meta} <- Pipeline.common_pipeline(data, local: false) do + {:ok, activity} end end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 175d690cc..b36be4d2a 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -421,8 +421,7 @@ defmodule Pleroma.Web.CommonAPI do {:ok, _pin, _} <- Pipeline.common_pipeline(pin_data, local: true, - activity_id: id, - featured_address: user.featured_address + activity_id: id ) do {:ok, activity} else -- cgit v1.2.3 From 1885268c9c242aca2a51bd15ed839bd65d6a52dc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 25 Mar 2021 13:26:54 +0300 Subject: expanding validator --- .../object_validators/add_remove_validator.ex | 28 +++++++++++++++------- lib/pleroma/web/activity_pub/transmogrifier.ex | 18 +------------- 2 files changed, 20 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex index c38f86a0e..f885aabe4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do require Pleroma.Constants alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.User @primary_key false @@ -25,14 +26,17 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do end def cast_and_validate(data) do + {:ok, actor} = User.get_or_fetch_by_ap_id(data["actor"]) + + {:ok, actor} = maybe_refetch_user(actor) + data - |> maybe_fix_data_for_mastodon() + |> maybe_fix_data_for_mastodon(actor) |> cast_data() - |> validate_data() + |> validate_data(actor) end - defp maybe_fix_data_for_mastodon(data) do - {:ok, actor} = Pleroma.User.get_or_fetch_by_ap_id(data["actor"]) + defp maybe_fix_data_for_mastodon(data, actor) do # Mastodon sends pin/unpin objects without id, to, cc fields data |> Map.put_new("id", Pleroma.Web.ActivityPub.Utils.generate_activity_id()) @@ -44,18 +48,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do cast(%__MODULE__{}, data, __schema__(:fields)) end - defp validate_data(changeset) do + defp validate_data(changeset, actor) do changeset |> validate_required([:id, :target, :object, :actor, :type, :to, :cc]) |> validate_inclusion(:type, ~w(Add Remove)) |> validate_actor_presence() - |> validate_collection_belongs_to_actor() + |> validate_collection_belongs_to_actor(actor) |> validate_object_presence() end - defp validate_collection_belongs_to_actor(changeset) do - {:ok, actor} = Pleroma.User.get_or_fetch_by_ap_id(changeset.changes[:actor]) - + defp validate_collection_belongs_to_actor(changeset, actor) do validate_change(changeset, :target, fn :target, target -> if target == actor.featured_address do [] @@ -64,4 +66,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do end end) end + + defp maybe_refetch_user(%User{featured_address: address} = user) when is_binary(address) do + {:ok, user} + end + + defp maybe_refetch_user(%User{ap_id: ap_id}) do + Pleroma.Web.ActivityPub.Transmogrifier.upgrade_user_from_ap_id(ap_id) + end end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 2172e7736..c4caeff0a 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -534,7 +534,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming(%{"type" => type} = data, _options) - when type in ~w{Like EmojiReact Announce} do + when type in ~w{Like EmojiReact Announce Add Remove} do with :ok <- ObjectValidator.fetch_actor_and_object(data), {:ok, activity, _meta} <- Pipeline.common_pipeline(data, local: false) do @@ -556,16 +556,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def handle_incoming(%{"type" => type} = data, _options) when type in ~w(Add Remove) do - with :ok <- ObjectValidator.fetch_actor_and_object(data), - {:ok, actor} <- Pleroma.User.get_or_fetch_by_ap_id(data["actor"]), - # maybe locally user doesn't have featured_address - {:ok, _} <- maybe_refetch_user(actor), - {:ok, activity, _meta} <- Pipeline.common_pipeline(data, local: false) do - {:ok, activity} - end - end - def handle_incoming( %{"type" => "Delete"} = data, _options @@ -659,12 +649,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming(_, _), do: :error - defp maybe_refetch_user(%User{featured_address: address} = user) when is_binary(address) do - {:ok, user} - end - - defp maybe_refetch_user(%User{ap_id: ap_id}), do: upgrade_user_from_ap_id(ap_id) - @spec get_obj_helper(String.t(), Keyword.t()) :: {:ok, Object.t()} | nil def get_obj_helper(id, options \\ []) do options = Keyword.put(options, :fetch, true) -- cgit v1.2.3 From 6e108b8603de45d489d4aef7e3e271bc5e8c431d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 26 Mar 2021 19:19:19 +0300 Subject: reading the file, instead of config keyword --- lib/pleroma/config/release_runtime_provider.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/config/release_runtime_provider.ex b/lib/pleroma/config/release_runtime_provider.ex index 8227195dc..70ef3bcc1 100644 --- a/lib/pleroma/config/release_runtime_provider.ex +++ b/lib/pleroma/config/release_runtime_provider.ex @@ -39,7 +39,7 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do with_exported = if File.exists?(exported_config_path) do - exported_config = Config.Reader.read!(with_runtime_config) + exported_config = Config.Reader.read!(exported_config_path) Config.Reader.merge(with_runtime_config, exported_config) else with_runtime_config -- cgit v1.2.3 From 4d046afd2769cfdc16b2ee48e8c1d8f7f8e8ffa7 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 27 Mar 2021 09:05:33 +0300 Subject: tests for release config provider --- lib/pleroma/config/release_runtime_provider.ex | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config/release_runtime_provider.ex b/lib/pleroma/config/release_runtime_provider.ex index 70ef3bcc1..46fa35559 100644 --- a/lib/pleroma/config/release_runtime_provider.ex +++ b/lib/pleroma/config/release_runtime_provider.ex @@ -1,6 +1,6 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do @moduledoc """ - Imports `runtime.exs` and `{env}.exported_from_db.secret.exs` for elixir releases. + Imports runtime config and `{env}.exported_from_db.secret.exs` for releases. """ @behaviour Config.Provider @@ -8,13 +8,13 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do def init(opts), do: opts @impl true - def load(config, _opts) do + def load(config, opts) do with_defaults = Config.Reader.merge(config, Pleroma.Config.Holder.release_defaults()) - config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" + config_path = opts[:config_path] with_runtime_config = - if File.exists?(config_path) do + if config_path && File.exists?(config_path) do runtime_config = Config.Reader.read!(config_path) with_defaults @@ -24,7 +24,7 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do warning = [ IO.ANSI.red(), IO.ANSI.bright(), - "!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file", + "!!! Config path is not declared! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file", IO.ANSI.reset() ] @@ -32,13 +32,10 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do with_defaults end - exported_config_path = - config_path - |> Path.dirname() - |> Path.join("prod.exported_from_db.secret.exs") + exported_config_path = opts[:exported_config_path] with_exported = - if File.exists?(exported_config_path) do + if exported_config_path && File.exists?(exported_config_path) do exported_config = Config.Reader.read!(exported_config_path) Config.Reader.merge(with_runtime_config, exported_config) else -- cgit v1.2.3 From 8b81d6222773180c9632b7b53ebe7f5ee19f4f65 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 8 Oct 2020 11:55:35 -0500 Subject: Upstream original followbot implementation --- .../web/activity_pub/mrf/follow_bot_policy.ex | 41 ++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex new file mode 100644 index 000000000..fb123dbd3 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -0,0 +1,41 @@ +defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF + alias Pleroma.User + alias Pleroma.Web.CommonAPI + require Logger + + @impl true + def filter(message) do + Task.start(fn -> + follower_nickname = Pleroma.Config.get([:mrf_follow_bot, :follower_nickname]) + + with %User{} = follower <- User.get_cached_by_nickname(follower_nickname), + %{"type" => "Create", "object" => %{"type" => "Note"}} <- message do + to = Map.get(message, "to", []) + cc = Map.get(message, "cc", []) + actor = [message["actor"]] + + Enum.concat([to, cc, actor]) + |> List.flatten() + |> User.get_all_by_ap_id() + |> Enum.each(fn user -> + Logger.info("Checking if #{user.nickname} can be followed") + + with false <- User.following?(follower, user), + false <- user.locked, + false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot") do + Logger.info("Following #{user.nickname}") + CommonAPI.follow(follower, user) + end + end) + end + end) + + {:ok, message} + end + + @impl true + def describe do + {:ok, %{}} + end +end -- cgit v1.2.3 From fba770b3ea861d0fdf7811b61a297278a617136b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 8 Oct 2020 12:09:31 -0500 Subject: Try to handle misconfiguration scenarios gracefully --- .../web/activity_pub/mrf/follow_bot_policy.ex | 59 ++++++++++++++-------- 1 file changed, 37 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index fb123dbd3..52ac9aef7 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -1,34 +1,49 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do @behaviour Pleroma.Web.ActivityPub.MRF + alias Pleroma.Config alias Pleroma.User alias Pleroma.Web.CommonAPI require Logger @impl true def filter(message) do + with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]), + %User{} = follower <- User.get_cached_by_nickname(follower_nickname), + %{"type" => "Create", "object" => %{"type" => "Note"}} <- message do + try_follow(follower, message) + else + nil -> + Logger.warn( + "#{__MODULE__} skipped because of missing :mrf_follow_bot, :follower_nickname configuration or the account + does not exist." + ) + + {:ok, message} + + _ -> + {:ok, message} + end + end + + defp try_follow(follower, message) do Task.start(fn -> - follower_nickname = Pleroma.Config.get([:mrf_follow_bot, :follower_nickname]) - - with %User{} = follower <- User.get_cached_by_nickname(follower_nickname), - %{"type" => "Create", "object" => %{"type" => "Note"}} <- message do - to = Map.get(message, "to", []) - cc = Map.get(message, "cc", []) - actor = [message["actor"]] - - Enum.concat([to, cc, actor]) - |> List.flatten() - |> User.get_all_by_ap_id() - |> Enum.each(fn user -> - Logger.info("Checking if #{user.nickname} can be followed") - - with false <- User.following?(follower, user), - false <- user.locked, - false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot") do - Logger.info("Following #{user.nickname}") - CommonAPI.follow(follower, user) - end - end) - end + to = Map.get(message, "to", []) + cc = Map.get(message, "cc", []) + actor = [message["actor"]] + + Enum.concat([to, cc, actor]) + |> List.flatten() + |> User.get_all_by_ap_id() + |> Enum.each(fn user -> + Logger.info("Checking if #{user.nickname} can be followed") + + with false <- User.following?(follower, user), + false <- user.locked, + false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot") do + Logger.info("Following #{user.nickname}") + CommonAPI.follow(follower, user) + end + end) end) {:ok, message} -- cgit v1.2.3 From e78738173aefd512bbce33c12b4ee3372bdc904b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 8 Oct 2020 12:41:01 -0500 Subject: Enforce that the followbot must be marked as a bot. --- lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index 52ac9aef7..d10b7b480 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -8,14 +8,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do @impl true def filter(message) do with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]), - %User{} = follower <- User.get_cached_by_nickname(follower_nickname), + %User{actor_type: "Service"} = follower <- + User.get_cached_by_nickname(follower_nickname), %{"type" => "Create", "object" => %{"type" => "Note"}} <- message do try_follow(follower, message) else nil -> Logger.warn( - "#{__MODULE__} skipped because of missing :mrf_follow_bot, :follower_nickname configuration or the account - does not exist." + "#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname + account does not exist, or the account is not correctly configured as a bot." ) {:ok, message} -- cgit v1.2.3 From 3949cfdc249bb508c1171851fa2ec076126003cc Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 19 Feb 2021 09:47:25 -0600 Subject: Make the followbot only dispatch follow requests once per 30 day period --- .../web/activity_pub/mrf/follow_bot_policy.ex | 27 ++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index d10b7b480..044febe0c 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -1,10 +1,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do @behaviour Pleroma.Web.ActivityPub.MRF + alias Pleroma.Activity.Queries alias Pleroma.Config + alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI require Logger + import Ecto.Query + @impl true def filter(message) do with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]), @@ -36,12 +40,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do |> List.flatten() |> User.get_all_by_ap_id() |> Enum.each(fn user -> - Logger.info("Checking if #{user.nickname} can be followed") + since_thirty_days_ago = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(86_400 * 30)) with false <- User.following?(follower, user), - false <- user.locked, - false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot") do - Logger.info("Following #{user.nickname}") + false <- User.locked?(user), + false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot"), + false <- outstanding_follow_request_since?(follower, user, since_thirty_days_ago) do + Logger.info("#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}") CommonAPI.follow(follower, user) end end) @@ -50,6 +55,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do {:ok, message} end + defp outstanding_follow_request_since?( + %User{ap_id: follower_id}, + %User{ap_id: followee_id}, + since_datetime + ) do + followee_id + |> Queries.by_object_id() + |> Queries.by_type("Follow") + |> where([a], a.inserted_at > ^since_datetime) + |> where([a], fragment("? ->> 'state' = 'pending'", a.data)) + |> where([a], a.actor == ^follower_id) + |> Repo.exists?() + end + @impl true def describe do {:ok, %{}} -- cgit v1.2.3 From 3989ec508c00a66d9093ead06deb8b1272b0b985 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 19 Feb 2021 09:59:30 -0600 Subject: Prevent duplicates from being processed --- lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index 044febe0c..2fd5d5612 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -38,6 +38,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do Enum.concat([to, cc, actor]) |> List.flatten() + |> Enum.uniq() |> User.get_all_by_ap_id() |> Enum.each(fn user -> since_thirty_days_ago = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(86_400 * 30)) -- cgit v1.2.3 From a176914c73456eea7926235eb48e342ac1ab112d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 19 Feb 2021 14:42:20 -0600 Subject: Better checking of previous follow request attempts --- lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index 2fd5d5612..c7aaa6386 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -65,7 +65,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do |> Queries.by_object_id() |> Queries.by_type("Follow") |> where([a], a.inserted_at > ^since_datetime) - |> where([a], fragment("? ->> 'state' = 'pending'", a.data)) + |> where([a], fragment("? ->> 'state' != 'accept'", a.data)) |> where([a], a.actor == ^follower_id) |> Repo.exists?() end -- cgit v1.2.3 From f0dcc1ca692fb5d6a5aca4f8a9ccb255baef9c1d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 19 Feb 2021 14:55:05 -0600 Subject: Lint --- lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index c7aaa6386..441ce553e 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -47,7 +47,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do false <- User.locked?(user), false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot"), false <- outstanding_follow_request_since?(follower, user, since_thirty_days_ago) do - Logger.info("#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}") + Logger.info( + "#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}" + ) + CommonAPI.follow(follower, user) end end) -- cgit v1.2.3 From 1926d0804ba6ade106a509c027af6bf56e6a8791 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 19 Feb 2021 15:16:55 -0600 Subject: Add follow_requests_outstanding_since?/3 to Pleroma.Activity --- lib/pleroma/activity.ex | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index d59403884..b0f1a900d 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -345,6 +345,20 @@ defmodule Pleroma.Activity do |> Repo.all() end + def follow_requests_outstanding_since?( + %User{ap_id: follower_id}, + %User{ap_id: followee_id}, + since_datetime + ) do + followee_id + |> Queries.by_object_id() + |> Queries.by_type("Follow") + |> where([a], a.inserted_at > ^since_datetime) + |> where([a], fragment("? ->> 'state' != 'accept'", a.data)) + |> where([a], a.actor == ^follower_id) + |> Repo.exists?() + end + def restrict_deactivated_users(query) do deactivated_users = from(u in User.Query.build(%{deactivated: true}), select: u.ap_id) -- cgit v1.2.3 From 86182ef8e445ee8a89ce2e49f33cab3dac2d2b12 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 19 Feb 2021 15:17:33 -0600 Subject: Change module name to FollowbotPolicy --- .../web/activity_pub/mrf/follow_bot_policy.ex | 80 ---------------------- .../web/activity_pub/mrf/followbot_policy.ex | 65 ++++++++++++++++++ 2 files changed, 65 insertions(+), 80 deletions(-) delete mode 100644 lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex create mode 100644 lib/pleroma/web/activity_pub/mrf/followbot_policy.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex deleted file mode 100644 index 441ce553e..000000000 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ /dev/null @@ -1,80 +0,0 @@ -defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do - @behaviour Pleroma.Web.ActivityPub.MRF - alias Pleroma.Activity.Queries - alias Pleroma.Config - alias Pleroma.Repo - alias Pleroma.User - alias Pleroma.Web.CommonAPI - require Logger - - import Ecto.Query - - @impl true - def filter(message) do - with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]), - %User{actor_type: "Service"} = follower <- - User.get_cached_by_nickname(follower_nickname), - %{"type" => "Create", "object" => %{"type" => "Note"}} <- message do - try_follow(follower, message) - else - nil -> - Logger.warn( - "#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname - account does not exist, or the account is not correctly configured as a bot." - ) - - {:ok, message} - - _ -> - {:ok, message} - end - end - - defp try_follow(follower, message) do - Task.start(fn -> - to = Map.get(message, "to", []) - cc = Map.get(message, "cc", []) - actor = [message["actor"]] - - Enum.concat([to, cc, actor]) - |> List.flatten() - |> Enum.uniq() - |> User.get_all_by_ap_id() - |> Enum.each(fn user -> - since_thirty_days_ago = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(86_400 * 30)) - - with false <- User.following?(follower, user), - false <- User.locked?(user), - false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot"), - false <- outstanding_follow_request_since?(follower, user, since_thirty_days_ago) do - Logger.info( - "#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}" - ) - - CommonAPI.follow(follower, user) - end - end) - end) - - {:ok, message} - end - - defp outstanding_follow_request_since?( - %User{ap_id: follower_id}, - %User{ap_id: followee_id}, - since_datetime - ) do - followee_id - |> Queries.by_object_id() - |> Queries.by_type("Follow") - |> where([a], a.inserted_at > ^since_datetime) - |> where([a], fragment("? ->> 'state' != 'accept'", a.data)) - |> where([a], a.actor == ^follower_id) - |> Repo.exists?() - end - - @impl true - def describe do - {:ok, %{}} - end -end diff --git a/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex new file mode 100644 index 000000000..838d39c88 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex @@ -0,0 +1,65 @@ +defmodule Pleroma.Web.ActivityPub.MRF.FollowbotPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF + alias Pleroma.Activity + alias Pleroma.Config + alias Pleroma.User + alias Pleroma.Web.CommonAPI + + require Logger + + @impl true + def filter(message) do + with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]), + %User{actor_type: "Service"} = follower <- + User.get_cached_by_nickname(follower_nickname), + %{"type" => "Create", "object" => %{"type" => "Note"}} <- message do + try_follow(follower, message) + else + nil -> + Logger.warn( + "#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname + account does not exist, or the account is not correctly configured as a bot." + ) + + {:ok, message} + + _ -> + {:ok, message} + end + end + + defp try_follow(follower, message) do + Task.start(fn -> + to = Map.get(message, "to", []) + cc = Map.get(message, "cc", []) + actor = [message["actor"]] + + Enum.concat([to, cc, actor]) + |> List.flatten() + |> Enum.uniq() + |> User.get_all_by_ap_id() + |> Enum.each(fn user -> + since_thirty_days_ago = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(86_400 * 30)) + + with false <- User.following?(follower, user), + false <- User.locked?(user), + false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot"), + false <- + Activity.follow_requests_outstanding_since?(follower, user, since_thirty_days_ago) do + Logger.info( + "#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}" + ) + + CommonAPI.follow(follower, user) + end + end) + end) + + {:ok, message} + end + + @impl true + def describe do + {:ok, %{}} + end +end -- cgit v1.2.3 From 778010ef8e1f4509bd554e65556336e5e8457ef6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 19 Feb 2021 15:25:26 -0600 Subject: Do not try to follow local users. Their posts are already available locally on the instance. --- lib/pleroma/web/activity_pub/mrf/followbot_policy.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex index 838d39c88..5c8834536 100644 --- a/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex @@ -41,7 +41,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowbotPolicy do |> Enum.each(fn user -> since_thirty_days_ago = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(86_400 * 30)) - with false <- User.following?(follower, user), + with false <- user.local, + false <- User.following?(follower, user), false <- User.locked?(user), false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot"), false <- -- cgit v1.2.3 From c252ac71d4ea4f3b08bd3524f32ee3fe9308be06 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 19 Feb 2021 18:34:52 -0600 Subject: Revert --- lib/pleroma/activity.ex | 14 -------------- 1 file changed, 14 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index b0f1a900d..d59403884 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -345,20 +345,6 @@ defmodule Pleroma.Activity do |> Repo.all() end - def follow_requests_outstanding_since?( - %User{ap_id: follower_id}, - %User{ap_id: followee_id}, - since_datetime - ) do - followee_id - |> Queries.by_object_id() - |> Queries.by_type("Follow") - |> where([a], a.inserted_at > ^since_datetime) - |> where([a], fragment("? ->> 'state' != 'accept'", a.data)) - |> where([a], a.actor == ^follower_id) - |> Repo.exists?() - end - def restrict_deactivated_users(query) do deactivated_users = from(u in User.Query.build(%{deactivated: true}), select: u.ap_id) -- cgit v1.2.3 From 4796df0bc39a57b2581168cb8d8fde7779068f2d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 19 Feb 2021 18:36:35 -0600 Subject: Remove Task.async as it is broken here and probably a premature optimization anyway --- .../web/activity_pub/mrf/followbot_policy.ex | 43 +++++++++------------- 1 file changed, 18 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex index 5c8834536..ca99e429c 100644 --- a/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex @@ -1,6 +1,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowbotPolicy do @behaviour Pleroma.Web.ActivityPub.MRF - alias Pleroma.Activity alias Pleroma.Config alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -29,31 +28,25 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowbotPolicy do end defp try_follow(follower, message) do - Task.start(fn -> - to = Map.get(message, "to", []) - cc = Map.get(message, "cc", []) - actor = [message["actor"]] - - Enum.concat([to, cc, actor]) - |> List.flatten() - |> Enum.uniq() - |> User.get_all_by_ap_id() - |> Enum.each(fn user -> - since_thirty_days_ago = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(86_400 * 30)) - - with false <- user.local, - false <- User.following?(follower, user), - false <- User.locked?(user), - false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot"), - false <- - Activity.follow_requests_outstanding_since?(follower, user, since_thirty_days_ago) do - Logger.info( - "#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}" - ) + to = Map.get(message, "to", []) + cc = Map.get(message, "cc", []) + actor = [message["actor"]] + + Enum.concat([to, cc, actor]) + |> List.flatten() + |> Enum.uniq() + |> User.get_all_by_ap_id() + |> Enum.each(fn user -> + with false <- user.local, + false <- User.following?(follower, user), + false <- User.locked?(user), + false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot") do + Logger.debug( + "#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}" + ) - CommonAPI.follow(follower, user) - end - end) + CommonAPI.follow(follower, user) + end end) {:ok, message} -- cgit v1.2.3 From 03f38ac4ebd97e792b0ff2a6ac804adefed85a41 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 24 Feb 2021 11:59:11 -0600 Subject: Prefer FollowBot naming convention vs Followbot --- .../web/activity_pub/mrf/follow_bot_policy.ex | 59 ++++++++++++++++++++++ .../web/activity_pub/mrf/followbot_policy.ex | 59 ---------------------- 2 files changed, 59 insertions(+), 59 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex delete mode 100644 lib/pleroma/web/activity_pub/mrf/followbot_policy.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex new file mode 100644 index 000000000..7307c9c14 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -0,0 +1,59 @@ +defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF + alias Pleroma.Config + alias Pleroma.User + alias Pleroma.Web.CommonAPI + + require Logger + + @impl true + def filter(message) do + with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]), + %User{actor_type: "Service"} = follower <- + User.get_cached_by_nickname(follower_nickname), + %{"type" => "Create", "object" => %{"type" => "Note"}} <- message do + try_follow(follower, message) + else + nil -> + Logger.warn( + "#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname + account does not exist, or the account is not correctly configured as a bot." + ) + + {:ok, message} + + _ -> + {:ok, message} + end + end + + defp try_follow(follower, message) do + to = Map.get(message, "to", []) + cc = Map.get(message, "cc", []) + actor = [message["actor"]] + + Enum.concat([to, cc, actor]) + |> List.flatten() + |> Enum.uniq() + |> User.get_all_by_ap_id() + |> Enum.each(fn user -> + with false <- user.local, + false <- User.following?(follower, user), + false <- User.locked?(user), + false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot") do + Logger.debug( + "#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}" + ) + + CommonAPI.follow(follower, user) + end + end) + + {:ok, message} + end + + @impl true + def describe do + {:ok, %{}} + end +end diff --git a/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex deleted file mode 100644 index ca99e429c..000000000 --- a/lib/pleroma/web/activity_pub/mrf/followbot_policy.ex +++ /dev/null @@ -1,59 +0,0 @@ -defmodule Pleroma.Web.ActivityPub.MRF.FollowbotPolicy do - @behaviour Pleroma.Web.ActivityPub.MRF - alias Pleroma.Config - alias Pleroma.User - alias Pleroma.Web.CommonAPI - - require Logger - - @impl true - def filter(message) do - with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]), - %User{actor_type: "Service"} = follower <- - User.get_cached_by_nickname(follower_nickname), - %{"type" => "Create", "object" => %{"type" => "Note"}} <- message do - try_follow(follower, message) - else - nil -> - Logger.warn( - "#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname - account does not exist, or the account is not correctly configured as a bot." - ) - - {:ok, message} - - _ -> - {:ok, message} - end - end - - defp try_follow(follower, message) do - to = Map.get(message, "to", []) - cc = Map.get(message, "cc", []) - actor = [message["actor"]] - - Enum.concat([to, cc, actor]) - |> List.flatten() - |> Enum.uniq() - |> User.get_all_by_ap_id() - |> Enum.each(fn user -> - with false <- user.local, - false <- User.following?(follower, user), - false <- User.locked?(user), - false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot") do - Logger.debug( - "#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}" - ) - - CommonAPI.follow(follower, user) - end - end) - - {:ok, message} - end - - @impl true - def describe do - {:ok, %{}} - end -end -- cgit v1.2.3 From 863010ea637d6670076dba3f6da54daa144cce67 Mon Sep 17 00:00:00 2001 From: Miss Pasture Date: Wed, 31 Mar 2021 06:51:22 +0000 Subject: date-times are always strings --- lib/pleroma/web/api_spec/operations/account_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 54e5ebc76..08d68893a 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -482,7 +482,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do access_token: %Schema{type: :string}, refresh_token: %Schema{type: :string}, scope: %Schema{type: :string}, - created_at: %Schema{type: :integer, format: :"date-time"}, + created_at: %Schema{type: :string, format: :"date-time"}, me: %Schema{type: :string}, expires_in: %Schema{type: :integer}, # -- cgit v1.2.3 From af1cd28f9b8005dff563c0df7f80db47df4e8488 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 1 Apr 2021 11:50:45 +0200 Subject: object_validator: Refactor most of validate/2 to a generic block --- lib/pleroma/web/activity_pub/object_validator.ex | 135 ++++------------------- 1 file changed, 22 insertions(+), 113 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 297c19cc0..f75744203 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -37,37 +37,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do @impl true def validate(object, meta) - def validate(%{"type" => type} = object, meta) - when type in ~w[Accept Reject] do - with {:ok, object} <- - object - |> AcceptRejectValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object) - {:ok, object, meta} - end - end - - def validate(%{"type" => "Event"} = object, meta) do - with {:ok, object} <- - object - |> EventValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object) - {:ok, object, meta} - end - end - - def validate(%{"type" => "Follow"} = object, meta) do - with {:ok, object} <- - object - |> FollowValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object) - {:ok, object, meta} - end - end - def validate(%{"type" => "Block"} = block_activity, meta) do with {:ok, block_activity} <- block_activity @@ -87,16 +56,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end end - def validate(%{"type" => "Update"} = update_activity, meta) do - with {:ok, update_activity} <- - update_activity - |> UpdateValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - update_activity = stringify_keys(update_activity) - {:ok, update_activity, meta} - end - end - def validate(%{"type" => "Undo"} = object, meta) do with {:ok, object} <- object @@ -123,76 +82,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end end - def validate(%{"type" => "Like"} = object, meta) do - with {:ok, object} <- - object - |> LikeValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object) - {:ok, object, meta} - end - end - - def validate(%{"type" => "ChatMessage"} = object, meta) do - with {:ok, object} <- - object - |> ChatMessageValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object) - {:ok, object, meta} - end - end - - def validate(%{"type" => "Question"} = object, meta) do - with {:ok, object} <- - object - |> QuestionValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object) - {:ok, object, meta} - end - end - - def validate(%{"type" => type} = object, meta) when type in ~w[Audio Video] do - with {:ok, object} <- - object - |> AudioVideoValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object) - {:ok, object, meta} - end - end - - def validate(%{"type" => "Article"} = object, meta) do - with {:ok, object} <- - object - |> ArticleNoteValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object) - {:ok, object, meta} - end - end - - def validate(%{"type" => "Answer"} = object, meta) do - with {:ok, object} <- - object - |> AnswerValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object) - {:ok, object, meta} - end - end - - def validate(%{"type" => "EmojiReact"} = object, meta) do - with {:ok, object} <- - object - |> EmojiReactValidator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object) - {:ok, object, meta} - end - end - def validate( %{"type" => "Create", "object" => %{"type" => "ChatMessage"} = object} = create_activity, meta @@ -224,10 +113,30 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end end - def validate(%{"type" => "Announce"} = object, meta) do + def validate(%{"type" => type} = object, meta) + when type in ~w[Accept Reject Follow Update Like EmojiReact Announce + Event ChatMessage Question Audio Video Article Answer] do + validator = + case type do + "Accept" -> AcceptRejectValidator + "Reject" -> AcceptRejectValidator + "Follow" -> FollowValidator + "Update" -> UpdateValidator + "Like" -> LikeValidator + "EmojiReact" -> EmojiReactValidator + "Announce" -> AnnounceValidator + "Event" -> EventValidator + "ChatMessage" -> ChatMessageValidator + "Question" -> QuestionValidator + "Audio" -> AudioVideoValidator + "Video" -> AudioVideoValidator + "Article" -> ArticleNoteValidator + "Answer" -> AnswerValidator + end + with {:ok, object} <- object - |> AnnounceValidator.cast_and_validate() + |> validator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do object = stringify_keys(object) {:ok, object, meta} -- cgit v1.2.3 From 1e3db075861198417bca36f4f2b0f29c2367c77e Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Thu, 1 Apr 2021 12:00:58 +0000 Subject: Revert "Merge branch 'patch-fix-open-api-spec' into 'develop'" This reverts merge request !3382 --- lib/pleroma/web/api_spec/operations/account_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 08d68893a..54e5ebc76 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -482,7 +482,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do access_token: %Schema{type: :string}, refresh_token: %Schema{type: :string}, scope: %Schema{type: :string}, - created_at: %Schema{type: :string, format: :"date-time"}, + created_at: %Schema{type: :integer, format: :"date-time"}, me: %Schema{type: :string}, expires_in: %Schema{type: :integer}, # -- cgit v1.2.3 From 9015df22291ab60c0efad328557936fd14eab2e6 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 7 Jan 2021 18:23:01 +0100 Subject: TagValidator: New --- .../object_validators/article_note_validator.ex | 7 +- .../object_validators/attachment_validator.ex | 1 - .../object_validators/audio_video_validator.ex | 7 +- .../object_validators/event_validator.ex | 7 +- .../object_validators/question_validator.ex | 7 +- .../object_validators/tag_validator.ex | 77 ++++++++++++++++++++++ 6 files changed, 93 insertions(+), 13 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/object_validators/tag_validator.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex index b0388ef3b..5910f4060 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -22,8 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do field(:cc, ObjectValidators.Recipients, default: []) field(:bto, ObjectValidators.Recipients, default: []) field(:bcc, ObjectValidators.Recipients, default: []) - # TODO: Write type - field(:tag, {:array, :map}, default: []) + embeds_many(:tag, TagValidator) field(:type, :string) field(:name, :string) @@ -90,8 +90,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do data = fix(data) struct - |> cast(data, __schema__(:fields) -- [:attachment]) + |> cast(data, __schema__(:fields) -- [:attachment, :tag]) |> cast_embed(:attachment) + |> cast_embed(:tag) end def validate_data(data_cng) do diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index 3175427ad..e7b3a3922 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators - alias Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator import Ecto.Changeset diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index 4a96fef52..a04c95f4b 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -23,8 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do field(:cc, ObjectValidators.Recipients, default: []) field(:bto, ObjectValidators.Recipients, default: []) field(:bcc, ObjectValidators.Recipients, default: []) - # TODO: Write type - field(:tag, {:array, :map}, default: []) + embeds_many(:tag, TagValidator) field(:type, :string) field(:name, :string) @@ -132,8 +132,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do data = fix(data) struct - |> cast(data, __schema__(:fields) -- [:attachment]) + |> cast(data, __schema__(:fields) -- [:attachment, :tag]) |> cast_embed(:attachment) + |> cast_embed(:tag) end def validate_data(data_cng) do diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex index 2e26726f8..0112a074d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -23,8 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do field(:cc, ObjectValidators.Recipients, default: []) field(:bto, ObjectValidators.Recipients, default: []) field(:bcc, ObjectValidators.Recipients, default: []) - # TODO: Write type - field(:tag, {:array, :map}, default: []) + embeds_many(:tag, TagValidator) field(:type, :string) field(:name, :string) @@ -81,8 +81,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do data = fix(data) struct - |> cast(data, __schema__(:fields) -- [:attachment]) + |> cast(data, __schema__(:fields) -- [:attachment, :tag]) |> cast_embed(:attachment) + |> cast_embed(:tag) end def validate_data(data_cng) do diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 6b746c997..7acb1e928 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -24,8 +25,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do field(:cc, ObjectValidators.Recipients, default: []) field(:bto, ObjectValidators.Recipients, default: []) field(:bcc, ObjectValidators.Recipients, default: []) - # TODO: Write type - field(:tag, {:array, :map}, default: []) + embeds_many(:tag, TagValidator) field(:type, :string) field(:content, :string) field(:context, :string) @@ -93,10 +93,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do data = fix(data) struct - |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment]) + |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment, :tag]) |> cast_embed(:attachment) |> cast_embed(:anyOf) |> cast_embed(:oneOf) + |> cast_embed(:tag) end def validate_data(data_cng) do diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex new file mode 100644 index 000000000..751021585 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -0,0 +1,77 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + + import Ecto.Changeset + + @primary_key false + embedded_schema do + # Common + field(:type, :string) + field(:name, :string) + + # Mention, Hashtag + field(:href, ObjectValidators.Uri) + + # Emoji + embeds_one :icon, IconObjectValidator, primary_key: false do + field(:type, :string) + field(:url, ObjectValidators.Uri) + end + + field(:updated, ObjectValidators.DateTime) + field(:id, ObjectValidators.Uri) + end + + def cast_and_validate(data) do + data + |> cast_data() + end + + def cast_data(data) do + %__MODULE__{} + |> changeset(data) + end + + def changeset(struct, %{"type" => "Mention"} = data) do + struct + |> cast(data, [:type, :name, :href]) + |> validate_required([:type, :href]) + end + + def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) do + name = + cond do + "#" <> name -> name + name -> name + end + |> String.downcase() + + data = Map.put(data, "name", name) + + struct + |> cast(data, [:type, :name, :href]) + |> validate_required([:type, :name]) + end + + def changeset(struct, %{"type" => "Emoji"} = data) do + data = Map.put(data, "name", String.trim(data["name"], ":")) + + struct + |> cast(data, [:type, :name, :updated, :id]) + |> cast_embed(:icon, with: &icon_changeset/2) + |> validate_required([:type, :name, :icon]) + end + + def icon_changeset(struct, data) do + struct + |> cast(data, [:type, :url]) + |> validate_inclusion(:type, ~w[Image]) + |> validate_required([:type, :url]) + end +end -- cgit v1.2.3 From 5ae27c8451a7012b43ef9113713132158701364b Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 12 Jan 2021 14:11:29 +0100 Subject: pipeline_test: Fix usage of %Activity{} --- lib/pleroma/web/activity_pub/object_validator.ex | 2 +- lib/pleroma/web/activity_pub/pipeline.ex | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index f75744203..15784b28c 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -169,7 +169,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def cast_and_apply(o), do: {:error, {:validator_not_set, o}} - # is_struct/1 isn't present in Elixir 1.8.x + # is_struct/1 appears in Elixir 1.11 def stringify_keys(%{__struct__: _} = object) do object |> Map.from_struct() diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 195596f94..0aa504e72 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -40,6 +40,8 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do end end + def do_common_pipeline(%{__struct__: _}, _meta), do: {:error, :is_struct} + def do_common_pipeline(object, meta) do with {_, {:ok, validated_object, meta}} <- {:validate_object, @object_validator.validate(object, meta)}, -- cgit v1.2.3 From 37a7f521fd4778cde48f1b003ad9695e6ea45d1f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 12 Jan 2021 09:30:22 +0100 Subject: Insert string-hashtags in Pipeline Cannot be done in Ecto schemas because only one type is allowed in arrays, and needs to be done before the MRFs. --- lib/pleroma/web/activity_pub/pipeline.ex | 34 +++++++++++++++++--------- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- lib/pleroma/web/common_api.ex | 12 +-------- 3 files changed, 24 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 0aa504e72..e184a9376 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -42,23 +42,33 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do def do_common_pipeline(%{__struct__: _}, _meta), do: {:error, :is_struct} - def do_common_pipeline(object, meta) do - with {_, {:ok, validated_object, meta}} <- - {:validate_object, @object_validator.validate(object, meta)}, - {_, {:ok, mrfd_object, meta}} <- - {:mrf_object, @mrf.pipeline_filter(validated_object, meta)}, - {_, {:ok, activity, meta}} <- - {:persist_object, @activity_pub.persist(mrfd_object, meta)}, - {_, {:ok, activity, meta}} <- - {:execute_side_effects, @side_effects.handle(activity, meta)}, - {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do - {:ok, activity, meta} + def do_common_pipeline(message, meta) do + with {_, {:ok, message, meta}} <- {:validate, @object_validator.validate(message, meta)}, + {_, {:ok, message, meta}} <- {:fixup, validation_fixups(message, meta)}, + {_, {:ok, message, meta}} <- {:mrf, @mrf.pipeline_filter(message, meta)}, + {_, {:ok, message, meta}} <- {:persist, @activity_pub.persist(message, meta)}, + {_, {:ok, message, meta}} <- {:side_effects, @side_effects.handle(message, meta)}, + {_, {:ok, _}} <- {:federation, maybe_federate(message, meta)} do + {:ok, message, meta} else - {:mrf_object, {:reject, message, _}} -> {:reject, message} + {:mrf, {:reject, message, _}} -> {:reject, message} e -> {:error, e} end end + defp validation_fixups(message, meta) do + # Insert copy of hashtags as strings for the non-hashtag table indexing + message = + if message["tag"] do + tag = Object.hashtags(%Object{data: message}) ++ (message["tag"] || []) + Map.put(message, "tag", tag) + else + message + end + + {:ok, message, meta} + end + defp maybe_federate(%Object{}, _), do: {:ok, :not_federated} defp maybe_federate(%Activity{} = activity, meta) do diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 8c7d6a747..4070ed14d 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -564,7 +564,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Pipeline.common_pipeline(data, local: false) do {:ok, activity} else - {:error, {:validate_object, _}} = e -> + {:error, {:validate, _}} = e -> # Check if we have a create activity for this with {:ok, object_id} <- ObjectValidators.ObjectID.cast(data["object"]), %Activity{data: %{"actor" => actor}} <- diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index b003e30c7..895baebc9 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -228,17 +228,7 @@ defmodule Pleroma.Web.CommonAPI do {:find_object, _} -> {:error, :not_found} - {:common_pipeline, - { - :error, - { - :validate_object, - { - :error, - changeset - } - } - }} = e -> + {:common_pipeline, {:error, {:validate, {:error, changeset}}}} = e -> if {:object, {"already liked by this actor", []}} in changeset.errors do {:ok, :already_liked} else -- cgit v1.2.3 From 7ebfe899007002f5bbf8744a8f0b582e0e13342e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 12 Jan 2021 11:14:09 +0100 Subject: object_validators: Mark validate_data as private --- .../web/activity_pub/object_validators/accept_reject_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/announce_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/answer_validator.ex | 2 +- .../web/activity_pub/object_validators/article_note_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/block_validator.ex | 2 +- .../web/activity_pub/object_validators/chat_message_validator.ex | 2 +- .../web/activity_pub/object_validators/create_chat_message_validator.ex | 2 +- .../web/activity_pub/object_validators/create_generic_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/delete_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/event_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/follow_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/like_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/question_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/undo_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/update_validator.ex | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex index d31e780c3..b577a1044 100644 --- a/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator do |> cast(data, __schema__(:fields)) end - def validate_data(cng) do + defp validate_data(cng) do cng |> validate_required([:id, :type, :actor, :to, :cc, :object]) |> validate_inclusion(:type, ["Accept", "Reject"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex index b08a33e68..576341790 100644 --- a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex @@ -50,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do cng end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Announce"]) |> validate_required([:id, :type, :object, :actor, :to, :cc]) diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex index 15e4413cd..c9bd9e42d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex @@ -50,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do |> cast(data, __schema__(:fields)) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Answer"]) |> validate_required([:id, :inReplyTo, :name, :attributedTo, :actor]) diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex index 5910f4060..39ef6dc29 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex @@ -95,7 +95,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do |> cast_embed(:tag) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Article", "Note"]) |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index e7b3a3922..4a0d1473d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -89,7 +89,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do end end - def validate_data(cng) do + defp validate_data(cng) do cng |> validate_inclusion(:type, ~w[Document Audio Image Video]) |> validate_required([:mediaType, :url, :type]) diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index a04c95f4b..8a5a60526 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -137,7 +137,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do |> cast_embed(:tag) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Audio", "Video"]) |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment]) diff --git a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex index c5f77bb76..88948135f 100644 --- a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex @@ -26,7 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do |> cast(data, __schema__(:fields)) end - def validate_data(cng) do + defp validate_data(cng) do cng |> validate_required([:id, :type, :actor, :to, :cc, :object]) |> validate_inclusion(:type, ["Block"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex index 1189778f2..b153156b0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex @@ -67,7 +67,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do |> cast_embed(:attachment) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["ChatMessage"]) |> validate_required([:id, :actor, :to, :type, :published]) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex index 8384c16a7..7a31a99bf 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex @@ -39,7 +39,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do |> validate_data(meta) end - def validate_data(cng, meta \\ []) do + defp validate_data(cng, meta) do cng |> validate_required([:id, :actor, :to, :type, :object]) |> validate_inclusion(:type, ["Create"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index bf56a918c..e06e442f4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -79,7 +79,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do |> CommonFixes.fix_actor() end - def validate_data(cng, meta \\ []) do + defp validate_data(cng, meta) do cng |> validate_required([:actor, :type, :object]) |> validate_inclusion(:type, ["Create"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex index fc1a79a72..7da67bf16 100644 --- a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex @@ -53,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do Tombstone Video } - def validate_data(cng) do + defp validate_data(cng) do cng |> validate_required([:id, :type, :actor, :to, :cc, :object]) |> validate_inclusion(:type, ["Delete"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex index 1906e597e..ec7566515 100644 --- a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex @@ -70,7 +70,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do end end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["EmojiReact"]) |> validate_required([:id, :type, :object, :actor, :context, :to, :cc, :content]) diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex index 0112a074d..d42458ef5 100644 --- a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex @@ -86,7 +86,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do |> cast_embed(:tag) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Event"]) |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) diff --git a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex index 6e428bacc..239cee5e7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator do |> cast(data, __schema__(:fields)) end - def validate_data(cng) do + defp validate_data(cng) do cng |> validate_required([:id, :type, :actor, :to, :cc, :object]) |> validate_inclusion(:type, ["Follow"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex index 30c40b238..509da507b 100644 --- a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex @@ -76,7 +76,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do end end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Like"]) |> validate_required([:id, :type, :object, :actor, :context, :to, :cc]) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 7acb1e928..7012e2e1d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -100,7 +100,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do |> cast_embed(:tag) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Question"]) |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) diff --git a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex index 783a79ddb..e8af60ffa 100644 --- a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator do |> cast(data, __schema__(:fields)) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Undo"]) |> validate_required([:id, :type, :object, :actor, :to, :cc]) diff --git a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex index a66d41400..6bb1dc7fa 100644 --- a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator do |> cast(data, __schema__(:fields)) end - def validate_data(cng) do + defp validate_data(cng) do cng |> validate_required([:id, :type, :actor, :to, :cc, :object]) |> validate_inclusion(:type, ["Update"]) -- cgit v1.2.3 From ef36f7fa5cff0a0d364aff192954556b0d2b0d2a Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 1 Apr 2021 13:49:04 +0200 Subject: Move tag fixup to object_validator --- lib/pleroma/web/activity_pub/object_validator.ex | 32 +++++++++++++++++++----- lib/pleroma/web/activity_pub/pipeline.ex | 14 ----------- 2 files changed, 26 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 15784b28c..70d9a35a9 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -113,9 +113,34 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end end + def validate(%{"type" => type} = object, meta) + when type in ~w[Event Question Audio Video Article] do + validator = + case type do + "Event" -> EventValidator + "Question" -> QuestionValidator + "Audio" -> AudioVideoValidator + "Video" -> AudioVideoValidator + "Article" -> ArticleNoteValidator + end + + with {:ok, object} <- + object + |> validator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + + # Insert copy of hashtags as strings for the non-hashtag table indexing + tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object}) + object = Map.put(object, "tag", tag) + + {:ok, object, meta} + end + end + def validate(%{"type" => type} = object, meta) when type in ~w[Accept Reject Follow Update Like EmojiReact Announce - Event ChatMessage Question Audio Video Article Answer] do + ChatMessage Answer] do validator = case type do "Accept" -> AcceptRejectValidator @@ -125,12 +150,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do "Like" -> LikeValidator "EmojiReact" -> EmojiReactValidator "Announce" -> AnnounceValidator - "Event" -> EventValidator "ChatMessage" -> ChatMessageValidator - "Question" -> QuestionValidator - "Audio" -> AudioVideoValidator - "Video" -> AudioVideoValidator - "Article" -> ArticleNoteValidator "Answer" -> AnswerValidator end diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index e184a9376..377eccb92 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -44,7 +44,6 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do def do_common_pipeline(message, meta) do with {_, {:ok, message, meta}} <- {:validate, @object_validator.validate(message, meta)}, - {_, {:ok, message, meta}} <- {:fixup, validation_fixups(message, meta)}, {_, {:ok, message, meta}} <- {:mrf, @mrf.pipeline_filter(message, meta)}, {_, {:ok, message, meta}} <- {:persist, @activity_pub.persist(message, meta)}, {_, {:ok, message, meta}} <- {:side_effects, @side_effects.handle(message, meta)}, @@ -56,19 +55,6 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do end end - defp validation_fixups(message, meta) do - # Insert copy of hashtags as strings for the non-hashtag table indexing - message = - if message["tag"] do - tag = Object.hashtags(%Object{data: message}) ++ (message["tag"] || []) - Map.put(message, "tag", tag) - else - message - end - - {:ok, message, meta} - end - defp maybe_federate(%Object{}, _), do: {:ok, :not_federated} defp maybe_federate(%Activity{} = activity, meta) do -- cgit v1.2.3 From e56779dd8d1668177afa199aaa836bea70e68420 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 10 Sep 2020 11:09:11 +0200 Subject: Transmogrifier: Simplify fix_explicit_addressing and fix_implicit_addressing --- lib/pleroma/web/activity_pub/transmogrifier.ex | 51 ++++++++------------------ 1 file changed, 16 insertions(+), 35 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 4070ed14d..047f23918 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -72,17 +72,21 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def fix_explicit_addressing( - %{"to" => to, "cc" => cc} = object, - explicit_mentions, - follower_collection - ) do - explicit_to = Enum.filter(to, fn x -> x in explicit_mentions end) + # if directMessage flag is set to true, leave the addressing alone + def fix_explicit_addressing(%{"directMessage" => true} = object, _follower_collection), + do: object + def fix_explicit_addressing(%{"to" => to, "cc" => cc} = object, follower_collection) do + explicit_mentions = + Utils.determine_explicit_mentions(object) ++ + [Pleroma.Constants.as_public(), follower_collection] + + explicit_to = Enum.filter(to, fn x -> x in explicit_mentions end) explicit_cc = Enum.filter(to, fn x -> x not in explicit_mentions end) final_cc = (cc ++ explicit_cc) + |> Enum.filter(& &1) |> Enum.reject(fn x -> String.ends_with?(x, "/followers") and x != follower_collection end) |> Enum.uniq() @@ -91,29 +95,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put("cc", final_cc) end - def fix_explicit_addressing(object, _explicit_mentions, _followers_collection), do: object - - # if directMessage flag is set to true, leave the addressing alone - def fix_explicit_addressing(%{"directMessage" => true} = object), do: object - - def fix_explicit_addressing(object) do - explicit_mentions = Utils.determine_explicit_mentions(object) - - %User{follower_address: follower_collection} = - object - |> Containment.get_actor() - |> User.get_cached_by_ap_id() - - explicit_mentions = - explicit_mentions ++ - [ - Pleroma.Constants.as_public(), - follower_collection - ] - - fix_explicit_addressing(object, explicit_mentions, follower_collection) - end - # if as:Public is addressed, then make sure the followers collection is also addressed # so that the activities will be delivered to local users. def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collection) do @@ -137,19 +118,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def fix_implicit_addressing(object, _), do: object - def fix_addressing(object) do - {:ok, %User{} = user} = User.get_or_fetch_by_ap_id(object["actor"]) - followers_collection = User.ap_followers(user) + {:ok, %User{follower_address: follower_collection}} = + object + |> Containment.get_actor() + |> User.get_or_fetch_by_ap_id() object |> fix_addressing_list("to") |> fix_addressing_list("cc") |> fix_addressing_list("bto") |> fix_addressing_list("bcc") - |> fix_explicit_addressing() - |> fix_implicit_addressing(followers_collection) + |> fix_explicit_addressing(follower_collection) + |> fix_implicit_addressing(follower_collection) end def fix_actor(%{"attributedTo" => actor} = object) do -- cgit v1.2.3 From e2a3365b5ce86293a5fed28c06b2e7d9dd97c9d1 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 10 Sep 2020 11:08:05 +0200 Subject: ObjectValidator.CommonFixes: Introduce fix_objects_defaults and fix_activity_defaults --- .../activity_pub/object_validators/recipients.ex | 24 +++++++++++-------- .../object_validators/article_note_validator.ex | 3 +-- .../object_validators/audio_video_validator.ex | 3 +-- .../activity_pub/object_validators/common_fixes.ex | 28 ++++++++++++++++++---- .../object_validators/create_generic_validator.ex | 12 +--------- .../object_validators/event_validator.ex | 4 ++-- .../object_validators/question_validator.ex | 4 ++-- 7 files changed, 44 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex index af4b0e527..b76547e75 100644 --- a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex @@ -15,19 +15,23 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do def cast(data) when is_list(data) do data - |> Enum.reduce_while({:ok, []}, fn element, {:ok, list} -> - case ObjectID.cast(element) do - {:ok, id} -> - {:cont, {:ok, [id | list]}} - - _ -> - {:halt, :error} - end + |> Enum.reduce_while({:ok, []}, fn + nil, {:ok, list} -> + {:cont, {:ok, list}} + + element, {:ok, list} -> + case ObjectID.cast(element) do + {:ok, id} -> + {:cont, {:ok, [id | list]}} + + _ -> + {:halt, {:error, element}} + end end) end - def cast(_) do - :error + def cast(data) do + {:error, data} end def dump(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex index 39ef6dc29..d2026b5ea 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex @@ -79,9 +79,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do defp fix(data) do data - |> CommonFixes.fix_defaults() - |> CommonFixes.fix_attribution() |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() |> fix_url() |> Transmogrifier.fix_emoji() end diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index 8a5a60526..8ee432947 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -120,9 +120,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do defp fix(data) do data - |> CommonFixes.fix_defaults() - |> CommonFixes.fix_attribution() |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() |> Transmogrifier.fix_emoji() |> fix_url() |> fix_content() diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 5f2c633bc..950eb1494 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -3,26 +3,44 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object.Containment + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils - # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults - def fix_defaults(data) do + def fix_object_defaults(data) do %{data: %{"id" => context}, id: context_id} = Utils.create_context(data["context"] || data["conversation"]) + %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"]) + {:ok, to} = ObjectValidators.Recipients.cast(data["to"] || []) + {:ok, cc} = ObjectValidators.Recipients.cast(data["cc"] || []) + data |> Map.put("context", context) |> Map.put("context_id", context_id) + |> Map.put("to", to) + |> Map.put("cc", cc) + |> Transmogrifier.fix_explicit_addressing(follower_collection) + |> Transmogrifier.fix_implicit_addressing(follower_collection) end - def fix_attribution(data) do + def fix_activity_defaults(data, meta) do + object = meta[:object_data] || %{} + data - |> Map.put_new("actor", data["attributedTo"]) + |> Map.put_new("to", object["to"] || []) + |> Map.put_new("cc", object["cc"] || []) + |> Map.put_new("bto", object["bto"] || []) + |> Map.put_new("bcc", object["bcc"] || []) end def fix_actor(data) do - actor = Containment.get_actor(data) + actor = + data + |> Map.put_new("actor", data["attributedTo"]) + |> Containment.get_actor() data |> Map.put("actor", actor) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index e06e442f4..99e8dc6c7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -62,21 +62,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do end end - defp fix_addressing(data, meta) do - if object = meta[:object_data] do - data - |> Map.put_new("to", object["to"] || []) - |> Map.put_new("cc", object["cc"] || []) - else - data - end - end - defp fix(data, meta) do data |> fix_context(meta) - |> fix_addressing(meta) |> CommonFixes.fix_actor() + |> CommonFixes.fix_activity_defaults(meta) end defp validate_data(cng, meta) do diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex index d42458ef5..fee2e997a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex @@ -72,8 +72,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do defp fix(data) do data - |> CommonFixes.fix_defaults() - |> CommonFixes.fix_attribution() + |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() |> Transmogrifier.fix_emoji() end diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 7012e2e1d..083d08ec4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -83,8 +83,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do defp fix(data) do data - |> CommonFixes.fix_defaults() - |> CommonFixes.fix_attribution() + |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() |> Transmogrifier.fix_emoji() |> fix_closed() end -- cgit v1.2.3 From c9449326747f8d33357f5179e69d3024b39089a0 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 10 Sep 2020 11:11:10 +0200 Subject: Pipeline Ingestion: Note --- .../activity_pub/object_validators/recipients.ex | 27 +++++++++++--------- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/activity_pub/object_validator.ex | 7 +++--- .../object_validators/article_note_validator.ex | 29 +++++++++++++++++----- .../activity_pub/object_validators/common_fixes.ex | 18 +++++++++----- .../object_validators/common_validations.ex | 1 + .../object_validators/create_note_validator.ex | 29 ---------------------- lib/pleroma/web/activity_pub/side_effects.ex | 15 ++++++++++- lib/pleroma/web/activity_pub/transmogrifier.ex | 12 ++++----- lib/pleroma/web/federator.ex | 5 ++++ 10 files changed, 81 insertions(+), 64 deletions(-) delete mode 100644 lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex (limited to 'lib') diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex index b76547e75..a03471462 100644 --- a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex @@ -13,20 +13,23 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do cast([object]) end + def cast(object) when is_map(object) do + case ObjectID.cast(object) do + {:ok, data} -> {:ok, data} + _ -> :error + end + end + def cast(data) when is_list(data) do data - |> Enum.reduce_while({:ok, []}, fn - nil, {:ok, list} -> - {:cont, {:ok, list}} - - element, {:ok, list} -> - case ObjectID.cast(element) do - {:ok, id} -> - {:cont, {:ok, [id | list]}} - - _ -> - {:halt, {:error, element}} - end + |> Enum.reduce_while({:ok, []}, fn element, {:ok, list} -> + case ObjectID.cast(element) do + {:ok, id} -> + {:cont, {:ok, [id | list]}} + + _ -> + {:cont, {:ok, list}} + end end) end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index efbf92c70..b74af3f3b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -88,7 +88,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp increase_replies_count_if_reply(_create_data), do: :noop - @object_types ~w[ChatMessage Question Answer Audio Video Event Article] + @object_types ~w[ChatMessage Question Answer Audio Video Event Article Note] @impl true def persist(%{"type" => type} = object, meta) when type in @object_types do with {:ok, object} <- Object.create(object) do diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 70d9a35a9..e5b35cdd4 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -101,7 +101,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity, meta ) - when objtype in ~w[Question Answer Audio Video Event Article] do + when objtype in ~w[Question Answer Audio Video Event Article Note] do with {:ok, object_data} <- cast_and_apply(object), meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, create_activity} <- @@ -114,7 +114,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end def validate(%{"type" => type} = object, meta) - when type in ~w[Event Question Audio Video Article] do + when type in ~w[Event Question Audio Video Article Note] do validator = case type do "Event" -> EventValidator @@ -122,6 +122,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do "Audio" -> AudioVideoValidator "Video" -> AudioVideoValidator "Article" -> ArticleNoteValidator + "Note" -> ArticleNoteValidator end with {:ok, object} <- @@ -183,7 +184,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do EventValidator.cast_and_apply(object) end - def cast_and_apply(%{"type" => "Article"} = object) do + def cast_and_apply(%{"type" => type} = object) when type in ~w[Article Note] do ArticleNoteValidator.cast_and_apply(object) end diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex index d2026b5ea..193f85f49 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex @@ -50,6 +50,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do field(:likes, {:array, ObjectValidators.ObjectID}, default: []) field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) + + field(:replies, {:array, ObjectValidators.ObjectID}, default: []) end def cast_and_apply(data) do @@ -65,24 +67,39 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do end def cast_data(data) do - data = fix(data) - %__MODULE__{} |> changeset(data) end - defp fix_url(%{"url" => url} = data) when is_map(url) do - Map.put(data, "url", url["href"]) - end - + defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data + defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"]) defp fix_url(data), do: data + defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data + defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag]) + defp fix_tag(data), do: Map.drop(data, ["tag"]) + + defp fix_replies(%{"replies" => %{"first" => %{"items" => replies}}} = data) + when is_list(replies), + do: Map.put(data, "replies", replies) + + defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies), + do: Map.put(data, "replies", replies) + + defp fix_replies(%{"replies" => replies} = data) when is_bitstring(replies), + do: Map.drop(data, ["replies"]) + + defp fix_replies(data), do: data + defp fix(data) do data |> CommonFixes.fix_actor() |> CommonFixes.fix_object_defaults() |> fix_url() + |> fix_tag() + |> fix_replies() |> Transmogrifier.fix_emoji() + |> Transmogrifier.fix_content_map() end def changeset(struct, data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 950eb1494..7309f6af2 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -26,14 +26,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do |> Transmogrifier.fix_implicit_addressing(follower_collection) end - def fix_activity_defaults(data, meta) do + defp fix_activity_recipients(activity, field, object) do + {:ok, data} = ObjectValidators.Recipients.cast(activity[field] || object[field]) + + Map.put(activity, field, data) + end + + def fix_activity_defaults(activity, meta) do object = meta[:object_data] || %{} - data - |> Map.put_new("to", object["to"] || []) - |> Map.put_new("cc", object["cc"] || []) - |> Map.put_new("bto", object["bto"] || []) - |> Map.put_new("bcc", object["bcc"] || []) + activity + |> fix_activity_recipients("to", object) + |> fix_activity_recipients("cc", object) + |> fix_activity_recipients("bto", object) + |> fix_activity_recipients("bcc", object) end def fix_actor(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index 093549a45..85ac07044 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do fields |> Enum.map(fn field -> get_field(cng, field) end) |> Enum.any?(fn + nil -> false [] -> false _ -> true end) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex deleted file mode 100644 index a85a0298c..000000000 --- a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex +++ /dev/null @@ -1,29 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do - use Ecto.Schema - - alias Pleroma.EctoType.ActivityPub.ObjectValidators - alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator - - import Ecto.Changeset - - @primary_key false - - embedded_schema do - field(:id, ObjectValidators.ObjectID, primary_key: true) - field(:actor, ObjectValidators.ObjectID) - field(:type, :string) - field(:to, ObjectValidators.Recipients, default: []) - field(:cc, ObjectValidators.Recipients, default: []) - field(:bto, ObjectValidators.Recipients, default: []) - field(:bcc, ObjectValidators.Recipients, default: []) - embeds_one(:object, NoteValidator) - end - - def cast_data(data) do - cast(%__MODULE__{}, data, __schema__(:fields)) - end -end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 0b9a9f0c5..3234b9e43 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -203,6 +203,19 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Object.increase_replies_count(in_reply_to) end + reply_depth = (meta[:depth] || 0) + 1 + + # FIXME: Force inReplyTo to replies + if Pleroma.Web.Federator.allowed_thread_distance?(reply_depth) and + object.data["replies"] != nil do + for reply_id <- object.data["replies"] do + Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{ + "id" => reply_id, + "depth" => reply_depth + }) + end + end + ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn -> Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) @@ -366,7 +379,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end def handle_object_creation(%{"type" => objtype} = object, meta) - when objtype in ~w[Audio Video Question Event Article] do + when objtype in ~w[Audio Video Question Event Article Note] do with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do {:ok, object, meta} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 047f23918..28bc25363 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -404,10 +404,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # - tags # - emoji def handle_incoming( - %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, + %{"type" => "Create", "object" => %{"type" => "Page"} = object} = data, options - ) - when objtype in ~w{Note Page} do + ) do actor = Containment.get_actor(data) with nil <- Activity.get_create_by_object_ap_id(object["id"]), @@ -499,14 +498,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming( %{"type" => "Create", "object" => %{"type" => objtype, "id" => obj_id}} = data, - _options + options ) - when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do + when objtype in ~w{Question Answer ChatMessage Audio Video Event Article Note} do data = Map.put(data, "object", strip_internal_fields(data["object"])) + options = Keyword.put(options, :local, false) with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), nil <- Activity.get_create_by_object_ap_id(obj_id), - {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do + {:ok, activity, _} <- Pipeline.common_pipeline(data, options) do {:ok, activity} else %Activity{} = activity -> {:ok, activity} diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index f5ef76d32..69cfc2d52 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -96,6 +96,11 @@ defmodule Pleroma.Web.Federator do Logger.debug("Unhandled actor #{actor}, #{inspect(e)}") {:error, e} + {:error, {:validate_object, _}} = e -> + Logger.error("Incoming AP doc validation error: #{inspect(e)}") + Logger.debug(Jason.encode!(params, pretty: true)) + e + e -> # Just drop those for now Logger.debug(fn -> "Unhandled activity\n" <> Jason.encode!(params, pretty: true) end) -- cgit v1.2.3 From 641184fc7aff694e4e7e802b9204a1d313c0877c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 10 Sep 2020 19:45:42 +0200 Subject: recipients fixes/hardening for CreateGenericValidator --- .../activity_pub/object_validators/recipients.ex | 27 ++++++---- .../activity_pub/object_validators/common_fixes.ex | 34 ++++++------ .../object_validators/create_generic_validator.ex | 60 ++++++++++++++++------ 3 files changed, 77 insertions(+), 44 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex index a03471462..06fed8fb3 100644 --- a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex @@ -15,22 +15,27 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do def cast(object) when is_map(object) do case ObjectID.cast(object) do - {:ok, data} -> {:ok, data} + {:ok, data} -> {:ok, [data]} _ -> :error end end def cast(data) when is_list(data) do - data - |> Enum.reduce_while({:ok, []}, fn element, {:ok, list} -> - case ObjectID.cast(element) do - {:ok, id} -> - {:cont, {:ok, [id | list]}} - - _ -> - {:cont, {:ok, list}} - end - end) + data = + data + |> Enum.reduce_while([], fn element, list -> + case ObjectID.cast(element) do + {:ok, id} -> + {:cont, [id | list]} + + _ -> + {:cont, list} + end + end) + |> Enum.sort() + |> Enum.uniq() + + {:ok, data} end def cast(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 7309f6af2..009cd51b0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -9,37 +9,39 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils + def cast_recipients(message, field, field_fallback \\ []) do + {:ok, data} = ObjectValidators.Recipients.cast(message[field] || field_fallback) + + Map.put(message, field, data) + end + def fix_object_defaults(data) do %{data: %{"id" => context}, id: context_id} = Utils.create_context(data["context"] || data["conversation"]) %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"]) - {:ok, to} = ObjectValidators.Recipients.cast(data["to"] || []) - {:ok, cc} = ObjectValidators.Recipients.cast(data["cc"] || []) data |> Map.put("context", context) |> Map.put("context_id", context_id) - |> Map.put("to", to) - |> Map.put("cc", cc) + |> cast_recipients("to") + |> cast_recipients("cc") + |> cast_recipients("bto") + |> cast_recipients("bcc") |> Transmogrifier.fix_explicit_addressing(follower_collection) |> Transmogrifier.fix_implicit_addressing(follower_collection) end - defp fix_activity_recipients(activity, field, object) do - {:ok, data} = ObjectValidators.Recipients.cast(activity[field] || object[field]) - - Map.put(activity, field, data) - end - - def fix_activity_defaults(activity, meta) do - object = meta[:object_data] || %{} + def fix_activity_addressing(activity, _meta) do + %User{follower_address: follower_collection} = User.get_cached_by_ap_id(activity["actor"]) activity - |> fix_activity_recipients("to", object) - |> fix_activity_recipients("cc", object) - |> fix_activity_recipients("bto", object) - |> fix_activity_recipients("bcc", object) + |> cast_recipients("to") + |> cast_recipients("cc") + |> cast_recipients("bto") + |> cast_recipients("bcc") + |> Transmogrifier.fix_explicit_addressing(follower_collection) + |> Transmogrifier.fix_implicit_addressing(follower_collection) end def fix_actor(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index 99e8dc6c7..51d43e8d0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -10,8 +10,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object + alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -23,6 +25,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do field(:type, :string) field(:to, ObjectValidators.Recipients, default: []) field(:cc, ObjectValidators.Recipients, default: []) + field(:bto, ObjectValidators.Recipients, default: []) + field(:bcc, ObjectValidators.Recipients, default: []) field(:object, ObjectValidators.ObjectID) field(:expires_at, ObjectValidators.DateTime) @@ -54,29 +58,38 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do |> cast(data, __schema__(:fields)) end - defp fix_context(data, meta) do - if object = meta[:object_data] do - Map.put_new(data, "context", object["context"]) - else - data - end + # CommonFixes.fix_activity_addressing adapted for Create specific behavior + defp fix_addressing(data, object) do + %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["actor"]) + + data + |> CommonFixes.cast_recipients("to", object["to"]) + |> CommonFixes.cast_recipients("cc", object["cc"]) + |> CommonFixes.cast_recipients("bto", object["bto"]) + |> CommonFixes.cast_recipients("bcc", object["bcc"]) + |> Transmogrifier.fix_explicit_addressing(follower_collection) + |> Transmogrifier.fix_implicit_addressing(follower_collection) end - defp fix(data, meta) do + def fix(data, meta) do + object = meta[:object_data] + data - |> fix_context(meta) |> CommonFixes.fix_actor() - |> CommonFixes.fix_activity_defaults(meta) + |> Map.put_new("context", object["context"]) + |> fix_addressing(object) end defp validate_data(cng, meta) do + object = meta[:object_data] + cng - |> validate_required([:actor, :type, :object]) + |> validate_required([:actor, :type, :object, :to, :cc]) |> validate_inclusion(:type, ["Create"]) |> CommonValidations.validate_actor_presence() - |> CommonValidations.validate_any_presence([:to, :cc]) - |> validate_actors_match(meta) - |> validate_context_match(meta) + |> validate_actors_match(object) + |> validate_context_match(object) + |> validate_addressing_match(object) |> validate_object_nonexistence() |> validate_object_containment() end @@ -108,8 +121,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do end) end - def validate_actors_match(cng, meta) do - attributed_to = meta[:object_data]["attributedTo"] || meta[:object_data]["actor"] + def validate_actors_match(cng, object) do + attributed_to = object["attributedTo"] || object["actor"] cng |> validate_change(:actor, fn :actor, actor -> @@ -121,7 +134,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do end) end - def validate_context_match(cng, %{object_data: %{"context" => object_context}}) do + def validate_context_match(cng, %{"context" => object_context}) do cng |> validate_change(:context, fn :context, context -> if context == object_context do @@ -132,5 +145,18 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do end) end - def validate_context_match(cng, _), do: cng + def validate_addressing_match(cng, object) do + [:to, :cc, :bcc, :bto] + |> Enum.reduce(cng, fn field, cng -> + object_data = object[to_string(field)] + + validate_change(cng, field, fn field, data -> + if data == object_data do + [] + else + [{field, "field doesn't match with object (#{inspect(object_data)})"}] + end + end) + end) + end end -- cgit v1.2.3 From 96212b2e32e2542964c665f091158fb1ff1d987d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 15 Sep 2020 17:22:08 +0200 Subject: Fix addressing --- lib/pleroma/object/fetcher.ex | 7 ++++-- .../activity_pub/object_validators/common_fixes.ex | 25 ++++++++++++---------- .../object_validators/create_generic_validator.ex | 9 ++++---- 3 files changed, 23 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index bcccf1c4c..82d2c8bcb 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.HTTP + alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Object.Containment alias Pleroma.Repo @@ -124,12 +125,14 @@ defmodule Pleroma.Object.Fetcher do defp prepare_activity_params(data) do %{ "type" => "Create", - "to" => data["to"] || [], - "cc" => data["cc"] || [], # Should we seriously keep this attributedTo thing? "actor" => data["actor"] || data["attributedTo"], "object" => data } + |> Maps.put_if_present("to", data["to"]) + |> Maps.put_if_present("cc", data["cc"]) + |> Maps.put_if_present("bto", data["bto"]) + |> Maps.put_if_present("bcc", data["bcc"]) end def fetch_object_from_id!(id, options \\ []) do diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 009cd51b0..c958fcc5d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -9,9 +9,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils - def cast_recipients(message, field, field_fallback \\ []) do + def cast_and_filter_recipients(message, field, follower_collection, field_fallback \\ []) do {:ok, data} = ObjectValidators.Recipients.cast(message[field] || field_fallback) + data = + Enum.reject(data, fn x -> + String.ends_with?(x, "/followers") and x != follower_collection + end) + Map.put(message, field, data) end @@ -24,11 +29,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do data |> Map.put("context", context) |> Map.put("context_id", context_id) - |> cast_recipients("to") - |> cast_recipients("cc") - |> cast_recipients("bto") - |> cast_recipients("bcc") - |> Transmogrifier.fix_explicit_addressing(follower_collection) + |> cast_and_filter_recipients("to", follower_collection) + |> cast_and_filter_recipients("cc", follower_collection) + |> cast_and_filter_recipients("bto", follower_collection) + |> cast_and_filter_recipients("bcc", follower_collection) |> Transmogrifier.fix_implicit_addressing(follower_collection) end @@ -36,11 +40,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do %User{follower_address: follower_collection} = User.get_cached_by_ap_id(activity["actor"]) activity - |> cast_recipients("to") - |> cast_recipients("cc") - |> cast_recipients("bto") - |> cast_recipients("bcc") - |> Transmogrifier.fix_explicit_addressing(follower_collection) + |> cast_and_filter_recipients("to", follower_collection) + |> cast_and_filter_recipients("cc", follower_collection) + |> cast_and_filter_recipients("bto", follower_collection) + |> cast_and_filter_recipients("bcc", follower_collection) |> Transmogrifier.fix_implicit_addressing(follower_collection) end diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index 51d43e8d0..d2de53049 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -63,11 +63,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["actor"]) data - |> CommonFixes.cast_recipients("to", object["to"]) - |> CommonFixes.cast_recipients("cc", object["cc"]) - |> CommonFixes.cast_recipients("bto", object["bto"]) - |> CommonFixes.cast_recipients("bcc", object["bcc"]) - |> Transmogrifier.fix_explicit_addressing(follower_collection) + |> CommonFixes.cast_and_filter_recipients("to", follower_collection, object["to"]) + |> CommonFixes.cast_and_filter_recipients("cc", follower_collection, object["cc"]) + |> CommonFixes.cast_and_filter_recipients("bto", follower_collection, object["bto"]) + |> CommonFixes.cast_and_filter_recipients("bcc", follower_collection, object["bcc"]) |> Transmogrifier.fix_implicit_addressing(follower_collection) end -- cgit v1.2.3 From d1205406d9237c72d10df937dd8d2d4da2786cc5 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 15 Sep 2020 18:18:57 +0200 Subject: ActivityPubControllerTest: Apply same addr changes to object --- lib/pleroma/web/activity_pub/utils.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index a4dc469dc..e81623d83 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -97,7 +97,10 @@ defmodule Pleroma.Web.ActivityPub.Utils do if need_splice? do cc_list = extract_list(params["cc"]) - Map.put(params, "cc", [ap_id | cc_list]) + + params + |> Map.put("cc", [ap_id | cc_list]) + |> Kernel.put_in(["object", "cc"], [ap_id | cc_list]) else params end -- cgit v1.2.3 From 461123110b7cf47f4d2c01d1dd6992a2b63337fe Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 17 Sep 2020 16:17:16 +0200 Subject: Object.Fetcher: Fix getting transmogrifier reject reason --- lib/pleroma/object/fetcher.ex | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 82d2c8bcb..4ca67f0fd 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -102,6 +102,9 @@ defmodule Pleroma.Object.Fetcher do {:transmogrifier, {:error, {:reject, e}}} -> {:reject, e} + {:transmogrifier, {:reject, e}} -> + {:reject, e} + {:transmogrifier, _} = e -> {:error, e} -- cgit v1.2.3 From 6c9f6e62c8453f023c6ec9106d1a7c3e66ab95b7 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 28 Sep 2020 19:34:27 +0200 Subject: transmogrifier: Fixing votes from Note to Answer --- .../object_validators/answer_validator.ex | 7 +++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 22 ++++++++++++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex index c9bd9e42d..3451e1ff8 100644 --- a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations import Ecto.Changeset @@ -23,6 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do field(:name, :string) field(:inReplyTo, ObjectValidators.ObjectID) field(:attributedTo, ObjectValidators.ObjectID) + field(:context, :string) # TODO: Remove actor on objects field(:actor, ObjectValidators.ObjectID) @@ -46,6 +48,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do end def changeset(struct, data) do + data = + data + |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() + struct |> cast(data, __schema__(:fields)) end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 28bc25363..454bbce9d 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -43,7 +43,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> fix_content_map() |> fix_addressing() |> fix_summary() - |> fix_type(options) end def fix_summary(%{"summary" => nil} = object) do @@ -321,19 +320,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_content_map(object), do: object - def fix_type(object, options \\ []) + defp fix_type(%{"type" => "Note", "inReplyTo" => reply_id, "name" => _} = object, options) + when is_binary(reply_id) do + options = Keyword.put(options, :fetch, true) - def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options) - when is_binary(reply_id) do - with true <- Federator.allowed_thread_distance?(options[:depth]), - {:ok, %{data: %{"type" => "Question"} = _} = _} <- get_obj_helper(reply_id, options) do + with %Object{data: %{"type" => "Question"}} <- Object.normalize(reply_id, options) do Map.put(object, "type", "Answer") else _ -> object end end - def fix_type(object, _), do: object + defp fix_type(object, _options), do: object # Reduce the object list to find the reported user. defp get_reported(objects) do @@ -501,7 +499,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do options ) when objtype in ~w{Question Answer ChatMessage Audio Video Event Article Note} do - data = Map.put(data, "object", strip_internal_fields(data["object"])) + fetch_options = Keyword.put(options, :depth, (options[:depth] || 0) + 1) + + object = + data["object"] + |> strip_internal_fields() + |> fix_type(fetch_options) + |> fix_in_reply_to(fetch_options) + + data = Map.put(data, "object", object) options = Keyword.put(options, :local, false) with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), -- cgit v1.2.3 From 53193b84b1d07c9fd3c6b80c04e3eada4fb4cd59 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 27 Nov 2020 00:25:24 +0100 Subject: =?UTF-8?q?utils:=20Fix=20maybe=5Fsplice=5Frecipient=20when=20"obj?= =?UTF-8?q?ect"=20isn=E2=80=99t=20a=20map?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/pleroma/maps.ex | 6 ++++++ lib/pleroma/web/activity_pub/utils.ex | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/maps.ex b/lib/pleroma/maps.ex index 0d2e94248..b08b83305 100644 --- a/lib/pleroma/maps.ex +++ b/lib/pleroma/maps.ex @@ -12,4 +12,10 @@ defmodule Pleroma.Maps do _ -> map end end + + def safe_put_in(data, keys, value) when is_map(data) and is_list(keys) do + Kernel.put_in(data, keys, value) + rescue + _ -> data + end end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index e81623d83..0d1a6d0f1 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -96,11 +96,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do !label_in_collection?(ap_id, params["cc"]) if need_splice? do - cc_list = extract_list(params["cc"]) + cc = [ap_id | extract_list(params["cc"])] params - |> Map.put("cc", [ap_id | cc_list]) - |> Kernel.put_in(["object", "cc"], [ap_id | cc_list]) + |> Map.put("cc", cc) + |> Maps.safe_put_in(["object", "cc"], cc) else params end -- cgit v1.2.3 From 681a42c359b4fbae74285363c670dff18aac5918 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 8 Apr 2021 15:45:31 +0300 Subject: release runtime provider fix for paths --- lib/pleroma/config/release_runtime_provider.ex | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config/release_runtime_provider.ex b/lib/pleroma/config/release_runtime_provider.ex index 46fa35559..e5e9d3dcd 100644 --- a/lib/pleroma/config/release_runtime_provider.ex +++ b/lib/pleroma/config/release_runtime_provider.ex @@ -11,10 +11,11 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do def load(config, opts) do with_defaults = Config.Reader.merge(config, Pleroma.Config.Holder.release_defaults()) - config_path = opts[:config_path] + config_path = + opts[:config_path] || System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" with_runtime_config = - if config_path && File.exists?(config_path) do + if File.exists?(config_path) do runtime_config = Config.Reader.read!(config_path) with_defaults @@ -32,10 +33,14 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do with_defaults end - exported_config_path = opts[:exported_config_path] + exported_config_path = + opts[:exported_config_path] || + config_path + |> Path.dirname() + |> Path.join("#{Pleroma.Config.get(:env)}.exported_from_db.secret.exs") with_exported = - if exported_config_path && File.exists?(exported_config_path) do + if File.exists?(exported_config_path) do exported_config = Config.Reader.read!(exported_config_path) Config.Reader.merge(with_runtime_config, exported_config) else -- cgit v1.2.3 From 0feafcc20cec168258f592b9d509c1e6ccc8efba Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 9 Apr 2021 10:30:27 -0500 Subject: Use URI.merge to prevent concatenating two canonical URLs when a custom instance thumbnail was uploaded via AdminFE --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 73205fb6d..dac68d8e6 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -23,7 +23,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do streaming_api: Pleroma.Web.Endpoint.websocket_url() }, stats: Pleroma.Stats.get_stats(), - thumbnail: Pleroma.Web.base_url() <> Keyword.get(instance, :instance_thumbnail), + thumbnail: + URI.merge(Pleroma.Web.base_url(), Keyword.get(instance, :instance_thumbnail)) |> to_string, languages: ["en"], registrations: Keyword.get(instance, :registrations_open), approval_required: Keyword.get(instance, :account_approval_required), -- cgit v1.2.3 From ee53ad4d7705328a5a583680c6f551c4c3bf2302 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 13 Apr 2021 12:09:18 -0500 Subject: Add ConcurrentLimiter to module_name?/1 and apply string_to_elixir_types/1 to search_opts keys during update_or_create/1 --- lib/pleroma/config_db.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config_db.ex b/lib/pleroma/config_db.ex index b874e0e37..03905c06b 100644 --- a/lib/pleroma/config_db.ex +++ b/lib/pleroma/config_db.ex @@ -141,7 +141,9 @@ defmodule Pleroma.ConfigDB do @spec update_or_create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} def update_or_create(params) do params = Map.put(params, :value, to_elixir_types(params[:value])) - search_opts = Map.take(params, [:group, :key]) + + search_opts = + Map.take(params, [:group, :key]) |> Map.update!(:key, &string_to_elixir_types(&1)) with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts), {_, true, config} <- {:partial_update, can_be_partially_updated?(config), config}, @@ -387,6 +389,6 @@ defmodule Pleroma.ConfigDB do @spec module_name?(String.t()) :: boolean() def module_name?(string) do Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or - string in ["Oban", "Ueberauth", "ExSyslogger"] + string in ["Oban", "Ueberauth", "ExSyslogger", "ConcurrentLimiter"] end end -- cgit v1.2.3 From c3b8c77967b0c42f93286f864236b7d6f1471c13 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 13 Apr 2021 14:25:15 -0500 Subject: Improve string_to_elixir_types/1 with guards --- lib/pleroma/config_db.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config_db.ex b/lib/pleroma/config_db.ex index 03905c06b..eeeb026c1 100644 --- a/lib/pleroma/config_db.ex +++ b/lib/pleroma/config_db.ex @@ -327,7 +327,7 @@ defmodule Pleroma.ConfigDB do @spec string_to_elixir_types(String.t()) :: atom() | Regex.t() | module() | String.t() | no_return() - def string_to_elixir_types("~r" <> _pattern = regex) do + def string_to_elixir_types("~r" <> _pattern = regex) when is_binary(regex) do pattern = ~r/^~r(?'delimiter'[\/|"'([{<]{1})(?'pattern'.+)[\/|"')\]}>]{1}(?'modifier'[uismxfU]*)/u @@ -341,9 +341,9 @@ defmodule Pleroma.ConfigDB do end end - def string_to_elixir_types(":" <> atom), do: String.to_atom(atom) + def string_to_elixir_types(":" <> atom) when is_binary(atom), do: String.to_atom(atom) - def string_to_elixir_types(value) do + def string_to_elixir_types(value) when is_binary(value) do if module_name?(value) do String.to_existing_atom("Elixir." <> value) else @@ -351,6 +351,8 @@ defmodule Pleroma.ConfigDB do end end + def string_to_elixir_types(value) when is_atom(value), do: value + defp parse_host("localhost"), do: :localhost defp parse_host(host) do -- cgit v1.2.3 From f95b52255b2d7373a3e0bf4adff81f83c080b2ef Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 14 Apr 2021 09:39:57 -0500 Subject: Revert guards on string_to_elixir_types/1, remove unnecessary assignment in test --- lib/pleroma/config_db.ex | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config_db.ex b/lib/pleroma/config_db.ex index eeeb026c1..cb57673e3 100644 --- a/lib/pleroma/config_db.ex +++ b/lib/pleroma/config_db.ex @@ -141,9 +141,7 @@ defmodule Pleroma.ConfigDB do @spec update_or_create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} def update_or_create(params) do params = Map.put(params, :value, to_elixir_types(params[:value])) - - search_opts = - Map.take(params, [:group, :key]) |> Map.update!(:key, &string_to_elixir_types(&1)) + search_opts = Map.take(params, [:group, :key]) with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts), {_, true, config} <- {:partial_update, can_be_partially_updated?(config), config}, @@ -327,7 +325,7 @@ defmodule Pleroma.ConfigDB do @spec string_to_elixir_types(String.t()) :: atom() | Regex.t() | module() | String.t() | no_return() - def string_to_elixir_types("~r" <> _pattern = regex) when is_binary(regex) do + def string_to_elixir_types("~r" <> _pattern = regex) do pattern = ~r/^~r(?'delimiter'[\/|"'([{<]{1})(?'pattern'.+)[\/|"')\]}>]{1}(?'modifier'[uismxfU]*)/u @@ -341,9 +339,9 @@ defmodule Pleroma.ConfigDB do end end - def string_to_elixir_types(":" <> atom) when is_binary(atom), do: String.to_atom(atom) + def string_to_elixir_types(":" <> atom), do: String.to_atom(atom) - def string_to_elixir_types(value) when is_binary(value) do + def string_to_elixir_types(value) do if module_name?(value) do String.to_existing_atom("Elixir." <> value) else @@ -351,8 +349,6 @@ defmodule Pleroma.ConfigDB do end end - def string_to_elixir_types(value) when is_atom(value), do: value - defp parse_host("localhost"), do: :localhost defp parse_host(host) do -- cgit v1.2.3 From c6dcd863e28531e0d21ee64a8387bd27c2c0ed31 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 16 Apr 2021 09:59:50 +0000 Subject: Apply rinpatch's suggestion(s) to 1 file(s) --- lib/pleroma/web/api_spec/operations/twitter_util_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index 62c9826f6..decb6572a 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -59,7 +59,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do def change_password_operation do %Operation{ - tags: ["Accounts"], + tags: ["Account credentials"], summary: "Change account password", security: [%{"oAuth" => ["write:accounts"]}], operationId: "UtilController.change_password", -- cgit v1.2.3 From 2b4f958b2ad653ee8e294ade18aa4482e4d372e1 Mon Sep 17 00:00:00 2001 From: Sean King Date: Sun, 18 Apr 2021 14:00:18 -0600 Subject: Add opting out of Google FLoC to HTTPSecurityPlug headers --- lib/pleroma/web/plugs/http_security_plug.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index 0025b042a..d1e6cc9d3 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -48,7 +48,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do {"x-content-type-options", "nosniff"}, {"referrer-policy", referrer_policy}, {"x-download-options", "noopen"}, - {"content-security-policy", csp_string()} + {"content-security-policy", csp_string()}, + {"permissions-policy", "interest-cohort=()"} ] headers = -- cgit v1.2.3 From efed94a23e30260bcf1b297910906b11d6e4d895 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 19 Apr 2021 16:23:57 -0500 Subject: Fix error response which was breaking tests related to pinned posts --- lib/pleroma/web/common_api.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 3970c19a8..1b5f8491e 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -415,7 +415,7 @@ defmodule Pleroma.Web.CommonAPI do ) do {:ok, activity} else - {:error, {:execute_side_effects, error}} -> error + {:error, {:side_effects, error}} -> error error -> error end end -- cgit v1.2.3 From 30b1d5093808974310a52917e6ab85d528683fae Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Tue, 20 Apr 2021 21:06:32 +0000 Subject: Apply lanodan's suggestion(s) to 1 file(s) --- lib/pleroma/web/api_spec/operations/twitter_util_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index decb6572a..6ddc93a92 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -88,7 +88,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do def change_email_operation do %Operation{ - tags: ["Accounts"], + tags: ["Account credentials"], summary: "Change account email", security: [%{"oAuth" => ["write:accounts"]}], operationId: "UtilController.change_email", -- cgit v1.2.3 From e104829c2f5b3eae9133ea1a6a81d138c3a8e314 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Tue, 20 Apr 2021 21:06:39 +0000 Subject: Apply lanodan's suggestion(s) to 1 file(s) --- lib/pleroma/web/api_spec/operations/twitter_util_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index 6ddc93a92..dbed1b518 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -143,7 +143,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do def disable_account_operation do %Operation{ - tags: ["Accounts"], + tags: ["Account credentials"], summary: "Disable Account", security: [%{"oAuth" => ["write:accounts"]}], operationId: "UtilController.disable_account", -- cgit v1.2.3 From 42185d87504ea595138e8e3f5bf9ce6840edd2f1 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Tue, 20 Apr 2021 21:06:45 +0000 Subject: Apply lanodan's suggestion(s) to 1 file(s) --- lib/pleroma/web/api_spec/operations/twitter_util_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index dbed1b518..6e6d330de 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -163,7 +163,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do def delete_account_operation do %Operation{ - tags: ["Accounts"], + tags: ["Account credentials"], summary: "Delete Account", security: [%{"oAuth" => ["write:accounts"]}], operationId: "UtilController.delete_account", -- cgit v1.2.3 From f9bedf5597dd00ce4f429a4077e7bb4473c97410 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Tue, 20 Apr 2021 21:08:31 +0000 Subject: Apply lanodan's suggestion(s) to 1 file(s) --- lib/pleroma/web/api_spec/operations/twitter_util_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index 6e6d330de..0cafbc719 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -195,7 +195,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do def healthcheck_operation do %Operation{ tags: ["Accounts"], - summary: "Disable Account", + summary: "Quick status check on the instance", security: [%{"oAuth" => ["write:accounts"]}], operationId: "UtilController.healthcheck", parameters: [], -- cgit v1.2.3 From 0effcd2cfed36baec1d960b64c901da7e56710a8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 19 Apr 2021 15:43:17 -0500 Subject: Set Repo.transaction/2 timeout to infinity. Fixes pleroma.user delete_activities mix task. --- lib/pleroma/web/activity_pub/pipeline.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 377eccb92..400823094 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do @spec common_pipeline(map(), keyword()) :: {:ok, Activity.t() | Object.t(), keyword()} | {:error, any()} def common_pipeline(object, meta) do - case Repo.transaction(fn -> do_common_pipeline(object, meta) end) do + case Repo.transaction(fn -> do_common_pipeline(object, meta) end, timeout: :infinity) do {:ok, {:ok, activity, meta}} -> @side_effects.handle_after_transaction(meta) {:ok, activity, meta} -- cgit v1.2.3 From 9bc69196d5dfbd3fb37c0e62da19ce08fb9bf28d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 20 Apr 2021 11:10:39 -0500 Subject: Add utility function to return infinite timeout for SQL transactions if we detect it was called from a Mix Task --- lib/pleroma/utils.ex | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/utils.ex b/lib/pleroma/utils.ex index bc0c95332..5e2fa8bf7 100644 --- a/lib/pleroma/utils.ex +++ b/lib/pleroma/utils.ex @@ -63,4 +63,13 @@ defmodule Pleroma.Utils do end def posix_error_message(_), do: "" + + def query_timeout do + {parent, _, _, _} = Process.info(self(), :current_stacktrace) |> elem(1) |> Enum.fetch!(2) + + cond do + parent |> to_string |> String.starts_with?("Elixir.Mix.Task") -> [timeout: :infinity] + true -> [timeout: 15_000] + end + end end -- cgit v1.2.3 From 9f711ddcf84bdb5a5680e1b55afa83768014906d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 20 Apr 2021 11:16:24 -0500 Subject: Try to set query timeout intelligently --- lib/pleroma/web/activity_pub/pipeline.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 400823094..a0f2e0312 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do alias Pleroma.Config alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Utils alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.ObjectValidator @@ -24,7 +25,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do @spec common_pipeline(map(), keyword()) :: {:ok, Activity.t() | Object.t(), keyword()} | {:error, any()} def common_pipeline(object, meta) do - case Repo.transaction(fn -> do_common_pipeline(object, meta) end, timeout: :infinity) do + case Repo.transaction(fn -> do_common_pipeline(object, meta) end, Utils.query_timeout()) do {:ok, {:ok, activity, meta}} -> @side_effects.handle_after_transaction(meta) {:ok, activity, meta} -- cgit v1.2.3 From 99fd9c5e38ad08973f435f1a67d6af60d004c578 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 20 Apr 2021 12:00:02 -0500 Subject: OTP releases executing commands via pleroma_ctl show the parent of the process is :erl_eval --- lib/pleroma/utils.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/utils.ex b/lib/pleroma/utils.ex index 5e2fa8bf7..55aecc509 100644 --- a/lib/pleroma/utils.ex +++ b/lib/pleroma/utils.ex @@ -69,6 +69,7 @@ defmodule Pleroma.Utils do cond do parent |> to_string |> String.starts_with?("Elixir.Mix.Task") -> [timeout: :infinity] + parent == :erl_eval -> [timeout: :infinity] true -> [timeout: 15_000] end end -- cgit v1.2.3 From 959dc6e6fc95b33700fb7e08689afb701b2877f2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 22 Apr 2021 10:11:08 -0500 Subject: Cleanup and ensure we obey custom Repo timeout --- lib/pleroma/utils.ex | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/utils.ex b/lib/pleroma/utils.ex index 55aecc509..a446d3ae6 100644 --- a/lib/pleroma/utils.ex +++ b/lib/pleroma/utils.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Utils do eperm epipe erange erofs espipe esrch estale etxtbsy exdev )a + @repo_timeout Pleroma.Config.get([Pleroma.Repo, :timeout], 15_000) + def compile_dir(dir) when is_binary(dir) do dir |> File.ls!() @@ -64,13 +66,20 @@ defmodule Pleroma.Utils do def posix_error_message(_), do: "" + @doc """ + Returns [timeout: integer] suitable for passing as an option to Repo functions. + + This function detects if the execution was triggered from IEx shell, Mix task, or + ./bin/pleroma_ctl and sets the timeout to :infinity, else returns the default timeout value. + """ + @spec query_timeout() :: [timeout: integer] def query_timeout do {parent, _, _, _} = Process.info(self(), :current_stacktrace) |> elem(1) |> Enum.fetch!(2) cond do parent |> to_string |> String.starts_with?("Elixir.Mix.Task") -> [timeout: :infinity] parent == :erl_eval -> [timeout: :infinity] - true -> [timeout: 15_000] + true -> [timeout: @repo_timeout] end end end -- cgit v1.2.3 From 6727a3659f60c0e09fa6375b6c0843c01f5be3dc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 30 Apr 2021 12:27:06 -0500 Subject: Remove Pleroma.Formatter.minify/2 --- lib/pleroma/formatter.ex | 11 ----------- .../activity_pub/object_validators/audio_video_validator.ex | 1 - lib/pleroma/web/common_api/utils.ex | 1 - 3 files changed, 13 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 2aa236ca9..baf652a5a 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -142,17 +142,6 @@ defmodule Pleroma.Formatter do |> Enum.join("") end - def minify({text, mentions, hashtags}, type) do - {minify(text, type), mentions, hashtags} - end - - def minify(text, "text/html") do - text - |> String.replace(">\n", ">") - |> String.replace("> ", ">") - |> String.replace(" <", "<") - end - def truncate(text, max_length \\ 200, omission \\ "...") do # Remove trailing whitespace text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}") diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index fa3e2c026..9b38aa4c2 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -96,7 +96,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do content = content |> Pleroma.Formatter.markdown_to_html() - |> Pleroma.Formatter.minify("text/html") |> Pleroma.HTML.filter_tags() Map.put(data, "content", content) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index be86009af..4731e79be 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -296,7 +296,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do |> Formatter.mentions_escape(options) |> Formatter.markdown_to_html() |> Formatter.linkify(options) - |> Formatter.minify("text/html") |> Formatter.html_escape("text/html") end -- cgit v1.2.3 From 3d742c3c1af69a9526c12a171663630b3439b5cc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 18 Mar 2021 15:31:50 -0500 Subject: SimplePolicy: filter nested objects --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index bb3838d2c..b3e5d814d 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -181,6 +181,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do defp check_banner_removal(_actor_info, object), do: {:ok, object} + defp check_object(%{"object" => object} = activity) when is_map(object) do + with {:ok, _object} <- filter(object) do + {:ok, activity} + end + end + + defp check_object(object), do: {:ok, object} + @impl true def filter(%{"type" => "Delete", "actor" => actor} = object) do %{host: actor_host} = URI.parse(actor) @@ -206,7 +214,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} <- check_media_nsfw(actor_info, object), {:ok, object} <- check_ftl_removal(actor_info, object), {:ok, object} <- check_followers_only(actor_info, object), - {:ok, object} <- check_report_removal(actor_info, object) do + {:ok, object} <- check_report_removal(actor_info, object), + {:ok, object} <- check_object(object) do {:ok, object} else {:reject, nil} -> {:reject, "[SimplePolicy]"} -- cgit v1.2.3 From c16c7fdb8794df8558cf8fbe4231d8f9ec01bb6d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 29 Apr 2021 11:51:49 -0500 Subject: SimplePolicy: filter string Objects --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index b3e5d814d..b07d70401 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -181,7 +181,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do defp check_banner_removal(_actor_info, object), do: {:ok, object} - defp check_object(%{"object" => object} = activity) when is_map(object) do + defp check_object(%{"object" => object} = activity) do with {:ok, _object} <- filter(object) do {:ok, activity} end @@ -240,6 +240,19 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do end end + def filter(object) when is_binary(object) do + uri = URI.parse(object) + + with {:ok, object} <- check_accept(uri, object), + {:ok, object} <- check_reject(uri, object) do + {:ok, object} + else + {:reject, nil} -> {:reject, "[SimplePolicy]"} + {:reject, _} = e -> e + _ -> {:reject, "[SimplePolicy]"} + end + end + def filter(object), do: {:ok, object} @impl true -- cgit v1.2.3 From c80b1aaf514dec6b538a9833d48df027708b6b4d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 3 May 2021 14:27:03 -0500 Subject: Don't crash when email settings are invalid Fixes: https://git.pleroma.social/pleroma/pleroma/-/issues/2606 Fixes: https://gitlab.com/soapbox-pub/soapbox/-/issues/4 --- lib/pleroma/application_requirements.ex | 38 ++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 6ef65b263..c412dec5e 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -34,15 +34,16 @@ defmodule Pleroma.ApplicationRequirements do defp check_welcome_message_config!(:ok) do if Pleroma.Config.get([:welcome, :email, :enabled], false) and not Pleroma.Emails.Mailer.enabled?() do - Logger.error(""" - To send welcome email do you need to enable mail. - \nconfig :pleroma, Pleroma.Emails.Mailer, enabled: true - """) + Logger.warn(""" + To send welcome emails, you need to enable the mailer. + Welcome emails will NOT be sent with the current config. - {:error, "The mail disabled."} - else - :ok + Enable the mailer: + config :pleroma, Pleroma.Emails.Mailer, enabled: true + """) end + + :ok end defp check_welcome_message_config!(result), do: result @@ -51,18 +52,21 @@ defmodule Pleroma.ApplicationRequirements do # def check_confirmation_accounts!(:ok) do if Pleroma.Config.get([:instance, :account_activation_required]) && - not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do - Logger.error( - "Account activation enabled, but no Mailer settings enabled.\n" <> - "Please set config :pleroma, :instance, account_activation_required: false\n" <> - "Otherwise setup and enable Mailer." - ) + not Pleroma.Emails.Mailer.enabled?() do + Logger.warn(""" + Account activation is required, but the mailer is disabled. + Users will NOT be able to confirm their accounts with this config. + Either disable account activation or enable the mailer. - {:error, - "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails."} - else - :ok + Disable account activation: + config :pleroma, :instance, account_activation_required: false + + Enable the mailer: + config :pleroma, Pleroma.Emails.Mailer, enabled: true + """) end + + :ok end def check_confirmation_accounts!(result), do: result -- cgit v1.2.3 From a0c9a2b4cc8c22d6238b0f31239c1e655f47730f Mon Sep 17 00:00:00 2001 From: faried nawaz Date: Wed, 21 Apr 2021 02:38:59 +0500 Subject: mix prune_objects: remove unused hashtags after pruning remote objects --- lib/mix/tasks/pleroma/database.ex | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index e7f4b67a4..53ad58b64 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -96,6 +96,17 @@ defmodule Mix.Tasks.Pleroma.Database do ) |> Repo.delete_all(timeout: :infinity) + prune_hashtags_query = """ + delete from hashtags + where id in ( + select id from hashtags as ht + left join hashtags_objects as hto + on hto.hashtag_id = ht.id + where hto.hashtag_id is null) + """ + + Repo.query(prune_hashtags_query) + if Keyword.get(options, :vacuum) do Maintenance.vacuum("full") end -- cgit v1.2.3 From 5be9d139816fa40ff6227950b58f3c6cea01fc81 Mon Sep 17 00:00:00 2001 From: faried nawaz Date: Wed, 21 Apr 2021 03:52:32 +0500 Subject: a better query to delete from hashtags old query: Delete on hashtags (cost=5089.81..5521.63 rows=6160 width=18) -> Hash Semi Join (cost=5089.81..5521.63 rows=6160 width=18) Hash Cond: (hashtags.id = ht.id) -> Seq Scan on hashtags (cost=0.00..317.28 rows=17528 width=14) -> Hash (cost=5012.81..5012.81 rows=6160 width=20) -> Merge Anti Join (cost=0.70..5012.81 rows=6160 width=20) Merge Cond: (ht.id = hto.hashtag_id) -> Index Scan using hashtags_pkey on hashtags ht (cost=0.29..610.53 rows=17528 width=14) -> Index Scan using hashtags_objects_pkey on hashtags_objects hto (cost=0.42..3506.48 rows=68158 width=14) new query: Delete on hashtags ht (cost=0.70..5012.81 rows=6160 width=12) -> Merge Anti Join (cost=0.70..5012.81 rows=6160 width=12) Merge Cond: (ht.id = hto.hashtag_id) -> Index Scan using hashtags_pkey on hashtags ht (cost=0.29..610.53 rows=17528 width=14) -> Index Scan using hashtags_objects_pkey on hashtags_objects hto (cost=0.42..3506.48 rows=68158 width=14) --- lib/mix/tasks/pleroma/database.ex | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 53ad58b64..bcde07774 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -97,12 +97,10 @@ defmodule Mix.Tasks.Pleroma.Database do |> Repo.delete_all(timeout: :infinity) prune_hashtags_query = """ - delete from hashtags - where id in ( - select id from hashtags as ht - left join hashtags_objects as hto - on hto.hashtag_id = ht.id - where hto.hashtag_id is null) + delete from hashtags as ht + where not exists ( + select 1 from hashtags_objects hto + where ht.id = hto.hashtag_id) """ Repo.query(prune_hashtags_query) -- cgit v1.2.3 From ab9eabdf20180f2dd8539cf5d3dc0fdc6412496b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 May 2021 13:38:11 -0500 Subject: Add SetMeta filter to store uploaded image sizes --- lib/pleroma/upload.ex | 9 +++++- lib/pleroma/upload/filter/set_meta.ex | 36 +++++++++++++++++++++++ lib/pleroma/web/mastodon_api/views/status_view.ex | 16 ++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/upload/filter/set_meta.ex (limited to 'lib') diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 654711351..4d58abd48 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -23,6 +23,8 @@ defmodule Pleroma.Upload do is once created permanent and changing it (especially in uploaders) is probably a bad idea! * `:tempfile` - path to the temporary file. Prefer in-place changes on the file rather than changing the path as the temporary file is also tracked by `Plug.Upload{}` and automatically deleted once the request is over. + * `:width` - width of the media in pixels + * `:height` - height of the media in pixels Related behaviors: @@ -32,6 +34,7 @@ defmodule Pleroma.Upload do """ alias Ecto.UUID alias Pleroma.Config + alias Pleroma.Maps require Logger @type source :: @@ -53,9 +56,11 @@ defmodule Pleroma.Upload do name: String.t(), tempfile: String.t(), content_type: String.t(), + width: integer(), + height: integer(), path: String.t() } - defstruct [:id, :name, :tempfile, :content_type, :path] + defstruct [:id, :name, :tempfile, :content_type, :width, :height, :path] defp get_description(opts, upload) do case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do @@ -89,6 +94,8 @@ defmodule Pleroma.Upload do "mediaType" => upload.content_type, "href" => url_from_spec(upload, opts.base_url, url_spec) } + |> Maps.put_if_present("width", upload.width) + |> Maps.put_if_present("height", upload.height) ], "name" => description }} diff --git a/lib/pleroma/upload/filter/set_meta.ex b/lib/pleroma/upload/filter/set_meta.ex new file mode 100644 index 000000000..cccb6c371 --- /dev/null +++ b/lib/pleroma/upload/filter/set_meta.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.SetMeta do + @moduledoc """ + Extracts metadata about the upload, such as width/height + """ + require Logger + + @behaviour Pleroma.Upload.Filter + + @spec filter(Pleroma.Upload.t()) :: + {:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()} + def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do + try do + image = + file + |> Mogrify.open() + |> Mogrify.verbose() + + upload = + upload + |> Map.put(:width, image.width) + |> Map.put(:height, image.height) + + {:ok, :filtered, upload} + rescue + e in ErlangError -> + Logger.warn("#{__MODULE__}: #{inspect(e)}") + {:ok, :noop} + end + end + + def filter(_), do: {:ok, :noop} +end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index bac897a57..5dbdc309e 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -426,10 +426,26 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do type: type, description: attachment["name"], pleroma: %{mime_type: media_type}, + meta: render("attachment_meta.json", %{attachment: attachment}), blurhash: attachment["blurhash"] } end + def render("attachment_meta.json", %{ + attachment: %{"url" => [%{"width" => width, "height" => height} | _]} + }) + when is_integer(width) and is_integer(height) do + %{ + original: %{ + width: width, + height: height, + aspect: width / height + } + } + end + + def render("attachment_meta.json", _), do: %{} + def render("context.json", %{activity: activity, activities: activities, user: user}) do %{ancestors: ancestors, descendants: descendants} = activities -- cgit v1.2.3 From 4c060ae73371a8567468186e5d1333ec00fbdf41 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 May 2021 15:38:49 -0500 Subject: Ingest remote attachment width/height --- .../web/activity_pub/object_validators/attachment_validator.ex | 4 +++- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index 3175427ad..a99b40adc 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -21,6 +21,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do field(:type, :string) field(:href, ObjectValidators.Uri) field(:mediaType, :string, default: "application/octet-stream") + field(:width, :integer) + field(:height, :integer) end end @@ -52,7 +54,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do data = fix_media_type(data) struct - |> cast(data, [:type, :href, :mediaType]) + |> cast(data, [:type, :href, :mediaType, :width, :height]) |> validate_inclusion(:type, ["Link"]) |> validate_required([:type, :href, :mediaType]) end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 4d9a5617e..b5767863c 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -245,6 +245,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "type" => Map.get(url || %{}, "type", "Link") } |> Maps.put_if_present("mediaType", media_type) + |> Maps.put_if_present("width", (url || %{})["width"]) + |> Maps.put_if_present("height", (url || %{})["height"]) %{ "url" => [attachment_url], -- cgit v1.2.3 From 02b9436494998e441fe2119b78c0e4f68c45a9e1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 May 2021 16:16:10 -0500 Subject: Don't render media `meta` if nil --- lib/pleroma/web/mastodon_api/views/status_view.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 5dbdc309e..7f318e81b 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Activity alias Pleroma.HTML + alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -406,6 +407,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do media_type = attachment_url["mediaType"] || attachment_url["mimeType"] || "image" href = attachment_url["href"] |> MediaProxy.url() href_preview = attachment_url["href"] |> MediaProxy.preview_url() + meta = render("attachment_meta.json", %{attachment: attachment}) type = cond do @@ -426,9 +428,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do type: type, description: attachment["name"], pleroma: %{mime_type: media_type}, - meta: render("attachment_meta.json", %{attachment: attachment}), blurhash: attachment["blurhash"] } + |> Maps.put_if_present(:meta, meta) end def render("attachment_meta.json", %{ @@ -444,7 +446,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do } end - def render("attachment_meta.json", _), do: %{} + def render("attachment_meta.json", _), do: nil def render("context.json", %{activity: activity, activities: activities, user: user}) do %{ancestors: ancestors, descendants: descendants} = -- cgit v1.2.3 From 6f0b42656dcce9cd7e4c833be42b6544954ca93b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 May 2021 19:03:10 -0500 Subject: Federate attachments as Links instead of Documents --- lib/pleroma/web/activity_pub/transmogrifier.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b5767863c..acb4f4b3e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -245,8 +245,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "type" => Map.get(url || %{}, "type", "Link") } |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("width", (url || %{})["width"]) - |> Maps.put_if_present("height", (url || %{})["height"]) + |> Maps.put_if_present("width", (url || %{})["width"] || data["width"]) + |> Maps.put_if_present("height", (url || %{})["height"] || data["height"]) %{ "url" => [attachment_url], @@ -963,7 +963,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do object |> Map.get("attachment", []) |> Enum.map(fn data -> - [%{"mediaType" => media_type, "href" => href} | _] = data["url"] + [%{"mediaType" => media_type, "href" => href} = url | _] = data["url"] %{ "url" => href, @@ -971,6 +971,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "name" => data["name"], "type" => "Document" } + |> Maps.put_if_present("width", url["width"]) + |> Maps.put_if_present("height", url["height"]) end) Map.put(object, "attachment", attachments) -- cgit v1.2.3 From 543e9402d64bce556f85294f91dc690c9acec51f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 14 May 2021 08:38:23 -0500 Subject: Support blurhash --- lib/pleroma/upload.ex | 7 +++++-- lib/pleroma/upload/filter/set_meta.ex | 9 +++++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 4d58abd48..5570ed104 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -25,6 +25,7 @@ defmodule Pleroma.Upload do path as the temporary file is also tracked by `Plug.Upload{}` and automatically deleted once the request is over. * `:width` - width of the media in pixels * `:height` - height of the media in pixels + * `:blurhash` - string hash of the image encoded with the blurhash algorithm (https://blurha.sh/) Related behaviors: @@ -58,9 +59,10 @@ defmodule Pleroma.Upload do content_type: String.t(), width: integer(), height: integer(), + blurhash: String.t(), path: String.t() } - defstruct [:id, :name, :tempfile, :content_type, :width, :height, :path] + defstruct [:id, :name, :tempfile, :content_type, :width, :height, :blurhash, :path] defp get_description(opts, upload) do case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do @@ -98,7 +100,8 @@ defmodule Pleroma.Upload do |> Maps.put_if_present("height", upload.height) ], "name" => description - }} + } + |> Maps.put_if_present("blurhash", upload.blurhash)} else {:description_limit, _} -> {:error, :description_too_long} diff --git a/lib/pleroma/upload/filter/set_meta.ex b/lib/pleroma/upload/filter/set_meta.ex index cccb6c371..81c48228a 100644 --- a/lib/pleroma/upload/filter/set_meta.ex +++ b/lib/pleroma/upload/filter/set_meta.ex @@ -23,6 +23,7 @@ defmodule Pleroma.Upload.Filter.SetMeta do upload |> Map.put(:width, image.width) |> Map.put(:height, image.height) + |> Map.put(:blurhash, get_blurhash(file)) {:ok, :filtered, upload} rescue @@ -33,4 +34,12 @@ defmodule Pleroma.Upload.Filter.SetMeta do end def filter(_), do: {:ok, :noop} + + defp get_blurhash(file) do + with {:ok, blurhash} <- :eblurhash.magick(file) do + blurhash + else + _ -> nil + end + end end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index acb4f4b3e..f601d6111 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -973,6 +973,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do } |> Maps.put_if_present("width", url["width"]) |> Maps.put_if_present("height", url["height"]) + |> Maps.put_if_present("blurhash", data["blurhash"]) end) Map.put(object, "attachment", attachments) -- cgit v1.2.3 From 9b6b5ac196d9a2defb74902bffad67505b0de5c5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 18 May 2021 15:33:33 -0500 Subject: Rename upload filter to AnalyzeMetadata --- lib/pleroma/upload/filter/analyze_metadata.ex | 45 +++++++++++++++++++++++++++ lib/pleroma/upload/filter/set_meta.ex | 45 --------------------------- 2 files changed, 45 insertions(+), 45 deletions(-) create mode 100644 lib/pleroma/upload/filter/analyze_metadata.ex delete mode 100644 lib/pleroma/upload/filter/set_meta.ex (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex new file mode 100644 index 000000000..8c23076d4 --- /dev/null +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.AnalyzeMetadata do + @moduledoc """ + Extracts metadata about the upload, such as width/height + """ + require Logger + + @behaviour Pleroma.Upload.Filter + + @spec filter(Pleroma.Upload.t()) :: + {:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()} + def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do + try do + image = + file + |> Mogrify.open() + |> Mogrify.verbose() + + upload = + upload + |> Map.put(:width, image.width) + |> Map.put(:height, image.height) + |> Map.put(:blurhash, get_blurhash(file)) + + {:ok, :filtered, upload} + rescue + e in ErlangError -> + Logger.warn("#{__MODULE__}: #{inspect(e)}") + {:ok, :noop} + end + end + + def filter(_), do: {:ok, :noop} + + defp get_blurhash(file) do + with {:ok, blurhash} <- :eblurhash.magick(file) do + blurhash + else + _ -> nil + end + end +end diff --git a/lib/pleroma/upload/filter/set_meta.ex b/lib/pleroma/upload/filter/set_meta.ex deleted file mode 100644 index 81c48228a..000000000 --- a/lib/pleroma/upload/filter/set_meta.ex +++ /dev/null @@ -1,45 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Upload.Filter.SetMeta do - @moduledoc """ - Extracts metadata about the upload, such as width/height - """ - require Logger - - @behaviour Pleroma.Upload.Filter - - @spec filter(Pleroma.Upload.t()) :: - {:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()} - def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do - try do - image = - file - |> Mogrify.open() - |> Mogrify.verbose() - - upload = - upload - |> Map.put(:width, image.width) - |> Map.put(:height, image.height) - |> Map.put(:blurhash, get_blurhash(file)) - - {:ok, :filtered, upload} - rescue - e in ErlangError -> - Logger.warn("#{__MODULE__}: #{inspect(e)}") - {:ok, :noop} - end - end - - def filter(_), do: {:ok, :noop} - - defp get_blurhash(file) do - with {:ok, blurhash} <- :eblurhash.magick(file) do - blurhash - else - _ -> nil - end - end -end -- cgit v1.2.3 From 4ab3ef07d0f10815e7a91ba3143b7f97cd2a6058 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 18 May 2021 15:51:11 -0500 Subject: Check AnalyzeMetadata filter's required commands eblurhash:magick uses "convert" Fetching image metadata uses "mogrify" --- lib/pleroma/application_requirements.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index c412dec5e..294eb3b6b 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -166,7 +166,9 @@ defmodule Pleroma.ApplicationRequirements do filter_commands_statuses = [ check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"), check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"), - check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify") + check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify"), + check_filter(Pleroma.Upload.Filters.AnalyzeMetadata, "mogrify"), + check_filter(Pleroma.Upload.Filters.AnalyzeMetadata, "convert") ] preview_proxy_commands_status = -- cgit v1.2.3 From c64cbee26c7b78f9743b668724d4797faa6a942a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 18 May 2021 16:28:21 -0500 Subject: Fixed checking for Upload Filter required commands --- lib/pleroma/application_requirements.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 294eb3b6b..ee6ee9516 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -164,11 +164,11 @@ defmodule Pleroma.ApplicationRequirements do defp check_system_commands!(:ok) do filter_commands_statuses = [ - check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"), - check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"), - check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify"), - check_filter(Pleroma.Upload.Filters.AnalyzeMetadata, "mogrify"), - check_filter(Pleroma.Upload.Filters.AnalyzeMetadata, "convert") + check_filter(Pleroma.Upload.Filter.Exiftool, "exiftool"), + check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"), + check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"), + check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"), + check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "convert") ] preview_proxy_commands_status = -- cgit v1.2.3 From 07fed0fda2473fc4e1e3b01e863217391fd2902f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 May 2021 17:11:25 -0500 Subject: Switch to aliasing `Router.Helpers` instead of importing --- lib/pleroma/web.ex | 6 ++++-- lib/pleroma/web/feed/user_controller.ex | 2 +- lib/pleroma/web/mastodon_api/controllers/auth_controller.ex | 4 ++-- lib/pleroma/web/o_auth/o_auth_controller.ex | 4 ++-- lib/pleroma/web/templates/feed/feed/tag.atom.eex | 4 ++-- lib/pleroma/web/templates/feed/feed/tag.rss.eex | 2 +- lib/pleroma/web/templates/feed/feed/user.atom.eex | 6 +++--- lib/pleroma/web/templates/feed/feed/user.rss.eex | 6 +++--- lib/pleroma/web/templates/masto_fe/index.html.eex | 2 +- lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex | 4 ++-- lib/pleroma/web/templates/o_auth/mfa/totp.html.eex | 4 ++-- lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex | 2 +- lib/pleroma/web/templates/o_auth/o_auth/register.html.eex | 2 +- lib/pleroma/web/templates/o_auth/o_auth/show.html.eex | 2 +- lib/pleroma/web/templates/twitter_api/password/reset.html.eex | 2 +- lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex | 2 +- .../web/templates/twitter_api/remote_follow/follow_login.html.eex | 2 +- .../web/templates/twitter_api/remote_follow/follow_mfa.html.eex | 2 +- lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex | 2 +- lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex | 2 +- lib/pleroma/web/views/masto_fe_view.ex | 2 +- 21 files changed, 33 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web.ex b/lib/pleroma/web.ex index 8630f244b..24751faba 100644 --- a/lib/pleroma/web.ex +++ b/lib/pleroma/web.ex @@ -35,9 +35,10 @@ defmodule Pleroma.Web do import Plug.Conn import Pleroma.Web.Gettext - import Pleroma.Web.Router.Helpers import Pleroma.Web.TranslationHelpers + alias Pleroma.Web.Router.Helpers, as: Routes + plug(:set_put_layout) defp set_put_layout(conn, _) do @@ -131,7 +132,8 @@ defmodule Pleroma.Web do import Pleroma.Web.ErrorHelpers import Pleroma.Web.Gettext - import Pleroma.Web.Router.Helpers + + alias Pleroma.Web.Router.Helpers, as: Routes require Logger diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index 58d35da1e..fa7879caf 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.Feed.UserController do def feed_redirect(conn, %{"nickname" => nickname}) do with {_, %User{} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do - redirect(conn, external: "#{user_feed_url(conn, :feed, user.nickname)}.atom") + redirect(conn, external: "#{Routes.user_feed_url(conn, :feed, user.nickname)}.atom") end end diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex index eb6639fc5..4920d65da 100644 --- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex @@ -53,7 +53,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do defp redirect_to_oauth_form(conn, _params) do with {:ok, app} <- local_mastofe_app() do path = - o_auth_path(conn, :authorize, + Routes.o_auth_path(conn, :authorize, response_type: "code", client_id: app.client_id, redirect_uri: ".", @@ -90,7 +90,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do defp local_mastodon_post_login_path(conn) do case get_session(conn, :return_to) do nil -> - masto_fe_path(conn, :index, ["getting-started"]) + Routes.masto_fe_path(conn, :index, ["getting-started"]) return_to -> delete_session(conn, :return_to) diff --git a/lib/pleroma/web/o_auth/o_auth_controller.ex b/lib/pleroma/web/o_auth/o_auth_controller.ex index 215d97b3a..42f4d768f 100644 --- a/lib/pleroma/web/o_auth/o_auth_controller.ex +++ b/lib/pleroma/web/o_auth/o_auth_controller.ex @@ -427,7 +427,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do |> Map.put("state", state) # Handing the request to Ueberauth - redirect(conn, to: o_auth_path(conn, :request, provider, params)) + redirect(conn, to: Routes.o_auth_path(conn, :request, provider, params)) end def request(%Plug.Conn{} = conn, params) do @@ -601,7 +601,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do end # Special case: Local MastodonFE - defp redirect_uri(%Plug.Conn{} = conn, "."), do: auth_url(conn, :login) + defp redirect_uri(%Plug.Conn{} = conn, "."), do: Routes.auth_url(conn, :login) defp redirect_uri(%Plug.Conn{}, redirect_uri), do: redirect_uri diff --git a/lib/pleroma/web/templates/feed/feed/tag.atom.eex b/lib/pleroma/web/templates/feed/feed/tag.atom.eex index a288539ed..de0731085 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.atom.eex @@ -9,13 +9,13 @@ xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:statusnet="http://status.net/schema/api/1/"> - <%= '#{tag_feed_url(@conn, :feed, @tag)}.rss' %> + <%= '#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss' %> #<%= @tag %> These are public toots tagged with #<%= @tag %>. You can interact with them if you have an account anywhere in the fediverse. <%= feed_logo() %> <%= most_recent_update(@activities) %> - + <%= for activity <- @activities do %> <%= render @view_module, "_tag_activity.atom", Map.merge(assigns, prepare_activity(activity, actor: true)) %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/tag.rss.eex b/lib/pleroma/web/templates/feed/feed/tag.rss.eex index eeda01a04..9c3613feb 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.rss.eex @@ -5,7 +5,7 @@ #<%= @tag %> These are public toots tagged with #<%= @tag %>. You can interact with them if you have an account anywhere in the fediverse. - <%= '#{tag_feed_url(@conn, :feed, @tag)}.rss' %> + <%= '#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss' %> <%= feed_logo() %> 2b90d9 <%= for activity <- @activities do %> diff --git a/lib/pleroma/web/templates/feed/feed/user.atom.eex b/lib/pleroma/web/templates/feed/feed/user.atom.eex index c6acd848f..5c1f0ecbc 100644 --- a/lib/pleroma/web/templates/feed/feed/user.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/user.atom.eex @@ -6,16 +6,16 @@ xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:ostatus="http://ostatus.org/schema/1.0"> - <%= user_feed_url(@conn, :feed, @user.nickname) <> ".atom" %> + <%= Routes.user_feed_url(@conn, :feed, @user.nickname) <> ".atom" %> <%= @user.nickname <> "'s timeline" %> <%= most_recent_update(@activities, @user) %> <%= logo(@user) %> - + <%= render @view_module, "_author.atom", assigns %> <%= if last_activity(@activities) do %> - + <% end %> <%= for activity <- @activities do %> diff --git a/lib/pleroma/web/templates/feed/feed/user.rss.eex b/lib/pleroma/web/templates/feed/feed/user.rss.eex index d69120480..6b842a085 100644 --- a/lib/pleroma/web/templates/feed/feed/user.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/user.rss.eex @@ -1,16 +1,16 @@ - <%= user_feed_url(@conn, :feed, @user.nickname) <> ".rss" %> + <%= Routes.user_feed_url(@conn, :feed, @user.nickname) <> ".rss" %> <%= @user.nickname <> "'s timeline" %> <%= most_recent_update(@activities, @user) %> <%= logo(@user) %> - <%= '#{user_feed_url(@conn, :feed, @user.nickname)}.rss' %> + <%= '#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss' %> <%= render @view_module, "_author.rss", assigns %> <%= if last_activity(@activities) do %> - <%= '#{user_feed_url(@conn, :feed, @user.nickname)}.rss?max_id=#{last_activity(@activities).id}' %> + <%= '#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss?max_id=#{last_activity(@activities).id}' %> <% end %> <%= for activity <- @activities do %> diff --git a/lib/pleroma/web/templates/masto_fe/index.html.eex b/lib/pleroma/web/templates/masto_fe/index.html.eex index c330960fa..6f2b98957 100644 --- a/lib/pleroma/web/templates/masto_fe/index.html.eex +++ b/lib/pleroma/web/templates/masto_fe/index.html.eex @@ -7,7 +7,7 @@ <%= Config.get([:instance, :name]) %> - + diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex index 5ab59b57b..b9daa8d8b 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex @@ -7,7 +7,7 @@

    Two-factor recovery

    -<%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %> +<%= form_for @conn, Routes.mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
    <%= label f, :code, "Recovery code" %> <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, spellcheck: false] %> @@ -19,6 +19,6 @@ <%= submit "Verify" %> <% end %> -"> +"> Enter a two-factor code diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex index af85777eb..29ea7c5fb 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex @@ -7,7 +7,7 @@

    Two-factor authentication

    -<%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %> +<%= form_for @conn, Routes.mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
    <%= label f, :code, "Authentication code" %> <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, pattern: "[0-9]*", spellcheck: false] %> @@ -19,6 +19,6 @@ <%= submit "Verify" %> <% end %> -"> +"> Enter a two-factor recovery code diff --git a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex index 4a0718851..dc4521a62 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex @@ -1,6 +1,6 @@

    Sign in with external provider

    -<%= form_for @conn, o_auth_path(@conn, :prepare_request), [as: "authorization", method: "get"], fn f -> %> +<%= form_for @conn, Routes.o_auth_path(@conn, :prepare_request), [as: "authorization", method: "get"], fn f -> %>
    <%= render @view_module, "_scopes.html", Map.merge(assigns, %{form: f}) %>
    diff --git a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex index facedc8db..99f900fb7 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex @@ -8,7 +8,7 @@

    Registration Details

    If you'd like to register a new account, please provide the details below.

    -<%= form_for @conn, o_auth_path(@conn, :register), [as: "authorization"], fn f -> %> +<%= form_for @conn, Routes.o_auth_path(@conn, :register), [as: "authorization"], fn f -> %>
    <%= label f, :nickname, "Nickname" %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index 1a85818ec..2846ec7e7 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -5,7 +5,7 @@ <% end %> -<%= form_for @conn, o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %> +<%= form_for @conn, Routes.o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %> <%= if @user do %>