diff options
Diffstat (limited to 'lib')
24 files changed, 415 insertions, 91 deletions
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 40bff08c7..d67e2cdc8 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -6,11 +6,13 @@ defmodule Pleroma.Application do use Application import Supervisor.Spec - @name "Pleroma" + @name Mix.Project.config()[:name] @version Mix.Project.config()[:version] + @repository Mix.Project.config()[:source_url] def name, do: @name def version, do: @version def named_version(), do: @name <> " " <> @version + def repository, do: @repository def user_agent() do info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>" diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index dc50682ee..0eb1833aa 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -12,6 +12,13 @@ defmodule Pleroma.Config.DeprecationWarnings do You are using the old configuration mechanism for the frontend. Please check config.md. """) end + + if Pleroma.Config.get(:mrf_hellthread, :threshold) do + Logger.warn(""" + !!!DEPRECATION WARNING!!! + You are using the old configuration mechanism for the hellthread filter. Please check config.md. + """) + end end def warn do diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex new file mode 100644 index 000000000..5e107f4c9 --- /dev/null +++ b/lib/pleroma/instances.ex @@ -0,0 +1,36 @@ +defmodule Pleroma.Instances do + @moduledoc "Instances context." + + @adapter Pleroma.Instances.Instance + + defdelegate filter_reachable(urls_or_hosts), to: @adapter + defdelegate reachable?(url_or_host), to: @adapter + defdelegate set_reachable(url_or_host), to: @adapter + defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter + + def set_consistently_unreachable(url_or_host), + do: set_unreachable(url_or_host, reachability_datetime_threshold()) + + def reachability_datetime_threshold do + federation_reachability_timeout_days = + Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 0 + + if federation_reachability_timeout_days > 0 do + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -federation_reachability_timeout_days * 24 * 3600, + :second + ) + else + ~N[0000-01-01 00:00:00] + end + end + + def host(url_or_host) when is_binary(url_or_host) do + if url_or_host =~ ~r/^http/i do + URI.parse(url_or_host).host + else + url_or_host + end + end +end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex new file mode 100644 index 000000000..4a4ca26dd --- /dev/null +++ b/lib/pleroma/instances/instance.ex @@ -0,0 +1,113 @@ +defmodule Pleroma.Instances.Instance do + @moduledoc "Instance." + + alias Pleroma.Instances + alias Pleroma.Instances.Instance + + use Ecto.Schema + + import Ecto.{Query, Changeset} + + alias Pleroma.Repo + + schema "instances" do + field(:host, :string) + field(:unreachable_since, :naive_datetime) + + timestamps() + end + + defdelegate host(url_or_host), to: Instances + + def changeset(struct, params \\ %{}) do + struct + |> cast(params, [:host, :unreachable_since]) + |> validate_required([:host]) + |> unique_constraint(:host) + end + + def filter_reachable([]), do: %{} + + def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do + hosts = + urls_or_hosts + |> Enum.map(&(&1 && host(&1))) + |> Enum.filter(&(to_string(&1) != "")) + + unreachable_since_by_host = + Repo.all( + from(i in Instance, + where: i.host in ^hosts, + select: {i.host, i.unreachable_since} + ) + ) + |> Map.new(& &1) + + reachability_datetime_threshold = Instances.reachability_datetime_threshold() + + for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do + host = host(entry) + unreachable_since = unreachable_since_by_host[host] + + if !unreachable_since || + NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do + {entry, unreachable_since} + end + end + |> Enum.filter(& &1) + |> Map.new(& &1) + end + + def reachable?(url_or_host) when is_binary(url_or_host) do + !Repo.one( + from(i in Instance, + where: + i.host == ^host(url_or_host) and + i.unreachable_since <= ^Instances.reachability_datetime_threshold(), + select: true + ) + ) + end + + def reachable?(_), do: true + + def set_reachable(url_or_host) when is_binary(url_or_host) do + with host <- host(url_or_host), + %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do + {:ok, _instance} = + existing_record + |> changeset(%{unreachable_since: nil}) + |> Repo.update() + end + end + + def set_reachable(_), do: {:error, nil} + + def set_unreachable(url_or_host, unreachable_since \\ nil) + + def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do + unreachable_since = unreachable_since || DateTime.utc_now() + host = host(url_or_host) + existing_record = Repo.get_by(Instance, %{host: host}) + + changes = %{unreachable_since: unreachable_since} + + cond do + is_nil(existing_record) -> + %Instance{} + |> changeset(Map.put(changes, :host, host)) + |> Repo.insert() + + existing_record.unreachable_since && + NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt -> + {:ok, existing_record} + + true -> + existing_record + |> changeset(changes) + |> Repo.update() + end + end + + def set_unreachable(_, _), do: {:error, nil} +end diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 707a61f14..7b46a3b05 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -31,8 +31,8 @@ defmodule Pleroma.Object do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end - def normalize(obj) when is_map(obj), do: Object.get_by_ap_id(obj["id"]) - def normalize(ap_id) when is_binary(ap_id), do: Object.get_by_ap_id(ap_id) + def normalize(%{"id" => ap_id}), do: normalize(ap_id) + def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) def normalize(_), do: nil # Owned objects can only be mutated by their owner @@ -42,24 +42,18 @@ defmodule Pleroma.Object do # Legacy objects can be mutated by anybody def authorize_mutation(%Object{}, %User{}), do: true - if Mix.env() == :test do - def get_cached_by_ap_id(ap_id) do - get_by_ap_id(ap_id) - end - else - def get_cached_by_ap_id(ap_id) do - key = "object:#{ap_id}" - - Cachex.fetch!(:object_cache, key, fn _ -> - object = get_by_ap_id(ap_id) - - if object do - {:commit, object} - else - {:ignore, object} - end - end) - end + def get_cached_by_ap_id(ap_id) do + key = "object:#{ap_id}" + + Cachex.fetch!(:object_cache, key, fn _ -> + object = get_by_ap_id(ap_id) + + if object do + {:commit, object} + else + {:ignore, object} + end + end) end def context_mapping(context) do @@ -90,4 +84,17 @@ defmodule Pleroma.Object do {:ok, object} end end + + def set_cache(%Object{data: %{"id" => ap_id}} = object) do + Cachex.put(:object_cache, "object:#{ap_id}", object) + {:ok, object} + end + + def update_and_set_cache(changeset) do + with {:ok, object} <- Repo.update(changeset) do + set_cache(object) + else + e -> e + end + end end diff --git a/lib/pleroma/plugs/instance_static.ex b/lib/pleroma/plugs/instance_static.ex index af2f6f331..11f108de7 100644 --- a/lib/pleroma/plugs/instance_static.ex +++ b/lib/pleroma/plugs/instance_static.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Plugs.InstanceStatic do end end - @only ~w(index.html static emoji packs sounds images instance favicon.png) + @only ~w(index.html static emoji packs sounds images instance favicon.png sw.js sw-pleroma.js) def init(opts) do opts diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 0a19e737b..ce2a1b696 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -124,10 +124,10 @@ defmodule Pleroma.Upload do :pleroma, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Mogrify]] - :pleroma, Pleroma.Upload.Filter.Mogrify, args: "strip" + :pleroma, Pleroma.Upload.Filter.Mogrify, args: ["strip", "auto-orient"] """) - Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: "strip") + Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: ["strip", "auto-orient"]) Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Mogrify]) else opts diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 60d1d4811..33630ac7c 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -315,7 +315,16 @@ defmodule Pleroma.User do q = from(u in User, where: u.id == ^follower.id, - update: [set: [following: fragment("array_cat(?, ?)", u.following, ^followed_addresses)]] + update: [ + set: [ + following: + fragment( + "array(select distinct unnest (array_cat(?, ?)))", + u.following, + ^followed_addresses + ) + ] + ] ) {1, [follower]} = Repo.update_all(q, [], returning: true) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 0199ac9e7..4635e7fcd 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.ActivityPub do - alias Pleroma.{Activity, Repo, Object, Upload, User, Notification} + alias Pleroma.{Activity, Repo, Object, Upload, User, Notification, Instances} alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF} alias Pleroma.Web.WebFinger alias Pleroma.Web.Federator @@ -734,7 +734,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def publish(actor, activity) do - followers = + remote_followers = if actor.follower_address in activity.recipients do {:ok, followers} = User.get_followers(actor) followers |> Enum.filter(&(!&1.local)) @@ -744,29 +744,31 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do public = is_public?(activity) - remote_inboxes = - (Pleroma.Web.Salmon.remote_users(activity) ++ followers) + reachable_inboxes_metadata = + (Pleroma.Web.Salmon.remote_users(activity) ++ remote_followers) |> Enum.filter(fn user -> User.ap_enabled?(user) end) |> Enum.map(fn %{info: %{source_data: data}} -> (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"] end) |> Enum.uniq() |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) + |> Instances.filter_reachable() {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) json = Jason.encode!(data) - Enum.each(remote_inboxes, fn inbox -> + Enum.each(reachable_inboxes_metadata, fn {inbox, unreachable_since} -> Federator.enqueue(:publish_single_ap, %{ inbox: inbox, json: json, actor: actor, - id: activity.data["id"] + id: activity.data["id"], + unreachable_since: unreachable_since }) end) end - def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do + def publish_one(%{inbox: inbox, json: json, actor: actor, id: id} = params) do Logger.info("Federating #{id} to #{inbox}") host = URI.parse(inbox).host @@ -779,15 +781,26 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do digest: digest }) - @httpoison.post( - inbox, - json, - [ - {"Content-Type", "application/activity+json"}, - {"signature", signature}, - {"digest", digest} - ] - ) + with {:ok, %{status: code}} when code in 200..299 <- + result = + @httpoison.post( + inbox, + json, + [ + {"Content-Type", "application/activity+json"}, + {"signature", signature}, + {"digest", digest} + ] + ) do + if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since], + do: Instances.set_reachable(inbox) + + result + else + {_post_result, response} -> + unless params[:unreachable_since], do: Instances.set_unreachable(inbox) + {:error, response} + end end # TODO: diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 7eed0a600..4dea6ab83 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do use Pleroma.Web, :controller + alias Pleroma.{Activity, User, Object} alias Pleroma.Web.ActivityPub.{ObjectView, UserView} alias Pleroma.Web.ActivityPub.ActivityPub @@ -17,6 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do action_fallback(:errors) plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay]) + plug(:set_requester_reachable when action in [:inbox]) plug(:relay_active? when action in [:relay]) def relay_active?(conn, _) do @@ -289,4 +291,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> put_status(500) |> json("error") end + + defp set_requester_reachable(%Plug.Conn{} = conn, _) do + with actor <- conn.params["actor"], + true <- is_binary(actor) do + Pleroma.Instances.set_reachable(actor) + end + + conn + end end diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index a3f516ae7..4c6e612b2 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -3,20 +3,46 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do + alias Pleroma.User @behaviour Pleroma.Web.ActivityPub.MRF + defp delist_message(message) do + follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address + + message + |> Map.put("to", [follower_collection]) + |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) + end + @impl true - def filter(%{"type" => "Create"} = object) do - threshold = Pleroma.Config.get([:mrf_hellthread, :threshold]) - recipients = (object["to"] || []) ++ (object["cc"] || []) - - if length(recipients) > threshold do - {:reject, nil} - else - {:ok, object} + def filter(%{"type" => "Create"} = message) do + delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold]) + + reject_threshold = + Pleroma.Config.get( + [:mrf_hellthread, :reject_threshold], + Pleroma.Config.get([:mrf_hellthread, :threshold]) + ) + + recipients = (message["to"] || []) ++ (message["cc"] || []) + + cond do + length(recipients) > reject_threshold and reject_threshold > 0 -> + {:reject, nil} + + length(recipients) > delist_threshold and delist_threshold > 0 -> + if Enum.member?(message["to"], "https://www.w3.org/ns/activitystreams#Public") or + Enum.member?(message["cc"], "https://www.w3.org/ns/activitystreams#Public") do + {:ok, delist_message(message)} + else + {:ok, message} + end + + true -> + {:ok, message} end end @impl true - def filter(object), do: {:ok, object} + def filter(message), do: {:ok, message} end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 3b0cdfe71..4a2cc6738 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -285,7 +285,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> Map.put("#{property}_count", length(element)) |> Map.put("#{property}s", element), changeset <- Changeset.change(object, data: new_data), - {:ok, object} <- Repo.update(changeset), + {:ok, object} <- Object.update_and_set_cache(changeset), _ <- update_object_in_activities(object) do {:ok, object} end diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 0b4ce9cc4..3eed047ca 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -25,7 +25,7 @@ defmodule Pleroma.Web.Endpoint do at: "/", from: :pleroma, only: - ~w(index.html static finmoji emoji packs sounds images instance sw.js favicon.png schemas doc) + ~w(index.html static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc) ) # Code reloading can be explicitly enabled under the @@ -82,4 +82,8 @@ defmodule Pleroma.Web.Endpoint do port = System.get_env("PORT") || raise "expected the PORT environment variable to be set" {:ok, Keyword.put(config, :http, [:inet6, port: port])} end + + def websocket_url do + String.replace_leading(url(), "http", "ws") + end end diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f3a0e18b8..bb7676cf0 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.Federator do use GenServer alias Pleroma.User alias Pleroma.Activity - alias Pleroma.Web.{WebFinger, Websub} + alias Pleroma.Web.{WebFinger, Websub, Salmon} alias Pleroma.Web.Federator.RetryQueue alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Relay @@ -124,6 +124,10 @@ defmodule Pleroma.Web.Federator do end end + def handle(:publish_single_salmon, params) do + Salmon.send_to_user(params) + end + def handle(:publish_single_ap, params) do case ActivityPub.publish_one(params) do {:ok, _} -> diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 0726e6ac4..7f3fbff4a 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -138,7 +138,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", email: Keyword.get(instance, :email), urls: %{ - streaming_api: String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws") + streaming_api: Pleroma.Web.Endpoint.websocket_url() }, stats: Stats.get_stats(), thumbnail: Web.base_url() <> "/instance/thumbnail.jpeg", @@ -905,7 +905,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do res = ListView.render("list.json", list: list) json(conn, res) else - _e -> json(conn, "error") + _e -> + conn + |> put_status(404) + |> json(%{error: "Record not found"}) end end diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 11b97164d..21694a5ee 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -19,6 +19,10 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do %{ rel: "http://nodeinfo.diaspora.software/ns/schema/2.0", href: Web.base_url() <> "/nodeinfo/2.0.json" + }, + %{ + rel: "http://nodeinfo.diaspora.software/ns/schema/2.1", + href: Web.base_url() <> "/nodeinfo/2.1.json" } ] } @@ -26,8 +30,9 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do json(conn, response) end - # Schema definition: https://github.com/jhass/nodeinfo/blob/master/schemas/2.0/schema.json - def nodeinfo(conn, %{"version" => "2.0"}) do + # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field + # under software. + def raw_nodeinfo() do instance = Application.get_env(:pleroma, :instance) media_proxy = Application.get_env(:pleroma, :media_proxy) suggestions = Application.get_env(:pleroma, :suggestions) @@ -98,10 +103,10 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do ] |> Enum.filter(& &1) - response = %{ + %{ version: "2.0", software: %{ - name: Pleroma.Application.name(), + name: Pleroma.Application.name() |> String.downcase(), version: Pleroma.Application.version() }, protocols: ["ostatus", "activitypub"], @@ -142,12 +147,37 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do restrictedNicknames: Pleroma.Config.get([Pleroma.User, :restricted_nicknames]) } } + end + # Schema definition: https://github.com/jhass/nodeinfo/blob/master/schemas/2.0/schema.json + # and https://github.com/jhass/nodeinfo/blob/master/schemas/2.1/schema.json + def nodeinfo(conn, %{"version" => "2.0"}) do conn |> put_resp_header( "content-type", "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#; charset=utf-8" ) + |> json(raw_nodeinfo()) + end + + def nodeinfo(conn, %{"version" => "2.1"}) do + raw_response = raw_nodeinfo() + + updated_software = + raw_response + |> Map.get(:software) + |> Map.put(:repository, Pleroma.Application.repository()) + + response = + raw_response + |> Map.put(:software, updated_software) + |> Map.put(:version, "2.1") + + conn + |> put_resp_header( + "content-type", + "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.1#; charset=utf-8" + ) |> json(response) end diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index a3155b79d..a20ca17bb 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -48,6 +48,9 @@ defmodule Pleroma.Web.OStatus do def handle_incoming(xml_string) do with doc when doc != :error <- parse_document(xml_string) do + with {:ok, actor_user} <- find_make_or_update_user(doc), + do: Pleroma.Instances.set_reachable(actor_user.ap_id) + entries = :xmerl_xpath.string('//entry', doc) activities = diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 297aca2f9..302ff38a4 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.Web.ActivityPub.ActivityPub plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming]) + action_fallback(:errors) def feed_redirect(conn, %{"nickname" => nickname}) do diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 71fdddef9..521fa7ee0 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -7,7 +7,8 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Web.RichMedia.Parser def fetch_data_for_activity(%Activity{} = activity) do - with %Object{} = object <- Object.normalize(activity.data["object"]), + with true <- Pleroma.Config.get([:rich_media, :enabled]), + %Object{} = object <- Object.normalize(activity.data["object"]), {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]), {:ok, rich_media} <- Parser.parse(page_url) do %{page_url: page_url, rich_media: rich_media} diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index e67ecc47d..32dec9887 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -30,7 +30,7 @@ defmodule Pleroma.Web.RichMedia.Parser do try do {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: [pool: :media]) - html |> maybe_parse() |> get_parsed_data() + html |> maybe_parse() |> clean_parsed_data() |> check_parsed_data() rescue e -> {:error, "Parsing error: #{inspect(e)}"} @@ -46,11 +46,33 @@ defmodule Pleroma.Web.RichMedia.Parser do end) end - defp get_parsed_data(%{title: title} = data) when is_binary(title) and byte_size(title) > 0 do + defp check_parsed_data(%{title: title} = data) when is_binary(title) and byte_size(title) > 0 do {:ok, data} end - defp get_parsed_data(data) do + defp check_parsed_data(data) do {:error, "Found metadata was invalid or incomplete: #{inspect(data)}"} end + + defp string_is_valid_unicode(data) when is_binary(data) do + data + |> :unicode.characters_to_binary() + |> clean_string() + end + + defp string_is_valid_unicode(data), do: {:ok, data} + + defp clean_string({:error, _, _}), do: {:error, "Invalid data"} + defp clean_string(data), do: {:ok, data} + + defp clean_parsed_data(data) do + data + |> Enum.reject(fn {_, val} -> + case string_is_valid_unicode(val) do + {:ok, _} -> false + _ -> true + end + end) + |> Map.new() + end end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index e41657da1..b1c2dc7fa 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.Salmon do @httpoison Application.get_env(:pleroma, :httpoison) use Bitwise + alias Pleroma.Instances alias Pleroma.Web.XML alias Pleroma.Web.OStatus.ActivityRepresenter alias Pleroma.User @@ -161,25 +162,31 @@ defmodule Pleroma.Web.Salmon do |> Enum.filter(fn user -> user && !user.local end) end - # push an activity to remote accounts - # - defp send_to_user(%{info: %{salmon: salmon}}, feed, poster), - do: send_to_user(salmon, feed, poster) + @doc "Pushes an activity to remote account." + def send_to_user(%{recipient: %{info: %{salmon: salmon}}} = params), + do: send_to_user(Map.put(params, :recipient, salmon)) - defp send_to_user(url, feed, poster) when is_binary(url) do - with {:ok, %{status: code}} <- + def send_to_user(%{recipient: url, feed: feed, poster: poster} = params) when is_binary(url) do + with {:ok, %{status: code}} when code in 200..299 <- poster.( url, feed, [{"Content-Type", "application/magic-envelope+xml"}] ) do + if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since], + do: Instances.set_reachable(url) + Logger.debug(fn -> "Pushed to #{url}, code #{code}" end) + :ok else - e -> Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end) + e -> + unless params[:unreachable_since], do: Instances.set_reachable(url) + Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end) + :error end end - defp send_to_user(_, _, _), do: nil + def send_to_user(_), do: :noop @supported_activities [ "Create", @@ -209,12 +216,23 @@ defmodule Pleroma.Web.Salmon do {:ok, private, _} = keys_from_pem(keys) {:ok, feed} = encode(private, feed) - remote_users(activity) + remote_users = remote_users(activity) + + salmon_urls = Enum.map(remote_users, & &1.info.salmon) + reachable_urls_metadata = Instances.filter_reachable(salmon_urls) + reachable_urls = Map.keys(reachable_urls_metadata) + + remote_users + |> Enum.filter(&(&1.info.salmon in reachable_urls)) |> Enum.each(fn remote_user -> - Task.start(fn -> - Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end) - send_to_user(remote_user, feed, poster) - end) + Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end) + + Pleroma.Web.Federator.enqueue(:publish_single_salmon, %{ + recipient: remote_user, + feed: feed, + poster: poster, + unreachable_since: reachable_urls_metadata[remote_user.info.salmon] + }) end) end end diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index 2e96c1509..8dd3284d6 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -1,7 +1,8 @@ <!DOCTYPE html> <html> <head> - <meta charset=utf-8 /> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width,initial-scale=1,minimal-ui" /> <title> <%= Application.get_env(:pleroma, :instance)[:name] %> </title> diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 7ca62c83b..90ba79962 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.Websub do alias Ecto.Changeset alias Pleroma.Repo + alias Pleroma.Instances alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription} alias Pleroma.Web.OStatus.FeedRepresenter alias Pleroma.Web.{XML, Endpoint, OStatus} @@ -53,28 +54,34 @@ defmodule Pleroma.Web.Websub do ] def publish(topic, user, %{data: %{"type" => type}} = activity) when type in @supported_activities do - # TODO: Only send to still valid subscriptions. + response = + user + |> FeedRepresenter.to_simple_form([activity], [user]) + |> :xmerl.export_simple(:xmerl_xml) + |> to_string + query = from( sub in WebsubServerSubscription, where: sub.topic == ^topic and sub.state == "active", - where: fragment("? > NOW()", sub.valid_until) + where: fragment("? > (NOW() at time zone 'UTC')", sub.valid_until) ) subscriptions = Repo.all(query) - Enum.each(subscriptions, fn sub -> - response = - user - |> FeedRepresenter.to_simple_form([activity], [user]) - |> :xmerl.export_simple(:xmerl_xml) - |> to_string + callbacks = Enum.map(subscriptions, & &1.callback) + reachable_callbacks_metadata = Instances.filter_reachable(callbacks) + reachable_callbacks = Map.keys(reachable_callbacks_metadata) + subscriptions + |> Enum.filter(&(&1.callback in reachable_callbacks)) + |> Enum.each(fn sub -> data = %{ xml: response, topic: topic, callback: sub.callback, - secret: sub.secret + secret: sub.secret, + unreachable_since: reachable_callbacks_metadata[sub.callback] } Pleroma.Web.Federator.enqueue(:publish_single_websub, data) @@ -263,11 +270,11 @@ defmodule Pleroma.Web.Websub do end) end - def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) do + def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret} = params) do signature = sign(secret || "", xml) Logger.info(fn -> "Pushing #{topic} to #{callback}" end) - with {:ok, %{status: code}} <- + with {:ok, %{status: code}} when code in 200..299 <- @httpoison.post( callback, xml, @@ -276,12 +283,16 @@ defmodule Pleroma.Web.Websub do {"X-Hub-Signature", "sha1=#{signature}"} ] ) do + if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since], + do: Instances.set_reachable(callback) + Logger.info(fn -> "Pushed to #{callback}, code #{code}" end) {:ok, code} else - e -> - Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end) - {:error, e} + {_post_result, response} -> + unless params[:unreachable_since], do: Instances.set_reachable(callback) + Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(response)}" end) + {:error, response} end end end diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index e58f144e5..a92dfe87b 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -4,9 +4,11 @@ defmodule Pleroma.Web.Websub.WebsubController do use Pleroma.Web, :controller + alias Pleroma.{Repo, User} alias Pleroma.Web.{Websub, Federator} alias Pleroma.Web.Websub.WebsubClientSubscription + require Logger plug( |