diff options
Diffstat (limited to 'lib')
25 files changed, 949 insertions, 273 deletions
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index dabce771d..a339e2c48 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -102,10 +102,14 @@ defmodule Pleroma.Application do build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000), build_cachex("scrubber", limit: 2500), build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), - build_cachex("web_resp", limit: 2500) + build_cachex("web_resp", limit: 2500), + build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10) ] end + defp emoji_packs_expiration, + do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60)) + defp idempotency_expiration, do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 66e20f0e4..170a7d098 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -122,6 +122,9 @@ defmodule Pleroma.Emoji do fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end ) + # Clear out old emojis + :ets.delete_all_objects(@ets) + true = :ets.insert(@ets, emojis) end @@ -143,23 +146,38 @@ defmodule Pleroma.Emoji do defp load_pack(pack_dir, emoji_groups) do pack_name = Path.basename(pack_dir) - emoji_txt = Path.join(pack_dir, "emoji.txt") + pack_file = Path.join(pack_dir, "pack.json") + + if File.exists?(pack_file) do + contents = Jason.decode!(File.read!(pack_file)) - if File.exists?(emoji_txt) do - load_from_file(emoji_txt, emoji_groups) + contents["files"] + |> Enum.map(fn {name, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) + {name, filename, pack_name} + end) else - extensions = Pleroma.Config.get([:emoji, :pack_extensions]) + # Load from emoji.txt / all files + emoji_txt = Path.join(pack_dir, "emoji.txt") - Logger.info( - "No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji" - ) + if File.exists?(emoji_txt) do + load_from_file(emoji_txt, emoji_groups) + else + extensions = Pleroma.Config.get([:emoji, :pack_extensions]) - make_shortcode_to_file_map(pack_dir, extensions) - |> Enum.map(fn {shortcode, rel_file} -> - filename = Path.join("/emoji/#{pack_name}", rel_file) + Logger.info( + "No emoji.txt found for pack \"#{pack_name}\", assuming all #{ + Enum.join(extensions, ", ") + } files are emoji" + ) - {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} - end) + make_shortcode_to_file_map(pack_dir, extensions) + |> Enum.map(fn {shortcode, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) + + {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} + end) + end end end diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 607843a5b..23a5ac8fe 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -36,9 +36,9 @@ defmodule Pleroma.Formatter do nickname_text = get_nickname_text(nickname, opts) link = - "<span class='h-card'><a data-user='#{id}' class='u-url mention' href='#{ap_id}'>@<span>#{ + ~s(<span class="h-card"><a data-user="#{id}" class="u-url mention" href="#{ap_id}" rel="ugc">@<span>#{ nickname_text - }</span></a></span>" + }</span></a></span>) {link, %{acc | mentions: MapSet.put(acc.mentions, {"@" <> nickname, user})}} @@ -50,7 +50,7 @@ defmodule Pleroma.Formatter do def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do tag = String.downcase(tag) url = "#{Pleroma.Web.base_url()}/tag/#{tag}" - link = "<a class='hashtag' data-tag='#{tag}' href='#{url}' rel='tag'>#{tag_text}</a>" + link = ~s(<a class="hashtag" data-tag="#{tag}" href="#{url}" rel="tag ugc">#{tag_text}</a>) {link, %{acc | tags: MapSet.put(acc.tags, {tag_text, tag})}} end diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 3951f0f51..937bafed5 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -184,7 +184,8 @@ defmodule Pleroma.HTML.Scrubber.Default do "tag", "nofollow", "noopener", - "noreferrer" + "noreferrer", + "ugc" ]) Meta.allow_tag_with_these_attributes("a", ["name", "title"]) @@ -304,7 +305,8 @@ defmodule Pleroma.HTML.Scrubber.LinksOnly do "nofollow", "noopener", "noreferrer", - "me" + "me", + "ugc" ]) Meta.allow_tag_with_these_attributes("a", ["name", "title"]) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index cea33b5af..5e064fd87 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -31,6 +31,7 @@ defmodule Pleroma.Object.Fetcher do defp maybe_reinject_internal_fields(data, _), do: data + @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()} defp reinject_object(struct, data) do Logger.debug("Reinjecting object #{data["id"]}") @@ -61,52 +62,54 @@ defmodule Pleroma.Object.Fetcher do # TODO: # This will create a Create activity, which we need internally at the moment. def fetch_object_from_id(id, options \\ []) do - if object = Object.get_cached_by_ap_id(id) do + with {:fetch_object, nil} <- {:fetch_object, Object.get_cached_by_ap_id(id)}, + {:fetch, {:ok, data}} <- {:fetch, fetch_and_contain_remote_object_from_id(id)}, + {:normalize, nil} <- {:normalize, Object.normalize(data, false)}, + params <- prepare_activity_params(data), + {:containment, :ok} <- {:containment, Containment.contain_origin(id, params)}, + {:ok, activity} <- Transmogrifier.handle_incoming(params, options), + {:object, _data, %Object{} = object} <- + {:object, data, Object.normalize(activity, false)} do {:ok, object} else - Logger.info("Fetching #{id} via AP") - - with {:fetch, {:ok, data}} <- {:fetch, fetch_and_contain_remote_object_from_id(id)}, - {:normalize, nil} <- {:normalize, Object.normalize(data, false)}, - params <- %{ - "type" => "Create", - "to" => data["to"], - "cc" => data["cc"], - # Should we seriously keep this attributedTo thing? - "actor" => data["actor"] || data["attributedTo"], - "object" => data - }, - {:containment, :ok} <- {:containment, Containment.contain_origin(id, params)}, - {:ok, activity} <- Transmogrifier.handle_incoming(params, options), - {:object, _data, %Object{} = object} <- - {:object, data, Object.normalize(activity, false)} do - {:ok, object} - else - {:containment, _} -> - {:error, "Object containment failed."} + {:containment, _} -> + {:error, "Object containment failed."} - {:error, {:reject, nil}} -> - {:reject, nil} + {:error, {:reject, nil}} -> + {:reject, nil} - {:object, data, nil} -> - reinject_object(%Object{}, data) + {:object, data, nil} -> + reinject_object(%Object{}, data) - {:normalize, object = %Object{}} -> - {:ok, object} + {:normalize, object = %Object{}} -> + {:ok, object} - _e -> - # Only fallback when receiving a fetch/normalization error with ActivityPub - Logger.info("Couldn't get object via AP, trying out OStatus fetching...") + {:fetch_object, %Object{} = object} -> + {:ok, object} - # FIXME: OStatus Object Containment? - case OStatus.fetch_activity_from_url(id) do - {:ok, [activity | _]} -> {:ok, Object.normalize(activity, false)} - e -> e - end - end + _e -> + # Only fallback when receiving a fetch/normalization error with ActivityPub + Logger.info("Couldn't get object via AP, trying out OStatus fetching...") + + # FIXME: OStatus Object Containment? + case OStatus.fetch_activity_from_url(id) do + {:ok, [activity | _]} -> {:ok, Object.normalize(activity, false)} + e -> e + end end end + defp prepare_activity_params(data) do + %{ + "type" => "Create", + "to" => data["to"], + "cc" => data["cc"], + # Should we seriously keep this attributedTo thing? + "actor" => data["actor"] || data["attributedTo"], + "object" => data + } + end + def fetch_object_from_id!(id, options \\ []) do with {:ok, object} <- fetch_object_from_id(id, options) do object diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex index 8c353bed3..9876b6398 100644 --- a/lib/pleroma/uploaders/s3.ex +++ b/lib/pleroma/uploaders/s3.ex @@ -38,16 +38,26 @@ defmodule Pleroma.Uploaders.S3 do def put_file(%Pleroma.Upload{} = upload) do config = Config.get([__MODULE__]) bucket = Keyword.get(config, :bucket) + streaming = Keyword.get(config, :streaming_enabled) s3_name = strict_encode(upload.path) op = - upload.tempfile - |> ExAws.S3.Upload.stream_file() - |> ExAws.S3.upload(bucket, s3_name, [ - {:acl, :public_read}, - {:content_type, upload.content_type} - ]) + if streaming do + upload.tempfile + |> ExAws.S3.Upload.stream_file() + |> ExAws.S3.upload(bucket, s3_name, [ + {:acl, :public_read}, + {:content_type, upload.content_type} + ]) + else + {:ok, file_data} = File.read(upload.tempfile) + + ExAws.S3.put_object(bucket, s3_name, file_data, [ + {:acl, :public_read}, + {:content_type, upload.content_type} + ]) + end case ExAws.request(op) do {:ok, _} -> diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index fb1f24254..ab253a274 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -269,6 +269,7 @@ defmodule Pleroma.User do |> validate_required([:password, :password_confirmation]) |> validate_confirmation(:password) |> put_password_hash + |> put_embed(:info, User.Info.set_password_reset_pending(struct.info, false)) end @spec reset_password(User.t(), map) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} @@ -285,6 +286,20 @@ defmodule Pleroma.User do end end + def force_password_reset_async(user) do + BackgroundWorker.enqueue("force_password_reset", %{"user_id" => user.id}) + end + + @spec force_password_reset(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} + def force_password_reset(user) do + info_cng = User.Info.set_password_reset_pending(user.info, true) + + user + |> change() + |> put_embed(:info, info_cng) + |> update_and_set_cache() + end + def register_changeset(struct, params \\ %{}, opts \\ []) do bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) @@ -1115,6 +1130,8 @@ defmodule Pleroma.User do BackgroundWorker.enqueue("delete_user", %{"user_id" => user.id}) end + def perform(:force_password_reset, user), do: force_password_reset(user) + @spec perform(atom(), User.t()) :: {:ok, User.t()} def perform(:delete, %User{} = user) do {:ok, _user} = ActivityPub.delete(user) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index b150a57cd..99745f496 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -20,6 +20,7 @@ defmodule Pleroma.User.Info do field(:following_count, :integer, default: nil) field(:locked, :boolean, default: false) field(:confirmation_pending, :boolean, default: false) + field(:password_reset_pending, :boolean, default: false) field(:confirmation_token, :string, default: nil) field(:default_scope, :string, default: "public") field(:blocks, {:array, :string}, default: []) @@ -82,6 +83,14 @@ defmodule Pleroma.User.Info do |> validate_required([:deactivated]) end + def set_password_reset_pending(info, pending) do + params = %{password_reset_pending: pending} + + info + |> cast(params, [:password_reset_pending]) + |> validate_required([:password_reset_pending]) + end + def update_notification_settings(info, settings) do settings = settings diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index e1e90d667..1cf8b6151 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -520,9 +520,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def fetch_public_activities(opts \\ %{}) do - q = fetch_activities_query([Pleroma.Constants.as_public()], opts) + opts = Map.drop(opts, ["user"]) - q + [Pleroma.Constants.as_public()] + |> fetch_activities_query(opts) |> restrict_unlisted() |> Pagination.fetch_paginated(opts) |> Enum.reverse() diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 01b34fb1d..9eb86106f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -49,7 +49,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_content_type("application/activity+json") - |> json(UserView.render("user.json", %{user: user})) + |> put_view(UserView) + |> render("user.json", %{user: user}) else nil -> {:error, :not_found} end @@ -90,7 +91,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do conn |> put_resp_content_type("application/activity+json") - |> json(ObjectView.render("likes.json", ap_id, likes, page)) + |> put_view(ObjectView) + |> render("likes.json", %{ap_id: ap_id, likes: likes, page: page}) else {:public?, false} -> {:error, :not_found} @@ -104,7 +106,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do likes <- Utils.get_object_likes(object) do conn |> put_resp_content_type("application/activity+json") - |> json(ObjectView.render("likes.json", ap_id, likes)) + |> put_view(ObjectView) + |> render("likes.json", %{ap_id: ap_id, likes: likes}) else {:public?, false} -> {:error, :not_found} @@ -158,7 +161,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def following(%{assigns: %{relay: true}} = conn, _params) do conn |> put_resp_content_type("application/activity+json") - |> json(UserView.render("following.json", %{user: Relay.get_actor()})) + |> put_view(UserView) + |> render("following.json", %{user: Relay.get_actor()}) end def following(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname, "page" => page}) do @@ -170,7 +174,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do conn |> put_resp_content_type("application/activity+json") - |> json(UserView.render("following.json", %{user: user, page: page, for: for_user})) + |> put_view(UserView) + |> render("following.json", %{user: user, page: page, for: for_user}) else {:show_follows, _} -> conn @@ -184,7 +189,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do {user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do conn |> put_resp_content_type("application/activity+json") - |> json(UserView.render("following.json", %{user: user, for: for_user})) + |> put_view(UserView) + |> render("following.json", %{user: user, for: for_user}) end end @@ -192,7 +198,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def followers(%{assigns: %{relay: true}} = conn, _params) do conn |> put_resp_content_type("application/activity+json") - |> json(UserView.render("followers.json", %{user: Relay.get_actor()})) + |> put_view(UserView) + |> render("followers.json", %{user: Relay.get_actor()}) end def followers(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname, "page" => page}) do @@ -204,7 +211,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do conn |> put_resp_content_type("application/activity+json") - |> json(UserView.render("followers.json", %{user: user, page: page, for: for_user})) + |> put_view(UserView) + |> render("followers.json", %{user: user, page: page, for: for_user}) else {:show_followers, _} -> conn @@ -218,7 +226,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do {user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do conn |> put_resp_content_type("application/activity+json") - |> json(UserView.render("followers.json", %{user: user, for: for_user})) + |> put_view(UserView) + |> render("followers.json", %{user: user, for: for_user}) end end @@ -227,7 +236,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_content_type("application/activity+json") - |> json(UserView.render("outbox.json", %{user: user, max_id: params["max_id"]})) + |> put_view(UserView) + |> render("outbox.json", %{user: user, max_id: params["max_id"]}) end end @@ -275,7 +285,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do with {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_content_type("application/activity+json") - |> json(UserView.render("user.json", %{user: user})) + |> put_view(UserView) + |> render("user.json", %{user: user}) else nil -> {:error, :not_found} end @@ -296,7 +307,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def whoami(%{assigns: %{user: %User{} = user}} = conn, _params) do conn |> put_resp_content_type("application/activity+json") - |> json(UserView.render("user.json", %{user: user})) + |> put_view(UserView) + |> render("user.json", %{user: user}) end def whoami(_conn, _params), do: {:error, :not_found} diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5878fb4f8..dad2fead8 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -42,8 +42,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def fix_summary(%{"summary" => nil} = object) do - object - |> Map.put("summary", "") + Map.put(object, "summary", "") end def fix_summary(%{"summary" => _} = object) do @@ -51,10 +50,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do object end - def fix_summary(object) do - object - |> Map.put("summary", "") - end + def fix_summary(object), do: Map.put(object, "summary", "") def fix_addressing_list(map, field) do cond do @@ -74,13 +70,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do explicit_mentions, follower_collection ) do - explicit_to = - to - |> Enum.filter(fn x -> x in explicit_mentions end) + explicit_to = Enum.filter(to, fn x -> x in explicit_mentions end) - explicit_cc = - to - |> Enum.filter(fn x -> x not in explicit_mentions end) + explicit_cc = Enum.filter(to, fn x -> x not in explicit_mentions end) final_cc = (cc ++ explicit_cc) @@ -98,13 +90,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_explicit_addressing(%{"directMessage" => true} = object), do: object def fix_explicit_addressing(object) do - explicit_mentions = - object - |> Utils.determine_explicit_mentions() + explicit_mentions = Utils.determine_explicit_mentions(object) - follower_collection = User.get_cached_by_ap_id(Containment.get_actor(object)).follower_address + %User{follower_address: follower_collection} = + object + |> Containment.get_actor() + |> User.get_cached_by_ap_id() - explicit_mentions = explicit_mentions ++ [Pleroma.Constants.as_public(), follower_collection] + explicit_mentions = + explicit_mentions ++ + [ + Pleroma.Constants.as_public(), + follower_collection + ] fix_explicit_addressing(object, explicit_mentions, follower_collection) end @@ -148,48 +146,25 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def fix_actor(%{"attributedTo" => actor} = object) do - object - |> Map.put("actor", Containment.get_actor(%{"actor" => actor})) + Map.put(object, "actor", Containment.get_actor(%{"actor" => actor})) end def fix_in_reply_to(object, options \\ []) def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) when not is_nil(in_reply_to) do - in_reply_to_id = - cond do - is_bitstring(in_reply_to) -> - in_reply_to - - is_map(in_reply_to) && is_bitstring(in_reply_to["id"]) -> - in_reply_to["id"] - - is_list(in_reply_to) && is_bitstring(Enum.at(in_reply_to, 0)) -> - Enum.at(in_reply_to, 0) - - # Maybe I should output an error too? - true -> - "" - end - + in_reply_to_id = prepare_in_reply_to(in_reply_to) object = Map.put(object, "inReplyToAtomUri", in_reply_to_id) if Federator.allowed_incoming_reply_depth?(options[:depth]) do - case get_obj_helper(in_reply_to_id, options) do - {:ok, replied_object} -> - with %Activity{} = _activity <- - Activity.get_create_by_object_ap_id(replied_object.data["id"]) do - object - |> Map.put("inReplyTo", replied_object.data["id"]) - |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id) - |> Map.put("conversation", replied_object.data["context"] || object["conversation"]) - |> Map.put("context", replied_object.data["context"] || object["conversation"]) - else - e -> - Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") - object - end - + with {:ok, replied_object} <- get_obj_helper(in_reply_to_id, options), + %Activity{} = _ <- Activity.get_create_by_object_ap_id(replied_object.data["id"]) do + object + |> Map.put("inReplyTo", replied_object.data["id"]) + |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id) + |> Map.put("conversation", replied_object.data["context"] || object["conversation"]) + |> Map.put("context", replied_object.data["context"] || object["conversation"]) + else e -> Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") object @@ -201,6 +176,22 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_in_reply_to(object, _options), do: object + defp prepare_in_reply_to(in_reply_to) do + cond do + is_bitstring(in_reply_to) -> + in_reply_to + + is_map(in_reply_to) && is_bitstring(in_reply_to["id"]) -> + in_reply_to["id"] + + is_list(in_reply_to) && is_bitstring(Enum.at(in_reply_to, 0)) -> + Enum.at(in_reply_to, 0) + + true -> + "" + end + end + def fix_context(object) do context = object["context"] || object["conversation"] || Utils.generate_context_id() @@ -211,11 +202,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do attachments = - attachment - |> Enum.map(fn data -> + Enum.map(attachment, fn data -> media_type = data["mediaType"] || data["mimeType"] href = data["url"] || data["href"] - url = [%{"type" => "Link", "mediaType" => media_type, "href" => href}] data @@ -223,30 +212,25 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put("url", url) end) - object - |> Map.put("attachment", attachments) + Map.put(object, "attachment", attachments) end def fix_attachments(%{"attachment" => attachment} = object) when is_map(attachment) do - Map.put(object, "attachment", [attachment]) + object + |> Map.put("attachment", [attachment]) |> fix_attachments() end def fix_attachments(object), do: object def fix_url(%{"url" => url} = object) when is_map(url) do - object - |> Map.put("url", url["href"]) + Map.put(object, "url", url["href"]) end def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do first_element = Enum.at(url, 0) - link_element = - url - |> Enum.filter(fn x -> is_map(x) end) - |> Enum.filter(fn x -> x["mimeType"] == "text/html" end) - |> Enum.at(0) + link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end) object |> Map.put("attachment", [first_element]) @@ -264,36 +248,32 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do true -> "" end - object - |> Map.put("url", url_string) + Map.put(object, "url", url_string) end def fix_url(object), do: object def fix_emoji(%{"tag" => tags} = object) when is_list(tags) do - emoji = tags |> Enum.filter(fn data -> data["type"] == "Emoji" and data["icon"] end) - emoji = - emoji + tags + |> Enum.filter(fn data -> data["type"] == "Emoji" and data["icon"] end) |> Enum.reduce(%{}, fn data, mapping -> name = String.trim(data["name"], ":") - mapping |> Map.put(name, data["icon"]["url"]) + Map.put(mapping, name, data["icon"]["url"]) end) # we merge mastodon and pleroma emoji into a single mapping, to allow for both wire formats emoji = Map.merge(object["emoji"] || %{}, emoji) - object - |> Map.put("emoji", emoji) + Map.put(object, "emoji", emoji) end def fix_emoji(%{"tag" => %{"type" => "Emoji"} = tag} = object) do name = String.trim(tag["name"], ":") emoji = %{name => tag["icon"]["url"]} - object - |> Map.put("emoji", emoji) + Map.put(object, "emoji", emoji) end def fix_emoji(object), do: object @@ -304,17 +284,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end) |> Enum.map(fn data -> String.slice(data["name"], 1..-1) end) - combined = tag ++ tags - - object - |> Map.put("tag", combined) + Map.put(object, "tag", tag ++ tags) end def fix_tag(%{"tag" => %{"type" => "Hashtag", "name" => hashtag} = tag} = object) do combined = [tag, String.slice(hashtag, 1..-1)] - object - |> Map.put("tag", combined) + Map.put(object, "tag", combined) end def fix_tag(%{"tag" => %{} = tag} = object), do: Map.put(object, "tag", [tag]) @@ -326,8 +302,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do content_groups = Map.to_list(content_map) {_, content} = Enum.at(content_groups, 0) - object - |> Map.put("content", content) + Map.put(object, "content", content) end def fix_content_map(object), do: object @@ -336,16 +311,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options) when is_binary(reply_id) do - reply = - with true <- Federator.allowed_incoming_reply_depth?(options[:depth]), - {:ok, object} <- get_obj_helper(reply_id, options) do - object - end - - if reply && reply.data["type"] == "Question" do + with true <- Federator.allowed_incoming_reply_depth?(options[:depth]), + {:ok, %{data: %{"type" => "Question"} = _} = _} <- get_obj_helper(reply_id, options) do Map.put(object, "type", "Answer") else - object + _ -> object end end @@ -377,6 +347,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end + # Reduce the object list to find the reported user. + defp get_reported(objects) do + Enum.reduce_while(objects, nil, fn ap_id, _ -> + with %User{} = user <- User.get_cached_by_ap_id(ap_id) do + {:halt, user} + else + _ -> {:cont, nil} + end + end) + end + def handle_incoming(data, options \\ []) # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them @@ -385,31 +366,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do with context <- data["context"] || Utils.generate_context_id(), content <- data["content"] || "", %User{} = actor <- User.get_cached_by_ap_id(actor), - # Reduce the object list to find the reported user. - %User{} = account <- - Enum.reduce_while(objects, nil, fn ap_id, _ -> - with %User{} = user <- User.get_cached_by_ap_id(ap_id) do - {:halt, user} - else - _ -> {:cont, nil} - end - end), - + %User{} = account <- get_reported(objects), # Remove the reported user from the object list. statuses <- Enum.filter(objects, fn ap_id -> ap_id != account.ap_id end) do - params = %{ + %{ actor: actor, context: context, account: account, statuses: statuses, content: content, - additional: %{ - "cc" => [account.ap_id] - } + additional: %{"cc" => [account.ap_id]} } - - ActivityPub.flag(params) + |> ActivityPub.flag() end end @@ -756,8 +725,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming(_, _), do: :error + @spec get_obj_helper(String.t(), Keyword.t()) :: {:ok, Object.t()} | nil def get_obj_helper(id, options \\ []) do - if object = Object.normalize(id, true, options), do: {:ok, object}, else: nil + case Object.normalize(id, true, options) do + %Object{} = object -> {:ok, object} + _ -> nil + end end def set_reply_to_uri(%{"inReplyTo" => in_reply_to} = object) when is_binary(in_reply_to) do @@ -856,27 +829,24 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do {:ok, data} end - def maybe_fix_object_url(data) do - if is_binary(data["object"]) and not String.starts_with?(data["object"], "http") do - case get_obj_helper(data["object"]) do - {:ok, relative_object} -> - if relative_object.data["external_url"] do - _data = - data - |> Map.put("object", relative_object.data["external_url"]) - else - data - end - - e -> - Logger.error("Couldn't fetch #{data["object"]} #{inspect(e)}") - data - end + def maybe_fix_object_url(%{"object" => object} = data) when is_binary(object) do + with false <- String.starts_with?(object, "http"), + {:fetch, {:ok, relative_object}} <- {:fetch, get_obj_helper(object)}, + %{data: %{"external_url" => external_url}} when not is_nil(external_url) <- + relative_object do + Map.put(data, "object", external_url) else - data + {:fetch, e} -> + Logger.error("Couldn't fetch #{object} #{inspect(e)}") + data + + _ -> + data end end + def maybe_fix_object_url(data), do: data + def add_hashtags(object) do tags = (object["tag"] || []) @@ -894,53 +864,49 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do tag end) - object - |> Map.put("tag", tags) + Map.put(object, "tag", tags) end def add_mention_tags(object) do mentions = object |> Utils.get_notified_from_object() - |> Enum.map(fn user -> - %{"type" => "Mention", "href" => user.ap_id, "name" => "@#{user.nickname}"} - end) + |> Enum.map(&build_mention_tag/1) tags = object["tag"] || [] - object - |> Map.put("tag", tags ++ mentions) + Map.put(object, "tag", tags ++ mentions) end - def add_emoji_tags(%User{info: %{"emoji" => _emoji} = user_info} = object) do - user_info = add_emoji_tags(user_info) + defp build_mention_tag(%{ap_id: ap_id, nickname: nickname} = _) do + %{"type" => "Mention", "href" => ap_id, "name" => "@#{nickname}"} + end - object - |> Map.put(:info, user_info) + def take_emoji_tags(%User{info: %{emoji: emoji} = _user_info} = _user) do + emoji + |> Enum.flat_map(&Map.to_list/1) + |> Enum.map(&build_emoji_tag/1) end # TODO: we should probably send mtime instead of unix epoch time for updated def add_emoji_tags(%{"emoji" => emoji} = object) do tags = object["tag"] || [] - out = - emoji - |> Enum.map(fn {name, url} -> - %{ - "icon" => %{"url" => url, "type" => "Image"}, - "name" => ":" <> name <> ":", - "type" => "Emoji", - "updated" => "1970-01-01T00:00:00Z", - "id" => url - } - end) + out = Enum.map(emoji, &build_emoji_tag/1) - object - |> Map.put("tag", tags ++ out) + Map.put(object, "tag", tags ++ out) end - def add_emoji_tags(object) do - object + def add_emoji_tags(object), do: object + + defp build_emoji_tag({name, url}) do + %{ + "icon" => %{"url" => url, "type" => "Image"}, + "name" => ":" <> name <> ":", + "type" => "Emoji", + "updated" => "1970-01-01T00:00:00Z", + "id" => url + } end def set_conversation(object) do @@ -960,9 +926,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def add_attributed_to(object) do attributed_to = object["attributedTo"] || object["actor"] - - object - |> Map.put("attributedTo", attributed_to) + Map.put(object, "attributedTo", attributed_to) end def prepare_attachments(object) do @@ -973,8 +937,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do %{"url" => href, "mediaType" => media_type, "name" => data["name"], "type" => "Document"} end) - object - |> Map.put("attachment", attachments) + Map.put(object, "attachment", attachments) end defp strip_internal_fields(object) do @@ -983,12 +946,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end defp strip_internal_tags(%{"tag" => tags} = object) do - tags = - tags - |> Enum.filter(fn x -> is_map(x) end) + tags = Enum.filter(tags, fn x -> is_map(x) end) - object - |> Map.put("tag", tags) + Map.put(object, "tag", tags) end defp strip_internal_tags(object), do: object @@ -1073,16 +1033,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def maybe_fix_user_url(data) do - if is_map(data["url"]) do - Map.put(data, "url", data["url"]["href"]) - else - data - end + def maybe_fix_user_url(%{"url" => url} = data) when is_map(url) do + Map.put(data, "url", url["href"]) end - def maybe_fix_user_object(data) do - data - |> maybe_fix_user_url - end + def maybe_fix_user_url(data), do: data + + def maybe_fix_user_object(data), do: maybe_fix_user_url(data) end diff --git a/lib/pleroma/web/activity_pub/views/object_view.ex b/lib/pleroma/web/activity_pub/views/object_view.ex index 94d05f49b..0d63f0707 100644 --- a/lib/pleroma/web/activity_pub/views/object_view.ex +++ b/lib/pleroma/web/activity_pub/views/object_view.ex @@ -37,12 +37,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do Map.merge(base, additional) end - def render("likes.json", ap_id, likes, page) do + def render("likes.json", %{ap_id: ap_id, likes: likes, page: page}) do collection(likes, "#{ap_id}/likes", page) |> Map.merge(Pleroma.Web.ActivityPub.Utils.make_json_ld_header()) end - def render("likes.json", ap_id, likes) do + def render("likes.json", %{ap_id: ap_id, likes: likes}) do %{ "id" => "#{ap_id}/likes", "type" => "OrderedCollection", diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index a2f73e140..352d856fa 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -75,10 +75,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do endpoints = render("endpoints.json", %{user: user}) - user_tags = - user - |> Transmogrifier.add_emoji_tags() - |> Map.get("tag", []) + emoji_tags = Transmogrifier.take_emoji_tags(user) fields = user.info @@ -110,7 +107,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do }, "endpoints" => endpoints, "attachment" => fields, - "tag" => (user.info.source_data["tag"] || []) ++ user_tags + "tag" => (user.info.source_data["tag"] || []) ++ emoji_tags } |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user)) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 460188390..6b6d124a5 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do alias Pleroma.Web.AdminAPI.Config alias Pleroma.Web.AdminAPI.ConfigView alias Pleroma.Web.AdminAPI.ModerationLogView + alias Pleroma.Web.AdminAPI.Report alias Pleroma.Web.AdminAPI.ReportView alias Pleroma.Web.AdminAPI.Search alias Pleroma.Web.CommonAPI @@ -141,7 +142,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do def user_show(conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do conn - |> json(AccountView.render("show.json", %{user: user})) + |> put_view(AccountView) + |> render("show.json", %{user: user}) else _ -> {:error, :not_found} end @@ -160,7 +162,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do }) conn - |> json(StatusView.render("index.json", %{activities: activities, as: :activity})) + |> put_view(StatusView) + |> render("index.json", %{activities: activities, as: :activity}) else _ -> {:error, :not_found} end @@ -180,7 +183,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do }) conn - |> json(AccountView.render("show.json", %{user: updated_user})) + |> put_view(AccountView) + |> render("show.json", %{user: updated_user}) end def tag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do @@ -426,7 +430,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do invites = UserInviteToken.list_invites() conn - |> json(AccountView.render("invites.json", %{invites: invites})) + |> put_view(AccountView) + |> render("invites.json", %{invites: invites}) end @doc "Revokes invite by token" @@ -434,7 +439,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do with {:ok, invite} <- UserInviteToken.find_by_token(token), {:ok, updated_invite} = UserInviteToken.update_invite(invite, %{used: true}) do conn - |> json(AccountView.render("invite.json", %{invite: updated_invite})) + |> put_view(AccountView) + |> render("invite.json", %{invite: updated_invite}) else nil -> {:error, :not_found} end @@ -452,6 +458,15 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do }) end + @doc "Force password reset for a given user" + def force_password_reset(conn, %{"nickname" => nickname}) do + (%User{local: true} = user) = User.get_cached_by_nickname(nickname) + + User.force_password_reset_async(user) + + json_response(conn, :no_content, "") + end + def list_reports(conn, params) do params = params @@ -470,7 +485,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do with %Activity{} = report <- Activity.get_by_id(id) do conn |> put_view(ReportView) - |> render("show.json", %{report: report}) + |> render("show.json", Report.extract_report_info(report)) else _ -> {:error, :not_found} end @@ -486,7 +501,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do conn |> put_view(ReportView) - |> render("show.json", %{report: report}) + |> render("show.json", Report.extract_report_info(report)) end end @@ -604,6 +619,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> render("index.json", %{configs: updated}) end + def reload_emoji(conn, _params) do + Pleroma.Emoji.reload() + + conn |> json("ok") + end + def errors(conn, {:error, :not_found}) do conn |> put_status(:not_found) diff --git a/lib/pleroma/web/admin_api/report.ex b/lib/pleroma/web/admin_api/report.ex new file mode 100644 index 000000000..c751dc2be --- /dev/null +++ b/lib/pleroma/web/admin_api/report.ex @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.Report do + alias Pleroma.Activity + alias Pleroma.User + + def extract_report_info( + %{data: %{"actor" => actor, "object" => [account_ap_id | status_ap_ids]}} = report + ) do + user = User.get_cached_by_ap_id(actor) + account = User.get_cached_by_ap_id(account_ap_id) + + statuses = + Enum.map(status_ap_ids, fn ap_id -> + Activity.get_by_ap_id_with_object(ap_id) + end) + + %{report: report, user: user, account: account, statuses: statuses} + end +end diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index 51b95ad5e..8c06364a3 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -4,27 +4,26 @@ defmodule Pleroma.Web.AdminAPI.ReportView do use Pleroma.Web, :view - alias Pleroma.Activity alias Pleroma.HTML alias Pleroma.User + alias Pleroma.Web.AdminAPI.Report alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.StatusView def render("index.json", %{reports: reports}) do %{ reports: - render_many(reports[:items], __MODULE__, "show.json", as: :report) |> Enum.reverse(), + reports[:items] + |> Enum.map(&Report.extract_report_info(&1)) + |> Enum.map(&render(__MODULE__, "show.json", &1)) + |> Enum.reverse(), total: reports[:total] } end - def render("show.json", %{report: report}) do - user = User.get_cached_by_ap_id(report.data["actor"]) + def render("show.json", %{report: report, user: user, account: account, statuses: statuses}) do created_at = Utils.to_masto_date(report.data["published"]) - [account_ap_id | status_ap_ids] = report.data["object"] - account = User.get_cached_by_ap_id(account_ap_id) - content = unless is_nil(report.data["content"]) do HTML.filter_tags(report.data["content"]) @@ -32,11 +31,6 @@ defmodule Pleroma.Web.AdminAPI.ReportView do nil end - statuses = - Enum.map(status_ap_ids, fn ap_id -> - Activity.get_by_ap_id_with_object(ap_id) - end) - %{ id: report.id, account: merge_account_views(account), diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 6704ee7e8..6421c2c53 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -381,7 +381,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> Map.put("local_only", local_only) |> Map.put("blocking_user", user) |> Map.put("muting_user", user) - |> Map.put("user", user) |> ActivityPub.fetch_public_activities() |> Enum.reverse() diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 9072aa7a4..c91713773 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -19,9 +19,10 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do def account_search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do accounts = User.search(query, search_options(params, user)) - res = AccountView.render("accounts.json", users: accounts, for: user, as: :user) - json(conn, res) + conn + |> put_view(AccountView) + |> render("accounts.json", users: accounts, for: user, as: :user) end def search2(conn, params), do: do_search(:v2, conn, params) diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index ee14cfd6b..192984242 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -57,6 +57,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do "mastodon_api_streaming", "polls", "pleroma_explicit_addressing", + "shareable_emoji_packs", if Config.get([:media_proxy, :enabled]) do "media_proxy" end, diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 81eae2c8b..a57670e02 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -202,6 +202,8 @@ defmodule Pleroma.Web.OAuth.OAuthController do {:ok, app} <- Token.Utils.fetch_app(conn), {:auth_active, true} <- {:auth_active, User.auth_active?(user)}, {:user_active, true} <- {:user_active, !user.info.deactivated}, + {:password_reset_pending, false} <- + {:password_reset_pending, user.info.password_reset_pending}, {:ok, scopes} <- validate_scopes(app, params), {:ok, auth} <- Authorization.create_authorization(app, user, scopes), {:ok, token} <- Token.exchange_token(app, auth) do @@ -215,6 +217,9 @@ defmodule Pleroma.Web.OAuth.OAuthController do {:user_active, false} -> render_error(conn, :forbidden, "Your account is currently disabled") + {:password_reset_pending, true} -> + render_error(conn, :forbidden, "Password reset is required") + _error -> render_invalid_credentials_error(conn) end diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 64b2c64b3..8f325b28e 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -216,7 +216,8 @@ defmodule Pleroma.Web.OStatus.OStatusController do conn |> put_resp_header("content-type", "application/activity+json") - |> json(ObjectView.render("object.json", %{object: object})) + |> put_view(ObjectView) + |> render("object.json", %{object: object}) end defp represent_activity(_conn, "activity+json", _, _) do diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex new file mode 100644 index 000000000..3ad29bd38 --- /dev/null +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -0,0 +1,575 @@ +defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do + use Pleroma.Web, :controller + + require Logger + + @emoji_dir_path Path.join( + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ) + + @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + + @doc """ + Lists the packs available on the instance as JSON. + + The information is public and does not require authentification. The format is + a map of "pack directory name" to pack.json contents. + """ + def list_packs(conn, _params) do + with {:ok, results} <- File.ls(@emoji_dir_path) do + pack_infos = + results + |> Enum.filter(&has_pack_json?/1) + |> Enum.map(&load_pack/1) + # Check if all the files are in place and can be sent + |> Enum.map(&validate_pack/1) + # Transform into a map of pack-name => pack-data + |> Enum.into(%{}) + + json(conn, pack_infos) + end + end + + defp has_pack_json?(file) do + dir_path = Path.join(@emoji_dir_path, file) + # Filter to only use the pack.json packs + File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) + end + + defp load_pack(pack_name) do + pack_path = Path.join(@emoji_dir_path, pack_name) + pack_file = Path.join(pack_path, "pack.json") + + {pack_name, Jason.decode!(File.read!(pack_file))} + end + + defp validate_pack({name, pack}) do + pack_path = Path.join(@emoji_dir_path, name) + + if can_download?(pack, pack_path) do + archive_for_sha = make_archive(name, pack, pack_path) + archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() + + pack = + pack + |> put_in(["pack", "can-download"], true) + |> put_in(["pack", "download-sha256"], archive_sha) + + {name, pack} + else + {name, put_in(pack, ["pack", "can-download"], false)} + end + end + + defp can_download?(pack, pack_path) do + # If the pack is set as shared, check if it can be downloaded + # That means that when asked, the pack can be packed and sent to the remote + # Otherwise, they'd have to download it from external-src + pack["pack"]["share-files"] && + Enum.all?(pack["files"], fn {_, path} -> + File.exists?(Path.join(pack_path, path)) + end) + end + + defp create_archive_and_cache(name, pack, pack_dir, md5) do + files = + ['pack.json'] ++ + (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) + + {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) + + cache_ms = :timer.seconds(@cache_seconds_per_file * Enum.count(files)) + + Cachex.put!( + :emoji_packs_cache, + name, + # if pack.json MD5 changes, the cache is not valid anymore + %{pack_json_md5: md5, pack_data: zip_result}, + # Add a minute to cache time for every file in the pack + ttl: cache_ms + ) + + Logger.debug("Created an archive for the '#{name}' emoji pack, \ +keeping it in cache for #{div(cache_ms, 1000)}s") + + zip_result + end + + defp make_archive(name, pack, pack_dir) do + # Having a different pack.json md5 invalidates cache + pack_file_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.json"))) + + case Cachex.get!(:emoji_packs_cache, name) do + %{pack_file_md5: ^pack_file_md5, pack_data: zip_result} -> + Logger.debug("Using cache for the '#{name}' shared emoji pack") + zip_result + + _ -> + create_archive_and_cache(name, pack, pack_dir, pack_file_md5) + end + end + + @doc """ + An endpoint for other instances (via admin UI) or users (via browser) + to download packs that the instance shares. + """ + def download_shared(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + pack_file = Path.join(pack_dir, "pack.json") + + with {_, true} <- {:exists?, File.exists?(pack_file)}, + pack = Jason.decode!(File.read!(pack_file)), + {_, true} <- {:can_download?, can_download?(pack, pack_dir)} do + zip_result = make_archive(name, pack, pack_dir) + send_download(conn, {:binary, zip_result}, filename: "#{name}.zip") + else + {:can_download?, _} -> + conn + |> put_status(:forbidden) + |> json(%{ + error: "Pack #{name} cannot be downloaded from this instance, either pack sharing\ + was disabled for this pack or some files are missing" + }) + + {:exists?, _} -> + conn + |> put_status(:not_found) + |> json(%{error: "Pack #{name} does not exist"}) + end + end + + @doc """ + An admin endpoint to request downloading a pack named `pack_name` from the instance + `instance_address`. + + If the requested instance's admin chose to share the pack, it will be downloaded + from that instance, otherwise it will be downloaded from the fallback source, if there is one. + """ + def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do + shareable_packs_available = + "#{address}/.well-known/nodeinfo" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> List.last() + |> Map.get("href") + # Get the actual nodeinfo address and fetch it + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> get_in(["metadata", "features"]) + |> Enum.member?("shareable_emoji_packs") + + if shareable_packs_available do + full_pack = + "#{address}/api/pleroma/emoji/packs/list" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> Map.get(name) + + pack_info_res = + case full_pack["pack"] do + %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> + {:ok, + %{ + sha: sha, + uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" + }} + + %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> + {:ok, + %{ + sha: sha, + uri: src, + fallback: true + }} + + _ -> + {:error, + "The pack was not set as shared and there is no fallback src to download from"} + end + + with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, + %{body: emoji_archive} <- Tesla.get!(uri), + {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do + local_name = data["as"] || name + pack_dir = Path.join(@emoji_dir_path, local_name) + File.mkdir_p!(pack_dir) + + files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) + # Fallback cannot contain a pack.json file + files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files + + {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) + + # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 + # in it to depend on itself + if pinfo[:fallback] do + pack_file_path = Path.join(pack_dir, "pack.json") + + File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) + end + + json(conn, "ok") + else + {:error, e} -> + conn |> put_status(:internal_server_error) |> json(%{error: e}) + + {:checksum, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) + end + else + conn + |> put_status(:internal_server_error) + |> json(%{error: "The requested instance does not support sharing emoji packs"}) + end + end + + @doc """ + Creates an empty pack named `name` which then can be updated via the admin UI. + """ + def create(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + + if not File.exists?(pack_dir) do + File.mkdir_p!(pack_dir) + + pack_file_p = Path.join(pack_dir, "pack.json") + + File.write!( + pack_file_p, + Jason.encode!(%{pack: %{}, files: %{}}, pretty: true) + ) + + conn |> json("ok") + else + conn + |> put_status(:conflict) + |> json(%{error: "A pack named \"#{name}\" already exists"}) + end + end + + @doc """ + Deletes the pack `name` and all it's files. + """ + def delete(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + + case File.rm_rf(pack_dir) do + {:ok, _} -> + conn |> json("ok") + + {:error, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "Couldn't delete the pack #{name}"}) + end + end + + @doc """ + An endpoint to update `pack_names`'s metadata. + + `new_data` is the new metadata for the pack, that will replace the old metadata. + """ + def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do + pack_file_p = Path.join([@emoji_dir_path, name, "pack.json"]) + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + # The new fallback-src is in the new data and it's not the same as it was in the old data + should_update_fb_sha = + not is_nil(new_data["fallback-src"]) and + new_data["fallback-src"] != full_pack["pack"]["fallback-src"] + + with {_, true} <- {:should_update?, should_update_fb_sha}, + %{body: pack_arch} <- Tesla.get!(new_data["fallback-src"]), + {:ok, flist} <- :zip.unzip(pack_arch, [:memory]), + {_, true} <- {:has_all_files?, has_all_files?(full_pack, flist)} do + fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16() + + new_data = Map.put(new_data, "fallback-src-sha256", fallback_sha) + update_metadata_and_send(conn, full_pack, new_data, pack_file_p) + else + {:should_update?, _} -> + update_metadata_and_send(conn, full_pack, new_data, pack_file_p) + + {:has_all_files?, _} -> + conn + |> put_status(:bad_request) + |> json(%{error: "The fallback archive does not have all files specified in pack.json"}) + end + end + + # Check if all files from the pack.json are in the archive + defp has_all_files?(%{"files" => files}, flist) do + Enum.all?(files, fn {_, from_manifest} -> + Enum.find(flist, fn {from_archive, _} -> + to_string(from_archive) == from_manifest + end) + end) + end + + defp update_metadata_and_send(conn, full_pack, new_data, pack_file_p) do + full_pack = Map.put(full_pack, "pack", new_data) + File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true)) + + # Send new data back with fallback sha filled + json(conn, new_data) + end + + defp get_filename(%{"filename" => filename}), do: filename + + defp get_filename(%{"file" => file}) do + case file do + %Plug.Upload{filename: filename} -> filename + url when is_binary(url) -> Path.basename(url) + end + end + + defp empty?(str), do: String.trim(str) == "" + + defp update_file_and_send(conn, updated_full_pack, pack_file_p) do + # Write the emoji pack file + File.write!(pack_file_p, Jason.encode!(updated_full_pack, pretty: true)) + + # Return the modified file list + json(conn, updated_full_pack["files"]) + end + + @doc """ + Updates a file in a pack. + + Updating can mean three things: + + - `add` adds an emoji named `shortcode` to the pack `pack_name`, + that means that the emoji file needs to be uploaded with the request + (thus requiring it to be a multipart request) and be named `file`. + There can also be an optional `filename` that will be the new emoji file name + (if it's not there, the name will be taken from the uploaded file). + - `update` changes emoji shortcode (from `shortcode` to `new_shortcode` or moves the file + (from the current filename to `new_filename`) + - `remove` removes the emoji named `shortcode` and it's associated file + """ + + # Add + def update_file( + conn, + %{"pack_name" => pack_name, "action" => "add", "shortcode" => shortcode} = params + ) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + with {_, false} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, + filename <- get_filename(params), + false <- empty?(shortcode), + false <- empty?(filename) do + file_path = Path.join(pack_dir, filename) + + # If the name contains directories, create them + if String.contains?(file_path, "/") do + File.mkdir_p!(Path.dirname(file_path)) + end + + case params["file"] do + %Plug.Upload{path: upload_path} -> + # Copy the uploaded file from the temporary directory + File.copy!(upload_path, file_path) + + url when is_binary(url) -> + # Download and write the file + file_contents = Tesla.get!(url).body + File.write!(file_path, file_contents) + end + + updated_full_pack = put_in(full_pack, ["files", shortcode], filename) + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + {:has_shortcode, _} -> + conn + |> put_status(:conflict) + |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"}) + + true -> + conn + |> put_status(:bad_request) + |> json(%{error: "shortcode or filename cannot be empty"}) + end + end + + # Remove + def update_file(conn, %{ + "pack_name" => pack_name, + "action" => "remove", + "shortcode" => shortcode + }) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + if Map.has_key?(full_pack["files"], shortcode) do + {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + + emoji_file_path = Path.join(pack_dir, emoji_file_path) + + # Delete the emoji file + File.rm!(emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(emoji_file_path, "/") do + dir = Path.dirname(emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + conn + |> put_status(:bad_request) + |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) + end + end + + # Update + def update_file( + conn, + %{"pack_name" => pack_name, "action" => "update", "shortcode" => shortcode} = params + ) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + with {_, true} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, + %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params, + false <- empty?(new_shortcode), + false <- empty?(new_filename) do + # First, remove the old shortcode, saving the old path + {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) + new_emoji_file_path = Path.join(pack_dir, new_filename) + + # If the name contains directories, create them + if String.contains?(new_emoji_file_path, "/") do + File.mkdir_p!(Path.dirname(new_emoji_file_path)) + end + + # Move/Rename the old filename to a new filename + # These are probably on the same filesystem, so just rename should work + :ok = File.rename(old_emoji_file_path, new_emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(old_emoji_file_path, "/") do + dir = Path.dirname(old_emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + # Then, put in the new shortcode with the new path + updated_full_pack = put_in(updated_full_pack, ["files", new_shortcode], new_filename) + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + {:has_shortcode, _} -> + conn + |> put_status(:bad_request) + |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) + + true -> + conn + |> put_status(:bad_request) + |> json(%{error: "new_shortcode or new_filename cannot be empty"}) + + _ -> + conn + |> put_status(:bad_request) + |> json(%{error: "new_shortcode or new_file were not specified"}) + end + end + + def update_file(conn, %{"action" => action}) do + conn + |> put_status(:bad_request) + |> json(%{error: "Unknown action: #{action}"}) + end + + @doc """ + Imports emoji from the filesystem. + + Importing means checking all the directories in the + `$instance_static/emoji/` for directories which do not have + `pack.json`. If one has an emoji.txt file, that file will be used + to create a `pack.json` file with it's contents. If the directory has + neither, all the files with specific configured extenstions will be + assumed to be emojis and stored in the new `pack.json` file. + """ + def import_from_fs(conn, _params) do + with {:ok, results} <- File.ls(@emoji_dir_path) do + imported_pack_names = + results + |> Enum.filter(fn file -> + dir_path = Path.join(@emoji_dir_path, file) + # Find the directories that do NOT have pack.json + File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) + end) + |> Enum.map(&write_pack_json_contents/1) + + json(conn, imported_pack_names) + else + {:error, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "Error accessing emoji pack directory"}) + end + end + + defp write_pack_json_contents(dir) do + dir_path = Path.join(@emoji_dir_path, dir) + emoji_txt_path = Path.join(dir_path, "emoji.txt") + + files_for_pack = files_for_pack(emoji_txt_path, dir_path) + pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack}) + + File.write!(Path.join(dir_path, "pack.json"), pack_json_contents) + + dir + end + + defp files_for_pack(emoji_txt_path, dir_path) do + if File.exists?(emoji_txt_path) do + # There's an emoji.txt file, it's likely from a pack installed by the pack manager. + # Make a pack.json file from the contents of that emoji.txt fileh + + # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2 + + # Create a map of shortcodes to filenames from emoji.txt + File.read!(emoji_txt_path) + |> String.split("\n") + |> Enum.map(&String.trim/1) + |> Enum.map(fn line -> + case String.split(line, ~r/,\s*/) do + # This matches both strings with and without tags + # and we don't care about tags here + [name, file | _] -> {name, file} + _ -> nil + end + end) + |> Enum.filter(fn x -> not is_nil(x) end) + |> Enum.into(%{}) + else + # If there's no emoji.txt, assume all files + # that are of certain extensions from the config are emojis and import them all + pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions]) + Pleroma.Emoji.make_shortcode_to_file_map(dir_path, pack_extensions) + end + end +end diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex index d17ccf84d..d17ccf84d 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index b9b85fd67..e583093d2 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -186,6 +186,7 @@ defmodule Pleroma.Web.Router do post("/users/email_invite", AdminAPIController, :email_invite) get("/users/:nickname/password_reset", AdminAPIController, :get_password_reset) + patch("/users/:nickname/force_password_reset", AdminAPIController, :force_password_reset) get("/users", AdminAPIController, :list_users) get("/users/:nickname", AdminAPIController, :user_show) @@ -205,6 +206,29 @@ defmodule Pleroma.Web.Router do get("/config/migrate_from_db", AdminAPIController, :migrate_from_db) get("/moderation_log", AdminAPIController, :list_log) + + post("/reload_emoji", AdminAPIController, :reload_emoji) + end + + scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do + scope "/packs" do + # Modifying packs + pipe_through([:admin_api, :oauth_write]) + + post("/import_from_fs", EmojiAPIController, :import_from_fs) + + post("/:pack_name/update_file", EmojiAPIController, :update_file) + post("/:pack_name/update_metadata", EmojiAPIController, :update_metadata) + put("/:name", EmojiAPIController, :create) + delete("/:name", EmojiAPIController, :delete) + post("/download_from", EmojiAPIController, :download_from) + end + + scope "/packs" do + # Pack info / downloading + get("/", EmojiAPIController, :list_packs) + get("/:name/download_shared/", EmojiAPIController, :download_shared) + end end scope "/", Pleroma.Web.TwitterAPI do diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 082f20ab7..7ffc8eabe 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -26,6 +26,11 @@ defmodule Pleroma.Workers.BackgroundWorker do User.perform(:delete, user) end + def perform(%{"op" => "force_password_reset", "user_id" => user_id}, _job) do + user = User.get_cached_by_id(user_id) + User.perform(:force_password_reset, user) + end + def perform( %{ "op" => "blocks_import", |