diff options
Diffstat (limited to 'lib')
39 files changed, 560 insertions, 626 deletions
diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 238d8dcd9..881a6f725 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -235,7 +235,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do cwd: tmp_pack_dir ) - emoji_map = Pleroma.Emoji.make_shortcode_to_file_map(tmp_pack_dir, exts) + emoji_map = Pleroma.Emoji.Loader.make_shortcode_to_file_map(tmp_pack_dir, exts) File.write!(files_name, Jason.encode!(emoji_map, pretty: true)) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index ec558168a..2c04a26f9 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Activity do @type t :: %__MODULE__{} @type actor :: String.t() - @primary_key {:id, Pleroma.FlakeId, autogenerate: true} + @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true} # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19 @mastodon_notification_types %{ diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex index bf57abca4..7ea5c48ca 100644 --- a/lib/pleroma/activity_expiration.ex +++ b/lib/pleroma/activity_expiration.ex @@ -7,7 +7,6 @@ defmodule Pleroma.ActivityExpiration do alias Pleroma.Activity alias Pleroma.ActivityExpiration - alias Pleroma.FlakeId alias Pleroma.Repo import Ecto.Changeset @@ -17,7 +16,7 @@ defmodule Pleroma.ActivityExpiration do @min_activity_lifetime :timer.hours(1) schema "activity_expirations" do - belongs_to(:activity, Activity, type: FlakeId) + belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType) field(:scheduled_at, :naive_datetime) end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index a339e2c48..7aec2c545 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -35,7 +35,6 @@ defmodule Pleroma.Application do Pleroma.Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, - Pleroma.FlakeId, Pleroma.Daemons.ScheduledActivityDaemon, Pleroma.Daemons.ActivityExpirationDaemon ] ++ diff --git a/lib/pleroma/bookmark.ex b/lib/pleroma/bookmark.ex index d976f949c..221a94f34 100644 --- a/lib/pleroma/bookmark.ex +++ b/lib/pleroma/bookmark.ex @@ -10,20 +10,20 @@ defmodule Pleroma.Bookmark do alias Pleroma.Activity alias Pleroma.Bookmark - alias Pleroma.FlakeId alias Pleroma.Repo alias Pleroma.User @type t :: %__MODULE__{} schema "bookmarks" do - belongs_to(:user, User, type: FlakeId) - belongs_to(:activity, Activity, type: FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) + belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType) timestamps() end - @spec create(FlakeId.t(), FlakeId.t()) :: {:ok, Bookmark.t()} | {:error, Changeset.t()} + @spec create(FlakeId.Ecto.CompatType.t(), FlakeId.Ecto.CompatType.t()) :: + {:ok, Bookmark.t()} | {:error, Changeset.t()} def create(user_id, activity_id) do attrs = %{ user_id: user_id, @@ -37,7 +37,7 @@ defmodule Pleroma.Bookmark do |> Repo.insert() end - @spec for_user_query(FlakeId.t()) :: Ecto.Query.t() + @spec for_user_query(FlakeId.Ecto.CompatType.t()) :: Ecto.Query.t() def for_user_query(user_id) do Bookmark |> where(user_id: ^user_id) @@ -52,7 +52,8 @@ defmodule Pleroma.Bookmark do |> Repo.one() end - @spec destroy(FlakeId.t(), FlakeId.t()) :: {:ok, Bookmark.t()} | {:error, Changeset.t()} + @spec destroy(FlakeId.Ecto.CompatType.t(), FlakeId.Ecto.CompatType.t()) :: + {:ok, Bookmark.t()} | {:error, Changeset.t()} def destroy(user_id, activity_id) do from(b in Bookmark, where: b.user_id == ^user_id, diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index ea5b9fe17..e946f6de2 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -13,10 +13,10 @@ defmodule Pleroma.Conversation.Participation do import Ecto.Query schema "conversation_participations" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:conversation, Conversation) field(:read, :boolean, default: false) - field(:last_activity_id, Pleroma.FlakeId, virtual: true) + field(:last_activity_id, FlakeId.Ecto.CompatType, virtual: true) has_many(:recipient_ships, RecipientShip) has_many(:recipients, through: [:recipient_ships, :user]) diff --git a/lib/pleroma/conversation/participation_recipient_ship.ex b/lib/pleroma/conversation/participation_recipient_ship.ex index 932cbd04c..e3d158cbc 100644 --- a/lib/pleroma/conversation/participation_recipient_ship.ex +++ b/lib/pleroma/conversation/participation_recipient_ship.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Conversation.Participation.RecipientShip do import Ecto.Changeset schema "conversation_participation_recipient_ships" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:participation, Participation) end diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex index 29a1e5a77..1d586a252 100644 --- a/lib/pleroma/delivery.ex +++ b/lib/pleroma/delivery.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Delivery do use Ecto.Schema alias Pleroma.Delivery - alias Pleroma.FlakeId alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -16,7 +15,7 @@ defmodule Pleroma.Delivery do import Ecto.Query schema "deliveries" do - belongs_to(:user, User, type: FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:object, Object) end diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 170a7d098..bafad2ae9 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -4,24 +4,37 @@ defmodule Pleroma.Emoji do @moduledoc """ - The emojis are loaded from: - - * emoji packs in INSTANCE-DIR/emoji - * the files: `config/emoji.txt` and `config/custom_emoji.txt` - * glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder - - This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime. + This GenServer stores in an ETS table the list of the loaded emojis, + and also allows to reload the list at runtime. """ use GenServer - require Logger + alias Pleroma.Emoji.Loader - @type pattern :: Regex.t() | module() | String.t() - @type patterns :: pattern() | [pattern()] - @type group_patterns :: keyword(patterns()) + require Logger @ets __MODULE__.Ets - @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] + @ets_options [ + :ordered_set, + :protected, + :named_table, + {:read_concurrency, true} + ] + + defstruct [:code, :file, :tags, :safe_code, :safe_file] + + @doc "Build emoji struct" + def build({code, file, tags}) do + %__MODULE__{ + code: code, + file: file, + tags: tags, + safe_code: Pleroma.HTML.strip_tags(code), + safe_file: Pleroma.HTML.strip_tags(file) + } + end + + def build({code, file}), do: build({code, file, []}) @doc false def start_link(_) do @@ -44,11 +57,14 @@ defmodule Pleroma.Emoji do end @doc "Returns all the emojos!!" - @spec get_all() :: [{String.t(), String.t()}, ...] + @spec get_all() :: list({String.t(), String.t(), String.t()}) def get_all do :ets.tab2list(@ets) end + @doc "Clear out old emojis" + def clear_all, do: :ets.delete_all_objects(@ets) + @doc false def init(_) do @ets = :ets.new(@ets, @ets_options) @@ -58,13 +74,13 @@ defmodule Pleroma.Emoji do @doc false def handle_cast(:reload, state) do - load() + update_emojis(Loader.load()) {:noreply, state} end @doc false def handle_call(:reload, _from, state) do - load() + update_emojis(Loader.load()) {:reply, :ok, state} end @@ -75,207 +91,11 @@ defmodule Pleroma.Emoji do @doc false def code_change(_old_vsn, state, _extra) do - load() + update_emojis(Loader.load()) {:ok, state} end - defp load do - emoji_dir_path = - Path.join( - Pleroma.Config.get!([:instance, :static_dir]), - "emoji" - ) - - emoji_groups = Pleroma.Config.get([:emoji, :groups]) - - case File.ls(emoji_dir_path) do - {:error, :enoent} -> - # The custom emoji directory doesn't exist, - # don't do anything - nil - - {:error, e} -> - # There was some other error - Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}") - - {:ok, results} -> - grouped = - Enum.group_by(results, fn file -> File.dir?(Path.join(emoji_dir_path, file)) end) - - packs = grouped[true] || [] - files = grouped[false] || [] - - # Print the packs we've found - Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}") - - if not Enum.empty?(files) do - Logger.warn( - "Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{ - Enum.join(files, ", ") - }" - ) - end - - emojis = - Enum.flat_map( - packs, - fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end - ) - - # Clear out old emojis - :ets.delete_all_objects(@ets) - - true = :ets.insert(@ets, emojis) - end - - # Compat thing for old custom emoji handling & default emoji, - # it should run even if there are no emoji packs - shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], []) - - emojis = - (load_from_file("config/emoji.txt", emoji_groups) ++ - load_from_file("config/custom_emoji.txt", emoji_groups) ++ - load_from_globs(shortcode_globs, emoji_groups)) - |> Enum.reject(fn value -> value == nil end) - - true = :ets.insert(@ets, emojis) - - :ok - end - - defp load_pack(pack_dir, emoji_groups) do - pack_name = Path.basename(pack_dir) - - pack_file = Path.join(pack_dir, "pack.json") - - if File.exists?(pack_file) do - contents = Jason.decode!(File.read!(pack_file)) - - contents["files"] - |> Enum.map(fn {name, rel_file} -> - filename = Path.join("/emoji/#{pack_name}", rel_file) - {name, filename, pack_name} - end) - else - # Load from emoji.txt / all files - emoji_txt = Path.join(pack_dir, "emoji.txt") - - if File.exists?(emoji_txt) do - load_from_file(emoji_txt, emoji_groups) - else - extensions = Pleroma.Config.get([:emoji, :pack_extensions]) - - Logger.info( - "No emoji.txt found for pack \"#{pack_name}\", assuming all #{ - Enum.join(extensions, ", ") - } files are emoji" - ) - - make_shortcode_to_file_map(pack_dir, extensions) - |> Enum.map(fn {shortcode, rel_file} -> - filename = Path.join("/emoji/#{pack_name}", rel_file) - - {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} - end) - end - end - end - - def make_shortcode_to_file_map(pack_dir, exts) do - find_all_emoji(pack_dir, exts) - |> Enum.map(&Path.relative_to(&1, pack_dir)) - |> Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end) - |> Enum.into(%{}) - end - - def find_all_emoji(dir, exts) do - Enum.reduce( - File.ls!(dir), - [], - fn f, acc -> - filepath = Path.join(dir, f) - - if File.dir?(filepath) do - acc ++ find_all_emoji(filepath, exts) - else - acc ++ [filepath] - end - end - ) - |> Enum.filter(fn f -> Path.extname(f) in exts end) - end - - defp load_from_file(file, emoji_groups) do - if File.exists?(file) do - load_from_file_stream(File.stream!(file), emoji_groups) - else - [] - end - end - - defp load_from_file_stream(stream, emoji_groups) do - stream - |> Stream.map(&String.trim/1) - |> Stream.map(fn line -> - case String.split(line, ~r/,\s*/) do - [name, file] -> - {name, file, [to_string(match_extra(emoji_groups, file))]} - - [name, file | tags] -> - {name, file, tags} - - _ -> - nil - end - end) - |> Enum.to_list() - end - - defp load_from_globs(globs, emoji_groups) do - static_path = Path.join(:code.priv_dir(:pleroma), "static") - - paths = - Enum.map(globs, fn glob -> - Path.join(static_path, glob) - |> Path.wildcard() - end) - |> Enum.concat() - - Enum.map(paths, fn path -> - tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path))) - shortcode = Path.basename(path, Path.extname(path)) - external_path = Path.join("/", Path.relative_to(path, static_path)) - {shortcode, external_path, [to_string(tag)]} - end) - end - - @doc """ - Finds a matching group for the given emoji filename - """ - @spec match_extra(group_patterns(), String.t()) :: atom() | nil - def match_extra(group_patterns, filename) do - match_group_patterns(group_patterns, fn pattern -> - case pattern do - %Regex{} = regex -> Regex.match?(regex, filename) - string when is_binary(string) -> filename == string - end - end) - end - - defp match_group_patterns(group_patterns, matcher) do - Enum.find_value(group_patterns, fn {group, patterns} -> - patterns = - patterns - |> List.wrap() - |> Enum.map(fn pattern -> - if String.contains?(pattern, "*") do - ~r(#{String.replace(pattern, "*", ".*")}) - else - pattern - end - end) - - Enum.any?(patterns, matcher) && group - end) + defp update_emojis(emojis) do + :ets.insert(@ets, emojis) end end diff --git a/lib/pleroma/emoji/formatter.ex b/lib/pleroma/emoji/formatter.ex new file mode 100644 index 000000000..4869d073e --- /dev/null +++ b/lib/pleroma/emoji/formatter.ex @@ -0,0 +1,59 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Emoji.Formatter do + alias Pleroma.Emoji + alias Pleroma.HTML + alias Pleroma.Web.MediaProxy + + def emojify(text) do + emojify(text, Emoji.get_all()) + end + + def emojify(text, nil), do: text + + def emojify(text, emoji, strip \\ false) do + Enum.reduce(emoji, text, fn + {_, %Emoji{safe_code: emoji, safe_file: file}}, text -> + String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) + + {unsafe_emoji, unsafe_file}, text -> + emoji = HTML.strip_tags(unsafe_emoji) + file = HTML.strip_tags(unsafe_file) + String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) + end) + |> HTML.filter_tags() + end + + defp prepare_emoji_html(_emoji, _file, true), do: "" + + defp prepare_emoji_html(emoji, file, _strip) do + "<img class='emoji' alt='#{emoji}' title='#{emoji}' src='#{MediaProxy.url(file)}' />" + end + + def demojify(text) do + emojify(text, Emoji.get_all(), true) + end + + def demojify(text, nil), do: text + + @doc "Outputs a list of the emoji-shortcodes in a text" + def get_emoji(text) when is_binary(text) do + Enum.filter(Emoji.get_all(), fn {emoji, %Emoji{}} -> + String.contains?(text, ":#{emoji}:") + end) + end + + def get_emoji(_), do: [] + + @doc "Outputs a list of the emoji-Maps in a text" + def get_emoji_map(text) when is_binary(text) do + get_emoji(text) + |> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc -> + Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") + end) + end + + def get_emoji_map(_), do: [] +end diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex new file mode 100644 index 000000000..4f4ee51d1 --- /dev/null +++ b/lib/pleroma/emoji/loader.ex @@ -0,0 +1,224 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Emoji.Loader do + @moduledoc """ + The Loader emoji from: + + * emoji packs in INSTANCE-DIR/emoji + * the files: `config/emoji.txt` and `config/custom_emoji.txt` + * glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder + """ + alias Pleroma.Config + alias Pleroma.Emoji + + require Logger + + @type pattern :: Regex.t() | module() | String.t() + @type patterns :: pattern() | [pattern()] + @type group_patterns :: keyword(patterns()) + @type emoji :: {String.t(), Emoji.t()} + + @doc """ + Loads emojis from files/packs. + + returns list emojis in format: + `{"000", "/emoji/freespeechextremist.com/000.png", ["Custom"]}` + """ + @spec load() :: list(emoji) + def load do + emoji_dir_path = Path.join(Config.get!([:instance, :static_dir]), "emoji") + + emoji_groups = Config.get([:emoji, :groups]) + + emojis = + case File.ls(emoji_dir_path) do + {:error, :enoent} -> + # The custom emoji directory doesn't exist, + # don't do anything + [] + + {:error, e} -> + # There was some other error + Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}") + [] + + {:ok, results} -> + grouped = + Enum.group_by(results, fn file -> + File.dir?(Path.join(emoji_dir_path, file)) + end) + + packs = grouped[true] || [] + files = grouped[false] || [] + + # Print the packs we've found + Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}") + + if not Enum.empty?(files) do + Logger.warn( + "Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{ + Enum.join(files, ", ") + }" + ) + end + + emojis = + Enum.flat_map(packs, fn pack -> + load_pack(Path.join(emoji_dir_path, pack), emoji_groups) + end) + + Emoji.clear_all() + emojis + end + + # Compat thing for old custom emoji handling & default emoji, + # it should run even if there are no emoji packs + shortcode_globs = Config.get([:emoji, :shortcode_globs], []) + + emojis_txt = + (load_from_file("config/emoji.txt", emoji_groups) ++ + load_from_file("config/custom_emoji.txt", emoji_groups) ++ + load_from_globs(shortcode_globs, emoji_groups)) + |> Enum.reject(fn value -> value == nil end) + + Enum.map(emojis ++ emojis_txt, &prepare_emoji/1) + end + + defp prepare_emoji({code, _, _} = emoji), do: {code, Emoji.build(emoji)} + + defp load_pack(pack_dir, emoji_groups) do + pack_name = Path.basename(pack_dir) + + pack_file = Path.join(pack_dir, "pack.json") + + if File.exists?(pack_file) do + contents = Jason.decode!(File.read!(pack_file)) + + contents["files"] + |> Enum.map(fn {name, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) + {name, filename, ["pack:#{pack_name}"]} + end) + else + # Load from emoji.txt / all files + emoji_txt = Path.join(pack_dir, "emoji.txt") + + if File.exists?(emoji_txt) do + load_from_file(emoji_txt, emoji_groups) + else + extensions = Pleroma.Config.get([:emoji, :pack_extensions]) + + Logger.info( + "No emoji.txt found for pack \"#{pack_name}\", assuming all #{ + Enum.join(extensions, ", ") + } files are emoji" + ) + + make_shortcode_to_file_map(pack_dir, extensions) + |> Enum.map(fn {shortcode, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) + + {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} + end) + end + end + end + + def make_shortcode_to_file_map(pack_dir, exts) do + find_all_emoji(pack_dir, exts) + |> Enum.map(&Path.relative_to(&1, pack_dir)) + |> Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end) + |> Enum.into(%{}) + end + + def find_all_emoji(dir, exts) do + dir + |> File.ls!() + |> Enum.flat_map(fn f -> + filepath = Path.join(dir, f) + + if File.dir?(filepath) do + find_all_emoji(filepath, exts) + else + [filepath] + end + end) + |> Enum.filter(fn f -> Path.extname(f) in exts end) + end + + defp load_from_file(file, emoji_groups) do + if File.exists?(file) do + load_from_file_stream(File.stream!(file), emoji_groups) + else + [] + end + end + + defp load_from_file_stream(stream, emoji_groups) do + stream + |> Stream.map(&String.trim/1) + |> Stream.map(fn line -> + case String.split(line, ~r/,\s*/) do + [name, file] -> + {name, file, [to_string(match_extra(emoji_groups, file))]} + + [name, file | tags] -> + {name, file, tags} + + _ -> + nil + end + end) + |> Enum.to_list() + end + + defp load_from_globs(globs, emoji_groups) do + static_path = Path.join(:code.priv_dir(:pleroma), "static") + + paths = + Enum.map(globs, fn glob -> + Path.join(static_path, glob) + |> Path.wildcard() + end) + |> Enum.concat() + + Enum.map(paths, fn path -> + tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path))) + shortcode = Path.basename(path, Path.extname(path)) + external_path = Path.join("/", Path.relative_to(path, static_path)) + {shortcode, external_path, [to_string(tag)]} + end) + end + + @doc """ + Finds a matching group for the given emoji filename + """ + @spec match_extra(group_patterns(), String.t()) :: atom() | nil + def match_extra(group_patterns, filename) do + match_group_patterns(group_patterns, fn pattern -> + case pattern do + %Regex{} = regex -> Regex.match?(regex, filename) + string when is_binary(string) -> filename == string + end + end) + end + + defp match_group_patterns(group_patterns, matcher) do + Enum.find_value(group_patterns, fn {group, patterns} -> + patterns = + patterns + |> List.wrap() + |> Enum.map(fn pattern -> + if String.contains?(pattern, "*") do + ~r(#{String.replace(pattern, "*", ".*")}) + else + pattern + end + end) + + Enum.any?(patterns, matcher) && group + end) + end +end diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index 90457dadf..c87141582 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Filter do alias Pleroma.User schema "filters" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:filter_id, :integer) field(:hide, :boolean, default: false) field(:whole_word, :boolean, default: true) diff --git a/lib/pleroma/flake_id.ex b/lib/pleroma/flake_id.ex deleted file mode 100644 index 042cf8659..000000000 --- a/lib/pleroma/flake_id.ex +++ /dev/null @@ -1,182 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.FlakeId do - @moduledoc """ - Flake is a decentralized, k-ordered id generation service. - - Adapted from: - - * [flaky](https://github.com/nirvana/flaky), released under the terms of the Truly Free License, - * [Flake](https://github.com/boundary/flake), Copyright 2012, Boundary, Apache License, Version 2.0 - """ - - @type t :: binary - - use Ecto.Type - use GenServer - require Logger - alias __MODULE__ - import Kernel, except: [to_string: 1] - - defstruct node: nil, time: 0, sq: 0 - - @doc "Converts a binary Flake to a String" - def to_string(<<0::integer-size(64), id::integer-size(64)>>) do - Kernel.to_string(id) - end - - def to_string(<<_::integer-size(64), _::integer-size(48), _::integer-size(16)>> = flake) do - encode_base62(flake) - end - - def to_string(s), do: s - - def from_string(int) when is_integer(int) do - from_string(Kernel.to_string(int)) - end - - for i <- [-1, 0] do - def from_string(unquote(i)), do: <<0::integer-size(128)>> - def from_string(unquote(Kernel.to_string(i))), do: <<0::integer-size(128)>> - end - - def from_string(<<_::integer-size(128)>> = flake), do: flake - - def from_string(string) when is_binary(string) and byte_size(string) < 18 do - case Integer.parse(string) do - {id, ""} -> <<0::integer-size(64), id::integer-size(64)>> - _ -> nil - end - end - - def from_string(string) do - string |> decode_base62 |> from_integer - end - - def to_integer(<<integer::integer-size(128)>>), do: integer - - def from_integer(integer) do - <<_time::integer-size(64), _node::integer-size(48), _seq::integer-size(16)>> = - <<integer::integer-size(128)>> - end - - @doc "Generates a Flake" - @spec get :: binary - def get, do: to_string(:gen_server.call(:flake, :get)) - - # checks that ID is is valid FlakeID - # - @spec is_flake_id?(String.t()) :: boolean - def is_flake_id?(id), do: is_flake_id?(String.to_charlist(id), true) - defp is_flake_id?([c | cs], true) when c >= ?0 and c <= ?9, do: is_flake_id?(cs, true) - defp is_flake_id?([c | cs], true) when c >= ?A and c <= ?Z, do: is_flake_id?(cs, true) - defp is_flake_id?([c | cs], true) when c >= ?a and c <= ?z, do: is_flake_id?(cs, true) - defp is_flake_id?([], true), do: true - defp is_flake_id?(_, _), do: false - - # -- Ecto.Type API - @impl Ecto.Type - def type, do: :uuid - - @impl Ecto.Type - def cast(value) do - {:ok, FlakeId.to_string(value)} - end - - @impl Ecto.Type - def load(value) do - {:ok, FlakeId.to_string(value)} - end - - @impl Ecto.Type - def dump(value) do - {:ok, FlakeId.from_string(value)} - end - - def autogenerate, do: get() - - # -- GenServer API - def start_link(_) do - :gen_server.start_link({:local, :flake}, __MODULE__, [], []) - end - - @impl GenServer - def init([]) do - {:ok, %FlakeId{node: worker_id(), time: time()}} - end - - @impl GenServer - def handle_call(:get, _from, state) do - {flake, new_state} = get(time(), state) - {:reply, flake, new_state} - end - - # Matches when the calling time is the same as the state time. Incr. sq - defp get(time, %FlakeId{time: time, node: node, sq: seq}) do - new_state = %FlakeId{time: time, node: node, sq: seq + 1} - {gen_flake(new_state), new_state} - end - - # Matches when the times are different, reset sq - defp get(newtime, %FlakeId{time: time, node: node}) when newtime > time do - new_state = %FlakeId{time: newtime, node: node, sq: 0} - {gen_flake(new_state), new_state} - end - - # Error when clock is running backwards - defp get(newtime, %FlakeId{time: time}) when newtime < time do - {:error, :clock_running_backwards} - end - - defp gen_flake(%FlakeId{time: time, node: node, sq: seq}) do - <<time::integer-size(64), node::integer-size(48), seq::integer-size(16)>> - end - - defp nthchar_base62(n) when n <= 9, do: ?0 + n - defp nthchar_base62(n) when n <= 35, do: ?A + n - 10 - defp nthchar_base62(n), do: ?a + n - 36 - - defp encode_base62(<<integer::integer-size(128)>>) do - integer - |> encode_base62([]) - |> List.to_string() - end - - defp encode_base62(int, acc) when int < 0, do: encode_base62(-int, acc) - defp encode_base62(int, []) when int == 0, do: '0' - defp encode_base62(int, acc) when int == 0, do: acc - - defp encode_base62(int, acc) do - r = rem(int, 62) - id = div(int, 62) - acc = [nthchar_base62(r) | acc] - encode_base62(id, acc) - end - - defp decode_base62(s) do - decode_base62(String.to_charlist(s), 0) - end - - defp decode_base62([c | cs], acc) when c >= ?0 and c <= ?9, - do: decode_base62(cs, 62 * acc + (c - ?0)) - - defp decode_base62([c | cs], acc) when c >= ?A and c <= ?Z, - do: decode_base62(cs, 62 * acc + (c - ?A + 10)) - - defp decode_base62([c | cs], acc) when c >= ?a and c <= ?z, - do: decode_base62(cs, 62 * acc + (c - ?a + 36)) - - defp decode_base62([], acc), do: acc - - defp time do - {mega_seconds, seconds, micro_seconds} = :erlang.timestamp() - 1_000_000_000 * mega_seconds + seconds * 1000 + :erlang.trunc(micro_seconds / 1000) - end - - defp worker_id do - <<worker::integer-size(48)>> = :crypto.strong_rand_bytes(6) - worker - end -end diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 23a5ac8fe..931b9af2b 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -3,10 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Formatter do - alias Pleroma.Emoji alias Pleroma.HTML alias Pleroma.User - alias Pleroma.Web.MediaProxy @safe_mention_regex ~r/^(\s*(?<mentions>(@.+?\s+){1,})+)(?<rest>.*)/s @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @@ -100,51 +98,6 @@ defmodule Pleroma.Formatter do end end - def emojify(text) do - emojify(text, Emoji.get_all()) - end - - def emojify(text, nil), do: text - - def emojify(text, emoji, strip \\ false) do - Enum.reduce(emoji, text, fn emoji_data, text -> - emoji = HTML.strip_tags(elem(emoji_data, 0)) - file = HTML.strip_tags(elem(emoji_data, 1)) - - html = - if not strip do - "<img class='emoji' alt='#{emoji}' title='#{emoji}' src='#{MediaProxy.url(file)}' />" - else - "" - end - - String.replace(text, ":#{emoji}:", html) |> HTML.filter_tags() - end) - end - - def demojify(text) do - emojify(text, Emoji.get_all(), true) - end - - def demojify(text, nil), do: text - - @doc "Outputs a list of the emoji-shortcodes in a text" - def get_emoji(text) when is_binary(text) do - Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end) - end - - def get_emoji(_), do: [] - - @doc "Outputs a list of the emoji-Maps in a text" - def get_emoji_map(text) when is_binary(text) do - get_emoji(text) - |> Enum.reduce(%{}, fn {name, file, _group}, acc -> - Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") - end) - end - - def get_emoji_map(_), do: [] - def html_escape({text, mentions, hashtags}, type) do {html_escape(text, type), mentions, hashtags} end diff --git a/lib/pleroma/list.ex b/lib/pleroma/list.ex index c572380c2..c5db1cb62 100644 --- a/lib/pleroma/list.ex +++ b/lib/pleroma/list.ex @@ -13,7 +13,7 @@ defmodule Pleroma.List do alias Pleroma.User schema "lists" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:title, :string) field(:following, {:array, :string}, default: []) field(:ap_id, :string) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 8012389ac..d94ae5971 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -22,8 +22,8 @@ defmodule Pleroma.Notification do schema "notifications" do field(:seen, :boolean, default: false) - belongs_to(:user, User, type: Pleroma.FlakeId) - belongs_to(:activity, Activity, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) + belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType) timestamps() end diff --git a/lib/pleroma/pagination.ex b/lib/pleroma/pagination.ex index b55379c4a..9d279fba7 100644 --- a/lib/pleroma/pagination.ex +++ b/lib/pleroma/pagination.ex @@ -64,6 +64,7 @@ defmodule Pleroma.Pagination do def paginate(query, options, :offset) do query + |> restrict(:order, options) |> restrict(:offset, options) |> restrict(:limit, options) end diff --git a/lib/pleroma/password_reset_token.ex b/lib/pleroma/password_reset_token.ex index 4a833f6a5..db398b1fc 100644 --- a/lib/pleroma/password_reset_token.ex +++ b/lib/pleroma/password_reset_token.ex @@ -12,7 +12,7 @@ defmodule Pleroma.PasswordResetToken do alias Pleroma.User schema "password_reset_tokens" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:token, :string) field(:used, :boolean, default: false) diff --git a/lib/pleroma/registration.ex b/lib/pleroma/registration.ex index 21fd1fc3f..8544461db 100644 --- a/lib/pleroma/registration.ex +++ b/lib/pleroma/registration.ex @@ -11,10 +11,10 @@ defmodule Pleroma.Registration do alias Pleroma.Repo alias Pleroma.User - @primary_key {:id, Pleroma.FlakeId, autogenerate: true} + @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true} schema "registrations" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:provider, :string) field(:uid, :string) field(:info, :map, default: %{}) diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index de0e54699..fea2cf3ff 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -17,7 +17,7 @@ defmodule Pleroma.ScheduledActivity do @min_offset :timer.minutes(5) schema "scheduled_activities" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:scheduled_at, :naive_datetime) field(:params, :map) diff --git a/lib/pleroma/thread_mute.ex b/lib/pleroma/thread_mute.ex index 10d31679d..65cbbede3 100644 --- a/lib/pleroma/thread_mute.ex +++ b/lib/pleroma/thread_mute.ex @@ -12,7 +12,7 @@ defmodule Pleroma.ThreadMute do require Ecto.Query schema "thread_mutes" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:context, :string) end @@ -24,7 +24,7 @@ defmodule Pleroma.ThreadMute do end def query(user_id, context) do - user_id = Pleroma.FlakeId.from_string(user_id) + {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id) ThreadMute |> Ecto.Query.where(user_id: ^user_id) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 640ef05c4..f3dcf7ad4 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -34,7 +34,7 @@ defmodule Pleroma.User do @type t :: %__MODULE__{} - @primary_key {:id, Pleroma.FlakeId, autogenerate: true} + @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true} # credo:disable-for-next-line Credo.Check.Readability.MaxLineLength @email_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/ @@ -575,7 +575,7 @@ defmodule Pleroma.User do restrict_to_local = Pleroma.Config.get([:instance, :limit_to_local_content]) cond do - is_integer(nickname_or_id) or Pleroma.FlakeId.is_flake_id?(nickname_or_id) -> + is_integer(nickname_or_id) or FlakeId.flake_id?(nickname_or_id) -> get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id) restrict_to_local == false -> diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 92e3944f7..eef985d0d 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -54,6 +54,7 @@ defmodule Pleroma.User.Info do field(:pleroma_settings_store, :map, default: %{}) field(:fields, {:array, :map}, default: nil) field(:raw_fields, {:array, :map}, default: []) + field(:discoverable, :boolean, default: false) field(:notification_settings, :map, default: %{ @@ -269,7 +270,8 @@ defmodule Pleroma.User.Info do :hide_follows_count, :follower_count, :fields, - :following_count + :following_count, + :discoverable ]) |> validate_fields(true) end @@ -287,6 +289,7 @@ defmodule Pleroma.User.Info do :hide_follows, :fields, :hide_followers, + :discoverable, :hide_followers_count, :hide_follows_count ]) @@ -310,7 +313,8 @@ defmodule Pleroma.User.Info do :skip_thread_containment, :fields, :raw_fields, - :pleroma_settings_store + :pleroma_settings_store, + :discoverable ]) |> validate_fields() end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1cf8b6151..8d0a57623 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -510,7 +510,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end @spec fetch_latest_activity_id_for_context(String.t(), keyword() | map()) :: - Pleroma.FlakeId.t() | nil + FlakeId.Ecto.CompatType.t() | nil def fetch_latest_activity_id_for_context(context, opts \\ %{}) do context |> fetch_activities_for_context_query(Map.merge(%{"skip_preload" => true}, opts)) @@ -519,13 +519,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Repo.one() end - def fetch_public_activities(opts \\ %{}) do + def fetch_public_activities(opts \\ %{}, pagination \\ :keyset) do opts = Map.drop(opts, ["user"]) [Pleroma.Constants.as_public()] |> fetch_activities_query(opts) |> restrict_unlisted() - |> Pagination.fetch_paginated(opts) + |> Pagination.fetch_paginated(opts, pagination) |> Enum.reverse() end @@ -834,7 +834,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query - defp exclude_poll_votes(query, %{"include_poll_votes" => "true"}), do: query + defp exclude_poll_votes(query, %{"include_poll_votes" => true}), do: query defp exclude_poll_votes(query, _) do if has_named_binding?(query, :object) do @@ -918,11 +918,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> exclude_poll_votes(opts) end - def fetch_activities(recipients, opts \\ %{}) do + def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do list_memberships = Pleroma.List.memberships(opts["user"]) fetch_activities_query(recipients ++ list_memberships, opts) - |> Pagination.fetch_paginated(opts) + |> Pagination.fetch_paginated(opts, pagination) |> Enum.reverse() |> maybe_update_cc(list_memberships, opts["user"]) end @@ -953,10 +953,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do ) end - def fetch_activities_bounded(recipients, recipients_with_public, opts \\ %{}) do + def fetch_activities_bounded( + recipients, + recipients_with_public, + opts \\ %{}, + pagination \\ :keyset + ) do fetch_activities_query([], opts) |> fetch_activities_bounded_query(recipients, recipients_with_public) - |> Pagination.fetch_paginated(opts) + |> Pagination.fetch_paginated(opts, pagination) |> Enum.reverse() end @@ -996,6 +1001,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do locked = data["manuallyApprovesFollowers"] || false data = Transmogrifier.maybe_fix_user_object(data) + discoverable = data["discoverable"] || false user_data = %{ ap_id: data["id"], @@ -1004,7 +1010,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do source_data: data, banner: banner, fields: fields, - locked: locked + locked: locked, + discoverable: discoverable }, avatar: avatar, name: data["name"], diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 9eb86106f..8112f6642 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -231,13 +231,43 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end end - def outbox(conn, %{"nickname" => nickname} = params) do + def outbox(conn, %{"nickname" => nickname, "page" => page?} = params) + when page? in [true, "true"] do with %User{} = user <- User.get_cached_by_nickname(nickname), {:ok, user} <- User.ensure_keys_present(user) do + activities = + if params["max_id"] do + ActivityPub.fetch_user_activities(user, nil, %{ + "max_id" => params["max_id"], + # This is a hack because postgres generates inefficient queries when filtering by + # 'Answer', poll votes will be hidden by the visibility filter in this case anyway + "include_poll_votes" => true, + "limit" => 10 + }) + else + ActivityPub.fetch_user_activities(user, nil, %{ + "limit" => 10, + "include_poll_votes" => true + }) + end + conn |> put_resp_content_type("application/activity+json") |> put_view(UserView) - |> render("outbox.json", %{user: user, max_id: params["max_id"]}) + |> render("activity_collection_page.json", %{ + activities: activities, + iri: "#{user.ap_id}/outbox" + }) + end + end + + def outbox(conn, %{"nickname" => nickname}) do + with %User{} = user <- User.get_cached_by_nickname(nickname), + {:ok, user} <- User.ensure_keys_present(user) do + conn + |> put_resp_content_type("application/activity+json") + |> put_view(UserView) + |> render("activity_collection.json", %{iri: "#{user.ap_id}/outbox"}) end end @@ -315,12 +345,37 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def read_inbox( %{assigns: %{user: %{nickname: nickname} = user}} = conn, - %{"nickname" => nickname} = params - ) do + %{"nickname" => nickname, "page" => page?} = params + ) + when page? in [true, "true"] do + activities = + if params["max_id"] do + ActivityPub.fetch_activities([user.ap_id | user.following], %{ + "max_id" => params["max_id"], + "limit" => 10 + }) + else + ActivityPub.fetch_activities([user.ap_id | user.following], %{"limit" => 10}) + end + conn |> put_resp_content_type("application/activity+json") |> put_view(UserView) - |> render("inbox.json", user: user, max_id: params["max_id"]) + |> render("activity_collection_page.json", %{ + activities: activities, + iri: "#{user.ap_id}/inbox" + }) + end + + def read_inbox(%{assigns: %{user: %{nickname: nickname} = user}} = conn, %{ + "nickname" => nickname + }) do + with {:ok, user} <- User.ensure_keys_present(user) do + conn + |> put_resp_content_type("application/activity+json") + |> put_view(UserView) + |> render("activity_collection.json", %{iri: "#{user.ap_id}/inbox"}) + end end def read_inbox(%{assigns: %{user: nil}} = conn, %{"nickname" => nickname}) do diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 352d856fa..993307287 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do alias Pleroma.Keys alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Endpoint @@ -107,7 +106,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do }, "endpoints" => endpoints, "attachment" => fields, - "tag" => (user.info.source_data["tag"] || []) ++ emoji_tags + "tag" => (user.info.source_data["tag"] || []) ++ emoji_tags, + "discoverable" => user.info.discoverable } |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user)) @@ -210,20 +210,16 @@ defmodule Pleroma.Web.ActivityPub.UserView do |> Map.merge(Utils.make_json_ld_header()) end - def render("outbox.json", %{user: user, max_id: max_qid}) do - params = %{ - "limit" => "10" + def render("activity_collection.json", %{iri: iri}) do + %{ + "id" => iri, + "type" => "OrderedCollection", + "first" => "#{iri}?page=true" } + |> Map.merge(Utils.make_json_ld_header()) + end - params = - if max_qid != nil do - Map.put(params, "max_id", max_qid) - else - params - end - - activities = ActivityPub.fetch_user_activities(user, nil, params) - + def render("activity_collection_page.json", %{activities: activities, iri: iri}) do # this is sorted chronologically, so first activity is the newest (max) {max_id, min_id, collection} = if length(activities) > 0 do @@ -243,71 +239,14 @@ defmodule Pleroma.Web.ActivityPub.UserView do } end - iri = "#{user.ap_id}/outbox" - - page = %{ - "id" => "#{iri}?max_id=#{max_id}", - "type" => "OrderedCollectionPage", - "partOf" => iri, - "orderedItems" => collection, - "next" => "#{iri}?max_id=#{min_id}" - } - - if max_qid == nil do - %{ - "id" => iri, - "type" => "OrderedCollection", - "first" => page - } - |> Map.merge(Utils.make_json_ld_header()) - else - page |> Map.merge(Utils.make_json_ld_header()) - end - end - - def render("inbox.json", %{user: user, max_id: max_qid}) do - params = %{ - "limit" => "10" - } - - params = - if max_qid != nil do - Map.put(params, "max_id", max_qid) - else - params - end - - activities = ActivityPub.fetch_activities([user.ap_id | user.following], params) - - min_id = Enum.at(Enum.reverse(activities), 0).id - max_id = Enum.at(activities, 0).id - - collection = - Enum.map(activities, fn act -> - {:ok, data} = Transmogrifier.prepare_outgoing(act.data) - data - end) - - iri = "#{user.ap_id}/inbox" - - page = %{ - "id" => "#{iri}?max_id=#{max_id}", + %{ + "id" => "#{iri}?max_id=#{max_id}&page=true", "type" => "OrderedCollectionPage", "partOf" => iri, "orderedItems" => collection, - "next" => "#{iri}?max_id=#{min_id}" + "next" => "#{iri}?max_id=#{min_id}&page=true" } - - if max_qid == nil do - %{ - "id" => iri, - "type" => "OrderedCollection", - "first" => page - } - |> Map.merge(Utils.make_json_ld_header()) - else - page |> Map.merge(Utils.make_json_ld_header()) - end + |> Map.merge(Utils.make_json_ld_header()) end def collection(collection, iri, page, show_items \\ true, total \\ nil) do diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 6e703d169..9f85f0292 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -448,13 +448,17 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end def list_reports(conn, params) do + {page, page_size} = page_params(params) + params = params |> Map.put("type", "Flag") |> Map.put("skip_preload", true) |> Map.put("total", true) + |> Map.put("limit", page_size) + |> Map.put("offset", (page - 1) * page_size) - reports = ActivityPub.fetch_activities([], params) + reports = ActivityPub.fetch_activities([], params, :offset) conn |> put_view(ReportView) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 40eebe2aa..4a74dc16f 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Activity alias Pleroma.ActivityExpiration alias Pleroma.Conversation.Participation - alias Pleroma.Formatter + alias Pleroma.Emoji alias Pleroma.Object alias Pleroma.ThreadMute alias Pleroma.User @@ -261,12 +261,7 @@ defmodule Pleroma.Web.CommonAPI do sensitive, poll ), - object <- - Map.put( - object, - "emoji", - Map.merge(Formatter.get_emoji_map(full_payload), poll_emoji) - ) do + object <- put_emoji(object, full_payload, poll_emoji) do preview? = Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false direct? = visibility == "direct" @@ -300,6 +295,15 @@ defmodule Pleroma.Web.CommonAPI do end end + # parse and put emoji to object data + defp put_emoji(map, text, emojis) do + Map.put( + map, + "emoji", + Map.merge(Emoji.Formatter.get_emoji_map(text), emojis) + ) + end + # Updates the emojis for a user based on their profile def update(user) do emoji = emoji_from_profile(user) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 6958c7511..52fbc162b 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Conversation.Participation + alias Pleroma.Emoji alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.Plugs.AuthenticationPlug @@ -25,7 +26,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do # This is a hack for twidere. def get_by_id_or_ap_id(id) do activity = - with true <- Pleroma.FlakeId.is_flake_id?(id), + with true <- FlakeId.flake_id?(id), %Activity{} = activity <- Activity.get_by_id_with_object(id) do activity else @@ -184,7 +185,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do "name" => option, "type" => "Note", "replies" => %{"type" => "Collection", "totalItems" => 0} - }, Map.merge(emoji, Formatter.get_emoji_map(option))} + }, Map.merge(emoji, Emoji.Formatter.get_emoji_map(option))} end) case expires_in do @@ -434,8 +435,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do end def emoji_from_profile(%{info: _info} = user) do - (Formatter.get_emoji(user.bio) ++ Formatter.get_emoji(user.name)) - |> Enum.map(fn {shortcode, url, _} -> + (Emoji.Formatter.get_emoji(user.bio) ++ Emoji.Formatter.get_emoji(user.name)) + |> Enum.map(fn {shortcode, %Emoji{file: url}} -> %{ "type" => "Emoji", "icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{url}"}, diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 873ef20bc..18a83fd34 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -13,8 +13,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Bookmark alias Pleroma.Config alias Pleroma.Conversation.Participation + alias Pleroma.Emoji alias Pleroma.Filter - alias Pleroma.Formatter alias Pleroma.HTTP alias Pleroma.Notification alias Pleroma.Object @@ -140,7 +140,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do user_info_emojis = user.info |> Map.get(:emoji, []) - |> Enum.concat(Formatter.get_emoji_map(emojis_text)) + |> Enum.concat(Emoji.Formatter.get_emoji_map(emojis_text)) |> Enum.dedup() info_params = @@ -153,7 +153,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do :hide_follows, :hide_favorites, :show_role, - :skip_thread_containment + :skip_thread_containment, + :discoverable ] |> Enum.reduce(%{}, fn key, acc -> add_if_present(acc, params, to_string(key), key, fn value -> @@ -325,7 +326,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do defp mastodonized_emoji do Pleroma.Emoji.get_all() - |> Enum.map(fn {shortcode, relative_url, tags} -> + |> Enum.map(fn {shortcode, %Pleroma.Emoji{file: relative_url, tags: tags}} -> url = to_string(URI.merge(Web.base_url(), relative_url)) %{ diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 195dd124b..a23aeea9b 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -116,6 +116,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do bio = HTML.filter_tags(user.bio, User.html_filter_policy(opts[:for])) relationship = render("relationship.json", %{user: opts[:for], target: user}) + discoverable = user.info.discoverable + %{ id: to_string(user.id), username: username_from_nickname(user.nickname), @@ -139,7 +141,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do note: HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")), sensitive: false, fields: raw_fields, - pleroma: %{} + pleroma: %{ + discoverable: discoverable + } }, # Pleroma extension diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex index 720bd4519..382ecf426 100644 --- a/lib/pleroma/web/metadata/utils.ex +++ b/lib/pleroma/web/metadata/utils.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata.Utils do + alias Pleroma.Emoji alias Pleroma.Formatter alias Pleroma.HTML alias Pleroma.Web.MediaProxy @@ -13,7 +14,7 @@ defmodule Pleroma.Web.Metadata.Utils do |> HtmlEntities.decode() |> String.replace(~r/<br\s?\/?>/, " ") |> HTML.get_cached_stripped_html_for_activity(object, "metadata") - |> Formatter.demojify() + |> Emoji.Formatter.demojify() |> Formatter.truncate() end @@ -23,7 +24,7 @@ defmodule Pleroma.Web.Metadata.Utils do |> HtmlEntities.decode() |> String.replace(~r/<br\s?\/?>/, " ") |> HTML.strip_tags() - |> Formatter.demojify() + |> Emoji.Formatter.demojify() |> Formatter.truncate(max_length) end diff --git a/lib/pleroma/web/oauth/authorization.ex b/lib/pleroma/web/oauth/authorization.ex index d53e20d12..ed42a34f3 100644 --- a/lib/pleroma/web/oauth/authorization.ex +++ b/lib/pleroma/web/oauth/authorization.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.OAuth.Authorization do field(:scopes, {:array, :string}, default: []) field(:valid_until, :naive_datetime_usec) field(:used, :boolean, default: false) - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:app, App) timestamps() diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 40f131b57..8ea373805 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Web.OAuth.Token do field(:refresh_token, :string) field(:scopes, {:array, :string}, default: []) field(:valid_until, :naive_datetime_usec) - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:app, App) timestamps() diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex index 3ad29bd38..545ad80c9 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -3,12 +3,33 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do require Logger - @emoji_dir_path Path.join( - Pleroma.Config.get!([:instance, :static_dir]), - "emoji" - ) + def emoji_dir_path do + Path.join( + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ) + end + + @doc """ + Lists packs from the remote instance. + + Since JS cannot ask remote instances for their packs due to CPS, it has to + be done by the server + """ + def list_from(conn, %{"instance_address" => address}) do + address = String.trim(address) - @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + if shareable_packs_available(address) do + list_resp = + "#{address}/api/pleroma/emoji/packs" |> Tesla.get!() |> Map.get(:body) |> Jason.decode!() + + json(conn, list_resp) + else + conn + |> put_status(:internal_server_error) + |> json(%{error: "The requested instance does not support sharing emoji packs"}) + end + end @doc """ Lists the packs available on the instance as JSON. @@ -17,7 +38,10 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do a map of "pack directory name" to pack.json contents. """ def list_packs(conn, _params) do - with {:ok, results} <- File.ls(@emoji_dir_path) do + # Create the directory first if it does not exist. This is probably the first request made + # with the API so it should be sufficient + with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_dir_path())}, + {:ls, {:ok, results}} <- {:ls, File.ls(emoji_dir_path())} do pack_infos = results |> Enum.filter(&has_pack_json?/1) @@ -28,24 +52,37 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do |> Enum.into(%{}) json(conn, pack_infos) + else + {:create_dir, {:error, e}} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "Failed to create the emoji pack directory at #{emoji_dir_path()}: #{e}"}) + + {:ls, {:error, e}} -> + conn + |> put_status(:internal_server_error) + |> json(%{ + error: + "Failed to get the contents of the emoji pack directory at #{emoji_dir_path()}: #{e}" + }) end end defp has_pack_json?(file) do - dir_path = Path.join(@emoji_dir_path, file) + dir_path = Path.join(emoji_dir_path(), file) # Filter to only use the pack.json packs File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) end defp load_pack(pack_name) do - pack_path = Path.join(@emoji_dir_path, pack_name) + pack_path = Path.join(emoji_dir_path(), pack_name) pack_file = Path.join(pack_path, "pack.json") {pack_name, Jason.decode!(File.read!(pack_file))} end defp validate_pack({name, pack}) do - pack_path = Path.join(@emoji_dir_path, name) + pack_path = Path.join(emoji_dir_path(), name) if can_download?(pack, pack_path) do archive_for_sha = make_archive(name, pack, pack_path) @@ -79,7 +116,8 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) - cache_ms = :timer.seconds(@cache_seconds_per_file * Enum.count(files)) + cache_seconds_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + cache_ms = :timer.seconds(cache_seconds_per_file * Enum.count(files)) Cachex.put!( :emoji_packs_cache, @@ -115,7 +153,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") to download packs that the instance shares. """ def download_shared(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) + pack_dir = Path.join(emoji_dir_path(), name) pack_file = Path.join(pack_dir, "pack.json") with {_, true} <- {:exists?, File.exists?(pack_file)}, @@ -139,6 +177,22 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + defp shareable_packs_available(address) do + "#{address}/.well-known/nodeinfo" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> Map.get("links") + |> List.last() + |> Map.get("href") + # Get the actual nodeinfo address and fetch it + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> get_in(["metadata", "features"]) + |> Enum.member?("shareable_emoji_packs") + end + @doc """ An admin endpoint to request downloading a pack named `pack_name` from the instance `instance_address`. @@ -147,21 +201,9 @@ keeping it in cache for #{div(cache_ms, 1000)}s") from that instance, otherwise it will be downloaded from the fallback source, if there is one. """ def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do - shareable_packs_available = - "#{address}/.well-known/nodeinfo" - |> Tesla.get!() - |> Map.get(:body) - |> Jason.decode!() - |> List.last() - |> Map.get("href") - # Get the actual nodeinfo address and fetch it - |> Tesla.get!() - |> Map.get(:body) - |> Jason.decode!() - |> get_in(["metadata", "features"]) - |> Enum.member?("shareable_emoji_packs") - - if shareable_packs_available do + address = String.trim(address) + + if shareable_packs_available(address) do full_pack = "#{address}/api/pleroma/emoji/packs/list" |> Tesla.get!() @@ -195,7 +237,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") %{body: emoji_archive} <- Tesla.get!(uri), {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do local_name = data["as"] || name - pack_dir = Path.join(@emoji_dir_path, local_name) + pack_dir = Path.join(emoji_dir_path(), local_name) File.mkdir_p!(pack_dir) files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) @@ -233,7 +275,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") Creates an empty pack named `name` which then can be updated via the admin UI. """ def create(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) + pack_dir = Path.join(emoji_dir_path(), name) if not File.exists?(pack_dir) do File.mkdir_p!(pack_dir) @@ -257,7 +299,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") Deletes the pack `name` and all it's files. """ def delete(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) + pack_dir = Path.join(emoji_dir_path(), name) case File.rm_rf(pack_dir) do {:ok, _} -> @@ -276,7 +318,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") `new_data` is the new metadata for the pack, that will replace the old metadata. """ def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do - pack_file_p = Path.join([@emoji_dir_path, name, "pack.json"]) + pack_file_p = Path.join([emoji_dir_path(), name, "pack.json"]) full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -360,7 +402,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") conn, %{"pack_name" => pack_name, "action" => "add", "shortcode" => shortcode} = params ) do - pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_dir = Path.join(emoji_dir_path(), pack_name) pack_file_p = Path.join(pack_dir, "pack.json") full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -408,7 +450,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") "action" => "remove", "shortcode" => shortcode }) do - pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_dir = Path.join(emoji_dir_path(), pack_name) pack_file_p = Path.join(pack_dir, "pack.json") full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -443,7 +485,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") conn, %{"pack_name" => pack_name, "action" => "update", "shortcode" => shortcode} = params ) do - pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_dir = Path.join(emoji_dir_path(), pack_name) pack_file_p = Path.join(pack_dir, "pack.json") full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -513,11 +555,11 @@ keeping it in cache for #{div(cache_ms, 1000)}s") assumed to be emojis and stored in the new `pack.json` file. """ def import_from_fs(conn, _params) do - with {:ok, results} <- File.ls(@emoji_dir_path) do + with {:ok, results} <- File.ls(emoji_dir_path()) do imported_pack_names = results |> Enum.filter(fn file -> - dir_path = Path.join(@emoji_dir_path, file) + dir_path = Path.join(emoji_dir_path(), file) # Find the directories that do NOT have pack.json File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) end) @@ -533,7 +575,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end defp write_pack_json_contents(dir) do - dir_path = Path.join(@emoji_dir_path, dir) + dir_path = Path.join(emoji_dir_path(), dir) emoji_txt_path = Path.join(dir_path, "emoji.txt") files_for_pack = files_for_pack(emoji_txt_path, dir_path) @@ -569,7 +611,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") # If there's no emoji.txt, assume all files # that are of certain extensions from the config are emojis and import them all pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions]) - Pleroma.Emoji.make_shortcode_to_file_map(dir_path, pack_extensions) + Pleroma.Emoji.Loader.make_shortcode_to_file_map(dir_path, pack_extensions) end end end diff --git a/lib/pleroma/web/push/subscription.ex b/lib/pleroma/web/push/subscription.ex index da301fbbc..988fabaeb 100644 --- a/lib/pleroma/web/push/subscription.ex +++ b/lib/pleroma/web/push/subscription.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Web.Push.Subscription do @type t :: %__MODULE__{} schema "push_subscriptions" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:token, Token) field(:endpoint, :string) field(:key_p256dh, :string) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e583093d2..8bc051936 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -222,6 +222,7 @@ defmodule Pleroma.Web.Router do put("/:name", EmojiAPIController, :create) delete("/:name", EmojiAPIController, :delete) post("/download_from", EmojiAPIController, :download_from) + post("/list_from", EmojiAPIController, :list_from) end scope "/packs" do diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index d7745ae7a..f05a84c7f 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -239,11 +239,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do def emoji(conn, _params) do emoji = - Emoji.get_all() - |> Enum.map(fn {short_code, path, tags} -> - {short_code, %{image_url: path, tags: tags}} + Enum.reduce(Emoji.get_all(), %{}, fn {code, %Emoji{file: file, tags: tags}}, acc -> + Map.put(acc, code, %{image_url: file, tags: tags}) end) - |> Enum.into(%{}) json(conn, emoji) end diff --git a/lib/pleroma/web/websub/websub_client_subscription.ex b/lib/pleroma/web/websub/websub_client_subscription.ex index 77703c496..23a04b87d 100644 --- a/lib/pleroma/web/websub/websub_client_subscription.ex +++ b/lib/pleroma/web/websub/websub_client_subscription.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do field(:state, :string) field(:subscribers, {:array, :string}, default: []) field(:hub, :string) - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) timestamps() end |