diff options
Diffstat (limited to 'lib')
57 files changed, 956 insertions, 434 deletions
diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index cc5425362..faa605d9b 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -24,7 +24,7 @@ defmodule Mix.Tasks.Pleroma.Config do |> Enum.reject(fn {k, _v} -> k in [Pleroma.Repo, :env] end) |> Enum.each(fn {k, v} -> key = to_string(k) |> String.replace("Elixir.", "") - {:ok, _} = Config.update_or_create(%{key: key, value: v}) + {:ok, _} = Config.update_or_create(%{group: "pleroma", key: key, value: v}) Mix.shell().info("#{key} is migrated.") end) @@ -36,9 +36,11 @@ defmodule Mix.Tasks.Pleroma.Config do end end - def run(["migrate_from_db", env]) do + def run(["migrate_from_db", env, delete?]) do start_pleroma() + delete? = if delete? == "true", do: true, else: false + if Pleroma.Config.get([:instance, :dynamic_configuration]) do config_path = "config/#{env}.exported_from_db.secret.exs" @@ -47,15 +49,23 @@ defmodule Mix.Tasks.Pleroma.Config do Repo.all(Config) |> Enum.each(fn config -> - mark = if String.starts_with?(config.key, "Pleroma."), do: ",", else: ":" + mark = + if String.starts_with?(config.key, "Pleroma.") or + String.starts_with?(config.key, "Ueberauth"), + do: ",", + else: ":" IO.write( file, - "config :pleroma, #{config.key}#{mark} #{inspect(Config.from_binary(config.value))}\r\n" + "config :#{config.group}, #{config.key}#{mark} #{ + inspect(Config.from_binary(config.value)) + }\r\n" ) - {:ok, _} = Repo.delete(config) - Mix.shell().info("#{config.key} deleted from DB.") + if delete? do + {:ok, _} = Repo.delete(config) + Mix.shell().info("#{config.key} deleted from DB.") + end end) File.close(file) diff --git a/lib/mix/tasks/pleroma/ecto/ecto.ex b/lib/mix/tasks/pleroma/ecto/ecto.ex index 324f57fdd..b66f63376 100644 --- a/lib/mix/tasks/pleroma/ecto/ecto.ex +++ b/lib/mix/tasks/pleroma/ecto/ecto.ex @@ -1,6 +1,7 @@ # Pleroma: A lightweight social networking server # Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/> # SPDX-License-Identifier: AGPL-3.0-onl + defmodule Mix.Tasks.Pleroma.Ecto do @doc """ Ensures the given repository's migrations path exists on the file system. diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 997eabbeb..2ae16adc0 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -114,12 +114,12 @@ defmodule Mix.Tasks.Pleroma.Instance do options, :db_configurable, "Do you want to store the configuration in the database (allows controlling it from admin-fe)? (y/n)", - "y" + "n" ) === "y" dbhost = get_option(options, :dbhost, "What is the hostname of your database?", "localhost") - dbname = get_option(options, :dbname, "What is the name of your database?", "pleroma_dev") + dbname = get_option(options, :dbname, "What is the name of your database?", "pleroma") dbuser = get_option( @@ -149,7 +149,7 @@ defmodule Mix.Tasks.Pleroma.Instance do uploads_dir = get_option( options, - :upload_dir, + :uploads_dir, "What directory should media uploads go in (when using the local uploader)?", Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads]) ) @@ -198,28 +198,16 @@ defmodule Mix.Tasks.Pleroma.Instance do rum_enabled: rum_enabled ) - shell_info( - "Writing config to #{config_path}. You should rename it to config/prod.secret.exs or config/dev.secret.exs." - ) + shell_info("Writing config to #{config_path}.") File.write(config_path, result_config) - shell_info("Writing #{psql_path}.") + shell_info("Writing the postgres script to #{psql_path}.") File.write(psql_path, result_psql) write_robots_txt(indexable, template_dir) shell_info( - "\n" <> - """ - To get started: - 1. Verify the contents of the generated files. - 2. Run `sudo -u postgres psql -f #{escape_sh_path(psql_path)}`. - """ <> - if config_path in ["config/dev.secret.exs", "config/prod.secret.exs"] do - "" - else - "3. Run `mv #{escape_sh_path(config_path)} 'config/prod.secret.exs'`." - end + "\n All files successfully written! Refer to the installation instructions for your platform for next steps" ) else shell_error( diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index ab158f57e..8a78b4fe6 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -204,9 +204,9 @@ defmodule Mix.Tasks.Pleroma.User do IO.puts( "URL: #{ - Pleroma.Web.Router.Helpers.util_url( + Pleroma.Web.Router.Helpers.reset_password_url( Pleroma.Web.Endpoint, - :show_password_reset, + :reset, token.token ) }" diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index ba4cf8486..86c348a0d 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -151,7 +151,11 @@ defmodule Pleroma.Application do start: {Pleroma.Web.Endpoint, :start_link, []}, type: :supervisor }, - %{id: Pleroma.Gopher.Server, start: {Pleroma.Gopher.Server, :start_link, []}} + %{id: Pleroma.Gopher.Server, start: {Pleroma.Gopher.Server, :start_link, []}}, + %{ + id: Pleroma.User.SynchronizationWorker, + start: {Pleroma.User.SynchronizationWorker, :start_link, []} + } ] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index 71a47b9fb..fcc039710 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Config do def put([parent_key | keys], value) do parent = - Application.get_env(:pleroma, parent_key) + Application.get_env(:pleroma, parent_key, []) |> put_in(keys, value) Application.put_env(:pleroma, parent_key, parent) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index a8cbfa52a..cf880aa22 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -11,8 +11,17 @@ defmodule Pleroma.Config.TransferTask do def load_and_update_env do if Pleroma.Config.get([:instance, :dynamic_configuration]) and Ecto.Adapters.SQL.table_exists?(Pleroma.Repo, "config") do - Pleroma.Repo.all(Config) - |> Enum.each(&update_env(&1)) + for_restart = + Pleroma.Repo.all(Config) + |> Enum.map(&update_env(&1)) + + # We need to restart applications for loaded settings take effect + for_restart + |> Enum.reject(&(&1 in [:pleroma, :ok])) + |> Enum.each(fn app -> + Application.stop(app) + :ok = Application.start(app) + end) end end @@ -25,11 +34,15 @@ defmodule Pleroma.Config.TransferTask do setting.key end + group = String.to_existing_atom(setting.group) + Application.put_env( - :pleroma, + group, String.to_existing_atom(key), Config.from_binary(setting.value) ) + + group rescue e -> require Logger diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 8502a0d0c..934620765 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -23,13 +23,8 @@ defmodule Pleroma.Emails.UserEmail do defp recipient(email, name), do: {name, email} defp recipient(%Pleroma.User{} = user), do: recipient(user.email, user.name) - def password_reset_email(user, password_reset_token) when is_binary(password_reset_token) do - password_reset_url = - Router.Helpers.util_url( - Endpoint, - :show_password_reset, - password_reset_token - ) + def password_reset_email(user, token) when is_binary(token) do + password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token) html_body = """ <h3>Reset your password at #{instance_name()}</h3> diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 854d46b1a..052501642 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -148,11 +148,13 @@ defmodule Pleroma.Emoji do if File.exists?(emoji_txt) do load_from_file(emoji_txt, emoji_groups) else + extensions = Pleroma.Config.get([:emoji, :pack_extensions]) + Logger.info( - "No emoji.txt found for pack \"#{pack_name}\", assuming all .png files are emoji" + "No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji" ) - make_shortcode_to_file_map(pack_dir, [".png"]) + make_shortcode_to_file_map(pack_dir, extensions) |> Enum.map(fn {shortcode, rel_file} -> filename = Path.join("/emoji/#{pack_name}", rel_file) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 4b181ec59..b8647dd26 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -44,20 +44,20 @@ defmodule Pleroma.Object do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end - def normalize(_, fetch_remote \\ true) + def normalize(_, fetch_remote \\ true, options \\ []) # If we pass an Activity to Object.normalize(), we can try to use the preloaded object. # Use this whenever possible, especially when walking graphs in an O(N) loop! - def normalize(%Object{} = object, _), do: object - def normalize(%Activity{object: %Object{} = object}, _), do: object + def normalize(%Object{} = object, _, _), do: object + def normalize(%Activity{object: %Object{} = object}, _, _), do: object # A hack for fake activities - def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _) do + def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _, _) do %Object{id: "pleroma:fake_object_id", data: data} end # Catch and log Object.normalize() calls where the Activity's child object is not # preloaded. - def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote) do + def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote, _) do Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) @@ -67,7 +67,7 @@ defmodule Pleroma.Object do normalize(ap_id, fetch_remote) end - def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote) do + def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote, _) do Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) @@ -78,10 +78,14 @@ defmodule Pleroma.Object do end # Old way, try fetching the object through cache. - def normalize(%{"id" => ap_id}, fetch_remote), do: normalize(ap_id, fetch_remote) - def normalize(ap_id, false) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) - def normalize(ap_id, true) when is_binary(ap_id), do: Fetcher.fetch_object_from_id!(ap_id) - def normalize(_, _), do: nil + def normalize(%{"id" => ap_id}, fetch_remote, _), do: normalize(ap_id, fetch_remote) + def normalize(ap_id, false, _) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) + + def normalize(ap_id, true, options) when is_binary(ap_id) do + Fetcher.fetch_object_from_id!(ap_id, options) + end + + def normalize(_, _, _), do: nil # Owned objects can only be mutated by their owner def authorize_mutation(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}), diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index c422490ac..fffbf2bbb 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Object.Fetcher do # TODO: # This will create a Create activity, which we need internally at the moment. - def fetch_object_from_id(id) do + def fetch_object_from_id(id, options \\ []) do if object = Object.get_cached_by_ap_id(id) do {:ok, object} else @@ -38,7 +38,7 @@ defmodule Pleroma.Object.Fetcher do "object" => data }, :ok <- Containment.contain_origin(id, params), - {:ok, activity} <- Transmogrifier.handle_incoming(params), + {:ok, activity} <- Transmogrifier.handle_incoming(params, options), {:object, _data, %Object{} = object} <- {:object, data, Object.normalize(activity, false)} do {:ok, object} @@ -63,8 +63,8 @@ defmodule Pleroma.Object.Fetcher do end end - def fetch_object_from_id!(id) do - with {:ok, object} <- fetch_object_from_id(id) do + def fetch_object_from_id!(id, options \\ []) do + with {:ok, object} <- fetch_object_from_id(id, options) do object else _e -> diff --git a/lib/pleroma/PasswordResetToken.ex b/lib/pleroma/password_reset_token.ex index f31ea5bc5..4a833f6a5 100644 --- a/lib/pleroma/PasswordResetToken.ex +++ b/lib/pleroma/password_reset_token.ex @@ -37,6 +37,7 @@ defmodule Pleroma.PasswordResetToken do |> put_change(:used, true) end + @spec reset_password(binary(), map()) :: {:ok, User.t()} | {:error, binary()} def reset_password(token, data) do with %{used: false} = token <- Repo.get_by(PasswordResetToken, %{token: token}), %User{} = user <- User.get_cached_by_id(token.user_id), diff --git a/lib/pleroma/plugs/idempotency_plug.ex b/lib/pleroma/plugs/idempotency_plug.ex new file mode 100644 index 000000000..e99c5d279 --- /dev/null +++ b/lib/pleroma/plugs/idempotency_plug.ex @@ -0,0 +1,84 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Plugs.IdempotencyPlug do + import Phoenix.Controller, only: [json: 2] + import Plug.Conn + + @behaviour Plug + + @impl true + def init(opts), do: opts + + # Sending idempotency keys in `GET` and `DELETE` requests has no effect + # and should be avoided, as these requests are idempotent by definition. + + @impl true + def call(%{method: method} = conn, _) when method in ["POST", "PUT", "PATCH"] do + case get_req_header(conn, "idempotency-key") do + [key] -> process_request(conn, key) + _ -> conn + end + end + + def call(conn, _), do: conn + + def process_request(conn, key) do + case Cachex.get(:idempotency_cache, key) do + {:ok, nil} -> + cache_resposnse(conn, key) + + {:ok, record} -> + send_cached(conn, key, record) + + {atom, message} when atom in [:ignore, :error] -> + render_error(conn, message) + end + end + + defp cache_resposnse(conn, key) do + register_before_send(conn, fn conn -> + [request_id] = get_resp_header(conn, "x-request-id") + content_type = get_content_type(conn) + + record = {request_id, content_type, conn.status, conn.resp_body} + {:ok, _} = Cachex.put(:idempotency_cache, key, record) + + conn + |> put_resp_header("idempotency-key", key) + |> put_resp_header("x-original-request-id", request_id) + end) + end + + defp send_cached(conn, key, record) do + {request_id, content_type, status, body} = record + + conn + |> put_resp_header("idempotency-key", key) + |> put_resp_header("idempotent-replayed", "true") + |> put_resp_header("x-original-request-id", request_id) + |> put_resp_content_type(content_type) + |> send_resp(status, body) + |> halt() + end + + defp render_error(conn, message) do + conn + |> put_status(:unprocessable_entity) + |> json(%{error: message}) + |> halt() + end + + defp get_content_type(conn) do + [content_type] = get_resp_header(conn, "content-type") + + if String.contains?(content_type, ";") do + content_type + |> String.split(";") + |> hd() + else + content_type + end + end +end diff --git a/lib/pleroma/repo_streamer.ex b/lib/pleroma/repo_streamer.ex new file mode 100644 index 000000000..a4b71a1bb --- /dev/null +++ b/lib/pleroma/repo_streamer.ex @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.RepoStreamer do + alias Pleroma.Repo + import Ecto.Query + + def chunk_stream(query, chunk_size) do + Stream.unfold(0, fn + :halt -> + {[], :halt} + + last_id -> + query + |> order_by(asc: :id) + |> where([r], r.id > ^last_id) + |> limit(^chunk_size) + |> Repo.all() + |> case do + [] -> + {[], :halt} + + records -> + last_id = List.last(records).id + {records, last_id} + end + end) + |> Stream.take_while(fn + [] -> false + _ -> true + end) + end +end diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex new file mode 100644 index 000000000..57c2d2cfd --- /dev/null +++ b/lib/pleroma/reverse_proxy/client.ex @@ -0,0 +1,24 @@ +defmodule Pleroma.ReverseProxy.Client do + @callback request(atom(), String.t(), [tuple()], String.t(), list()) :: + {:ok, pos_integer(), [tuple()], reference() | map()} + | {:ok, pos_integer(), [tuple()]} + | {:ok, reference()} + | {:error, term()} + + @callback stream_body(reference() | pid() | map()) :: + {:ok, binary()} | :done | {:error, String.t()} + + @callback close(reference() | pid() | map()) :: :ok + + def request(method, url, headers, "", opts \\ []) do + client().request(method, url, headers, "", opts) + end + + def stream_body(ref), do: client().stream_body(ref) + + def close(ref), do: client().close(ref) + + defp client do + Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney) + end +end diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index de0f6e1bc..bf31e9cba 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -146,7 +146,7 @@ defmodule Pleroma.ReverseProxy do Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}") method = method |> String.downcase() |> String.to_existing_atom() - case hackney().request(method, url, headers, "", hackney_opts) do + case client().request(method, url, headers, "", hackney_opts) do {:ok, code, headers, client} when code in @valid_resp_codes -> {:ok, code, downcase_headers(headers), client} @@ -173,7 +173,7 @@ defmodule Pleroma.ReverseProxy do halt(conn) {:error, :closed, conn} -> - :hackney.close(client) + client().close(client) halt(conn) {:error, error, conn} -> @@ -181,7 +181,7 @@ defmodule Pleroma.ReverseProxy do "#{__MODULE__} request to #{url} failed while reading/chunking: #{inspect(error)}" ) - :hackney.close(client) + client().close(client) halt(conn) end end @@ -196,7 +196,7 @@ defmodule Pleroma.ReverseProxy do duration, Keyword.get(opts, :max_read_duration, @max_read_duration) ), - {:ok, data} <- hackney().stream_body(client), + {:ok, data} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)), @@ -378,5 +378,5 @@ defmodule Pleroma.ReverseProxy do {:ok, :no_duration_limit, :no_duration_limit} end - defp hackney, do: Pleroma.Config.get(:hackney, :hackney) + defp client, do: Pleroma.ReverseProxy.Client end diff --git a/lib/pleroma/upload/filter/anonymize_filename.ex b/lib/pleroma/upload/filter/anonymize_filename.ex index 5ca53a79b..a8516811c 100644 --- a/lib/pleroma/upload/filter/anonymize_filename.ex +++ b/lib/pleroma/upload/filter/anonymize_filename.ex @@ -10,10 +10,19 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do """ @behaviour Pleroma.Upload.Filter - def filter(upload) do - extension = List.last(String.split(upload.name, ".")) - name = Pleroma.Config.get([__MODULE__, :text], random(extension)) - {:ok, %Pleroma.Upload{upload | name: name}} + alias Pleroma.Config + alias Pleroma.Upload + + def filter(%Upload{name: name} = upload) do + extension = List.last(String.split(name, ".")) + name = predefined_name(extension) || random(extension) + {:ok, %Upload{upload | name: name}} + end + + @spec predefined_name(String.t()) :: String.t() | nil + defp predefined_name(extension) do + with name when not is_nil(name) <- Config.get([__MODULE__, :text]), + do: String.replace(name, "{extension}", extension) end defp random(extension) do diff --git a/lib/pleroma/uploaders/swift/keystone.ex b/lib/pleroma/uploaders/swift/keystone.ex deleted file mode 100644 index dd44c7561..000000000 --- a/lib/pleroma/uploaders/swift/keystone.ex +++ /dev/null @@ -1,51 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Uploaders.Swift.Keystone do - use HTTPoison.Base - - def process_url(url) do - Enum.join( - [Pleroma.Config.get!([Pleroma.Uploaders.Swift, :auth_url]), url], - "/" - ) - end - - def process_response_body(body) do - body - |> Jason.decode!() - end - - def get_token do - settings = Pleroma.Config.get(Pleroma.Uploaders.Swift) - username = Keyword.fetch!(settings, :username) - password = Keyword.fetch!(settings, :password) - tenant_id = Keyword.fetch!(settings, :tenant_id) - - case post( - "/tokens", - make_auth_body(username, password, tenant_id), - ["Content-Type": "application/json"], - hackney: [:insecure] - ) do - {:ok, %Tesla.Env{status: 200, body: body}} -> - body["access"]["token"]["id"] - - {:ok, %Tesla.Env{status: _}} -> - "" - end - end - - def make_auth_body(username, password, tenant) do - Jason.encode!(%{ - :auth => %{ - :passwordCredentials => %{ - :username => username, - :password => password - }, - :tenantId => tenant - } - }) - end -end diff --git a/lib/pleroma/uploaders/swift/swift.ex b/lib/pleroma/uploaders/swift/swift.ex deleted file mode 100644 index 2b0f2ad04..000000000 --- a/lib/pleroma/uploaders/swift/swift.ex +++ /dev/null @@ -1,29 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Uploaders.Swift.Client do - use HTTPoison.Base - - def process_url(url) do - Enum.join( - [Pleroma.Config.get!([Pleroma.Uploaders.Swift, :storage_url]), url], - "/" - ) - end - - def upload_file(filename, body, content_type) do - token = Pleroma.Uploaders.Swift.Keystone.get_token() - - case put("#{filename}", body, "X-Auth-Token": token, "Content-Type": content_type) do - {:ok, %Tesla.Env{status: 201}} -> - {:ok, {:file, filename}} - - {:ok, %Tesla.Env{status: 401}} -> - {:error, "Unauthorized, Bad Token"} - - {:error, _} -> - {:error, "Swift Upload Error"} - end - end -end diff --git a/lib/pleroma/uploaders/swift/uploader.ex b/lib/pleroma/uploaders/swift/uploader.ex deleted file mode 100644 index d122b09e7..000000000 --- a/lib/pleroma/uploaders/swift/uploader.ex +++ /dev/null @@ -1,19 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Uploaders.Swift do - @behaviour Pleroma.Uploaders.Uploader - - def get_file(name) do - {:ok, {:url, Path.join([Pleroma.Config.get!([__MODULE__, :object_url]), name])}} - end - - def put_file(upload) do - Pleroma.Uploaders.Swift.Client.upload_file( - upload.path, - File.read!(upload.tmpfile), - upload.content_type - ) - end -end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3a9ae8d73..d03810d1a 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -9,12 +9,14 @@ defmodule Pleroma.User do import Ecto.Query alias Comeonin.Pbkdf2 + alias Ecto.Multi alias Pleroma.Activity alias Pleroma.Keys alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Registration alias Pleroma.Repo + alias Pleroma.RepoStreamer alias Pleroma.User alias Pleroma.Web alias Pleroma.Web.ActivityPub.ActivityPub @@ -105,15 +107,25 @@ defmodule Pleroma.User do def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers" - def user_info(%User{} = user) do + def user_info(%User{} = user, args \\ %{}) do + following_count = + if args[:following_count], do: args[:following_count], else: following_count(user) + + follower_count = + if args[:follower_count], do: args[:follower_count], else: user.info.follower_count + %{ - following_count: following_count(user), note_count: user.info.note_count, - follower_count: user.info.follower_count, locked: user.info.locked, confirmation_pending: user.info.confirmation_pending, default_scope: user.info.default_scope } + |> Map.put(:following_count, following_count) + |> Map.put(:follower_count, follower_count) + end + + def set_info_cache(user, args) do + Cachex.put(:user_cache, "user_info:#{user.id}", user_info(user, args)) end def restrict_deactivated(query) do @@ -193,29 +205,26 @@ defmodule Pleroma.User do end def password_update_changeset(struct, params) do - changeset = - struct - |> cast(params, [:password, :password_confirmation]) - |> validate_required([:password, :password_confirmation]) - |> validate_confirmation(:password) - - OAuth.Token.delete_user_tokens(struct) - OAuth.Authorization.delete_user_authorizations(struct) - - if changeset.valid? do - hashed = Pbkdf2.hashpwsalt(changeset.changes[:password]) - - changeset - |> put_change(:password_hash, hashed) - else - changeset + struct + |> cast(params, [:password, :password_confirmation]) + |> validate_required([:password, :password_confirmation]) + |> validate_confirmation(:password) + |> put_password_hash + end + + def reset_password(%User{id: user_id} = user, data) do + multi = + Multi.new() + |> Multi.update(:user, password_update_changeset(user, data)) + |> Multi.delete_all(:tokens, OAuth.Token.Query.get_by_user(user_id)) + |> Multi.delete_all(:auth, OAuth.Authorization.delete_by_user_query(user)) + + case Repo.transaction(multi) do + {:ok, %{user: user} = _} -> set_cache(user) + {:error, _, changeset, _} -> {:error, changeset} end end - def reset_password(user, data) do - update_and_set_cache(password_update_changeset(user, data)) - end - def register_changeset(struct, params \\ %{}, opts \\ []) do need_confirmation? = if is_nil(opts[:need_confirmation]) do @@ -249,12 +258,11 @@ defmodule Pleroma.User do end if changeset.valid? do - hashed = Pbkdf2.hashpwsalt(changeset.changes[:password]) ap_id = User.ap_id(%User{nickname: changeset.changes[:nickname]}) followers = User.ap_followers(%User{nickname: changeset.changes[:nickname]}) changeset - |> put_change(:password_hash, hashed) + |> put_password_hash |> put_change(:ap_id, ap_id) |> unique_constraint(:ap_id) |> put_change(:following, [followers]) @@ -838,15 +846,12 @@ defmodule Pleroma.User do def mutes?(nil, _), do: false def mutes?(user, %{ap_id: ap_id}), do: Enum.member?(user.info.mutes, ap_id) - def blocks?(user, %{ap_id: ap_id}) do - blocks = user.info.blocks - domain_blocks = user.info.domain_blocks + def blocks?(%User{info: info} = _user, %{ap_id: ap_id}) do + blocks = info.blocks + domain_blocks = info.domain_blocks %{host: host} = URI.parse(ap_id) - Enum.member?(blocks, ap_id) || - Enum.any?(domain_blocks, fn domain -> - host == domain - end) + Enum.member?(blocks, ap_id) || Enum.any?(domain_blocks, &(&1 == host)) end def subscribed_to?(user, %{ap_id: ap_id}) do @@ -932,18 +937,24 @@ defmodule Pleroma.User do @spec perform(atom(), User.t()) :: {:ok, User.t()} def perform(:delete, %User{} = user) do - {:ok, user} = User.deactivate(user) - # Remove all relationships {:ok, followers} = User.get_followers(user) - Enum.each(followers, fn follower -> User.unfollow(follower, user) end) + Enum.each(followers, fn follower -> + ActivityPub.unfollow(follower, user) + User.unfollow(follower, user) + end) {:ok, friends} = User.get_friends(user) - Enum.each(friends, fn followed -> User.unfollow(user, followed) end) + Enum.each(friends, fn followed -> + ActivityPub.unfollow(user, followed) + User.unfollow(user, followed) + end) delete_user_activities(user) + + {:ok, _user} = Repo.delete(user) end @spec perform(atom(), User.t()) :: {:ok, User.t()} @@ -999,6 +1010,56 @@ defmodule Pleroma.User do ) end + @spec sync_follow_counter() :: :ok + def sync_follow_counter, + do: PleromaJobQueue.enqueue(:background, __MODULE__, [:sync_follow_counters]) + + @spec perform(:sync_follow_counters) :: :ok + def perform(:sync_follow_counters) do + {:ok, _pid} = Agent.start_link(fn -> %{} end, name: :domain_errors) + config = Pleroma.Config.get([:instance, :external_user_synchronization]) + + :ok = sync_follow_counters(config) + Agent.stop(:domain_errors) + end + + @spec sync_follow_counters(keyword()) :: :ok + def sync_follow_counters(opts \\ []) do + users = external_users(opts) + + if length(users) > 0 do + errors = Agent.get(:domain_errors, fn state -> state end) + {last, updated_errors} = User.Synchronization.call(users, errors, opts) + Agent.update(:domain_errors, fn _state -> updated_errors end) + sync_follow_counters(max_id: last.id, limit: opts[:limit]) + else + :ok + end + end + + @spec external_users(keyword()) :: [User.t()] + def external_users(opts \\ []) do + query = + User.Query.build(%{ + external: true, + active: true, + order_by: :id, + select: [:id, :ap_id, :info] + }) + + query = + if opts[:max_id], + do: where(query, [u], u.id > ^opts[:max_id]), + else: query + + query = + if opts[:limit], + do: limit(query, ^opts[:limit]), + else: query + + Repo.all(query) + end + def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers), do: PleromaJobQueue.enqueue(:background, __MODULE__, [ @@ -1016,18 +1077,35 @@ defmodule Pleroma.User do ]) def delete_user_activities(%User{ap_id: ap_id} = user) do - stream = - ap_id - |> Activity.query_by_actor() - |> Repo.stream() - - Repo.transaction(fn -> Enum.each(stream, &delete_activity(&1)) end, timeout: :infinity) + ap_id + |> Activity.query_by_actor() + |> RepoStreamer.chunk_stream(50) + |> Stream.each(fn activities -> + Enum.each(activities, &delete_activity(&1)) + end) + |> Stream.run() {:ok, user} end defp delete_activity(%{data: %{"type" => "Create"}} = activity) do - Object.normalize(activity) |> ActivityPub.delete() + activity + |> Object.normalize() + |> ActivityPub.delete() + end + + defp delete_activity(%{data: %{"type" => "Like"}} = activity) do + user = get_cached_by_ap_id(activity.actor) + object = Object.normalize(activity) + + ActivityPub.unlike(user, object) + end + + defp delete_activity(%{data: %{"type" => "Announce"}} = activity) do + user = get_cached_by_ap_id(activity.actor) + object = Object.normalize(activity) + + ActivityPub.unannounce(user, object) end defp delete_activity(_activity), do: "Doing nothing" @@ -1325,4 +1403,12 @@ defmodule Pleroma.User do end defdelegate search(query, opts \\ []), to: User.Search + + defp put_password_hash( + %Ecto.Changeset{valid?: true, changes: %{password: password}} = changeset + ) do + change(changeset, password_hash: Pbkdf2.hashpwsalt(password)) + end + + defp put_password_hash(changeset), do: changeset end diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index ace9c05f2..f9bcc9e19 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -7,7 +7,7 @@ defmodule Pleroma.User.Query do User query builder module. Builds query from new query or another user query. ## Example: - query = Pleroma.User.Query(%{nickname: "nickname"}) + query = Pleroma.User.Query.build(%{nickname: "nickname"}) another_query = Pleroma.User.Query.build(query, %{email: "email@example.com"}) Pleroma.Repo.all(query) Pleroma.Repo.all(another_query) @@ -47,7 +47,10 @@ defmodule Pleroma.User.Query do friends: User.t(), recipients_from_activity: [String.t()], nickname: [String.t()], - ap_id: [String.t()] + ap_id: [String.t()], + order_by: term(), + select: term(), + limit: pos_integer() } | %{} @@ -141,6 +144,18 @@ defmodule Pleroma.User.Query do where(query, [u], u.ap_id in ^to or fragment("? && ?", u.following, ^to)) end + defp compose_query({:order_by, key}, query) do + order_by(query, [u], field(u, ^key)) + end + + defp compose_query({:select, keys}, query) do + select(query, [u], ^keys) + end + + defp compose_query({:limit, limit}, query) do + limit(query, ^limit) + end + defp compose_query(_unsupported_param, query), do: query defp prepare_tag_criteria(tag, query) do diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index ed06c2ab9..64eb6d2bc 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -43,6 +43,8 @@ defmodule Pleroma.User.Search do defp search_query(query_string, for_user, following) do for_user |> base_query(following) + |> filter_blocked_user(for_user) + |> filter_blocked_domains(for_user) |> search_subqueries(query_string) |> union_subqueries |> distinct_query() @@ -55,6 +57,25 @@ defmodule Pleroma.User.Search do defp base_query(_user, false), do: User defp base_query(user, true), do: User.get_followers_query(user) + defp filter_blocked_user(query, %User{info: %{blocks: blocks}}) + when length(blocks) > 0 do + from(q in query, where: not (q.ap_id in ^blocks)) + end + + defp filter_blocked_user(query, _), do: query + + defp filter_blocked_domains(query, %User{info: %{domain_blocks: domain_blocks}}) + when length(domain_blocks) > 0 do + domains = Enum.join(domain_blocks, ",") + + from( + q in query, + where: fragment("substring(ap_id from '.*://([^/]*)') NOT IN (?)", ^domains) + ) + end + + defp filter_blocked_domains(query, _), do: query + defp paginate(query, limit, offset) do from(q in query, limit: ^limit, offset: ^offset) end @@ -129,7 +150,7 @@ defmodule Pleroma.User.Search do @spec fts_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t() defp fts_search_subquery(query, term) do processed_query = - term + String.trim_trailing(term, "@" <> local_domain()) |> String.replace(~r/\W+/, " ") |> String.trim() |> String.split() @@ -171,6 +192,8 @@ defmodule Pleroma.User.Search do @spec trigram_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t() defp trigram_search_subquery(query, term) do + term = String.trim_trailing(term, "@" <> local_domain()) + from( u in query, select_merge: %{ @@ -188,4 +211,6 @@ defmodule Pleroma.User.Search do ) |> User.restrict_deactivated() end + + defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) end diff --git a/lib/pleroma/user/synchronization.ex b/lib/pleroma/user/synchronization.ex new file mode 100644 index 000000000..93660e08c --- /dev/null +++ b/lib/pleroma/user/synchronization.ex @@ -0,0 +1,60 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.User.Synchronization do + alias Pleroma.HTTP + alias Pleroma.User + + @spec call([User.t()], map(), keyword()) :: {User.t(), map()} + def call(users, errors, opts \\ []) do + do_call(users, errors, opts) + end + + defp do_call([user | []], errors, opts) do + updated = fetch_counters(user, errors, opts) + {user, updated} + end + + defp do_call([user | others], errors, opts) do + updated = fetch_counters(user, errors, opts) + do_call(others, updated, opts) + end + + defp fetch_counters(user, errors, opts) do + %{host: host} = URI.parse(user.ap_id) + + info = %{} + {following, errors} = fetch_counter(user.ap_id <> "/following", host, errors, opts) + info = if following, do: Map.put(info, :following_count, following), else: info + + {followers, errors} = fetch_counter(user.ap_id <> "/followers", host, errors, opts) + info = if followers, do: Map.put(info, :follower_count, followers), else: info + + User.set_info_cache(user, info) + errors + end + + defp available_domain?(domain, errors, opts) do + max_retries = Keyword.get(opts, :max_retries, 3) + not (Map.has_key?(errors, domain) && errors[domain] >= max_retries) + end + + defp fetch_counter(url, host, errors, opts) do + with true <- available_domain?(host, errors, opts), + {:ok, %{body: body, status: code}} when code in 200..299 <- + HTTP.get( + url, + [{:Accept, "application/activity+json"}] + ), + {:ok, data} <- Jason.decode(body) do + {data["totalItems"], errors} + else + false -> + {nil, errors} + + _ -> + {nil, Map.update(errors, host, 1, &(&1 + 1))} + end + end +end diff --git a/lib/pleroma/user/synchronization_worker.ex b/lib/pleroma/user/synchronization_worker.ex new file mode 100644 index 000000000..ba9cc3556 --- /dev/null +++ b/lib/pleroma/user/synchronization_worker.ex @@ -0,0 +1,32 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-onl + +defmodule Pleroma.User.SynchronizationWorker do + use GenServer + + def start_link do + config = Pleroma.Config.get([:instance, :external_user_synchronization]) + + if config[:enabled] do + GenServer.start_link(__MODULE__, interval: config[:interval]) + else + :ignore + end + end + + def init(opts) do + schedule_next(opts) + {:ok, opts} + end + + def handle_info(:sync_follow_counters, opts) do + Pleroma.User.sync_follow_counter() + schedule_next(opts) + {:noreply, opts} + end + + defp schedule_next(opts) do + Process.send_after(self(), :sync_follow_counters, opts[:interval]) + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c0e3d1478..55315d66e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -189,6 +189,22 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) end + def stream_out_participations(%Object{data: %{"context" => context}}, user) do + with %Conversation{} = conversation <- Conversation.get_for_ap_id(context), + conversation = Repo.preload(conversation, :participations), + last_activity_id = + fetch_latest_activity_id_for_context(conversation.ap_id, %{ + "user" => user, + "blocking_user" => user + }) do + if last_activity_id do + stream_out_participations(conversation.participations) + end + end + end + + def stream_out_participations(_, _), do: :noop + def stream_out(activity) do public = "https://www.w3.org/ns/activitystreams#Public" @@ -401,7 +417,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do "to" => to, "deleted_activity_id" => activity && activity.id }, - {:ok, activity} <- insert(data, local), + {:ok, activity} <- insert(data, local, false), + stream_out_participations(object, user), _ <- decrease_replies_count_if_reply(object), # Changing note count prior to enqueuing federation task in order to avoid # race conditions on updating user.info diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex new file mode 100644 index 000000000..01d21a299 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do + @moduledoc "Preloads any attachments in the MediaProxy cache by prefetching them" + @behaviour Pleroma.Web.ActivityPub.MRF + + alias Pleroma.HTTP + alias Pleroma.Web.MediaProxy + + require Logger + + @hackney_options [ + pool: :media, + recv_timeout: 10_000 + ] + + def perform(:prefetch, url) do + Logger.info("Prefetching #{inspect(url)}") + + url + |> MediaProxy.url() + |> HTTP.get([], adapter: @hackney_options) + end + + def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do + Enum.each(attachments, fn + %{"url" => url} when is_list(url) -> + url + |> Enum.each(fn + %{"href" => href} -> + PleromaJobQueue.enqueue(:background, __MODULE__, [:prefetch, href]) + + x -> + Logger.debug("Unhandled attachment URL object #{inspect(x)}") + end) + + x -> + Logger.debug("Unhandled attachment #{inspect(x)}") + end) + end + + @impl true + def filter( + %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message + ) + when is_list(attachments) and length(attachments) > 0 do + PleromaJobQueue.enqueue(:background, __MODULE__, [:preload, message]) + + {:ok, message} + end + + @impl true + def filter(message), do: {:ok, message} +end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 3bb8b40b5..543d4bb7d 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.Federator import Ecto.Query @@ -22,20 +23,20 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ - def fix_object(object) do + def fix_object(object, options \\ []) do object |> fix_actor |> fix_url |> fix_attachments |> fix_context - |> fix_in_reply_to + |> fix_in_reply_to(options) |> fix_emoji |> fix_tag |> fix_content_map |> fix_likes |> fix_addressing |> fix_summary - |> fix_type + |> fix_type(options) end def fix_summary(%{"summary" => nil} = object) do @@ -164,7 +165,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do object end - def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object) + def fix_in_reply_to(object, options \\ []) + + def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) when not is_nil(in_reply_to) do in_reply_to_id = cond do @@ -182,28 +185,34 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "" end - case get_obj_helper(in_reply_to_id) do - {:ok, replied_object} -> - with %Activity{} = _activity <- - Activity.get_create_by_object_ap_id(replied_object.data["id"]) do - object - |> Map.put("inReplyTo", replied_object.data["id"]) - |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id) - |> Map.put("conversation", replied_object.data["context"] || object["conversation"]) - |> Map.put("context", replied_object.data["context"] || object["conversation"]) - else - e -> - Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") + object = Map.put(object, "inReplyToAtomUri", in_reply_to_id) + + if Federator.allowed_incoming_reply_depth?(options[:depth]) do + case get_obj_helper(in_reply_to_id, options) do + {:ok, replied_object} -> + with %Activity{} = _activity <- + Activity.get_create_by_object_ap_id(replied_object.data["id"]) do object - end + |> Map.put("inReplyTo", replied_object.data["id"]) + |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id) + |> Map.put("conversation", replied_object.data["context"] || object["conversation"]) + |> Map.put("context", replied_object.data["context"] || object["conversation"]) + else + e -> + Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") + object + end - e -> - Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") - object + e -> + Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") + object + end + else + object end end - def fix_in_reply_to(object), do: object + def fix_in_reply_to(object, _options), do: object def fix_context(object) do context = object["context"] || object["conversation"] || Utils.generate_context_id() @@ -336,8 +345,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_content_map(object), do: object - def fix_type(%{"inReplyTo" => reply_id} = object) when is_binary(reply_id) do - reply = Object.normalize(reply_id) + def fix_type(object, options \\ []) + + def fix_type(%{"inReplyTo" => reply_id} = object, options) when is_binary(reply_id) do + reply = + if Federator.allowed_incoming_reply_depth?(options[:depth]) do + Object.normalize(reply_id, true) + end if reply && (reply.data["type"] == "Question" and object["name"]) do Map.put(object, "type", "Answer") @@ -346,7 +360,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def fix_type(object), do: object + def fix_type(object, _), do: object defp mastodon_follow_hack(%{"id" => id, "actor" => follower_id}, followed) do with true <- id =~ "follows", @@ -374,9 +388,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end + def handle_incoming(data, options \\ []) + # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them # with nil ID. - def handle_incoming(%{"type" => "Flag", "object" => objects, "actor" => actor} = data) do + def handle_incoming(%{"type" => "Flag", "object" => objects, "actor" => actor} = data, _options) do with context <- data["context"] || Utils.generate_context_id(), content <- data["content"] || "", %User{} = actor <- User.get_cached_by_ap_id(actor), @@ -409,15 +425,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end # disallow objects with bogus IDs - def handle_incoming(%{"id" => nil}), do: :error - def handle_incoming(%{"id" => ""}), do: :error + def handle_incoming(%{"id" => nil}, _options), do: :error + def handle_incoming(%{"id" => ""}, _options), do: :error # length of https:// = 8, should validate better, but good enough for now. - def handle_incoming(%{"id" => id}) when not (is_binary(id) and length(id) > 8), do: :error + def handle_incoming(%{"id" => id}, _options) when not (is_binary(id) and length(id) > 8), + do: :error # TODO: validate those with a Ecto scheme # - tags # - emoji - def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data) + def handle_incoming( + %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, + options + ) when objtype in ["Article", "Note", "Video", "Page", "Question", "Answer"] do actor = Containment.get_actor(data) @@ -427,7 +447,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do with nil <- Activity.get_create_by_object_ap_id(object["id"]), {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do - object = fix_object(data["object"]) + options = Keyword.put(options, :depth, (options[:depth] || 0) + 1) + object = fix_object(data["object"], options) params = %{ to: data["to"], @@ -452,7 +473,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data + %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data, + _options ) do with %User{local: true} = followed <- User.get_cached_by_ap_id(followed), {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower), @@ -503,7 +525,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data + %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor), @@ -524,7 +547,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data + %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor), @@ -548,7 +572,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data + %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -561,7 +586,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data + %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -576,7 +602,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming( %{"type" => "Update", "object" => %{"type" => object_type} = object, "actor" => actor_id} = - data + data, + _options ) when object_type in ["Person", "Application", "Service", "Organization"] do with %User{ap_id: ^actor_id} = actor <- User.get_cached_by_ap_id(object["id"]) do @@ -614,7 +641,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # an error or a tombstone. This would allow us to verify that a deletion actually took # place. def handle_incoming( - %{"type" => "Delete", "object" => object_id, "actor" => _actor, "id" => _id} = data + %{"type" => "Delete", "object" => object_id, "actor" => _actor, "id" => _id} = data, + _options ) do object_id = Utils.get_ap_id(object_id) @@ -635,7 +663,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "object" => %{"type" => "Announce", "object" => object_id}, "actor" => _actor, "id" => id - } = data + } = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -653,7 +682,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "object" => %{"type" => "Follow", "object" => followed}, "actor" => follower, "id" => id - } = _data + } = _data, + _options ) do with %User{local: true} = followed <- User.get_cached_by_ap_id(followed), {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower), @@ -671,7 +701,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "object" => %{"type" => "Block", "object" => blocked}, "actor" => blocker, "id" => id - } = _data + } = _data, + _options ) do with true <- Pleroma.Config.get([:activitypub, :accept_blocks]), %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked), @@ -685,7 +716,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data + %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data, + _options ) do with true <- Pleroma.Config.get([:activitypub, :accept_blocks]), %User{local: true} = blocked = User.get_cached_by_ap_id(blocked), @@ -705,7 +737,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "object" => %{"type" => "Like", "object" => object_id}, "actor" => _actor, "id" => id - } = data + } = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -717,10 +750,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def handle_incoming(_), do: :error + def handle_incoming(_, _), do: :error - def get_obj_helper(id) do - if object = Object.normalize(id), do: {:ok, object}, else: nil + def get_obj_helper(id, options \\ []) do + if object = Object.normalize(id, true, options), do: {:ok, object}, else: nil end def set_reply_to_uri(%{"inReplyTo" => in_reply_to} = object) when is_binary(in_reply_to) do diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 10ff572a2..4288ea4c8 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -151,16 +151,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do def create_context(context) do context = context || generate_id("contexts") - changeset = Object.context_mapping(context) - case Repo.insert(changeset) do - {:ok, object} -> - object + # Ecto has problems accessing the constraint inside the jsonb, + # so we explicitly check for the existed object before insert + object = Object.get_cached_by_ap_id(context) - # This should be solved by an upsert, but it seems ecto - # has problems accessing the constraint inside the jsonb. - {:error, _} -> - Object.get_cached_by_ap_id(context) + with true <- is_nil(object), + changeset <- Object.context_mapping(context), + {:ok, inserted_object} <- Repo.insert(changeset) do + inserted_object + else + _ -> + object end end @@ -168,14 +170,17 @@ defmodule Pleroma.Web.ActivityPub.Utils do Enqueues an activity for federation if it's local """ def maybe_federate(%Activity{local: true} = activity) do - priority = - case activity.data["type"] do - "Delete" -> 10 - "Create" -> 1 - _ -> 5 - end + if Pleroma.Config.get!([:instance, :federating]) do + priority = + case activity.data["type"] do + "Delete" -> 10 + "Create" -> 1 + _ -> 5 + end + + Pleroma.Web.Federator.publish(activity, priority) + end - Pleroma.Web.Federator.publish(activity, priority) :ok end diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 03dfdca82..498beb56a 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -377,18 +377,18 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do if Pleroma.Config.get([:instance, :dynamic_configuration]) do updated = Enum.map(configs, fn - %{"key" => key, "value" => value} -> - {:ok, config} = Config.update_or_create(%{key: key, value: value}) + %{"group" => group, "key" => key, "value" => value} -> + {:ok, config} = Config.update_or_create(%{group: group, key: key, value: value}) config - %{"key" => key, "delete" => "true"} -> - {:ok, _} = Config.delete(key) + %{"group" => group, "key" => key, "delete" => "true"} -> + {:ok, _} = Config.delete(%{group: group, key: key}) nil end) |> Enum.reject(&is_nil(&1)) Pleroma.Config.TransferTask.load_and_update_env() - Mix.Tasks.Pleroma.Config.run(["migrate_from_db", Pleroma.Config.get(:env)]) + Mix.Tasks.Pleroma.Config.run(["migrate_from_db", Pleroma.Config.get(:env), "false"]) updated else [] diff --git a/lib/pleroma/web/admin_api/config.ex b/lib/pleroma/web/admin_api/config.ex index b7072f050..8b9b658a9 100644 --- a/lib/pleroma/web/admin_api/config.ex +++ b/lib/pleroma/web/admin_api/config.ex @@ -12,26 +12,27 @@ defmodule Pleroma.Web.AdminAPI.Config do schema "config" do field(:key, :string) + field(:group, :string) field(:value, :binary) timestamps() end - @spec get_by_key(String.t()) :: Config.t() | nil - def get_by_key(key), do: Repo.get_by(Config, key: key) + @spec get_by_params(map()) :: Config.t() | nil + def get_by_params(params), do: Repo.get_by(Config, params) @spec changeset(Config.t(), map()) :: Changeset.t() def changeset(config, params \\ %{}) do config - |> cast(params, [:key, :value]) - |> validate_required([:key, :value]) - |> unique_constraint(:key) + |> cast(params, [:key, :group, :value]) + |> validate_required([:key, :group, :value]) + |> unique_constraint(:key, name: :config_group_key_index) end @spec create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()} - def create(%{key: key, value: value}) do + def create(params) do %Config{} - |> changeset(%{key: key, value: transform(value)}) + |> changeset(Map.put(params, :value, transform(params[:value]))) |> Repo.insert() end @@ -43,20 +44,20 @@ defmodule Pleroma.Web.AdminAPI.Config do end @spec update_or_create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()} - def update_or_create(%{key: key} = params) do - with %Config{} = config <- Config.get_by_key(key) do + def update_or_create(params) do + with %Config{} = config <- Config.get_by_params(Map.take(params, [:group, :key])) do Config.update(config, params) else nil -> Config.create(params) end end - @spec delete(String.t()) :: {:ok, Config.t()} | {:error, Changeset.t()} - def delete(key) do - with %Config{} = config <- Config.get_by_key(key) do + @spec delete(map()) :: {:ok, Config.t()} | {:error, Changeset.t()} + def delete(params) do + with %Config{} = config <- Config.get_by_params(params) do Repo.delete(config) else - nil -> {:error, "Config with key #{key} not found"} + nil -> {:error, "Config with params #{inspect(params)} not found"} end end @@ -77,10 +78,21 @@ defmodule Pleroma.Web.AdminAPI.Config do defp do_convert({k, v} = value) when is_tuple(value), do: %{k => do_convert(v)} - defp do_convert(value) when is_binary(value) or is_atom(value) or is_map(value), - do: value + defp do_convert(value) when is_tuple(value), do: %{"tuple" => do_convert(Tuple.to_list(value))} + + defp do_convert(value) when is_binary(value) or is_map(value) or is_number(value), do: value + + defp do_convert(value) when is_atom(value) do + string = to_string(value) + + if String.starts_with?(string, "Elixir."), + do: String.trim_leading(string, "Elixir."), + else: value + end @spec transform(any()) :: binary() + def transform(%{"tuple" => _} = entity), do: :erlang.term_to_binary(do_transform(entity)) + def transform(entity) when is_map(entity) do tuples = for {k, v} <- entity, @@ -101,11 +113,16 @@ defmodule Pleroma.Web.AdminAPI.Config do defp do_transform(%Regex{} = value) when is_map(value), do: value + defp do_transform(%{"tuple" => [k, values] = entity}) when length(entity) == 2 do + {do_transform(k), do_transform(values)} + end + + defp do_transform(%{"tuple" => values}) do + Enum.reduce(values, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end) + end + defp do_transform(value) when is_map(value) do - values = - for {key, val} <- value, - into: [], - do: {String.to_atom(key), do_transform(val)} + values = for {key, val} <- value, into: [], do: {String.to_atom(key), do_transform(val)} Enum.sort(values) end @@ -117,28 +134,27 @@ defmodule Pleroma.Web.AdminAPI.Config do defp do_transform(entity) when is_list(entity) and length(entity) == 1, do: hd(entity) defp do_transform(value) when is_binary(value) do - value = String.trim(value) + String.trim(value) + |> do_transform_string() + end - case String.length(value) do - 0 -> - nil + defp do_transform(value), do: value - _ -> - cond do - String.starts_with?(value, "Pleroma") -> - String.to_existing_atom("Elixir." <> value) + defp do_transform_string(value) when byte_size(value) == 0, do: nil - String.starts_with?(value, ":") -> - String.replace(value, ":", "") |> String.to_existing_atom() + defp do_transform_string(value) do + cond do + String.starts_with?(value, "Pleroma") or String.starts_with?(value, "Phoenix") -> + String.to_existing_atom("Elixir." <> value) - String.starts_with?(value, "i:") -> - String.replace(value, "i:", "") |> String.to_integer() + String.starts_with?(value, ":") -> + String.replace(value, ":", "") |> String.to_existing_atom() - true -> - value - end + String.starts_with?(value, "i:") -> + String.replace(value, "i:", "") |> String.to_integer() + + true -> + value end end - - defp do_transform(value), do: value end diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index 28bb667d8..7e1b9c431 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -5,8 +5,11 @@ defmodule Pleroma.Web.AdminAPI.AccountView do use Pleroma.Web, :view + alias Pleroma.HTML + alias Pleroma.User alias Pleroma.User.Info alias Pleroma.Web.AdminAPI.AccountView + alias Pleroma.Web.MediaProxy def render("index.json", %{users: users, count: count, page_size: page_size}) do %{ @@ -17,9 +20,14 @@ defmodule Pleroma.Web.AdminAPI.AccountView do end def render("show.json", %{user: user}) do + avatar = User.avatar_url(user) |> MediaProxy.url() + display_name = HTML.strip_tags(user.name || user.nickname) + %{ "id" => user.id, + "avatar" => avatar, "nickname" => user.nickname, + "display_name" => display_name, "deactivated" => user.info.deactivated, "local" => user.local, "roles" => Info.roles(user.info), diff --git a/lib/pleroma/web/admin_api/views/config_view.ex b/lib/pleroma/web/admin_api/views/config_view.ex index c8560033e..3ccc9ca46 100644 --- a/lib/pleroma/web/admin_api/views/config_view.ex +++ b/lib/pleroma/web/admin_api/views/config_view.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigView do def render("show.json", %{config: config}) do %{ key: config.key, + group: config.group, value: Pleroma.Web.AdminAPI.Config.from_binary_to_map(config.value) } end diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index e7db3a8ff..a25f3f1fe 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.ReportView do alias Pleroma.HTML alias Pleroma.User alias Pleroma.Web.CommonAPI.Utils - alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.StatusView def render("index.json", %{reports: reports}) do @@ -38,12 +37,19 @@ defmodule Pleroma.Web.AdminAPI.ReportView do %{ id: report.id, - account: AccountView.render("account.json", %{user: account}), - actor: AccountView.render("account.json", %{user: user}), + account: merge_account_views(account), + actor: merge_account_views(user), content: content, created_at: created_at, statuses: StatusView.render("index.json", %{activities: statuses, as: :activity}), state: report.data["state"] } end + + defp merge_account_views(%User{} = user) do + Pleroma.Web.MastodonAPI.AccountView.render("account.json", %{user: user}) + |> Map.merge(Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user})) + end + + defp merge_account_views(_), do: %{} end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 42b78494d..f71c67a3d 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.ActivityPub.Visibility import Pleroma.Web.CommonAPI.Utils @@ -247,6 +248,7 @@ defmodule Pleroma.Web.CommonAPI do res else + {:private_to_public, true} -> {:error, "The message visibility must be direct"} {:error, _} = e -> e e -> {:error, e} end @@ -283,12 +285,11 @@ defmodule Pleroma.Web.CommonAPI do }, object: %Object{ data: %{ - "to" => object_to, "type" => "Note" } } } = activity <- get_by_id_or_ap_id(id_or_ap_id), - true <- Enum.member?(object_to, "https://www.w3.org/ns/activitystreams#Public"), + true <- Visibility.is_public?(activity), %{valid?: true} = info_changeset <- User.Info.add_pinnned_activity(user.info, activity), changeset <- diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f4c9fe284..f4f9e83e0 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -22,6 +22,18 @@ defmodule Pleroma.Web.Federator do refresh_subscriptions() end + @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)" + # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength + def allowed_incoming_reply_depth?(depth) do + max_replies_depth = Pleroma.Config.get([:instance, :federation_incoming_replies_max_depth]) + + if max_replies_depth do + (depth || 1) <= max_replies_depth + else + true + end + end + # Client API def incoming_doc(doc) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index d6aacd288..0d3a878bb 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -419,6 +419,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do def user_statuses(%{assigns: %{user: reading_user}} = conn, params) do with %User{} = user <- User.get_cached_by_id(params["id"]) do + params = + params + |> Map.put("tag", params["tagged"]) + activities = ActivityPub.fetch_user_activities(user, reading_user, params) conn @@ -624,18 +628,13 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do else params = Map.drop(params, ["scheduled_at"]) - case get_cached_status_or_post(conn, params) do - {:ignore, message} -> - conn - |> put_status(422) - |> json(%{error: message}) - + case CommonAPI.post(user, params) do {:error, message} -> conn - |> put_status(422) + |> put_status(:unprocessable_entity) |> json(%{error: message}) - {_, activity} -> + {:ok, activity} -> conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) @@ -643,21 +642,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end end - defp get_cached_status_or_post(%{assigns: %{user: user}} = conn, params) do - idempotency_key = - case get_req_header(conn, "idempotency-key") do - [key] -> key - _ -> Ecto.UUID.generate() - end - - Cachex.fetch(:idempotency_cache, idempotency_key, fn _ -> - case CommonAPI.post(user, params) do - {:ok, activity} -> activity - {:error, message} -> {:ignore, message} - end - end) - end - def delete_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do json(conn, %{}) @@ -907,7 +891,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do conn |> put_view(AccountView) - |> render(AccountView, "accounts.json", %{for: user, users: users, as: :user}) + |> render("accounts.json", %{for: user, users: users, as: :user}) else _ -> json(conn, []) end diff --git a/lib/pleroma/web/mastodon_api/search_controller.ex b/lib/pleroma/web/mastodon_api/search_controller.ex index 0d1e2355d..efa9cc788 100644 --- a/lib/pleroma/web/mastodon_api/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/search_controller.ex @@ -17,8 +17,8 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do plug(Pleroma.Plugs.RateLimiter, :search when action in [:search, :search2, :account_search]) def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do - accounts = User.search(query, search_options(params, user)) - statuses = Activity.search(user, query) + accounts = with_fallback(fn -> User.search(query, search_options(params, user)) end, []) + statuses = with_fallback(fn -> Activity.search(user, query) end, []) tags_path = Web.base_url() <> "/tag/" tags = @@ -40,8 +40,8 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do end def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do - accounts = User.search(query, search_options(params, user)) - statuses = Activity.search(user, query) + accounts = with_fallback(fn -> User.search(query, search_options(params, user)) end, []) + statuses = with_fallback(fn -> Activity.search(user, query) end, []) tags = query @@ -76,4 +76,14 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do for_user: user ] end + + defp with_fallback(f, fallback) do + try do + f.() + rescue + error -> + Logger.error("#{__MODULE__} search error: #{inspect(error)}") + fallback + end + end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 6836d331a..ec582b919 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -104,7 +104,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do id: to_string(activity.id), uri: activity_object.data["id"], url: activity_object.data["id"], - account: AccountView.render("account.json", %{user: user}), + account: AccountView.render("account.json", %{user: user, for: opts[:for]}), in_reply_to_id: nil, in_reply_to_account_id: nil, reblog: reblogged, @@ -221,7 +221,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do id: to_string(activity.id), uri: object.data["id"], url: url, - account: AccountView.render("account.json", %{user: user}), + account: AccountView.render("account.json", %{user: user, for: opts[:for]}), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), reblog: nil, diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 3299e1721..dbd3542ea 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -29,9 +29,10 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do def init(%{qs: qs} = req, state) do with params <- :cow_qs.parse_qs(qs), + sec_websocket <- :cowboy_req.header("sec-websocket-protocol", req, nil), access_token <- List.keyfind(params, "access_token", 0), {_, stream} <- List.keyfind(params, "stream", 0), - {:ok, user} <- allow_request(stream, access_token), + {:ok, user} <- allow_request(stream, [access_token, sec_websocket]), topic when is_binary(topic) <- expand_topic(stream, params) do {:cowboy_websocket, req, %{user: user, topic: topic}, %{idle_timeout: @timeout}} else @@ -84,13 +85,21 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do end # Public streams without authentication. - defp allow_request(stream, nil) when stream in @anonymous_streams do + defp allow_request(stream, [nil, nil]) when stream in @anonymous_streams do {:ok, nil} end # Authenticated streams. - defp allow_request(stream, {"access_token", access_token}) when stream in @streams do - with %Token{user_id: user_id} <- Repo.get_by(Token, token: access_token), + defp allow_request(stream, [access_token, sec_websocket]) when stream in @streams do + token = + with {"access_token", token} <- access_token do + token + else + _ -> sec_websocket + end + + with true <- is_bitstring(token), + %Token{user_id: user_id} <- Repo.get_by(Token, token: token), user = %User{} <- User.get_cached_by_id(user_id) do {:ok, user} else diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index cee6d8481..dd8888a02 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -33,20 +33,7 @@ defmodule Pleroma.Web.MediaProxy do def encode_url(url) do secret = Pleroma.Config.get([Pleroma.Web.Endpoint, :secret_key_base]) - - # Must preserve `%2F` for compatibility with S3 - # https://git.pleroma.social/pleroma/pleroma/issues/580 - replacement = get_replacement(url, ":2F:") - - # The URL is url-decoded and encoded again to ensure it is correctly encoded and not twice. - base64 = - url - |> String.replace("%2F", replacement) - |> URI.decode() - |> URI.encode() - |> String.replace(replacement, "%2F") - |> Base.url_encode64(@base64_opts) - + base64 = Base.url_encode64(url, @base64_opts) sig = :crypto.hmac(:sha, secret, base64) sig64 = sig |> Base.url_encode64(@base64_opts) @@ -80,12 +67,4 @@ defmodule Pleroma.Web.MediaProxy do |> Enum.filter(fn value -> value end) |> Path.join() end - - defp get_replacement(url, replacement) do - if String.contains?(url, replacement) do - get_replacement(url, replacement <> replacement) - else - replacement - end - end end diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 357b80a2d..4033ec38f 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -121,4 +121,6 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do acc ++ rendered_tags end) end + + defp build_attachments(_), do: [] end diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex index 040b872e7..8dd01e0d5 100644 --- a/lib/pleroma/web/metadata/twitter_card.ex +++ b/lib/pleroma/web/metadata/twitter_card.ex @@ -117,6 +117,8 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do end) end + defp build_attachments(_id, _object), do: [] + defp player_url(id) do Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :notice_player, id) end diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 32be430b7..869dda5c5 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -162,7 +162,8 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do accountActivationRequired: Config.get([:instance, :account_activation_required], false), invitesEnabled: Config.get([:instance, :invites_enabled], false), features: features, - restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]) + restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]), + skipThreadContainment: Config.get([:instance, :skip_thread_containment], false) } } end diff --git a/lib/pleroma/web/oauth/authorization.ex b/lib/pleroma/web/oauth/authorization.ex index 18973413e..d53e20d12 100644 --- a/lib/pleroma/web/oauth/authorization.ex +++ b/lib/pleroma/web/oauth/authorization.ex @@ -76,14 +76,16 @@ defmodule Pleroma.Web.OAuth.Authorization do def use_token(%Authorization{used: true}), do: {:error, "already used"} @spec delete_user_authorizations(User.t()) :: {integer(), any()} - def delete_user_authorizations(%User{id: user_id}) do - from( - a in Pleroma.Web.OAuth.Authorization, - where: a.user_id == ^user_id - ) + def delete_user_authorizations(%User{} = user) do + user + |> delete_by_user_query |> Repo.delete_all() end + def delete_by_user_query(%User{id: user_id}) do + from(a in __MODULE__, where: a.user_id == ^user_id) + end + @doc "gets auth for app by token" @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} def get_by_token(%App{id: app_id} = _app, token) do diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index ec6e5cfaf..8e0adad91 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Federator alias Pleroma.Web.OStatus alias Pleroma.Web.XML @@ -88,14 +89,15 @@ defmodule Pleroma.Web.OStatus.NoteHandler do Map.put(note, "external_url", url) end - def fetch_replied_to_activity(entry, in_reply_to) do + def fetch_replied_to_activity(entry, in_reply_to, options \\ []) do with %Activity{} = activity <- Activity.get_create_by_object_ap_id(in_reply_to) do activity else _e -> - with in_reply_to_href when not is_nil(in_reply_to_href) <- + with true <- Federator.allowed_incoming_reply_depth?(options[:depth]), + in_reply_to_href when not is_nil(in_reply_to_href) <- XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry), - {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href) do + {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href, options) do activity else _e -> nil @@ -104,7 +106,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do end # TODO: Clean this up a bit. - def handle_note(entry, doc \\ nil) do + def handle_note(entry, doc \\ nil, options \\ []) do with id <- XML.string_from_xpath("//id", entry), activity when is_nil(activity) <- Activity.get_create_by_object_ap_id_with_object(id), [author] <- :xmerl_xpath.string('//author[1]', doc), @@ -112,7 +114,8 @@ defmodule Pleroma.Web.OStatus.NoteHandler do content_html <- OStatus.get_content(entry), cw <- OStatus.get_cw(entry), in_reply_to <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry), - in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to), + options <- Keyword.put(options, :depth, (options[:depth] || 0) + 1), + in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to, options), in_reply_to_object <- (in_reply_to_activity && Object.normalize(in_reply_to_activity)) || nil, in_reply_to <- (in_reply_to_object && in_reply_to_object.data["id"]) || in_reply_to, diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 6ed089d84..502410c83 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -54,7 +54,7 @@ defmodule Pleroma.Web.OStatus do "#{Web.base_url()}/ostatus_subscribe?acct={uri}" end - def handle_incoming(xml_string) do + def handle_incoming(xml_string, options \\ []) do with doc when doc != :error <- parse_document(xml_string) do with {:ok, actor_user} <- find_make_or_update_user(doc), do: Pleroma.Instances.set_reachable(actor_user.ap_id) @@ -91,10 +91,12 @@ defmodule Pleroma.Web.OStatus do _ -> case object_type do 'http://activitystrea.ms/schema/1.0/note' -> - with {:ok, activity} <- NoteHandler.handle_note(entry, doc), do: activity + with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options), + do: activity 'http://activitystrea.ms/schema/1.0/comment' -> - with {:ok, activity} <- NoteHandler.handle_note(entry, doc), do: activity + with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options), + do: activity _ -> Logger.error("Couldn't parse incoming document") @@ -359,7 +361,7 @@ defmodule Pleroma.Web.OStatus do end end - def fetch_activity_from_atom_url(url) do + def fetch_activity_from_atom_url(url, options \\ []) do with true <- String.starts_with?(url, "http"), {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get( @@ -367,7 +369,7 @@ defmodule Pleroma.Web.OStatus do [{:Accept, "application/atom+xml"}] ) do Logger.debug("Got document from #{url}, handling...") - handle_incoming(body) + handle_incoming(body, options) else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}") @@ -375,13 +377,13 @@ defmodule Pleroma.Web.OStatus do end end - def fetch_activity_from_html_url(url) do + def fetch_activity_from_html_url(url, options \\ []) do Logger.debug("Trying to fetch #{url}") with true <- String.starts_with?(url, "http"), {:ok, %{body: body}} <- HTTP.get(url, []), {:ok, atom_url} <- get_atom_url(body) do - fetch_activity_from_atom_url(atom_url) + fetch_activity_from_atom_url(atom_url, options) else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}") @@ -389,11 +391,11 @@ defmodule Pleroma.Web.OStatus do end end - def fetch_activity_from_url(url) do - with {:ok, [_ | _] = activities} <- fetch_activity_from_atom_url(url) do + def fetch_activity_from_url(url, options \\ []) do + with {:ok, [_ | _] = activities} <- fetch_activity_from_atom_url(url, options) do {:ok, activities} else - _e -> fetch_activity_from_html_url(url) + _e -> fetch_activity_from_html_url(url, options) end rescue e -> diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 94f56f70d..6506de46c 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -4,27 +4,53 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Web.RichMedia.Parser + @spec validate_page_url(any()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld] - if AutoLinker.Parser.url?(page_url, scheme: true, validate_tld: validate_tld) do - URI.parse(page_url) |> validate_page_url - else - :error + page_url + |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld) + |> parse_uri(page_url) + end + + defp validate_page_url(%URI{host: host, scheme: scheme, authority: authority}) + when scheme == "https" and not is_nil(authority) do + cond do + host in Config.get([:rich_media, :ignore_hosts], []) -> + :error + + get_tld(host) in Config.get([:rich_media, :ignore_tld], []) -> + :error + + true -> + :ok end end - defp validate_page_url(%URI{authority: nil}), do: :error - defp validate_page_url(%URI{scheme: nil}), do: :error - defp validate_page_url(%URI{}), do: :ok defp validate_page_url(_), do: :error + defp parse_uri(true, url) do + url + |> URI.parse() + |> validate_page_url + end + + defp parse_uri(_, _), do: :error + + defp get_tld(host) do + host + |> String.split(".") + |> Enum.reverse() + |> hd + end + def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do - with true <- Pleroma.Config.get([:rich_media, :enabled]), + with true <- Config.get([:rich_media, :enabled]), %Object{} = object <- Object.normalize(activity), false <- object.data["sensitive"] || false, {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]), diff --git a/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex b/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex index 4a7c5eae0..fb79630e4 100644 --- a/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex +++ b/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex @@ -1,15 +1,19 @@ defmodule Pleroma.Web.RichMedia.Parsers.MetaTagsParser do def parse(html, data, prefix, error_message, key_name, value_name \\ "content") do - with elements = [_ | _] <- get_elements(html, key_name, prefix), - meta_data = - Enum.reduce(elements, data, fn el, acc -> - attributes = normalize_attributes(el, prefix, key_name, value_name) + meta_data = + html + |> get_elements(key_name, prefix) + |> Enum.reduce(data, fn el, acc -> + attributes = normalize_attributes(el, prefix, key_name, value_name) - Map.merge(acc, attributes) - end) do - {:ok, meta_data} + Map.merge(acc, attributes) + end) + |> maybe_put_title(html) + + if Enum.empty?(meta_data) do + {:error, error_message} else - _e -> {:error, error_message} + {:ok, meta_data} end end @@ -27,4 +31,19 @@ defmodule Pleroma.Web.RichMedia.Parsers.MetaTagsParser do %{String.to_atom(data[key_name]) => data[value_name]} end + + defp maybe_put_title(%{title: _} = meta, _), do: meta + + defp maybe_put_title(meta, html) when meta != %{} do + case get_page_title(html) do + "" -> meta + title -> Map.put_new(meta, :title, title) + end + end + + defp maybe_put_title(meta, _), do: meta + + defp get_page_title(html) do + Floki.find(html, "title") |> Floki.text() + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 36458b2f4..d53fa8a35 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -27,6 +27,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Plugs.UserEnabledPlug) plug(Pleroma.Plugs.SetUserSessionIdPlug) plug(Pleroma.Plugs.EnsureUserKeyPlug) + plug(Pleroma.Plugs.IdempotencyPlug) end pipeline :authenticated_api do @@ -41,6 +42,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Plugs.UserEnabledPlug) plug(Pleroma.Plugs.SetUserSessionIdPlug) plug(Pleroma.Plugs.EnsureAuthenticatedPlug) + plug(Pleroma.Plugs.IdempotencyPlug) end pipeline :admin_api do @@ -57,6 +59,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Plugs.SetUserSessionIdPlug) plug(Pleroma.Plugs.EnsureAuthenticatedPlug) plug(Pleroma.Plugs.UserIsAdminPlug) + plug(Pleroma.Plugs.IdempotencyPlug) end pipeline :mastodon_html do @@ -133,8 +136,8 @@ defmodule Pleroma.Web.Router do scope "/api/pleroma", Pleroma.Web.TwitterAPI do pipe_through(:pleroma_api) - get("/password_reset/:token", UtilController, :show_password_reset) - post("/password_reset", UtilController, :password_reset) + get("/password_reset/:token", PasswordController, :reset, as: :reset_password) + post("/password_reset", PasswordController, :do_reset, as: :reset_password) get("/emoji", UtilController, :emoji) get("/captcha", UtilController, :captcha) get("/healthcheck", UtilController, :healthcheck) @@ -725,6 +728,7 @@ end defmodule Fallback.RedirectController do use Pleroma.Web, :controller + require Logger alias Pleroma.User alias Pleroma.Web.Metadata @@ -751,7 +755,20 @@ defmodule Fallback.RedirectController do def redirector_with_meta(conn, params) do {:ok, index_content} = File.read(index_file_path()) - tags = Metadata.build_tags(params) + + tags = + try do + Metadata.build_tags(params) + rescue + e -> + Logger.error( + "Metadata rendering for #{conn.request_path} failed.\n" <> + Exception.format(:error, e, __STACKTRACE__) + ) + + "" + end + response = String.replace(index_content, "<!--server-generated-meta-->", tags) conn diff --git a/lib/pleroma/web/templates/twitter_api/util/invalid_token.html.eex b/lib/pleroma/web/templates/twitter_api/password/invalid_token.html.eex index ee84750c7..ee84750c7 100644 --- a/lib/pleroma/web/templates/twitter_api/util/invalid_token.html.eex +++ b/lib/pleroma/web/templates/twitter_api/password/invalid_token.html.eex diff --git a/lib/pleroma/web/templates/twitter_api/util/password_reset.html.eex b/lib/pleroma/web/templates/twitter_api/password/reset.html.eex index a3facf017..7d3ef6b0d 100644 --- a/lib/pleroma/web/templates/twitter_api/util/password_reset.html.eex +++ b/lib/pleroma/web/templates/twitter_api/password/reset.html.eex @@ -1,5 +1,5 @@ <h2>Password Reset for <%= @user.nickname %></h2> -<%= form_for @conn, util_path(@conn, :password_reset), [as: "data"], fn f -> %> +<%= form_for @conn, reset_password_path(@conn, :do_reset), [as: "data"], fn f -> %> <div class="form-row"> <%= label f, :password, "Password" %> <%= password_input f, :password %> diff --git a/lib/pleroma/web/templates/twitter_api/util/password_reset_failed.html.eex b/lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex index df037c01e..df037c01e 100644 --- a/lib/pleroma/web/templates/twitter_api/util/password_reset_failed.html.eex +++ b/lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex diff --git a/lib/pleroma/web/templates/twitter_api/util/password_reset_success.html.eex b/lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex index f30ba3274..f30ba3274 100644 --- a/lib/pleroma/web/templates/twitter_api/util/password_reset_success.html.eex +++ b/lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex diff --git a/lib/pleroma/web/twitter_api/controllers/password_controller.ex b/lib/pleroma/web/twitter_api/controllers/password_controller.ex new file mode 100644 index 000000000..1941e6143 --- /dev/null +++ b/lib/pleroma/web/twitter_api/controllers/password_controller.ex @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.TwitterAPI.PasswordController do + @moduledoc """ + The module containts functions for reset password. + """ + + use Pleroma.Web, :controller + + require Logger + + alias Pleroma.PasswordResetToken + alias Pleroma.Repo + alias Pleroma.User + + def reset(conn, %{"token" => token}) do + with %{used: false} = token <- Repo.get_by(PasswordResetToken, %{token: token}), + %User{} = user <- User.get_cached_by_id(token.user_id) do + render(conn, "reset.html", %{ + token: token, + user: user + }) + else + _e -> render(conn, "invalid_token.html") + end + end + + def do_reset(conn, %{"data" => data}) do + with {:ok, _} <- PasswordResetToken.reset_password(data["token"], data) do + render(conn, "reset_success.html") + else + _e -> render(conn, "reset_failed.html") + end + end +end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 489170d80..b1863528f 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -11,8 +11,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Activity alias Pleroma.Emoji alias Pleroma.Notification - alias Pleroma.PasswordResetToken - alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web alias Pleroma.Web.ActivityPub.ActivityPub @@ -20,26 +18,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Web.OStatus alias Pleroma.Web.WebFinger - def show_password_reset(conn, %{"token" => token}) do - with %{used: false} = token <- Repo.get_by(PasswordResetToken, %{token: token}), - %User{} = user <- User.get_cached_by_id(token.user_id) do - render(conn, "password_reset.html", %{ - token: token, - user: user - }) - else - _e -> render(conn, "invalid_token.html") - end - end - - def password_reset(conn, %{"data" => data}) do - with {:ok, _} <- PasswordResetToken.reset_password(data["token"], data) do - render(conn, "password_reset_success.html") - else - _e -> render(conn, "password_reset_failed.html") - end - end - def help_test(conn, _params) do json(conn, "ok") end diff --git a/lib/pleroma/web/twitter_api/views/password_view.ex b/lib/pleroma/web/twitter_api/views/password_view.ex new file mode 100644 index 000000000..b166b925d --- /dev/null +++ b/lib/pleroma/web/twitter_api/views/password_view.ex @@ -0,0 +1,8 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.TwitterAPI.PasswordView do + use Pleroma.Web, :view + import Phoenix.HTML.Form +end |