From 6c39fa20b191f985a2be704089c20acbcfe0035a Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 17 Dec 2019 17:00:46 +0700 Subject: Add support for `account_id` param to filter notifications by the account --- lib/pleroma/web/mastodon_api/mastodon_api.ex | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index b1816370e..6c13d4df6 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -56,6 +56,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do user |> Notification.for_user_query(options) |> restrict(:exclude_types, options) + |> restrict(:account_id, options) |> Pagination.fetch_paginated(params) end @@ -71,7 +72,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do exclude_visibilities: {:array, :string}, reblogs: :boolean, with_muted: :boolean, - with_move: :boolean + with_move: :boolean, + account_id: :string } changeset = cast({%{}, param_types}, params, Map.keys(param_types)) @@ -88,5 +90,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do |> where([q, a], not fragment("? @> ARRAY[?->>'type']::varchar[]", ^ap_types, a.data)) end + defp restrict(query, :account_id, %{account_id: account_id}) do + case User.get_cached_by_id(account_id) do + %{ap_id: ap_id} -> where(query, [n, a], a.actor == ^ap_id) + _ -> where(query, [n, a], a.actor == "fake ap id") + end + end + defp restrict(query, _, _), do: query end -- cgit v1.2.3 From 34d85f8a5473fe0f85e8a8e9e8f58e40b3964ba4 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 19 Dec 2019 20:45:44 +0700 Subject: Return 404 if account to filter notifications from is not found --- .../mastodon_api/controllers/notification_controller.ex | 17 +++++++++++++++++ lib/pleroma/web/mastodon_api/mastodon_api.ex | 11 ++++------- 2 files changed, 21 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index 16759be6a..f2508aca4 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -23,6 +23,23 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug) # GET /api/v1/notifications + def index(conn, %{"account_id" => account_id} = params) do + case Pleroma.User.get_cached_by_id(account_id) do + %{ap_id: account_ap_id} -> + params = + params + |> Map.delete("account_id") + |> Map.put("account_ap_id", account_ap_id) + + index(conn, params) + + _ -> + conn + |> put_status(:not_found) + |> json(%{"error" => "Account is not found"}) + end + end + def index(%{assigns: %{user: user}} = conn, params) do notifications = MastodonAPI.get_notifications(user, params) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index 6c13d4df6..390a2b190 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -56,7 +56,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do user |> Notification.for_user_query(options) |> restrict(:exclude_types, options) - |> restrict(:account_id, options) + |> restrict(:account_ap_id, options) |> Pagination.fetch_paginated(params) end @@ -73,7 +73,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do reblogs: :boolean, with_muted: :boolean, with_move: :boolean, - account_id: :string + account_ap_id: :string } changeset = cast({%{}, param_types}, params, Map.keys(param_types)) @@ -90,11 +90,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do |> where([q, a], not fragment("? @> ARRAY[?->>'type']::varchar[]", ^ap_types, a.data)) end - defp restrict(query, :account_id, %{account_id: account_id}) do - case User.get_cached_by_id(account_id) do - %{ap_id: ap_id} -> where(query, [n, a], a.actor == ^ap_id) - _ -> where(query, [n, a], a.actor == "fake ap id") - end + defp restrict(query, :account_ap_id, %{account_ap_id: account_ap_id}) do + where(query, [n, a], a.actor == ^account_ap_id) end defp restrict(query, _, _), do: query -- cgit v1.2.3 From 385356aad0dd7eac0695bb1597ba1e52b5f17b40 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 24 Dec 2019 20:45:46 +0300 Subject: fix oauth scopes for AdminApi#reports_update --- lib/pleroma/web/admin_api/admin_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index c8abeff06..ddae139c6 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -66,7 +66,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do plug( OAuthScopesPlug, %{scopes: ["write:reports"], admin: true} - when action in [:report_update_state, :report_respond] + when action in [:reports_update] ) plug( -- cgit v1.2.3 From 6c94b7498b889ffe13691123c94bbe5440786852 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 10 Jan 2020 10:52:21 +0300 Subject: [#1478] OAuth admin tweaks: enforced OAuth admin scopes usage by default, migrated existing OAuth records. Adjusted tests. --- lib/pleroma/plugs/user_is_admin_plug.ex | 1 + lib/pleroma/user.ex | 23 +++++++---------------- lib/pleroma/web/oauth/oauth_controller.ex | 10 +++++----- lib/pleroma/web/oauth/scopes.ex | 24 +++--------------------- 4 files changed, 16 insertions(+), 42 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/plugs/user_is_admin_plug.ex b/lib/pleroma/plugs/user_is_admin_plug.ex index 582fb1f92..3190163d3 100644 --- a/lib/pleroma/plugs/user_is_admin_plug.ex +++ b/lib/pleroma/plugs/user_is_admin_plug.ex @@ -23,6 +23,7 @@ defmodule Pleroma.Plugs.UserIsAdminPlug do token && OAuth.Scopes.contains_admin_scopes?(token.scopes) -> # Note: checking for _any_ admin scope presence, not necessarily fitting requested action. # Thus, controller must explicitly invoke OAuthScopesPlug to verify scope requirements. + # Admin might opt out of admin scope for some apps to block any admin actions from them. conn true -> diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 706aee2ff..40cc93e03 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1847,22 +1847,13 @@ defmodule Pleroma.User do end def admin_api_update(user, params) do - changeset = - cast(user, params, [ - :is_moderator, - :is_admin, - :show_role - ]) - - with {:ok, updated_user} <- update_and_set_cache(changeset) do - if user.is_admin && !updated_user.is_admin do - # Tokens & authorizations containing any admin scopes must be revoked (revoking all). - # This is an extra safety measure (tokens' admin scopes won't be accepted for non-admins). - global_sign_out(user) - end - - {:ok, updated_user} - end + user + |> cast(params, [ + :is_moderator, + :is_admin, + :show_role + ]) + |> update_and_set_cache() end @doc "Signs user out of all applications" diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 87acdec97..d31a3d91c 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -222,7 +222,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do {:user_active, true} <- {:user_active, !user.deactivated}, {:password_reset_pending, false} <- {:password_reset_pending, user.password_reset_pending}, - {:ok, scopes} <- validate_scopes(app, params, user), + {:ok, scopes} <- validate_scopes(app, params), {:ok, auth} <- Authorization.create_authorization(app, user, scopes), {:ok, token} <- Token.exchange_token(app, auth) do json(conn, Token.Response.build(user, token)) @@ -471,7 +471,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do {:get_user, (user && {:ok, user}) || Authenticator.get_user(conn)}, %App{} = app <- Repo.get_by(App, client_id: client_id), true <- redirect_uri in String.split(app.redirect_uris), - {:ok, scopes} <- validate_scopes(app, auth_attrs, user), + {:ok, scopes} <- validate_scopes(app, auth_attrs), {:auth_active, true} <- {:auth_active, User.auth_active?(user)} do Authorization.create_authorization(app, user, scopes) end @@ -487,12 +487,12 @@ defmodule Pleroma.Web.OAuth.OAuthController do defp put_session_registration_id(%Plug.Conn{} = conn, registration_id), do: put_session(conn, :registration_id, registration_id) - @spec validate_scopes(App.t(), map(), User.t()) :: + @spec validate_scopes(App.t(), map()) :: {:ok, list()} | {:error, :missing_scopes | :unsupported_scopes} - defp validate_scopes(%App{} = app, params, %User{} = user) do + defp validate_scopes(%App{} = app, params) do params |> Scopes.fetch_scopes(app.scopes) - |> Scopes.validate(app.scopes, user) + |> Scopes.validate(app.scopes) end def default_redirect_uri(%App{} = app) do diff --git a/lib/pleroma/web/oauth/scopes.ex b/lib/pleroma/web/oauth/scopes.ex index 00da225b9..151467494 100644 --- a/lib/pleroma/web/oauth/scopes.ex +++ b/lib/pleroma/web/oauth/scopes.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.OAuth.Scopes do """ alias Pleroma.Plugs.OAuthScopesPlug - alias Pleroma.User @doc """ Fetch scopes from request params. @@ -56,35 +55,18 @@ defmodule Pleroma.Web.OAuth.Scopes do @doc """ Validates scopes. """ - @spec validate(list() | nil, list(), User.t()) :: + @spec validate(list() | nil, list()) :: {:ok, list()} | {:error, :missing_scopes | :unsupported_scopes} - def validate(blank_scopes, _app_scopes, _user) when blank_scopes in [nil, []], + def validate(blank_scopes, _app_scopes) when blank_scopes in [nil, []], do: {:error, :missing_scopes} - def validate(scopes, app_scopes, %User{} = user) do - with {:ok, _} <- ensure_scopes_support(scopes, app_scopes), - {:ok, scopes} <- authorize_admin_scopes(scopes, app_scopes, user) do - {:ok, scopes} - end - end - - defp ensure_scopes_support(scopes, app_scopes) do + def validate(scopes, app_scopes) do case OAuthScopesPlug.filter_descendants(scopes, app_scopes) do ^scopes -> {:ok, scopes} _ -> {:error, :unsupported_scopes} end end - defp authorize_admin_scopes(scopes, app_scopes, %User{} = user) do - if user.is_admin || !contains_admin_scopes?(scopes) || !contains_admin_scopes?(app_scopes) do - {:ok, scopes} - else - # Gracefully dropping admin scopes from requested scopes if user isn't an admin (not raising) - scopes = scopes -- OAuthScopesPlug.filter_descendants(scopes, ["admin"]) - validate(scopes, app_scopes, user) - end - end - def contains_admin_scopes?(scopes) do scopes |> OAuthScopesPlug.filter_descendants(["admin"]) -- cgit v1.2.3 From 2ee6754095f5b81807efe97c73ada42e2c990ede Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 14 Jan 2020 17:24:26 +0100 Subject: Mix Tasks: Add pleroma.benchmarks.tags --- lib/mix/tasks/pleroma/benchmarks/tags.ex | 57 ++++++++++++++++++++++ .../controllers/timeline_controller.ex | 14 ++++-- 2 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 lib/mix/tasks/pleroma/benchmarks/tags.ex (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/benchmarks/tags.ex b/lib/mix/tasks/pleroma/benchmarks/tags.ex new file mode 100644 index 000000000..73796b5f9 --- /dev/null +++ b/lib/mix/tasks/pleroma/benchmarks/tags.ex @@ -0,0 +1,57 @@ +defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do + use Mix.Task + alias Pleroma.Repo + alias Pleroma.LoadTesting.Generator + import Ecto.Query + + def run(_args) do + Mix.Pleroma.start_pleroma() + activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id) + + if activities_count == 0 do + IO.puts("Did not find any activities, cleaning and generating") + clean_tables() + Generator.generate_users(users_max: 10) + Generator.generate_tagged_activities() + else + IO.puts("Found #{activities_count} activities, won't generate new ones") + end + + tags = Enum.map(0..20, fn i -> {"For #tag_#{i}", "tag_#{i}"} end) + + Enum.each(tags, fn {_, tag} -> + query = + from(o in Pleroma.Object, + where: fragment("(?)->'tag' \\? (?)", o.data, ^tag) + ) + + count = Repo.aggregate(query, :count, :id) + IO.puts("Database contains #{count} posts tagged with #{tag}") + end) + + user = Repo.all(Pleroma.User) |> List.first() + + Benchee.run( + %{ + "Hashtag fetching" => fn tag -> + Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( + %{ + "tag" => tag + }, + user, + false + ) + end + }, + inputs: tags, + time: 5 + ) + end + + defp clean_tables do + IO.puts("Deleting old data...\n") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;") + end +end diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index 384159336..29964a1d4 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -77,10 +77,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do |> render("index.json", activities: activities, for: user, as: :activity) end - # GET /api/v1/timelines/tag/:tag - def hashtag(%{assigns: %{user: user}} = conn, params) do - local_only = truthy_param?(params["local"]) - + def hashtag_fetching(params, user, local_only) do tags = [params["tag"], params["any"]] |> List.flatten() @@ -98,7 +95,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do |> Map.get("none", []) |> Enum.map(&String.downcase(&1)) - activities = + _activities = params |> Map.put("type", "Create") |> Map.put("local_only", local_only) @@ -109,6 +106,13 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do |> Map.put("tag_all", tag_all) |> Map.put("tag_reject", tag_reject) |> ActivityPub.fetch_public_activities() + end + + # GET /api/v1/timelines/tag/:tag + def hashtag(%{assigns: %{user: user}} = conn, params) do + local_only = truthy_param?(params["local"]) + + activities = hashtag_fetching(params, user, local_only) conn |> add_link_headers(activities, %{"local" => local_only}) -- cgit v1.2.3 From 0709757e47a546225163482474a46e8bc3d09837 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Tue, 14 Jan 2020 19:53:28 +0300 Subject: Check if object->data is array when looking up attachment objects to delete --- lib/pleroma/object.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 2452a7389..6c56a9c62 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -203,7 +203,8 @@ defmodule Pleroma.Object do from(o in Object, where: fragment( - "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href'))::jsonb \\?| (?)", + "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", + o.data, o.data, ^hrefs ) -- cgit v1.2.3 From 76c1948880687ed74e0275e51808c0ddc6be887d Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 15 Jan 2020 12:11:23 +0300 Subject: [#1478] Adjusted AdminAPIController admin subscope requirements. --- lib/pleroma/web/admin_api/admin_api_controller.ex | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index c8abeff06..529169c1b 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -32,19 +32,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do plug( OAuthScopesPlug, %{scopes: ["read:accounts"], admin: true} - when action in [:list_users, :user_show, :right_get, :invites] + when action in [:list_users, :user_show, :right_get] ) plug( OAuthScopesPlug, %{scopes: ["write:accounts"], admin: true} when action in [ - :get_invite_token, - :revoke_invite, - :email_invite, :get_password_reset, - :user_follow, - :user_unfollow, :user_delete, :users_create, :user_toggle_activation, @@ -57,6 +52,20 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do ] ) + plug(OAuthScopesPlug, %{scopes: ["read:invites"], admin: true} when action == :invites) + + plug( + OAuthScopesPlug, + %{scopes: ["write:invites"], admin: true} + when action in [:create_invite_token, :revoke_invite, :email_invite] + ) + + plug( + OAuthScopesPlug, + %{scopes: ["write:follows"], admin: true} + when action in [:user_follow, :user_unfollow, :relay_follow, :relay_unfollow] + ) + plug( OAuthScopesPlug, %{scopes: ["read:reports"], admin: true} @@ -90,7 +99,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do plug( OAuthScopesPlug, %{scopes: ["write"], admin: true} - when action in [:relay_follow, :relay_unfollow, :config_update] + when action == :config_update ) @users_page_size 50 -- cgit v1.2.3 From 167e9c45eccf5ddb89077c979b1d587318f78cc0 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 15 Jan 2020 12:37:50 +0100 Subject: Benchmarks: Move to correct folder --- lib/mix/tasks/pleroma/benchmarks/tags.ex | 57 -------------------------------- 1 file changed, 57 deletions(-) delete mode 100644 lib/mix/tasks/pleroma/benchmarks/tags.ex (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/benchmarks/tags.ex b/lib/mix/tasks/pleroma/benchmarks/tags.ex deleted file mode 100644 index 73796b5f9..000000000 --- a/lib/mix/tasks/pleroma/benchmarks/tags.ex +++ /dev/null @@ -1,57 +0,0 @@ -defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do - use Mix.Task - alias Pleroma.Repo - alias Pleroma.LoadTesting.Generator - import Ecto.Query - - def run(_args) do - Mix.Pleroma.start_pleroma() - activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id) - - if activities_count == 0 do - IO.puts("Did not find any activities, cleaning and generating") - clean_tables() - Generator.generate_users(users_max: 10) - Generator.generate_tagged_activities() - else - IO.puts("Found #{activities_count} activities, won't generate new ones") - end - - tags = Enum.map(0..20, fn i -> {"For #tag_#{i}", "tag_#{i}"} end) - - Enum.each(tags, fn {_, tag} -> - query = - from(o in Pleroma.Object, - where: fragment("(?)->'tag' \\? (?)", o.data, ^tag) - ) - - count = Repo.aggregate(query, :count, :id) - IO.puts("Database contains #{count} posts tagged with #{tag}") - end) - - user = Repo.all(Pleroma.User) |> List.first() - - Benchee.run( - %{ - "Hashtag fetching" => fn tag -> - Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( - %{ - "tag" => tag - }, - user, - false - ) - end - }, - inputs: tags, - time: 5 - ) - end - - defp clean_tables do - IO.puts("Deleting old data...\n") - Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;") - Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;") - Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;") - end -end -- cgit v1.2.3 From 81133702d4246e5e5c17ea4d4119747dc880edd6 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Fri, 17 Jan 2020 20:20:37 +0300 Subject: Delete attachments after object and cache cleanup --- lib/pleroma/object.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 6c56a9c62..499339982 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -180,10 +180,10 @@ defmodule Pleroma.Object do def delete(%Object{data: %{"id" => id}} = object) do with {:ok, _obj} = swap_object_with_tombstone(object), - :ok <- delete_attachments(object), deleted_activity = Activity.delete_all_by_object_ap_id(id), {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), - {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do + {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path), + :ok <- delete_attachments(object) do {:ok, object, deleted_activity} end end -- cgit v1.2.3 From 6cf3b19a38d08b37a686cd1059604816d11708f1 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 19 Jan 2020 07:02:16 +0100 Subject: transmogrifier.ex: simplify handle_incoming of actors --- lib/pleroma/web/activity_pub/transmogrifier.ex | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 3fa789d53..2b8bfc3bd 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -658,24 +658,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do with %User{ap_id: ^actor_id} = actor <- User.get_cached_by_ap_id(object["id"]) do {:ok, new_user_data} = ActivityPub.user_data_from_user_object(object) - locked = new_user_data[:locked] || false - attachment = get_in(new_user_data, [:source_data, "attachment"]) || [] - invisible = new_user_data[:invisible] || false - - fields = - attachment - |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) - |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) - - update_data = - new_user_data - |> Map.take([:avatar, :banner, :bio, :name, :also_known_as]) - |> Map.put(:fields, fields) - |> Map.put(:locked, locked) - |> Map.put(:invisible, invisible) - actor - |> User.upgrade_changeset(update_data, true) + |> User.upgrade_changeset(new_user_data, true) |> User.update_and_set_cache() ActivityPub.update(%{ -- cgit v1.2.3 From d6a532bf0f280cc191a9f2c1f53af31c451481d9 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sun, 19 Jan 2020 19:45:20 +0300 Subject: Delete attachments asynchronously --- lib/pleroma/object.ex | 79 ++------------------ lib/pleroma/workers/attachments_cleanup_worker.ex | 87 +++++++++++++++++++++++ 2 files changed, 93 insertions(+), 73 deletions(-) create mode 100644 lib/pleroma/workers/attachments_cleanup_worker.ex (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 499339982..38e372f6d 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -19,6 +19,8 @@ defmodule Pleroma.Object do @type t() :: %__MODULE__{} + @derive {Jason.Encoder, only: [:data]} + schema "objects" do field(:data, :map) @@ -183,83 +185,14 @@ defmodule Pleroma.Object do deleted_activity = Activity.delete_all_by_object_ap_id(id), {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path), - :ok <- delete_attachments(object) do + {:ok, _} <- + Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{ + "object" => object + }) do {:ok, object, deleted_activity} end end - defp delete_attachments(%{data: %{"attachment" => [_ | _] = attachments, "actor" => actor}}) do - hrefs = - Enum.flat_map(attachments, fn attachment -> - Enum.map(attachment["url"], & &1["href"]) - end) - - names = Enum.map(attachments, & &1["name"]) - - uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) - - # find all objects for copies of the attachments, name and actor doesn't matter here - delete_ids = - from(o in Object, - where: - fragment( - "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", - o.data, - o.data, - ^hrefs - ) - ) - |> Repo.all() - # we should delete 1 object for any given attachment, but don't delete files if - # there are more than 1 object for it - |> Enum.reduce(%{}, fn %{ - id: id, - data: %{ - "url" => [%{"href" => href}], - "actor" => obj_actor, - "name" => name - } - }, - acc -> - Map.update(acc, href, %{id: id, count: 1}, fn val -> - case obj_actor == actor and name in names do - true -> - # set id of the actor's object that will be deleted - %{val | id: id, count: val.count + 1} - - false -> - # another actor's object, just increase count to not delete file - %{val | count: val.count + 1} - end - end) - end) - |> Enum.map(fn {href, %{id: id, count: count}} -> - # only delete files that have single instance - with 1 <- count do - prefix = - case Pleroma.Config.get([Pleroma.Upload, :base_url]) do - nil -> "media" - _ -> "" - end - - base_url = Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url()) - - file_path = String.trim_leading(href, "#{base_url}/#{prefix}") - - uploader.delete_file(file_path) - end - - id - end) - - from(o in Object, where: o.id in ^delete_ids) - |> Repo.delete_all() - - :ok - end - - defp delete_attachments(%{data: _data}), do: :ok - def prune(%Object{data: %{"id" => id}} = object) do with {:ok, object} <- Repo.delete(object), {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex new file mode 100644 index 000000000..f8239ece6 --- /dev/null +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.AttachmentsCleanupWorker do + import Ecto.Query + + alias Pleroma.Object + alias Pleroma.Repo + + use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup" + + @impl Oban.Worker + def perform( + %{"object" => %{"data" => %{"attachment" => [_ | _] = attachments, "actor" => actor}}}, + _job + ) do + hrefs = + Enum.flat_map(attachments, fn attachment -> + Enum.map(attachment["url"], & &1["href"]) + end) + + names = Enum.map(attachments, & &1["name"]) + + uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + + # find all objects for copies of the attachments, name and actor doesn't matter here + delete_ids = + from(o in Object, + where: + fragment( + "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", + o.data, + o.data, + ^hrefs + ) + ) + # The query above can be time consumptive on large instances until we refactor how uploads are stored + |> Repo.all(timout: :infinity) + # we should delete 1 object for any given attachment, but don't delete files if + # there are more than 1 object for it + |> Enum.reduce(%{}, fn %{ + id: id, + data: %{ + "url" => [%{"href" => href}], + "actor" => obj_actor, + "name" => name + } + }, + acc -> + Map.update(acc, href, %{id: id, count: 1}, fn val -> + case obj_actor == actor and name in names do + true -> + # set id of the actor's object that will be deleted + %{val | id: id, count: val.count + 1} + + false -> + # another actor's object, just increase count to not delete file + %{val | count: val.count + 1} + end + end) + end) + |> Enum.map(fn {href, %{id: id, count: count}} -> + # only delete files that have single instance + with 1 <- count do + prefix = + case Pleroma.Config.get([Pleroma.Upload, :base_url]) do + nil -> "media" + _ -> "" + end + + base_url = Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url()) + + file_path = String.trim_leading(href, "#{base_url}/#{prefix}") + + uploader.delete_file(file_path) + end + + id + end) + + from(o in Object, where: o.id in ^delete_ids) + |> Repo.delete_all() + end + + def perform(%{"object" => _object}, _job), do: :ok +end -- cgit v1.2.3 From cde828ff7df64740f36b2fc9dfdbfc8d76a8a78d Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sun, 19 Jan 2020 22:09:47 +0300 Subject: Fix credo warning --- lib/pleroma/workers/attachments_cleanup_worker.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index f8239ece6..3f421db40 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -35,10 +35,11 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do ^hrefs ) ) - # The query above can be time consumptive on large instances until we refactor how uploads are stored + # The query above can be time consumptive on large instances until we + # refactor how uploads are stored |> Repo.all(timout: :infinity) - # we should delete 1 object for any given attachment, but don't delete files if - # there are more than 1 object for it + # we should delete 1 object for any given attachment, but don't delete + # files if there are more than 1 object for it |> Enum.reduce(%{}, fn %{ id: id, data: %{ -- cgit v1.2.3 From dc0498ab2b7cee930f426e37ed4f167c2dc83262 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 16 Jan 2020 19:01:31 +0300 Subject: Check for unapplied migrations on startup Closes #1328 --- lib/pleroma/application.ex | 1 + lib/pleroma/repo.ex | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 2ae052069..e17068876 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -33,6 +33,7 @@ defmodule Pleroma.Application do def start(_type, _args) do Pleroma.HTML.compile_scrubbers() Pleroma.Config.DeprecationWarnings.warn() + Pleroma.Repo.check_migrations_applied!() setup_instrumenters() load_custom_modules() diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index f57e088bc..cb0b6653c 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Repo do adapter: Ecto.Adapters.Postgres, migration_timestamps: [type: :naive_datetime_usec] + require Logger + defmodule Instrumenter do use Prometheus.EctoInstrumenter end @@ -47,4 +49,37 @@ defmodule Pleroma.Repo do _ -> {:error, :not_found} end end + + def check_migrations_applied!() do + unless Pleroma.Config.get( + [:i_am_aware_this_may_cause_data_loss, :disable_migration_check], + false + ) do + Ecto.Migrator.with_repo(__MODULE__, fn repo -> + down_migrations = + Ecto.Migrator.migrations(repo) + |> Enum.reject(fn + {:up, _, _} -> true + {:down, _, _} -> false + end) + + if length(down_migrations) > 0 do + down_migrations_text = + Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end) + + Logger.error( + "The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true" + ) + + raise Pleroma.Repo.UnappliedMigrationsError + end + end) + else + :ok + end + end +end + +defmodule Pleroma.Repo.UnappliedMigrationsError do + defexception message: "Unapplied Migrations detected" end -- cgit v1.2.3 From 5c533e10e73d92b753d9ac20b2d7ab14f42649b2 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 20 Jan 2020 11:53:14 +0100 Subject: Bump credo to 1.1.5 --- lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex | 2 +- lib/pleroma/web/admin_api/admin_api_controller.ex | 2 +- lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex | 2 +- lib/pleroma/web/oauth/oauth_controller.ex | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index 4eaea00d8..c184c3b66 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do with accepted_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :accept]), rejected_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :reject]), true <- - length(accepted_vocabulary) == 0 || Enum.member?(accepted_vocabulary, message_type), + Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, message_type), false <- length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type), {:ok, _} <- filter(message["object"]) do diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 529169c1b..78b77a97a 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -639,7 +639,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do def force_password_reset(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do users = nicknames |> Enum.map(&User.get_cached_by_nickname/1) - Enum.map(users, &User.force_password_reset_async/1) + Enum.each(users, &User.force_password_reset_async/1) ModerationLog.insert_log(%{ actor: admin, diff --git a/lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex b/lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex index fc7d52824..11f7b85d3 100644 --- a/lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex @@ -6,9 +6,9 @@ defmodule Pleroma.Web.MastodonAPI.SubscriptionController do @moduledoc "The module represents functions to manage user subscriptions." use Pleroma.Web, :controller + alias Pleroma.Web.MastodonAPI.PushSubscriptionView, as: View alias Pleroma.Web.Push alias Pleroma.Web.Push.Subscription - alias Pleroma.Web.MastodonAPI.PushSubscriptionView, as: View action_fallback(:errors) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index d31a3d91c..5292aedf2 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -14,10 +14,10 @@ defmodule Pleroma.Web.OAuth.OAuthController do alias Pleroma.Web.ControllerHelper alias Pleroma.Web.OAuth.App alias Pleroma.Web.OAuth.Authorization + alias Pleroma.Web.OAuth.Scopes alias Pleroma.Web.OAuth.Token alias Pleroma.Web.OAuth.Token.Strategy.RefreshToken alias Pleroma.Web.OAuth.Token.Strategy.Revoke, as: RevokeToken - alias Pleroma.Web.OAuth.Scopes require Logger -- cgit v1.2.3 From 5fddf988ea45382d8e2307f11205afcc4b468d45 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 20 Jan 2020 13:05:35 +0100 Subject: Pleroma API: `emoji_reactions_by` does not need authorization --- lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex index 772c535a4..3285dc11b 100644 --- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do plug( OAuthScopesPlug, %{scopes: ["read:statuses"]} - when action in [:conversation, :conversation_statuses, :emoji_reactions_by] + when action in [:conversation, :conversation_statuses] ) plug( -- cgit v1.2.3 From 510776ba31f0bb01217fd6585848657f5bceb5f7 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 20 Jan 2020 14:27:59 +0100 Subject: CommonAPI: Don't error out on double favs/repeats --- lib/pleroma/web/common_api/common_api.ex | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 2f3bcfc3c..c05a6c544 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -85,9 +85,13 @@ defmodule Pleroma.Web.CommonAPI do def repeat(id_or_ap_id, user, params \\ %{}) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), object <- Object.normalize(activity), - nil <- Utils.get_existing_announce(user.ap_id, object), + announce_activity <- Utils.get_existing_announce(user.ap_id, object), public <- public_announce?(object, params) do - ActivityPub.announce(user, object, nil, true, public) + if announce_activity do + {:ok, announce_activity, object} + else + ActivityPub.announce(user, object, nil, true, public) + end else _ -> {:error, dgettext("errors", "Could not repeat")} end @@ -105,8 +109,12 @@ defmodule Pleroma.Web.CommonAPI do def favorite(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), object <- Object.normalize(activity), - nil <- Utils.get_existing_like(user.ap_id, object) do - ActivityPub.like(user, object) + like_activity <- Utils.get_existing_like(user.ap_id, object) do + if like_activity do + {:ok, like_activity, object} + else + ActivityPub.like(user, object) + end else _ -> {:error, dgettext("errors", "Could not favorite")} end -- cgit v1.2.3 From 4c5b5f14dcec51cba8e86bcfcf2943ee9b49b0e4 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 20 Jan 2020 16:24:20 +0100 Subject: StatusView: Add `emoji_reactions` --- lib/pleroma/web/mastodon_api/views/status_view.ex | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index e9590224b..b59ac39bc 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -253,6 +253,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do nil end + emoji_reactions = + with %{data: %{"reactions" => emoji_reactions}} <- object do + Enum.map(emoji_reactions, fn {emoji, users} -> + {emoji, length(users)} + end) + |> Enum.into(%{}) + else + _ -> %{} + end + %{ id: to_string(activity.id), uri: object.data["id"], @@ -293,7 +303,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do spoiler_text: %{"text/plain" => summary_plaintext}, expires_at: expires_at, direct_conversation_id: direct_conversation_id, - thread_muted: thread_muted? + thread_muted: thread_muted?, + emoji_reactions: emoji_reactions } } end -- cgit v1.2.3 From 6e88a7e5917f558fe3e9d639297086a9e8b98939 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 20 Jan 2020 21:47:44 +0300 Subject: exclude blocked user posts from search results --- lib/pleroma/activity.ex | 4 +--- lib/pleroma/activity/queries.ex | 10 ++++++++++ lib/pleroma/activity/search.ex | 11 ++++++++--- lib/pleroma/web/mastodon_api/controllers/search_controller.ex | 2 +- 4 files changed, 20 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 510d3273c..896cbb3c5 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -312,9 +312,7 @@ defmodule Pleroma.Activity do from(u in User.Query.build(deactivated: true), select: u.ap_id) |> Repo.all() - from(activity in query, - where: activity.actor not in ^deactivated_users - ) + Activity.Queries.exclude_authors(query, deactivated_users) end defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index 26bc1099d..79f305201 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -12,6 +12,7 @@ defmodule Pleroma.Activity.Queries do @type query :: Ecto.Queryable.t() | Activity.t() alias Pleroma.Activity + alias Pleroma.User @spec by_ap_id(query, String.t()) :: query def by_ap_id(query \\ Activity, ap_id) do @@ -29,6 +30,11 @@ defmodule Pleroma.Activity.Queries do ) end + @spec by_author(query, String.t()) :: query + def by_author(query \\ Activity, %User{ap_id: ap_id}) do + from(a in query, where: a.actor == ^ap_id) + end + @spec by_object_id(query, String.t() | [String.t()]) :: query def by_object_id(query \\ Activity, object_id) @@ -72,4 +78,8 @@ defmodule Pleroma.Activity.Queries do where: fragment("(?)->>'type' != ?", activity.data, ^activity_type) ) end + + def exclude_authors(query \\ Activity, actors) do + from(activity in query, where: activity.actor not in ^actors) + end end diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex index d30a5a6a5..f96e208da 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/activity/search.ex @@ -26,18 +26,23 @@ defmodule Pleroma.Activity.Search do |> query_with(index_type, search_query) |> maybe_restrict_local(user) |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => limit}, :offset) |> maybe_fetch(user, search_query) end def maybe_restrict_author(query, %User{} = author) do - from([a, o] in query, - where: a.actor == ^author.ap_id - ) + Activity.Queries.by_author(query, author) end def maybe_restrict_author(query, _), do: query + def maybe_restrict_blocked(query, %User{} = user) do + Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user)) + end + + def maybe_restrict_blocked(query, _), do: query + defp restrict_public(q) do from([a, o] in q, where: fragment("?->>'type' = 'Create'", a.data), diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 0a929f55b..5a5db8e00 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -43,7 +43,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do result = default_values |> Enum.map(fn {resource, default_value} -> - if params["type"] == nil or params["type"] == resource do + if params["type"] in [nil, resource] do {resource, fn -> resource_search(version, resource, query, options) end} else {resource, fn -> default_value end} -- cgit v1.2.3