diff options
| -rw-r--r-- | CHANGELOG.md | 1 | ||||
| -rw-r--r-- | benchmarks/load_testing/generator.ex | 44 | ||||
| -rw-r--r-- | benchmarks/mix/tasks/pleroma/benchmarks/tags.ex | 87 | ||||
| -rw-r--r-- | lib/pleroma/application.ex | 1 | ||||
| -rw-r--r-- | lib/pleroma/repo.ex | 35 | ||||
| -rw-r--r-- | lib/pleroma/web/activity_pub/transmogrifier.ex | 18 | ||||
| -rw-r--r-- | lib/pleroma/web/admin_api/admin_api_controller.ex | 2 | ||||
| -rw-r--r-- | lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex | 14 | ||||
| -rw-r--r-- | lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex | 2 | ||||
| -rw-r--r-- | mix.lock | 2 | ||||
| -rw-r--r-- | test/repo_test.exs | 43 | ||||
| -rw-r--r-- | test/web/admin_api/admin_api_controller_test.exs | 24 | ||||
| -rw-r--r-- | test/web/pleroma_api/controllers/pleroma_api_controller_test.exs | 5 |
13 files changed, 247 insertions, 31 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 182f5e579..af497dcba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking**: MDII uploader ### Changed +- **Breaking:** Pleroma won't start if it detects unapplied migrations - **Breaking:** attachments are removed along with statuses when there are no other references to it - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** attachment links (`config :pleroma, :instance, no_attachment_links` and `config :pleroma, Pleroma.Upload, link_name`) disabled by default diff --git a/benchmarks/load_testing/generator.ex b/benchmarks/load_testing/generator.ex index a957e0ffb..3f88fefd7 100644 --- a/benchmarks/load_testing/generator.ex +++ b/benchmarks/load_testing/generator.ex @@ -9,7 +9,7 @@ defmodule Pleroma.LoadTesting.Generator do {time, _} = :timer.tc(fn -> Task.async_stream( - Enum.take_random(posts, count_likes), + Enum.take_random(posts, count_likes), fn post -> {:ok, _, _} = CommonAPI.favorite(post.id, user) end, max_concurrency: 10, timeout: 30_000 @@ -142,6 +142,48 @@ defmodule Pleroma.LoadTesting.Generator do CommonAPI.post(Enum.random(users), post) end + def generate_power_intervals(opts \\ []) do + count = Keyword.get(opts, :count, 20) + power = Keyword.get(opts, :power, 2) + IO.puts("Generating #{count} intervals for a power #{power} series...") + counts = Enum.map(1..count, fn n -> :math.pow(n, power) end) + sum = Enum.sum(counts) + + densities = + Enum.map(counts, fn c -> + c / sum + end) + + densities + |> Enum.reduce(0, fn density, acc -> + if acc == 0 do + [{0, density}] + else + [{_, lower} | _] = acc + [{lower, lower + density} | acc] + end + end) + |> Enum.reverse() + end + + def generate_tagged_activities(opts \\ []) do + tag_count = Keyword.get(opts, :tag_count, 20) + users = Keyword.get(opts, :users, Repo.all(User)) + activity_count = Keyword.get(opts, :count, 200_000) + + intervals = generate_power_intervals(count: tag_count) + + IO.puts( + "Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0" + ) + + Enum.each(1..activity_count, fn _ -> + random = :rand.uniform() + i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end) + CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"}) + end) + end + defp do_generate_activity_with_mention(user, users) do mentions_cnt = Enum.random([2, 3, 4, 5]) with_user = Enum.random([true, false]) diff --git a/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex b/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex new file mode 100644 index 000000000..fd1506907 --- /dev/null +++ b/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex @@ -0,0 +1,87 @@ +defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do + use Mix.Task + alias Pleroma.Repo + alias Pleroma.LoadTesting.Generator + import Ecto.Query + + def run(_args) do + Mix.Pleroma.start_pleroma() + activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id) + + if activities_count == 0 do + IO.puts("Did not find any activities, cleaning and generating") + clean_tables() + Generator.generate_users(users_max: 10) + Generator.generate_tagged_activities() + else + IO.puts("Found #{activities_count} activities, won't generate new ones") + end + + tags = Enum.map(0..20, fn i -> {"For #tag_#{i}", "tag_#{i}"} end) + + Enum.each(tags, fn {_, tag} -> + query = + from(o in Pleroma.Object, + where: fragment("(?)->'tag' \\? (?)", o.data, ^tag) + ) + + count = Repo.aggregate(query, :count, :id) + IO.puts("Database contains #{count} posts tagged with #{tag}") + end) + + user = Repo.all(Pleroma.User) |> List.first() + + Benchee.run( + %{ + "Hashtag fetching, any" => fn tags -> + Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( + %{ + "any" => tags + }, + user, + false + ) + end, + # Will always return zero results because no overlapping hashtags are generated. + "Hashtag fetching, all" => fn tags -> + Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( + %{ + "all" => tags + }, + user, + false + ) + end + }, + inputs: + tags + |> Enum.map(fn {_, v} -> v end) + |> Enum.chunk_every(2) + |> Enum.map(fn tags -> {"For #{inspect(tags)}", tags} end), + time: 5 + ) + + Benchee.run( + %{ + "Hashtag fetching" => fn tag -> + Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( + %{ + "tag" => tag + }, + user, + false + ) + end + }, + inputs: tags, + time: 5 + ) + end + + defp clean_tables do + IO.puts("Deleting old data...\n") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;") + end +end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 2ae052069..e17068876 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -33,6 +33,7 @@ defmodule Pleroma.Application do def start(_type, _args) do Pleroma.HTML.compile_scrubbers() Pleroma.Config.DeprecationWarnings.warn() + Pleroma.Repo.check_migrations_applied!() setup_instrumenters() load_custom_modules() diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index f57e088bc..cb0b6653c 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Repo do adapter: Ecto.Adapters.Postgres, migration_timestamps: [type: :naive_datetime_usec] + require Logger + defmodule Instrumenter do use Prometheus.EctoInstrumenter end @@ -47,4 +49,37 @@ defmodule Pleroma.Repo do _ -> {:error, :not_found} end end + + def check_migrations_applied!() do + unless Pleroma.Config.get( + [:i_am_aware_this_may_cause_data_loss, :disable_migration_check], + false + ) do + Ecto.Migrator.with_repo(__MODULE__, fn repo -> + down_migrations = + Ecto.Migrator.migrations(repo) + |> Enum.reject(fn + {:up, _, _} -> true + {:down, _, _} -> false + end) + + if length(down_migrations) > 0 do + down_migrations_text = + Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end) + + Logger.error( + "The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true" + ) + + raise Pleroma.Repo.UnappliedMigrationsError + end + end) + else + :ok + end + end +end + +defmodule Pleroma.Repo.UnappliedMigrationsError do + defexception message: "Unapplied Migrations detected" end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 3fa789d53..2b8bfc3bd 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -658,24 +658,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do with %User{ap_id: ^actor_id} = actor <- User.get_cached_by_ap_id(object["id"]) do {:ok, new_user_data} = ActivityPub.user_data_from_user_object(object) - locked = new_user_data[:locked] || false - attachment = get_in(new_user_data, [:source_data, "attachment"]) || [] - invisible = new_user_data[:invisible] || false - - fields = - attachment - |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) - |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) - - update_data = - new_user_data - |> Map.take([:avatar, :banner, :bio, :name, :also_known_as]) - |> Map.put(:fields, fields) - |> Map.put(:locked, locked) - |> Map.put(:invisible, invisible) - actor - |> User.upgrade_changeset(update_data, true) + |> User.upgrade_changeset(new_user_data, true) |> User.update_and_set_cache() ActivityPub.update(%{ diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 78b77a97a..7118faf94 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -75,7 +75,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do plug( OAuthScopesPlug, %{scopes: ["write:reports"], admin: true} - when action in [:report_update_state, :report_respond] + when action in [:reports_update] ) plug( diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index 384159336..29964a1d4 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -77,10 +77,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do |> render("index.json", activities: activities, for: user, as: :activity) end - # GET /api/v1/timelines/tag/:tag - def hashtag(%{assigns: %{user: user}} = conn, params) do - local_only = truthy_param?(params["local"]) - + def hashtag_fetching(params, user, local_only) do tags = [params["tag"], params["any"]] |> List.flatten() @@ -98,7 +95,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do |> Map.get("none", []) |> Enum.map(&String.downcase(&1)) - activities = + _activities = params |> Map.put("type", "Create") |> Map.put("local_only", local_only) @@ -109,6 +106,13 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do |> Map.put("tag_all", tag_all) |> Map.put("tag_reject", tag_reject) |> ActivityPub.fetch_public_activities() + end + + # GET /api/v1/timelines/tag/:tag + def hashtag(%{assigns: %{user: user}} = conn, params) do + local_only = truthy_param?(params["local"]) + + activities = hashtag_fetching(params, user, local_only) conn |> add_link_headers(activities, %{"local" => local_only}) diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex index 772c535a4..3285dc11b 100644 --- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do plug( OAuthScopesPlug, %{scopes: ["read:statuses"]} - when action in [:conversation, :conversation_statuses, :emoji_reactions_by] + when action in [:conversation, :conversation_statuses] ) plug( @@ -63,7 +63,7 @@ "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"}, "mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"}, - "mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, + "mock": {:hex, :mock, "0.3.4", "c5862eb3b8c64237f45f586cf00c9d892ba07bb48305a43319d428ce3c2897dd", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]}, diff --git a/test/repo_test.exs b/test/repo_test.exs index 85b64d4d1..5526b0327 100644 --- a/test/repo_test.exs +++ b/test/repo_test.exs @@ -4,7 +4,10 @@ defmodule Pleroma.RepoTest do use Pleroma.DataCase + import ExUnit.CaptureLog import Pleroma.Factory + import Mock + alias Pleroma.User describe "find_resource/1" do @@ -46,4 +49,44 @@ defmodule Pleroma.RepoTest do assert Repo.get_assoc(token, :user) == {:error, :not_found} end end + + describe "check_migrations_applied!" do + setup_with_mocks([ + {Ecto.Migrator, [], + [ + with_repo: fn repo, fun -> passthrough([repo, fun]) end, + migrations: fn Pleroma.Repo -> + [ + {:up, 20_191_128_153_944, "fix_missing_following_count"}, + {:up, 20_191_203_043_610, "create_report_notes"}, + {:down, 20_191_220_174_645, "add_scopes_to_pleroma_feo_auth_records"} + ] + end + ]} + ]) do + :ok + end + + test "raises if it detects unapplied migrations" do + assert_raise Pleroma.Repo.UnappliedMigrationsError, fn -> + capture_log(&Repo.check_migrations_applied!/0) + end + end + + test "doesn't do anything if disabled" do + disable_migration_check = + Pleroma.Config.get([:i_am_aware_this_may_cause_data_loss, :disable_migration_check]) + + Pleroma.Config.put([:i_am_aware_this_may_cause_data_loss, :disable_migration_check], true) + + on_exit(fn -> + Pleroma.Config.put( + [:i_am_aware_this_may_cause_data_loss, :disable_migration_check], + disable_migration_check + ) + end) + + assert :ok == Repo.check_migrations_applied!() + end + end end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index d70fe54b2..d11b26207 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1363,6 +1363,30 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do } end + test "requires write:reports scope", %{conn: conn, id: id, admin: admin} do + read_token = insert(:oauth_token, user: admin, scopes: ["read"]) + write_token = insert(:oauth_token, user: admin, scopes: ["write:reports"]) + + response = + conn + |> assign(:token, read_token) + |> patch("/api/pleroma/admin/reports", %{ + "reports" => [%{"state" => "resolved", "id" => id}] + }) + |> json_response(403) + + assert response == %{ + "error" => "Insufficient permissions: admin:write:reports | write:reports." + } + + conn + |> assign(:token, write_token) + |> patch("/api/pleroma/admin/reports", %{ + "reports" => [%{"state" => "resolved", "id" => id}] + }) + |> json_response(:no_content) + end + test "mark report as resolved", %{conn: conn, id: id, admin: admin} do conn |> patch("/api/pleroma/admin/reports", %{ diff --git a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs index 3f7ef13bc..fb7500134 100644 --- a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs @@ -57,11 +57,6 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do {:ok, activity} = CommonAPI.post(user, %{"status" => "#cofe"}) - conn = - conn - |> assign(:user, user) - |> assign(:token, insert(:oauth_token, user: user, scopes: ["read:statuses"])) - result = conn |> get("/api/v1/pleroma/statuses/#{activity.id}/emoji_reactions_by") |