aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorlain <lain@soykaf.club>2020-06-20 09:33:18 +0000
committerlain <lain@soykaf.club>2020-06-20 09:33:18 +0000
commit7d542450b13742c6efa93246c172c8306e730d50 (patch)
tree86998336bba747f1ce605a1c06e39ac96f38b396
parent4733f6a3371504ebb3eeb447d7c20d56c10b43bf (diff)
parentf9282677731cca96a1316e2830507837535c06e0 (diff)
downloadpleroma-7d542450b13742c6efa93246c172c8306e730d50.tar.gz
Merge branch 'develop' into '1865-media-only'
# Conflicts: # CHANGELOG.md
-rw-r--r--.gitlab-ci.yml19
-rw-r--r--CHANGELOG.md2
-rw-r--r--README.md10
-rw-r--r--config/config.exs7
-rw-r--r--config/description.exs64
-rw-r--r--docs/API/admin_api.md64
-rw-r--r--docs/configuration/cheatsheet.md6
-rw-r--r--elixir_buildpack.config4
-rwxr-xr-xinstallation/nginx-cache-purge.sh.example4
-rw-r--r--lib/mix/tasks/pleroma/config.ex10
-rw-r--r--lib/pleroma/application.ex3
-rw-r--r--lib/pleroma/config/config_db.ex281
-rw-r--r--lib/pleroma/config/transfer_task.ex12
-rw-r--r--lib/pleroma/conversation/participation.ex11
-rw-r--r--lib/pleroma/ecto_type/activity_pub/object_validators/date_time.ex (renamed from lib/pleroma/web/activity_pub/object_validators/types/date_time.ex)6
-rw-r--r--lib/pleroma/ecto_type/activity_pub/object_validators/object_id.ex (renamed from lib/pleroma/web/activity_pub/object_validators/types/object_id.ex)6
-rw-r--r--lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex (renamed from lib/pleroma/web/activity_pub/object_validators/types/recipients.ex)8
-rw-r--r--lib/pleroma/ecto_type/activity_pub/object_validators/safe_text.ex (renamed from lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex)2
-rw-r--r--lib/pleroma/ecto_type/activity_pub/object_validators/uri.ex (renamed from lib/pleroma/web/activity_pub/object_validators/types/uri.ex)6
-rw-r--r--lib/pleroma/ecto_type/config/atom.ex26
-rw-r--r--lib/pleroma/ecto_type/config/binary_value.ex27
-rw-r--r--lib/pleroma/migration_helper/notification_backfill.ex2
-rw-r--r--lib/pleroma/pagination.ex6
-rw-r--r--lib/pleroma/plugs/uploaded_media.ex16
-rw-r--r--lib/pleroma/repo.ex28
-rw-r--r--lib/pleroma/signature.ex4
-rw-r--r--lib/pleroma/user.ex49
-rw-r--r--lib/pleroma/web/activity_pub/activity_pub.ex7
-rw-r--r--lib/pleroma/web/activity_pub/object_validator.ex4
-rw-r--r--lib/pleroma/web/activity_pub/object_validators/announce_validator.ex14
-rw-r--r--lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex12
-rw-r--r--lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex10
-rw-r--r--lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex6
-rw-r--r--lib/pleroma/web/activity_pub/object_validators/delete_validator.ex14
-rw-r--r--lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex8
-rw-r--r--lib/pleroma/web/activity_pub/object_validators/like_validator.ex14
-rw-r--r--lib/pleroma/web/activity_pub/object_validators/note_validator.ex12
-rw-r--r--lib/pleroma/web/activity_pub/object_validators/undo_validator.ex8
-rw-r--r--lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex8
-rw-r--r--lib/pleroma/web/activity_pub/transmogrifier.ex4
-rw-r--r--lib/pleroma/web/admin_api/controllers/config_controller.ex34
-rw-r--r--lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex63
-rw-r--r--lib/pleroma/web/admin_api/views/config_view.ex19
-rw-r--r--lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex11
-rw-r--r--lib/pleroma/web/api_spec/helpers.ex6
-rw-r--r--lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex109
-rw-r--r--lib/pleroma/web/api_spec/operations/notification_operation.ex1
-rw-r--r--lib/pleroma/web/mastodon_api/controllers/account_controller.ex1
-rw-r--r--lib/pleroma/web/mastodon_api/controllers/search_controller.ex31
-rw-r--r--lib/pleroma/web/mastodon_api/views/account_view.ex13
-rw-r--r--lib/pleroma/web/mastodon_api/views/conversation_view.ex11
-rw-r--r--lib/pleroma/web/media_proxy/invalidation.ex26
-rw-r--r--lib/pleroma/web/media_proxy/invalidations/http.ex10
-rw-r--r--lib/pleroma/web/media_proxy/invalidations/script.ex36
-rw-r--r--lib/pleroma/web/media_proxy/media_proxy.ex35
-rw-r--r--lib/pleroma/web/media_proxy/media_proxy_controller.ex3
-rw-r--r--lib/pleroma/web/router.ex4
-rw-r--r--lib/pleroma/workers/attachments_cleanup_worker.ex134
-rw-r--r--mix.exs2
-rw-r--r--priv/repo/migrations/20200322174133_user_raw_bio.exs9
-rw-r--r--priv/repo/migrations/20200328193433_populate_user_raw_bio.exs25
-rw-r--r--test/config/config_db_test.exs591
-rw-r--r--test/config/transfer_task_test.exs94
-rw-r--r--test/support/factory.ex20
-rw-r--r--test/tasks/config_test.exs41
-rw-r--r--test/upload/filter/mogrify_test.exs8
-rw-r--r--test/web/activity_pub/object_validators/types/date_time_test.exs2
-rw-r--r--test/web/activity_pub/object_validators/types/object_id_test.exs2
-rw-r--r--test/web/activity_pub/object_validators/types/recipients_test.exs2
-rw-r--r--test/web/activity_pub/object_validators/types/safe_text_test.exs2
-rw-r--r--test/web/admin_api/controllers/config_controller_test.exs256
-rw-r--r--test/web/admin_api/controllers/media_proxy_cache_controller_test.exs145
-rw-r--r--test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs13
-rw-r--r--test/web/mastodon_api/controllers/search_controller_test.exs16
-rw-r--r--test/web/mastodon_api/controllers/status_controller_test.exs2
-rw-r--r--test/web/mastodon_api/views/account_view_test.exs3
-rw-r--r--test/web/mastodon_api/views/conversation_view_test.exs11
-rw-r--r--test/web/media_proxy/invalidation_test.exs64
-rw-r--r--test/web/media_proxy/invalidations/http_test.exs12
-rw-r--r--test/web/media_proxy/invalidations/script_test.exs20
-rw-r--r--test/web/media_proxy/media_proxy_controller_test.exs16
81 files changed, 1656 insertions, 1051 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index aad28a2d8..b4bd59b43 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,4 @@
-image: elixir:1.8.1
+image: elixir:1.9.4
variables: &global_variables
POSTGRES_DB: pleroma_test
@@ -170,8 +170,7 @@ stop_review_app:
amd64:
stage: release
- # TODO: Replace with upstream image when 1.9.0 comes out
- image: rinpatch/elixir:1.9.0-rc.0
+ image: elixir:1.10.3
only: &release-only
- stable@pleroma/pleroma
- develop@pleroma/pleroma
@@ -208,8 +207,7 @@ amd64-musl:
stage: release
artifacts: *release-artifacts
only: *release-only
- # TODO: Replace with upstream image when 1.9.0 comes out
- image: rinpatch/elixir:1.9.0-rc.0-alpine
+ image: elixir:1.10.3-alpine
cache: *release-cache
variables: *release-variables
before_script: &before-release-musl
@@ -225,8 +223,7 @@ arm:
only: *release-only
tags:
- arm32
- # TODO: Replace with upstream image when 1.9.0 comes out
- image: rinpatch/elixir:1.9.0-rc.0-arm
+ image: elixir:1.10.3
cache: *release-cache
variables: *release-variables
before_script: *before-release
@@ -238,8 +235,7 @@ arm-musl:
only: *release-only
tags:
- arm32
- # TODO: Replace with upstream image when 1.9.0 comes out
- image: rinpatch/elixir:1.9.0-rc.0-arm-alpine
+ image: elixir:1.10.3-alpine
cache: *release-cache
variables: *release-variables
before_script: *before-release-musl
@@ -251,8 +247,7 @@ arm64:
only: *release-only
tags:
- arm
- # TODO: Replace with upstream image when 1.9.0 comes out
- image: rinpatch/elixir:1.9.0-rc.0-arm64
+ image: elixir:1.10.3
cache: *release-cache
variables: *release-variables
before_script: *before-release
@@ -265,7 +260,7 @@ arm64-musl:
tags:
- arm
# TODO: Replace with upstream image when 1.9.0 comes out
- image: rinpatch/elixir:1.9.0-rc.0-arm64-alpine
+ image: elixir:1.10.3-alpine
cache: *release-cache
variables: *release-variables
before_script: *before-release-musl
diff --git a/CHANGELOG.md b/CHANGELOG.md
index eee442817..07bd9461f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## [unreleased]
### Changed
+- **Breaking:** Elixir >=1.9 is now required (was >= 1.8)
+- In Conversations, return only direct messages as `last_status`
- Using the `only_media` filter on timelines will now exclude reblog media
- MFR policy to set global expiration for all local Create activities
- OGP rich media parser merged with TwitterCard
diff --git a/README.md b/README.md
index 7fc1fd381..6ca3118fb 100644
--- a/README.md
+++ b/README.md
@@ -34,6 +34,16 @@ Currently Pleroma is not packaged by any OS/Distros, but if you want to package
### Docker
While we don’t provide docker files, other people have written very good ones. Take a look at <https://github.com/angristan/docker-pleroma> or <https://glitch.sh/sn0w/pleroma-docker>.
+### Compilation Troubleshooting
+If you ever encounter compilation issues during the updating of Pleroma, you can try these commands and see if they fix things:
+
+- `mix deps.clean --all`
+- `mix local.rebar`
+- `mix local.hex`
+- `rm -r _build`
+
+If you are not developing Pleroma, it is better to use the OTP release, which comes with everything precompiled.
+
## Documentation
- Latest Released revision: <https://docs.pleroma.social>
- Latest Git revision: <https://docs-develop.pleroma.social>
diff --git a/config/config.exs b/config/config.exs
index 6a7bb9e06..4bf31f3fc 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -407,6 +407,13 @@ config :pleroma, :media_proxy,
],
whitelist: []
+config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
+ method: :purge,
+ headers: [],
+ options: []
+
+config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil
+
config :pleroma, :chat, enabled: true
config :phoenix, :format_encoders, json: Jason
diff --git a/config/description.exs b/config/description.exs
index b21d7840c..f9523936a 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -1651,6 +1651,31 @@ config :pleroma, :config_description, [
suggestions: ["https://example.com"]
},
%{
+ key: :invalidation,
+ type: :keyword,
+ descpiption: "",
+ suggestions: [
+ enabled: true,
+ provider: Pleroma.Web.MediaProxy.Invalidation.Script
+ ],
+ children: [
+ %{
+ key: :enabled,
+ type: :boolean,
+ description: "Enables invalidate media cache"
+ },
+ %{
+ key: :provider,
+ type: :module,
+ description: "Module which will be used to cache purge.",
+ suggestions: [
+ Pleroma.Web.MediaProxy.Invalidation.Script,
+ Pleroma.Web.MediaProxy.Invalidation.Http
+ ]
+ }
+ ]
+ },
+ %{
key: :proxy_opts,
type: :keyword,
description: "Options for Pleroma.ReverseProxy",
@@ -1724,6 +1749,45 @@ config :pleroma, :config_description, [
},
%{
group: :pleroma,
+ key: Pleroma.Web.MediaProxy.Invalidation.Http,
+ type: :group,
+ description: "HTTP invalidate settings",
+ children: [
+ %{
+ key: :method,
+ type: :atom,
+ description: "HTTP method of request. Default: :purge"
+ },
+ %{
+ key: :headers,
+ type: {:list, :tuple},
+ description: "HTTP headers of request.",
+ suggestions: [{"x-refresh", 1}]
+ },
+ %{
+ key: :options,
+ type: :keyword,
+ description: "Request options.",
+ suggestions: [params: %{ts: "xxx"}]
+ }
+ ]
+ },
+ %{
+ group: :pleroma,
+ key: Pleroma.Web.MediaProxy.Invalidation.Script,
+ type: :group,
+ description: "Script invalidate settings",
+ children: [
+ %{
+ key: :script_path,
+ type: :string,
+ description: "Path to shell script. Which will run purge cache.",
+ suggestions: ["./installation/nginx-cache-purge.sh.example"]
+ }
+ ]
+ },
+ %{
+ group: :pleroma,
key: :gopher,
type: :group,
description: "Gopher settings",
diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md
index 92816baf9..c7f56cf5f 100644
--- a/docs/API/admin_api.md
+++ b/docs/API/admin_api.md
@@ -1224,4 +1224,66 @@ Loads json generated from `config/descriptions.exs`.
- Response:
- On success: `204`, empty response
- On failure:
- - 400 Bad Request `"Invalid parameters"` when `status` is missing \ No newline at end of file
+ - 400 Bad Request `"Invalid parameters"` when `status` is missing
+
+## `GET /api/pleroma/admin/media_proxy_caches`
+
+### Get a list of all banned MediaProxy URLs in Cachex
+
+- Authentication: required
+- Params:
+- *optional* `page`: **integer** page number
+- *optional* `page_size`: **integer** number of log entries per page (default is `50`)
+
+- Response:
+
+``` json
+{
+ "urls": [
+ "http://example.com/media/a688346.jpg",
+ "http://example.com/media/fb1f4d.jpg"
+ ]
+}
+
+```
+
+## `POST /api/pleroma/admin/media_proxy_caches/delete`
+
+### Remove a banned MediaProxy URL from Cachex
+
+- Authentication: required
+- Params:
+ - `urls` (array)
+
+- Response:
+
+``` json
+{
+ "urls": [
+ "http://example.com/media/a688346.jpg",
+ "http://example.com/media/fb1f4d.jpg"
+ ]
+}
+
+```
+
+## `POST /api/pleroma/admin/media_proxy_caches/purge`
+
+### Purge a MediaProxy URL
+
+- Authentication: required
+- Params:
+ - `urls` (array)
+ - `ban` (boolean)
+
+- Response:
+
+``` json
+{
+ "urls": [
+ "http://example.com/media/a688346.jpg",
+ "http://example.com/media/fb1f4d.jpg"
+ ]
+}
+
+```
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index fad67fc4d..6ebdab546 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -268,7 +268,7 @@ This section describe PWA manifest instance-specific values. Currently this opti
#### Pleroma.Web.MediaProxy.Invalidation.Script
-This strategy allow perform external bash script to purge cache.
+This strategy allow perform external shell script to purge cache.
Urls of attachments pass to script as arguments.
* `script_path`: path to external script.
@@ -284,8 +284,8 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script,
This strategy allow perform custom http request to purge cache.
* `method`: http method. default is `purge`
-* `headers`: http headers. default is empty
-* `options`: request options. default is empty
+* `headers`: http headers.
+* `options`: request options.
Example:
```elixir
diff --git a/elixir_buildpack.config b/elixir_buildpack.config
index c23b08fb8..946408c12 100644
--- a/elixir_buildpack.config
+++ b/elixir_buildpack.config
@@ -1,2 +1,2 @@
-elixir_version=1.8.2
-erlang_version=21.3.7
+elixir_version=1.9.4
+erlang_version=22.3.4.1
diff --git a/installation/nginx-cache-purge.sh.example b/installation/nginx-cache-purge.sh.example
index b2915321c..5f6cbb128 100755
--- a/installation/nginx-cache-purge.sh.example
+++ b/installation/nginx-cache-purge.sh.example
@@ -13,7 +13,7 @@ CACHE_DIRECTORY="/tmp/pleroma-media-cache"
## $3 - (optional) the number of parallel processes to run for grep.
get_cache_files() {
local max_parallel=${3-16}
- find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E Rl "^KEY:.*$1" | sort -u
+ find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E -Rl "^KEY:.*$1" | sort -u
}
## Removes an item from the given cache zone.
@@ -37,4 +37,4 @@ purge() {
}
-purge $1
+purge $@
diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex
index 5c9ef6904..f1b3a8766 100644
--- a/lib/mix/tasks/pleroma/config.ex
+++ b/lib/mix/tasks/pleroma/config.ex
@@ -72,8 +72,7 @@ defmodule Mix.Tasks.Pleroma.Config do
group
|> Pleroma.Config.Loader.filter_group(settings)
|> Enum.each(fn {key, value} ->
- key = inspect(key)
- {:ok, _} = ConfigDB.update_or_create(%{group: inspect(group), key: key, value: value})
+ {:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value})
shell_info("Settings for key #{key} migrated.")
end)
@@ -131,12 +130,9 @@ defmodule Mix.Tasks.Pleroma.Config do
end
defp write(config, file) do
- value =
- config.value
- |> ConfigDB.from_binary()
- |> inspect(limit: :infinity)
+ value = inspect(config.value, limit: :infinity)
- IO.write(file, "config #{config.group}, #{config.key}, #{value}\r\n\r\n")
+ IO.write(file, "config #{inspect(config.group)}, #{inspect(config.key)}, #{value}\r\n\r\n")
config
end
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 9d3d92b38..4a21bf138 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -148,7 +148,8 @@ defmodule Pleroma.Application do
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
build_cachex("web_resp", limit: 2500),
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
- build_cachex("failed_proxy_url", limit: 2500)
+ build_cachex("failed_proxy_url", limit: 2500),
+ build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
]
end
diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex
index 2b43d4c36..2f4eb8581 100644
--- a/lib/pleroma/config/config_db.ex
+++ b/lib/pleroma/config/config_db.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.ConfigDB do
use Ecto.Schema
import Ecto.Changeset
- import Ecto.Query
+ import Ecto.Query, only: [select: 3]
import Pleroma.Web.Gettext
alias __MODULE__
@@ -14,16 +14,6 @@ defmodule Pleroma.ConfigDB do
@type t :: %__MODULE__{}
- @full_key_update [
- {:pleroma, :ecto_repos},
- {:quack, :meta},
- {:mime, :types},
- {:cors_plug, [:max_age, :methods, :expose, :headers]},
- {:auto_linker, :opts},
- {:swarm, :node_blacklist},
- {:logger, :backends}
- ]
-
@full_subkey_update [
{:pleroma, :assets, :mascots},
{:pleroma, :emoji, :groups},
@@ -32,14 +22,10 @@ defmodule Pleroma.ConfigDB do
{:pleroma, :mrf_keyword, :replace}
]
- @regex ~r/^~r(?'delimiter'[\/|"'([{<]{1})(?'pattern'.+)[\/|"')\]}>]{1}(?'modifier'[uismxfU]*)/u
-
- @delimiters ["/", "|", "\"", "'", {"(", ")"}, {"[", "]"}, {"{", "}"}, {"<", ">"}]
-
schema "config" do
- field(:key, :string)
- field(:group, :string)
- field(:value, :binary)
+ field(:key, Pleroma.EctoType.Config.Atom)
+ field(:group, Pleroma.EctoType.Config.Atom)
+ field(:value, Pleroma.EctoType.Config.BinaryValue)
field(:db, {:array, :string}, virtual: true, default: [])
timestamps()
@@ -51,10 +37,6 @@ defmodule Pleroma.ConfigDB do
|> select([c], {c.group, c.key, c.value})
|> Repo.all()
|> Enum.reduce([], fn {group, key, value}, acc ->
- group = ConfigDB.from_string(group)
- key = ConfigDB.from_string(key)
- value = from_binary(value)
-
Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}]))
end)
end
@@ -64,50 +46,41 @@ defmodule Pleroma.ConfigDB do
@spec changeset(ConfigDB.t(), map()) :: Changeset.t()
def changeset(config, params \\ %{}) do
- params = Map.put(params, :value, transform(params[:value]))
-
config
|> cast(params, [:key, :group, :value])
|> validate_required([:key, :group, :value])
|> unique_constraint(:key, name: :config_group_key_index)
end
- @spec create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()}
- def create(params) do
+ defp create(params) do
%ConfigDB{}
|> changeset(params)
|> Repo.insert()
end
- @spec update(ConfigDB.t(), map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()}
- def update(%ConfigDB{} = config, %{value: value}) do
+ defp update(%ConfigDB{} = config, %{value: value}) do
config
|> changeset(%{value: value})
|> Repo.update()
end
- @spec get_db_keys(ConfigDB.t()) :: [String.t()]
- def get_db_keys(%ConfigDB{} = config) do
- config.value
- |> ConfigDB.from_binary()
- |> get_db_keys(config.key)
- end
-
@spec get_db_keys(keyword(), any()) :: [String.t()]
def get_db_keys(value, key) do
- if Keyword.keyword?(value) do
- value |> Keyword.keys() |> Enum.map(&convert(&1))
- else
- [convert(key)]
- end
+ keys =
+ if Keyword.keyword?(value) do
+ Keyword.keys(value)
+ else
+ [key]
+ end
+
+ Enum.map(keys, &to_json_types(&1))
end
@spec merge_group(atom(), atom(), keyword(), keyword()) :: keyword()
def merge_group(group, key, old_value, new_value) do
- new_keys = to_map_set(new_value)
+ new_keys = to_mapset(new_value)
- intersect_keys =
- old_value |> to_map_set() |> MapSet.intersection(new_keys) |> MapSet.to_list()
+ intersect_keys = old_value |> to_mapset() |> MapSet.intersection(new_keys) |> MapSet.to_list()
merged_value = ConfigDB.merge(old_value, new_value)
@@ -120,12 +93,10 @@ defmodule Pleroma.ConfigDB do
[]
end)
|> List.flatten()
- |> Enum.reduce(merged_value, fn subkey, acc ->
- Keyword.put(acc, subkey, new_value[subkey])
- end)
+ |> Enum.reduce(merged_value, &Keyword.put(&2, &1, new_value[&1]))
end
- defp to_map_set(keyword) do
+ defp to_mapset(keyword) do
keyword
|> Keyword.keys()
|> MapSet.new()
@@ -159,43 +130,40 @@ defmodule Pleroma.ConfigDB do
@spec update_or_create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()}
def update_or_create(params) do
+ params = Map.put(params, :value, to_elixir_types(params[:value]))
search_opts = Map.take(params, [:group, :key])
with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts),
- {:partial_update, true, config} <-
- {:partial_update, can_be_partially_updated?(config), config},
- old_value <- from_binary(config.value),
- transformed_value <- do_transform(params[:value]),
- {:can_be_merged, true, config} <- {:can_be_merged, is_list(transformed_value), config},
- new_value <-
- merge_group(
- ConfigDB.from_string(config.group),
- ConfigDB.from_string(config.key),
- old_value,
- transformed_value
- ) do
- ConfigDB.update(config, %{value: new_value})
+ {_, true, config} <- {:partial_update, can_be_partially_updated?(config), config},
+ {_, true, config} <-
+ {:can_be_merged, is_list(params[:value]) and is_list(config.value), config} do
+ new_value = merge_group(config.group, config.key, config.value, params[:value])
+ update(config, %{value: new_value})
else
{reason, false, config} when reason in [:partial_update, :can_be_merged] ->
- ConfigDB.update(config, params)
+ update(config, params)
nil ->
- ConfigDB.create(params)
+ create(params)
end
end
defp can_be_partially_updated?(%ConfigDB{} = config), do: not only_full_update?(config)
- defp only_full_update?(%ConfigDB{} = config) do
- config_group = ConfigDB.from_string(config.group)
- config_key = ConfigDB.from_string(config.key)
-
- Enum.any?(@full_key_update, fn
- {group, key} when is_list(key) ->
- config_group == group and config_key in key
-
- {group, key} ->
- config_group == group and config_key == key
+ defp only_full_update?(%ConfigDB{group: group, key: key}) do
+ full_key_update = [
+ {:pleroma, :ecto_repos},
+ {:quack, :meta},
+ {:mime, :types},
+ {:cors_plug, [:max_age, :methods, :expose, :headers]},
+ {:auto_linker, :opts},
+ {:swarm, :node_blacklist},
+ {:logger, :backends}
+ ]
+
+ Enum.any?(full_key_update, fn
+ {s_group, s_key} ->
+ group == s_group and ((is_list(s_key) and key in s_key) or key == s_key)
end)
end
@@ -205,11 +173,10 @@ defmodule Pleroma.ConfigDB do
with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts),
{config, sub_keys} when is_list(sub_keys) <- {config, params[:subkeys]},
- old_value <- from_binary(config.value),
- keys <- Enum.map(sub_keys, &do_transform_string(&1)),
- {:partial_remove, config, new_value} when new_value != [] <-
- {:partial_remove, config, Keyword.drop(old_value, keys)} do
- ConfigDB.update(config, %{value: new_value})
+ keys <- Enum.map(sub_keys, &string_to_elixir_types(&1)),
+ {_, config, new_value} when new_value != [] <-
+ {:partial_remove, config, Keyword.drop(config.value, keys)} do
+ update(config, %{value: new_value})
else
{:partial_remove, config, []} ->
Repo.delete(config)
@@ -225,37 +192,32 @@ defmodule Pleroma.ConfigDB do
end
end
- @spec from_binary(binary()) :: term()
- def from_binary(binary), do: :erlang.binary_to_term(binary)
-
- @spec from_binary_with_convert(binary()) :: any()
- def from_binary_with_convert(binary) do
- binary
- |> from_binary()
- |> do_convert()
+ @spec to_json_types(term()) :: map() | list() | boolean() | String.t()
+ def to_json_types(entity) when is_list(entity) do
+ Enum.map(entity, &to_json_types/1)
end
- @spec from_string(String.t()) :: atom() | no_return()
- def from_string(string), do: do_transform_string(string)
+ def to_json_types(%Regex{} = entity), do: inspect(entity)
- @spec convert(any()) :: any()
- def convert(entity), do: do_convert(entity)
-
- defp do_convert(entity) when is_list(entity) do
- for v <- entity, into: [], do: do_convert(v)
+ def to_json_types(entity) when is_map(entity) do
+ Map.new(entity, fn {k, v} -> {to_json_types(k), to_json_types(v)} end)
end
- defp do_convert(%Regex{} = entity), do: inspect(entity)
+ def to_json_types({:args, args}) when is_list(args) do
+ arguments =
+ Enum.map(args, fn
+ arg when is_tuple(arg) -> inspect(arg)
+ arg -> to_json_types(arg)
+ end)
- defp do_convert(entity) when is_map(entity) do
- for {k, v} <- entity, into: %{}, do: {do_convert(k), do_convert(v)}
+ %{"tuple" => [":args", arguments]}
end
- defp do_convert({:proxy_url, {type, :localhost, port}}) do
- %{"tuple" => [":proxy_url", %{"tuple" => [do_convert(type), "localhost", port]}]}
+ def to_json_types({:proxy_url, {type, :localhost, port}}) do
+ %{"tuple" => [":proxy_url", %{"tuple" => [to_json_types(type), "localhost", port]}]}
end
- defp do_convert({:proxy_url, {type, host, port}}) when is_tuple(host) do
+ def to_json_types({:proxy_url, {type, host, port}}) when is_tuple(host) do
ip =
host
|> :inet_parse.ntoa()
@@ -264,66 +226,64 @@ defmodule Pleroma.ConfigDB do
%{
"tuple" => [
":proxy_url",
- %{"tuple" => [do_convert(type), ip, port]}
+ %{"tuple" => [to_json_types(type), ip, port]}
]
}
end
- defp do_convert({:proxy_url, {type, host, port}}) do
+ def to_json_types({:proxy_url, {type, host, port}}) do
%{
"tuple" => [
":proxy_url",
- %{"tuple" => [do_convert(type), to_string(host), port]}
+ %{"tuple" => [to_json_types(type), to_string(host), port]}
]
}
end
- defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]}
+ def to_json_types({:partial_chain, entity}),
+ do: %{"tuple" => [":partial_chain", inspect(entity)]}
- defp do_convert(entity) when is_tuple(entity) do
+ def to_json_types(entity) when is_tuple(entity) do
value =
entity
|> Tuple.to_list()
- |> do_convert()
+ |> to_json_types()
%{"tuple" => value}
end
- defp do_convert(entity) when is_boolean(entity) or is_number(entity) or is_nil(entity) do
+ def to_json_types(entity) when is_binary(entity), do: entity
+
+ def to_json_types(entity) when is_boolean(entity) or is_number(entity) or is_nil(entity) do
entity
end
- defp do_convert(entity)
- when is_atom(entity) and entity in [:"tlsv1.1", :"tlsv1.2", :"tlsv1.3"] do
+ def to_json_types(entity) when entity in [:"tlsv1.1", :"tlsv1.2", :"tlsv1.3"] do
":#{entity}"
end
- defp do_convert(entity) when is_atom(entity), do: inspect(entity)
+ def to_json_types(entity) when is_atom(entity), do: inspect(entity)
- defp do_convert(entity) when is_binary(entity), do: entity
+ @spec to_elixir_types(boolean() | String.t() | map() | list()) :: term()
+ def to_elixir_types(%{"tuple" => [":args", args]}) when is_list(args) do
+ arguments =
+ Enum.map(args, fn arg ->
+ if String.contains?(arg, ["{", "}"]) do
+ {elem, []} = Code.eval_string(arg)
+ elem
+ else
+ to_elixir_types(arg)
+ end
+ end)
- @spec transform(any()) :: binary() | no_return()
- def transform(entity) when is_binary(entity) or is_map(entity) or is_list(entity) do
- entity
- |> do_transform()
- |> to_binary()
+ {:args, arguments}
end
- def transform(entity), do: to_binary(entity)
-
- @spec transform_with_out_binary(any()) :: any()
- def transform_with_out_binary(entity), do: do_transform(entity)
-
- @spec to_binary(any()) :: binary()
- def to_binary(entity), do: :erlang.term_to_binary(entity)
-
- defp do_transform(%Regex{} = entity), do: entity
-
- defp do_transform(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}) do
- {:proxy_url, {do_transform_string(type), parse_host(host), port}}
+ def to_elixir_types(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}) do
+ {:proxy_url, {string_to_elixir_types(type), parse_host(host), port}}
end
- defp do_transform(%{"tuple" => [":partial_chain", entity]}) do
+ def to_elixir_types(%{"tuple" => [":partial_chain", entity]}) do
{partial_chain, []} =
entity
|> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "")
@@ -332,25 +292,51 @@ defmodule Pleroma.ConfigDB do
{:partial_chain, partial_chain}
end
- defp do_transform(%{"tuple" => entity}) do
- Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end)
+ def to_elixir_types(%{"tuple" => entity}) do
+ Enum.reduce(entity, {}, &Tuple.append(&2, to_elixir_types(&1)))
end
- defp do_transform(entity) when is_map(entity) do
- for {k, v} <- entity, into: %{}, do: {do_transform(k), do_transform(v)}
+ def to_elixir_types(entity) when is_map(entity) do
+ Map.new(entity, fn {k, v} -> {to_elixir_types(k), to_elixir_types(v)} end)
end
- defp do_transform(entity) when is_list(entity) do
- for v <- entity, into: [], do: do_transform(v)
+ def to_elixir_types(entity) when is_list(entity) do
+ Enum.map(entity, &to_elixir_types/1)
end
- defp do_transform(entity) when is_binary(entity) do
+ def to_elixir_types(entity) when is_binary(entity) do
entity
|> String.trim()
- |> do_transform_string()
+ |> string_to_elixir_types()
+ end
+
+ def to_elixir_types(entity), do: entity
+
+ @spec string_to_elixir_types(String.t()) ::
+ atom() | Regex.t() | module() | String.t() | no_return()
+ def string_to_elixir_types("~r" <> _pattern = regex) do
+ pattern =
+ ~r/^~r(?'delimiter'[\/|"'([{<]{1})(?'pattern'.+)[\/|"')\]}>]{1}(?'modifier'[uismxfU]*)/u
+
+ delimiters = ["/", "|", "\"", "'", {"(", ")"}, {"[", "]"}, {"{", "}"}, {"<", ">"}]
+
+ with %{"modifier" => modifier, "pattern" => pattern, "delimiter" => regex_delimiter} <-
+ Regex.named_captures(pattern, regex),
+ {:ok, {leading, closing}} <- find_valid_delimiter(delimiters, pattern, regex_delimiter),
+ {result, _} <- Code.eval_string("~r#{leading}#{pattern}#{closing}#{modifier}") do
+ result
+ end
end
- defp do_transform(entity), do: entity
+ def string_to_elixir_types(":" <> atom), do: String.to_atom(atom)
+
+ def string_to_elixir_types(value) do
+ if module_name?(value) do
+ String.to_existing_atom("Elixir." <> value)
+ else
+ value
+ end
+ end
defp parse_host("localhost"), do: :localhost
@@ -387,27 +373,8 @@ defmodule Pleroma.ConfigDB do
end
end
- defp do_transform_string("~r" <> _pattern = regex) do
- with %{"modifier" => modifier, "pattern" => pattern, "delimiter" => regex_delimiter} <-
- Regex.named_captures(@regex, regex),
- {:ok, {leading, closing}} <- find_valid_delimiter(@delimiters, pattern, regex_delimiter),
- {result, _} <- Code.eval_string("~r#{leading}#{pattern}#{closing}#{modifier}") do
- result
- end
- end
-
- defp do_transform_string(":" <> atom), do: String.to_atom(atom)
-
- defp do_transform_string(value) do
- if is_module_name?(value) do
- String.to_existing_atom("Elixir." <> value)
- else
- value
- end
- end
-
- @spec is_module_name?(String.t()) :: boolean()
- def is_module_name?(string) do
+ @spec module_name?(String.t()) :: boolean()
+ def module_name?(string) do
Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or
string in ["Oban", "Ueberauth", "ExSyslogger"]
end
diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex
index c02b70e96..eb86b8ff4 100644
--- a/lib/pleroma/config/transfer_task.ex
+++ b/lib/pleroma/config/transfer_task.ex
@@ -28,10 +28,6 @@ defmodule Pleroma.Config.TransferTask do
{:pleroma, Pleroma.Captcha, [:seconds_valid]},
{:pleroma, Pleroma.Upload, [:proxy_remote]},
{:pleroma, :instance, [:upload_limit]},
- {:pleroma, :email_notifications, [:digest]},
- {:pleroma, :oauth2, [:clean_expired_tokens]},
- {:pleroma, Pleroma.ActivityExpiration, [:enabled]},
- {:pleroma, Pleroma.ScheduledActivity, [:enabled]},
{:pleroma, :gopher, [:enabled]}
]
@@ -48,7 +44,7 @@ defmodule Pleroma.Config.TransferTask do
{logger, other} =
(Repo.all(ConfigDB) ++ deleted_settings)
- |> Enum.map(&transform_and_merge/1)
+ |> Enum.map(&merge_with_default/1)
|> Enum.split_with(fn {group, _, _, _} -> group in [:logger, :quack] end)
logger
@@ -92,11 +88,7 @@ defmodule Pleroma.Config.TransferTask do
end
end
- defp transform_and_merge(%{group: group, key: key, value: value} = setting) do
- group = ConfigDB.from_string(group)
- key = ConfigDB.from_string(key)
- value = ConfigDB.from_binary(value)
-
+ defp merge_with_default(%{group: group, key: key, value: value} = setting) do
default = Config.Holder.default_config(group, key)
merged =
diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex
index ce7bd2396..8bc3e85d6 100644
--- a/lib/pleroma/conversation/participation.ex
+++ b/lib/pleroma/conversation/participation.ex
@@ -162,10 +162,13 @@ defmodule Pleroma.Conversation.Participation do
for_user(user, params)
|> Enum.map(fn participation ->
activity_id =
- ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{
- user: user,
- blocking_user: user
- })
+ ActivityPub.fetch_latest_direct_activity_id_for_context(
+ participation.conversation.ap_id,
+ %{
+ user: user,
+ blocking_user: user
+ }
+ )
%{
participation
diff --git a/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/date_time.ex
index 4f412fcde..d852c0abd 100644
--- a/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex
+++ b/lib/pleroma/ecto_type/activity_pub/object_validators/date_time.ex
@@ -1,4 +1,8 @@
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.DateTime do
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime do
@moduledoc """
The AP standard defines the date fields in AP as xsd:DateTime. Elixir's
DateTime can't parse this, but it can parse the related iso8601. This
diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/object_id.ex
index f71f76370..8034235b0 100644
--- a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex
+++ b/lib/pleroma/ecto_type/activity_pub/object_validators/object_id.ex
@@ -1,4 +1,8 @@
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID do
use Ecto.Type
def type, do: :string
diff --git a/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex
index 408e0f6ee..205527a96 100644
--- a/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex
+++ b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex
@@ -1,7 +1,11 @@
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.Recipients do
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do
use Ecto.Type
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID
def type, do: {:array, ObjectID}
diff --git a/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/safe_text.ex
index 95c948123..7f0405c7b 100644
--- a/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex
+++ b/lib/pleroma/ecto_type/activity_pub/object_validators/safe_text.ex
@@ -2,7 +2,7 @@
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.SafeText do
+defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.SafeText do
use Ecto.Type
alias Pleroma.HTML
diff --git a/lib/pleroma/web/activity_pub/object_validators/types/uri.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/uri.ex
index 24845bcc0..2054c26be 100644
--- a/lib/pleroma/web/activity_pub/object_validators/types/uri.ex
+++ b/lib/pleroma/ecto_type/activity_pub/object_validators/uri.ex
@@ -1,4 +1,8 @@
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.Uri do
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Uri do
use Ecto.Type
def type, do: :string
diff --git a/lib/pleroma/ecto_type/config/atom.ex b/lib/pleroma/ecto_type/config/atom.ex
new file mode 100644
index 000000000..df565d432
--- /dev/null
+++ b/lib/pleroma/ecto_type/config/atom.ex
@@ -0,0 +1,26 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.Config.Atom do
+ use Ecto.Type
+
+ def type, do: :atom
+
+ def cast(key) when is_atom(key) do
+ {:ok, key}
+ end
+
+ def cast(key) when is_binary(key) do
+ {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)}
+ end
+
+ def cast(_), do: :error
+
+ def load(key) do
+ {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)}
+ end
+
+ def dump(key) when is_atom(key), do: {:ok, inspect(key)}
+ def dump(_), do: :error
+end
diff --git a/lib/pleroma/ecto_type/config/binary_value.ex b/lib/pleroma/ecto_type/config/binary_value.ex
new file mode 100644
index 000000000..bbd2608c5
--- /dev/null
+++ b/lib/pleroma/ecto_type/config/binary_value.ex
@@ -0,0 +1,27 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.Config.BinaryValue do
+ use Ecto.Type
+
+ def type, do: :term
+
+ def cast(value) when is_binary(value) do
+ if String.valid?(value) do
+ {:ok, value}
+ else
+ {:ok, :erlang.binary_to_term(value)}
+ end
+ end
+
+ def cast(value), do: {:ok, value}
+
+ def load(value) when is_binary(value) do
+ {:ok, :erlang.binary_to_term(value)}
+ end
+
+ def dump(value) do
+ {:ok, :erlang.term_to_binary(value)}
+ end
+end
diff --git a/lib/pleroma/migration_helper/notification_backfill.ex b/lib/pleroma/migration_helper/notification_backfill.ex
index 09647d12a..b3770307a 100644
--- a/lib/pleroma/migration_helper/notification_backfill.ex
+++ b/lib/pleroma/migration_helper/notification_backfill.ex
@@ -18,7 +18,7 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do
)
query
- |> Repo.all()
+ |> Repo.chunk_stream(100)
|> Enum.each(fn notification ->
type =
notification.activity
diff --git a/lib/pleroma/pagination.ex b/lib/pleroma/pagination.ex
index 1b99e44f9..9a3795769 100644
--- a/lib/pleroma/pagination.ex
+++ b/lib/pleroma/pagination.ex
@@ -64,6 +64,12 @@ defmodule Pleroma.Pagination do
@spec paginate(Ecto.Query.t(), map(), type(), atom() | nil) :: [Ecto.Schema.t()]
def paginate(query, options, method \\ :keyset, table_binding \\ nil)
+ def paginate(list, options, _method, _table_binding) when is_list(list) do
+ offset = options[:offset] || 0
+ limit = options[:limit] || 0
+ Enum.slice(list, offset, limit)
+ end
+
def paginate(query, options, :keyset, table_binding) do
query
|> restrict(:min_id, options, table_binding)
diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex
index 94147e0c4..40984cfc0 100644
--- a/lib/pleroma/plugs/uploaded_media.ex
+++ b/lib/pleroma/plugs/uploaded_media.ex
@@ -10,6 +10,8 @@ defmodule Pleroma.Plugs.UploadedMedia do
import Pleroma.Web.Gettext
require Logger
+ alias Pleroma.Web.MediaProxy
+
@behaviour Plug
# no slashes
@path "media"
@@ -35,8 +37,7 @@ defmodule Pleroma.Plugs.UploadedMedia do
%{query_params: %{"name" => name}} = conn ->
name = String.replace(name, "\"", "\\\"")
- conn
- |> put_resp_header("content-disposition", "filename=\"#{name}\"")
+ put_resp_header(conn, "content-disposition", "filename=\"#{name}\"")
conn ->
conn
@@ -47,7 +48,8 @@ defmodule Pleroma.Plugs.UploadedMedia do
with uploader <- Keyword.fetch!(config, :uploader),
proxy_remote = Keyword.get(config, :proxy_remote, false),
- {:ok, get_method} <- uploader.get_file(file) do
+ {:ok, get_method} <- uploader.get_file(file),
+ false <- media_is_banned(conn, get_method) do
get_media(conn, get_method, proxy_remote, opts)
else
_ ->
@@ -59,6 +61,14 @@ defmodule Pleroma.Plugs.UploadedMedia do
def call(conn, _opts), do: conn
+ defp media_is_banned(%{request_path: path} = _conn, {:static_dir, _}) do
+ MediaProxy.in_banned_urls(Pleroma.Web.base_url() <> path)
+ end
+
+ defp media_is_banned(_, {:url, url}), do: MediaProxy.in_banned_urls(url)
+
+ defp media_is_banned(_, _), do: false
+
defp get_media(conn, {:static_dir, directory}, _, opts) do
static_opts =
Map.get(opts, :static_plug_opts)
diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex
index f62138466..6d85d70bc 100644
--- a/lib/pleroma/repo.ex
+++ b/lib/pleroma/repo.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Repo do
adapter: Ecto.Adapters.Postgres,
migration_timestamps: [type: :naive_datetime_usec]
+ import Ecto.Query
require Logger
defmodule Instrumenter do
@@ -78,6 +79,33 @@ defmodule Pleroma.Repo do
:ok
end
end
+
+ def chunk_stream(query, chunk_size) do
+ # We don't actually need start and end funcitons of resource streaming,
+ # but it seems to be the only way to not fetch records one-by-one and
+ # have individual records be the elements of the stream, instead of
+ # lists of records
+ Stream.resource(
+ fn -> 0 end,
+ fn
+ last_id ->
+ query
+ |> order_by(asc: :id)
+ |> where([r], r.id > ^last_id)
+ |> limit(^chunk_size)
+ |> all()
+ |> case do
+ [] ->
+ {:halt, last_id}
+
+ records ->
+ last_id = List.last(records).id
+ {records, last_id}
+ end
+ end,
+ fn _ -> :ok end
+ )
+ end
end
defmodule Pleroma.Repo.UnappliedMigrationsError do
diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex
index d01728361..3aa6909d2 100644
--- a/lib/pleroma/signature.ex
+++ b/lib/pleroma/signature.ex
@@ -5,10 +5,10 @@
defmodule Pleroma.Signature do
@behaviour HTTPSignatures.Adapter
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Keys
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
def key_id_to_actor_id(key_id) do
uri =
@@ -24,7 +24,7 @@ defmodule Pleroma.Signature do
maybe_ap_id = URI.to_string(uri)
- case Types.ObjectID.cast(maybe_ap_id) do
+ case ObjectValidators.ObjectID.cast(maybe_ap_id) do
{:ok, ap_id} ->
{:ok, ap_id}
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 52ac9052b..f0ccc7c79 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -14,6 +14,7 @@ defmodule Pleroma.User do
alias Pleroma.Config
alias Pleroma.Conversation.Participation
alias Pleroma.Delivery
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Emoji
alias Pleroma.FollowingRelationship
alias Pleroma.Formatter
@@ -30,7 +31,6 @@ defmodule Pleroma.User do
alias Pleroma.Web
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Builder
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
alias Pleroma.Web.ActivityPub.Pipeline
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.CommonAPI
@@ -79,6 +79,7 @@ defmodule Pleroma.User do
schema "users" do
field(:bio, :string)
+ field(:raw_bio, :string)
field(:email, :string)
field(:name, :string)
field(:nickname, :string)
@@ -115,7 +116,7 @@ defmodule Pleroma.User do
field(:is_admin, :boolean, default: false)
field(:show_role, :boolean, default: true)
field(:settings, :map, default: nil)
- field(:uri, Types.Uri, default: nil)
+ field(:uri, ObjectValidators.Uri, default: nil)
field(:hide_followers_count, :boolean, default: false)
field(:hide_follows_count, :boolean, default: false)
field(:hide_followers, :boolean, default: false)
@@ -432,6 +433,7 @@ defmodule Pleroma.User do
params,
[
:bio,
+ :raw_bio,
:name,
:emoji,
:avatar,
@@ -607,7 +609,16 @@ defmodule Pleroma.User do
struct
|> confirmation_changeset(need_confirmation: need_confirmation?)
- |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation, :emoji])
+ |> cast(params, [
+ :bio,
+ :raw_bio,
+ :email,
+ :name,
+ :nickname,
+ :password,
+ :password_confirmation,
+ :emoji
+ ])
|> validate_required([:name, :nickname, :password, :password_confirmation])
|> validate_confirmation(:password)
|> unique_constraint(:email)
@@ -747,7 +758,6 @@ defmodule Pleroma.User do
follower
|> update_following_count()
- |> set_cache()
end
end
@@ -776,7 +786,6 @@ defmodule Pleroma.User do
{:ok, follower} =
follower
|> update_following_count()
- |> set_cache()
{:ok, follower, followed}
@@ -1128,35 +1137,25 @@ defmodule Pleroma.User do
])
end
+ @spec update_follower_count(User.t()) :: {:ok, User.t()}
def update_follower_count(%User{} = user) do
if user.local or !Pleroma.Config.get([:instance, :external_user_synchronization]) do
- follower_count_query =
- User.Query.build(%{followers: user, deactivated: false})
- |> select([u], %{count: count(u.id)})
-
- User
- |> where(id: ^user.id)
- |> join(:inner, [u], s in subquery(follower_count_query))
- |> update([u, s],
- set: [follower_count: s.count]
- )
- |> select([u], u)
- |> Repo.update_all([])
- |> case do
- {1, [user]} -> set_cache(user)
- _ -> {:error, user}
- end
+ follower_count = FollowingRelationship.follower_count(user)
+
+ user
+ |> follow_information_changeset(%{follower_count: follower_count})
+ |> update_and_set_cache
else
{:ok, maybe_fetch_follow_information(user)}
end
end
- @spec update_following_count(User.t()) :: User.t()
+ @spec update_following_count(User.t()) :: {:ok, User.t()}
def update_following_count(%User{local: false} = user) do
if Pleroma.Config.get([:instance, :external_user_synchronization]) do
- maybe_fetch_follow_information(user)
+ {:ok, maybe_fetch_follow_information(user)}
else
- user
+ {:ok, user}
end
end
@@ -1165,7 +1164,7 @@ defmodule Pleroma.User do
user
|> follow_information_changeset(%{following_count: following_count})
- |> Repo.update!()
+ |> update_and_set_cache()
end
def set_unread_conversation_count(%User{local: true} = user) do
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index efb8b81db..3e4d0a2be 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -210,7 +210,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
conversation = Repo.preload(conversation, :participations)
last_activity_id =
- fetch_latest_activity_id_for_context(conversation.ap_id, %{
+ fetch_latest_direct_activity_id_for_context(conversation.ap_id, %{
user: user,
blocking_user: user
})
@@ -517,11 +517,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|> Repo.all()
end
- @spec fetch_latest_activity_id_for_context(String.t(), keyword() | map()) ::
+ @spec fetch_latest_direct_activity_id_for_context(String.t(), keyword() | map()) ::
FlakeId.Ecto.CompatType.t() | nil
- def fetch_latest_activity_id_for_context(context, opts \\ %{}) do
+ def fetch_latest_direct_activity_id_for_context(context, opts \\ %{}) do
context
|> fetch_activities_for_context_query(Map.merge(%{skip_preload: true}, opts))
+ |> restrict_visibility(%{visibility: "direct"})
|> limit(1)
|> select([a], a.id)
|> Repo.one()
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index c01c5f780..6a83a2c33 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
the system.
"""
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Object
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
@@ -17,7 +18,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator
alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator
alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator
@spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
@@ -120,7 +120,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
def stringify_keys(object), do: object
def fetch_actor(object) do
- with {:ok, actor} <- Types.ObjectID.cast(object["actor"]) do
+ with {:ok, actor} <- ObjectValidators.ObjectID.cast(object["actor"]) do
User.get_or_fetch_by_ap_id(actor)
end
end
diff --git a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex
index 40f861f47..6f757f49c 100644
--- a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex
@@ -5,9 +5,9 @@
defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do
use Ecto.Schema
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Object
alias Pleroma.User
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.ActivityPub.Visibility
@@ -19,14 +19,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do
@primary_key false
embedded_schema do
- field(:id, Types.ObjectID, primary_key: true)
+ field(:id, ObjectValidators.ObjectID, primary_key: true)
field(:type, :string)
- field(:object, Types.ObjectID)
- field(:actor, Types.ObjectID)
+ field(:object, ObjectValidators.ObjectID)
+ field(:actor, ObjectValidators.ObjectID)
field(:context, :string, autogenerate: {Utils, :generate_context_id, []})
- field(:to, Types.Recipients, default: [])
- field(:cc, Types.Recipients, default: [])
- field(:published, Types.DateTime)
+ field(:to, ObjectValidators.Recipients, default: [])
+ field(:cc, ObjectValidators.Recipients, default: [])
+ field(:published, ObjectValidators.DateTime)
end
def cast_and_validate(data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index 138736f23..c481d79e0 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -5,9 +5,9 @@
defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
use Ecto.Schema
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
import Ecto.Changeset
import Pleroma.Web.ActivityPub.Transmogrifier, only: [fix_emoji: 1]
@@ -16,12 +16,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
@derive Jason.Encoder
embedded_schema do
- field(:id, Types.ObjectID, primary_key: true)
- field(:to, Types.Recipients, default: [])
+ field(:id, ObjectValidators.ObjectID, primary_key: true)
+ field(:to, ObjectValidators.Recipients, default: [])
field(:type, :string)
- field(:content, Types.SafeText)
- field(:actor, Types.ObjectID)
- field(:published, Types.DateTime)
+ field(:content, ObjectValidators.SafeText)
+ field(:actor, ObjectValidators.ObjectID)
+ field(:published, ObjectValidators.DateTime)
field(:emoji, :map, default: %{})
embeds_one(:attachment, AttachmentValidator)
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
index fc582400b..7269f9ff0 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
@@ -7,9 +7,9 @@
# - doesn't embed, will only get the object id
defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do
use Ecto.Schema
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Object
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
import Ecto.Changeset
import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@@ -17,11 +17,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do
@primary_key false
embedded_schema do
- field(:id, Types.ObjectID, primary_key: true)
- field(:actor, Types.ObjectID)
+ field(:id, ObjectValidators.ObjectID, primary_key: true)
+ field(:actor, ObjectValidators.ObjectID)
field(:type, :string)
- field(:to, Types.Recipients, default: [])
- field(:object, Types.ObjectID)
+ field(:to, ObjectValidators.Recipients, default: [])
+ field(:object, ObjectValidators.ObjectID)
end
def cast_and_apply(data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex
index 926804ce7..316bd0c07 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex
@@ -5,16 +5,16 @@
defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do
use Ecto.Schema
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
import Ecto.Changeset
@primary_key false
embedded_schema do
- field(:id, Types.ObjectID, primary_key: true)
- field(:actor, Types.ObjectID)
+ field(:id, ObjectValidators.ObjectID, primary_key: true)
+ field(:actor, ObjectValidators.ObjectID)
field(:type, :string)
field(:to, {:array, :string})
field(:cc, {:array, :string})
diff --git a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex
index e5d08eb5c..93a7b0e0b 100644
--- a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex
@@ -6,8 +6,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
use Ecto.Schema
alias Pleroma.Activity
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.User
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
import Ecto.Changeset
import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@@ -15,13 +15,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
@primary_key false
embedded_schema do
- field(:id, Types.ObjectID, primary_key: true)
+ field(:id, ObjectValidators.ObjectID, primary_key: true)
field(:type, :string)
- field(:actor, Types.ObjectID)
- field(:to, Types.Recipients, default: [])
- field(:cc, Types.Recipients, default: [])
- field(:deleted_activity_id, Types.ObjectID)
- field(:object, Types.ObjectID)
+ field(:actor, ObjectValidators.ObjectID)
+ field(:to, ObjectValidators.Recipients, default: [])
+ field(:cc, ObjectValidators.Recipients, default: [])
+ field(:deleted_activity_id, ObjectValidators.ObjectID)
+ field(:object, ObjectValidators.ObjectID)
end
def cast_data(data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex
index e87519c59..a543af1f8 100644
--- a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex
@@ -5,8 +5,8 @@
defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do
use Ecto.Schema
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Object
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
import Ecto.Changeset
import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@@ -14,10 +14,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do
@primary_key false
embedded_schema do
- field(:id, Types.ObjectID, primary_key: true)
+ field(:id, ObjectValidators.ObjectID, primary_key: true)
field(:type, :string)
- field(:object, Types.ObjectID)
- field(:actor, Types.ObjectID)
+ field(:object, ObjectValidators.ObjectID)
+ field(:actor, ObjectValidators.ObjectID)
field(:context, :string)
field(:content, :string)
field(:to, {:array, :string}, default: [])
diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex
index 034f25492..493e4c247 100644
--- a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex
@@ -5,8 +5,8 @@
defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do
use Ecto.Schema
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Object
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
alias Pleroma.Web.ActivityPub.Utils
import Ecto.Changeset
@@ -15,13 +15,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do
@primary_key false
embedded_schema do
- field(:id, Types.ObjectID, primary_key: true)
+ field(:id, ObjectValidators.ObjectID, primary_key: true)
field(:type, :string)
- field(:object, Types.ObjectID)
- field(:actor, Types.ObjectID)
+ field(:object, ObjectValidators.ObjectID)
+ field(:actor, ObjectValidators.ObjectID)
field(:context, :string)
- field(:to, Types.Recipients, default: [])
- field(:cc, Types.Recipients, default: [])
+ field(:to, ObjectValidators.Recipients, default: [])
+ field(:cc, ObjectValidators.Recipients, default: [])
end
def cast_and_validate(data) do
@@ -67,7 +67,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do
with {[], []} <- {to, cc},
%Object{data: %{"actor" => actor}} <- Object.get_cached_by_ap_id(object),
- {:ok, actor} <- Types.ObjectID.cast(actor) do
+ {:ok, actor} <- ObjectValidators.ObjectID.cast(actor) do
cng
|> put_change(:to, [actor])
else
diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
index 462a5620a..a10728ac6 100644
--- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
@@ -5,14 +5,14 @@
defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
use Ecto.Schema
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
import Ecto.Changeset
@primary_key false
embedded_schema do
- field(:id, Types.ObjectID, primary_key: true)
+ field(:id, ObjectValidators.ObjectID, primary_key: true)
field(:to, {:array, :string}, default: [])
field(:cc, {:array, :string}, default: [])
field(:bto, {:array, :string}, default: [])
@@ -22,10 +22,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
field(:type, :string)
field(:content, :string)
field(:context, :string)
- field(:actor, Types.ObjectID)
- field(:attributedTo, Types.ObjectID)
+ field(:actor, ObjectValidators.ObjectID)
+ field(:attributedTo, ObjectValidators.ObjectID)
field(:summary, :string)
- field(:published, Types.DateTime)
+ field(:published, ObjectValidators.DateTime)
# TODO: Write type
field(:emoji, :map, default: %{})
field(:sensitive, :boolean, default: false)
@@ -35,7 +35,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
field(:like_count, :integer, default: 0)
field(:announcement_count, :integer, default: 0)
field(:inRepyTo, :string)
- field(:uri, Types.Uri)
+ field(:uri, ObjectValidators.Uri)
field(:likes, {:array, :string}, default: [])
field(:announcements, {:array, :string}, default: [])
diff --git a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex
index d0ba418e8..e8d2d39c1 100644
--- a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator do
use Ecto.Schema
alias Pleroma.Activity
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
import Ecto.Changeset
import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@@ -14,10 +14,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator do
@primary_key false
embedded_schema do
- field(:id, Types.ObjectID, primary_key: true)
+ field(:id, ObjectValidators.ObjectID, primary_key: true)
field(:type, :string)
- field(:object, Types.ObjectID)
- field(:actor, Types.ObjectID)
+ field(:object, ObjectValidators.ObjectID)
+ field(:actor, ObjectValidators.ObjectID)
field(:to, {:array, :string}, default: [])
field(:cc, {:array, :string}, default: [])
end
diff --git a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex
index 47e231150..f64fac46d 100644
--- a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex
@@ -1,14 +1,18 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do
use Ecto.Schema
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
import Ecto.Changeset
@primary_key false
embedded_schema do
field(:type, :string)
- field(:href, Types.Uri)
+ field(:href, ObjectValidators.Uri)
field(:mediaType, :string)
end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 985921aa0..851f474b8 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
"""
alias Pleroma.Activity
alias Pleroma.EarmarkRenderer
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.FollowingRelationship
alias Pleroma.Maps
alias Pleroma.Notification
@@ -18,7 +19,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Builder
alias Pleroma.Web.ActivityPub.ObjectValidator
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types
alias Pleroma.Web.ActivityPub.Pipeline
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.ActivityPub.Visibility
@@ -725,7 +725,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
else
{:error, {:validate_object, _}} = e ->
# Check if we have a create activity for this
- with {:ok, object_id} <- Types.ObjectID.cast(data["object"]),
+ with {:ok, object_id} <- ObjectValidators.ObjectID.cast(data["object"]),
%Activity{data: %{"actor" => actor}} <-
Activity.create_by_object_ap_id(object_id) |> Repo.one(),
# We have one, insert a tombstone and retry
diff --git a/lib/pleroma/web/admin_api/controllers/config_controller.ex b/lib/pleroma/web/admin_api/controllers/config_controller.ex
index d6e2019bc..7f60470cb 100644
--- a/lib/pleroma/web/admin_api/controllers/config_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/config_controller.ex
@@ -33,7 +33,11 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do
def show(conn, %{only_db: true}) do
with :ok <- configurable_from_database() do
configs = Pleroma.Repo.all(ConfigDB)
- render(conn, "index.json", %{configs: configs})
+
+ render(conn, "index.json", %{
+ configs: configs,
+ need_reboot: Restarter.Pleroma.need_reboot?()
+ })
end
end
@@ -61,17 +65,20 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do
value
end
- %{
- group: ConfigDB.convert(group),
- key: ConfigDB.convert(key),
- value: ConfigDB.convert(merged_value)
+ %ConfigDB{
+ group: group,
+ key: key,
+ value: merged_value
}
|> Pleroma.Maps.put_if_present(:db, db)
end)
end)
|> List.flatten()
- json(conn, %{configs: merged, need_reboot: Restarter.Pleroma.need_reboot?()})
+ render(conn, "index.json", %{
+ configs: merged,
+ need_reboot: Restarter.Pleroma.need_reboot?()
+ })
end
end
@@ -91,24 +98,17 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do
{deleted, updated} =
results
- |> Enum.map(fn {:ok, config} ->
- Map.put(config, :db, ConfigDB.get_db_keys(config))
- end)
- |> Enum.split_with(fn config ->
- Ecto.get_meta(config, :state) == :deleted
+ |> Enum.map(fn {:ok, %{key: key, value: value} = config} ->
+ Map.put(config, :db, ConfigDB.get_db_keys(value, key))
end)
+ |> Enum.split_with(&(Ecto.get_meta(&1, :state) == :deleted))
Config.TransferTask.load_and_update_env(deleted, false)
if not Restarter.Pleroma.need_reboot?() do
changed_reboot_settings? =
(updated ++ deleted)
- |> Enum.any?(fn config ->
- group = ConfigDB.from_string(config.group)
- key = ConfigDB.from_string(config.key)
- value = ConfigDB.from_binary(config.value)
- Config.TransferTask.pleroma_need_restart?(group, key, value)
- end)
+ |> Enum.any?(&Config.TransferTask.pleroma_need_restart?(&1.group, &1.key, &1.value))
if changed_reboot_settings?, do: Restarter.Pleroma.need_reboot()
end
diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex
new file mode 100644
index 000000000..e2759d59f
--- /dev/null
+++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex
@@ -0,0 +1,63 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
+ use Pleroma.Web, :controller
+
+ alias Pleroma.Plugs.OAuthScopesPlug
+ alias Pleroma.Web.ApiSpec.Admin, as: Spec
+ alias Pleroma.Web.MediaProxy
+
+ plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+ plug(
+ OAuthScopesPlug,
+ %{scopes: ["read:media_proxy_caches"], admin: true} when action in [:index]
+ )
+
+ plug(
+ OAuthScopesPlug,
+ %{scopes: ["write:media_proxy_caches"], admin: true} when action in [:purge, :delete]
+ )
+
+ action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+ defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
+
+ def index(%{assigns: %{user: _}} = conn, params) do
+ cursor =
+ :banned_urls_cache
+ |> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}])
+ |> :qlc.cursor()
+
+ urls =
+ case params.page do
+ 1 ->
+ :qlc.next_answers(cursor, params.page_size)
+
+ _ ->
+ :qlc.next_answers(cursor, (params.page - 1) * params.page_size)
+ :qlc.next_answers(cursor, params.page_size)
+ end
+
+ :qlc.delete_cursor(cursor)
+
+ render(conn, "index.json", urls: urls)
+ end
+
+ def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
+ MediaProxy.remove_from_banned_urls(urls)
+ render(conn, "index.json", urls: urls)
+ end
+
+ def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do
+ MediaProxy.Invalidation.purge(urls)
+
+ if ban do
+ MediaProxy.put_in_banned_urls(urls)
+ end
+
+ render(conn, "index.json", urls: urls)
+ end
+end
diff --git a/lib/pleroma/web/admin_api/views/config_view.ex b/lib/pleroma/web/admin_api/views/config_view.ex
index 587ef760e..d2d8b5907 100644
--- a/lib/pleroma/web/admin_api/views/config_view.ex
+++ b/lib/pleroma/web/admin_api/views/config_view.ex
@@ -5,23 +5,20 @@
defmodule Pleroma.Web.AdminAPI.ConfigView do
use Pleroma.Web, :view
+ alias Pleroma.ConfigDB
+
def render("index.json", %{configs: configs} = params) do
- map = %{
- configs: render_many(configs, __MODULE__, "show.json", as: :config)
+ %{
+ configs: render_many(configs, __MODULE__, "show.json", as: :config),
+ need_reboot: params[:need_reboot]
}
-
- if params[:need_reboot] do
- Map.put(map, :need_reboot, true)
- else
- map
- end
end
def render("show.json", %{config: config}) do
map = %{
- key: config.key,
- group: config.group,
- value: Pleroma.ConfigDB.from_binary_with_convert(config.value)
+ key: ConfigDB.to_json_types(config.key),
+ group: ConfigDB.to_json_types(config.group),
+ value: ConfigDB.to_json_types(config.value)
}
if config.db != [] do
diff --git a/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex
new file mode 100644
index 000000000..c97400beb
--- /dev/null
+++ b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex
@@ -0,0 +1,11 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do
+ use Pleroma.Web, :view
+
+ def render("index.json", %{urls: urls}) do
+ %{urls: urls}
+ end
+end
diff --git a/lib/pleroma/web/api_spec/helpers.ex b/lib/pleroma/web/api_spec/helpers.ex
index a9cfe0fed..a258e8421 100644
--- a/lib/pleroma/web/api_spec/helpers.ex
+++ b/lib/pleroma/web/api_spec/helpers.ex
@@ -40,6 +40,12 @@ defmodule Pleroma.Web.ApiSpec.Helpers do
"Return the newest items newer than this ID"
),
Operation.parameter(
+ :offset,
+ :query,
+ %Schema{type: :integer, default: 0},
+ "Return items past this number of items"
+ ),
+ Operation.parameter(
:limit,
:query,
%Schema{type: :integer, default: 20},
diff --git a/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex
new file mode 100644
index 000000000..0358cfbad
--- /dev/null
+++ b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex
@@ -0,0 +1,109 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
+ alias OpenApiSpex.Operation
+ alias OpenApiSpex.Schema
+ alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+ import Pleroma.Web.ApiSpec.Helpers
+
+ def open_api_operation(action) do
+ operation = String.to_existing_atom("#{action}_operation")
+ apply(__MODULE__, operation, [])
+ end
+
+ def index_operation do
+ %Operation{
+ tags: ["Admin", "MediaProxyCache"],
+ summary: "Fetch a paginated list of all banned MediaProxy URLs in Cachex",
+ operationId: "AdminAPI.MediaProxyCacheController.index",
+ security: [%{"oAuth" => ["read:media_proxy_caches"]}],
+ parameters: [
+ Operation.parameter(
+ :page,
+ :query,
+ %Schema{type: :integer, default: 1},
+ "Page"
+ ),
+ Operation.parameter(
+ :page_size,
+ :query,
+ %Schema{type: :integer, default: 50},
+ "Number of statuses to return"
+ )
+ ],
+ responses: %{
+ 200 => success_response()
+ }
+ }
+ end
+
+ def delete_operation do
+ %Operation{
+ tags: ["Admin", "MediaProxyCache"],
+ summary: "Remove a banned MediaProxy URL from Cachex",
+ operationId: "AdminAPI.MediaProxyCacheController.delete",
+ security: [%{"oAuth" => ["write:media_proxy_caches"]}],
+ requestBody:
+ request_body(
+ "Parameters",
+ %Schema{
+ type: :object,
+ required: [:urls],
+ properties: %{
+ urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}}
+ }
+ },
+ required: true
+ ),
+ responses: %{
+ 200 => success_response(),
+ 400 => Operation.response("Error", "application/json", ApiError)
+ }
+ }
+ end
+
+ def purge_operation do
+ %Operation{
+ tags: ["Admin", "MediaProxyCache"],
+ summary: "Purge and optionally ban a MediaProxy URL",
+ operationId: "AdminAPI.MediaProxyCacheController.purge",
+ security: [%{"oAuth" => ["write:media_proxy_caches"]}],
+ requestBody:
+ request_body(
+ "Parameters",
+ %Schema{
+ type: :object,
+ required: [:urls],
+ properties: %{
+ urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}},
+ ban: %Schema{type: :boolean, default: true}
+ }
+ },
+ required: true
+ ),
+ responses: %{
+ 200 => success_response(),
+ 400 => Operation.response("Error", "application/json", ApiError)
+ }
+ }
+ end
+
+ defp success_response do
+ Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{
+ type: :object,
+ properties: %{
+ urls: %Schema{
+ type: :array,
+ items: %Schema{
+ type: :string,
+ format: :uri,
+ description: "MediaProxy URLs"
+ }
+ }
+ }
+ })
+ end
+end
diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex
index c966b553a..41328b5f2 100644
--- a/lib/pleroma/web/api_spec/operations/notification_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex
@@ -183,7 +183,6 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
"favourite",
"reblog",
"mention",
- "poll",
"pleroma:emoji_reaction",
"pleroma:chat_mention",
"move",
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 7cdd8f458..c38c2b895 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -165,6 +165,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
end)
|> Maps.put_if_present(:name, params[:display_name])
|> Maps.put_if_present(:bio, params[:note])
+ |> Maps.put_if_present(:raw_bio, params[:note])
|> Maps.put_if_present(:avatar, params[:avatar])
|> Maps.put_if_present(:banner, params[:header])
|> Maps.put_if_present(:background, params[:pleroma_background_image])
diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
index 3be0ca095..e50980122 100644
--- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
@@ -107,21 +107,21 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
)
end
- defp resource_search(:v2, "hashtags", query, _options) do
+ defp resource_search(:v2, "hashtags", query, options) do
tags_path = Web.base_url() <> "/tag/"
query
- |> prepare_tags()
+ |> prepare_tags(options)
|> Enum.map(fn tag ->
%{name: tag, url: tags_path <> tag}
end)
end
- defp resource_search(:v1, "hashtags", query, _options) do
- prepare_tags(query)
+ defp resource_search(:v1, "hashtags", query, options) do
+ prepare_tags(query, options)
end
- defp prepare_tags(query, add_joined_tag \\ true) do
+ defp prepare_tags(query, options) do
tags =
query
|> preprocess_uri_query()
@@ -139,13 +139,20 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end)
- if Enum.empty?(explicit_tags) && add_joined_tag do
- tags
- |> Kernel.++([joined_tag(tags)])
- |> Enum.uniq_by(&String.downcase/1)
- else
- tags
- end
+ tags =
+ if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do
+ add_joined_tag(tags)
+ else
+ tags
+ end
+
+ Pleroma.Pagination.paginate(tags, options)
+ end
+
+ defp add_joined_tag(tags) do
+ tags
+ |> Kernel.++([joined_tag(tags)])
+ |> Enum.uniq_by(&String.downcase/1)
end
# If `query` is a URI, returns last component of its path, otherwise returns `query`
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 9fc06bf9d..68beb69b8 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -224,7 +224,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
fields: user.fields,
bot: bot,
source: %{
- note: prepare_user_bio(user),
+ note: user.raw_bio || "",
sensitive: false,
fields: user.raw_fields,
pleroma: %{
@@ -260,17 +260,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
|> maybe_put_unread_notification_count(user, opts[:for])
end
- defp prepare_user_bio(%User{bio: ""}), do: ""
-
- defp prepare_user_bio(%User{bio: bio}) when is_binary(bio) do
- bio
- |> String.replace(~r(<br */?>), "\n")
- |> Pleroma.HTML.strip_tags()
- |> HtmlEntities.decode()
- end
-
- defp prepare_user_bio(_), do: ""
-
defp username_from_nickname(string) when is_binary(string) do
hd(String.split(string, "@"))
end
diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
index fbe618377..06f0c1728 100644
--- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
@@ -23,10 +23,13 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do
last_activity_id =
with nil <- participation.last_activity_id do
- ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{
- user: user,
- blocking_user: user
- })
+ ActivityPub.fetch_latest_direct_activity_id_for_context(
+ participation.conversation.ap_id,
+ %{
+ user: user,
+ blocking_user: user
+ }
+ )
end
activity = Activity.get_by_id_with_object(last_activity_id)
diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex
index c037ff13e..5808861e6 100644
--- a/lib/pleroma/web/media_proxy/invalidation.ex
+++ b/lib/pleroma/web/media_proxy/invalidation.ex
@@ -5,22 +5,34 @@
defmodule Pleroma.Web.MediaProxy.Invalidation do
@moduledoc false
- @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()}
+ @callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()}
alias Pleroma.Config
+ alias Pleroma.Web.MediaProxy
- @spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()}
+ @spec enabled?() :: boolean()
+ def enabled?, do: Config.get([:media_proxy, :invalidation, :enabled])
+
+ @spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()}
def purge(urls) do
- [:media_proxy, :invalidation, :enabled]
- |> Config.get()
- |> do_purge(urls)
+ prepared_urls = prepare_urls(urls)
+
+ if enabled?() do
+ do_purge(prepared_urls)
+ else
+ {:ok, prepared_urls}
+ end
end
- defp do_purge(true, urls) do
+ defp do_purge(urls) do
provider = Config.get([:media_proxy, :invalidation, :provider])
options = Config.get(provider)
provider.purge(urls, options)
end
- defp do_purge(_, _), do: :ok
+ def prepare_urls(urls) do
+ urls
+ |> List.wrap()
+ |> Enum.map(&MediaProxy.url/1)
+ end
end
diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex
index 07248df6e..bb81d8888 100644
--- a/lib/pleroma/web/media_proxy/invalidations/http.ex
+++ b/lib/pleroma/web/media_proxy/invalidations/http.ex
@@ -9,10 +9,10 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do
require Logger
@impl Pleroma.Web.MediaProxy.Invalidation
- def purge(urls, opts) do
- method = Map.get(opts, :method, :purge)
- headers = Map.get(opts, :headers, [])
- options = Map.get(opts, :options, [])
+ def purge(urls, opts \\ []) do
+ method = Keyword.get(opts, :method, :purge)
+ headers = Keyword.get(opts, :headers, [])
+ options = Keyword.get(opts, :options, [])
Logger.debug("Running cache purge: #{inspect(urls)}")
@@ -22,7 +22,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do
end
end)
- {:ok, "success"}
+ {:ok, urls}
end
defp do_purge(method, url, headers, options) do
diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex
index 6be782132..d32ffc50b 100644
--- a/lib/pleroma/web/media_proxy/invalidations/script.ex
+++ b/lib/pleroma/web/media_proxy/invalidations/script.ex
@@ -10,32 +10,34 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
require Logger
@impl Pleroma.Web.MediaProxy.Invalidation
- def purge(urls, %{script_path: script_path} = _options) do
+ def purge(urls, opts \\ []) do
args =
urls
|> List.wrap()
|> Enum.uniq()
|> Enum.join(" ")
- path = Path.expand(script_path)
-
- Logger.debug("Running cache purge: #{inspect(urls)}, #{path}")
-
- case do_purge(path, [args]) do
- {result, exit_status} when exit_status > 0 ->
- Logger.error("Error while cache purge: #{inspect(result)}")
- {:error, inspect(result)}
-
- _ ->
- {:ok, "success"}
- end
+ opts
+ |> Keyword.get(:script_path)
+ |> do_purge([args])
+ |> handle_result(urls)
end
- def purge(_, _), do: {:error, "not found script path"}
-
- defp do_purge(path, args) do
+ defp do_purge(script_path, args) when is_binary(script_path) do
+ path = Path.expand(script_path)
+ Logger.debug("Running cache purge: #{inspect(args)}, #{inspect(path)}")
System.cmd(path, args)
rescue
- error -> {inspect(error), 1}
+ error -> error
+ end
+
+ defp do_purge(_, _), do: {:error, "not found script path"}
+
+ defp handle_result({_result, 0}, urls), do: {:ok, urls}
+ defp handle_result({:error, error}, urls), do: handle_result(error, urls)
+
+ defp handle_result(error, _) do
+ Logger.error("Error while cache purge: #{inspect(error)}")
+ {:error, inspect(error)}
end
end
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index b2b524524..077fabe47 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -6,20 +6,53 @@ defmodule Pleroma.Web.MediaProxy do
alias Pleroma.Config
alias Pleroma.Upload
alias Pleroma.Web
+ alias Pleroma.Web.MediaProxy.Invalidation
@base64_opts [padding: false]
+ @spec in_banned_urls(String.t()) :: boolean()
+ def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1)
+
+ def remove_from_banned_urls(urls) when is_list(urls) do
+ Cachex.execute!(:banned_urls_cache, fn cache ->
+ Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
+ end)
+ end
+
+ def remove_from_banned_urls(url) when is_binary(url) do
+ Cachex.del(:banned_urls_cache, url(url))
+ end
+
+ def put_in_banned_urls(urls) when is_list(urls) do
+ Cachex.execute!(:banned_urls_cache, fn cache ->
+ Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
+ end)
+ end
+
+ def put_in_banned_urls(url) when is_binary(url) do
+ Cachex.put(:banned_urls_cache, url(url), true)
+ end
+
def url(url) when is_nil(url) or url == "", do: nil
def url("/" <> _ = url), do: url
def url(url) do
- if disabled?() or local?(url) or whitelisted?(url) do
+ if disabled?() or not url_proxiable?(url) do
url
else
encode_url(url)
end
end
+ @spec url_proxiable?(String.t()) :: boolean()
+ def url_proxiable?(url) do
+ if local?(url) or whitelisted?(url) do
+ false
+ else
+ true
+ end
+ end
+
defp disabled?, do: !Config.get([:media_proxy, :enabled], false)
defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 4657a4383..9a64b0ef3 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -14,10 +14,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
with config <- Pleroma.Config.get([:media_proxy], []),
true <- Keyword.get(config, :enabled, false),
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
+ {_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)},
:ok <- filename_matches(params, conn.request_path, url) do
ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
else
- false ->
+ error when error in [false, {:in_banned_urls, true}] ->
send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
{:error, :invalid_signature} ->
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 57570b672..eda74a171 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -209,6 +209,10 @@ defmodule Pleroma.Web.Router do
post("/oauth_app", OAuthAppController, :create)
patch("/oauth_app/:id", OAuthAppController, :update)
delete("/oauth_app/:id", OAuthAppController, :delete)
+
+ get("/media_proxy_caches", MediaProxyCacheController, :index)
+ post("/media_proxy_caches/delete", MediaProxyCacheController, :delete)
+ post("/media_proxy_caches/purge", MediaProxyCacheController, :purge)
end
scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do
diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex
index 49352db2a..8deeabda0 100644
--- a/lib/pleroma/workers/attachments_cleanup_worker.ex
+++ b/lib/pleroma/workers/attachments_cleanup_worker.ex
@@ -18,13 +18,19 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
},
_job
) do
- hrefs =
- Enum.flat_map(attachments, fn attachment ->
- Enum.map(attachment["url"], & &1["href"])
- end)
+ attachments
+ |> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end)
+ |> fetch_objects
+ |> prepare_objects(actor, Enum.map(attachments, & &1["name"]))
+ |> filter_objects
+ |> do_clean
- names = Enum.map(attachments, & &1["name"])
+ {:ok, :success}
+ end
+
+ def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
+ defp do_clean({object_ids, attachment_urls}) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
prefix =
@@ -39,68 +45,70 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
"/"
)
- # find all objects for copies of the attachments, name and actor doesn't matter here
- object_ids_and_hrefs =
- from(o in Object,
- where:
- fragment(
- "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
- o.data,
- o.data,
- ^hrefs
- )
- )
- # The query above can be time consumptive on large instances until we
- # refactor how uploads are stored
- |> Repo.all(timeout: :infinity)
- # we should delete 1 object for any given attachment, but don't delete
- # files if there are more than 1 object for it
- |> Enum.reduce(%{}, fn %{
- id: id,
- data: %{
- "url" => [%{"href" => href}],
- "actor" => obj_actor,
- "name" => name
- }
- },
- acc ->
- Map.update(acc, href, %{id: id, count: 1}, fn val ->
- case obj_actor == actor and name in names do
- true ->
- # set id of the actor's object that will be deleted
- %{val | id: id, count: val.count + 1}
-
- false ->
- # another actor's object, just increase count to not delete file
- %{val | count: val.count + 1}
- end
- end)
- end)
- |> Enum.map(fn {href, %{id: id, count: count}} ->
- # only delete files that have single instance
- with 1 <- count do
- href
- |> String.trim_leading("#{base_url}/#{prefix}")
- |> uploader.delete_file()
-
- {id, href}
- else
- _ -> {id, nil}
- end
- end)
+ Enum.each(attachment_urls, fn href ->
+ href
+ |> String.trim_leading("#{base_url}/#{prefix}")
+ |> uploader.delete_file()
+ end)
- object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end)
+ delete_objects(object_ids)
+ end
- from(o in Object, where: o.id in ^object_ids)
- |> Repo.delete_all()
+ defp delete_objects([_ | _] = object_ids) do
+ Repo.delete_all(from(o in Object, where: o.id in ^object_ids))
+ end
- object_ids_and_hrefs
- |> Enum.filter(fn {_, href} -> not is_nil(href) end)
- |> Enum.map(&elem(&1, 1))
- |> Pleroma.Web.MediaProxy.Invalidation.purge()
+ defp delete_objects(_), do: :ok
- {:ok, :success}
+ # we should delete 1 object for any given attachment, but don't delete
+ # files if there are more than 1 object for it
+ defp filter_objects(objects) do
+ Enum.reduce(objects, {[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} ->
+ with 1 <- count do
+ {ids ++ [id], hrefs ++ [href]}
+ else
+ _ -> {ids ++ [id], hrefs}
+ end
+ end)
end
- def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
+ defp prepare_objects(objects, actor, names) do
+ objects
+ |> Enum.reduce(%{}, fn %{
+ id: id,
+ data: %{
+ "url" => [%{"href" => href}],
+ "actor" => obj_actor,
+ "name" => name
+ }
+ },
+ acc ->
+ Map.update(acc, href, %{id: id, count: 1}, fn val ->
+ case obj_actor == actor and name in names do
+ true ->
+ # set id of the actor's object that will be deleted
+ %{val | id: id, count: val.count + 1}
+
+ false ->
+ # another actor's object, just increase count to not delete file
+ %{val | count: val.count + 1}
+ end
+ end)
+ end)
+ end
+
+ defp fetch_objects(hrefs) do
+ from(o in Object,
+ where:
+ fragment(
+ "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
+ o.data,
+ o.data,
+ ^hrefs
+ )
+ )
+ # The query above can be time consumptive on large instances until we
+ # refactor how uploads are stored
+ |> Repo.all(timeout: :infinity)
+ end
end
diff --git a/mix.exs b/mix.exs
index 03b060bc0..6040c994e 100644
--- a/mix.exs
+++ b/mix.exs
@@ -5,7 +5,7 @@ defmodule Pleroma.Mixfile do
[
app: :pleroma,
version: version("2.0.50"),
- elixir: "~> 1.8",
+ elixir: "~> 1.9",
elixirc_paths: elixirc_paths(Mix.env()),
compilers: [:phoenix, :gettext] ++ Mix.compilers(),
elixirc_options: [warnings_as_errors: warnings_as_errors(Mix.env())],
diff --git a/priv/repo/migrations/20200322174133_user_raw_bio.exs b/priv/repo/migrations/20200322174133_user_raw_bio.exs
new file mode 100644
index 000000000..ddf9be4f5
--- /dev/null
+++ b/priv/repo/migrations/20200322174133_user_raw_bio.exs
@@ -0,0 +1,9 @@
+defmodule Pleroma.Repo.Migrations.UserRawBio do
+ use Ecto.Migration
+
+ def change do
+ alter table(:users) do
+ add_if_not_exists(:raw_bio, :text)
+ end
+ end
+end
diff --git a/priv/repo/migrations/20200328193433_populate_user_raw_bio.exs b/priv/repo/migrations/20200328193433_populate_user_raw_bio.exs
new file mode 100644
index 000000000..cb35db3f5
--- /dev/null
+++ b/priv/repo/migrations/20200328193433_populate_user_raw_bio.exs
@@ -0,0 +1,25 @@
+defmodule Pleroma.Repo.Migrations.PopulateUserRawBio do
+ use Ecto.Migration
+ import Ecto.Query
+ alias Pleroma.User
+ alias Pleroma.Repo
+
+ def change do
+ {:ok, _} = Application.ensure_all_started(:fast_sanitize)
+
+ User.Query.build(%{local: true})
+ |> select([u], struct(u, [:id, :ap_id, :bio]))
+ |> Repo.stream()
+ |> Enum.each(fn %{bio: bio} = user ->
+ if bio do
+ raw_bio =
+ bio
+ |> String.replace(~r(<br */?>), "\n")
+ |> Pleroma.HTML.strip_tags()
+
+ Ecto.Changeset.cast(user, %{raw_bio: raw_bio}, [:raw_bio])
+ |> Repo.update()
+ end
+ end)
+ end
+end
diff --git a/test/config/config_db_test.exs b/test/config/config_db_test.exs
index 336de7359..3895e2cda 100644
--- a/test/config/config_db_test.exs
+++ b/test/config/config_db_test.exs
@@ -7,40 +7,28 @@ defmodule Pleroma.ConfigDBTest do
import Pleroma.Factory
alias Pleroma.ConfigDB
- test "get_by_key/1" do
+ test "get_by_params/1" do
config = insert(:config)
insert(:config)
assert config == ConfigDB.get_by_params(%{group: config.group, key: config.key})
end
- test "create/1" do
- {:ok, config} = ConfigDB.create(%{group: ":pleroma", key: ":some_key", value: "some_value"})
- assert config == ConfigDB.get_by_params(%{group: ":pleroma", key: ":some_key"})
- end
-
- test "update/1" do
- config = insert(:config)
- {:ok, updated} = ConfigDB.update(config, %{value: "some_value"})
- loaded = ConfigDB.get_by_params(%{group: config.group, key: config.key})
- assert loaded == updated
- end
-
test "get_all_as_keyword/0" do
saved = insert(:config)
- insert(:config, group: ":quack", key: ":level", value: ConfigDB.to_binary(:info))
- insert(:config, group: ":quack", key: ":meta", value: ConfigDB.to_binary([:none]))
+ insert(:config, group: ":quack", key: ":level", value: :info)
+ insert(:config, group: ":quack", key: ":meta", value: [:none])
insert(:config,
group: ":quack",
key: ":webhook_url",
- value: ConfigDB.to_binary("https://hooks.slack.com/services/KEY/some_val")
+ value: "https://hooks.slack.com/services/KEY/some_val"
)
config = ConfigDB.get_all_as_keyword()
assert config[:pleroma] == [
- {ConfigDB.from_string(saved.key), ConfigDB.from_binary(saved.value)}
+ {saved.key, saved.value}
]
assert config[:quack][:level] == :info
@@ -51,11 +39,11 @@ defmodule Pleroma.ConfigDBTest do
describe "update_or_create/1" do
test "common" do
config = insert(:config)
- key2 = "another_key"
+ key2 = :another_key
params = [
- %{group: "pleroma", key: key2, value: "another_value"},
- %{group: config.group, key: config.key, value: "new_value"}
+ %{group: :pleroma, key: key2, value: "another_value"},
+ %{group: :pleroma, key: config.key, value: [a: 1, b: 2, c: "new_value"]}
]
assert Repo.all(ConfigDB) |> length() == 1
@@ -65,16 +53,16 @@ defmodule Pleroma.ConfigDBTest do
assert Repo.all(ConfigDB) |> length() == 2
config1 = ConfigDB.get_by_params(%{group: config.group, key: config.key})
- config2 = ConfigDB.get_by_params(%{group: "pleroma", key: key2})
+ config2 = ConfigDB.get_by_params(%{group: :pleroma, key: key2})
- assert config1.value == ConfigDB.transform("new_value")
- assert config2.value == ConfigDB.transform("another_value")
+ assert config1.value == [a: 1, b: 2, c: "new_value"]
+ assert config2.value == "another_value"
end
test "partial update" do
- config = insert(:config, value: ConfigDB.to_binary(key1: "val1", key2: :val2))
+ config = insert(:config, value: [key1: "val1", key2: :val2])
- {:ok, _config} =
+ {:ok, config} =
ConfigDB.update_or_create(%{
group: config.group,
key: config.key,
@@ -83,15 +71,14 @@ defmodule Pleroma.ConfigDBTest do
updated = ConfigDB.get_by_params(%{group: config.group, key: config.key})
- value = ConfigDB.from_binary(updated.value)
- assert length(value) == 3
- assert value[:key1] == :val1
- assert value[:key2] == :val2
- assert value[:key3] == :val3
+ assert config.value == updated.value
+ assert updated.value[:key1] == :val1
+ assert updated.value[:key2] == :val2
+ assert updated.value[:key3] == :val3
end
test "deep merge" do
- config = insert(:config, value: ConfigDB.to_binary(key1: "val1", key2: [k1: :v1, k2: "v2"]))
+ config = insert(:config, value: [key1: "val1", key2: [k1: :v1, k2: "v2"]])
{:ok, config} =
ConfigDB.update_or_create(%{
@@ -103,18 +90,15 @@ defmodule Pleroma.ConfigDBTest do
updated = ConfigDB.get_by_params(%{group: config.group, key: config.key})
assert config.value == updated.value
-
- value = ConfigDB.from_binary(updated.value)
- assert value[:key1] == :val1
- assert value[:key2] == [k1: :v1, k2: :v2, k3: :v3]
- assert value[:key3] == :val3
+ assert updated.value[:key1] == :val1
+ assert updated.value[:key2] == [k1: :v1, k2: :v2, k3: :v3]
+ assert updated.value[:key3] == :val3
end
test "only full update for some keys" do
- config1 = insert(:config, key: ":ecto_repos", value: ConfigDB.to_binary(repo: Pleroma.Repo))
+ config1 = insert(:config, key: :ecto_repos, value: [repo: Pleroma.Repo])
- config2 =
- insert(:config, group: ":cors_plug", key: ":max_age", value: ConfigDB.to_binary(18))
+ config2 = insert(:config, group: :cors_plug, key: :max_age, value: 18)
{:ok, _config} =
ConfigDB.update_or_create(%{
@@ -133,8 +117,8 @@ defmodule Pleroma.ConfigDBTest do
updated1 = ConfigDB.get_by_params(%{group: config1.group, key: config1.key})
updated2 = ConfigDB.get_by_params(%{group: config2.group, key: config2.key})
- assert ConfigDB.from_binary(updated1.value) == [another_repo: [Pleroma.Repo]]
- assert ConfigDB.from_binary(updated2.value) == 777
+ assert updated1.value == [another_repo: [Pleroma.Repo]]
+ assert updated2.value == 777
end
test "full update if value is not keyword" do
@@ -142,7 +126,7 @@ defmodule Pleroma.ConfigDBTest do
insert(:config,
group: ":tesla",
key: ":adapter",
- value: ConfigDB.to_binary(Tesla.Adapter.Hackney)
+ value: Tesla.Adapter.Hackney
)
{:ok, _config} =
@@ -154,20 +138,20 @@ defmodule Pleroma.ConfigDBTest do
updated = ConfigDB.get_by_params(%{group: config.group, key: config.key})
- assert ConfigDB.from_binary(updated.value) == Tesla.Adapter.Httpc
+ assert updated.value == Tesla.Adapter.Httpc
end
test "only full update for some subkeys" do
config1 =
insert(:config,
key: ":emoji",
- value: ConfigDB.to_binary(groups: [a: 1, b: 2], key: [a: 1])
+ value: [groups: [a: 1, b: 2], key: [a: 1]]
)
config2 =
insert(:config,
key: ":assets",
- value: ConfigDB.to_binary(mascots: [a: 1, b: 2], key: [a: 1])
+ value: [mascots: [a: 1, b: 2], key: [a: 1]]
)
{:ok, _config} =
@@ -187,8 +171,8 @@ defmodule Pleroma.ConfigDBTest do
updated1 = ConfigDB.get_by_params(%{group: config1.group, key: config1.key})
updated2 = ConfigDB.get_by_params(%{group: config2.group, key: config2.key})
- assert ConfigDB.from_binary(updated1.value) == [groups: [c: 3, d: 4], key: [a: 1, b: 2]]
- assert ConfigDB.from_binary(updated2.value) == [mascots: [c: 3, d: 4], key: [a: 1, b: 2]]
+ assert updated1.value == [groups: [c: 3, d: 4], key: [a: 1, b: 2]]
+ assert updated2.value == [mascots: [c: 3, d: 4], key: [a: 1, b: 2]]
end
end
@@ -206,14 +190,14 @@ defmodule Pleroma.ConfigDBTest do
end
test "partial subkeys delete" do
- config = insert(:config, value: ConfigDB.to_binary(groups: [a: 1, b: 2], key: [a: 1]))
+ config = insert(:config, value: [groups: [a: 1, b: 2], key: [a: 1]])
{:ok, deleted} =
ConfigDB.delete(%{group: config.group, key: config.key, subkeys: [":groups"]})
assert Ecto.get_meta(deleted, :state) == :loaded
- assert deleted.value == ConfigDB.to_binary(key: [a: 1])
+ assert deleted.value == [key: [a: 1]]
updated = ConfigDB.get_by_params(%{group: config.group, key: config.key})
@@ -221,7 +205,7 @@ defmodule Pleroma.ConfigDBTest do
end
test "full delete if remaining value after subkeys deletion is empty list" do
- config = insert(:config, value: ConfigDB.to_binary(groups: [a: 1, b: 2]))
+ config = insert(:config, value: [groups: [a: 1, b: 2]])
{:ok, deleted} =
ConfigDB.delete(%{group: config.group, key: config.key, subkeys: [":groups"]})
@@ -232,234 +216,159 @@ defmodule Pleroma.ConfigDBTest do
end
end
- describe "transform/1" do
+ describe "to_elixir_types/1" do
test "string" do
- binary = ConfigDB.transform("value as string")
- assert binary == :erlang.term_to_binary("value as string")
- assert ConfigDB.from_binary(binary) == "value as string"
+ assert ConfigDB.to_elixir_types("value as string") == "value as string"
end
test "boolean" do
- binary = ConfigDB.transform(false)
- assert binary == :erlang.term_to_binary(false)
- assert ConfigDB.from_binary(binary) == false
+ assert ConfigDB.to_elixir_types(false) == false
end
test "nil" do
- binary = ConfigDB.transform(nil)
- assert binary == :erlang.term_to_binary(nil)
- assert ConfigDB.from_binary(binary) == nil
+ assert ConfigDB.to_elixir_types(nil) == nil
end
test "integer" do
- binary = ConfigDB.transform(150)
- assert binary == :erlang.term_to_binary(150)
- assert ConfigDB.from_binary(binary) == 150
+ assert ConfigDB.to_elixir_types(150) == 150
end
test "atom" do
- binary = ConfigDB.transform(":atom")
- assert binary == :erlang.term_to_binary(:atom)
- assert ConfigDB.from_binary(binary) == :atom
+ assert ConfigDB.to_elixir_types(":atom") == :atom
end
test "ssl options" do
- binary = ConfigDB.transform([":tlsv1", ":tlsv1.1", ":tlsv1.2"])
- assert binary == :erlang.term_to_binary([:tlsv1, :"tlsv1.1", :"tlsv1.2"])
- assert ConfigDB.from_binary(binary) == [:tlsv1, :"tlsv1.1", :"tlsv1.2"]
+ assert ConfigDB.to_elixir_types([":tlsv1", ":tlsv1.1", ":tlsv1.2"]) == [
+ :tlsv1,
+ :"tlsv1.1",
+ :"tlsv1.2"
+ ]
end
test "pleroma module" do
- binary = ConfigDB.transform("Pleroma.Bookmark")
- assert binary == :erlang.term_to_binary(Pleroma.Bookmark)
- assert ConfigDB.from_binary(binary) == Pleroma.Bookmark
+ assert ConfigDB.to_elixir_types("Pleroma.Bookmark") == Pleroma.Bookmark
end
test "pleroma string" do
- binary = ConfigDB.transform("Pleroma")
- assert binary == :erlang.term_to_binary("Pleroma")
- assert ConfigDB.from_binary(binary) == "Pleroma"
+ assert ConfigDB.to_elixir_types("Pleroma") == "Pleroma"
end
test "phoenix module" do
- binary = ConfigDB.transform("Phoenix.Socket.V1.JSONSerializer")
- assert binary == :erlang.term_to_binary(Phoenix.Socket.V1.JSONSerializer)
- assert ConfigDB.from_binary(binary) == Phoenix.Socket.V1.JSONSerializer
+ assert ConfigDB.to_elixir_types("Phoenix.Socket.V1.JSONSerializer") ==
+ Phoenix.Socket.V1.JSONSerializer
end
test "tesla module" do
- binary = ConfigDB.transform("Tesla.Adapter.Hackney")
- assert binary == :erlang.term_to_binary(Tesla.Adapter.Hackney)
- assert ConfigDB.from_binary(binary) == Tesla.Adapter.Hackney
+ assert ConfigDB.to_elixir_types("Tesla.Adapter.Hackney") == Tesla.Adapter.Hackney
end
test "ExSyslogger module" do
- binary = ConfigDB.transform("ExSyslogger")
- assert binary == :erlang.term_to_binary(ExSyslogger)
- assert ConfigDB.from_binary(binary) == ExSyslogger
+ assert ConfigDB.to_elixir_types("ExSyslogger") == ExSyslogger
end
test "Quack.Logger module" do
- binary = ConfigDB.transform("Quack.Logger")
- assert binary == :erlang.term_to_binary(Quack.Logger)
- assert ConfigDB.from_binary(binary) == Quack.Logger
+ assert ConfigDB.to_elixir_types("Quack.Logger") == Quack.Logger
end
test "Swoosh.Adapters modules" do
- binary = ConfigDB.transform("Swoosh.Adapters.SMTP")
- assert binary == :erlang.term_to_binary(Swoosh.Adapters.SMTP)
- assert ConfigDB.from_binary(binary) == Swoosh.Adapters.SMTP
- binary = ConfigDB.transform("Swoosh.Adapters.AmazonSES")
- assert binary == :erlang.term_to_binary(Swoosh.Adapters.AmazonSES)
- assert ConfigDB.from_binary(binary) == Swoosh.Adapters.AmazonSES
+ assert ConfigDB.to_elixir_types("Swoosh.Adapters.SMTP") == Swoosh.Adapters.SMTP
+ assert ConfigDB.to_elixir_types("Swoosh.Adapters.AmazonSES") == Swoosh.Adapters.AmazonSES
end
test "sigil" do
- binary = ConfigDB.transform("~r[comp[lL][aA][iI][nN]er]")
- assert binary == :erlang.term_to_binary(~r/comp[lL][aA][iI][nN]er/)
- assert ConfigDB.from_binary(binary) == ~r/comp[lL][aA][iI][nN]er/
+ assert ConfigDB.to_elixir_types("~r[comp[lL][aA][iI][nN]er]") == ~r/comp[lL][aA][iI][nN]er/
end
test "link sigil" do
- binary = ConfigDB.transform("~r/https:\/\/example.com/")
- assert binary == :erlang.term_to_binary(~r/https:\/\/example.com/)
- assert ConfigDB.from_binary(binary) == ~r/https:\/\/example.com/
+ assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/") == ~r/https:\/\/example.com/
end
test "link sigil with um modifiers" do
- binary = ConfigDB.transform("~r/https:\/\/example.com/um")
- assert binary == :erlang.term_to_binary(~r/https:\/\/example.com/um)
- assert ConfigDB.from_binary(binary) == ~r/https:\/\/example.com/um
+ assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/um") ==
+ ~r/https:\/\/example.com/um
end
test "link sigil with i modifier" do
- binary = ConfigDB.transform("~r/https:\/\/example.com/i")
- assert binary == :erlang.term_to_binary(~r/https:\/\/example.com/i)
- assert ConfigDB.from_binary(binary) == ~r/https:\/\/example.com/i
+ assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/i") == ~r/https:\/\/example.com/i
end
test "link sigil with s modifier" do
- binary = ConfigDB.transform("~r/https:\/\/example.com/s")
- assert binary == :erlang.term_to_binary(~r/https:\/\/example.com/s)
- assert ConfigDB.from_binary(binary) == ~r/https:\/\/example.com/s
+ assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/s") == ~r/https:\/\/example.com/s
end
test "raise if valid delimiter not found" do
assert_raise ArgumentError, "valid delimiter for Regex expression not found", fn ->
- ConfigDB.transform("~r/https://[]{}<>\"'()|example.com/s")
+ ConfigDB.to_elixir_types("~r/https://[]{}<>\"'()|example.com/s")
end
end
test "2 child tuple" do
- binary = ConfigDB.transform(%{"tuple" => ["v1", ":v2"]})
- assert binary == :erlang.term_to_binary({"v1", :v2})
- assert ConfigDB.from_binary(binary) == {"v1", :v2}
+ assert ConfigDB.to_elixir_types(%{"tuple" => ["v1", ":v2"]}) == {"v1", :v2}
end
test "proxy tuple with localhost" do
- binary =
- ConfigDB.transform(%{
- "tuple" => [":proxy_url", %{"tuple" => [":socks5", "localhost", 1234]}]
- })
-
- assert binary == :erlang.term_to_binary({:proxy_url, {:socks5, :localhost, 1234}})
- assert ConfigDB.from_binary(binary) == {:proxy_url, {:socks5, :localhost, 1234}}
+ assert ConfigDB.to_elixir_types(%{
+ "tuple" => [":proxy_url", %{"tuple" => [":socks5", "localhost", 1234]}]
+ }) == {:proxy_url, {:socks5, :localhost, 1234}}
end
test "proxy tuple with domain" do
- binary =
- ConfigDB.transform(%{
- "tuple" => [":proxy_url", %{"tuple" => [":socks5", "domain.com", 1234]}]
- })
-
- assert binary == :erlang.term_to_binary({:proxy_url, {:socks5, 'domain.com', 1234}})
- assert ConfigDB.from_binary(binary) == {:proxy_url, {:socks5, 'domain.com', 1234}}
+ assert ConfigDB.to_elixir_types(%{
+ "tuple" => [":proxy_url", %{"tuple" => [":socks5", "domain.com", 1234]}]
+ }) == {:proxy_url, {:socks5, 'domain.com', 1234}}
end
test "proxy tuple with ip" do
- binary =
- ConfigDB.transform(%{
- "tuple" => [":proxy_url", %{"tuple" => [":socks5", "127.0.0.1", 1234]}]
- })
-
- assert binary == :erlang.term_to_binary({:proxy_url, {:socks5, {127, 0, 0, 1}, 1234}})
- assert ConfigDB.from_binary(binary) == {:proxy_url, {:socks5, {127, 0, 0, 1}, 1234}}
+ assert ConfigDB.to_elixir_types(%{
+ "tuple" => [":proxy_url", %{"tuple" => [":socks5", "127.0.0.1", 1234]}]
+ }) == {:proxy_url, {:socks5, {127, 0, 0, 1}, 1234}}
end
test "tuple with n childs" do
- binary =
- ConfigDB.transform(%{
- "tuple" => [
- "v1",
- ":v2",
- "Pleroma.Bookmark",
- 150,
- false,
- "Phoenix.Socket.V1.JSONSerializer"
- ]
- })
-
- assert binary ==
- :erlang.term_to_binary(
- {"v1", :v2, Pleroma.Bookmark, 150, false, Phoenix.Socket.V1.JSONSerializer}
- )
-
- assert ConfigDB.from_binary(binary) ==
- {"v1", :v2, Pleroma.Bookmark, 150, false, Phoenix.Socket.V1.JSONSerializer}
+ assert ConfigDB.to_elixir_types(%{
+ "tuple" => [
+ "v1",
+ ":v2",
+ "Pleroma.Bookmark",
+ 150,
+ false,
+ "Phoenix.Socket.V1.JSONSerializer"
+ ]
+ }) == {"v1", :v2, Pleroma.Bookmark, 150, false, Phoenix.Socket.V1.JSONSerializer}
end
test "map with string key" do
- binary = ConfigDB.transform(%{"key" => "value"})
- assert binary == :erlang.term_to_binary(%{"key" => "value"})
- assert ConfigDB.from_binary(binary) == %{"key" => "value"}
+ assert ConfigDB.to_elixir_types(%{"key" => "value"}) == %{"key" => "value"}
end
test "map with atom key" do
- binary = ConfigDB.transform(%{":key" => "value"})
- assert binary == :erlang.term_to_binary(%{key: "value"})
- assert ConfigDB.from_binary(binary) == %{key: "value"}
+ assert ConfigDB.to_elixir_types(%{":key" => "value"}) == %{key: "value"}
end
test "list of strings" do
- binary = ConfigDB.transform(["v1", "v2", "v3"])
- assert binary == :erlang.term_to_binary(["v1", "v2", "v3"])
- assert ConfigDB.from_binary(binary) == ["v1", "v2", "v3"]
+ assert ConfigDB.to_elixir_types(["v1", "v2", "v3"]) == ["v1", "v2", "v3"]
end
test "list of modules" do
- binary = ConfigDB.transform(["Pleroma.Repo", "Pleroma.Activity"])
- assert binary == :erlang.term_to_binary([Pleroma.Repo, Pleroma.Activity])
- assert ConfigDB.from_binary(binary) == [Pleroma.Repo, Pleroma.Activity]
+ assert ConfigDB.to_elixir_types(["Pleroma.Repo", "Pleroma.Activity"]) == [
+ Pleroma.Repo,
+ Pleroma.Activity
+ ]
end
test "list of atoms" do
- binary = ConfigDB.transform([":v1", ":v2", ":v3"])
- assert binary == :erlang.term_to_binary([:v1, :v2, :v3])
- assert ConfigDB.from_binary(binary) == [:v1, :v2, :v3]
+ assert ConfigDB.to_elixir_types([":v1", ":v2", ":v3"]) == [:v1, :v2, :v3]
end
test "list of mixed values" do
- binary =
- ConfigDB.transform([
- "v1",
- ":v2",
- "Pleroma.Repo",
- "Phoenix.Socket.V1.JSONSerializer",
- 15,
- false
- ])
-
- assert binary ==
- :erlang.term_to_binary([
- "v1",
- :v2,
- Pleroma.Repo,
- Phoenix.Socket.V1.JSONSerializer,
- 15,
- false
- ])
-
- assert ConfigDB.from_binary(binary) == [
+ assert ConfigDB.to_elixir_types([
+ "v1",
+ ":v2",
+ "Pleroma.Repo",
+ "Phoenix.Socket.V1.JSONSerializer",
+ 15,
+ false
+ ]) == [
"v1",
:v2,
Pleroma.Repo,
@@ -470,40 +379,17 @@ defmodule Pleroma.ConfigDBTest do
end
test "simple keyword" do
- binary = ConfigDB.transform([%{"tuple" => [":key", "value"]}])
- assert binary == :erlang.term_to_binary([{:key, "value"}])
- assert ConfigDB.from_binary(binary) == [{:key, "value"}]
- assert ConfigDB.from_binary(binary) == [key: "value"]
- end
-
- test "keyword with partial_chain key" do
- binary =
- ConfigDB.transform([%{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}])
-
- assert binary == :erlang.term_to_binary(partial_chain: &:hackney_connect.partial_chain/1)
- assert ConfigDB.from_binary(binary) == [partial_chain: &:hackney_connect.partial_chain/1]
+ assert ConfigDB.to_elixir_types([%{"tuple" => [":key", "value"]}]) == [key: "value"]
end
test "keyword" do
- binary =
- ConfigDB.transform([
- %{"tuple" => [":types", "Pleroma.PostgresTypes"]},
- %{"tuple" => [":telemetry_event", ["Pleroma.Repo.Instrumenter"]]},
- %{"tuple" => [":migration_lock", nil]},
- %{"tuple" => [":key1", 150]},
- %{"tuple" => [":key2", "string"]}
- ])
-
- assert binary ==
- :erlang.term_to_binary(
- types: Pleroma.PostgresTypes,
- telemetry_event: [Pleroma.Repo.Instrumenter],
- migration_lock: nil,
- key1: 150,
- key2: "string"
- )
-
- assert ConfigDB.from_binary(binary) == [
+ assert ConfigDB.to_elixir_types([
+ %{"tuple" => [":types", "Pleroma.PostgresTypes"]},
+ %{"tuple" => [":telemetry_event", ["Pleroma.Repo.Instrumenter"]]},
+ %{"tuple" => [":migration_lock", nil]},
+ %{"tuple" => [":key1", 150]},
+ %{"tuple" => [":key2", "string"]}
+ ]) == [
types: Pleroma.PostgresTypes,
telemetry_event: [Pleroma.Repo.Instrumenter],
migration_lock: nil,
@@ -512,86 +398,60 @@ defmodule Pleroma.ConfigDBTest do
]
end
+ test "trandformed keyword" do
+ assert ConfigDB.to_elixir_types(a: 1, b: 2, c: "string") == [a: 1, b: 2, c: "string"]
+ end
+
test "complex keyword with nested mixed childs" do
- binary =
- ConfigDB.transform([
- %{"tuple" => [":uploader", "Pleroma.Uploaders.Local"]},
- %{"tuple" => [":filters", ["Pleroma.Upload.Filter.Dedupe"]]},
- %{"tuple" => [":link_name", true]},
- %{"tuple" => [":proxy_remote", false]},
- %{"tuple" => [":common_map", %{":key" => "value"}]},
- %{
- "tuple" => [
- ":proxy_opts",
- [
- %{"tuple" => [":redirect_on_failure", false]},
- %{"tuple" => [":max_body_length", 1_048_576]},
- %{
- "tuple" => [
- ":http",
- [%{"tuple" => [":follow_redirect", true]}, %{"tuple" => [":pool", ":upload"]}]
- ]
- }
- ]
- ]
- }
- ])
-
- assert binary ==
- :erlang.term_to_binary(
- uploader: Pleroma.Uploaders.Local,
- filters: [Pleroma.Upload.Filter.Dedupe],
- link_name: true,
- proxy_remote: false,
- common_map: %{key: "value"},
- proxy_opts: [
- redirect_on_failure: false,
- max_body_length: 1_048_576,
- http: [
- follow_redirect: true,
- pool: :upload
+ assert ConfigDB.to_elixir_types([
+ %{"tuple" => [":uploader", "Pleroma.Uploaders.Local"]},
+ %{"tuple" => [":filters", ["Pleroma.Upload.Filter.Dedupe"]]},
+ %{"tuple" => [":link_name", true]},
+ %{"tuple" => [":proxy_remote", false]},
+ %{"tuple" => [":common_map", %{":key" => "value"}]},
+ %{
+ "tuple" => [
+ ":proxy_opts",
+ [
+ %{"tuple" => [":redirect_on_failure", false]},
+ %{"tuple" => [":max_body_length", 1_048_576]},
+ %{
+ "tuple" => [
+ ":http",
+ [
+ %{"tuple" => [":follow_redirect", true]},
+ %{"tuple" => [":pool", ":upload"]}
+ ]
+ ]
+ }
]
]
- )
-
- assert ConfigDB.from_binary(binary) ==
- [
- uploader: Pleroma.Uploaders.Local,
- filters: [Pleroma.Upload.Filter.Dedupe],
- link_name: true,
- proxy_remote: false,
- common_map: %{key: "value"},
- proxy_opts: [
- redirect_on_failure: false,
- max_body_length: 1_048_576,
- http: [
- follow_redirect: true,
- pool: :upload
- ]
+ }
+ ]) == [
+ uploader: Pleroma.Uploaders.Local,
+ filters: [Pleroma.Upload.Filter.Dedupe],
+ link_name: true,
+ proxy_remote: false,
+ common_map: %{key: "value"},
+ proxy_opts: [
+ redirect_on_failure: false,
+ max_body_length: 1_048_576,
+ http: [
+ follow_redirect: true,
+ pool: :upload
]
]
+ ]
end
test "common keyword" do
- binary =
- ConfigDB.transform([
- %{"tuple" => [":level", ":warn"]},
- %{"tuple" => [":meta", [":all"]]},
- %{"tuple" => [":path", ""]},
- %{"tuple" => [":val", nil]},
- %{"tuple" => [":webhook_url", "https://hooks.slack.com/services/YOUR-KEY-HERE"]}
- ])
-
- assert binary ==
- :erlang.term_to_binary(
- level: :warn,
- meta: [:all],
- path: "",
- val: nil,
- webhook_url: "https://hooks.slack.com/services/YOUR-KEY-HERE"
- )
-
- assert ConfigDB.from_binary(binary) == [
+ assert ConfigDB.to_elixir_types([
+ %{"tuple" => [":level", ":warn"]},
+ %{"tuple" => [":meta", [":all"]]},
+ %{"tuple" => [":path", ""]},
+ %{"tuple" => [":val", nil]},
+ %{"tuple" => [":webhook_url", "https://hooks.slack.com/services/YOUR-KEY-HERE"]}
+ ]) == [
level: :warn,
meta: [:all],
path: "",
@@ -601,98 +461,73 @@ defmodule Pleroma.ConfigDBTest do
end
test "complex keyword with sigil" do
- binary =
- ConfigDB.transform([
- %{"tuple" => [":federated_timeline_removal", []]},
- %{"tuple" => [":reject", ["~r/comp[lL][aA][iI][nN]er/"]]},
- %{"tuple" => [":replace", []]}
- ])
-
- assert binary ==
- :erlang.term_to_binary(
- federated_timeline_removal: [],
- reject: [~r/comp[lL][aA][iI][nN]er/],
- replace: []
- )
-
- assert ConfigDB.from_binary(binary) ==
- [federated_timeline_removal: [], reject: [~r/comp[lL][aA][iI][nN]er/], replace: []]
+ assert ConfigDB.to_elixir_types([
+ %{"tuple" => [":federated_timeline_removal", []]},
+ %{"tuple" => [":reject", ["~r/comp[lL][aA][iI][nN]er/"]]},
+ %{"tuple" => [":replace", []]}
+ ]) == [
+ federated_timeline_removal: [],
+ reject: [~r/comp[lL][aA][iI][nN]er/],
+ replace: []
+ ]
end
test "complex keyword with tuples with more than 2 values" do
- binary =
- ConfigDB.transform([
- %{
- "tuple" => [
- ":http",
- [
- %{
- "tuple" => [
- ":key1",
- [
- %{
- "tuple" => [
- ":_",
- [
- %{
- "tuple" => [
- "/api/v1/streaming",
- "Pleroma.Web.MastodonAPI.WebsocketHandler",
- []
- ]
- },
- %{
- "tuple" => [
- "/websocket",
- "Phoenix.Endpoint.CowboyWebSocket",
- %{
- "tuple" => [
- "Phoenix.Transports.WebSocket",
- %{
- "tuple" => [
- "Pleroma.Web.Endpoint",
- "Pleroma.Web.UserSocket",
- []
- ]
- }
- ]
- }
- ]
- },
- %{
- "tuple" => [
- ":_",
- "Phoenix.Endpoint.Cowboy2Handler",
- %{"tuple" => ["Pleroma.Web.Endpoint", []]}
- ]
- }
- ]
- ]
- }
- ]
- ]
- }
- ]
- ]
- }
- ])
-
- assert binary ==
- :erlang.term_to_binary(
- http: [
- key1: [
- _: [
- {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []},
- {"/websocket", Phoenix.Endpoint.CowboyWebSocket,
- {Phoenix.Transports.WebSocket,
- {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, []}}},
- {:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}}
- ]
+ assert ConfigDB.to_elixir_types([
+ %{
+ "tuple" => [
+ ":http",
+ [
+ %{
+ "tuple" => [
+ ":key1",
+ [
+ %{
+ "tuple" => [
+ ":_",
+ [
+ %{
+ "tuple" => [
+ "/api/v1/streaming",
+ "Pleroma.Web.MastodonAPI.WebsocketHandler",
+ []
+ ]
+ },
+ %{
+ "tuple" => [
+ "/websocket",
+ "Phoenix.Endpoint.CowboyWebSocket",
+ %{
+ "tuple" => [
+ "Phoenix.Transports.WebSocket",
+ %{
+ "tuple" => [
+ "Pleroma.Web.Endpoint",
+ "Pleroma.Web.UserSocket",
+ []
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ %{
+ "tuple" => [
+ ":_",
+ "Phoenix.Endpoint.Cowboy2Handler",
+ %{"tuple" => ["Pleroma.Web.Endpoint", []]}
+ ]
+ }
+ ]
+ ]
+ }
+ ]
+ ]
+ }
]
]
- )
-
- assert ConfigDB.from_binary(binary) == [
+ }
+ ]) == [
http: [
key1: [
{:_,
diff --git a/test/config/transfer_task_test.exs b/test/config/transfer_task_test.exs
index 473899d1d..f53829e09 100644
--- a/test/config/transfer_task_test.exs
+++ b/test/config/transfer_task_test.exs
@@ -6,9 +6,9 @@ defmodule Pleroma.Config.TransferTaskTest do
use Pleroma.DataCase
import ExUnit.CaptureLog
+ import Pleroma.Factory
alias Pleroma.Config.TransferTask
- alias Pleroma.ConfigDB
setup do: clear_config(:configurable_from_database, true)
@@ -19,31 +19,11 @@ defmodule Pleroma.Config.TransferTaskTest do
refute Application.get_env(:postgrex, :test_key)
initial = Application.get_env(:logger, :level)
- ConfigDB.create(%{
- group: ":pleroma",
- key: ":test_key",
- value: [live: 2, com: 3]
- })
-
- ConfigDB.create(%{
- group: ":idna",
- key: ":test_key",
- value: [live: 15, com: 35]
- })
-
- ConfigDB.create(%{
- group: ":quack",
- key: ":test_key",
- value: [:test_value1, :test_value2]
- })
-
- ConfigDB.create(%{
- group: ":postgrex",
- key: ":test_key",
- value: :value
- })
-
- ConfigDB.create(%{group: ":logger", key: ":level", value: :debug})
+ insert(:config, key: :test_key, value: [live: 2, com: 3])
+ insert(:config, group: :idna, key: :test_key, value: [live: 15, com: 35])
+ insert(:config, group: :quack, key: :test_key, value: [:test_value1, :test_value2])
+ insert(:config, group: :postgrex, key: :test_key, value: :value)
+ insert(:config, group: :logger, key: :level, value: :debug)
TransferTask.start_link([])
@@ -66,17 +46,8 @@ defmodule Pleroma.Config.TransferTaskTest do
level = Application.get_env(:quack, :level)
meta = Application.get_env(:quack, :meta)
- ConfigDB.create(%{
- group: ":quack",
- key: ":level",
- value: :info
- })
-
- ConfigDB.create(%{
- group: ":quack",
- key: ":meta",
- value: [:none]
- })
+ insert(:config, group: :quack, key: :level, value: :info)
+ insert(:config, group: :quack, key: :meta, value: [:none])
TransferTask.start_link([])
@@ -95,17 +66,8 @@ defmodule Pleroma.Config.TransferTaskTest do
clear_config(:emoji)
clear_config(:assets)
- ConfigDB.create(%{
- group: ":pleroma",
- key: ":emoji",
- value: [groups: [a: 1, b: 2]]
- })
-
- ConfigDB.create(%{
- group: ":pleroma",
- key: ":assets",
- value: [mascots: [a: 1, b: 2]]
- })
+ insert(:config, key: :emoji, value: [groups: [a: 1, b: 2]])
+ insert(:config, key: :assets, value: [mascots: [a: 1, b: 2]])
TransferTask.start_link([])
@@ -122,12 +84,7 @@ defmodule Pleroma.Config.TransferTaskTest do
test "don't restart if no reboot time settings were changed" do
clear_config(:emoji)
-
- ConfigDB.create(%{
- group: ":pleroma",
- key: ":emoji",
- value: [groups: [a: 1, b: 2]]
- })
+ insert(:config, key: :emoji, value: [groups: [a: 1, b: 2]])
refute String.contains?(
capture_log(fn -> TransferTask.start_link([]) end),
@@ -137,25 +94,13 @@ defmodule Pleroma.Config.TransferTaskTest do
test "on reboot time key" do
clear_config(:chat)
-
- ConfigDB.create(%{
- group: ":pleroma",
- key: ":chat",
- value: [enabled: false]
- })
-
+ insert(:config, key: :chat, value: [enabled: false])
assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted"
end
test "on reboot time subkey" do
clear_config(Pleroma.Captcha)
-
- ConfigDB.create(%{
- group: ":pleroma",
- key: "Pleroma.Captcha",
- value: [seconds_valid: 60]
- })
-
+ insert(:config, key: Pleroma.Captcha, value: [seconds_valid: 60])
assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted"
end
@@ -163,17 +108,8 @@ defmodule Pleroma.Config.TransferTaskTest do
clear_config(:chat)
clear_config(Pleroma.Captcha)
- ConfigDB.create(%{
- group: ":pleroma",
- key: ":chat",
- value: [enabled: false]
- })
-
- ConfigDB.create(%{
- group: ":pleroma",
- key: "Pleroma.Captcha",
- value: [seconds_valid: 60]
- })
+ insert(:config, key: :chat, value: [enabled: false])
+ insert(:config, key: Pleroma.Captcha, value: [seconds_valid: 60])
refute String.contains?(
capture_log(fn -> TransferTask.load_and_update_env([], false) end),
diff --git a/test/support/factory.ex b/test/support/factory.ex
index 6e3676aca..6e22b66a4 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -42,7 +42,8 @@ defmodule Pleroma.Factory do
user
| ap_id: User.ap_id(user),
follower_address: User.ap_followers(user),
- following_address: User.ap_following(user)
+ following_address: User.ap_following(user),
+ raw_bio: user.bio
}
end
@@ -396,24 +397,17 @@ defmodule Pleroma.Factory do
}
end
- def config_factory do
+ def config_factory(attrs \\ %{}) do
%Pleroma.ConfigDB{
- key:
- sequence(:key, fn key ->
- # Atom dynamic registration hack in tests
- "some_key_#{key}"
- |> String.to_atom()
- |> inspect()
- end),
- group: ":pleroma",
+ key: sequence(:key, &String.to_atom("some_key_#{&1}")),
+ group: :pleroma,
value:
sequence(
:value,
- fn key ->
- :erlang.term_to_binary(%{another_key: "#{key}somevalue", another: "#{key}somevalue"})
- end
+ &%{another_key: "#{&1}somevalue", another: "#{&1}somevalue"}
)
}
+ |> merge_attributes(attrs)
end
def marker_factory do
diff --git a/test/tasks/config_test.exs b/test/tasks/config_test.exs
index 04bc947a9..e1bddfebf 100644
--- a/test/tasks/config_test.exs
+++ b/test/tasks/config_test.exs
@@ -5,6 +5,8 @@
defmodule Mix.Tasks.Pleroma.ConfigTest do
use Pleroma.DataCase
+ import Pleroma.Factory
+
alias Pleroma.ConfigDB
alias Pleroma.Repo
@@ -49,24 +51,19 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do
refute ConfigDB.get_by_params(%{group: ":pleroma", key: "Pleroma.Repo"})
refute ConfigDB.get_by_params(%{group: ":postgrex", key: ":json_library"})
- assert ConfigDB.from_binary(config1.value) == [key: "value", key2: [Repo]]
- assert ConfigDB.from_binary(config2.value) == [key: "value2", key2: ["Activity"]]
- assert ConfigDB.from_binary(config3.value) == :info
+ assert config1.value == [key: "value", key2: [Repo]]
+ assert config2.value == [key: "value2", key2: ["Activity"]]
+ assert config3.value == :info
end
test "config table is truncated before migration" do
- ConfigDB.create(%{
- group: ":pleroma",
- key: ":first_setting",
- value: [key: "value", key2: ["Activity"]]
- })
-
+ insert(:config, key: :first_setting, value: [key: "value", key2: ["Activity"]])
assert Repo.aggregate(ConfigDB, :count, :id) == 1
Mix.Tasks.Pleroma.Config.migrate_to_db("test/fixtures/config/temp.secret.exs")
config = ConfigDB.get_by_params(%{group: ":pleroma", key: ":first_setting"})
- assert ConfigDB.from_binary(config.value) == [key: "value", key2: [Repo]]
+ assert config.value == [key: "value", key2: [Repo]]
end
end
@@ -82,19 +79,9 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do
end
test "settings are migrated to file and deleted from db", %{temp_file: temp_file} do
- ConfigDB.create(%{
- group: ":pleroma",
- key: ":setting_first",
- value: [key: "value", key2: ["Activity"]]
- })
-
- ConfigDB.create(%{
- group: ":pleroma",
- key: ":setting_second",
- value: [key: "value2", key2: [Repo]]
- })
-
- ConfigDB.create(%{group: ":quack", key: ":level", value: :info})
+ insert(:config, key: :setting_first, value: [key: "value", key2: ["Activity"]])
+ insert(:config, key: :setting_second, value: [key: "value2", key2: [Repo]])
+ insert(:config, group: :quack, key: :level, value: :info)
Mix.Tasks.Pleroma.Config.run(["migrate_from_db", "--env", "temp", "-d"])
@@ -107,9 +94,8 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do
end
test "load a settings with large values and pass to file", %{temp_file: temp_file} do
- ConfigDB.create(%{
- group: ":pleroma",
- key: ":instance",
+ insert(:config,
+ key: :instance,
value: [
name: "Pleroma",
email: "example@example.com",
@@ -163,7 +149,6 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do
extended_nickname_format: true,
multi_factor_authentication: [
totp: [
- # digits 6 or 8
digits: 6,
period: 30
],
@@ -173,7 +158,7 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do
]
]
]
- })
+ )
Mix.Tasks.Pleroma.Config.run(["migrate_from_db", "--env", "temp", "-d"])
diff --git a/test/upload/filter/mogrify_test.exs b/test/upload/filter/mogrify_test.exs
index b6a463e8c..62ca30487 100644
--- a/test/upload/filter/mogrify_test.exs
+++ b/test/upload/filter/mogrify_test.exs
@@ -6,21 +6,17 @@ defmodule Pleroma.Upload.Filter.MogrifyTest do
use Pleroma.DataCase
import Mock
- alias Pleroma.Config
- alias Pleroma.Upload
alias Pleroma.Upload.Filter
- setup do: clear_config([Filter.Mogrify, :args])
-
test "apply mogrify filter" do
- Config.put([Filter.Mogrify, :args], [{"tint", "40"}])
+ clear_config(Filter.Mogrify, args: [{"tint", "40"}])
File.cp!(
"test/fixtures/image.jpg",
"test/fixtures/image_tmp.jpg"
)
- upload = %Upload{
+ upload = %Pleroma.Upload{
name: "an… image.jpg",
content_type: "image/jpg",
path: Path.absname("test/fixtures/image_tmp.jpg"),
diff --git a/test/web/activity_pub/object_validators/types/date_time_test.exs b/test/web/activity_pub/object_validators/types/date_time_test.exs
index 3e17a9497..43be8e936 100644
--- a/test/web/activity_pub/object_validators/types/date_time_test.exs
+++ b/test/web/activity_pub/object_validators/types/date_time_test.exs
@@ -1,5 +1,5 @@
defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.DateTimeTest do
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types.DateTime
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime
use Pleroma.DataCase
test "it validates an xsd:Datetime" do
diff --git a/test/web/activity_pub/object_validators/types/object_id_test.exs b/test/web/activity_pub/object_validators/types/object_id_test.exs
index c8911948e..e0ab76379 100644
--- a/test/web/activity_pub/object_validators/types/object_id_test.exs
+++ b/test/web/activity_pub/object_validators/types/object_id_test.exs
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ObjectValidators.Types.ObjectIDTest do
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID
use Pleroma.DataCase
@uris [
diff --git a/test/web/activity_pub/object_validators/types/recipients_test.exs b/test/web/activity_pub/object_validators/types/recipients_test.exs
index f278f039b..053916bdd 100644
--- a/test/web/activity_pub/object_validators/types/recipients_test.exs
+++ b/test/web/activity_pub/object_validators/types/recipients_test.exs
@@ -1,5 +1,5 @@
defmodule Pleroma.Web.ObjectValidators.Types.RecipientsTest do
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types.Recipients
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients
use Pleroma.DataCase
test "it asserts that all elements of the list are object ids" do
diff --git a/test/web/activity_pub/object_validators/types/safe_text_test.exs b/test/web/activity_pub/object_validators/types/safe_text_test.exs
index d4a574554..9c08606f6 100644
--- a/test/web/activity_pub/object_validators/types/safe_text_test.exs
+++ b/test/web/activity_pub/object_validators/types/safe_text_test.exs
@@ -5,7 +5,7 @@
defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.SafeTextTest do
use Pleroma.DataCase
- alias Pleroma.Web.ActivityPub.ObjectValidators.Types.SafeText
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators.SafeText
test "it lets normal text go through" do
text = "hey how are you"
diff --git a/test/web/admin_api/controllers/config_controller_test.exs b/test/web/admin_api/controllers/config_controller_test.exs
index 780de8d18..064ef9bc7 100644
--- a/test/web/admin_api/controllers/config_controller_test.exs
+++ b/test/web/admin_api/controllers/config_controller_test.exs
@@ -57,12 +57,12 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
]
} = json_response_and_validate_schema(conn, 200)
- assert key1 == config1.key
- assert key2 == config2.key
+ assert key1 == inspect(config1.key)
+ assert key2 == inspect(config2.key)
end
test "db is added to settings that are in db", %{conn: conn} do
- _config = insert(:config, key: ":instance", value: ConfigDB.to_binary(name: "Some name"))
+ _config = insert(:config, key: ":instance", value: [name: "Some name"])
%{"configs" => configs} =
conn
@@ -83,7 +83,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
config3 =
insert(:config,
- value: ConfigDB.to_binary(k1: :v1, k2: :v2)
+ value: [k1: :v1, k2: :v2]
)
%{"configs" => configs} =
@@ -93,42 +93,45 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
assert length(configs) > 3
+ saved_configs = [config1, config2, config3]
+ keys = Enum.map(saved_configs, &inspect(&1.key))
+
received_configs =
Enum.filter(configs, fn %{"group" => group, "key" => key} ->
- group == ":pleroma" and key in [config1.key, config2.key, config3.key]
+ group == ":pleroma" and key in keys
end)
assert length(received_configs) == 3
db_keys =
config3.value
- |> ConfigDB.from_binary()
|> Keyword.keys()
- |> ConfigDB.convert()
+ |> ConfigDB.to_json_types()
+
+ keys = Enum.map(saved_configs -- [config3], &inspect(&1.key))
+
+ values = Enum.map(saved_configs, &ConfigDB.to_json_types(&1.value))
+
+ mapset_keys = MapSet.new(keys ++ db_keys)
Enum.each(received_configs, fn %{"value" => value, "db" => db} ->
- assert db in [[config1.key], [config2.key], db_keys]
+ db = MapSet.new(db)
+ assert MapSet.subset?(db, mapset_keys)
- assert value in [
- ConfigDB.from_binary_with_convert(config1.value),
- ConfigDB.from_binary_with_convert(config2.value),
- ConfigDB.from_binary_with_convert(config3.value)
- ]
+ assert value in values
end)
end
test "subkeys with full update right merge", %{conn: conn} do
- config1 =
- insert(:config,
- key: ":emoji",
- value: ConfigDB.to_binary(groups: [a: 1, b: 2], key: [a: 1])
- )
+ insert(:config,
+ key: ":emoji",
+ value: [groups: [a: 1, b: 2], key: [a: 1]]
+ )
- config2 =
- insert(:config,
- key: ":assets",
- value: ConfigDB.to_binary(mascots: [a: 1, b: 2], key: [a: 1])
- )
+ insert(:config,
+ key: ":assets",
+ value: [mascots: [a: 1, b: 2], key: [a: 1]]
+ )
%{"configs" => configs} =
conn
@@ -137,14 +140,14 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
vals =
Enum.filter(configs, fn %{"group" => group, "key" => key} ->
- group == ":pleroma" and key in [config1.key, config2.key]
+ group == ":pleroma" and key in [":emoji", ":assets"]
end)
emoji = Enum.find(vals, fn %{"key" => key} -> key == ":emoji" end)
assets = Enum.find(vals, fn %{"key" => key} -> key == ":assets" end)
- emoji_val = ConfigDB.transform_with_out_binary(emoji["value"])
- assets_val = ConfigDB.transform_with_out_binary(assets["value"])
+ emoji_val = ConfigDB.to_elixir_types(emoji["value"])
+ assets_val = ConfigDB.to_elixir_types(assets["value"])
assert emoji_val[:groups] == [a: 1, b: 2]
assert assets_val[:mascots] == [a: 1, b: 2]
@@ -277,7 +280,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
"value" => %{"tuple" => ["string", "Pleroma.Captcha.NotReal", []]},
"db" => [":key5"]
}
- ]
+ ],
+ "need_reboot" => false
}
assert Application.get_env(:pleroma, :key1) == "value1"
@@ -357,7 +361,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
"value" => "https://hooks.slack.com/services/KEY",
"db" => [":webhook_url"]
}
- ]
+ ],
+ "need_reboot" => false
}
assert Application.get_env(:quack, :level) == :info
@@ -366,14 +371,14 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
end
test "saving config with partial update", %{conn: conn} do
- config = insert(:config, key: ":key1", value: :erlang.term_to_binary(key1: 1, key2: 2))
+ insert(:config, key: ":key1", value: :erlang.term_to_binary(key1: 1, key2: 2))
conn =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/pleroma/admin/config", %{
configs: [
- %{group: config.group, key: config.key, value: [%{"tuple" => [":key3", 3]}]}
+ %{group: ":pleroma", key: ":key1", value: [%{"tuple" => [":key3", 3]}]}
]
})
@@ -389,7 +394,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
],
"db" => [":key1", ":key2", ":key3"]
}
- ]
+ ],
+ "need_reboot" => false
}
end
@@ -500,8 +506,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
end
test "saving config with nested merge", %{conn: conn} do
- config =
- insert(:config, key: ":key1", value: :erlang.term_to_binary(key1: 1, key2: [k1: 1, k2: 2]))
+ insert(:config, key: :key1, value: [key1: 1, key2: [k1: 1, k2: 2]])
conn =
conn
@@ -509,8 +514,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
|> post("/api/pleroma/admin/config", %{
configs: [
%{
- group: config.group,
- key: config.key,
+ group: ":pleroma",
+ key: ":key1",
value: [
%{"tuple" => [":key3", 3]},
%{
@@ -548,7 +553,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
],
"db" => [":key1", ":key3", ":key2"]
}
- ]
+ ],
+ "need_reboot" => false
}
end
@@ -588,7 +594,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
],
"db" => [":ssl_options"]
}
- ]
+ ],
+ "need_reboot" => false
}
assert Application.get_env(:pleroma, :key1) == [
@@ -600,12 +607,11 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
backends = Application.get_env(:logger, :backends)
on_exit(fn -> Application.put_env(:logger, :backends, backends) end)
- config =
- insert(:config,
- group: ":logger",
- key: ":backends",
- value: :erlang.term_to_binary([])
- )
+ insert(:config,
+ group: :logger,
+ key: :backends,
+ value: []
+ )
Pleroma.Config.TransferTask.load_and_update_env([], false)
@@ -617,8 +623,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
|> post("/api/pleroma/admin/config", %{
configs: [
%{
- group: config.group,
- key: config.key,
+ group: ":logger",
+ key: ":backends",
value: [":console"]
}
]
@@ -634,7 +640,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
],
"db" => [":backends"]
}
- ]
+ ],
+ "need_reboot" => false
}
assert Application.get_env(:logger, :backends) == [
@@ -643,19 +650,18 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
end
test "saving full setting if value is not keyword", %{conn: conn} do
- config =
- insert(:config,
- group: ":tesla",
- key: ":adapter",
- value: :erlang.term_to_binary(Tesla.Adapter.Hackey)
- )
+ insert(:config,
+ group: :tesla,
+ key: :adapter,
+ value: Tesla.Adapter.Hackey
+ )
conn =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/pleroma/admin/config", %{
configs: [
- %{group: config.group, key: config.key, value: "Tesla.Adapter.Httpc"}
+ %{group: ":tesla", key: ":adapter", value: "Tesla.Adapter.Httpc"}
]
})
@@ -667,7 +673,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
"value" => "Tesla.Adapter.Httpc",
"db" => [":adapter"]
}
- ]
+ ],
+ "need_reboot" => false
}
end
@@ -677,13 +684,13 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
token: token
} do
ueberauth = Application.get_env(:ueberauth, Ueberauth)
- config1 = insert(:config, key: ":keyaa1")
- config2 = insert(:config, key: ":keyaa2")
+ insert(:config, key: :keyaa1)
+ insert(:config, key: :keyaa2)
config3 =
insert(:config,
- group: ":ueberauth",
- key: "Ueberauth"
+ group: :ueberauth,
+ key: Ueberauth
)
conn =
@@ -691,8 +698,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
|> put_req_header("content-type", "application/json")
|> post("/api/pleroma/admin/config", %{
configs: [
- %{group: config1.group, key: config1.key, value: "another_value"},
- %{group: config2.group, key: config2.key, value: "another_value"}
+ %{group: ":pleroma", key: ":keyaa1", value: "another_value"},
+ %{group: ":pleroma", key: ":keyaa2", value: "another_value"}
]
})
@@ -700,22 +707,23 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
"configs" => [
%{
"group" => ":pleroma",
- "key" => config1.key,
+ "key" => ":keyaa1",
"value" => "another_value",
"db" => [":keyaa1"]
},
%{
"group" => ":pleroma",
- "key" => config2.key,
+ "key" => ":keyaa2",
"value" => "another_value",
"db" => [":keyaa2"]
}
- ]
+ ],
+ "need_reboot" => false
}
assert Application.get_env(:pleroma, :keyaa1) == "another_value"
assert Application.get_env(:pleroma, :keyaa2) == "another_value"
- assert Application.get_env(:ueberauth, Ueberauth) == ConfigDB.from_binary(config3.value)
+ assert Application.get_env(:ueberauth, Ueberauth) == config3.value
conn =
build_conn()
@@ -724,7 +732,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
|> put_req_header("content-type", "application/json")
|> post("/api/pleroma/admin/config", %{
configs: [
- %{group: config2.group, key: config2.key, delete: true},
+ %{group: ":pleroma", key: ":keyaa2", delete: true},
%{
group: ":ueberauth",
key: "Ueberauth",
@@ -734,7 +742,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
})
assert json_response_and_validate_schema(conn, 200) == %{
- "configs" => []
+ "configs" => [],
+ "need_reboot" => false
}
assert Application.get_env(:ueberauth, Ueberauth) == ueberauth
@@ -801,7 +810,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
":name"
]
}
- ]
+ ],
+ "need_reboot" => false
}
end
@@ -935,7 +945,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
],
"db" => [":http"]
}
- ]
+ ],
+ "need_reboot" => false
}
end
@@ -1000,7 +1011,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
],
"db" => [":key2", ":key3"]
}
- ]
+ ],
+ "need_reboot" => false
}
end
@@ -1027,7 +1039,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
"value" => %{"key" => "some_val"},
"db" => [":key1"]
}
- ]
+ ],
+ "need_reboot" => false
}
end
@@ -1077,16 +1090,16 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
":background"
]
}
- ]
+ ],
+ "need_reboot" => false
}
end
test "delete part of settings by atom subkeys", %{conn: conn} do
- config =
- insert(:config,
- key: ":keyaa1",
- value: :erlang.term_to_binary(subkey1: "val1", subkey2: "val2", subkey3: "val3")
- )
+ insert(:config,
+ key: :keyaa1,
+ value: [subkey1: "val1", subkey2: "val2", subkey3: "val3"]
+ )
conn =
conn
@@ -1094,8 +1107,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
|> post("/api/pleroma/admin/config", %{
configs: [
%{
- group: config.group,
- key: config.key,
+ group: ":pleroma",
+ key: ":keyaa1",
subkeys: [":subkey1", ":subkey3"],
delete: true
}
@@ -1110,7 +1123,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
"value" => [%{"tuple" => [":subkey2", "val2"]}],
"db" => [":subkey2"]
}
- ]
+ ],
+ "need_reboot" => false
}
end
@@ -1236,6 +1250,90 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
assert Application.get_env(:pleroma, Pleroma.Captcha.NotReal) == "value5"
assert Application.get_env(:not_real, :anything) == "value6"
end
+
+ test "args for Pleroma.Upload.Filter.Mogrify with custom tuples", %{conn: conn} do
+ clear_config(Pleroma.Upload.Filter.Mogrify)
+
+ assert conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/pleroma/admin/config", %{
+ configs: [
+ %{
+ group: ":pleroma",
+ key: "Pleroma.Upload.Filter.Mogrify",
+ value: [
+ %{"tuple" => [":args", ["auto-orient", "strip"]]}
+ ]
+ }
+ ]
+ })
+ |> json_response_and_validate_schema(200) == %{
+ "configs" => [
+ %{
+ "group" => ":pleroma",
+ "key" => "Pleroma.Upload.Filter.Mogrify",
+ "value" => [
+ %{"tuple" => [":args", ["auto-orient", "strip"]]}
+ ],
+ "db" => [":args"]
+ }
+ ],
+ "need_reboot" => false
+ }
+
+ assert Config.get(Pleroma.Upload.Filter.Mogrify) == [args: ["auto-orient", "strip"]]
+
+ assert conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/pleroma/admin/config", %{
+ configs: [
+ %{
+ group: ":pleroma",
+ key: "Pleroma.Upload.Filter.Mogrify",
+ value: [
+ %{
+ "tuple" => [
+ ":args",
+ [
+ "auto-orient",
+ "strip",
+ "{\"implode\", \"1\"}",
+ "{\"resize\", \"3840x1080>\"}"
+ ]
+ ]
+ }
+ ]
+ }
+ ]
+ })
+ |> json_response(200) == %{
+ "configs" => [
+ %{
+ "group" => ":pleroma",
+ "key" => "Pleroma.Upload.Filter.Mogrify",
+ "value" => [
+ %{
+ "tuple" => [
+ ":args",
+ [
+ "auto-orient",
+ "strip",
+ "{\"implode\", \"1\"}",
+ "{\"resize\", \"3840x1080>\"}"
+ ]
+ ]
+ }
+ ],
+ "db" => [":args"]
+ }
+ ],
+ "need_reboot" => false
+ }
+
+ assert Config.get(Pleroma.Upload.Filter.Mogrify) == [
+ args: ["auto-orient", "strip", {"implode", "1"}, {"resize", "3840x1080>"}]
+ ]
+ end
end
describe "GET /api/pleroma/admin/config/descriptions" do
diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
new file mode 100644
index 000000000..5ab6cb78a
--- /dev/null
+++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
@@ -0,0 +1,145 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
+ use Pleroma.Web.ConnCase
+
+ import Pleroma.Factory
+ import Mock
+
+ alias Pleroma.Web.MediaProxy
+
+ setup do: clear_config([:media_proxy])
+
+ setup do
+ on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
+ end
+
+ setup do
+ admin = insert(:user, is_admin: true)
+ token = insert(:oauth_admin_token, user: admin)
+
+ conn =
+ build_conn()
+ |> assign(:user, admin)
+ |> assign(:token, token)
+
+ Config.put([:media_proxy, :enabled], true)
+ Config.put([:media_proxy, :invalidation, :enabled], true)
+ Config.put([:media_proxy, :invalidation, :provider], MediaProxy.Invalidation.Script)
+
+ {:ok, %{admin: admin, token: token, conn: conn}}
+ end
+
+ describe "GET /api/pleroma/admin/media_proxy_caches" do
+ test "shows banned MediaProxy URLs", %{conn: conn} do
+ MediaProxy.put_in_banned_urls([
+ "http://localhost:4001/media/a688346.jpg",
+ "http://localhost:4001/media/fb1f4d.jpg"
+ ])
+
+ MediaProxy.put_in_banned_urls("http://localhost:4001/media/gb1f44.jpg")
+ MediaProxy.put_in_banned_urls("http://localhost:4001/media/tb13f47.jpg")
+ MediaProxy.put_in_banned_urls("http://localhost:4001/media/wb1f46.jpg")
+
+ response =
+ conn
+ |> get("/api/pleroma/admin/media_proxy_caches?page_size=2")
+ |> json_response_and_validate_schema(200)
+
+ assert response["urls"] == [
+ "http://localhost:4001/media/fb1f4d.jpg",
+ "http://localhost:4001/media/a688346.jpg"
+ ]
+
+ response =
+ conn
+ |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=2")
+ |> json_response_and_validate_schema(200)
+
+ assert response["urls"] == [
+ "http://localhost:4001/media/gb1f44.jpg",
+ "http://localhost:4001/media/tb13f47.jpg"
+ ]
+
+ response =
+ conn
+ |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3")
+ |> json_response_and_validate_schema(200)
+
+ assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"]
+ end
+ end
+
+ describe "POST /api/pleroma/admin/media_proxy_caches/delete" do
+ test "deleted MediaProxy URLs from banned", %{conn: conn} do
+ MediaProxy.put_in_banned_urls([
+ "http://localhost:4001/media/a688346.jpg",
+ "http://localhost:4001/media/fb1f4d.jpg"
+ ])
+
+ response =
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/pleroma/admin/media_proxy_caches/delete", %{
+ urls: ["http://localhost:4001/media/a688346.jpg"]
+ })
+ |> json_response_and_validate_schema(200)
+
+ assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"]
+ refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg")
+ assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg")
+ end
+ end
+
+ describe "POST /api/pleroma/admin/media_proxy_caches/purge" do
+ test "perform invalidates cache of MediaProxy", %{conn: conn} do
+ urls = [
+ "http://example.com/media/a688346.jpg",
+ "http://example.com/media/fb1f4d.jpg"
+ ]
+
+ with_mocks [
+ {MediaProxy.Invalidation.Script, [],
+ [
+ purge: fn _, _ -> {"ok", 0} end
+ ]}
+ ] do
+ response =
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
+ |> json_response_and_validate_schema(200)
+
+ assert response["urls"] == urls
+
+ refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
+ refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
+ end
+ end
+
+ test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: conn} do
+ urls = [
+ "http://example.com/media/a688346.jpg",
+ "http://example.com/media/fb1f4d.jpg"
+ ]
+
+ with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do
+ response =
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/pleroma/admin/media_proxy_caches/purge", %{
+ urls: urls,
+ ban: true
+ })
+ |> json_response_and_validate_schema(200)
+
+ assert response["urls"] == urls
+
+ assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
+ assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
+ end
+ end
+ end
+end
diff --git a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
index 7c420985d..76e6d603a 100644
--- a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
+++ b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
@@ -83,10 +83,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do
test "updates the user's bio", %{conn: conn} do
user2 = insert(:user)
- conn =
- patch(conn, "/api/v1/accounts/update_credentials", %{
- "note" => "I drink #cofe with @#{user2.nickname}\n\nsuya.."
- })
+ raw_bio = "I drink #cofe with @#{user2.nickname}\n\nsuya.."
+
+ conn = patch(conn, "/api/v1/accounts/update_credentials", %{"note" => raw_bio})
assert user_data = json_response_and_validate_schema(conn, 200)
@@ -94,6 +93,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do
~s(I drink <a class="hashtag" data-tag="cofe" href="http://localhost:4001/tag/cofe">#cofe</a> with <span class="h-card"><a class="u-url mention" data-user="#{
user2.id
}" href="#{user2.ap_id}" rel="ugc">@<span>#{user2.nickname}</span></a></span><br/><br/>suya..)
+
+ assert user_data["source"]["note"] == raw_bio
+
+ user = Repo.get(User, user_data["id"])
+
+ assert user.raw_bio == raw_bio
end
test "updates the user's locking status", %{conn: conn} do
diff --git a/test/web/mastodon_api/controllers/search_controller_test.exs b/test/web/mastodon_api/controllers/search_controller_test.exs
index c605957b1..826f37fbc 100644
--- a/test/web/mastodon_api/controllers/search_controller_test.exs
+++ b/test/web/mastodon_api/controllers/search_controller_test.exs
@@ -151,6 +151,22 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do
]
end
+ test "supports pagination of hashtags search results", %{conn: conn} do
+ results =
+ conn
+ |> get(
+ "/api/v2/search?#{
+ URI.encode_query(%{q: "#some #text #with #hashtags", limit: 2, offset: 1})
+ }"
+ )
+ |> json_response_and_validate_schema(200)
+
+ assert results["hashtags"] == [
+ %{"name" => "text", "url" => "#{Web.base_url()}/tag/text"},
+ %{"name" => "with", "url" => "#{Web.base_url()}/tag/with"}
+ ]
+ end
+
test "excludes a blocked users from search results", %{conn: conn} do
user = insert(:user)
user_smith = insert(:user, %{nickname: "Agent", name: "I love 2hu"})
diff --git a/test/web/mastodon_api/controllers/status_controller_test.exs b/test/web/mastodon_api/controllers/status_controller_test.exs
index 648e6f2ce..a98e939e8 100644
--- a/test/web/mastodon_api/controllers/status_controller_test.exs
+++ b/test/web/mastodon_api/controllers/status_controller_test.exs
@@ -1561,7 +1561,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
# Using the header for pagination works correctly
[next, _] = get_resp_header(result, "link") |> hd() |> String.split(", ")
- [_, max_id] = Regex.run(~r/max_id=(.*)>;/, next)
+ [_, max_id] = Regex.run(~r/max_id=([^&]+)/, next)
assert max_id == third_favorite.id
diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs
index 044f088a4..80b1f734c 100644
--- a/test/web/mastodon_api/views/account_view_test.exs
+++ b/test/web/mastodon_api/views/account_view_test.exs
@@ -33,7 +33,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
bio:
"<script src=\"invalid-html\"></script><span>valid html</span>. a<br>b<br/>c<br >d<br />f '&<>\"",
inserted_at: ~N[2017-08-15 15:47:06.597036],
- emoji: %{"karjalanpiirakka" => "/file.png"}
+ emoji: %{"karjalanpiirakka" => "/file.png"},
+ raw_bio: "valid html. a\nb\nc\nd\nf '&<>\""
})
expected = %{
diff --git a/test/web/mastodon_api/views/conversation_view_test.exs b/test/web/mastodon_api/views/conversation_view_test.exs
index 6f84366f8..2e8203c9b 100644
--- a/test/web/mastodon_api/views/conversation_view_test.exs
+++ b/test/web/mastodon_api/views/conversation_view_test.exs
@@ -15,8 +15,17 @@ defmodule Pleroma.Web.MastodonAPI.ConversationViewTest do
user = insert(:user)
other_user = insert(:user)
+ {:ok, parent} = CommonAPI.post(user, %{status: "parent"})
+
{:ok, activity} =
- CommonAPI.post(user, %{status: "hey @#{other_user.nickname}", visibility: "direct"})
+ CommonAPI.post(user, %{
+ status: "hey @#{other_user.nickname}",
+ visibility: "direct",
+ in_reply_to_id: parent.id
+ })
+
+ {:ok, _reply_activity} =
+ CommonAPI.post(user, %{status: "hu", visibility: "public", in_reply_to_id: parent.id})
[participation] = Participation.for_user_with_last_activity_id(user)
diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs
new file mode 100644
index 000000000..926ae74ca
--- /dev/null
+++ b/test/web/media_proxy/invalidation_test.exs
@@ -0,0 +1,64 @@
+defmodule Pleroma.Web.MediaProxy.InvalidationTest do
+ use ExUnit.Case
+ use Pleroma.Tests.Helpers
+
+ alias Pleroma.Config
+ alias Pleroma.Web.MediaProxy.Invalidation
+
+ import ExUnit.CaptureLog
+ import Mock
+ import Tesla.Mock
+
+ setup do: clear_config([:media_proxy])
+
+ setup do
+ on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
+ end
+
+ describe "Invalidation.Http" do
+ test "perform request to clear cache" do
+ Config.put([:media_proxy, :enabled], false)
+ Config.put([:media_proxy, :invalidation, :enabled], true)
+ Config.put([:media_proxy, :invalidation, :provider], Invalidation.Http)
+
+ Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}])
+ image_url = "http://example.com/media/example.jpg"
+ Pleroma.Web.MediaProxy.put_in_banned_urls(image_url)
+
+ mock(fn
+ %{
+ method: :purge,
+ url: "http://example.com/media/example.jpg",
+ headers: [{"x-refresh", 1}]
+ } ->
+ %Tesla.Env{status: 200}
+ end)
+
+ assert capture_log(fn ->
+ assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
+ assert Invalidation.purge([image_url]) == {:ok, [image_url]}
+ assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
+ end) =~ "Running cache purge: [\"#{image_url}\"]"
+ end
+ end
+
+ describe "Invalidation.Script" do
+ test "run script to clear cache" do
+ Config.put([:media_proxy, :enabled], false)
+ Config.put([:media_proxy, :invalidation, :enabled], true)
+ Config.put([:media_proxy, :invalidation, :provider], Invalidation.Script)
+ Config.put([Invalidation.Script], script_path: "purge-nginx")
+
+ image_url = "http://example.com/media/example.jpg"
+ Pleroma.Web.MediaProxy.put_in_banned_urls(image_url)
+
+ with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do
+ assert capture_log(fn ->
+ assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
+ assert Invalidation.purge([image_url]) == {:ok, [image_url]}
+ assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
+ end) =~ "Running cache purge: [\"#{image_url}\"]"
+ end
+ end
+ end
+end
diff --git a/test/web/media_proxy/invalidations/http_test.exs b/test/web/media_proxy/invalidations/http_test.exs
index 8a3b4141c..a1bef5237 100644
--- a/test/web/media_proxy/invalidations/http_test.exs
+++ b/test/web/media_proxy/invalidations/http_test.exs
@@ -5,6 +5,10 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
import ExUnit.CaptureLog
import Tesla.Mock
+ setup do
+ on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
+ end
+
test "logs hasn't error message when request is valid" do
mock(fn
%{method: :purge, url: "http://example.com/media/example.jpg"} ->
@@ -14,8 +18,8 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
refute capture_log(fn ->
assert Invalidation.Http.purge(
["http://example.com/media/example.jpg"],
- %{}
- ) == {:ok, "success"}
+ []
+ ) == {:ok, ["http://example.com/media/example.jpg"]}
end) =~ "Error while cache purge"
end
@@ -28,8 +32,8 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
assert capture_log(fn ->
assert Invalidation.Http.purge(
["http://example.com/media/example1.jpg"],
- %{}
- ) == {:ok, "success"}
+ []
+ ) == {:ok, ["http://example.com/media/example1.jpg"]}
end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg"
end
end
diff --git a/test/web/media_proxy/invalidations/script_test.exs b/test/web/media_proxy/invalidations/script_test.exs
index 1358963ab..51833ab18 100644
--- a/test/web/media_proxy/invalidations/script_test.exs
+++ b/test/web/media_proxy/invalidations/script_test.exs
@@ -4,17 +4,23 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do
import ExUnit.CaptureLog
+ setup do
+ on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
+ end
+
test "it logger error when script not found" do
assert capture_log(fn ->
assert Invalidation.Script.purge(
["http://example.com/media/example.jpg"],
- %{script_path: "./example"}
- ) == {:error, "\"%ErlangError{original: :enoent}\""}
- end) =~ "Error while cache purge: \"%ErlangError{original: :enoent}\""
+ script_path: "./example"
+ ) == {:error, "%ErlangError{original: :enoent}"}
+ end) =~ "Error while cache purge: %ErlangError{original: :enoent}"
- assert Invalidation.Script.purge(
- ["http://example.com/media/example.jpg"],
- %{}
- ) == {:error, "not found script path"}
+ capture_log(fn ->
+ assert Invalidation.Script.purge(
+ ["http://example.com/media/example.jpg"],
+ []
+ ) == {:error, "\"not found script path\""}
+ end)
end
end
diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs
index da79d38a5..d61cef83b 100644
--- a/test/web/media_proxy/media_proxy_controller_test.exs
+++ b/test/web/media_proxy/media_proxy_controller_test.exs
@@ -10,6 +10,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
setup do: clear_config(:media_proxy)
setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base])
+ setup do
+ on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
+ end
+
test "it returns 404 when MediaProxy disabled", %{conn: conn} do
Config.put([:media_proxy, :enabled], false)
@@ -66,4 +70,16 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
assert %Plug.Conn{status: :success} = get(conn, url)
end
end
+
+ test "it returns 404 when url contains in banned_urls cache", %{conn: conn} do
+ Config.put([:media_proxy, :enabled], true)
+ Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
+ url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png")
+ Pleroma.Web.MediaProxy.put_in_banned_urls("https://google.fn/test.png")
+
+ with_mock Pleroma.ReverseProxy,
+ call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do
+ assert %Plug.Conn{status: 404, resp_body: "Not Found"} = get(conn, url)
+ end
+ end
end