diff options
| -rw-r--r-- | CHANGELOG.md | 6 | ||||
| -rw-r--r-- | config/config.exs | 6 | ||||
| -rw-r--r-- | config/test.exs | 6 | ||||
| -rw-r--r-- | docs/config.md | 3 | ||||
| -rw-r--r-- | lib/pleroma/emoji.ex | 6 | ||||
| -rw-r--r-- | lib/pleroma/web/rich_media/helpers.ex | 42 | ||||
| -rw-r--r-- | test/support/http_request_mock.ex | 16 | ||||
| -rw-r--r-- | test/web/mastodon_api/mastodon_api_controller_test.exs | 9 | ||||
| -rw-r--r-- | test/web/rich_media/helpers_test.exs | 47 |
9 files changed, 109 insertions, 32 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index d3c554245..be0b74a16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Security - Mastodon API: Fix display names not being sanitized +- Rich media: Do not crawl private IP ranges + ### Added - Add a generic settings store for frontends / clients to use. - Explicit addressing option for posting. @@ -26,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Federation: Support for `Question` and `Answer` objects - Federation: Support for reports - Configuration: `poll_limits` option +- Configuration: `pack_extensions` option - Configuration: `safe_dm_mentions` option - Configuration: `link_name` option - Configuration: `fetch_initial_posts` option @@ -63,6 +66,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: `skip_thread_containment` option - Configuration: `rate_limit` option. See `Pleroma.Plugs.RateLimiter` documentation for details. - MRF: Support for filtering out likely spam messages by rejecting posts from new users that contain links. +- Configuration: `ignore_hosts` option +- Configuration: `ignore_tld` option ### Changed - **Breaking:** bind to 127.0.0.1 instead of 0.0.0.0 by default @@ -102,6 +107,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Posts which are marked sensitive or tagged nsfw no longer have link previews. - HTTP connection timeout is now set to 10 seconds. - Respond with a 404 Not implemented JSON error message when requested API is not implemented +- Rich Media: crawl only https URLs. ### Fixed - Follow requests don't get 'stuck' anymore. diff --git a/config/config.exs b/config/config.exs index 0d07fc692..d381a8dc2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -99,6 +99,7 @@ config :pleroma, Pleroma.Uploaders.MDII, config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"], + pack_extensions: [".png", ".gif"], groups: [ # Put groups that have higher priority than defaults here. Example in `docs/config/custom_emoji.md` Custom: ["/emoji/*.png", "/emoji/**/*.png"] @@ -330,7 +331,10 @@ config :pleroma, :mrf_keyword, config :pleroma, :mrf_subchain, match_actor: %{} -config :pleroma, :rich_media, enabled: true +config :pleroma, :rich_media, + enabled: true, + ignore_hosts: [], + ignore_tld: ["local", "localdomain", "lan"] config :pleroma, :media_proxy, enabled: false, diff --git a/config/test.exs b/config/test.exs index 73a8b82a1..9d441a7f5 100644 --- a/config/test.exs +++ b/config/test.exs @@ -43,7 +43,11 @@ config :pleroma, Pleroma.Repo, config :pbkdf2_elixir, rounds: 1 config :tesla, adapter: Tesla.Mock -config :pleroma, :rich_media, enabled: false + +config :pleroma, :rich_media, + enabled: false, + ignore_hosts: [], + ignore_tld: ["local", "localdomain", "lan"] config :web_push_encryption, :vapid_details, subject: "mailto:administrator@example.com", diff --git a/docs/config.md b/docs/config.md index b08c37e84..7a53364ef 100644 --- a/docs/config.md +++ b/docs/config.md @@ -417,6 +417,8 @@ This config contains two queues: `federator_incoming` and `federator_outgoing`. ## :rich_media * `enabled`: if enabled the instance will parse metadata from attached links to generate link previews +* `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`. +* `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"] ## :fetch_initial_posts * `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts @@ -604,6 +606,7 @@ Configure OAuth 2 provider capabilities: ## :emoji * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` +* `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]` * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays). diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 854d46b1a..052501642 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -148,11 +148,13 @@ defmodule Pleroma.Emoji do if File.exists?(emoji_txt) do load_from_file(emoji_txt, emoji_groups) else + extensions = Pleroma.Config.get([:emoji, :pack_extensions]) + Logger.info( - "No emoji.txt found for pack \"#{pack_name}\", assuming all .png files are emoji" + "No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji" ) - make_shortcode_to_file_map(pack_dir, [".png"]) + make_shortcode_to_file_map(pack_dir, extensions) |> Enum.map(fn {shortcode, rel_file} -> filename = Path.join("/emoji/#{pack_name}", rel_file) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 94f56f70d..6506de46c 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -4,27 +4,53 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Web.RichMedia.Parser + @spec validate_page_url(any()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld] - if AutoLinker.Parser.url?(page_url, scheme: true, validate_tld: validate_tld) do - URI.parse(page_url) |> validate_page_url - else - :error + page_url + |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld) + |> parse_uri(page_url) + end + + defp validate_page_url(%URI{host: host, scheme: scheme, authority: authority}) + when scheme == "https" and not is_nil(authority) do + cond do + host in Config.get([:rich_media, :ignore_hosts], []) -> + :error + + get_tld(host) in Config.get([:rich_media, :ignore_tld], []) -> + :error + + true -> + :ok end end - defp validate_page_url(%URI{authority: nil}), do: :error - defp validate_page_url(%URI{scheme: nil}), do: :error - defp validate_page_url(%URI{}), do: :ok defp validate_page_url(_), do: :error + defp parse_uri(true, url) do + url + |> URI.parse() + |> validate_page_url + end + + defp parse_uri(_, _), do: :error + + defp get_tld(host) do + host + |> String.split(".") + |> Enum.reverse() + |> hd + end + def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do - with true <- Pleroma.Config.get([:rich_media, :enabled]), + with true <- Config.get([:rich_media, :enabled]), %Object{} = object <- Object.normalize(activity), false <- object.data["sensitive"] || false, {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]), diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index f7f55a11a..30169edb0 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -757,6 +757,14 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}} end + def get("https://example.com/ogp", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}} + end + + def get("https://pleroma.local/notice/9kCP7V", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}} + end + def get("http://example.com/ogp-missing-data", _, _, _) do {:ok, %Tesla.Env{ @@ -765,6 +773,14 @@ defmodule HttpRequestMock do }} end + def get("https://example.com/ogp-missing-data", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/rich_media/ogp-missing-data.html") + }} + end + def get("http://example.com/malformed", _, _, _) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/malformed-data.html")}} diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 707723421..17e723528 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -312,7 +312,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do conn |> assign(:user, user) |> post("/api/v1/statuses", %{ - "status" => "http://example.com/ogp" + "status" => "https://example.com/ogp" }) assert %{"id" => id, "card" => %{"title" => "The Rock"}} = json_response(conn, 200) @@ -2557,7 +2557,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end test "returns rich-media card", %{conn: conn, user: user} do - {:ok, activity} = CommonAPI.post(user, %{"status" => "http://example.com/ogp"}) + {:ok, activity} = CommonAPI.post(user, %{"status" => "https://example.com/ogp"}) card_data = %{ "image" => "http://ia.media-imdb.com/images/rock.jpg", @@ -2589,7 +2589,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do # works with private posts {:ok, activity} = - CommonAPI.post(user, %{"status" => "http://example.com/ogp", "visibility" => "direct"}) + CommonAPI.post(user, %{"status" => "https://example.com/ogp", "visibility" => "direct"}) response_two = conn @@ -2601,7 +2601,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end test "replaces missing description with an empty string", %{conn: conn, user: user} do - {:ok, activity} = CommonAPI.post(user, %{"status" => "http://example.com/ogp-missing-data"}) + {:ok, activity} = + CommonAPI.post(user, %{"status" => "https://example.com/ogp-missing-data"}) response = conn diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs index 53b0596f5..c8f442b05 100644 --- a/test/web/rich_media/helpers_test.exs +++ b/test/web/rich_media/helpers_test.exs @@ -1,14 +1,19 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do use Pleroma.DataCase + alias Pleroma.Config alias Pleroma.Object alias Pleroma.Web.CommonAPI + alias Pleroma.Web.RichMedia.Helpers import Pleroma.Factory import Tesla.Mock setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + rich_media = Config.get([:rich_media, :enabled]) + on_exit(fn -> Config.put([:rich_media, :enabled], rich_media) end) + :ok end @@ -21,11 +26,9 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do "content_type" => "text/markdown" }) - Pleroma.Config.put([:rich_media, :enabled], true) + Config.put([:rich_media, :enabled], true) assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) - - Pleroma.Config.put([:rich_media, :enabled], false) end test "refuses to crawl malformed URLs" do @@ -37,11 +40,9 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do "content_type" => "text/markdown" }) - Pleroma.Config.put([:rich_media, :enabled], true) + Config.put([:rich_media, :enabled], true) assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) - - Pleroma.Config.put([:rich_media, :enabled], false) end test "crawls valid, complete URLs" do @@ -49,16 +50,14 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do {:ok, activity} = CommonAPI.post(user, %{ - "status" => "[test](http://example.com/ogp)", + "status" => "[test](https://example.com/ogp)", "content_type" => "text/markdown" }) - Pleroma.Config.put([:rich_media, :enabled], true) + Config.put([:rich_media, :enabled], true) - assert %{page_url: "http://example.com/ogp", rich_media: _} = + assert %{page_url: "https://example.com/ogp", rich_media: _} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) - - Pleroma.Config.put([:rich_media, :enabled], false) end test "refuses to crawl URLs from posts marked sensitive" do @@ -74,11 +73,9 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do assert object.data["sensitive"] - Pleroma.Config.put([:rich_media, :enabled], true) + Config.put([:rich_media, :enabled], true) assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) - - Pleroma.Config.put([:rich_media, :enabled], false) end test "refuses to crawl URLs from posts tagged NSFW" do @@ -93,10 +90,28 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do assert object.data["sensitive"] - Pleroma.Config.put([:rich_media, :enabled], true) + Config.put([:rich_media, :enabled], true) assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) + end + + test "refuses to crawl URLs of private network from posts" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{"status" => "http://127.0.0.1:4000/notice/9kCP7VNyPJXFOXDrgO"}) + + {:ok, activity2} = CommonAPI.post(user, %{"status" => "https://10.111.10.1/notice/9kCP7V"}) + {:ok, activity3} = CommonAPI.post(user, %{"status" => "https://172.16.32.40/notice/9kCP7V"}) + {:ok, activity4} = CommonAPI.post(user, %{"status" => "https://192.168.10.40/notice/9kCP7V"}) + {:ok, activity5} = CommonAPI.post(user, %{"status" => "https://pleroma.local/notice/9kCP7V"}) + + Config.put([:rich_media, :enabled], true) - Pleroma.Config.put([:rich_media, :enabled], false) + assert %{} = Helpers.fetch_data_for_activity(activity) + assert %{} = Helpers.fetch_data_for_activity(activity2) + assert %{} = Helpers.fetch_data_for_activity(activity3) + assert %{} = Helpers.fetch_data_for_activity(activity4) + assert %{} = Helpers.fetch_data_for_activity(activity5) end end |