diff options
| -rw-r--r-- | .gitlab-ci.yml | 2 | ||||
| -rw-r--r-- | CHANGELOG.md | 2 | ||||
| -rw-r--r-- | config/config.exs | 3 | ||||
| -rw-r--r-- | docs/configuration/cheatsheet.md | 1 | ||||
| -rw-r--r-- | lib/pleroma/activity/search.ex | 31 | ||||
| -rw-r--r-- | test/pleroma/activity/search_test.exs | 45 | ||||
| -rw-r--r-- | test/pleroma/web/mastodon_api/controllers/search_controller_test.exs | 2 |
7 files changed, 81 insertions, 5 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a754ed78..1b05e4a08 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,7 +57,7 @@ unit-testing: policy: pull services: - - name: postgres:9.6 + - name: postgres:13 alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] script: diff --git a/CHANGELOG.md b/CHANGELOG.md index fc4013fc3..a682036f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Polls now always return a `voters_count`, even if they are single-choice. - Admin Emails: The ap id is used as the user link in emails now. +- *Breaking* Configuration: Use `websearch` function by default. If you're using a PostgreSQL version below 11, set `:instance, :search_function` to `:plain` in your configuration. ### Added @@ -22,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Ability to view remote timelines, with ex. `/api/v1/timelines/public?instance=lain.com` and streams `public:remote` and `public:remote:media`. - The site title is now injected as a `title` tag like preloads or metadata. - Password reset tokens now are not accepted after a certain age. +- Added a configuration option to use the postgresql `websearch` function for more complicated search queries. <details> <summary>API Changes</summary> diff --git a/config/config.exs b/config/config.exs index be5257663..8d0545704 100644 --- a/config/config.exs +++ b/config/config.exs @@ -264,7 +264,8 @@ config :pleroma, :instance, ] ], show_reactions: true, - password_reset_token_validity: 60 * 60 * 24 + password_reset_token_validity: 60 * 60 * 24, + search_function: :websearch config :pleroma, :welcome, direct_message: [ diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 85551362c..1b321d103 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -64,6 +64,7 @@ To add configuration to your config file, you can copy it from the base config. * `cleanup_attachments`: Remove attachments along with statuses. Does not affect duplicate files and attachments without status. Enabling this will increase load to database when deleting statuses on larger instances. * `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`). * `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day). +* `search_function`: What search function to use for fulltext search. Possible values are `:websearch` and `:plain`. `:websearch` enables more complex search queries, but requires at least PostgreSQL 11. (default: `websearch`) ## Welcome * `direct_message`: - welcome message sent as a direct message. diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex index 382c81118..cc98e2d06 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/activity/search.ex @@ -19,11 +19,13 @@ defmodule Pleroma.Activity.Search do offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) + search_function = Pleroma.Config.get([:instance, :search_function], :plain) + Activity |> Activity.with_preloaded_object() |> Activity.restrict_deactivated_users() |> restrict_public() - |> query_with(index_type, search_query) + |> query_with(index_type, search_query, search_function) |> maybe_restrict_local(user) |> maybe_restrict_author(author) |> maybe_restrict_blocked(user) @@ -53,7 +55,7 @@ defmodule Pleroma.Activity.Search do ) end - defp query_with(q, :gin, search_query) do + defp query_with(q, :gin, search_query, :plain) do from([a, o] in q, where: fragment( @@ -64,7 +66,18 @@ defmodule Pleroma.Activity.Search do ) end - defp query_with(q, :rum, search_query) do + defp query_with(q, :gin, search_query, :websearch) do + from([a, o] in q, + where: + fragment( + "to_tsvector('english', ?->>'content') @@ websearch_to_tsquery('english', ?)", + o.data, + ^search_query + ) + ) + end + + defp query_with(q, :rum, search_query, :plain) do from([a, o] in q, where: fragment( @@ -76,6 +89,18 @@ defmodule Pleroma.Activity.Search do ) end + defp query_with(q, :rum, search_query, :websearch) do + from([a, o] in q, + where: + fragment( + "? @@ websearch_to_tsquery('english', ?)", + o.fts_content, + ^search_query + ), + order_by: [fragment("? <=> now()::date", o.inserted_at)] + ) + end + defp maybe_restrict_local(q, user) do limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) diff --git a/test/pleroma/activity/search_test.exs b/test/pleroma/activity/search_test.exs new file mode 100644 index 000000000..15591b726 --- /dev/null +++ b/test/pleroma/activity/search_test.exs @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Activity.SearchTest do + alias Pleroma.Activity.Search + alias Pleroma.Web.CommonAPI + import Pleroma.Factory + + use Pleroma.DataCase + + test "it finds something" do + user = insert(:user) + {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"}) + + [result] = Search.search(nil, "wednesday") + + assert result.id == post.id + end + + test "using plainto_tsquery" do + clear_config([:instance, :search_function], :plain) + + user = insert(:user) + {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"}) + {:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"}) + + # plainto doesn't understand complex queries + assert [result] = Search.search(nil, "wednesday -dudes") + + assert result.id == post.id + end + + test "using websearch_to_tsquery" do + clear_config([:instance, :search_function], :websearch) + + user = insert(:user) + {:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"}) + {:ok, other_post} = CommonAPI.post(user, %{status: "it's wednesday my bros"}) + + assert [result] = Search.search(nil, "wednesday -dudes") + + assert result.id == other_post.id + end +end diff --git a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs index 04dc6f445..b77614b7c 100644 --- a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs @@ -279,6 +279,8 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do end test "search fetches remote statuses and prefers them over other results", %{conn: conn} do + clear_config([:instance, :search_function], :plain) + capture_log(fn -> {:ok, %{id: activity_id}} = CommonAPI.post(insert(:user), %{ |