Merge branch 'auto_linker' into 'develop'

AutoLinker

Closes #609

See merge request pleroma/pleroma!839

4 files changed, 107 insertions, 209 deletions

diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index f31aafa0d..51d08c5ee 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -8,33 +8,51 @@ defmodule Pleroma.Formatter do
   alias Pleroma.User
   alias Pleroma.Web.MediaProxy
 
-  @tag_regex ~r/((?<=[^&])|\A)(\#)(\w+)/u
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
+  @link_regex ~r{((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+}ui
 
-  # Modified from https://www.w3.org/TR/html5/forms.html#valid-e-mail-address
-  @mentions_regex ~r/@[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]*@?[a-zA-Z0-9_-](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/u
-
-  def parse_tags(text, data \\ %{}) do
-    Regex.scan(@tag_regex, text)
-    |> Enum.map(fn ["#" <> tag = full_tag | _] -> {full_tag, String.downcase(tag)} end)
-    |> (fn map ->
-          if data["sensitive"] in [true, "True", "true", "1"],
-            do: [{"#nsfw", "nsfw"}] ++ map,
-            else: map
-        end).()
+  @auto_linker_config hashtag: true,
+                      hashtag_handler: &Pleroma.Formatter.hashtag_handler/4,
+                      mention: true,
+                      mention_handler: &Pleroma.Formatter.mention_handler/4
+
+  def mention_handler("@" <> nickname, buffer, opts, acc) do
+    case User.get_cached_by_nickname(nickname) do
+      %User{id: id} = user ->
+        ap_id = get_ap_id(user)
+        nickname_text = get_nickname_text(nickname, opts) |> maybe_escape(opts)
+
+        link =
+          "<span class='h-card'><a data-user='#{id}' class='u-url mention' href='#{ap_id}'>@<span>#{
+            nickname_text
+          }</span></a></span>"
+
+        {link, %{acc | mentions: MapSet.put(acc.mentions, {"@" <> nickname, user})}}
+
+      _ ->
+        {buffer, acc}
+    end
   end
 
-  @doc "Parses mentions text and returns list {nickname, user}."
-  @spec parse_mentions(binary()) :: list({binary(), User.t()})
-  def parse_mentions(text) do
-    Regex.scan(@mentions_regex, text)
-    |> List.flatten()
-    |> Enum.uniq()
-    |> Enum.map(fn nickname ->
-      with nickname <- String.trim_leading(nickname, "@"),
-           do: {"@" <> nickname, User.get_cached_by_nickname(nickname)}
-    end)
-    |> Enum.filter(fn {_match, user} -> user end)
+  def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do
+    tag = String.downcase(tag)
+    url = "#{Pleroma.Web.base_url()}/tag/#{tag}"
+    link = "<a class='hashtag' data-tag='#{tag}' href='#{url}' rel='tag'>#{tag_text}</a>"
+
+    {link, %{acc | tags: MapSet.put(acc.tags, {tag_text, tag})}}
+  end
+
+  @doc """
+  Parses a text and replace plain text links with HTML. Returns a tuple with a result text, mentions, and hashtags.
+  """
+  @spec linkify(String.t(), keyword()) ::
+          {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]}
+  def linkify(text, options \\ []) do
+    options = options ++ @auto_linker_config
+    acc = %{mentions: MapSet.new(), tags: MapSet.new()}
+    {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options)
+
+    {text, MapSet.to_list(mentions), MapSet.to_list(tags)}
   end
 
   def emojify(text) do
@@ -48,9 +66,7 @@ defmodule Pleroma.Formatter do
       emoji = HTML.strip_tags(emoji)
       file = HTML.strip_tags(file)
 
-      String.replace(
-        text,
-        ":#{emoji}:",
+      html =
         if not strip do
           "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{
             MediaProxy.url(file)
@@ -58,8 +74,8 @@ defmodule Pleroma.Formatter do
         else
           ""
         end
-      )
-      |> HTML.filter_tags()
+
+      String.replace(text, ":#{emoji}:", html) |> HTML.filter_tags()
     end)
   end
 
@@ -75,12 +91,6 @@ defmodule Pleroma.Formatter do
 
   def get_emoji(_), do: []
 
-  @link_regex ~r/[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+/ui
-
-  @uri_schemes Application.get_env(:pleroma, :uri_schemes, [])
-  @valid_schemes Keyword.get(@uri_schemes, :valid_schemes, [])
-
-  # TODO: make it use something other than @link_regex
   def html_escape(text, "text/html") do
     HTML.filter_tags(text)
   end
@@ -94,112 +104,6 @@ defmodule Pleroma.Formatter do
     |> Enum.join("")
   end
 
-  @doc """
-  Escapes a special characters in mention names.
-  """
-  @spec mentions_escape(String.t(), list({String.t(), any()})) :: String.t()
-  def mentions_escape(text, mentions) do
-    mentions
-    |> Enum.reduce(text, fn {name, _}, acc ->
-      escape_name = String.replace(name, @markdown_characters_regex, "\\\\\\1")
-      String.replace(acc, name, escape_name)
-    end)
-  end
-
-  @doc "changes scheme:... urls to html links"
-  def add_links({subs, text}) do
-    links =
-      text
-      |> String.split([" ", "\t", "<br>"])
-      |> Enum.filter(fn word -> String.starts_with?(word, @valid_schemes) end)
-      |> Enum.filter(fn word -> Regex.match?(@link_regex, word) end)
-      |> Enum.map(fn url -> {Ecto.UUID.generate(), url} end)
-      |> Enum.sort_by(fn {_, url} -> -String.length(url) end)
-
-    uuid_text =
-      links
-      |> Enum.reduce(text, fn {uuid, url}, acc -> String.replace(acc, url, uuid) end)
-
-    subs =
-      subs ++
-        Enum.map(links, fn {uuid, url} ->
-          {uuid, "<a href=\"#{url}\">#{url}</a>"}
-        end)
-
-    {subs, uuid_text}
-  end
-
-  @doc "Adds the links to mentioned users"
-  def add_user_links({subs, text}, mentions, options \\ []) do
-    mentions =
-      mentions
-      |> Enum.sort_by(fn {name, _} -> -String.length(name) end)
-      |> Enum.map(fn {name, user} -> {name, user, Ecto.UUID.generate()} end)
-
-    uuid_text =
-      mentions
-      |> Enum.reduce(text, fn {match, _user, uuid}, text ->
-        String.replace(text, match, uuid)
-      end)
-
-    subs =
-      subs ++
-        Enum.map(mentions, fn {match, %User{id: id, ap_id: ap_id, info: info}, uuid} ->
-          ap_id =
-            if is_binary(info.source_data["url"]) do
-              info.source_data["url"]
-            else
-              ap_id
-            end
-
-          nickname =
-            if options[:format] == :full do
-              User.full_nickname(match)
-            else
-              User.local_nickname(match)
-            end
-
-          {uuid,
-           "<span class='h-card'><a data-user='#{id}' class='u-url mention' href='#{ap_id}'>" <>
-             "@<span>#{nickname}</span></a></span>"}
-        end)
-
-    {subs, uuid_text}
-  end
-
-  @doc "Adds the hashtag links"
-  def add_hashtag_links({subs, text}, tags) do
-    tags =
-      tags
-      |> Enum.sort_by(fn {name, _} -> -String.length(name) end)
-      |> Enum.map(fn {name, short} -> {name, short, Ecto.UUID.generate()} end)
-
-    uuid_text =
-      tags
-      |> Enum.reduce(text, fn {match, _short, uuid}, text ->
-        String.replace(text, ~r/((?<=[^&])|(\A))#{match}/, uuid)
-      end)
-
-    subs =
-      subs ++
-        Enum.map(tags, fn {tag_text, tag, uuid} ->
-          url =
-            "<a class='hashtag' data-tag='#{tag}' href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>#{
-              tag_text
-            }</a>"
-
-          {uuid, url}
-        end)
-
-    {subs, uuid_text}
-  end
-
-  def finalize({subs, text}) do
-    Enum.reduce(subs, text, fn {uuid, replacement}, result_text ->
-      String.replace(result_text, uuid, replacement)
-    end)
-  end
-
   def truncate(text, max_length \\ 200, omission \\ "...") do
     # Remove trailing whitespace
     text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}")
@@ -211,4 +115,16 @@ defmodule Pleroma.Formatter do
       String.slice(text, 0, length_with_omission) <> omission
     end
   end
+
+  defp get_ap_id(%User{info: %{source_data: %{"url" => url}}}) when is_binary(url), do: url
+  defp get_ap_id(%User{ap_id: ap_id}), do: ap_id
+
+  defp get_nickname_text(nickname, %{mentions_format: :full}), do: User.full_nickname(nickname)
+  defp get_nickname_text(nickname, _), do: User.local_nickname(nickname)
+
+  defp maybe_escape(str, %{mentions_escape: true}) do
+    String.replace(str, @markdown_characters_regex, "\\\\\\1")
+  end
+
+  defp maybe_escape(str, _), do: str
 end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 12e0e818e..01d532ab3 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1193,9 +1193,6 @@ defmodule Pleroma.User do
   def parse_bio(bio, _user) when bio == "", do: bio
 
   def parse_bio(bio, user) do
-    mentions = Formatter.parse_mentions(bio)
-    tags = Formatter.parse_tags(bio)
-
     emoji =
       (user.info.source_data["tag"] || [])
       |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
@@ -1204,7 +1201,8 @@ defmodule Pleroma.User do
       end)
 
     bio
-    |> CommonUtils.format_input(mentions, tags, "text/plain", user_links: [format: :full])
+    |> CommonUtils.format_input("text/plain", mentions_format: :full)
+    |> elem(0)
     |> Formatter.emojify(emoji)
   end
 
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index e788337cc..7114d6de6 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -82,40 +82,20 @@ defmodule Pleroma.Web.CommonAPI do
 
   def get_visibility(_), do: "public"
 
-  defp get_content_type(content_type) do
-    if Enum.member?(Pleroma.Config.get([:instance, :allowed_post_formats]), content_type) do
-      content_type
-    else
-      "text/plain"
-    end
-  end
-
   def post(user, %{"status" => status} = data) do
     visibility = get_visibility(data)
     limit = Pleroma.Config.get([:instance, :limit])
 
     with status <- String.trim(status),
          attachments <- attachments_from_ids(data),
-         mentions <- Formatter.parse_mentions(status),
          inReplyTo <- get_replied_to_activity(data["in_reply_to_status_id"]),
-         {to, cc} <- to_for_user_and_mentions(user, mentions, inReplyTo, visibility),
-         tags <- Formatter.parse_tags(status, data),
-         content_html <-
+         {content_html, mentions, tags} <-
            make_content_html(
              status,
-             mentions,
              attachments,
-             tags,
-             get_content_type(data["content_type"]),
-             Enum.member?(
-               [true, "true"],
-               Map.get(
-                 data,
-                 "no_attachment_links",
-                 Pleroma.Config.get([:instance, :no_attachment_links], false)
-               )
-             )
+             data
            ),
+         {to, cc} <- to_for_user_and_mentions(user, mentions, inReplyTo, visibility),
          context <- make_context(inReplyTo),
          cw <- data["spoiler_text"],
          full_payload <- String.trim(status <> (data["spoiler_text"] || "")),
@@ -247,7 +227,7 @@ defmodule Pleroma.Web.CommonAPI do
   def report(user, data) do
     with {:account_id, %{"account_id" => account_id}} <- {:account_id, data},
          {:account, %User{} = account} <- {:account, User.get_by_id(account_id)},
-         {:ok, content_html} <- make_report_content_html(data["comment"]),
+         {:ok, {content_html, _, _}} <- make_report_content_html(data["comment"]),
          {:ok, statuses} <- get_report_statuses(account, data),
          {:ok, activity} <-
            ActivityPub.flag(%{
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 1d3a314ce..20123854d 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
-  alias Pleroma.Web
+  alias Pleroma.Config
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.MediaProxy
   alias Pleroma.Web.ActivityPub.Utils
@@ -100,24 +100,45 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   def make_content_html(
         status,
-        mentions,
         attachments,
-        tags,
-        content_type,
-        no_attachment_links \\ false
+        data
       ) do
+    no_attachment_links =
+      data
+      |> Map.get("no_attachment_links", Config.get([:instance, :no_attachment_links]))
+      |> Kernel.in([true, "true"])
+
+    content_type = get_content_type(data["content_type"])
+
     status
-    |> format_input(mentions, tags, content_type)
+    |> format_input(content_type)
     |> maybe_add_attachments(attachments, no_attachment_links)
+    |> maybe_add_nsfw_tag(data)
+  end
+
+  defp get_content_type(content_type) do
+    if Enum.member?(Config.get([:instance, :allowed_post_formats]), content_type) do
+      content_type
+    else
+      "text/plain"
+    end
   end
 
+  defp maybe_add_nsfw_tag({text, mentions, tags}, %{"sensitive" => sensitive})
+       when sensitive in [true, "True", "true", "1"] do
+    {text, mentions, [{"#nsfw", "nsfw"} | tags]}
+  end
+
+  defp maybe_add_nsfw_tag(data, _), do: data
+
   def make_context(%Activity{data: %{"context" => context}}), do: context
   def make_context(_), do: Utils.generate_context_id()
 
-  def maybe_add_attachments(text, _attachments, true = _no_links), do: text
+  def maybe_add_attachments(parsed, _attachments, true = _no_links), do: parsed
 
-  def maybe_add_attachments(text, attachments, _no_links) do
-    add_attachments(text, attachments)
+  def maybe_add_attachments({text, mentions, tags}, attachments, _no_links) do
+    text = add_attachments(text, attachments)
+    {text, mentions, tags}
   end
 
   def add_attachments(text, attachments) do
@@ -135,56 +156,39 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     Enum.join([text | attachment_text], "<br>")
   end
 
-  def format_input(text, mentions, tags, format, options \\ [])
+  def format_input(text, format, options \\ [])
 
   @doc """
   Formatting text to plain text.
   """
-  def format_input(text, mentions, tags, "text/plain", options) do
+  def format_input(text, "text/plain", options) do
     text
     |> Formatter.html_escape("text/plain")
-    |> String.replace(~r/\r?\n/, "<br>")
-    |> (&{[], &1}).()
-    |> Formatter.add_links()
-    |> Formatter.add_user_links(mentions, options[:user_links] || [])
-    |> Formatter.add_hashtag_links(tags)
-    |> Formatter.finalize()
+    |> Formatter.linkify(options)
+    |> (fn {text, mentions, tags} ->
+          {String.replace(text, ~r/\r?\n/, "<br>"), mentions, tags}
+        end).()
   end
 
   @doc """
   Formatting text to html.
   """
-  def format_input(text, mentions, _tags, "text/html", options) do
+  def format_input(text, "text/html", options) do
     text
     |> Formatter.html_escape("text/html")
-    |> (&{[], &1}).()
-    |> Formatter.add_user_links(mentions, options[:user_links] || [])
-    |> Formatter.finalize()
+    |> Formatter.linkify(options)
   end
 
   @doc """
   Formatting text to markdown.
   """
-  def format_input(text, mentions, tags, "text/markdown", options) do
+  def format_input(text, "text/markdown", options) do
+    options = Keyword.put(options, :mentions_escape, true)
+
     text
-    |> Formatter.mentions_escape(mentions)
-    |> Earmark.as_html!()
     |> Formatter.html_escape("text/html")
-    |> (&{[], &1}).()
-    |> Formatter.add_user_links(mentions, options[:user_links] || [])
-    |> Formatter.add_hashtag_links(tags)
-    |> Formatter.finalize()
-  end
-
-  def add_tag_links(text, tags) do
-    tags =
-      tags
-      |> Enum.sort_by(fn {tag, _} -> -String.length(tag) end)
-
-    Enum.reduce(tags, text, fn {full, tag}, text ->
-      url = "<a href='#{Web.base_url()}/tag/#{tag}' rel='tag'>##{tag}</a>"
-      String.replace(text, full, url)
-    end)
+    |> Formatter.linkify(options)
+    |> (fn {text, mentions, tags} -> {Earmark.as_html!(text), mentions, tags} end).()
   end
 
   def make_note_data(
@@ -323,13 +327,13 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   def maybe_extract_mentions(_), do: []
 
-  def make_report_content_html(nil), do: {:ok, nil}
+  def make_report_content_html(nil), do: {:ok, {nil, [], []}}
 
   def make_report_content_html(comment) do
     max_size = Pleroma.Config.get([:instance, :max_report_comment_size], 1000)
 
     if String.length(comment) <= max_size do
-      {:ok, format_input(comment, [], [], "text/plain")}
+      {:ok, format_input(comment, "text/plain")}
     else
       {:error, "Comment must be up to #{max_size} characters"}
     end