Merge branch 'develop' into 'feature/relay'

# Conflicts:
#   lib/pleroma/web/activity_pub/utils.ex

24 files changed, 403 insertions, 93 deletions

diff --git a/lib/mix/tasks/generate_invite_token.ex b/lib/mix/tasks/generate_invite_token.ex
new file mode 100644
index 000000000..c4daa9a6c
--- /dev/null
+++ b/lib/mix/tasks/generate_invite_token.ex
@@ -0,0 +1,25 @@
+defmodule Mix.Tasks.GenerateInviteToken do
+  use Mix.Task
+
+  @shortdoc "Generate invite token for user"
+  def run([]) do
+    Mix.Task.run("app.start")
+
+    with {:ok, token} <- Pleroma.UserInviteToken.create_token() do
+      IO.puts("Generated user invite token")
+
+      IO.puts(
+        "Url: #{
+          Pleroma.Web.Router.Helpers.redirect_url(
+            Pleroma.Web.Endpoint,
+            :registration_page,
+            token.token
+          )
+        }"
+      )
+    else
+      _ ->
+        IO.puts("Error creating token")
+    end
+  end
+end
diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index 0aaf21538..cf2944c38 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -16,7 +16,7 @@ defmodule Pleroma.Formatter do
   def parse_mentions(text) do
     # Modified from https://www.w3.org/TR/html5/forms.html#valid-e-mail-address
     regex =
-      ~r/@[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@?[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/u
+      ~r/@[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]*@?[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/u
 
     Regex.scan(regex, text)
     |> List.flatten()
@@ -165,8 +165,29 @@ defmodule Pleroma.Formatter do
     @emoji
   end
 
-  @link_regex ~r/https?:\/\/[\w\.\/?=\-#\+%&@~'\(\):]+[\w\/]/u
+  @link_regex ~r/[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+/ui
+
+  # IANA got a list https://www.iana.org/assignments/uri-schemes/ but
+  # Stuff like ipfs isn’t in it
+  # There is very niche stuff
+  @uri_schemes [
+    "https://",
+    "http://",
+    "dat://",
+    "dweb://",
+    "gopher://",
+    "ipfs://",
+    "ipns://",
+    "irc:",
+    "ircs:",
+    "magnet:",
+    "mailto:",
+    "mumble:",
+    "ssb://",
+    "xmpp:"
+  ]
 
+  # TODO: make it use something other than @link_regex
   def html_escape(text) do
     Regex.split(@link_regex, text, include_captures: true)
     |> Enum.map_every(2, fn chunk ->
@@ -176,11 +197,18 @@ defmodule Pleroma.Formatter do
     |> Enum.join("")
   end
 
-  @doc "changes http:... links to html links"
+  @doc "changes scheme:... urls to html links"
   def add_links({subs, text}) do
+    additionnal_schemes =
+      Application.get_env(:pleroma, :uri_schemes, [])
+      |> Keyword.get(:additionnal_schemes, [])
+
     links =
-      Regex.scan(@link_regex, text)
-      |> Enum.map(fn [url] -> {Ecto.UUID.generate(), url} end)
+      text
+      |> String.split([" ", "\t", "<br>"])
+      |> Enum.filter(fn word -> String.starts_with?(word, @uri_schemes ++ additionnal_schemes) end)
+      |> Enum.filter(fn word -> Regex.match?(@link_regex, word) end)
+      |> Enum.map(fn url -> {Ecto.UUID.generate(), url} end)
       |> Enum.sort_by(fn {_, url} -> -String.length(url) end)
 
     uuid_text =
@@ -244,8 +272,8 @@ defmodule Pleroma.Formatter do
 
     subs =
       subs ++
-        Enum.map(tags, fn {_, tag, uuid} ->
-          url = "<a href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>##{tag}</a>"
+        Enum.map(tags, fn {tag_text, tag, uuid} ->
+          url = "<a href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>#{tag_text}</a>"
           {uuid, url}
         end)
 
diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex
index f6abcd4d0..97a1dea77 100644
--- a/lib/pleroma/gopher/server.ex
+++ b/lib/pleroma/gopher/server.ex
@@ -54,7 +54,7 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do
 
     String.split(text, "\r")
     |> Enum.map(fn text ->
-      "i#{text}\tfake\(NULL)\t0\r\n"
+      "i#{text}\tfake\t(NULL)\t0\r\n"
     end)
     |> Enum.join("")
   end
@@ -77,14 +77,14 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do
 
       link("Post ##{activity.id} by #{user.nickname}", "/notices/#{activity.id}") <>
         info("#{like_count} likes, #{announcement_count} repeats") <>
-        "\r\n" <>
+        "i\tfake\t(NULL)\t0\r\n" <>
         info(
           HtmlSanitizeEx.strip_tags(
             String.replace(activity.data["object"]["content"], "<br>", "\r")
           )
         )
     end)
-    |> Enum.join("\r\n")
+    |> Enum.join("i\tfake\t(NULL)\t0\r\n")
   end
 
   def response("") do
diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex
index 84f34eb4a..c19bccf60 100644
--- a/lib/pleroma/http/http.ex
+++ b/lib/pleroma/http/http.ex
@@ -1,5 +1,23 @@
 defmodule Pleroma.HTTP do
-  use HTTPoison.Base
+  require HTTPoison
+
+  def request(method, url, body \\ "", headers \\ [], options \\ []) do
+    options =
+      process_request_options(options)
+      |> process_sni_options(url)
+
+    HTTPoison.request(method, url, body, headers, options)
+  end
+
+  defp process_sni_options(options, url) do
+    uri = URI.parse(url)
+    host = uri.host |> to_charlist()
+
+    case uri.scheme do
+      "https" -> options ++ [ssl: [server_name_indication: host]]
+      _ -> options
+    end
+  end
 
   def process_request_options(options) do
     config = Application.get_env(:pleroma, :http, [])
@@ -10,4 +28,9 @@ defmodule Pleroma.HTTP do
       _ -> options ++ [proxy: proxy]
     end
   end
+
+  def get(url, headers \\ [], options \\ []), do: request(:get, url, "", headers, options)
+
+  def post(url, body, headers \\ [], options \\ []),
+    do: request(:post, url, body, headers, options)
 end
diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
index 408a3fc56..e0cb545b0 100644
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@@ -124,20 +124,20 @@ defmodule Pleroma.Upload do
     if should_dedupe do
       create_name(uuid, List.last(String.split(file.filename, ".")), type)
     else
-      unless String.contains?(file.filename, ".") do
-        case type do
-          "image/png" -> file.filename <> ".png"
-          "image/jpeg" -> file.filename <> ".jpg"
-          "image/gif" -> file.filename <> ".gif"
-          "video/webm" -> file.filename <> ".webm"
-          "video/mp4" -> file.filename <> ".mp4"
-          "audio/mpeg" -> file.filename <> ".mp3"
-          "audio/ogg" -> file.filename <> ".ogg"
-          "audio/wav" -> file.filename <> ".wav"
-          _ -> file.filename
+      parts = String.split(file.filename, ".")
+
+      new_filename =
+        if length(parts) > 1 do
+          Enum.drop(parts, -1) |> Enum.join(".")
+        else
+          Enum.join(parts)
         end
-      else
-        file.filename
+
+      case type do
+        "application/octet-stream" -> file.filename
+        "audio/mpeg" -> new_filename <> ".mp3"
+        "image/jpeg" -> new_filename <> ".jpg"
+        _ -> Enum.join([new_filename, String.split(type, "/") |> List.last()], ".")
       end
     end
   end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 748fdbca4..3bcfcdd91 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -398,6 +398,7 @@ defmodule Pleroma.User do
       Enum.map(reqs, fn req -> req.actor end)
       |> Enum.uniq()
       |> Enum.map(fn ap_id -> get_by_ap_id(ap_id) end)
+      |> Enum.filter(fn u -> !following?(u, user) end)
 
     {:ok, users}
   end
diff --git a/lib/pleroma/user_invite_token.ex b/lib/pleroma/user_invite_token.ex
new file mode 100644
index 000000000..48ee1019a
--- /dev/null
+++ b/lib/pleroma/user_invite_token.ex
@@ -0,0 +1,40 @@
+defmodule Pleroma.UserInviteToken do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+
+  alias Pleroma.{User, UserInviteToken, Repo}
+
+  schema "user_invite_tokens" do
+    field(:token, :string)
+    field(:used, :boolean, default: false)
+
+    timestamps()
+  end
+
+  def create_token do
+    token = :crypto.strong_rand_bytes(32) |> Base.url_encode64()
+
+    token = %UserInviteToken{
+      used: false,
+      token: token
+    }
+
+    Repo.insert(token)
+  end
+
+  def used_changeset(struct) do
+    struct
+    |> cast(%{}, [])
+    |> put_change(:used, true)
+  end
+
+  def mark_as_used(token) do
+    with %{used: false} = token <- Repo.get_by(UserInviteToken, %{token: token}),
+         {:ok, token} <- Repo.update(used_changeset(token)) do
+      {:ok, token}
+    else
+      _e -> {:error, token}
+    end
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index a07bf1629..3a25f614e 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -576,7 +576,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def fetch_and_prepare_user_from_ap_id(ap_id) do
     with {:ok, %{status_code: 200, body: body}} <-
-           @httpoison.get(ap_id, Accept: "application/activity+json"),
+           @httpoison.get(ap_id, [Accept: "application/activity+json"], follow_redirect: true),
          {:ok, data} <- Jason.decode(body) do
       user_data_from_user_object(data)
     else
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 2ebc526df..1367bc7e3 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -18,12 +18,16 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def get_actor(%{"actor" => actor}) when is_list(actor) do
-    Enum.at(actor, 0)
+    if is_binary(Enum.at(actor, 0)) do
+      Enum.at(actor, 0)
+    else
+      Enum.find(actor, fn %{"type" => type} -> type == "Person" end)
+      |> Map.get("id")
+    end
   end
 
-  def get_actor(%{"actor" => actor_list}) do
-    Enum.find(actor_list, fn %{"type" => type} -> type == "Person" end)
-    |> Map.get("id")
+  def get_actor(%{"actor" => actor}) when is_map(actor) do
+    actor["id"]
   end
 
   @doc """
@@ -38,6 +42,25 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> fix_emoji
     |> fix_tag
     |> fix_content_map
+    |> fix_likes
+    |> fix_addressing
+  end
+
+  def fix_addressing_list(map, field) do
+    if is_binary(map[field]) do
+      map
+      |> Map.put(field, [map[field]])
+    else
+      map
+    end
+  end
+
+  def fix_addressing(map) do
+    map
+    |> fix_addressing_list("to")
+    |> fix_addressing_list("cc")
+    |> fix_addressing_list("bto")
+    |> fix_addressing_list("bcc")
   end
 
   def fix_actor(%{"attributedTo" => actor} = object) do
@@ -45,6 +68,20 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> Map.put("actor", get_actor(%{"actor" => actor}))
   end
 
+  def fix_likes(%{"likes" => likes} = object)
+      when is_bitstring(likes) do
+    # Check for standardisation
+    # This is what Peertube does
+    # curl -H 'Accept: application/activity+json' $likes | jq .totalItems
+    object
+    |> Map.put("likes", [])
+    |> Map.put("like_count", 0)
+  end
+
+  def fix_likes(object) do
+    object
+  end
+
   def fix_in_reply_to(%{"inReplyTo" => in_reply_to_id} = object)
       when not is_nil(in_reply_to_id) do
     case ActivityPub.fetch_object_from_id(in_reply_to_id) do
@@ -72,8 +109,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def fix_in_reply_to(object), do: object
 
   def fix_context(object) do
+    context = object["context"] || object["conversation"] || Utils.generate_context_id()
+
     object
-    |> Map.put("context", object["conversation"])
+    |> Map.put("context", context)
+    |> Map.put("conversation", context)
   end
 
   def fix_attachments(object) do
@@ -137,13 +177,22 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def fix_content_map(object), do: object
 
+  # disallow objects with bogus IDs
+  def handle_incoming(%{"id" => nil}), do: :error
+  def handle_incoming(%{"id" => ""}), do: :error
+  # length of https:// = 8, should validate better, but good enough for now.
+  def handle_incoming(%{"id" => id}) when not (is_binary(id) and length(id) > 8), do: :error
+
   # TODO: validate those with a Ecto scheme
   # - tags
   # - emoji
   def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data)
-      when objtype in ["Article", "Note"] do
+      when objtype in ["Article", "Note", "Video"] do
     actor = get_actor(data)
-    data = Map.put(data, "actor", actor)
+
+    data =
+      Map.put(data, "actor", actor)
+      |> fix_addressing
 
     with nil <- Activity.get_create_activity_by_object_ap_id(object["id"]),
          %User{} = user <- User.get_or_fetch_by_ap_id(data["actor"]) do
diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
index a2e5c5002..0664b5a2e 100644
--- a/lib/pleroma/web/activity_pub/utils.ex
+++ b/lib/pleroma/web/activity_pub/utils.ex
@@ -128,7 +128,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   Inserts a full object if it is contained in an activity.
   """
   def insert_full_object(%{"object" => %{"type" => type} = object_data})
-      when is_map(object_data) and type in ["Article", "Note"] do
+      when is_map(object_data) and type in ["Article", "Note", "Video"] do
     with {:ok, _} <- Object.create(object_data) do
       :ok
     end
@@ -204,13 +204,17 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   end
 
   def add_like_to_object(%Activity{data: %{"actor" => actor}}, object) do
-    with likes <- [actor | object.data["likes"] || []] |> Enum.uniq() do
+    likes = if is_list(object.data["likes"]), do: object.data["likes"], else: []
+
+    with likes <- [actor | likes] |> Enum.uniq() do
       update_likes_in_object(likes, object)
     end
   end
 
   def remove_like_from_object(%Activity{data: %{"actor" => actor}}, object) do
-    with likes <- (object.data["likes"] || []) |> List.delete(actor) do
+    likes = if is_list(object.data["likes"]), do: object.data["likes"], else: []
+
+    with likes <- likes |> List.delete(actor) do
       update_likes_in_object(likes, object)
     end
   end
@@ -380,7 +384,10 @@ defmodule Pleroma.Web.ActivityPub.Utils do
         },
         object
       ) do
-    with announcements <- [actor | object.data["announcements"] || []] |> Enum.uniq() do
+    announcements =
+      if is_list(object.data["announcements"]), do: object.data["announcements"], else: []
+
+    with announcements <- [actor | announcements] |> Enum.uniq() do
       update_element_in_object("announcement", announcements, object)
     end
   end
@@ -388,7 +395,10 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   def add_announce_to_object(_, object), do: {:ok, object}
 
   def remove_announce_from_object(%Activity{data: %{"actor" => actor}}, object) do
-    with announcements <- (object.data["announcements"] || []) |> List.delete(actor) do
+    announcements =
+      if is_list(object.data["announcements"]), do: object.data["announcements"], else: []
+
+    with announcements <- announcements |> List.delete(actor) do
       update_element_in_object("announcement", announcements, object)
     end
   end
diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex
index 6ecb8862e..b57cbb9ac 100644
--- a/lib/pleroma/web/activity_pub/views/user_view.ex
+++ b/lib/pleroma/web/activity_pub/views/user_view.ex
@@ -127,9 +127,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do
     info = User.user_info(user)
 
     params = %{
-      "type" => ["Create", "Announce"],
-      "actor_id" => user.ap_id,
-      "whole_db" => true,
       "limit" => "10"
     }
 
@@ -140,10 +137,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do
         params
       end
 
-    activities = ActivityPub.fetch_public_activities(params)
-    min_id = Enum.at(activities, 0).id
-
-    activities = Enum.reverse(activities)
+    activities = ActivityPub.fetch_user_activities(user, nil, params)
+    min_id = Enum.at(Enum.reverse(activities), 0).id
     max_id = Enum.at(activities, 0).id
 
     collection =
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 30089f553..869f4c566 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -64,7 +64,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   def make_content_html(status, mentions, attachments, tags, no_attachment_links \\ false) do
     status
-    |> String.replace("\r", "")
     |> format_input(mentions, tags)
     |> maybe_add_attachments(attachments, no_attachment_links)
   end
@@ -95,7 +94,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   def format_input(text, mentions, tags) do
     text
     |> Formatter.html_escape()
-    |> String.replace("\n", "<br>")
+    |> String.replace(~r/\r?\n/, "<br>")
     |> (&{[], &1}).()
     |> Formatter.add_links()
     |> Formatter.add_user_links(mentions)
@@ -109,7 +108,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
       |> Enum.sort_by(fn {tag, _} -> -String.length(tag) end)
 
     Enum.reduce(tags, text, fn {full, tag}, text ->
-      url = "#<a href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>#{tag}</a>"
+      url = "<a href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>##{tag}</a>"
       String.replace(text, full, url)
     end)
   end
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 956787d5a..f482de6fd 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -5,21 +5,26 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.Web.MastodonAPI.{StatusView, AccountView, MastodonView, ListView}
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.{CommonAPI, OStatus}
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.OAuth.{Authorization, Token, App}
   alias Comeonin.Pbkdf2
   import Ecto.Query
   require Logger
 
+  @httpoison Application.get_env(:pleroma, :httpoison)
+
   action_fallback(:errors)
 
   def create_app(conn, params) do
     with cs <- App.register_changeset(%App{}, params) |> IO.inspect(),
          {:ok, app} <- Repo.insert(cs) |> IO.inspect() do
       res = %{
-        id: app.id,
+        id: app.id |> to_string,
+        name: app.client_name,
         client_id: app.client_id,
-        client_secret: app.client_secret
+        client_secret: app.client_secret,
+        redirect_uri: app.redirect_uris,
+        website: app.website
       }
 
       json(conn, res)
@@ -653,12 +658,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
     fetched =
       if Regex.match?(~r/https?:/, query) do
-        with {:ok, activities} <- OStatus.fetch_activity_from_url(query) do
-          activities
-          |> Enum.filter(fn
-            %{data: %{"type" => "Create"}} -> true
-            _ -> false
-          end)
+        with {:ok, object} <- ActivityPub.fetch_object_from_id(query) do
+          [Activity.get_create_activity_by_object_ap_id(object.data["id"])]
         else
           _e -> []
         end
@@ -705,12 +706,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
     fetched =
       if Regex.match?(~r/https?:/, query) do
-        with {:ok, activities} <- OStatus.fetch_activity_from_url(query) do
-          activities
-          |> Enum.filter(fn
-            %{data: %{"type" => "Create"}} -> true
-            _ -> false
-          end)
+        with {:ok, object} <- ActivityPub.fetch_object_from_id(query) do
+          [Activity.get_create_activity_by_object_ap_id(object.data["id"])]
         else
           _e -> []
         end
@@ -1097,4 +1094,45 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     |> put_status(500)
     |> json("Something went wrong")
   end
+
+  @suggestions Application.get_env(:pleroma, :suggestions)
+
+  def suggestions(%{assigns: %{user: user}} = conn, _) do
+    if Keyword.get(@suggestions, :enabled, false) do
+      api = Keyword.get(@suggestions, :third_party_engine, "")
+      timeout = Keyword.get(@suggestions, :timeout, 5000)
+
+      host =
+        Application.get_env(:pleroma, Pleroma.Web.Endpoint)
+        |> Keyword.get(:url)
+        |> Keyword.get(:host)
+
+      user = user.nickname
+      url = String.replace(api, "{{host}}", host) |> String.replace("{{user}}", user)
+
+      with {:ok, %{status_code: 200, body: body}} <-
+             @httpoison.get(url, [], timeout: timeout, recv_timeout: timeout),
+           {:ok, data} <- Jason.decode(body) do
+        data2 =
+          Enum.slice(data, 0, 40)
+          |> Enum.map(fn x ->
+            Map.put(
+              x,
+              "id",
+              case User.get_or_fetch(x["acct"]) do
+                %{id: id} -> id
+                _ -> 0
+              end
+            )
+          end)
+
+        conn
+        |> json(data2)
+      else
+        e -> Logger.error("Could not retrieve suggestions at fetch #{url}, #{inspect(e)}")
+      end
+    else
+      json(conn, [])
+    end
+  end
 end
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index f33d615cf..d9edcae7f 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -14,6 +14,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
     header = User.banner_url(user) |> MediaProxy.url()
     user_info = User.user_info(user)
 
+    emojis =
+      (user.info["source_data"]["tag"] || [])
+      |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
+      |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} ->
+        %{
+          "shortcode" => String.trim(name, ":"),
+          "url" => MediaProxy.url(url),
+          "static_url" => MediaProxy.url(url),
+          "visible_in_picker" => false
+        }
+      end)
+
     %{
       id: to_string(user.id),
       username: hd(String.split(user.nickname, "@")),
@@ -24,13 +36,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       followers_count: user_info.follower_count,
       following_count: user_info.following_count,
       statuses_count: user_info.note_count,
-      note: user.bio || "",
+      note: HtmlSanitizeEx.basic_html(user.bio) || "",
       url: user.ap_id,
       avatar: image,
       avatar_static: image,
       header: header,
       header_static: header,
-      emojis: [],
+      emojis: emojis,
       fields: [],
       source: %{
         note: "",
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 5dbd59dd9..6962aa54f 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -99,8 +99,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || [])
     favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || [])
 
-    attachments =
-      render_many(object["attachment"] || [], StatusView, "attachment.json", as: :attachment)
+    attachment_data = object["attachment"] || []
+    attachment_data = attachment_data ++ if object["type"] == "Video", do: [object], else: []
+    attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)
 
     created_at = Utils.to_masto_date(object["published"])
 
@@ -151,7 +152,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   end
 
   def render("attachment.json", %{attachment: attachment}) do
-    [%{"mediaType" => media_type, "href" => href} | _] = attachment["url"]
+    [attachment_url | _] = attachment["url"]
+    media_type = attachment_url["mediaType"] || attachment_url["mimeType"]
+    href = attachment_url["href"]
 
     type =
       cond do
@@ -208,6 +211,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     end
   end
 
+  def render_content(%{"type" => "Video"} = object) do
+    name = object["name"]
+
+    content =
+      if !!name and name != "" do
+        "<p><a href=\"#{object["id"]}\">#{name}</a></p>#{object["content"]}"
+      else
+        object["content"]
+      end
+
+    HtmlSanitizeEx.basic_html(content)
+  end
+
   def render_content(%{"type" => "Article"} = object) do
     summary = object["name"]
 
diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
index 7c67bbf1c..2fab60274 100644
--- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
+++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
@@ -21,6 +21,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
   def nodeinfo(conn, %{"version" => "2.0"}) do
     instance = Application.get_env(:pleroma, :instance)
     media_proxy = Application.get_env(:pleroma, :media_proxy)
+    suggestions = Application.get_env(:pleroma, :suggestions)
     stats = Stats.get_stats()
 
     response = %{
@@ -45,7 +46,13 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
         nodeName: Keyword.get(instance, :name),
         nodeDescription: Keyword.get(instance, :description),
         mediaProxy: Keyword.get(media_proxy, :enabled),
-        private: !Keyword.get(instance, :public, true)
+        private: !Keyword.get(instance, :public, true),
+        suggestions: %{
+          enabled: Keyword.get(suggestions, :enabled, false),
+          thirdPartyEngine: Keyword.get(suggestions, :third_party_engine, ""),
+          timeout: Keyword.get(suggestions, :timeout, 5000),
+          web: Keyword.get(suggestions, :web, "")
+        }
       }
     }
 
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index eb9860864..7b4c81e00 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -142,6 +142,8 @@ defmodule Pleroma.Web.Router do
     get("/domain_blocks", MastodonAPIController, :domain_blocks)
     post("/domain_blocks", MastodonAPIController, :block_domain)
     delete("/domain_blocks", MastodonAPIController, :unblock_domain)
+
+    get("/suggestions", MastodonAPIController, :suggestions)
   end
 
   scope "/api/web", Pleroma.Web.MastodonAPI do
@@ -203,9 +205,7 @@ defmodule Pleroma.Web.Router do
     get("/statuses/show/:id", TwitterAPI.Controller, :fetch_status)
     get("/statusnet/conversation/:id", TwitterAPI.Controller, :fetch_conversation)
 
-    if @registrations_open do
-      post("/account/register", TwitterAPI.Controller, :register)
-    end
+    post("/account/register", TwitterAPI.Controller, :register)
 
     get("/search", TwitterAPI.Controller, :search)
     get("/statusnet/tags/timeline/:tag", TwitterAPI.Controller, :public_and_external_timeline)
@@ -368,6 +368,7 @@ defmodule Pleroma.Web.Router do
   end
 
   scope "/", Fallback do
+    get("/registration/:token", RedirectController, :registration_page)
     get("/*path", RedirectController, :redirector)
   end
 end
@@ -382,4 +383,8 @@ defmodule Fallback.RedirectController do
       |> send_file(200, "priv/static/index.html")
     end
   end
+
+  def registration_page(conn, params) do
+    redirector(conn, params)
+  end
 end
diff --git a/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex b/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex
index 6a00b9e2c..0862412ea 100644
--- a/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex
+++ b/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex
@@ -19,7 +19,7 @@
 <script id='initial-state' type='application/json'><%= raw @initial_state %></script>
 <script src="/packs/application.js"></script>
 </head>
-<body class='app-body no-reduce-motion'>
+<body class='app-body no-reduce-motion system-font'>
   <div class='app-holder' data-props='{&quot;locale&quot;:&quot;en&quot;}' id='mastodon'>
   </div>
 </body>
diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
index 47fc79350..d1ecebf61 100644
--- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex
+++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
@@ -99,6 +99,10 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
       conn
       |> render("followed.html", %{error: false})
     else
+      # Was already following user
+      {:error, "Could not follow user:" <> _rest} ->
+        render(conn, "followed.html", %{error: false})
+
       _e ->
         conn
         |> render("follow_login.html", %{
@@ -117,6 +121,11 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
       conn
       |> render("followed.html", %{error: false})
     else
+      # Was already following user
+      {:error, "Could not follow user:" <> _rest} ->
+        conn
+        |> render("followed.html", %{error: false})
+
       e ->
         Logger.debug("Remote follow failed with error #{inspect(e)}")
 
@@ -163,10 +172,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
               redirectRootLogin: Keyword.get(@instance_fe, :redirect_root_login),
               chatDisabled: !Keyword.get(@instance_chat, :enabled),
               showInstanceSpecificPanel: Keyword.get(@instance_fe, :show_instance_panel),
-              showWhoToFollowPanel: Keyword.get(@instance_fe, :show_who_to_follow_panel),
               scopeOptionsEnabled: Keyword.get(@instance_fe, :scope_options_enabled),
-              whoToFollowProvider: Keyword.get(@instance_fe, :who_to_follow_provider),
-              whoToFollowLink: Keyword.get(@instance_fe, :who_to_follow_link)
+              collapseMessageWithSubject:
+                Keyword.get(@instance_fe, :collapse_message_with_subject)
             }
           }
         })
diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
index 26bfb79af..9abea59a7 100644
--- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex
+++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
@@ -170,6 +170,15 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
       HtmlSanitizeEx.basic_html(content)
       |> Formatter.emojify(object["emoji"])
 
+    video =
+      if object["type"] == "Video" do
+        vid = [object]
+      else
+        []
+      end
+
+    attachments = (object["attachment"] || []) ++ video
+
     %{
       "id" => activity.id,
       "uri" => activity.data["object"]["id"],
@@ -181,7 +190,7 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
       "created_at" => created_at,
       "in_reply_to_status_id" => object["inReplyToStatusId"],
       "statusnet_conversation_id" => conversation_id,
-      "attachments" => (object["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts),
+      "attachments" => attachments |> ObjectRepresenter.enum_to_list(opts),
       "attentions" => attentions,
       "fave_num" => like_count,
       "repeat_num" => announcement_count,
diff --git a/lib/pleroma/web/twitter_api/representers/object_representer.ex b/lib/pleroma/web/twitter_api/representers/object_representer.ex
index 9af8a1691..6aa794a59 100644
--- a/lib/pleroma/web/twitter_api/representers/object_representer.ex
+++ b/lib/pleroma/web/twitter_api/representers/object_representer.ex
@@ -7,18 +7,20 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter do
 
     %{
       url: url["href"] |> Pleroma.Web.MediaProxy.url(),
-      mimetype: url["mediaType"],
+      mimetype: url["mediaType"] || url["mimeType"],
       id: data["uuid"],
-      oembed: false
+      oembed: false,
+      description: data["name"]
     }
   end
 
   def to_map(%Object{data: %{"url" => url} = data}, _opts) when is_binary(url) do
     %{
       url: url |> Pleroma.Web.MediaProxy.url(),
-      mimetype: data["mediaType"],
+      mimetype: data["mediaType"] || url["mimeType"],
       id: data["uuid"],
-      oembed: false
+      oembed: false,
+      description: data["name"]
     }
   end
 
diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex
index c23b3c2c4..dbad08e66 100644
--- a/lib/pleroma/web/twitter_api/twitter_api.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api.ex
@@ -1,11 +1,13 @@
 defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
-  alias Pleroma.{User, Activity, Repo, Object}
+  alias Pleroma.{UserInviteToken, User, Activity, Repo, Object}
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.TwitterAPI.UserView
   alias Pleroma.Web.{OStatus, CommonAPI}
   import Ecto.Query
 
+  @instance Application.get_env(:pleroma, :instance)
   @httpoison Application.get_env(:pleroma, :httpoison)
+  @registrations_open Keyword.get(@instance, :registrations_open)
 
   def create_status(%User{} = user, %{"status" => _} = data) do
     CommonAPI.post(user, data)
@@ -120,6 +122,8 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
   end
 
   def register_user(params) do
+    tokenString = params["token"]
+
     params = %{
       nickname: params["nickname"],
       name: params["fullname"],
@@ -129,17 +133,33 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
       password_confirmation: params["confirm"]
     }
 
-    changeset = User.register_changeset(%User{}, params)
+    # no need to query DB if registration is open
+    token =
+      unless @registrations_open || is_nil(tokenString) do
+        Repo.get_by(UserInviteToken, %{token: tokenString})
+      end
 
-    with {:ok, user} <- Repo.insert(changeset) do
-      {:ok, user}
-    else
-      {:error, changeset} ->
-        errors =
-          Ecto.Changeset.traverse_errors(changeset, fn {msg, _opts} -> msg end)
-          |> Jason.encode!()
+    cond do
+      @registrations_open || (!is_nil(token) && !token.used) ->
+        changeset = User.register_changeset(%User{}, params)
+
+        with {:ok, user} <- Repo.insert(changeset) do
+          !@registrations_open && UserInviteToken.mark_as_used(token.token)
+          {:ok, user}
+        else
+          {:error, changeset} ->
+            errors =
+              Ecto.Changeset.traverse_errors(changeset, fn {msg, _opts} -> msg end)
+              |> Jason.encode!()
+
+            {:error, %{error: errors}}
+        end
+
+      !@registrations_open && is_nil(token) ->
+        {:error, "Invalid token"}
 
-        {:error, %{error: errors}}
+      !@registrations_open && token.used ->
+        {:error, "Expired token"}
     end
   end
 
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index 65e67396b..b3a56b27e 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -1,7 +1,9 @@
 defmodule Pleroma.Web.TwitterAPI.Controller do
   use Pleroma.Web, :controller
+  alias Pleroma.Formatter
   alias Pleroma.Web.TwitterAPI.{TwitterAPI, UserView, ActivityView, NotificationView}
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.CommonAPI.Utils, as: CommonUtils
   alias Pleroma.{Repo, Activity, User, Notification}
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
@@ -411,8 +413,18 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   def update_profile(%{assigns: %{user: user}} = conn, params) do
     params =
       if bio = params["description"] do
-        bio_brs = Regex.replace(~r/\r?\n/, bio, "<br>")
-        Map.put(params, "bio", bio_brs)
+        mentions = Formatter.parse_mentions(bio)
+        tags = Formatter.parse_tags(bio)
+
+        emoji =
+          (user.info["source_data"]["tag"] || [])
+          |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
+          |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} ->
+            {String.trim(name, ":"), url}
+          end)
+
+        bio_html = CommonUtils.format_input(bio, mentions, tags)
+        Map.put(params, "bio", bio_html |> Formatter.emojify(emoji))
       else
         params
       end
diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex
index 9c8460378..32f93153d 100644
--- a/lib/pleroma/web/twitter_api/views/user_view.ex
+++ b/lib/pleroma/web/twitter_api/views/user_view.ex
@@ -1,6 +1,7 @@
 defmodule Pleroma.Web.TwitterAPI.UserView do
   use Pleroma.Web, :view
   alias Pleroma.User
+  alias Pleroma.Formatter
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MediaProxy
 
@@ -28,9 +29,18 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
 
     user_info = User.get_cached_user_info(user)
 
+    emoji =
+      (user.info["source_data"]["tag"] || [])
+      |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
+      |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} ->
+        {String.trim(name, ":"), url}
+      end)
+
     data = %{
       "created_at" => user.inserted_at |> Utils.format_naive_asctime(),
-      "description" => HtmlSanitizeEx.strip_tags(user.bio),
+      "description" =>
+        HtmlSanitizeEx.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
+      "description_html" => HtmlSanitizeEx.basic_html(user.bio),
       "favourites_count" => 0,
       "followers_count" => user_info[:follower_count],
       "following" => following,
@@ -39,6 +49,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
       "friends_count" => user_info[:following_count],
       "id" => user.id,
       "name" => user.name,
+      "name_html" => HtmlSanitizeEx.strip_tags(user.name) |> Formatter.emojify(emoji),
       "profile_image_url" => image,
       "profile_image_url_https" => image,
       "profile_image_url_profile_size" => image,