Merge branch 'develop' into feature/compat/push-subscriptions

# Conflicts:
#	lib/mix/tasks/sample_config.eex
#	lib/pleroma/web/twitter_api/controllers/util_controller.ex
#	mix.exs
#	mix.lock

6 files changed, 145 insertions, 46 deletions

diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
index f06020a3e..092779010 100644
--- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex
+++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.CommonAPI
   alias Comeonin.Pbkdf2
-  alias Pleroma.Formatter
+  alias Pleroma.{Formatter, Emoji}
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.{Repo, PasswordResetToken, User}
 
@@ -134,19 +134,20 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
     end
   end
 
-  @instance Application.get_env(:pleroma, :instance)
-  @instance_fe Application.get_env(:pleroma, :fe)
-  @instance_chat Application.get_env(:pleroma, :chat)
   def config(conn, _params) do
+    instance = Pleroma.Config.get(:instance)
+    instance_fe = Pleroma.Config.get(:fe)
+    instance_chat = Pleroma.Config.get(:chat)
+
     case get_format(conn) do
       "xml" ->
         response = """
         <config>
           <site>
-            <name>#{Keyword.get(@instance, :name)}</name>
+            <name>#{Keyword.get(instance, :name)}</name>
             <site>#{Web.base_url()}</site>
-            <textlimit>#{Keyword.get(@instance, :limit)}</textlimit>
-            <closed>#{!Keyword.get(@instance, :registrations_open)}</closed>
+            <textlimit>#{Keyword.get(instance, :limit)}</textlimit>
+            <closed>#{!Keyword.get(instance, :registrations_open)}</closed>
           </site>
         </config>
         """
@@ -160,30 +161,33 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
           Keyword.get(Application.get_env(:web_push_encryption, :vapid_details), :public_key)
 
         data = %{
-          name: Keyword.get(@instance, :name),
-          description: Keyword.get(@instance, :description),
+          name: Keyword.get(instance, :name),
+          description: Keyword.get(instance, :description),
           server: Web.base_url(),
-          textlimit: to_string(Keyword.get(@instance, :limit)),
-          closed: if(Keyword.get(@instance, :registrations_open), do: "0", else: "1"),
-          private: if(Keyword.get(@instance, :public, true), do: "0", else: "1"),
+          textlimit: to_string(Keyword.get(instance, :limit)),
+          closed: if(Keyword.get(instance, :registrations_open), do: "0", else: "1"),
+          private: if(Keyword.get(instance, :public, true), do: "0", else: "1"),
           vapidPublicKey: vapid_public_key
         }
 
         pleroma_fe = %{
-          theme: Keyword.get(@instance_fe, :theme),
-          background: Keyword.get(@instance_fe, :background),
-          logo: Keyword.get(@instance_fe, :logo),
-          logoMask: Keyword.get(@instance_fe, :logo_mask),
-          logoMargin: Keyword.get(@instance_fe, :logo_margin),
-          redirectRootNoLogin: Keyword.get(@instance_fe, :redirect_root_no_login),
-          redirectRootLogin: Keyword.get(@instance_fe, :redirect_root_login),
-          chatDisabled: !Keyword.get(@instance_chat, :enabled),
-          showInstanceSpecificPanel: Keyword.get(@instance_fe, :show_instance_panel),
-          scopeOptionsEnabled: Keyword.get(@instance_fe, :scope_options_enabled),
-          collapseMessageWithSubject: Keyword.get(@instance_fe, :collapse_message_with_subject)
+          theme: Keyword.get(instance_fe, :theme),
+          background: Keyword.get(instance_fe, :background),
+          logo: Keyword.get(instance_fe, :logo),
+          logoMask: Keyword.get(instance_fe, :logo_mask),
+          logoMargin: Keyword.get(instance_fe, :logo_margin),
+          redirectRootNoLogin: Keyword.get(instance_fe, :redirect_root_no_login),
+          redirectRootLogin: Keyword.get(instance_fe, :redirect_root_login),
+          chatDisabled: !Keyword.get(instance_chat, :enabled),
+          showInstanceSpecificPanel: Keyword.get(instance_fe, :show_instance_panel),
+          scopeOptionsEnabled: Keyword.get(instance_fe, :scope_options_enabled),
+          formattingOptionsEnabled: Keyword.get(instance_fe, :formatting_options_enabled),
+          collapseMessageWithSubject: Keyword.get(instance_fe, :collapse_message_with_subject),
+          hidePostStats: Keyword.get(instance_fe, :hide_post_stats),
+          hideUserStats: Keyword.get(instance_fe, :hide_user_stats)
         }
 
-        managed_config = Keyword.get(@instance, :managed_config)
+        managed_config = Keyword.get(instance, :managed_config)
 
         data =
           if managed_config do
@@ -197,7 +201,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   end
 
   def version(conn, _params) do
-    version = Keyword.get(@instance, :version)
+    version = Pleroma.Application.named_version()
 
     case get_format(conn) do
       "xml" ->
@@ -213,7 +217,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   end
 
   def emoji(conn, _params) do
-    json(conn, Enum.into(Formatter.get_custom_emoji(), %{}))
+    json(conn, Enum.into(Emoji.get_all(), %{}))
   end
 
   def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do
@@ -226,7 +230,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
       |> Enum.map(fn account ->
         with %User{} = follower <- User.get_cached_by_ap_id(user.ap_id),
              %User{} = followed <- User.get_or_fetch(account),
-             {:ok, follower} <- User.follow(follower, followed) do
+             {:ok, follower} <- User.maybe_direct_follow(follower, followed) do
           ActivityPub.follow(follower, followed)
         else
           err -> Logger.debug("follow_import: following #{account} failed with #{inspect(err)}")
diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
index b21bbb205..fbd33f07e 100644
--- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex
+++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
@@ -180,6 +180,10 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
 
     attachments = (object["attachment"] || []) ++ video
 
+    reply_parent = Activity.get_in_reply_to_activity(activity)
+
+    reply_user = reply_parent && User.get_cached_by_ap_id(reply_parent.actor)
+
     %{
       "id" => activity.id,
       "uri" => activity.data["object"]["id"],
@@ -190,6 +194,10 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
       "is_post_verb" => true,
       "created_at" => created_at,
       "in_reply_to_status_id" => object["inReplyToStatusId"],
+      "in_reply_to_screen_name" => reply_user && reply_user.nickname,
+      "in_reply_to_profileurl" => User.profile_url(reply_user),
+      "in_reply_to_ostatus_uri" => reply_user && reply_user.ap_id,
+      "in_reply_to_user_id" => reply_user && reply_user.id,
       "statusnet_conversation_id" => conversation_id,
       "attachments" => attachments |> ObjectRepresenter.enum_to_list(opts),
       "attentions" => attentions,
diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex
index dbad08e66..5bfb83b1e 100644
--- a/lib/pleroma/web/twitter_api/twitter_api.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api.ex
@@ -3,11 +3,10 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.TwitterAPI.UserView
   alias Pleroma.Web.{OStatus, CommonAPI}
+  alias Pleroma.Web.MediaProxy
   import Ecto.Query
 
-  @instance Application.get_env(:pleroma, :instance)
   @httpoison Application.get_env(:pleroma, :httpoison)
-  @registrations_open Keyword.get(@instance, :registrations_open)
 
   def create_status(%User{} = user, %{"status" => _} = data) do
     CommonAPI.post(user, data)
@@ -23,7 +22,13 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
   def follow(%User{} = follower, params) do
     with {:ok, %User{} = followed} <- get_user(params),
          {:ok, follower} <- User.maybe_direct_follow(follower, followed),
-         {:ok, activity} <- ActivityPub.follow(follower, followed) do
+         {:ok, activity} <- ActivityPub.follow(follower, followed),
+         {:ok, follower, followed} <-
+           User.wait_and_refresh(
+             Pleroma.Config.get([:activitypub, :follow_handshake_timeout]),
+             follower,
+             followed
+           ) do
       {:ok, follower, followed, activity}
     else
       err -> err
@@ -92,7 +97,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
     {:ok, object} = ActivityPub.upload(file)
 
     url = List.first(object.data["url"])
-    href = url["href"]
+    href = url["href"] |> MediaProxy.url()
     type = url["mediaType"]
 
     case format do
@@ -133,18 +138,20 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
       password_confirmation: params["confirm"]
     }
 
+    registrations_open = Pleroma.Config.get([:instance, :registrations_open])
+
     # no need to query DB if registration is open
     token =
-      unless @registrations_open || is_nil(tokenString) do
+      unless registrations_open || is_nil(tokenString) do
         Repo.get_by(UserInviteToken, %{token: tokenString})
       end
 
     cond do
-      @registrations_open || (!is_nil(token) && !token.used) ->
+      registrations_open || (!is_nil(token) && !token.used) ->
         changeset = User.register_changeset(%User{}, params)
 
         with {:ok, user} <- Repo.insert(changeset) do
-          !@registrations_open && UserInviteToken.mark_as_used(token.token)
+          !registrations_open && UserInviteToken.mark_as_used(token.token)
           {:ok, user}
         else
           {:error, changeset} ->
@@ -155,10 +162,10 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
             {:error, %{error: errors}}
         end
 
-      !@registrations_open && is_nil(token) ->
+      !registrations_open && is_nil(token) ->
         {:error, "Invalid token"}
 
-      !@registrations_open && token.used ->
+      !registrations_open && token.used ->
         {:error, "Expired token"}
     end
   end
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index cd2bb5b57..cd0e2121c 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -11,6 +11,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
   require Logger
 
+  plug(:only_if_public_instance when action in [:public_timeline, :public_and_external_timeline])
   action_fallback(:errors)
 
   def verify_credentials(%{assigns: %{user: user}} = conn, _params) do
@@ -79,7 +80,9 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
       |> Map.put("blocking_user", user)
       |> Map.put("user", user)
 
-    activities = ActivityPub.fetch_activities([user.ap_id | user.following], params)
+    activities =
+      ActivityPub.fetch_activities([user.ap_id | user.following], params)
+      |> ActivityPub.contain_timeline(user)
 
     conn
     |> render(ActivityView, "index.json", %{activities: activities, for: user})
@@ -123,6 +126,19 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
     |> render(ActivityView, "index.json", %{activities: activities, for: user})
   end
 
+  def dm_timeline(%{assigns: %{user: user}} = conn, params) do
+    query =
+      ActivityPub.fetch_activities_query(
+        [user.ap_id],
+        Map.merge(params, %{"type" => "Create", "user" => user, visibility: "direct"})
+      )
+
+    activities = Repo.all(query)
+
+    conn
+    |> render(ActivityView, "index.json", %{activities: activities, for: user})
+  end
+
   def notifications(%{assigns: %{user: user}} = conn, params) do
     notifications = Notification.for_user(user, params)
 
@@ -130,6 +146,19 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
     |> render(NotificationView, "notification.json", %{notifications: notifications, for: user})
   end
 
+  def notifications_read(%{assigns: %{user: user}} = conn, %{"latest_id" => latest_id} = params) do
+    Notification.set_read_up_to(user, latest_id)
+
+    notifications = Notification.for_user(user, params)
+
+    conn
+    |> render(NotificationView, "notification.json", %{notifications: notifications, for: user})
+  end
+
+  def notifications_read(%{assigns: %{user: user}} = conn, _) do
+    bad_request_reply(conn, "You need to specify latest_id")
+  end
+
   def follow(%{assigns: %{user: user}} = conn, params) do
     case TwitterAPI.follow(user, params) do
       {:ok, user, followed, _activity} ->
@@ -261,7 +290,11 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   end
 
   def update_avatar(%{assigns: %{user: user}} = conn, params) do
-    {:ok, object} = ActivityPub.upload(params)
+    upload_limit =
+      Application.get_env(:pleroma, :instance)
+      |> Keyword.fetch(:avatar_upload_limit)
+
+    {:ok, object} = ActivityPub.upload(params, upload_limit)
     change = Changeset.change(user, %{avatar: object.data})
     {:ok, user} = User.update_and_set_cache(change)
     CommonAPI.update(user)
@@ -270,7 +303,11 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   end
 
   def update_banner(%{assigns: %{user: user}} = conn, params) do
-    with {:ok, object} <- ActivityPub.upload(%{"img" => params["banner"]}),
+    upload_limit =
+      Application.get_env(:pleroma, :instance)
+      |> Keyword.fetch(:banner_upload_limit)
+
+    with {:ok, object} <- ActivityPub.upload(%{"img" => params["banner"]}, upload_limit),
          new_info <- Map.put(user.info, "banner", object.data),
          change <- User.info_changeset(user, %{info: new_info}),
          {:ok, user} <- User.update_and_set_cache(change) do
@@ -284,7 +321,11 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   end
 
   def update_background(%{assigns: %{user: user}} = conn, params) do
-    with {:ok, object} <- ActivityPub.upload(params),
+    upload_limit =
+      Application.get_env(:pleroma, :instance)
+      |> Keyword.fetch(:background_upload_limit)
+
+    with {:ok, object} <- ActivityPub.upload(params, upload_limit),
          new_info <- Map.put(user.info, "background", object.data),
          change <- User.info_changeset(user, %{info: new_info}),
          {:ok, _user} <- User.update_and_set_cache(change) do
@@ -423,7 +464,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
             {String.trim(name, ":"), url}
           end)
 
-        bio_html = CommonUtils.format_input(bio, mentions, tags)
+        bio_html = CommonUtils.format_input(bio, mentions, tags, "text/plain")
         Map.put(params, "bio", bio_html |> Formatter.emojify(emoji))
       else
         params
@@ -488,6 +529,13 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
     |> render(ActivityView, "index.json", %{activities: activities, for: user})
   end
 
+  def search_user(%{assigns: %{user: user}} = conn, %{"query" => query}) do
+    users = User.search(query, true)
+
+    conn
+    |> render(UserView, "index.json", %{users: users, for: user})
+  end
+
   defp bad_request_reply(conn, error_message) do
     json = error_json(conn, error_message)
     json_reply(conn, 400, json)
@@ -504,6 +552,18 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
     json_reply(conn, 403, json)
   end
 
+  def only_if_public_instance(conn = %{conn: %{assigns: %{user: _user}}}, _), do: conn
+
+  def only_if_public_instance(conn, _) do
+    if Keyword.get(Application.get_env(:pleroma, :instance), :public) do
+      conn
+    else
+      conn
+      |> forbidden_json_reply("Invalid credentials.")
+      |> halt()
+    end
+  end
+
   defp error_json(conn, error_message) do
     %{"error" => error_message, "request" => conn.request_path} |> Jason.encode!()
   end
diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex
index b9fd062d6..83e8fb765 100644
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@@ -236,6 +236,10 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
       HTML.filter_tags(content, User.html_filter_policy(opts[:for]))
       |> Formatter.emojify(object["emoji"])
 
+    reply_parent = Activity.get_in_reply_to_activity(activity)
+
+    reply_user = reply_parent && User.get_cached_by_ap_id(reply_parent.actor)
+
     %{
       "id" => activity.id,
       "uri" => activity.data["object"]["id"],
@@ -246,6 +250,10 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
       "is_post_verb" => true,
       "created_at" => created_at,
       "in_reply_to_status_id" => object["inReplyToStatusId"],
+      "in_reply_to_screen_name" => reply_user && reply_user.nickname,
+      "in_reply_to_profileurl" => User.profile_url(reply_user),
+      "in_reply_to_ostatus_uri" => reply_user && reply_user.ap_id,
+      "in_reply_to_user_id" => reply_user && reply_user.id,
       "statusnet_conversation_id" => conversation_id,
       "attachments" => (object["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts),
       "attentions" => attentions,
@@ -275,11 +283,11 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
     {summary, content}
   end
 
-  def render_content(%{"type" => "Article"} = object) do
+  def render_content(%{"type" => object_type} = object) when object_type in ["Article", "Page"] do
     summary = object["name"] || object["summary"]
 
     content =
-      if !!summary and summary != "" do
+      if !!summary and summary != "" and is_bitstring(object["url"]) do
         "<p><a href=\"#{object["url"]}\">#{summary}</a></p>#{object["content"]}"
       else
         object["content"]
diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex
index cfbaef4c9..a100a1127 100644
--- a/lib/pleroma/web/twitter_api/views/user_view.ex
+++ b/lib/pleroma/web/twitter_api/views/user_view.ex
@@ -37,6 +37,13 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
         {String.trim(name, ":"), url}
       end)
 
+    # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``.
+    # For example: [{"name": "Pronoun", "value": "she/her"}, …]
+    fields =
+      (user.info["source_data"]["attachment"] || [])
+      |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end)
+      |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end)
+
     data = %{
       "created_at" => user.inserted_at |> Utils.format_naive_asctime(),
       "description" => HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
@@ -48,8 +55,12 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
       "statusnet_blocking" => statusnet_blocking,
       "friends_count" => user_info[:following_count],
       "id" => user.id,
-      "name" => user.name,
-      "name_html" => HTML.strip_tags(user.name) |> Formatter.emojify(emoji),
+      "name" => user.name || user.nickname,
+      "name_html" =>
+        if(user.name,
+          do: HTML.strip_tags(user.name) |> Formatter.emojify(emoji),
+          else: user.nickname
+        ),
       "profile_image_url" => image,
       "profile_image_url_https" => image,
       "profile_image_url_profile_size" => image,
@@ -65,7 +76,8 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
       "is_local" => user.local,
       "locked" => !!user.info["locked"],
       "default_scope" => user.info["default_scope"] || "public",
-      "no_rich_text" => user.info["no_rich_text"] || false
+      "no_rich_text" => user.info["no_rich_text"] || false,
+      "fields" => fields
     }
 
     if assigns[:token] do