Merge branch 'develop' into feature/activitypub

34 files changed, 449 insertions, 58 deletions

diff --git a/lib/mix/tasks/generate_config.ex b/lib/mix/tasks/generate_config.ex
index f20f93e4d..2d962124f 100644
--- a/lib/mix/tasks/generate_config.ex
+++ b/lib/mix/tasks/generate_config.ex
@@ -8,11 +8,20 @@ defmodule Mix.Tasks.GenerateConfig do
     domain = IO.gets("What is your domain name? (e.g. pleroma.soykaf.com): ") |> String.trim
     name = IO.gets("What is the name of your instance? (e.g. Pleroma/Soykaf): ") |> String.trim
     email = IO.gets("What's your admin email address: ") |> String.trim
+    mediaproxy = IO.gets("Do you want to activate the mediaproxy? (y/N): ")
+    |> String.trim()
+    |> String.downcase()
+    |> String.starts_with?("y")
+    proxy_url = if mediaproxy do
+      IO.gets("What is the mediaproxy's URL? (e.g. https://cache.example.com): ") |> String.trim
+    else
+      "https://cache.example.com"
+    end
     secret =  :crypto.strong_rand_bytes(64) |> Base.encode64 |> binary_part(0, 64)
     dbpass =  :crypto.strong_rand_bytes(64) |> Base.encode64 |> binary_part(0, 64)
 
     resultSql = EEx.eval_file("lib/mix/tasks/sample_psql.eex", [dbpass: dbpass])
-    result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret, dbpass: dbpass])
+    result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret, mediaproxy: mediaproxy, proxy_url: proxy_url, dbpass: dbpass])
 
     IO.puts("\nWriting config to config/generated_config.exs.\n\nCheck it and configure your database, then copy it to either config/dev.secret.exs or config/prod.secret.exs")
     File.write("config/generated_config.exs", result)
diff --git a/lib/mix/tasks/sample_config.eex b/lib/mix/tasks/sample_config.eex
index 85a7c554e..9330fae2d 100644
--- a/lib/mix/tasks/sample_config.eex
+++ b/lib/mix/tasks/sample_config.eex
@@ -10,6 +10,11 @@ config :pleroma, :instance,
   limit: 5000,
   registrations_open: true
 
+config :pleroma, :media_proxy,
+  enabled: <%= mediaproxy %>,
+  redirect_on_failure: true,
+  base_url: "<%= proxy_url %>"
+
 # Configure your database
 config :pleroma, Pleroma.Repo,
   adapter: Ecto.Adapters.Postgres,
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 2969ca3c4..79b9dee9d 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -20,13 +20,18 @@ defmodule Pleroma.Application do
                          limit: 2500
                        ]]),
       worker(Pleroma.Web.Federator, []),
-      worker(Pleroma.Web.ChatChannel.ChatChannelState, []),
+      worker(Pleroma.Stats, []),
     ]
     ++ if Mix.env == :test, do: [], else: [worker(Pleroma.Web.Streamer, [])]
+    ++ if !chat_enabled(), do: [], else: [worker(Pleroma.Web.ChatChannel.ChatChannelState, [])]
 
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
     Supervisor.start_link(children, opts)
   end
+
+  defp chat_enabled do
+    Application.get_env(:pleroma, :chat, []) |> Keyword.get(:enabled)
+  end
 end
diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index c98db2d94..fdf91f56e 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -1,5 +1,6 @@
 defmodule Pleroma.Formatter do
   alias Pleroma.User
+  alias Pleroma.Web.MediaProxy
 
   @link_regex ~r/https?:\/\/[\w\.\/?=\-#%&@~\(\)]+[\w\/]/u
   def linkify(text) do
@@ -10,7 +11,7 @@ defmodule Pleroma.Formatter do
   def parse_tags(text, data \\ %{}) do
     Regex.scan(@tag_regex, text)
     |> Enum.map(fn (["#" <> tag = full_tag]) -> {full_tag, String.downcase(tag)} end)
-    |> (fn map -> if data["sensitive"], do: [{"#nsfw", "nsfw"}] ++ map, else: map end).()
+    |> (fn map -> if data["sensitive"] in [true, "True", "true", "1"], do: [{"#nsfw", "nsfw"}] ++ map, else: map end).()
   end
 
   def parse_mentions(text) do
@@ -103,13 +104,19 @@ defmodule Pleroma.Formatter do
     {finmoji, "/finmoji/128px/#{finmoji}-128.png"}
   end)
 
-  @emoji_from_file (with {:ok, file} <- File.read("config/emoji.txt") do
-                     file
-                     |> String.trim
-                     |> String.split("\n")
-                     |> Enum.map(fn(line) ->
-                       [name, file] = String.split(line, ", ")
-                       {name, file}
+  @emoji_from_file (with {:ok, default} <- File.read("config/emoji.txt") do
+                      custom =
+                        with {:ok, custom} <- File.read("config/custom_emoji.txt") do
+                          custom
+                        else
+                          _e -> ""
+                        end
+                      (default <> "\n" <> custom)
+                      |> String.trim()
+                      |> String.split(~r/\n+/)
+                      |> Enum.map(fn(line) ->
+                        [name, file] = String.split(line, ~r/,\s*/)
+                        {name, file}
                      end)
                     else
                       _ -> []
@@ -125,7 +132,7 @@ defmodule Pleroma.Formatter do
     end
 
     Enum.reduce(all_emoji, text, fn ({emoji, file}, text) ->
-      String.replace(text, ":#{emoji}:", "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{file}' />")
+      String.replace(text, ":#{emoji}:", "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{MediaProxy.url(file)}' />")
     end)
   end
 
diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex
new file mode 100644
index 000000000..8b8a82353
--- /dev/null
+++ b/lib/pleroma/http/http.ex
@@ -0,0 +1,14 @@
+
+defmodule Pleroma.HTTP do
+  use HTTPoison.Base
+
+  def process_request_options(options) do
+    config = Application.get_env(:pleroma, :http, [])
+    proxy = Keyword.get(config, :proxy_url, nil)
+    case proxy do
+      nil -> options
+      _ -> options ++ [proxy: proxy]
+    end
+  end
+
+end
diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex
new file mode 100644
index 000000000..737e9b62e
--- /dev/null
+++ b/lib/pleroma/stats.ex
@@ -0,0 +1,41 @@
+defmodule Pleroma.Stats do
+  import Ecto.Query
+  alias Pleroma.{User, Repo, Activity}
+
+  def start_link do
+    agent = Agent.start_link(fn -> {[], %{}} end, name: __MODULE__)
+    spawn(fn -> schedule_update() end)
+    agent
+  end
+
+  def get_stats do
+    Agent.get(__MODULE__, fn {_, stats} -> stats end)
+  end
+
+  def get_peers do
+    Agent.get(__MODULE__, fn {peers, _} -> peers end)
+  end
+
+  def schedule_update do
+    spawn(fn ->
+      Process.sleep(1000 * 60 * 60 * 1) # 1 hour
+      schedule_update()
+    end)
+    update_stats()
+  end
+
+  def update_stats do
+    peers = from(u in Pleroma.User,
+      select: fragment("distinct ?->'host'", u.info),
+      where: u.local != ^true)
+    |> Repo.all()
+    domain_count = Enum.count(peers)
+    status_query = from(u in User.local_user_query,
+      select: fragment("sum((?->>'note_count')::int)", u.info))
+    status_count = Repo.one(status_query) |> IO.inspect
+    user_count = Repo.aggregate(User.local_user_query, :count, :id)
+    Agent.update(__MODULE__, fn _ ->
+      {peers, %{domain_count: domain_count, status_count: status_count, user_count: user_count}}
+    end)
+  end
+end
diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
index 3567c6c88..c41617c68 100644
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Upload do
     File.cp!(file.path, result_file)
 
     # fix content type on some image uploads
-    content_type = if file.content_type == "application/octet-stream" do
+    content_type = if file.content_type in [nil, "application/octet-stream"] do
       get_content_type(file.path)
     else
       file.content_type
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 4580f30fb..e544d3772 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -29,14 +29,14 @@ defmodule Pleroma.User do
   def avatar_url(user) do
     case user.avatar do
       %{"url" => [%{"href" => href} | _]} -> href
-      _ -> "https://placehold.it/48x48"
+      _ -> "#{Web.base_url()}/images/avi.png"
     end
   end
 
   def banner_url(user) do
     case user.info["banner"] do
       %{"url" => [%{"href" => href} | _]} -> href
-      _ -> nil
+      _ -> "#{Web.base_url()}/images/banner.png"
     end
   end
 
diff --git a/lib/pleroma/web/channels/user_socket.ex b/lib/pleroma/web/channels/user_socket.ex
index 4a9bb8e22..f18b3cb80 100644
--- a/lib/pleroma/web/channels/user_socket.ex
+++ b/lib/pleroma/web/channels/user_socket.ex
@@ -5,7 +5,9 @@ defmodule Pleroma.Web.UserSocket do
 
   ## Channels
   # channel "room:*", Pleroma.Web.RoomChannel
-  channel "chat:*", Pleroma.Web.ChatChannel
+  if Application.get_env(:pleroma, :chat) |> Keyword.get(:enabled) do
+    channel "chat:*", Pleroma.Web.ChatChannel
+  end
 
   ## Transports
   transport :websocket, Phoenix.Transports.WebSocket
diff --git a/lib/pleroma/web/chat_channel.ex b/lib/pleroma/web/chat_channel.ex
index 268bef17d..48a3553bf 100644
--- a/lib/pleroma/web/chat_channel.ex
+++ b/lib/pleroma/web/chat_channel.ex
@@ -24,7 +24,6 @@ defmodule Pleroma.Web.ChatChannel do
 end
 
 defmodule Pleroma.Web.ChatChannel.ChatChannelState do
-  use Agent
   @max_messages 20
 
   def start_link do
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index e60dff7dc..2b359dd72 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -95,7 +95,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
     Enum.reduce(mentions, step_one, fn ({match, %User{ap_id: ap_id}, uuid}, text) ->
       short_match = String.split(match, "@") |> tl() |> hd()
-      String.replace(text, uuid, "<a href='#{ap_id}'>@#{short_match}</a>")
+      String.replace(text, uuid, "<span><a href='#{ap_id}'>@<span>#{short_match}</span></a></span>")
     end)
   end
 
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index b57cf3917..93b37dc74 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -1,7 +1,9 @@
 defmodule Pleroma.Web.Endpoint do
   use Phoenix.Endpoint, otp_app: :pleroma
 
-  socket "/socket", Pleroma.Web.UserSocket
+  if Application.get_env(:pleroma, :chat) |> Keyword.get(:enabled) do
+    socket "/socket", Pleroma.Web.UserSocket
+  end
   socket "/api/v1", Pleroma.Web.MastodonAPI.MastodonSocket
 
   # Serve at "/" the static files from "priv/static" directory.
@@ -12,7 +14,7 @@ defmodule Pleroma.Web.Endpoint do
     at: "/media", from: "uploads", gzip: false
   plug Plug.Static,
     at: "/", from: :pleroma,
-    only: ~w(index.html static finmoji emoji packs sounds sw.js)
+    only: ~w(index.html static finmoji emoji packs sounds images instance sw.js)
 
   # Code reloading can be explicitly enabled under the
   # :code_reloader configuration of your endpoint.
diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index b23ed5fcc..c9f9dc7a1 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -41,12 +41,12 @@ defmodule Pleroma.Web.Federator do
   def handle(:publish, activity) do
     Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end)
     with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do
-      Logger.debug(fn -> "Sending #{activity.data["id"]} out via websub" end)
-      Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity)
-
       {:ok, actor} = WebFinger.ensure_keys_present(actor)
       Logger.debug(fn -> "Sending #{activity.data["id"]} out via salmon" end)
       Pleroma.Web.Salmon.publish(actor, activity)
+
+      Logger.debug(fn -> "Sending #{activity.data["id"]} out via websub" end)
+      Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity)
     end
   end
 
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index e50f53ba4..e16a2a092 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1,6 +1,6 @@
 defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   use Pleroma.Web, :controller
-  alias Pleroma.{Repo, Activity, User, Notification}
+  alias Pleroma.{Repo, Activity, User, Notification, Stats}
   alias Pleroma.Web
   alias Pleroma.Web.MastodonAPI.{StatusView, AccountView, MastodonView}
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -93,7 +93,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   @instance Application.get_env(:pleroma, :instance)
 
   def masto_instance(conn, _params) do
-    user_count = Repo.aggregate(User.local_user_query, :count, :id)
     response = %{
       uri: Web.base_url,
       title: Keyword.get(@instance, :name),
@@ -103,17 +102,18 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       urls: %{
         streaming_api: String.replace(Web.base_url, ["http","https"], "wss")
       },
-      stats: %{
-        status_count: 2,
-        user_count: user_count,
-        domain_count: 3
-      },
+      stats: Stats.get_stats,
+      thumbnail: Web.base_url <> "/instance/thumbnail.jpeg",
       max_toot_chars: Keyword.get(@instance, :limit)
     }
 
     json(conn, response)
   end
 
+  def peers(conn, _params) do
+    json(conn, Stats.get_peers)
+  end
+
   defp mastodonized_emoji do
     Pleroma.Formatter.get_custom_emoji()
     |> Enum.map(fn {shortcode, relative_url} ->
@@ -162,7 +162,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   def public_timeline(%{assigns: %{user: user}} = conn, params) do
     params = params
     |> Map.put("type", ["Create", "Announce"])
-    |> Map.put("local_only", !!params["local"])
+    |> Map.put("local_only", params["local"] in [true, "True", "true", "1"])
     |> Map.put("blocking_user", user)
 
     activities = ActivityPub.fetch_public_activities(params)
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 02f1e60bb..d2a4dd366 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -3,20 +3,17 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
   alias Pleroma.User
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.CommonAPI.Utils
-
-  defp image_url(%{"url" => [ %{ "href" => href } | _ ]}), do: href
-  defp image_url(_), do: nil
+  alias Pleroma.Web.MediaProxy
 
   def render("accounts.json", %{users: users} = opts) do
     render_many(users, AccountView, "account.json", opts)
   end
 
   def render("account.json", %{user: user}) do
-    image = User.avatar_url(user)
+    image = User.avatar_url(user) |> MediaProxy.url()
+    header = User.banner_url(user) |> MediaProxy.url()
     user_info = User.user_info(user)
 
-    header = image_url(user.info["banner"]) || "https://placehold.it/700x335"
-
     %{
       id: to_string(user.id),
       username: hd(String.split(user.nickname, "@")),
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 5585a5605..64f315597 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -3,6 +3,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   alias Pleroma.Web.MastodonAPI.{AccountView, StatusView}
   alias Pleroma.{User, Activity}
   alias Pleroma.Web.CommonAPI.Utils
+  alias Pleroma.Web.MediaProxy
 
   def render("index.json", opts) do
     render_many(opts.activities, StatusView, "status.json", opts)
@@ -121,9 +122,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
     %{
       id: to_string(attachment["id"] || hash_id),
-      url: href,
+      url: MediaProxy.url(href),
       remote_url: href,
-      preview_url: href,
+      preview_url: MediaProxy.url(href),
       text_url: href,
       type: type
     }
diff --git a/lib/pleroma/web/media_proxy/controller.ex b/lib/pleroma/web/media_proxy/controller.ex
new file mode 100644
index 000000000..9327e7253
--- /dev/null
+++ b/lib/pleroma/web/media_proxy/controller.ex
@@ -0,0 +1,84 @@
+defmodule Pleroma.Web.MediaProxy.MediaProxyController do
+  use Pleroma.Web, :controller
+  require Logger
+
+  @httpoison Application.get_env(:pleroma, :httpoison)
+  
+  @max_body_length 25 * 1048576
+
+  @cache_control %{
+    default: "public, max-age=1209600",
+    error:   "public, must-revalidate, max-age=160",
+  }
+
+  def remote(conn, %{"sig" => sig, "url" => url}) do
+    config = Application.get_env(:pleroma, :media_proxy, [])
+    with \
+      true <- Keyword.get(config, :enabled, false),
+      {:ok, url} <- Pleroma.Web.MediaProxy.decode_url(sig, url),
+      {:ok, content_type, body} <- proxy_request(url)
+    do
+      conn
+      |> put_resp_content_type(content_type)
+      |> set_cache_header(:default)
+      |> send_resp(200, body)
+    else
+      false -> send_error(conn, 404)
+      {:error, :invalid_signature} -> send_error(conn, 403)
+      {:error, {:http, _, url}} -> redirect_or_error(conn, url, Keyword.get(config, :redirect_on_failure, true))
+    end
+  end
+
+  defp proxy_request(link) do
+    headers = [{"user-agent", "Pleroma/MediaProxy; #{Pleroma.Web.base_url()} <#{Application.get_env(:pleroma, :instance)[:email]}>"}]
+    options = @httpoison.process_request_options([:insecure, {:follow_redirect, true}])
+    with \
+      {:ok, 200, headers, client} <- :hackney.request(:get, link, headers, "", options),
+      headers = Enum.into(headers, Map.new),
+      {:ok, body} <- proxy_request_body(client),
+      content_type <- proxy_request_content_type(headers, body)
+    do
+      {:ok, content_type, body}
+    else
+      {:ok, status, _, _} ->
+        Logger.warn "MediaProxy: request failed, status #{status}, link: #{link}"
+        {:error, {:http, :bad_status, link}}
+      {:error, error} ->
+        Logger.warn "MediaProxy: request failed, error #{inspect error}, link: #{link}"
+        {:error, {:http, error, link}}
+    end
+  end
+
+  defp set_cache_header(conn, key) do
+    Plug.Conn.put_resp_header(conn, "cache-control", @cache_control[key])
+  end
+
+  defp redirect_or_error(conn, url, true), do: redirect(conn, external: url)
+  defp redirect_or_error(conn, url, _), do: send_error(conn, 502, "Media proxy error: " <> url)
+
+  defp send_error(conn, code, body \\ "") do
+    conn
+    |> set_cache_header(:error)
+    |> send_resp(code, body)
+  end
+
+  defp proxy_request_body(client), do: proxy_request_body(client, <<>>)
+  defp proxy_request_body(client, body) when byte_size(body) < @max_body_length do
+    case :hackney.stream_body(client) do
+      {:ok, data} -> proxy_request_body(client, <<body :: binary, data :: binary>>)
+      :done -> {:ok, body}
+      {:error, reason} -> {:error, reason}
+    end
+  end
+  defp proxy_request_body(client, _) do
+    :hackney.close(client)
+    {:error, :body_too_large}
+  end
+
+  # TODO: the body is passed here as well because some hosts do not provide a content-type.
+  # At some point we may want to use magic numbers to discover the content-type and reply a proper one.
+  defp proxy_request_content_type(headers, _body) do
+    headers["Content-Type"] || headers["content-type"] || "image/jpeg"
+  end
+
+end
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
new file mode 100644
index 000000000..23efc18fa
--- /dev/null
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -0,0 +1,32 @@
+defmodule Pleroma.Web.MediaProxy do
+  @base64_opts [padding: false]
+
+  def url(nil), do: nil
+
+  def url(url = "/" <> _), do: url
+
+  def url(url) do
+    config = Application.get_env(:pleroma, :media_proxy, [])
+    if !Keyword.get(config, :enabled, false) or String.starts_with?(url, Pleroma.Web.base_url) do
+      url
+    else
+      secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
+      base64 = Base.url_encode64(url, @base64_opts)
+      sig = :crypto.hmac(:sha, secret, base64)
+      sig64 = sig |> Base.url_encode64(@base64_opts)
+      Keyword.get(config, :base_url, Pleroma.Web.base_url) <> "/proxy/#{sig64}/#{base64}"
+    end
+  end
+
+  def decode_url(sig, url) do
+    secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
+    sig = Base.url_decode64!(sig, @base64_opts)
+    local_sig = :crypto.hmac(:sha, secret, url)
+    if local_sig == sig do
+      {:ok, Base.url_decode64!(url, @base64_opts)}
+    else
+      {:error, :invalid_signature}
+    end
+  end
+
+end
diff --git a/lib/pleroma/web/oauth/fallback_controller.ex b/lib/pleroma/web/oauth/fallback_controller.ex
new file mode 100644
index 000000000..daa110532
--- /dev/null
+++ b/lib/pleroma/web/oauth/fallback_controller.ex
@@ -0,0 +1,12 @@
+defmodule Pleroma.Web.OAuth.FallbackController do
+    use Pleroma.Web, :controller
+    alias Pleroma.Web.OAuth.OAuthController
+
+    # No user/password
+    def call(conn, _) do
+        conn
+        |> put_flash(:error, "Invalid Username/Password")
+        |> OAuthController.authorize(conn.params)
+    end
+
+end
\ No newline at end of file
diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index e8483dec0..94318bfa9 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -5,6 +5,11 @@ defmodule Pleroma.Web.OAuth.OAuthController do
   alias Pleroma.{Repo, User}
   alias Comeonin.Pbkdf2
 
+  plug :fetch_session
+  plug :fetch_flash
+
+  action_fallback Pleroma.Web.OAuth.FallbackController
+
   def authorize(conn, params) do
     render conn, "show.html", %{
       response_type: params["response_type"],
diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex
index 08710f246..10860ce04 100644
--- a/lib/pleroma/web/ostatus/feed_representer.ex
+++ b/lib/pleroma/web/ostatus/feed_representer.ex
@@ -1,6 +1,8 @@
 defmodule Pleroma.Web.OStatus.FeedRepresenter do
   alias Pleroma.Web.OStatus
   alias Pleroma.Web.OStatus.{UserRepresenter, ActivityRepresenter}
+  alias Pleroma.User
+  alias Pleroma.Web.MediaProxy
 
   def to_simple_form(user, activities, _users) do
     most_recent_update = (List.first(activities) || user).updated_at
@@ -25,6 +27,7 @@ defmodule Pleroma.Web.OStatus.FeedRepresenter do
         {:id, h.(OStatus.feed_path(user))},
         {:title, ['#{user.nickname}\'s timeline']},
         {:updated, h.(most_recent_update)},
+        {:logo, [to_charlist(User.avatar_url(user) |> MediaProxy.url())]},
         {:link, [rel: 'hub', href: h.(OStatus.pubsub_path(user))], []},
         {:link, [rel: 'salmon', href: h.(OStatus.salmon_path(user))], []},
         {:link, [rel: 'self', href: h.(OStatus.feed_path(user)), type: 'application/atom+xml'], []},
diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex
index 745539b3e..c35ba42be 100644
--- a/lib/pleroma/web/ostatus/ostatus.ex
+++ b/lib/pleroma/web/ostatus/ostatus.ex
@@ -22,6 +22,10 @@ defmodule Pleroma.Web.OStatus do
     "#{user.ap_id}/salmon"
   end
 
+  def remote_follow_path do
+    "#{Web.base_url}/ostatus_subscribe?acct={uri}"
+  end
+
   def handle_incoming(xml_string) do
     with doc when doc != :error <- parse_document(xml_string) do
       entries = :xmerl_xpath.string('//entry', doc)
@@ -159,8 +163,7 @@ defmodule Pleroma.Web.OStatus do
     Get the cw that mastodon uses.
   """
   def get_cw(entry) do
-    with scope when not is_nil(scope) <- string_from_xpath("//mastodon:scope", entry),
-         cw when not is_nil(cw) <- string_from_xpath("/*/summary", entry) do
+    with cw when not is_nil(cw) <- string_from_xpath("/*/summary", entry) do
       cw
     else _e -> nil
     end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 4f9ebf5e8..6455ff108 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -28,6 +28,13 @@ defmodule Pleroma.Web.Router do
     plug Pleroma.Plugs.AuthenticationPlug, %{fetcher: &Router.user_fetcher/1, optional: true}
   end
 
+  pipeline :pleroma_html do
+    plug :accepts, ["html"]
+    plug :fetch_session
+    plug Pleroma.Plugs.OAuthPlug
+    plug Pleroma.Plugs.AuthenticationPlug, %{fetcher: &Router.user_fetcher/1, optional: true}
+  end
+
   pipeline :well_known do
     plug :accepts, ["xml", "xrd+xml"]
   end
@@ -51,6 +58,18 @@ defmodule Pleroma.Web.Router do
     get "/emoji", UtilController, :emoji
   end
 
+  scope "/", Pleroma.Web.TwitterAPI do
+    pipe_through :pleroma_html
+    get "/ostatus_subscribe", UtilController, :remote_follow
+    post "/ostatus_subscribe", UtilController, :do_remote_follow
+    post "/main/ostatus", UtilController, :remote_subscribe
+  end
+
+  scope "/api/pleroma", Pleroma.Web.TwitterAPI do
+    pipe_through :authenticated_api
+    post "/follow_import", UtilController, :follow_import
+  end
+
   scope "/oauth", Pleroma.Web.OAuth do
     get "/authorize", OAuthController, :authorize
     post "/authorize", OAuthController, :create_authorization
@@ -101,6 +120,7 @@ defmodule Pleroma.Web.Router do
   scope "/api/v1", Pleroma.Web.MastodonAPI do
     pipe_through :api
     get "/instance", MastodonAPIController, :masto_instance
+    get "/instance/peers", MastodonAPIController, :peers
     post "/apps", MastodonAPIController, :create_app
     get "/custom_emojis", MastodonAPIController, :custom_emojis
 
@@ -142,6 +162,8 @@ defmodule Pleroma.Web.Router do
     get "/qvitter/statuses/user_timeline", TwitterAPI.Controller, :user_timeline
     get "/users/show", TwitterAPI.Controller, :show_user
 
+    get "/statuses/followers", TwitterAPI.Controller, :followers
+    get "/statuses/friends", TwitterAPI.Controller, :friends
     get "/statuses/show/:id", TwitterAPI.Controller, :fetch_status
     get "/statusnet/conversation/:id", TwitterAPI.Controller, :fetch_conversation
 
@@ -188,8 +210,6 @@ defmodule Pleroma.Web.Router do
 
     post "/qvitter/update_avatar", TwitterAPI.Controller, :update_avatar
 
-    get "/statuses/followers", TwitterAPI.Controller, :followers
-    get "/statuses/friends", TwitterAPI.Controller, :friends
     get "/friends/ids", TwitterAPI.Controller, :friends_ids
     get "/friendships/no_retweets/ids", TwitterAPI.Controller, :empty_array
 
@@ -243,6 +263,14 @@ defmodule Pleroma.Web.Router do
     delete "/auth/sign_out", MastodonAPIController, :logout
   end
 
+  pipeline :remote_media do
+    plug :accepts, ["html"]
+  end
+  scope "/proxy/", Pleroma.Web.MediaProxy do
+    pipe_through :remote_media
+    get "/:sig/:url", MediaProxyController, :remote
+  end
+
   scope "/", Fallback do
     get "/*path", RedirectController, :redirector
   end
diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex
index 3c6903a16..a7fa7523b 100644
--- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex
+++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex
@@ -1,3 +1,5 @@
+<p class="alert alert-info" role="alert"><%= get_flash(@conn, :info) %></p>
+<p class="alert alert-danger" role="alert"><%= get_flash(@conn, :error) %></p>
 <h2>OAuth Authorization</h2>
 <%= form_for @conn, o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %>
 <%= label f, :name, "Name" %>
diff --git a/lib/pleroma/web/templates/twitter_api/util/follow.html.eex b/lib/pleroma/web/templates/twitter_api/util/follow.html.eex
new file mode 100644
index 000000000..06359fa6c
--- /dev/null
+++ b/lib/pleroma/web/templates/twitter_api/util/follow.html.eex
@@ -0,0 +1,11 @@
+<%= if @error == :error do %>
+    <h2>Error fetching user</h2>
+<% else %>
+    <h2>Remote follow</h2>
+    <img width="128" height="128" src="<%= @avatar %>">
+    <p><%= @name %></p>
+    <%= form_for @conn, util_path(@conn, :do_remote_follow), [as: "user"], fn f -> %>
+    <%= hidden_input f, :id, value: @id %>
+    <%= submit "Authorize" %>
+    <% end %>
+<% end %>
diff --git a/lib/pleroma/web/templates/twitter_api/util/follow_login.html.eex b/lib/pleroma/web/templates/twitter_api/util/follow_login.html.eex
new file mode 100644
index 000000000..4e3a2be67
--- /dev/null
+++ b/lib/pleroma/web/templates/twitter_api/util/follow_login.html.eex
@@ -0,0 +1,14 @@
+<%= if @error do %>
+    <h2><%= @error %></h2>
+<% end %>
+<h2>Log in to follow</h2>
+<p><%= @name %></p>
+<img height="128" width="128" src="<%= @avatar %>">
+<%= form_for @conn, util_path(@conn, :do_remote_follow), [as: "authorization"], fn f -> %>
+<%= text_input f, :name, placeholder: "Username" %>
+<br>
+<%= password_input f, :password, placeholder: "Password" %>
+<br>
+<%= hidden_input f, :id, value: @id %>
+<%= submit "Authorize" %>
+<% end %>
diff --git a/lib/pleroma/web/templates/twitter_api/util/followed.html.eex b/lib/pleroma/web/templates/twitter_api/util/followed.html.eex
new file mode 100644
index 000000000..da473d502
--- /dev/null
+++ b/lib/pleroma/web/templates/twitter_api/util/followed.html.eex
@@ -0,0 +1,6 @@
+<%= if @error do %>
+<p>Error following account</p>
+<% else %>
+<h2>Account followed!</h2>
+<% end %>
+
diff --git a/lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex b/lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex
new file mode 100644
index 000000000..f60accebf
--- /dev/null
+++ b/lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex
@@ -0,0 +1,10 @@
+<%= if @error do %>
+  <h2>Error: <%= @error %></h2>
+<% else %>
+  <h2>Remotely follow <%= @nickname %></h2>
+  <%= form_for @conn, util_path(@conn, :remote_subscribe), [as: "user"], fn f -> %>
+  <%= hidden_input f, :nickname, value: @nickname %>
+  <%= text_input f, :profile, placeholder: "Your account ID, e.g. lain@quitter.se" %>
+  <%= submit "Follow" %>
+  <% end %>
+<% end %>
diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
index de2abd4d1..503719dbf 100644
--- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex
+++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
@@ -1,8 +1,12 @@
 defmodule Pleroma.Web.TwitterAPI.UtilController do
   use Pleroma.Web, :controller
+  require Logger
   alias Pleroma.Web
+  alias Pleroma.Web.OStatus
+  alias Pleroma.Web.WebFinger
+  alias Comeonin.Pbkdf2
   alias Pleroma.Formatter
-
+  alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.{Repo, PasswordResetToken, User}
 
   def show_password_reset(conn, %{"token" => token}) do
@@ -29,6 +33,72 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
     json(conn, "ok")
   end
 
+  def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do
+    with %User{} = user <- User.get_cached_by_nickname(nick),
+         avatar = User.avatar_url(user) do
+      conn
+      |> render("subscribe.html", %{nickname: nick, avatar: avatar, error: false})
+    else
+      _e -> render(conn, "subscribe.html", %{nickname: nick, avatar: nil, error: "Could not find user"})
+    end
+  end
+  def remote_subscribe(conn, %{"user" => %{"nickname" => nick, "profile" => profile}}) do
+    with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile),
+         %User{ap_id: ap_id} <- User.get_cached_by_nickname(nick) do
+      conn
+      |> Phoenix.Controller.redirect(external: String.replace(template, "{uri}", ap_id))
+    else
+      _e ->
+        render(conn, "subscribe.html", %{nickname: nick, avatar: nil, error: "Something went wrong."})
+    end
+  end
+
+  def remote_follow(%{assigns: %{user: user}} = conn, %{"acct" => acct}) do
+    {err, followee} = OStatus.find_or_make_user(acct)
+    avatar = User.avatar_url(followee)
+    name = followee.nickname
+    id = followee.id
+
+    if !!user do
+      conn
+      |> render("follow.html", %{error: err, acct: acct, avatar: avatar, name: name, id: id})
+    else
+      conn
+      |> render("follow_login.html", %{error: false, acct: acct, avatar: avatar, name: name, id: id})
+    end
+  end
+
+  def do_remote_follow(conn, %{"authorization" => %{"name" => username, "password" => password, "id" => id}}) do
+    followee = Repo.get(User, id)
+    avatar = User.avatar_url(followee)
+    name = followee.nickname
+    with %User{} = user <- User.get_cached_by_nickname(username),
+         true <- Pbkdf2.checkpw(password, user.password_hash),
+           %User{} = followed <- Repo.get(User, id),
+         {:ok, follower} <- User.follow(user, followee),
+         {:ok, _activity} <- ActivityPub.follow(follower, followee) do
+      conn
+      |> render("followed.html", %{error: false})
+    else
+      _e ->
+        conn
+        |> render("follow_login.html", %{error: "Wrong username or password", id: id, name: name, avatar: avatar})
+    end
+  end
+  def do_remote_follow(%{assigns: %{user: user}} = conn, %{"user" => %{"id" => id}}) do
+    with %User{} = followee <- Repo.get(User, id),
+         {:ok, follower} <- User.follow(user, followee),
+         {:ok, _activity} <- ActivityPub.follow(follower, followee) do
+      conn
+      |> render("followed.html", %{error: false})
+    else
+      e ->
+        Logger.debug("Remote follow failed with error #{inspect e}")
+      conn
+      |> render("followed.html", %{error: inspect(e)})
+    end
+  end
+
   @instance Application.get_env(:pleroma, :instance)
   def config(conn, _params) do
     case get_format(conn) do
@@ -51,7 +121,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
               site: %{
                 name: Keyword.get(@instance, :name),
                 server: Web.base_url,
-                textlimit: Keyword.get(@instance, :limit),
+                textlimit: to_string(Keyword.get(@instance, :limit)),
                 closed: if(Keyword.get(@instance, :registrations_open), do: "0", else: "1")
               }
              })
@@ -73,4 +143,24 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   def emoji(conn, _params) do
     json conn, Enum.into(Formatter.get_custom_emoji(), %{})
   end
+
+  def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do
+    follow_import(conn, %{"list" => File.read!(listfile.path)})
+  end
+  def follow_import(%{assigns: %{user: user}} = conn, %{"list" => list}) do
+    Task.start(fn ->
+    String.split(list)
+    |> Enum.map(fn nick ->
+        with %User{} = follower <- User.get_cached_by_ap_id(user.ap_id),
+        %User{} = followed <- User.get_or_fetch_by_nickname(nick),
+        {:ok, follower} <- User.follow(follower, followed) do
+          ActivityPub.follow(follower, followed)
+        else
+          _e -> Logger.debug "follow_import: following #{nick} failed"
+        end
+      end)
+    end)
+
+    json conn, "job started"
+  end
 end
diff --git a/lib/pleroma/web/twitter_api/representers/object_representer.ex b/lib/pleroma/web/twitter_api/representers/object_representer.ex
index c39b60760..69eaeb36c 100644
--- a/lib/pleroma/web/twitter_api/representers/object_representer.ex
+++ b/lib/pleroma/web/twitter_api/representers/object_representer.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter do
     data = object.data
     url = List.first(data["url"])
     %{
-      url: url["href"],
+      url: url["href"] |> Pleroma.Web.MediaProxy.url(),
       mimetype: url["mediaType"],
       id: data["uuid"],
       oembed: false
diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex
index d04a81cd4..faecebde0 100644
--- a/lib/pleroma/web/twitter_api/twitter_api.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api.ex
@@ -316,10 +316,12 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
 
   def get_external_profile(for_user, uri) do
     with {:ok, %User{} = user} <- OStatus.find_or_make_user(uri) do
-      with url <- user.info["topic"],
-           {:ok, %{body: body}} <- @httpoison.get(url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000) do
-        OStatus.handle_incoming(body)
-      end
+      spawn(fn ->
+        with url <- user.info["topic"],
+             {:ok, %{body: body}} <- @httpoison.get(url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000) do
+          OStatus.handle_incoming(body)
+        end
+      end)
       {:ok, UserView.render("show.json", %{user: user, for: for_user})}
     else _e ->
         {:error, "Couldn't find user"}
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index 73d96c73d..5284a8847 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -263,16 +263,18 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
     end
   end
 
-  def followers(%{assigns: %{user: user}} = conn, _params) do
-    with {:ok, followers} <- User.get_followers(user) do
+  def followers(conn, params) do
+    with {:ok, user} <- TwitterAPI.get_user(conn.assigns.user, params),
+         {:ok, followers} <- User.get_followers(user) do
       render(conn, UserView, "index.json", %{users: followers, for: user})
     else
       _e -> bad_request_reply(conn, "Can't get followers")
     end
   end
 
-  def friends(%{assigns: %{user: user}} = conn, _params) do
-    with {:ok, friends} <- User.get_friends(user) do
+  def friends(conn, params) do
+    with {:ok, user} <- TwitterAPI.get_user(conn.assigns.user, params),
+         {:ok, friends} <- User.get_friends(user) do
       render(conn, UserView, "index.json", %{users: friends, for: user})
     else
       _e -> bad_request_reply(conn, "Can't get friends")
diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex
index d1c7e6fbd..f49bcc0fb 100644
--- a/lib/pleroma/web/twitter_api/views/user_view.ex
+++ b/lib/pleroma/web/twitter_api/views/user_view.ex
@@ -2,6 +2,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
   use Pleroma.Web, :view
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
+  alias Pleroma.Web.MediaProxy
 
   def render("show.json", %{user: user = %User{}} = assigns) do
     render_one(user, Pleroma.Web.TwitterAPI.UserView, "user.json", assigns)
@@ -12,7 +13,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
   end
 
   def render("user.json", %{user: user = %User{}} = assigns) do
-    image = User.avatar_url(user)
+    image = User.avatar_url(user) |> MediaProxy.url()
     {following, follows_you, statusnet_blocking} = if assigns[:for] do
       {
         User.following?(assigns[:for], user),
@@ -44,8 +45,9 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
       "screen_name" => user.nickname,
       "statuses_count" => user_info[:note_count],
       "statusnet_profile_url" => user.ap_id,
-      "cover_photo" => image_url(user.info["banner"]),
-      "background_image" => image_url(user.info["background"])
+      "cover_photo" => User.banner_url(user) |> MediaProxy.url(),
+      "background_image" => image_url(user.info["background"]) |> MediaProxy.url(),
+      "is_local" => user.local
     }
 
     if assigns[:token] do
diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex
index 11b36d6ac..09957e133 100644
--- a/lib/pleroma/web/web_finger/web_finger.ex
+++ b/lib/pleroma/web/web_finger/web_finger.ex
@@ -45,7 +45,8 @@ defmodule Pleroma.Web.WebFinger do
         {:Link, %{rel: "http://webfinger.net/rel/profile-page", type: "text/html", href: user.ap_id}},
         {:Link, %{rel: "salmon", href: OStatus.salmon_path(user)}},
         {:Link, %{rel: "magic-public-key", href: "data:application/magic-public-key,#{magic_key}"}},
-        {:Link, %{rel: "self", type: "application/activity+json", href: user.ap_id}}
+        {:Link, %{rel: "self", type: "application/activity+json", href: user.ap_id}},
+        {:Link, %{rel: "http://ostatus.org/schema/1.0/subscribe", template: OStatus.remote_follow_path()}}
       ]
     }
     |> XmlBuilder.to_doc
@@ -69,11 +70,13 @@ defmodule Pleroma.Web.WebFinger do
     topic = XML.string_from_xpath(~s{//Link[@rel="http://schemas.google.com/g/2010#updates-from"]/@href}, doc)
     subject = XML.string_from_xpath("//Subject", doc)
     salmon = XML.string_from_xpath(~s{//Link[@rel="salmon"]/@href}, doc)
+    subscribe_address = XML.string_from_xpath(~s{//Link[@rel="http://ostatus.org/schema/1.0/subscribe"]/@template}, doc)
     data = %{
       "magic_key" => magic_key,
       "topic" => topic,
       "subject" => subject,
-      "salmon" => salmon
+      "salmon" => salmon,
+      "subscribe_address" => subscribe_address
     }
     {:ok, data}
   end