Merge branch 'develop' into dtluna/pleroma-refactor/1

22 files changed, 991 insertions, 193 deletions

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 46568bb13..d77c88997 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -5,6 +5,7 @@ defmodule Pleroma.Activity do
 
   schema "activities" do
     field :data, :map
+    field :local, :boolean, default: true
 
     timestamps()
   end
@@ -18,4 +19,9 @@ defmodule Pleroma.Activity do
     Repo.all(from activity in Activity,
       where: fragment("? @> ?", activity.data, ^%{object: %{id: ap_id}}))
   end
+
+  def get_create_activity_by_object_ap_id(ap_id) do
+    Repo.one(from activity in Activity,
+      where: fragment("? @> ?", activity.data, ^%{type: "Create", object: %{id: ap_id}}))
+  end
 end
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 86b6c0c1e..6267d0695 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -15,9 +15,9 @@ defmodule Pleroma.Application do
       # Start your own worker by calling: Pleroma.Worker.start_link(arg1, arg2, arg3)
       # worker(Pleroma.Worker, [arg1, arg2, arg3]),
       worker(Cachex, [:user_cache, [
-                         default_ttl: 5000,
+                         default_ttl: 25000,
                          ttl_interval: 1000,
-                         limit: 500
+                         limit: 2500
                        ]])
     ]
 
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index f932034d7..949ccb0f6 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -13,4 +13,24 @@ defmodule Pleroma.Object do
     Repo.one(from object in Object,
       where: fragment("? @> ?", object.data, ^%{id: ap_id}))
   end
+
+  def get_cached_by_ap_id(ap_id) do
+    if Mix.env == :test do
+      get_by_ap_id(ap_id)
+    else
+      key = "object:#{ap_id}"
+      Cachex.get!(:user_cache, key, fallback: fn(_) ->
+        object = get_by_ap_id(ap_id)
+        if object do
+          {:commit, object}
+        else
+          {:ignore, object}
+        end
+      end)
+    end
+  end
+
+  def context_mapping(context) do
+    %Object{data: %{"id" => context}}
+  end
 end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 65925caed..23be6276e 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1,8 +1,10 @@
 defmodule Pleroma.User do
   use Ecto.Schema
+
   import Ecto.{Changeset, Query}
   alias Pleroma.{Repo, User, Object, Web}
   alias Comeonin.Pbkdf2
+  alias Pleroma.Web.OStatus
 
   schema "users" do
     field :bio, :string
@@ -15,6 +17,8 @@ defmodule Pleroma.User do
     field :following, {:array, :string}, default: []
     field :ap_id, :string
     field :avatar, :map
+    field :local, :boolean, default: true
+    field :info, :map, default: %{}
 
     timestamps()
   end
@@ -118,6 +122,27 @@ defmodule Pleroma.User do
 
   def get_cached_by_nickname(nickname) do
     key = "nickname:#{nickname}"
-    Cachex.get!(:user_cache, key, fallback: fn(_) -> Repo.get_by(User, nickname: nickname) end)
+    Cachex.get!(:user_cache, key, fallback: fn(_) -> get_or_fetch_by_nickname(nickname) end)
+  end
+
+  def get_by_nickname(nickname) do
+    Repo.get_by(User, nickname: nickname)
+  end
+
+  def get_cached_user_info(user) do
+    key = "user_info:#{user.id}"
+    Cachex.get!(:user_cache, key, fallback: fn(_) -> user_info(user) end)
+  end
+
+  def get_or_fetch_by_nickname(nickname) do
+    with %User{} = user <- get_by_nickname(nickname)  do
+      user
+    else _e ->
+      with [nick, domain] <- String.split(nickname, "@"),
+           {:ok, user} <- OStatus.make_user(nickname) do
+        user
+      else _e -> nil
+      end
+    end
   end
 end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 02255e0a4..d7b490088 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -3,7 +3,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Ecto.{Changeset, UUID}
   import Ecto.Query
 
-  def insert(map) when is_map(map) do
+  def insert(map, local \\ true) when is_map(map) do
     map = map
     |> Map.put_new_lazy("id", &generate_activity_id/0)
     |> Map.put_new_lazy("published", &make_date/0)
@@ -16,7 +16,29 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       map
     end
 
-    Repo.insert(%Activity{data: map})
+    Repo.insert(%Activity{data: map, local: local})
+  end
+
+  def create(to, actor, context, object, additional \\ %{}, published \\ nil, local \\ true) do
+    published = published || make_date()
+
+    activity = %{
+      "type" => "Create",
+      "to" => to |> Enum.uniq,
+      "actor" => actor.ap_id,
+      "object" => object,
+      "published" => published,
+      "context" => context
+    }
+    |> Map.merge(additional)
+
+    with {:ok, activity} <- insert(activity, local) do
+      if actor.local do
+        Pleroma.Web.Federator.enqueue(:publish, activity)
+       end
+
+      {:ok, activity}
+    end
   end
 
   def like(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do
@@ -33,7 +55,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           "type" => "Like",
           "actor" => ap_id,
           "object" => id,
-          "to" => [User.ap_followers(user), object.data["actor"]]
+          "to" => [User.ap_followers(user), object.data["actor"]],
+          "context" => object.data["context"]
         }
 
         {:ok, activity} = insert(data)
@@ -49,6 +72,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
         update_object_in_activities(object)
 
+        if user.local do
+          Pleroma.Web.Federator.enqueue(:publish, activity)
+        end
+
         {:ok, activity, object}
     end
   end
@@ -99,7 +126,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def generate_object_id do
-    generate_id("objects")
+    Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :object, Ecto.UUID.generate)
   end
 
   def generate_id(type) do
@@ -127,6 +154,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     query = from activity in query,
       where: activity.id > ^since_id
 
+    query = if opts["local_only"] do
+      from activity in query, where: activity.local == true
+    else
+      query
+    end
+
     query = if opts["max_id"] do
       from activity in query, where: activity.id < ^opts["max_id"]
     else
@@ -143,15 +176,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     Enum.reverse(Repo.all(query))
   end
 
-  def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do
+  def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object, local \\ true) do
     data = %{
       "type" => "Announce",
       "actor" => ap_id,
       "object" => id,
-      "to" => [User.ap_followers(user), object.data["actor"]]
+      "to" => [User.ap_followers(user), object.data["actor"]],
+      "context" => object.data["context"]
     }
 
-    {:ok, activity} = insert(data)
+    {:ok, activity} = insert(data, local)
 
     announcements = [ap_id | (object.data["announcements"] || [])] |> Enum.uniq
 
@@ -164,6 +198,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
     update_object_in_activities(object)
 
+    if user.local do
+      Pleroma.Web.Federator.enqueue(:publish, activity)
+    end
+
     {:ok, activity, object}
   end
 
diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
new file mode 100644
index 000000000..675e804a2
--- /dev/null
+++ b/lib/pleroma/web/federator/federator.ex
@@ -0,0 +1,38 @@
+defmodule Pleroma.Web.Federator do
+  alias Pleroma.User
+  alias Pleroma.Web.WebFinger
+  require Logger
+
+  @websub Application.get_env(:pleroma, :websub)
+
+  def handle(:publish, activity) do
+    Logger.debug("Running publish for #{activity.data["id"]}")
+    with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do
+      Logger.debug("Sending #{activity.data["id"]} out via websub")
+      Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity)
+
+      {:ok, actor} = WebFinger.ensure_keys_present(actor)
+      Logger.debug("Sending #{activity.data["id"]} out via salmon")
+      Pleroma.Web.Salmon.publish(actor, activity)
+    end
+  end
+
+  def handle(:verify_websub, websub) do
+    Logger.debug("Running websub verification for #{websub.id} (#{websub.topic}, #{websub.callback})")
+    @websub.verify(websub)
+  end
+
+  def handle(type, payload) do
+    Logger.debug("Unknown task: #{type}")
+    {:error, "Don't know what do do with this"}
+  end
+
+  def enqueue(type, payload) do
+    # for now, just run immediately in a new process.
+    if Mix.env == :test do
+      handle(type, payload)
+    else
+      spawn(fn -> handle(type, payload) end)
+    end
+  end
+end
diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex
index d7ea61321..88781626c 100644
--- a/lib/pleroma/web/ostatus/activity_representer.ex
+++ b/lib/pleroma/web/ostatus/activity_representer.ex
@@ -1,5 +1,31 @@
 defmodule Pleroma.Web.OStatus.ActivityRepresenter do
-  def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) do
+  alias Pleroma.{Activity, User}
+  alias Pleroma.Web.OStatus.UserRepresenter
+  require Logger
+
+  defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do
+    [{:"thr:in-reply-to", [ref: to_charlist(in_reply_to)], []}]
+  end
+
+  defp get_in_reply_to(_), do: []
+
+  defp get_mentions(to) do
+    Enum.map(to, fn (id) ->
+      cond do
+        # Special handling for the AP/Ostatus public collections
+        "https://www.w3.org/ns/activitystreams#Public" == id ->
+          {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/collection", href: "http://activityschema.org/collection/public"], []}
+        # Ostatus doesn't handle follower collections, ignore these.
+        Regex.match?(~r/^#{Pleroma.Web.base_url}.+followers$/, id) ->
+          []
+        true ->
+          {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/person", href: id], []}
+      end
+    end)
+  end
+
+  def to_simple_form(activity, user, with_author \\ false)
+  def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do
     h = fn(str) -> [to_charlist(str)] end
 
     updated_at = activity.updated_at
@@ -12,16 +38,97 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
       {:link, [rel: 'enclosure', href: to_charlist(url["href"]), type: to_charlist(url["mediaType"])], []}
     end)
 
+    in_reply_to = get_in_reply_to(activity.data)
+    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
+    mentions = activity.data["to"] |> get_mentions
+
     [
       {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']},
       {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']},
-      {:id, h.(activity.data["object"]["id"])},
+      {:id, h.(activity.data["object"]["id"])}, # For notes, federate the object id.
       {:title, ['New note by #{user.nickname}']},
       {:content, [type: 'html'], h.(activity.data["object"]["content"])},
       {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)}
-    ] ++ attachments
+      {:updated, h.(updated_at)},
+      {:"ostatus:conversation", [], h.(activity.data["context"])},
+      {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []},
+      {:link, [type: ['application/atom+xml'], href: h.(activity.data["object"]["id"]), rel: 'self'], []}
+    ] ++ attachments ++ in_reply_to ++ author ++ mentions
+  end
+
+  def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) do
+    h = fn(str) -> [to_charlist(str)] end
+
+    updated_at = activity.updated_at
+    |> NaiveDateTime.to_iso8601
+    inserted_at = activity.inserted_at
+    |> NaiveDateTime.to_iso8601
+
+    in_reply_to = get_in_reply_to(activity.data)
+    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
+    mentions = activity.data["to"] |> get_mentions
+
+    [
+      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/favorite']},
+      {:id, h.(activity.data["id"])},
+      {:title, ['New favorite by #{user.nickname}']},
+      {:content, [type: 'html'], ['#{user.nickname} favorited something']},
+      {:published, h.(inserted_at)},
+      {:updated, h.(updated_at)},
+      {:"activity:object", [
+        {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']},
+        {:id, h.(activity.data["object"])}, # For notes, federate the object id.
+      ]},
+      {:"ostatus:conversation", [], h.(activity.data["context"])},
+      {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []},
+      {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []},
+      {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []}
+    ] ++ author ++ mentions
+  end
+
+  def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_author) do
+    h = fn(str) -> [to_charlist(str)] end
+
+    updated_at = activity.updated_at
+    |> NaiveDateTime.to_iso8601
+    inserted_at = activity.inserted_at
+    |> NaiveDateTime.to_iso8601
+
+    in_reply_to = get_in_reply_to(activity.data)
+    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
+
+    retweeted_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+    retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"])
+
+    retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true)
+
+    mentions = activity.data["to"] |> get_mentions
+    [
+      {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']},
+      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']},
+      {:id, h.(activity.data["id"])},
+      {:title, ['#{user.nickname} repeated a notice']},
+      {:content, [type: 'html'], ['RT #{retweeted_activity.data["object"]["content"]}']},
+      {:published, h.(inserted_at)},
+      {:updated, h.(updated_at)},
+      {:"ostatus:conversation", [], h.(activity.data["context"])},
+      {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []},
+      {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []},
+      {:"activity:object", retweeted_xml}
+    ] ++ mentions ++ author
+  end
+
+  def wrap_with_entry(simple_form) do
+    [{
+      :entry, [
+        xmlns: 'http://www.w3.org/2005/Atom',
+        "xmlns:thr": 'http://purl.org/syndication/thread/1.0',
+        "xmlns:activity": 'http://activitystrea.ms/spec/1.0/',
+        "xmlns:poco": 'http://portablecontacts.net/spec/1.0',
+        "xmlns:ostatus": 'http://ostatus.org/schema/1.0'
+      ], simple_form
+    }]
   end
 
-  def to_simple_form(_, _), do: nil
+  def to_simple_form(_, _, _), do: nil
 end
diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex
index 86c6f9d4f..6b67b8ddf 100644
--- a/lib/pleroma/web/ostatus/feed_representer.ex
+++ b/lib/pleroma/web/ostatus/feed_representer.ex
@@ -17,14 +17,17 @@ defmodule Pleroma.Web.OStatus.FeedRepresenter do
     [{
       :feed, [
         xmlns: 'http://www.w3.org/2005/Atom',
+        "xmlns:thr": 'http://purl.org/syndication/thread/1.0',
         "xmlns:activity": 'http://activitystrea.ms/spec/1.0/',
-        "xmlns:poco": 'http://portablecontacts.net/spec/1.0'
+        "xmlns:poco": 'http://portablecontacts.net/spec/1.0',
+        "xmlns:ostatus": 'http://ostatus.org/schema/1.0'
       ], [
         {:id, h.(OStatus.feed_path(user))},
         {:title, ['#{user.nickname}\'s timeline']},
         {:updated, h.(most_recent_update)},
         {:link, [rel: 'hub', href: h.(OStatus.pubsub_path(user))], []},
-        {:link, [rel: 'self', href: h.(OStatus.feed_path(user))], []},
+        {:link, [rel: 'salmon', href: h.(OStatus.salmon_path(user))], []},
+        {:link, [rel: 'self', href: h.(OStatus.feed_path(user)), type: 'application/atom+xml'], []},
         {:author, UserRepresenter.to_simple_form(user)},
       ] ++ entries
     }]
diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex
index d21b9078f..2fab67663 100644
--- a/lib/pleroma/web/ostatus/ostatus.ex
+++ b/lib/pleroma/web/ostatus/ostatus.ex
@@ -1,5 +1,11 @@
 defmodule Pleroma.Web.OStatus do
-  alias Pleroma.Web
+  import Ecto.Query
+  import Pleroma.Web.XML
+  require Logger
+
+  alias Pleroma.{Repo, User, Web, Object}
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.{WebFinger, Websub}
 
   def feed_path(user) do
     "#{user.ap_id}/feed.atom"
@@ -9,6 +15,199 @@ defmodule Pleroma.Web.OStatus do
     "#{Web.base_url}/push/hub/#{user.nickname}"
   end
 
-  def user_path(user) do
+  def salmon_path(user) do
+    "#{user.ap_id}/salmon"
+  end
+
+  def handle_incoming(xml_string) do
+    doc = parse_document(xml_string)
+    entries = :xmerl_xpath.string('//entry', doc)
+
+    activities = Enum.map(entries, fn (entry) ->
+      {:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry)
+      {:xmlObj, :string, verb } = :xmerl_xpath.string('string(/entry/activity:verb[1])', entry)
+
+      case verb do
+        'http://activitystrea.ms/schema/1.0/share' ->
+          with {:ok, activity, retweeted_activity} <- handle_share(entry, doc), do: [activity, retweeted_activity]
+        _ ->
+          case object_type do
+            'http://activitystrea.ms/schema/1.0/note' ->
+              with {:ok, activity} <- handle_note(entry, doc), do: activity
+            'http://activitystrea.ms/schema/1.0/comment' ->
+              with {:ok, activity} <- handle_note(entry, doc), do: activity
+            _ ->
+              Logger.error("Couldn't parse incoming document")
+              nil
+          end
+      end
+    end)
+    {:ok, activities}
+  end
+
+  def make_share(entry, doc, retweeted_activity) do
+    with {:ok, actor} <- find_make_or_update_user(doc),
+         %Object{} = object <- Object.get_cached_by_ap_id(retweeted_activity.data["object"]["id"]),
+         {:ok, activity, object} = ActivityPub.announce(actor, object, false) do
+      {:ok, activity}
+    end
+  end
+
+  def handle_share(entry, doc) do
+    with [object] <- :xmerl_xpath.string('/entry/activity:object', entry),
+         {:ok, retweeted_activity} <-  handle_note(object, object),
+         {:ok, activity} <- make_share(entry, doc, retweeted_activity) do
+      {:ok, activity, retweeted_activity}
+    else
+      e -> {:error, e}
+    end
+  end
+
+  def get_attachments(entry) do
+    :xmerl_xpath.string('/entry/link[@rel="enclosure"]', entry)
+    |> Enum.map(fn (enclosure) ->
+      with href when not is_nil(href) <- string_from_xpath("/link/@href", enclosure),
+           type when not is_nil(type) <- string_from_xpath("/link/@type", enclosure) do
+        %{
+          "type" => "Attachment",
+          "url" => [%{
+                       "type" => "Link",
+                       "mediaType" => type,
+                       "href" => href
+                    }]
+        }
+      end
+    end)
+    |> Enum.filter(&(&1))
+  end
+
+  def handle_note(entry, doc \\ nil) do
+    content_html = string_from_xpath("//content[1]", entry)
+
+    [author] = :xmerl_xpath.string('//author[1]', doc)
+    {:ok, actor} = find_make_or_update_user(author)
+    inReplyTo = string_from_xpath("//thr:in-reply-to[1]/@ref", entry)
+
+    context = (string_from_xpath("//ostatus:conversation[1]", entry) || "") |> String.trim
+
+    attachments = get_attachments(entry)
+
+    context = with %{data: %{"context" => context}} <- Object.get_cached_by_ap_id(inReplyTo) do
+                context
+              else _e ->
+                if String.length(context) > 0 do
+                  context
+                else
+                  ActivityPub.generate_context_id
+                end
+              end
+
+    to = [
+      "https://www.w3.org/ns/activitystreams#Public"
+    ]
+
+    mentions = :xmerl_xpath.string('//link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', entry)
+    |> Enum.map(fn(person) -> string_from_xpath("@href", person) end)
+
+    to = to ++ mentions
+
+    date = string_from_xpath("//published", entry)
+    id = string_from_xpath("//id", entry)
+
+    object = %{
+      "id" => id,
+      "type" => "Note",
+      "to" => to,
+      "content" => content_html,
+      "published" => date,
+      "context" => context,
+      "actor" => actor.ap_id,
+      "attachment" => attachments
+    }
+
+    object = if inReplyTo do
+      Map.put(object, "inReplyTo", inReplyTo)
+    else
+      object
+    end
+
+    # TODO: Bail out sooner and use transaction.
+    if Object.get_by_ap_id(id) do
+      {:error, "duplicate activity"}
+    else
+      ActivityPub.create(to, actor, context, object, %{}, date, false)
+    end
+  end
+
+  def find_make_or_update_user(doc) do
+    uri = string_from_xpath("//author/uri[1]", doc)
+    with {:ok, user} <- find_or_make_user(uri) do
+      avatar = make_avatar_object(doc)
+      if user.avatar != avatar do
+        change = Ecto.Changeset.change(user, %{avatar: avatar})
+        Repo.update(change)
+      else
+        {:ok, user}
+      end
+    end
+  end
+
+  def find_or_make_user(uri) do
+    query = from user in User,
+      where: user.local == false and fragment("? @> ?", user.info, ^%{uri: uri})
+
+    user = Repo.one(query)
+
+    if is_nil(user) do
+      make_user(uri)
+    else
+      {:ok, user}
+    end
+  end
+
+  def make_user(uri) do
+    with {:ok, info} <- gather_user_info(uri) do
+      data = %{
+        local: false,
+        name: info["name"],
+        nickname: info["nickname"] <> "@" <> info["host"],
+        ap_id: info["uri"],
+        info: info,
+        avatar: info["avatar"]
+      }
+      # TODO: Make remote user changeset
+      # SHould enforce fqn nickname
+      Repo.insert(Ecto.Changeset.change(%User{}, data))
+    end
+  end
+
+  # TODO: Just takes the first one for now.
+  def make_avatar_object(author_doc) do
+    href = string_from_xpath("//author[1]/link[@rel=\"avatar\"]/@href", author_doc)
+    type = string_from_xpath("//author[1]/link[@rel=\"avatar\"]/@type", author_doc)
+
+    if href do
+      %{
+        "type" => "Image",
+        "url" =>
+          [%{
+              "type" => "Link",
+              "mediaType" => type,
+              "href" => href
+           }]
+      }
+    else
+      nil
+    end
+  end
+
+  def gather_user_info(username) do
+    with {:ok, webfinger_data} <- WebFinger.finger(username),
+         {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data["topic"]) do
+      {:ok, Map.merge(webfinger_data, feed_data) |> Map.put("fqn", username)}
+    else e ->
+      Logger.debug("Couldn't gather info for #{username}")
+      {:error, e}
+    end
   end
 end
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index ed7618d2b..e442562d5 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -2,10 +2,16 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   use Pleroma.Web, :controller
 
   alias Pleroma.{User, Activity}
-  alias Pleroma.Web.OStatus.FeedRepresenter
+  alias Pleroma.Web.OStatus.{FeedRepresenter, ActivityRepresenter}
   alias Pleroma.Repo
+  alias Pleroma.Web.OStatus
   import Ecto.Query
 
+  def feed_redirect(conn, %{"nickname" => nickname}) do
+    user = User.get_cached_by_nickname(nickname)
+    redirect conn, external: OStatus.feed_path(user)
+  end
+
   def feed(conn, %{"nickname" => nickname}) do
     user = User.get_cached_by_nickname(nickname)
     query = from activity in Activity,
@@ -26,7 +32,29 @@ defmodule Pleroma.Web.OStatus.OStatusController do
     |> send_resp(200, response)
   end
 
-  def temp(_conn, params) do
-    IO.inspect(params)
+  def salmon_incoming(conn, params) do
+    {:ok, body, _conn} = read_body(conn)
+    {:ok, magic_key} = Pleroma.Web.Salmon.fetch_magic_key(body)
+    {:ok, doc} = Pleroma.Web.Salmon.decode_and_validate(magic_key, body)
+
+    Pleroma.Web.OStatus.handle_incoming(doc)
+
+    conn
+    |> send_resp(200, "")
+  end
+
+  def object(conn, %{"uuid" => uuid}) do
+    id = o_status_url(conn, :object, uuid)
+    activity = Activity.get_create_activity_by_object_ap_id(id)
+    user = User.get_cached_by_ap_id(activity.data["actor"])
+
+    response = ActivityRepresenter.to_simple_form(activity, user, true)
+    |> ActivityRepresenter.wrap_with_entry
+    |> :xmerl.export_simple(:xmerl_xml)
+    |> to_string
+
+    conn
+    |> put_resp_content_type("application/atom+xml")
+    |> send_resp(200, response)
   end
 end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 2c94d071f..327987d1d 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -30,7 +30,7 @@ defmodule Pleroma.Web.Router do
     get "/statusnet/config", TwitterAPI.Controller, :config
 
     get "/statuses/public_timeline", TwitterAPI.Controller, :public_timeline
-    get "/statuses/public_and_external_timeline", TwitterAPI.Controller, :public_timeline
+    get "/statuses/public_and_external_timeline", TwitterAPI.Controller, :public_and_external_timeline
     get "/statuses/user_timeline", TwitterAPI.Controller, :user_timeline
 
     get "/statuses/show/:id", TwitterAPI.Controller, :fetch_status
@@ -73,8 +73,14 @@ defmodule Pleroma.Web.Router do
   scope "/", Pleroma.Web do
     pipe_through :ostatus
 
+    get "/objects/:uuid", OStatus.OStatusController, :object
+
     get "/users/:nickname/feed", OStatus.OStatusController, :feed
+    get "/users/:nickname", OStatus.OStatusController, :feed_redirect
+    post "/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming
     post "/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request
+    get "/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation
+    post "/push/subscriptions/:id", Websub.WebsubController, :websub_incoming
   end
 
   scope "/.well-known", Pleroma.Web do
@@ -92,5 +98,5 @@ end
 
 defmodule Fallback.RedirectController do
   use Pleroma.Web, :controller
-  def redirector(conn, _params), do: send_file(conn, 200, "priv/static/index.html")
+  def redirector(conn, _params), do: (if Mix.env != :test, do: send_file(conn, 200, "priv/static/index.html"))
 end
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index f26daf824..fe529c4c0 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -1,8 +1,12 @@
 defmodule Pleroma.Web.Salmon do
   use Bitwise
+  alias Pleroma.Web.XML
+  alias Pleroma.Web.OStatus.ActivityRepresenter
+  alias Pleroma.User
+  require Logger
 
   def decode(salmon) do
-    {doc, _rest} = :xmerl_scan.string(to_charlist(salmon))
+    doc = XML.parse_document(salmon)
 
     {:xmlObj, :string, data} = :xmerl_xpath.string('string(//me:data[1])', doc)
     {:xmlObj, :string, sig} = :xmerl_xpath.string('string(//me:sig[1])', doc)
@@ -20,22 +24,12 @@ defmodule Pleroma.Web.Salmon do
   end
 
   def fetch_magic_key(salmon) do
-    [data, _, _, _, _] = decode(salmon)
-    {doc, _rest} = :xmerl_scan.string(to_charlist(data))
-    {:xmlObj, :string, uri} = :xmerl_xpath.string('string(//author[1]/uri)', doc)
-
-    uri = to_string(uri)
-    base = URI.parse(uri).host
-
-    # TODO: Find out if this endpoint is mandated by the standard.
-    {:ok, response} = HTTPoison.get(base <> "/.well-known/webfinger", ["Accept": "application/xrd+xml"], [params: [resource: uri]])
-
-    {doc, _rest} = :xmerl_scan.string(to_charlist(response.body))
-
-    {:xmlObj, :string, magickey} = :xmerl_xpath.string('string(//Link[@rel="magic-public-key"]/@href)', doc)
-    "data:application/magic-public-key," <> magickey = to_string(magickey)
-
-    magickey
+    with [data, _, _, _, _] <- decode(salmon),
+         doc <- XML.parse_document(data),
+         uri when not is_nil(uri) <- XML.string_from_xpath("/entry/author[1]/uri", doc),
+         {:ok, %{info: %{"magic_key" => magic_key}}} <- Pleroma.Web.OStatus.find_or_make_user(uri) do
+      {:ok, magic_key}
+    end
   end
 
   def decode_and_validate(magickey, salmon) do
@@ -56,7 +50,7 @@ defmodule Pleroma.Web.Salmon do
     end
   end
 
-  defp decode_key("RSA." <> magickey) do
+  def decode_key("RSA." <> magickey) do
     make_integer = fn(bin) ->
       list = :erlang.binary_to_list(bin)
       Enum.reduce(list, 0, fn (el, acc) -> (acc <<< 8) ||| el end)
@@ -69,4 +63,91 @@ defmodule Pleroma.Web.Salmon do
 
     {:RSAPublicKey, modulus, exponent}
   end
+
+  def encode_key({:RSAPublicKey, modulus, exponent}) do
+    modulus_enc = :binary.encode_unsigned(modulus) |> Base.url_encode64
+    exponent_enc = :binary.encode_unsigned(exponent) |> Base.url_encode64
+
+    "RSA.#{modulus_enc}.#{exponent_enc}"
+  end
+
+  def generate_rsa_pem do
+    port = Port.open({:spawn, "openssl genrsa"}, [:binary])
+    {:ok, pem} = receive do
+      {^port, {:data, pem}} -> {:ok, pem}
+    end
+    Port.close(port)
+    if Regex.match?(~r/RSA PRIVATE KEY/, pem) do
+      {:ok, pem}
+    else
+      :error
+    end
+  end
+
+  def keys_from_pem(pem) do
+    [private_key_code] = :public_key.pem_decode(pem)
+    private_key = :public_key.pem_entry_decode(private_key_code)
+    {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key
+    public_key = {:RSAPublicKey, modulus, exponent}
+    {:ok, private_key, public_key}
+  end
+
+  def encode(private_key, doc) do
+    type = "application/atom+xml"
+    encoding = "base64url"
+    alg = "RSA-SHA256"
+
+    signed_text = [doc, type, encoding, alg]
+    |> Enum.map(&Base.url_encode64/1)
+    |> Enum.join(".")
+
+    signature = :public_key.sign(signed_text, :sha256, private_key) |> to_string |> Base.url_encode64
+    doc_base64= doc |> Base.url_encode64
+
+    # Don't need proper xml building, these strings are safe to leave unescaped
+    salmon = """
+    <?xml version="1.0" encoding="UTF-8"?>
+    <me:env xmlns:me="http://salmon-protocol.org/ns/magic-env">
+      <me:data type="application/atom+xml">#{doc_base64}</me:data>
+      <me:encoding>#{encoding}</me:encoding>
+      <me:alg>#{alg}</me:alg>
+      <me:sig>#{signature}</me:sig>
+    </me:env>
+    """
+
+    {:ok, salmon}
+  end
+
+  def remote_users(%{data: %{"to" => to}}) do
+    to
+    |> Enum.map(fn(id) -> User.get_cached_by_ap_id(id) end)
+    |> Enum.filter(fn(user) -> user && !user.local end)
+  end
+
+  defp send_to_user(%{info: %{"salmon" => salmon}}, feed, poster) do
+    poster.(salmon, feed, [{"Content-Type", "application/magic-envelope+xml"}])
+  end
+
+  defp send_to_user(_,_,_), do: nil
+
+  def publish(user, activity, poster \\ &HTTPoison.post/3)
+  def publish(%{info: %{"keys" => keys}} = user, activity, poster) do
+    feed = ActivityRepresenter.to_simple_form(activity, user, true)
+    |> ActivityRepresenter.wrap_with_entry
+    |> :xmerl.export_simple(:xmerl_xml)
+    |> to_string
+
+    if feed do
+      {:ok, private, _} = keys_from_pem(keys)
+      {:ok, feed} = encode(private, feed)
+
+      remote_users(activity)
+      |> Enum.each(fn(remote_user) ->
+        Logger.debug("sending salmon to #{remote_user.ap_id}")
+        send_to_user(remote_user, feed, poster)
+      end)
+    end
+  end
+
+  def publish(%{id: id}, _, _), do: Logger.debug("Keys missing for user #{id}")
 end
diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
index b58572829..4d7ea0c5c 100644
--- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex
+++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
@@ -3,6 +3,7 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
   alias Pleroma.Web.TwitterAPI.Representers.{UserRepresenter, ObjectRepresenter}
   alias Pleroma.{Activity, User}
   alias Calendar.Strftime
+  alias Pleroma.Web.TwitterAPI.TwitterAPI
 
   defp user_by_ap_id(user_list, ap_id) do
     Enum.find(user_list, fn (%{ap_id: user_id}) -> ap_id == user_id end)
@@ -81,6 +82,12 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
     |> Enum.filter(&(&1))
     |> Enum.map(fn (user) -> UserRepresenter.to_map(user, opts) end)
 
+
+    conversation_id = with context when not is_nil(context) <- activity.data["context"] do
+      TwitterAPI.context_to_conversation_id(context)
+    else _e -> nil
+    end
+
     %{
       "id" => activity.id,
       "user" => UserRepresenter.to_map(user, opts),
@@ -91,7 +98,7 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
       "is_post_verb" => true,
       "created_at" => created_at,
       "in_reply_to_status_id" => object["inReplyToStatusId"],
-      "statusnet_conversation_id" => object["statusnetConversationId"],
+      "statusnet_conversation_id" => conversation_id,
       "attachments" => (object["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts),
       "attentions" => attentions,
       "fave_num" => like_count,
diff --git a/lib/pleroma/web/twitter_api/representers/user_representer.ex b/lib/pleroma/web/twitter_api/representers/user_representer.ex
index ab7d6d353..493077413 100644
--- a/lib/pleroma/web/twitter_api/representers/user_representer.ex
+++ b/lib/pleroma/web/twitter_api/representers/user_representer.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.Web.TwitterAPI.Representers.UserRepresenter do
       false
     end
 
-    user_info = User.user_info(user)
+    user_info = User.get_cached_user_info(user)
 
     map = %{
       "id" => user.id,
@@ -28,7 +28,8 @@ defmodule Pleroma.Web.TwitterAPI.Representers.UserRepresenter do
       "profile_image_url_https" => image,
       "profile_image_url_profile_size" => image,
       "profile_image_url_original" => image,
-      "rights" => %{}
+      "rights" => %{},
+      "statusnet_profile_url" => user.ap_id
     }
 
     map
diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex
index 649936b76..7656d4d33 100644
--- a/lib/pleroma/web/twitter_api/twitter_api.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api.ex
@@ -1,38 +1,81 @@
 defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
-  alias Ecto.Changeset
   alias Pleroma.{User, Activity, Repo, Object}
-  alias Pleroma.Web.{ActivityPub.ActivityPub, Websub, OStatus}
+  alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.TwitterAPI.Representers.{ActivityRepresenter, UserRepresenter}
 
   import Ecto.Query
 
-  def create_status(%User{} = user, %{} = data) do
-    attachments = Enum.map(data["media_ids"] || [], fn (media_id) ->
+  def to_for_user_and_mentions(user, mentions) do
+    default_to = [
+      User.ap_followers(user),
+      "https://www.w3.org/ns/activitystreams#Public"
+    ]
+
+    default_to ++ Enum.map(mentions, fn ({_, %{ap_id: ap_id}}) -> ap_id end)
+  end
+
+  def format_input(text, mentions) do
+    HtmlSanitizeEx.strip_tags(text)
+    |> String.replace("\n", "<br>")
+    |> add_user_links(mentions)
+  end
+
+  def attachments_from_ids(ids) do
+    Enum.map(ids || [], fn (media_id) ->
       Repo.get(Object, media_id).data
     end)
+  end
 
-    context = ActivityPub.generate_context_id
+  def get_replied_to_activity(id) when not is_nil(id) do
+    Repo.get(Activity, id)
+  end
 
-    content = data["status"] |> HtmlSanitizeEx.strip_tags |> String.replace("\n", "<br>")
+  def get_replied_to_activity(_), do: nil
 
-    mentions = parse_mentions(content)
+  def add_attachments(text, attachments) do
+    attachment_text = Enum.map(attachments, fn
+      (%{"url" => [%{"href" => href} | _]}) ->
+        "<a href='#{href}'>#{href}</a>"
+      _ -> ""
+    end)
+    Enum.join([text | attachment_text], "<br>")
+    end
 
-    default_to = [
-      User.ap_followers(user),
-      "https://www.w3.org/ns/activitystreams#Public"
-    ]
+  def create_status(user = %User{}, data = %{"status" => status}) do
+    attachments = attachments_from_ids(data["media_ids"])
+    context = ActivityPub.generate_context_id
+    mentions = parse_mentions(status)
+    content_html = status
+    |> format_input(mentions)
+    |> add_attachments(attachments)
 
-    to = default_to ++ Enum.map(mentions, fn ({_, %{ap_id: ap_id}}) -> ap_id end)
+    to = to_for_user_and_mentions(user, mentions)
+    date = make_date()
 
-    content_html = add_user_links(content, mentions)
+    inReplyTo = get_replied_to_activity(data["in_reply_to_status_id"])
 
-    date = make_date()
+    # Wire up reply info.
+    [to, context, object, additional] =
+      if inReplyTo do
+      context = inReplyTo.data["context"]
+      to = to ++ [inReplyTo.data["actor"]]
 
-    activity = %{
-      "type" => "Create",
-      "to" => to,
-      "actor" => user.ap_id,
-      "object" => %{
+      object = %{
+        "type" => "Note",
+        "to" => to,
+        "content" => content_html,
+        "published" => date,
+        "context" => context,
+        "attachment" => attachments,
+        "actor" => user.ap_id,
+        "inReplyTo" => inReplyTo.data["object"]["id"],
+        "inReplyToStatusId" => inReplyTo.id,
+      }
+      additional = %{}
+
+      [to, context, object, additional]
+      else
+      object = %{
         "type" => "Note",
         "to" => to,
         "content" => content_html,
@@ -40,65 +83,41 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
         "context" => context,
         "attachment" => attachments,
         "actor" => user.ap_id
-     },
-      "published" => date,
-      "context" => context
-   }
-
-    # Wire up reply info.
-    activity = with inReplyToId when not is_nil(inReplyToId) <- data["in_reply_to_status_id"],
-                    inReplyTo <- Repo.get(Activity, inReplyToId),
-                    context <- inReplyTo.data["context"]
-               do
-
-               to = activity["to"] ++ [inReplyTo.data["actor"]]
-
-               activity
-               |> put_in(["to"], to)
-               |> put_in(["context"], context)
-               |> put_in(["object", "context"], context)
-               |> put_in(["object", "inReplyTo"], inReplyTo.data["object"]["id"])
-               |> put_in(["object", "inReplyToStatusId"], inReplyToId)
-               |> put_in(["statusnetConversationId"], inReplyTo.data["statusnetConversationId"])
-               |> put_in(["object", "statusnetConversationId"], inReplyTo.data["statusnetConversationId"])
-               else _e ->
-                 activity
-               end
-
-    with {:ok, activity} <- ActivityPub.insert(activity) do
-      {:ok, activity} = add_conversation_id(activity)
-      Websub.publish(OStatus.feed_path(user), user, activity)
-      {:ok, activity}
+      }
+      [to, context, object, %{}]
     end
+
+    ActivityPub.create(to, user, context, object, additional, data)
   end
 
   def fetch_friend_statuses(user, opts \\ %{}) do
-    activities = ActivityPub.fetch_activities([user.ap_id | user.following], opts)
-    activities_to_statuses(activities, %{for: user})
+    ActivityPub.fetch_activities([user.ap_id | user.following], opts)
+    |> activities_to_statuses(%{for: user})
   end
 
   def fetch_public_statuses(user, opts \\ %{}) do
-    activities = ActivityPub.fetch_public_activities(opts)
-    activities_to_statuses(activities, %{for: user})
+    opts = Map.put(opts, "local_only", true)
+    ActivityPub.fetch_public_activities(opts)
+    |> activities_to_statuses(%{for: user})
+  end
+
+  def fetch_public_and_external_statuses(user, opts \\ %{}) do
+    ActivityPub.fetch_public_activities(opts)
+    |> activities_to_statuses(%{for: user})
   end
 
   def fetch_user_statuses(user, opts \\ %{}) do
-    activities = ActivityPub.fetch_activities([], opts)
-    activities_to_statuses(activities, %{for: user})
+    ActivityPub.fetch_activities([], opts)
+    |> activities_to_statuses(%{for: user})
   end
 
   def fetch_mentions(user, opts \\ %{}) do
-    activities = ActivityPub.fetch_activities([user.ap_id], opts)
-    activities_to_statuses(activities, %{for: user})
+    ActivityPub.fetch_activities([user.ap_id], opts)
+    |> activities_to_statuses(%{for: user})
   end
 
   def fetch_conversation(user, id) do
-    query = from activity in Activity,
-      where: fragment("? @> ?", activity.data, ^%{statusnetConversationId: id}),
-      limit: 1
-
-    with %Activity{} = activity <- Repo.one(query),
-         context <- activity.data["context"],
+    with context when is_binary(context) <- conversation_id_to_context(id),
          activities <- ActivityPub.fetch_activities_for_context(context),
          statuses <- activities |> activities_to_statuses(%{for: user})
     do
@@ -116,26 +135,26 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
   end
 
   def follow(%User{} = follower, params) do
-    with {:ok, %User{} = followed} <- get_user(params),
-         {:ok, follower} <- User.follow(follower, followed),
-         {:ok, activity} <- ActivityPub.insert(%{
-          "type" => "Follow",
-          "actor" => follower.ap_id,
-          "object" => followed.ap_id,
-          "published" => make_date()
-        })
+    with { :ok, %User{} = followed } <- get_user(params),
+         { :ok, follower } <- User.follow(follower, followed),
+         { :ok, activity } <- ActivityPub.insert(%{
+           "type" => "Follow",
+           "actor" => follower.ap_id,
+           "object" => followed.ap_id,
+           "published" => make_date()
+         })
     do
-      {:ok, follower, followed, activity}
+      { :ok, follower, followed, activity }
     else
       err -> err
     end
   end
 
   def unfollow(%User{} = follower, params) do
-    with {:ok, %User{} = unfollowed} <- get_user(params),
-         {:ok, follower} <- User.unfollow(follower, unfollowed)
+    with { :ok, %User{} = unfollowed } <- get_user(params),
+         { :ok, follower } <- User.unfollow(follower, unfollowed)
     do
-      {:ok, follower, unfollowed}
+      { :ok, follower, unfollowed}
     else
       err -> err
     end
@@ -207,7 +226,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
           media_id_string: "#{object.id}}",
           media_url: href,
           size: 0
-       } |> Poison.encode!
+        } |> Poison.encode!
     end
   end
 
@@ -215,36 +234,15 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
     # Modified from https://www.w3.org/TR/html5/forms.html#valid-e-mail-address
     regex = ~r/@[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@?[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/
 
-    regex
-    |> Regex.scan(text)
+    Regex.scan(regex, text)
     |> List.flatten
     |> Enum.uniq
-    |> Enum.map(fn ("@" <> match = full_match) ->
-      {full_match, User.get_cached_by_nickname(match)} end)
+    |> Enum.map(fn ("@" <> match = full_match) -> {full_match, User.get_cached_by_nickname(match)} end)
     |> Enum.filter(fn ({_match, user}) -> user end)
   end
 
   def add_user_links(text, mentions) do
-    Enum.reduce(mentions, text, fn ({match, %User{ap_id: ap_id}}, text) ->
-      String.replace(text, match, "<a href='#{ap_id}'>#{match}</a>") end)
-  end
-
-  defp add_conversation_id(activity) do
-    if is_integer(activity.data["statusnetConversationId"]) do
-      {:ok, activity}
-    else
-      data = activity.data
-      |> put_in(["object", "statusnetConversationId"], activity.id)
-      |> put_in(["statusnetConversationId"], activity.id)
-
-      object = Object.get_by_ap_id(activity.data["object"]["id"])
-
-      changeset = Changeset.change(object, data: data["object"])
-      Repo.update(changeset)
-
-      changeset = Changeset.change(activity, data: data)
-      Repo.update(changeset)
-    end
+    Enum.reduce(mentions, text, fn ({match, %User{ap_id: ap_id}}, text) -> String.replace(text, match, "<a href='#{ap_id}'>#{match}</a>") end)
   end
 
   def register_user(params) do
@@ -255,7 +253,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
       email: params["email"],
       password: params["password"],
       password_confirmation: params["confirm"]
-   }
+    }
 
     changeset = User.register_changeset(%User{}, params)
 
@@ -263,21 +261,22 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
       {:ok, UserRepresenter.to_map(user)}
     else
       {:error, changeset} ->
-        errors = Poison.encode!(Changeset.traverse_errors(changeset, fn {msg, _opts} -> msg end))
-        {:error, %{error: errors}}
+        errors = Ecto.Changeset.traverse_errors(changeset, fn {msg, _opts} -> msg end)
+      |> Poison.encode!
+      {:error, %{error: errors}}
     end
   end
 
   def get_user(user \\ nil, params) do
     case params do
-      %{"user_id" => user_id} ->
+      %{ "user_id" => user_id } ->
         case target = Repo.get(User, user_id) do
           nil ->
             {:error, "No user with such user_id"}
           _ ->
             {:ok, target}
         end
-      %{"screen_name" => nickname} ->
+      %{ "screen_name" => nickname } ->
         case target = Repo.get_by(User, nickname: nickname) do
           nil ->
             {:error, "No user with such screen_name"}
@@ -305,8 +304,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
     user = User.get_cached_by_ap_id(actor)
     [liked_activity] = Activity.all_by_object_ap_id(activity.data["object"])
 
-    ActivityRepresenter.to_map(activity,
-      Map.merge(opts, %{user: user, liked_activity: liked_activity}))
+    ActivityRepresenter.to_map(activity, Map.merge(opts, %{user: user, liked_activity: liked_activity}))
   end
 
   # For announces, fetch the announced activity and the user.
@@ -316,8 +314,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
     [announced_activity] = Activity.all_by_object_ap_id(activity.data["object"])
     announced_actor = User.get_cached_by_ap_id(announced_activity.data["actor"])
 
-    ActivityRepresenter.to_map(activity,
-      Map.merge(opts, %{users: [user, announced_actor], announced_activity: announced_activity}))
+    ActivityRepresenter.to_map(activity, Map.merge(opts, %{users: [user, announced_actor], announced_activity: announced_activity}))
   end
 
   defp activity_to_status(activity, opts) do
@@ -327,7 +324,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
     mentioned_users = Enum.map(activity.data["to"] || [], fn (ap_id) ->
       User.get_cached_by_ap_id(ap_id)
     end)
-    mentioned_users = mentioned_users |> Enum.filter(&(&1))
+    |> Enum.filter(&(&1))
 
     ActivityRepresenter.to_map(activity, Map.merge(opts, %{user: user, mentioned: mentioned_users}))
   end
@@ -335,4 +332,22 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
   defp make_date do
     DateTime.utc_now() |> DateTime.to_iso8601
   end
+
+  def context_to_conversation_id(context) do
+    with %Object{id: id} <- Object.get_cached_by_ap_id(context) do
+      id
+    else _e ->
+      changeset = Object.context_mapping(context)
+      {:ok, %{id: id}} = Repo.insert(changeset)
+      id
+    end
+  end
+
+  def conversation_id_to_context(id) do
+    with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do
+      context
+    else _e ->
+      {:error, "No such conversation"}
+    end
+  end
 end
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index bbfc04a6a..96a5f2151 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -42,6 +42,14 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
     end
   end
 
+  def public_and_external_timeline(%{assigns: %{user: user}} = conn, params) do
+    statuses = TwitterAPI.fetch_public_and_external_statuses(user, params)
+    {:ok, json} = Poison.encode(statuses)
+
+    conn
+    |> json_reply(200, json)
+  end
+
   def public_timeline(%{assigns: %{user: user}} = conn, params) do
     statuses = TwitterAPI.fetch_public_statuses(user, params)
     {:ok, json} = Poison.encode(statuses)
diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex
index 19b1ff848..2c343c2d7 100644
--- a/lib/pleroma/web/web.ex
+++ b/lib/pleroma/web/web.ex
@@ -58,28 +58,7 @@ defmodule Pleroma.Web do
     apply(__MODULE__, which, [])
   end
 
-  def host do
-    settings = Application.get_env(:pleroma, Pleroma.Web.Endpoint)
-    settings
-    |> Keyword.fetch!(:url)
-    |> Keyword.fetch!(:host)
-  end
-
   def base_url do
-    settings = Application.get_env(:pleroma, Pleroma.Web.Endpoint)
-
-    host = host()
-
-    protocol = settings |> Keyword.fetch!(:protocol)
-
-    port_fragment = with {:ok, protocol_info} <- settings
-                         |> Keyword.fetch(String.to_atom(protocol)),
-                         {:ok, port} <- protocol_info |> Keyword.fetch(:port)
-    do
-      ":#{port}"
-    else _e ->
-      ""
-    end
-    "#{protocol}://#{host}#{port_fragment}"
+    Pleroma.Web.Endpoint.url
   end
 end
diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex
index 3d6ca4e05..1eb26a89f 100644
--- a/lib/pleroma/web/web_finger/web_finger.ex
+++ b/lib/pleroma/web/web_finger/web_finger.ex
@@ -1,6 +1,9 @@
 defmodule Pleroma.Web.WebFinger do
-  alias Pleroma.{User, XmlBuilder}
-  alias Pleroma.{Web, Web.OStatus}
+
+  alias Pleroma.{Repo, User, XmlBuilder}
+  alias Pleroma.Web
+  alias Pleroma.Web.{XML, Salmon, OStatus}
+  require Logger
 
   def host_meta do
     base_url  = Web.base_url
@@ -14,25 +17,94 @@ defmodule Pleroma.Web.WebFinger do
   end
 
   def webfinger(resource) do
-    host = Web.host
-    regex = ~r/acct:(?<username>\w+)@#{host}/
-    case Regex.named_captures(regex, resource) do
-      %{"username" => username} ->
-        user = User.get_cached_by_nickname(username)
+    host = Pleroma.Web.Endpoint.host
+    regex = ~r/(acct:)?(?<username>\w+)@#{host}/
+    with %{"username" => username} <- Regex.named_captures(regex, resource) do
+      user = User.get_by_nickname(username)
+      {:ok, represent_user(user)}
+    else _e ->
+      with user when not is_nil(user) <- User.get_cached_by_ap_id(resource) do
         {:ok, represent_user(user)}
-      _ -> nil
+      else _e ->
+        {:error, "Couldn't find user"}
+      end
     end
   end
 
   def represent_user(user) do
+    {:ok, user} = ensure_keys_present(user)
+    {:ok, _private, public} = Salmon.keys_from_pem(user.info["keys"])
+    magic_key = Salmon.encode_key(public)
     {
       :XRD, %{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"},
       [
-        {:Subject, "acct:#{user.nickname}@#{Web.host}"},
+        {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.Endpoint.host}"},
         {:Alias, user.ap_id},
-        {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}}
+        {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}},
+        {:Link, %{rel: "http://webfinger.net/rel/profile-page", type: "text/html", href: user.ap_id}},
+        {:Link, %{rel: "salmon", href: OStatus.salmon_path(user)}},
+        {:Link, %{rel: "magic-public-key", href: "data:application/magic-public-key,#{magic_key}"}}
       ]
     }
     |> XmlBuilder.to_doc
   end
+
+  # This seems a better fit in Salmon
+  def ensure_keys_present(user) do
+    info = user.info || %{}
+    if info["keys"] do
+      {:ok, user}
+    else
+      {:ok, pem} = Salmon.generate_rsa_pem
+      info = Map.put(info, "keys", pem)
+      Repo.update(Ecto.Changeset.change(user, info: info))
+    end
+  end
+
+  # FIXME: Make this call the host-meta to find the actual address.
+  defp webfinger_address(domain) do
+    "//#{domain}/.well-known/webfinger"
+  end
+
+  defp webfinger_from_xml(doc) do
+    magic_key = XML.string_from_xpath(~s{//Link[@rel="magic-public-key"]/@href}, doc)
+    "data:application/magic-public-key," <> magic_key = magic_key
+    topic = XML.string_from_xpath(~s{//Link[@rel="http://schemas.google.com/g/2010#updates-from"]/@href}, doc)
+    subject = XML.string_from_xpath("//Subject", doc)
+    salmon = XML.string_from_xpath(~s{//Link[@rel="salmon"]/@href}, doc)
+    data = %{
+      "magic_key" => magic_key,
+      "topic" => topic,
+      "subject" => subject,
+      "salmon" => salmon
+    }
+    {:ok, data}
+  end
+
+  def finger(account, getter \\ &HTTPoison.get/3) do
+    domain = with [_name, domain] <- String.split(account, "@") do
+               domain
+             else _e ->
+               URI.parse(account).host
+             end
+    address = webfinger_address(domain)
+
+    # try https first
+    response = with {:ok, result} <- getter.("https:" <> address, ["Accept": "application/xrd+xml"], [params: [resource: account]]) do
+                 {:ok, result}
+               else _ ->
+                 getter.("http:" <> address, ["Accept": "application/xrd+xml"], [params: [resource: account], follow_redirect: true])
+               end
+
+    with {:ok, %{status_code: status_code, body: body}} when status_code in 200..299 <- response,
+         doc <- XML.parse_document(body),
+         {:ok, data} <- webfinger_from_xml(doc) do
+      {:ok, data}
+    else
+      e ->
+        Logger.debug("Couldn't finger #{account}.")
+        Logger.debug(inspect(e))
+        {:error, e}
+    end
+  end
 end
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index ba699db24..afbe944c5 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -1,9 +1,11 @@
 defmodule Pleroma.Web.Websub do
   alias Ecto.Changeset
   alias Pleroma.Repo
-  alias Pleroma.Web.Websub.WebsubServerSubscription
+  alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription}
   alias Pleroma.Web.OStatus.FeedRepresenter
-  alias Pleroma.Web.OStatus
+  alias Pleroma.Web.{XML, Endpoint, OStatus}
+  alias Pleroma.Web.Router.Helpers
+  require Logger
 
   import Ecto.Query
 
@@ -44,8 +46,10 @@ defmodule Pleroma.Web.Websub do
       response = user
       |> FeedRepresenter.to_simple_form([activity], [user])
       |> :xmerl.export_simple(:xmerl_xml)
+      |> to_string
 
-      signature = Base.encode16(:crypto.hmac(:sha, sub.secret, response))
+      signature = sign(sub.secret || "", response)
+      Logger.debug("Pushing to #{sub.callback}")
 
       HTTPoison.post(sub.callback, response, [
             {"Content-Type", "application/atom+xml"},
@@ -54,6 +58,10 @@ defmodule Pleroma.Web.Websub do
     end)
   end
 
+  def sign(secret, doc) do
+    :crypto.hmac(:sha, secret, to_string(doc)) |> Base.encode16 |> String.downcase
+  end
+
   def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) do
     with {:ok, topic} <- valid_topic(params, user),
          {:ok, lease_time} <- lease_time(params),
@@ -75,11 +83,13 @@ defmodule Pleroma.Web.Websub do
                                           NaiveDateTime.add(websub.updated_at, lease_time)})
       websub = Repo.update!(change)
 
-      # Just spawn that for now, maybe pool later.
-      spawn(fn -> @websub_verifier.verify(websub) end)
+      Pleroma.Web.Federator.enqueue(:verify_websub, websub)
 
       {:ok, websub}
     else {:error, reason} ->
+      Logger.debug("Couldn't create subscription.")
+      Logger.debug(inspect(reason))
+
       {:error, reason}
     end
   end
@@ -89,6 +99,11 @@ defmodule Pleroma.Web.Websub do
       %WebsubServerSubscription{}
   end
 
+  # Temp hack for mastodon.
+  defp lease_time(%{"hub.lease_seconds" => ""}) do
+    {:ok, 60 * 60 * 24 * 3} # three days
+  end
+
   defp lease_time(%{"hub.lease_seconds" => lease_seconds}) do
     {:ok, String.to_integer(lease_seconds)}
   end
@@ -99,9 +114,92 @@ defmodule Pleroma.Web.Websub do
 
   defp valid_topic(%{"hub.topic" => topic}, user) do
     if topic == OStatus.feed_path(user) do
-      {:ok, topic}
+      {:ok, OStatus.feed_path(user)}
     else
       {:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"}
     end
   end
+
+  def subscribe(subscriber, subscribed, requester \\ &request_subscription/1) do
+    topic = subscribed.info["topic"]
+    # FIXME: Race condition, use transactions
+    {:ok, subscription} = with subscription when not is_nil(subscription) <- Repo.get_by(WebsubClientSubscription, topic: topic) do
+      subscribers = [subscriber.ap_id, subscription.subscribers] |> Enum.uniq
+      change = Ecto.Changeset.change(subscription, %{subscribers: subscribers})
+      Repo.update(change)
+    else _e ->
+      subscription = %WebsubClientSubscription{
+        topic: topic,
+        hub: subscribed.info["hub"],
+        subscribers: [subscriber.ap_id],
+        state: "requested",
+        secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64,
+        user: subscribed
+      }
+      Repo.insert(subscription)
+    end
+    requester.(subscription)
+  end
+
+  def gather_feed_data(topic, getter \\ &HTTPoison.get/1) do
+    with {:ok, response} <- getter.(topic),
+         status_code when status_code in 200..299 <- response.status_code,
+         body <- response.body,
+         doc <- XML.parse_document(body),
+         uri when not is_nil(uri) <- XML.string_from_xpath("/feed/author[1]/uri", doc),
+         hub when not is_nil(hub) <- XML.string_from_xpath(~S{/feed/link[@rel="hub"]/@href}, doc) do
+
+      name = XML.string_from_xpath("/feed/author[1]/name", doc)
+      preferredUsername = XML.string_from_xpath("/feed/author[1]/poco:preferredUsername", doc)
+      displayName = XML.string_from_xpath("/feed/author[1]/poco:displayName", doc)
+      avatar = OStatus.make_avatar_object(doc)
+
+      {:ok, %{
+        "uri" => uri,
+        "hub" => hub,
+        "nickname" => preferredUsername || name,
+        "name" => displayName || name,
+        "host" => URI.parse(uri).host,
+        "avatar" => avatar
+      }}
+    else e ->
+      {:error, e}
+    end
+  end
+
+  def request_subscription(websub, poster \\ &HTTPoison.post/3, timeout \\ 10_000) do
+    data = [
+      "hub.mode": "subscribe",
+      "hub.topic": websub.topic,
+      "hub.secret": websub.secret,
+      "hub.callback": Helpers.websub_url(Endpoint, :websub_subscription_confirmation, websub.id)
+    ]
+
+    # This checks once a second if we are confirmed yet
+    websub_checker = fn ->
+      helper = fn (helper) ->
+        :timer.sleep(1000)
+        websub = Repo.get_by(WebsubClientSubscription, id: websub.id, state: "accepted")
+        if websub, do: websub, else: helper.(helper)
+      end
+      helper.(helper)
+    end
+
+    task = Task.async(websub_checker)
+
+    with {:ok, %{status_code: 202}} <- poster.(websub.hub, {:form, data}, ["Content-type": "application/x-www-form-urlencoded"]),
+         {:ok, websub} <- Task.yield(task, timeout) do
+      {:ok, websub}
+    else e ->
+      Task.shutdown(task)
+
+      change = Ecto.Changeset.change(websub, %{state: "rejected"})
+      {:ok, websub} = Repo.update(change)
+
+      Logger.debug("Couldn't confirm subscription: #{inspect(websub)}")
+      Logger.debug("error: #{inspect(e)}")
+
+      {:error, websub}
+    end
+  end
 end
diff --git a/lib/pleroma/web/websub/websub_client_subscription.ex b/lib/pleroma/web/websub/websub_client_subscription.ex
new file mode 100644
index 000000000..c7a25ea22
--- /dev/null
+++ b/lib/pleroma/web/websub/websub_client_subscription.ex
@@ -0,0 +1,16 @@
+defmodule Pleroma.Web.Websub.WebsubClientSubscription do
+  use Ecto.Schema
+  alias Pleroma.User
+
+  schema "websub_client_subscriptions" do
+    field :topic, :string
+    field :secret, :string
+    field :valid_until, :naive_datetime
+    field :state, :string
+    field :subscribers, {:array, :string}, default: []
+    field :hub, :string
+    belongs_to :user, User
+
+    timestamps()
+  end
+end
diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex
index 5d54c6ef5..e860ec9e5 100644
--- a/lib/pleroma/web/websub/websub_controller.ex
+++ b/lib/pleroma/web/websub/websub_controller.ex
@@ -1,7 +1,11 @@
 defmodule Pleroma.Web.Websub.WebsubController do
   use Pleroma.Web, :controller
-  alias Pleroma.User
+  alias Pleroma.{Repo, User}
   alias Pleroma.Web.Websub
+  alias Pleroma.Web.Websub.WebsubClientSubscription
+  require Logger
+
+  @ostatus Application.get_env(:pleroma, :ostatus)
 
   def websub_subscription_request(conn, %{"nickname" => nickname} = params) do
     user = User.get_cached_by_nickname(nickname)
@@ -15,4 +19,32 @@ defmodule Pleroma.Web.Websub.WebsubController do
       |> send_resp(500, reason)
     end
   end
+
+  def websub_subscription_confirmation(conn, %{"id" => id, "hub.mode" => "subscribe", "hub.challenge" => challenge, "hub.topic" => topic}) do
+    with %WebsubClientSubscription{} = websub <- Repo.get_by(WebsubClientSubscription, id: id, topic: topic) do
+      change = Ecto.Changeset.change(websub, %{state: "accepted"})
+      {:ok, _websub} = Repo.update(change)
+      conn
+      |> send_resp(200, challenge)
+    else _e ->
+      conn
+      |> send_resp(500, "Error")
+    end
+  end
+
+  def websub_incoming(conn, %{"id" => id}) do
+    with "sha1=" <> signature <- hd(get_req_header(conn, "x-hub-signature")),
+         signature <- String.downcase(signature),
+         %WebsubClientSubscription{} = websub <- Repo.get(WebsubClientSubscription, id),
+         {:ok, body, _conn} = read_body(conn),
+         ^signature <- Websub.sign(websub.secret, body) do
+      @ostatus.handle_incoming(body)
+      conn
+      |> send_resp(200, "OK")
+    else _e ->
+      Logger.debug("Can't handle incoming subscription post")
+      conn
+      |> send_resp(500, "Error")
+    end
+  end
 end
diff --git a/lib/pleroma/web/xml/xml.ex b/lib/pleroma/web/xml/xml.ex
new file mode 100644
index 000000000..22faf72df
--- /dev/null
+++ b/lib/pleroma/web/xml/xml.ex
@@ -0,0 +1,19 @@
+defmodule Pleroma.Web.XML do
+  def string_from_xpath(xpath, doc) do
+    {:xmlObj, :string, res} = :xmerl_xpath.string('string(#{xpath})', doc)
+
+    res = res
+    |> to_string
+    |> String.trim
+
+    if res == "", do: nil, else: res
+  end
+
+  def parse_document(text) do
+    {doc, _rest} = text
+    |> :binary.bin_to_list
+    |> :xmerl_scan.string
+
+    doc
+  end
+end