Merge remote-tracking branch 'remotes/origin/develop' into 1560-non-federating-instances-routes-restrictions

# Conflicts:
#	lib/pleroma/plugs/static_fe_plug.ex

9 files changed, 281 insertions, 22 deletions

diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex
index 4acc1a3e0..9e65bedad 100644
--- a/lib/pleroma/activity/ir/topics.ex
+++ b/lib/pleroma/activity/ir/topics.ex
@@ -39,7 +39,7 @@ defmodule Pleroma.Activity.Ir.Topics do
     end
   end
 
-  defp item_creation_tags(tags, %{data: %{"type" => "Create"}} = object, activity) do
+  defp item_creation_tags(tags, object, %{data: %{"type" => "Create"}} = activity) do
     tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
   end
 
diff --git a/lib/pleroma/earmark_renderer.ex b/lib/pleroma/earmark_renderer.ex
new file mode 100644
index 000000000..6211a3b4a
--- /dev/null
+++ b/lib/pleroma/earmark_renderer.ex
@@ -0,0 +1,256 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+#
+# This file is derived from Earmark, under the following copyright:
+# Copyright © 2014 Dave Thomas, The Pragmatic Programmers
+# SPDX-License-Identifier: Apache-2.0
+# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex
+defmodule Pleroma.EarmarkRenderer do
+  @moduledoc false
+
+  alias Earmark.Block
+  alias Earmark.Context
+  alias Earmark.HtmlRenderer
+  alias Earmark.Options
+
+  import Earmark.Inline, only: [convert: 3]
+  import Earmark.Helpers.HtmlHelpers
+  import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2]
+  import Earmark.Context, only: [append: 2, set_value: 2]
+  import Earmark.Options, only: [get_mapper: 1]
+
+  @doc false
+  def render(blocks, %Context{options: %Options{}} = context) do
+    messages = get_messages(context)
+
+    {contexts, html} =
+      get_mapper(context.options).(
+        blocks,
+        &render_block(&1, put_in(context.options.messages, []))
+      )
+      |> Enum.unzip()
+
+    all_messages =
+      contexts
+      |> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end)
+
+    {put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()}
+  end
+
+  #############
+  # Paragraph #
+  #############
+  defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do
+    lines = convert(lines, lnb, context)
+    add_attrs(lines, "<p>#{lines.value}</p>", attrs, [], lnb)
+  end
+
+  ########
+  # Html #
+  ########
+  defp render_block(%Block.Html{html: html}, context) do
+    {context, html}
+  end
+
+  defp render_block(%Block.HtmlComment{lines: lines}, context) do
+    {context, lines}
+  end
+
+  defp render_block(%Block.HtmlOneline{html: html}, context) do
+    {context, html}
+  end
+
+  #########
+  # Ruler #
+  #########
+  defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do
+    add_attrs(context, "<hr />", attrs, [], lnb)
+  end
+
+  ###########
+  # Heading #
+  ###########
+  defp render_block(
+         %Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs},
+         context
+       ) do
+    converted = convert(content, lnb, context)
+    html = "<h#{level}>#{converted.value}</h#{level}>"
+    add_attrs(converted, html, attrs, [], lnb)
+  end
+
+  ##############
+  # Blockquote #
+  ##############
+
+  defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
+    {context1, body} = render(blocks, context)
+    html = "<blockquote>#{body}</blockquote>"
+    add_attrs(context1, html, attrs, [], lnb)
+  end
+
+  #########
+  # Table #
+  #########
+
+  defp render_block(
+         %Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs},
+         context
+       ) do
+    {context1, html} = add_attrs(context, "<table>", attrs, [], lnb)
+    context2 = set_value(context1, html)
+
+    context3 =
+      if header do
+        append(add_trs(append(context2, "<thead>"), [header], "th", aligns, lnb), "</thead>")
+      else
+        # Maybe an error, needed append(context, html)
+        context2
+      end
+
+    context4 = append(add_trs(append(context3, "<tbody>"), rows, "td", aligns, lnb), "</tbody>")
+
+    {context4, [context4.value, "</table>"]}
+  end
+
+  ########
+  # Code #
+  ########
+
+  defp render_block(
+         %Block.Code{lnb: lnb, language: language, attrs: attrs} = block,
+         %Context{options: options} = context
+       ) do
+    class =
+      if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: ""
+
+    tag = ~s[<pre><code#{class}>]
+    lines = options.render_code.(block)
+    html = ~s[#{tag}#{lines}</code></pre>]
+    add_attrs(context, html, attrs, [], lnb)
+  end
+
+  #########
+  # Lists #
+  #########
+
+  defp render_block(
+         %Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start},
+         context
+       ) do
+    {context1, content} = render(items, context)
+    html = "<#{type}#{start}>#{content}</#{type}>"
+    add_attrs(context1, html, attrs, [], lnb)
+  end
+
+  # format a single paragraph list item, and remove the para tags
+  defp render_block(
+         %Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs},
+         context
+       )
+       when length(blocks) == 1 do
+    {context1, content} = render(blocks, context)
+    content = Regex.replace(~r{</?p>}, content, "")
+    html = "<li>#{content}</li>"
+    add_attrs(context1, html, attrs, [], lnb)
+  end
+
+  # format a spaced list item
+  defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
+    {context1, content} = render(blocks, context)
+    html = "<li>#{content}</li>"
+    add_attrs(context1, html, attrs, [], lnb)
+  end
+
+  ##################
+  # Footnote Block #
+  ##################
+
+  defp render_block(%Block.FnList{blocks: footnotes}, context) do
+    items =
+      Enum.map(footnotes, fn note ->
+        blocks = append_footnote_link(note)
+        %Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks}
+      end)
+
+    {context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context)
+    {context1, Enum.join([~s[<div class="footnotes">], "<hr />", html, "</div>"])}
+  end
+
+  #######################################
+  # Isolated IALs are rendered as paras #
+  #######################################
+
+  defp render_block(%Block.Ial{verbatim: verbatim}, context) do
+    {context, "<p>{:#{verbatim}}</p>"}
+  end
+
+  ####################
+  # IDDef is ignored #
+  ####################
+
+  defp render_block(%Block.IdDef{}, context), do: {context, ""}
+
+  #####################################
+  # And here are the inline renderers #
+  #####################################
+
+  defdelegate br, to: HtmlRenderer
+  defdelegate codespan(text), to: HtmlRenderer
+  defdelegate em(text), to: HtmlRenderer
+  defdelegate strong(text), to: HtmlRenderer
+  defdelegate strikethrough(text), to: HtmlRenderer
+
+  defdelegate link(url, text), to: HtmlRenderer
+  defdelegate link(url, text, title), to: HtmlRenderer
+
+  defdelegate image(path, alt, title), to: HtmlRenderer
+
+  defdelegate footnote_link(ref, backref, number), to: HtmlRenderer
+
+  # Table rows
+  defp add_trs(context, rows, tag, aligns, lnb) do
+    numbered_rows =
+      rows
+      |> Enum.zip(Stream.iterate(lnb, &(&1 + 1)))
+
+    numbered_rows
+    |> Enum.reduce(context, fn {row, lnb}, ctx ->
+      append(add_tds(append(ctx, "<tr>"), row, tag, aligns, lnb), "</tr>")
+    end)
+  end
+
+  defp add_tds(context, row, tag, aligns, lnb) do
+    Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb))
+  end
+
+  defp add_td_fn(row, tag, aligns, lnb) do
+    fn n, ctx ->
+      style =
+        case Enum.at(aligns, n - 1, :default) do
+          :default -> ""
+          align -> " style=\"text-align: #{align}\""
+        end
+
+      col = Enum.at(row, n - 1)
+      converted = convert(col, lnb, set_messages(ctx, []))
+      append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}</#{tag}>")
+    end
+  end
+
+  ###############################
+  # Append Footnote Return Link #
+  ###############################
+
+  defdelegate append_footnote_link(note), to: HtmlRenderer
+  defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer
+
+  defdelegate render_code(lines), to: HtmlRenderer
+
+  defp code_classes(language, prefix) do
+    ["" | String.split(prefix || "")]
+    |> Enum.map(fn pfx -> "#{pfx}#{language}" end)
+    |> Enum.join(" ")
+  end
+end
diff --git a/lib/pleroma/plugs/static_fe_plug.ex b/lib/pleroma/plugs/static_fe_plug.ex
index 10843b4c8..156e6788e 100644
--- a/lib/pleroma/plugs/static_fe_plug.ex
+++ b/lib/pleroma/plugs/static_fe_plug.ex
@@ -21,9 +21,9 @@ defmodule Pleroma.Plugs.StaticFEPlug do
   defp enabled?, do: Pleroma.Config.get([:static_fe, :enabled], false)
 
   defp accepts_html?(conn) do
-    conn
-    |> get_req_header("accept")
-    |> List.first()
-    |> String.contains?("text/html")
+    case get_req_header(conn, "accept") do
+      [accept | _] -> String.contains?(accept, "text/html")
+      _ -> false
+    end
   end
 end
diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex
index f372829a2..36ff024a7 100644
--- a/lib/pleroma/plugs/uploaded_media.ex
+++ b/lib/pleroma/plugs/uploaded_media.ex
@@ -14,9 +14,14 @@ defmodule Pleroma.Plugs.UploadedMedia do
   # no slashes
   @path "media"
 
+  @default_cache_control_header "public, max-age=1209600"
+
   def init(_opts) do
     static_plug_opts =
-      []
+      [
+        headers: %{"cache-control" => @default_cache_control_header},
+        cache_control_for_etags: @default_cache_control_header
+      ]
       |> Keyword.put(:from, "__unconfigured_media_plug")
       |> Keyword.put(:at, "/__unconfigured_media_plug")
       |> Plug.Static.init()
diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex
index a281a00dc..8b713b8f4 100644
--- a/lib/pleroma/reverse_proxy/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex
@@ -7,7 +7,7 @@ defmodule Pleroma.ReverseProxy do
 
   @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++
                       ~w(if-unmodified-since if-none-match if-range range)
-  @resp_cache_headers ~w(etag date last-modified cache-control)
+  @resp_cache_headers ~w(etag date last-modified)
   @keep_resp_headers @resp_cache_headers ++
                        ~w(content-type content-disposition content-encoding content-range) ++
                        ~w(accept-ranges vary)
@@ -34,9 +34,6 @@ defmodule Pleroma.ReverseProxy do
   * request: `#{inspect(@keep_req_headers)}`
   * response: `#{inspect(@keep_resp_headers)}`
 
-  If no caching headers (`#{inspect(@resp_cache_headers)}`) are returned by upstream, `cache-control` will be
-  set to `#{inspect(@default_cache_control_header)}`.
-
   Options:
 
   * `redirect_on_failure` (default `false`). Redirects the client to the real remote URL if there's any HTTP
@@ -297,16 +294,17 @@ defmodule Pleroma.ReverseProxy do
 
   defp build_resp_cache_headers(headers, _opts) do
     has_cache? = Enum.any?(headers, fn {k, _} -> k in @resp_cache_headers end)
-    has_cache_control? = List.keymember?(headers, "cache-control", 0)
 
     cond do
-      has_cache? && has_cache_control? ->
-        headers
-
       has_cache? ->
-        # There's caching header present but no cache-control -- we need to explicitely override it
-        # to public as Plug defaults to "max-age=0, private, must-revalidate"
-        List.keystore(headers, "cache-control", 0, {"cache-control", "public"})
+        # There's caching header present but no cache-control -- we need to set our own
+        # as Plug defaults to "max-age=0, private, must-revalidate"
+        List.keystore(
+          headers,
+          "cache-control",
+          0,
+          {"cache-control", @default_cache_control_header}
+        )
 
       true ->
         List.keystore(
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index 47b7d2da3..175260bc2 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -745,14 +745,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
-  def list_statuses(%{assigns: %{user: admin}} = conn, params) do
+  def list_statuses(%{assigns: %{user: _admin}} = conn, params) do
     godmode = params["godmode"] == "true" || params["godmode"] == true
     local_only = params["local_only"] == "true" || params["local_only"] == true
     with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true
     {page, page_size} = page_params(params)
 
     activities =
-      ActivityPub.fetch_statuses(admin, %{
+      ActivityPub.fetch_statuses(nil, %{
         "godmode" => godmode,
         "local_only" => local_only,
         "limit" => page_size,
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 348fdedf1..635e7cd38 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -331,7 +331,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   def format_input(text, "text/markdown", options) do
     text
     |> Formatter.mentions_escape(options)
-    |> Earmark.as_html!()
+    |> Earmark.as_html!(%Earmark.Options{renderer: Pleroma.EarmarkRenderer})
     |> Formatter.linkify(options)
     |> Formatter.html_escape("text/html")
   end
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 118c3ac6f..72cb3ee27 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -12,7 +12,7 @@ defmodule Pleroma.Web.Endpoint do
   plug(Pleroma.Plugs.HTTPSecurityPlug)
   plug(Pleroma.Plugs.UploadedMedia)
 
-  @static_cache_control "public max-age=86400 must-revalidate"
+  @static_cache_control "public, no-cache"
 
   # InstanceStatic needs to be before Plug.Static to be able to override shipped-static files
   # If you're adding new paths to `only:` you'll need to configure them in InstanceStatic as well
diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
index f165c9965..37b389382 100644
--- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
@@ -86,6 +86,6 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do
   @spec get_or_make_app() :: {:ok, App.t()} | {:error, Ecto.Changeset.t()}
   defp get_or_make_app do
     %{client_name: @local_mastodon_name, redirect_uris: "."}
-    |> App.get_or_make(["read", "write", "follow", "push"])
+    |> App.get_or_make(["read", "write", "follow", "push", "admin"])
   end
 end