Merge branch 'develop' into feature/matstodon-statuses-by-name

1 files changed, 34 insertions, 0 deletions

diff --git a/lib/pleroma/repo_streamer.ex b/lib/pleroma/repo_streamer.ex
new file mode 100644
index 000000000..a4b71a1bb
--- /dev/null
+++ b/lib/pleroma/repo_streamer.ex
@@ -0,0 +1,34 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.RepoStreamer do
+  alias Pleroma.Repo
+  import Ecto.Query
+
+  def chunk_stream(query, chunk_size) do
+    Stream.unfold(0, fn
+      :halt ->
+        {[], :halt}
+
+      last_id ->
+        query
+        |> order_by(asc: :id)
+        |> where([r], r.id > ^last_id)
+        |> limit(^chunk_size)
+        |> Repo.all()
+        |> case do
+          [] ->
+            {[], :halt}
+
+          records ->
+            last_id = List.last(records).id
+            {records, last_id}
+        end
+    end)
+    |> Stream.take_while(fn
+      [] -> false
+      _ -> true
+    end)
+  end
+end