From c26724cc5580a13d9e7e7468860eff8e49e02ba2 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 17 Apr 2019 23:54:09 +0300 Subject: Remove finmoji and add a way to download emojis in packs These packs are stored in a git repo on pleroma gitlab --- lib/mix/tasks/pleroma/emoji.ex | 98 ++++++++++++++++++++++++++++++++ lib/pleroma/emoji.ex | 126 ++++++++++++++--------------------------- 2 files changed, 139 insertions(+), 85 deletions(-) create mode 100644 lib/mix/tasks/pleroma/emoji.ex (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex new file mode 100644 index 000000000..ffe733617 --- /dev/null +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -0,0 +1,98 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Emoji do + use Mix.Task + + @shortdoc "Manages Pleroma instance" + @moduledoc """ + """ + + defp fetch_manifest do + Tesla.get!("https://git.pleroma.social/vaartis/emoji-index/raw/master/index.json").body + |> Poison.decode!() + end + + def run(["ls-packs"]) do + Application.ensure_all_started(:hackney) + + manifest = fetch_manifest() + + Enum.each(manifest, fn {name, info} -> + to_print = [ + {"Name", name}, + {"Homepage", info["homepage"]}, + {"Description", info["description"]}, + {"License", info["license"]}, + {"Source", info["src"]} + ] + + for {param, value} <- to_print do + IO.puts(IO.ANSI.format([:bright, param, :normal, ": ", value])) + end + end) + end + + def run(["get-pack", pack_name]) do + Application.ensure_all_started(:hackney) + + manifest = fetch_manifest() + + if Map.has_key?(manifest, pack_name) do + pack = manifest[pack_name] + src_url = pack["src"] + + IO.puts( + IO.ANSI.format([ + "Downloading pack ", + :bright, + pack_name, + :normal, + " from ", + :underline, + src_url + ]) + ) + + binary_archive = Tesla.get!(src_url).body + + IO.puts("Unpacking #{pack_name} pack") + + static_path = Path.join(:code.priv_dir(:pleroma), "static") + + pack_path = + Path.join([ + static_path, + Pleroma.Config.get!([:instance, :static_dir]), + "emoji", + pack_name + ]) + + files_to_unzip = + Enum.map( + pack["files"], + fn {_, f} -> to_charlist(f) end + ) + + {:ok, _} = + :zip.unzip(binary_archive, + cwd: pack_path, + file_list: files_to_unzip + ) + + IO.puts("Wriring emoji.txt for the #{pack_name} pack") + + emoji_txt_str = + Enum.map( + pack["files"], + fn {shortcode, path} -> "#{shortcode}, /instance/static/emoji/#{pack_name}/#{path}" end + ) + |> Enum.join("\n") + + File.write!(Path.join(pack_path, "emoji.txt"), emoji_txt_str) + else + IO.puts(IO.ANSI.format([:bright, :red, "No pack named \"#{pack_name}\" found"])) + end + end +end diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 87c7f2cec..8ead7ea1e 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -14,6 +14,8 @@ defmodule Pleroma.Emoji do """ use GenServer + require Logger + @type pattern :: Regex.t() | module() | String.t() @type patterns :: pattern() | [pattern()] @type group_patterns :: keyword(patterns()) @@ -79,96 +81,50 @@ defmodule Pleroma.Emoji do end defp load do - finmoji_enabled = Keyword.get(Application.get_env(:pleroma, :instance), :finmoji_enabled) - shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || [] + static_path = Path.join(:code.priv_dir(:pleroma), "static") - emojis = - (load_finmoji(finmoji_enabled) ++ - load_from_file("config/emoji.txt") ++ - load_from_file("config/custom_emoji.txt") ++ - load_from_globs(shortcode_globs)) - |> Enum.reject(fn value -> value == nil end) + emoji_dir_path = + Path.join([ + static_path, + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ]) + + case File.ls(emoji_dir_path) do + {:error, :enoent} -> + # The custom emoji directory doesn't exist, + # don't do anything + nil + + {:error, e} -> + # There was some other error + Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}") + + {:ok, packs} -> + # Print the packs we've found + Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}") + + # compat thing for old custom emoji handling + shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || [] + + emojis = + # Add the things fro + # Deprecated? + (Enum.flat_map( + packs, + fn pack -> load_from_file(Path.join([emoji_dir_path, pack, "emoji.txt"])) end + ) ++ + load_from_file("config/emoji.txt") ++ + load_from_file("config/custom_emoji.txt") ++ + load_from_globs(shortcode_globs)) + |> Enum.reject(fn value -> value == nil end) + + true = :ets.insert(@ets, emojis) + end - true = :ets.insert(@ets, emojis) :ok end - @finmoji [ - "a_trusted_friend", - "alandislands", - "association", - "auroraborealis", - "baby_in_a_box", - "bear", - "black_gold", - "christmasparty", - "crosscountryskiing", - "cupofcoffee", - "education", - "fashionista_finns", - "finnishlove", - "flag", - "forest", - "four_seasons_of_bbq", - "girlpower", - "handshake", - "happiness", - "headbanger", - "icebreaker", - "iceman", - "joulutorttu", - "kaamos", - "kalsarikannit_f", - "kalsarikannit_m", - "karjalanpiirakka", - "kicksled", - "kokko", - "lavatanssit", - "losthopes_f", - "losthopes_m", - "mattinykanen", - "meanwhileinfinland", - "moominmamma", - "nordicfamily", - "out_of_office", - "peacemaker", - "perkele", - "pesapallo", - "polarbear", - "pusa_hispida_saimensis", - "reindeer", - "sami", - "sauna_f", - "sauna_m", - "sauna_whisk", - "sisu", - "stuck", - "suomimainittu", - "superfood", - "swan", - "the_cap", - "the_conductor", - "the_king", - "the_voice", - "theoriginalsanta", - "tomoffinland", - "torillatavataan", - "unbreakable", - "waiting", - "white_nights", - "woollysocks" - ] - - defp load_finmoji(true) do - Enum.map(@finmoji, fn finmoji -> - file_name = "/finmoji/128px/#{finmoji}-128.png" - group = match_extra(@groups, file_name) - {finmoji, file_name, to_string(group)} - end) - end - - defp load_finmoji(_), do: [] - defp load_from_file(file) do if File.exists?(file) do load_from_file_stream(File.stream!(file)) -- cgit v1.2.3 From 21b39c54a36c265fee89a9c2f2312ad925f82263 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 18 Apr 2019 10:57:20 +0300 Subject: Allow using a custom manfest and getting multiple packs at once A custom manifest can be provided as a command-line options --manifest/-m --- lib/mix/tasks/pleroma/emoji.ex | 143 ++++++++++++++++++++++++----------------- 1 file changed, 83 insertions(+), 60 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index ffe733617..902bddc65 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -9,15 +9,31 @@ defmodule Mix.Tasks.Pleroma.Emoji do @moduledoc """ """ - defp fetch_manifest do - Tesla.get!("https://git.pleroma.social/vaartis/emoji-index/raw/master/index.json").body - |> Poison.decode!() + @default_manifest "https://git.pleroma.social/vaartis/emoji-index/raw/master/index.json" + + defp fetch_manifest(from) do + Tesla.get!(from).body |> Poison.decode!() + end + + defp parse_global_opts(args) do + OptionParser.parse( + args, + strict: [ + manifest: :string + ], + aliases: [ + m: :manifest + ] + ) end - def run(["ls-packs"]) do + def run(["ls-packs" | args]) do Application.ensure_all_started(:hackney) - manifest = fetch_manifest() + {options, [], []} = parse_global_opts(args) + + manifest = + fetch_manifest(if options[:manifest], do: options[:manifest], else: @default_manifest) Enum.each(manifest, fn {name, info} -> to_print = [ @@ -34,65 +50,72 @@ defmodule Mix.Tasks.Pleroma.Emoji do end) end - def run(["get-pack", pack_name]) do + def run(["get-packs" | args]) do Application.ensure_all_started(:hackney) - manifest = fetch_manifest() - - if Map.has_key?(manifest, pack_name) do - pack = manifest[pack_name] - src_url = pack["src"] - - IO.puts( - IO.ANSI.format([ - "Downloading pack ", - :bright, - pack_name, - :normal, - " from ", - :underline, - src_url - ]) - ) - - binary_archive = Tesla.get!(src_url).body - - IO.puts("Unpacking #{pack_name} pack") - - static_path = Path.join(:code.priv_dir(:pleroma), "static") - - pack_path = - Path.join([ - static_path, - Pleroma.Config.get!([:instance, :static_dir]), - "emoji", - pack_name - ]) - - files_to_unzip = - Enum.map( - pack["files"], - fn {_, f} -> to_charlist(f) end + {options, pack_names, []} = parse_global_opts(args) + + manifest = + fetch_manifest(if options[:manifest], do: options[:manifest], else: @default_manifest) + + for pack_name <- pack_names do + if Map.has_key?(manifest, pack_name) do + pack = manifest[pack_name] + src_url = pack["src"] + + IO.puts( + IO.ANSI.format([ + "Downloading ", + :bright, + pack_name, + :normal, + " from ", + :underline, + src_url + ]) ) - {:ok, _} = - :zip.unzip(binary_archive, - cwd: pack_path, - file_list: files_to_unzip - ) - - IO.puts("Wriring emoji.txt for the #{pack_name} pack") - - emoji_txt_str = - Enum.map( - pack["files"], - fn {shortcode, path} -> "#{shortcode}, /instance/static/emoji/#{pack_name}/#{path}" end - ) - |> Enum.join("\n") - - File.write!(Path.join(pack_path, "emoji.txt"), emoji_txt_str) - else - IO.puts(IO.ANSI.format([:bright, :red, "No pack named \"#{pack_name}\" found"])) + binary_archive = Tesla.get!(src_url).body + + IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name])) + + static_path = Path.join(:code.priv_dir(:pleroma), "static") + + pack_path = + Path.join([ + static_path, + Pleroma.Config.get!([:instance, :static_dir]), + "emoji", + pack_name + ]) + + files_to_unzip = + Enum.map( + pack["files"], + fn {_, f} -> to_charlist(f) end + ) + + {:ok, _} = + :zip.unzip(binary_archive, + cwd: pack_path, + file_list: files_to_unzip + ) + + IO.puts(IO.ANSI.format(["Writing emoji.txt for ", :bright, pack_name])) + + emoji_txt_str = + Enum.map( + pack["files"], + fn {shortcode, path} -> + "#{shortcode}, /instance/static/emoji/#{pack_name}/#{path}" + end + ) + |> Enum.join("\n") + + File.write!(Path.join(pack_path, "emoji.txt"), emoji_txt_str) + else + IO.puts(IO.ANSI.format([:bright, :red, "No pack named \"#{pack_name}\" found"])) + end end end end -- cgit v1.2.3 From af5494f942636bc6d2baa2638502974ed8cb7846 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 18 Apr 2019 15:32:18 +0300 Subject: Separate emoji pack file lists in a different file The file should be in the same directory as the manifest file --- lib/mix/tasks/pleroma/emoji.ex | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 902bddc65..526b09b11 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -55,8 +55,9 @@ defmodule Mix.Tasks.Pleroma.Emoji do {options, pack_names, []} = parse_global_opts(args) - manifest = - fetch_manifest(if options[:manifest], do: options[:manifest], else: @default_manifest) + manifest_url = if options[:manifest], do: options[:manifest], else: @default_manifest + + manifest = fetch_manifest(manifest_url) for pack_name <- pack_names do if Map.has_key?(manifest, pack_name) do @@ -77,6 +78,23 @@ defmodule Mix.Tasks.Pleroma.Emoji do binary_archive = Tesla.get!(src_url).body + # The url specified in files should be in the same directory + files_url = Path.join(Path.dirname(manifest_url), pack["files"]) + + IO.puts( + IO.ANSI.format([ + "Fetching the file list for ", + :bright, + pack_name, + :normal, + " from ", + :underline, + files_url + ]) + ) + + files = Tesla.get!(files_url).body |> Poison.decode!() + IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name])) static_path = Path.join(:code.priv_dir(:pleroma), "static") @@ -91,7 +109,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do files_to_unzip = Enum.map( - pack["files"], + files, fn {_, f} -> to_charlist(f) end ) @@ -105,7 +123,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do emoji_txt_str = Enum.map( - pack["files"], + files, fn {shortcode, path} -> "#{shortcode}, /instance/static/emoji/#{pack_name}/#{path}" end -- cgit v1.2.3 From 06db3ee1a8a443316196e6d8f55f4d5fc0cac694 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 18 Apr 2019 15:46:07 +0300 Subject: Add MD5 verification for emoji pack source --- lib/mix/tasks/pleroma/emoji.ex | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 526b09b11..29c5d0c93 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -77,6 +77,16 @@ defmodule Mix.Tasks.Pleroma.Emoji do ) binary_archive = Tesla.get!(src_url).body + archive_md5 = :crypto.hash(:md5, binary_archive) |> Base.encode16() + + md5_status_text = ["MD5 of ", :bright, pack_name, :normal, " source file is ", :bright] + if archive_md5 == String.upcase(pack["src_md5"]) do + IO.puts(IO.ANSI.format(md5_status_text ++ [:green, "OK"])) + else + IO.puts(IO.ANSI.format(md5_status_text ++ [:red, "BAD"])) + + raise "Bad MD5 for #{pack_name}" + end # The url specified in files should be in the same directory files_url = Path.join(Path.dirname(manifest_url), pack["files"]) -- cgit v1.2.3 From c5b7286b5f3fd2a3eb91eea74bebb684575682bd Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 18 Apr 2019 15:47:49 +0300 Subject: Move helper functions of emoji manager task down in the file --- lib/mix/tasks/pleroma/emoji.ex | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 29c5d0c93..71d08411f 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -11,22 +11,6 @@ defmodule Mix.Tasks.Pleroma.Emoji do @default_manifest "https://git.pleroma.social/vaartis/emoji-index/raw/master/index.json" - defp fetch_manifest(from) do - Tesla.get!(from).body |> Poison.decode!() - end - - defp parse_global_opts(args) do - OptionParser.parse( - args, - strict: [ - manifest: :string - ], - aliases: [ - m: :manifest - ] - ) - end - def run(["ls-packs" | args]) do Application.ensure_all_started(:hackney) @@ -146,4 +130,20 @@ defmodule Mix.Tasks.Pleroma.Emoji do end end end + + defp fetch_manifest(from) do + Tesla.get!(from).body |> Poison.decode!() + end + + defp parse_global_opts(args) do + OptionParser.parse( + args, + strict: [ + manifest: :string + ], + aliases: [ + m: :manifest + ] + ) + end end -- cgit v1.2.3 From 1e44b5478da83f5841cbe292812b27b4539b975c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 18 Apr 2019 15:57:41 +0300 Subject: Remove useless comment --- lib/pleroma/emoji.ex | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 8ead7ea1e..35e6eee93 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -108,8 +108,6 @@ defmodule Pleroma.Emoji do shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || [] emojis = - # Add the things fro - # Deprecated? (Enum.flat_map( packs, fn pack -> load_from_file(Path.join([emoji_dir_path, pack, "emoji.txt"])) end -- cgit v1.2.3 From eff725c3af4537b2d993f85a236636cd5d5e17d0 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 18 Apr 2019 17:02:22 +0300 Subject: Add a task to generate emoji packs --- lib/mix/tasks/pleroma/emoji.ex | 103 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 71d08411f..2126588b1 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -131,6 +131,109 @@ defmodule Mix.Tasks.Pleroma.Emoji do end end + def run(["gen-pack", src]) do + Application.ensure_all_started(:hackney) + + proposed_name = Path.basename(src) |> Path.rootname() + name = String.trim(IO.gets("Pack name [#{proposed_name}]: ")) + # If there's no name, use the default one + name = if String.length(name) > 0, do: name, else: proposed_name + + license = String.trim(IO.gets("License: ")) + homepage = String.trim(IO.gets("Homepage: ")) + description = String.trim(IO.gets("Description: ")) + + proposed_files_name = "#{name}.json" + files_name = String.trim(IO.gets("Save file list to [#{proposed_files_name}]: ")) + files_name = if String.length(files_name) > 0, do: files_name, else: proposed_files_name + + default_exts = [".png", ".gif"] + default_exts_str = Enum.join(default_exts, " ") + exts = + String.trim(IO.gets("Emoji file extensions (separated with spaces) [#{default_exts_str}]: ")) + exts = if String.length(exts) > 0 do + String.split(exts, " ") |> Enum.filter(fn e -> (e |> String.trim() |> String.length()) > 0 end) + else + default_exts + end + + IO.puts "Downloading the pack and generating MD5" + + binary_archive = Tesla.get!(src).body + archive_md5 = :crypto.hash(:md5, binary_archive) |> Base.encode16() + + IO.puts "MD5 is #{archive_md5}" + + pack_json = %{ + name => %{ + license: license, + homepage: homepage, + description: description, + src: src, + src_md5: archive_md5, + files: files_name + } + } + + tmp_pack_dir = Path.join(System.tmp_dir!(), "emoji-pack-#{name}") + {:ok, _} = + :zip.unzip( + binary_archive, + cwd: tmp_pack_dir + ) + + emoji_map = + find_all_emoji(tmp_pack_dir, exts) |> + Enum.map(&Path.relative_to(&1, tmp_pack_dir)) |> + Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end) |> + Enum.into(%{}) + + File.write!(files_name, Poison.encode!(emoji_map, pretty: true)) + + IO.puts """ + + #{files_name} has been created and contains the list of all found emojis in the pack. + Please review the files in the remove those not needed. + """ + + if File.exists?("index.json") do + existing_data = File.read!("index.json") |> Poison.decode!() + + File.write!( + "index.json", + Poison.encode!( + Map.merge( + existing_data, + pack_json + ), + pretty: true + ) + ) + + IO.puts "index.json file has been update with the #{name} pack" + else + File.write!("index.json", Poison.encode!(pack_json, pretty: true)) + + IO.puts "index.json has been created with the #{name} pack" + end + + end + + defp find_all_emoji(dir, exts) do + Enum.reduce( + File.ls!(dir), + [], + fn f, acc -> + filepath = Path.join(dir, f) + if File.dir?(filepath) do + acc ++ find_all_emoji(filepath, exts) + else + acc ++ [filepath] + end + end + ) |> Enum.filter(fn f -> Path.extname(f) in exts end) + end + defp fetch_manifest(from) do Tesla.get!(from).body |> Poison.decode!() end -- cgit v1.2.3 From a141f0807bc84868fb84e3d628ab8f99f429d5c0 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 18 Apr 2019 18:04:02 +0300 Subject: Assume packs without emoji.txt only have emoji pictures, unhardcode unhardcode: remove hardcoded /instance/static and actually use the config option as it is used in other places. packs without emoji.txt: these are now assumed to have .png files that are all emojis, their names are used as shortcodes --- lib/mix/tasks/pleroma/emoji.ex | 27 +++++--------------- lib/pleroma/emoji.ex | 58 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 57 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 2126588b1..4fb383b61 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -115,11 +115,14 @@ defmodule Mix.Tasks.Pleroma.Emoji do IO.puts(IO.ANSI.format(["Writing emoji.txt for ", :bright, pack_name])) + common_pack_path = Path.join([ + "/", Pleroma.Config.get!([:instance, :static_dir]), "emoji", pack_name + ]) emoji_txt_str = Enum.map( files, fn {shortcode, path} -> - "#{shortcode}, /instance/static/emoji/#{pack_name}/#{path}" + "#{shortcode}, #{Path.join(common_pack_path, path)}" end ) |> Enum.join("\n") @@ -182,11 +185,8 @@ defmodule Mix.Tasks.Pleroma.Emoji do cwd: tmp_pack_dir ) - emoji_map = - find_all_emoji(tmp_pack_dir, exts) |> - Enum.map(&Path.relative_to(&1, tmp_pack_dir)) |> - Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end) |> - Enum.into(%{}) + emoji_map = Pleroma.Emoji.make_shortcode_to_file_map(tmp_pack_dir, exts) + File.write!(files_name, Poison.encode!(emoji_map, pretty: true)) @@ -219,21 +219,6 @@ defmodule Mix.Tasks.Pleroma.Emoji do end - defp find_all_emoji(dir, exts) do - Enum.reduce( - File.ls!(dir), - [], - fn f, acc -> - filepath = Path.join(dir, f) - if File.dir?(filepath) do - acc ++ find_all_emoji(filepath, exts) - else - acc ++ [filepath] - end - end - ) |> Enum.filter(fn f -> Path.extname(f) in exts end) - end - defp fetch_manifest(from) do Tesla.get!(from).body |> Poison.decode!() end diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 35e6eee93..c7620b573 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -108,13 +108,13 @@ defmodule Pleroma.Emoji do shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || [] emojis = - (Enum.flat_map( - packs, - fn pack -> load_from_file(Path.join([emoji_dir_path, pack, "emoji.txt"])) end - ) ++ - load_from_file("config/emoji.txt") ++ - load_from_file("config/custom_emoji.txt") ++ - load_from_globs(shortcode_globs)) + (Enum.flat_map( + packs, + fn pack -> load_pack(Path.join(emoji_dir_path, pack)) end + ) ++ + load_from_file("config/emoji.txt") ++ + load_from_file("config/custom_emoji.txt") ++ + load_from_globs(shortcode_globs)) |> Enum.reject(fn value -> value == nil end) true = :ets.insert(@ets, emojis) @@ -123,6 +123,50 @@ defmodule Pleroma.Emoji do :ok end + defp load_pack(pack_dir) do + pack_name = Path.basename(pack_dir) + + emoji_txt = Path.join(pack_dir, "emoji.txt") + if File.exists?(emoji_txt) do + load_from_file(emoji_txt) + else + Logger.info("No emoji.txt found for pack \"#{pack_name}\", assuming all .png files are emoji") + + common_pack_path = Path.join([ + "/", Pleroma.Config.get!([:instance, :static_dir]), "emoji", pack_name + ]) + make_shortcode_to_file_map(pack_dir, [".png"]) |> + Enum.map(fn {shortcode, rel_file} -> + filename = Path.join(common_pack_path, rel_file) + + # If no tag matches, use the pack name as a tag + {shortcode, filename, to_string(match_extra(@groups, filename))} + end) + end + end + + def make_shortcode_to_file_map(pack_dir, exts) do + find_all_emoji(pack_dir, exts) |> + Enum.map(&Path.relative_to(&1, pack_dir)) |> + Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end) |> + Enum.into(%{}) + end + + def find_all_emoji(dir, exts) do + Enum.reduce( + File.ls!(dir), + [], + fn f, acc -> + filepath = Path.join(dir, f) + if File.dir?(filepath) do + acc ++ find_all_emoji(filepath, exts) + else + acc ++ [filepath] + end + end + ) |> Enum.filter(fn f -> Path.extname(f) in exts end) + end + defp load_from_file(file) do if File.exists?(file) do load_from_file_stream(File.stream!(file)) -- cgit v1.2.3 From aaaa428512db8ace56ca5ab7ebf1488d64ac5e35 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 18 Apr 2019 18:09:43 +0300 Subject: mix format --- lib/mix/tasks/pleroma/emoji.ex | 45 +++++++++++++++++++------------- lib/pleroma/emoji.ex | 58 +++++++++++++++++++++++++----------------- 2 files changed, 62 insertions(+), 41 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 4fb383b61..fed3dcb40 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -64,6 +64,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do archive_md5 = :crypto.hash(:md5, binary_archive) |> Base.encode16() md5_status_text = ["MD5 of ", :bright, pack_name, :normal, " source file is ", :bright] + if archive_md5 == String.upcase(pack["src_md5"]) do IO.puts(IO.ANSI.format(md5_status_text ++ [:green, "OK"])) else @@ -115,9 +116,14 @@ defmodule Mix.Tasks.Pleroma.Emoji do IO.puts(IO.ANSI.format(["Writing emoji.txt for ", :bright, pack_name])) - common_pack_path = Path.join([ - "/", Pleroma.Config.get!([:instance, :static_dir]), "emoji", pack_name - ]) + common_pack_path = + Path.join([ + "/", + Pleroma.Config.get!([:instance, :static_dir]), + "emoji", + pack_name + ]) + emoji_txt_str = Enum.map( files, @@ -152,20 +158,26 @@ defmodule Mix.Tasks.Pleroma.Emoji do default_exts = [".png", ".gif"] default_exts_str = Enum.join(default_exts, " ") + exts = - String.trim(IO.gets("Emoji file extensions (separated with spaces) [#{default_exts_str}]: ")) - exts = if String.length(exts) > 0 do - String.split(exts, " ") |> Enum.filter(fn e -> (e |> String.trim() |> String.length()) > 0 end) - else - default_exts - end + String.trim( + IO.gets("Emoji file extensions (separated with spaces) [#{default_exts_str}]: ") + ) - IO.puts "Downloading the pack and generating MD5" + exts = + if String.length(exts) > 0 do + String.split(exts, " ") + |> Enum.filter(fn e -> e |> String.trim() |> String.length() > 0 end) + else + default_exts + end + + IO.puts("Downloading the pack and generating MD5") binary_archive = Tesla.get!(src).body archive_md5 = :crypto.hash(:md5, binary_archive) |> Base.encode16() - IO.puts "MD5 is #{archive_md5}" + IO.puts("MD5 is #{archive_md5}") pack_json = %{ name => %{ @@ -179,6 +191,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do } tmp_pack_dir = Path.join(System.tmp_dir!(), "emoji-pack-#{name}") + {:ok, _} = :zip.unzip( binary_archive, @@ -187,14 +200,13 @@ defmodule Mix.Tasks.Pleroma.Emoji do emoji_map = Pleroma.Emoji.make_shortcode_to_file_map(tmp_pack_dir, exts) - File.write!(files_name, Poison.encode!(emoji_map, pretty: true)) - IO.puts """ + IO.puts(""" #{files_name} has been created and contains the list of all found emojis in the pack. Please review the files in the remove those not needed. - """ + """) if File.exists?("index.json") do existing_data = File.read!("index.json") |> Poison.decode!() @@ -210,13 +222,12 @@ defmodule Mix.Tasks.Pleroma.Emoji do ) ) - IO.puts "index.json file has been update with the #{name} pack" + IO.puts("index.json file has been update with the #{name} pack") else File.write!("index.json", Poison.encode!(pack_json, pretty: true)) - IO.puts "index.json has been created with the #{name} pack" + IO.puts("index.json has been created with the #{name} pack") end - end defp fetch_manifest(from) do diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index c7620b573..9a23bd81b 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -108,13 +108,13 @@ defmodule Pleroma.Emoji do shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || [] emojis = - (Enum.flat_map( - packs, - fn pack -> load_pack(Path.join(emoji_dir_path, pack)) end - ) ++ - load_from_file("config/emoji.txt") ++ - load_from_file("config/custom_emoji.txt") ++ - load_from_globs(shortcode_globs)) + (Enum.flat_map( + packs, + fn pack -> load_pack(Path.join(emoji_dir_path, pack)) end + ) ++ + load_from_file("config/emoji.txt") ++ + load_from_file("config/custom_emoji.txt") ++ + load_from_globs(shortcode_globs)) |> Enum.reject(fn value -> value == nil end) true = :ets.insert(@ets, emojis) @@ -127,29 +127,37 @@ defmodule Pleroma.Emoji do pack_name = Path.basename(pack_dir) emoji_txt = Path.join(pack_dir, "emoji.txt") + if File.exists?(emoji_txt) do load_from_file(emoji_txt) else - Logger.info("No emoji.txt found for pack \"#{pack_name}\", assuming all .png files are emoji") - - common_pack_path = Path.join([ - "/", Pleroma.Config.get!([:instance, :static_dir]), "emoji", pack_name - ]) - make_shortcode_to_file_map(pack_dir, [".png"]) |> - Enum.map(fn {shortcode, rel_file} -> - filename = Path.join(common_pack_path, rel_file) - - # If no tag matches, use the pack name as a tag - {shortcode, filename, to_string(match_extra(@groups, filename))} - end) + Logger.info( + "No emoji.txt found for pack \"#{pack_name}\", assuming all .png files are emoji" + ) + + common_pack_path = + Path.join([ + "/", + Pleroma.Config.get!([:instance, :static_dir]), + "emoji", + pack_name + ]) + + make_shortcode_to_file_map(pack_dir, [".png"]) + |> Enum.map(fn {shortcode, rel_file} -> + filename = Path.join(common_pack_path, rel_file) + + # If no tag matches, use the pack name as a tag + {shortcode, filename, to_string(match_extra(@groups, filename))} + end) end end def make_shortcode_to_file_map(pack_dir, exts) do - find_all_emoji(pack_dir, exts) |> - Enum.map(&Path.relative_to(&1, pack_dir)) |> - Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end) |> - Enum.into(%{}) + find_all_emoji(pack_dir, exts) + |> Enum.map(&Path.relative_to(&1, pack_dir)) + |> Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end) + |> Enum.into(%{}) end def find_all_emoji(dir, exts) do @@ -158,13 +166,15 @@ defmodule Pleroma.Emoji do [], fn f, acc -> filepath = Path.join(dir, f) + if File.dir?(filepath) do acc ++ find_all_emoji(filepath, exts) else acc ++ [filepath] end end - ) |> Enum.filter(fn f -> Path.extname(f) in exts end) + ) + |> Enum.filter(fn f -> Path.extname(f) in exts end) end defp load_from_file(file) do -- cgit v1.2.3 From 98d4b3de53a5eaf412e3b200d4f0ed04c9c4622d Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 18 Apr 2019 20:06:59 +0300 Subject: Treat the manifest path as a file if it doesn't start with http --- lib/mix/tasks/pleroma/emoji.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index fed3dcb40..9cb6fb88d 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -231,7 +231,13 @@ defmodule Mix.Tasks.Pleroma.Emoji do end defp fetch_manifest(from) do - Tesla.get!(from).body |> Poison.decode!() + Poison.decode!( + if String.starts_with?(from, "http") do + Tesla.get!(from).body + else + File.read!(from) + end + ) end defp parse_global_opts(args) do -- cgit v1.2.3 From 9bd5e2dec9ce0b23f287b3ea6ad375280d92bb7b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 18 Apr 2019 20:48:57 +0300 Subject: Make emoji default_manifest a config option --- lib/mix/tasks/pleroma/emoji.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 9cb6fb88d..8261c2122 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do @moduledoc """ """ - @default_manifest "https://git.pleroma.social/vaartis/emoji-index/raw/master/index.json" + @default_manifest Pleroma.Config.get!([:emoji, :default_manifest]) def run(["ls-packs" | args]) do Application.ensure_all_started(:hackney) -- cgit v1.2.3 From 52ed287e87ea18fdbf14695ccfafae00768299dc Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 18 Apr 2019 21:17:52 +0300 Subject: Fix incorrect multiple emoji tag handling introduced in initial impl --- lib/pleroma/emoji.ex | 13 ++++++------- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 +- lib/pleroma/web/twitter_api/controllers/util_controller.ex | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 9a23bd81b..15455b779 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -147,8 +147,7 @@ defmodule Pleroma.Emoji do |> Enum.map(fn {shortcode, rel_file} -> filename = Path.join(common_pack_path, rel_file) - # If no tag matches, use the pack name as a tag - {shortcode, filename, to_string(match_extra(@groups, filename))} + {shortcode, filename, [to_string(match_extra(@groups, filename))]} end) end end @@ -190,11 +189,11 @@ defmodule Pleroma.Emoji do |> Stream.map(&String.trim/1) |> Stream.map(fn line -> case String.split(line, ~r/,\s*/) do - [name, file, tags] -> - {name, file, tags} - [name, file] -> - {name, file, to_string(match_extra(@groups, file))} + {name, file, [to_string(match_extra(@groups, file))]} + + [name, file | tags] -> + {name, file, tags} _ -> nil @@ -217,7 +216,7 @@ defmodule Pleroma.Emoji do tag = match_extra(@groups, Path.join("/", Path.relative_to(path, static_path))) shortcode = Path.basename(path, Path.extname(path)) external_path = Path.join("/", Path.relative_to(path, static_path)) - {shortcode, external_path, to_string(tag)} + {shortcode, external_path, [to_string(tag)]} end) end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 3916d7c41..2a81dcc67 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -190,7 +190,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do "static_url" => url, "visible_in_picker" => true, "url" => url, - "tags" => String.split(tags, ",") + "tags" => tags } end) end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 9441984c7..8665e058a 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -286,7 +286,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do emoji = Emoji.get_all() |> Enum.map(fn {short_code, path, tags} -> - {short_code, %{image_url: path, tags: String.split(tags, ",")}} + {short_code, %{image_url: path, tags: tags}} end) |> Enum.into(%{}) -- cgit v1.2.3 From 1e311d6662812377f2ed8c4483754d5876d82631 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 20 Apr 2019 00:22:11 +0300 Subject: Add a newline at the end of pack in ls-packs --- lib/mix/tasks/pleroma/emoji.ex | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 8261c2122..f4da183ad 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -31,6 +31,9 @@ defmodule Mix.Tasks.Pleroma.Emoji do for {param, value} <- to_print do IO.puts(IO.ANSI.format([:bright, param, :normal, ": ", value])) end + + # A newline + IO.puts("") end) end -- cgit v1.2.3 From 31cff7dbcaf7d8087fb2c8eef2b949820fd5767c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 20 Apr 2019 10:57:31 +0300 Subject: Document the pleroma.emoji task --- lib/mix/tasks/pleroma/emoji.ex | 44 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index f4da183ad..0a1bf24e2 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -5,8 +5,50 @@ defmodule Mix.Tasks.Pleroma.Emoji do use Mix.Task - @shortdoc "Manages Pleroma instance" + @shortdoc "Manages emoji packs" @moduledoc """ + Manages emoji packs + + ## ls-packs + + mix pleroma.emoji ls-packs [OPTION...] + + Lists the emoji packs and metadata specified in the manifest. + + ### Options + + - `-m, --manifest PATH/URL` - path to a custom manifest, it can either be an URL + starting with `http`, in that case the manifest will be fetched from that address, + or a local path + + ## get-packs + + mix pleroma.emoji get-packs [OPTION...] PACKS + + Fetches, verifies and installs the specified PACKS from the manifest into + the `STATIC-DIR/emoji/PACK-NAME + + ### Options + + - `-m, --manifest PATH/URL` - same as ls-packs + + ## gen-pack + + mix pleroma.emoji gen-pack PACK-URL + + Creates a new manifest entry and a file list from the specified remote pack file. + Currently, only .zip archives are recognized as remote pack files and packs are therefore + assumed to be zip archives. This command is intended to run interactively and + will first ask you some basic questions about the pack, then download the remote + file and generate an MD5 signature for it, then generate an emoji file list for you. + + The manifest entry will either be written to a newly created `index.json` file or appended to the existing one, + *replacing* the old pack with the same name if it was in the file previously. + + The file list will be written to the file specified previously, *replacing* that file. + You _should_ check that the file list doesn't contain anything you don't need in the pack, that is, + anything that is not an emoji (the whole pack is downloaded, but only emoji files are extracted). + """ @default_manifest Pleroma.Config.get!([:emoji, :default_manifest]) -- cgit v1.2.3 From 53a3e61016592b25cca4876c4f8f7be8aa6efa9b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 20 Apr 2019 15:06:56 +0300 Subject: Fix priv/static/instance/static to be just instance/static It was a misunderstanding --- lib/mix/tasks/pleroma/emoji.ex | 14 ++------------ lib/pleroma/emoji.ex | 25 +++++++------------------ 2 files changed, 9 insertions(+), 30 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 0a1bf24e2..02cfaa774 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -137,11 +137,8 @@ defmodule Mix.Tasks.Pleroma.Emoji do IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name])) - static_path = Path.join(:code.priv_dir(:pleroma), "static") - pack_path = Path.join([ - static_path, Pleroma.Config.get!([:instance, :static_dir]), "emoji", pack_name @@ -161,19 +158,12 @@ defmodule Mix.Tasks.Pleroma.Emoji do IO.puts(IO.ANSI.format(["Writing emoji.txt for ", :bright, pack_name])) - common_pack_path = - Path.join([ - "/", - Pleroma.Config.get!([:instance, :static_dir]), - "emoji", - pack_name - ]) - emoji_txt_str = Enum.map( files, fn {shortcode, path} -> - "#{shortcode}, #{Path.join(common_pack_path, path)}" + emojo_path = Path.join("/emoji/#{pack_name}", path) + "#{shortcode}, #{emojo_path}" end ) |> Enum.join("\n") diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 15455b779..169094438 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -81,14 +81,11 @@ defmodule Pleroma.Emoji do end defp load do - static_path = Path.join(:code.priv_dir(:pleroma), "static") - emoji_dir_path = - Path.join([ - static_path, + Path.join( Pleroma.Config.get!([:instance, :static_dir]), "emoji" - ]) + ) case File.ls(emoji_dir_path) do {:error, :enoent} -> @@ -135,20 +132,12 @@ defmodule Pleroma.Emoji do "No emoji.txt found for pack \"#{pack_name}\", assuming all .png files are emoji" ) - common_pack_path = - Path.join([ - "/", - Pleroma.Config.get!([:instance, :static_dir]), - "emoji", - pack_name - ]) + make_shortcode_to_file_map(pack_dir, [".png"]) |> + Enum.map(fn {shortcode, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) - make_shortcode_to_file_map(pack_dir, [".png"]) - |> Enum.map(fn {shortcode, rel_file} -> - filename = Path.join(common_pack_path, rel_file) - - {shortcode, filename, [to_string(match_extra(@groups, filename))]} - end) + {shortcode, filename, [to_string(match_extra(@groups, filename))]} + end) end end -- cgit v1.2.3 From 8f93d4c42366ccce1949717ad2a9e1de0f91011b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 20 Apr 2019 15:34:45 +0300 Subject: Remove most finmoji mentions where appropriate --- lib/pleroma/emoji.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 169094438..6f9bd3eda 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Emoji do @moduledoc """ The emojis are loaded from: - * the built-in Finmojis (if enabled in configuration), + * emoji packs in INSTANCE-DIR/emoji * the files: `config/emoji.txt` and `config/custom_emoji.txt` * glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder -- cgit v1.2.3 From d5c0fd35e1486040d4c57ba18942b2a228d6a4a8 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 21 Apr 2019 22:16:46 +0300 Subject: Wrap the docstrings to 70 characters --- lib/mix/tasks/pleroma/emoji.ex | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 02cfaa774..92d62b6de 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -17,16 +17,16 @@ defmodule Mix.Tasks.Pleroma.Emoji do ### Options - - `-m, --manifest PATH/URL` - path to a custom manifest, it can either be an URL - starting with `http`, in that case the manifest will be fetched from that address, - or a local path + - `-m, --manifest PATH/URL` - path to a custom manifest, it can + either be an URL starting with `http`, in that case the + manifest will be fetched from that address, or a local path ## get-packs mix pleroma.emoji get-packs [OPTION...] PACKS - Fetches, verifies and installs the specified PACKS from the manifest into - the `STATIC-DIR/emoji/PACK-NAME + Fetches, verifies and installs the specified PACKS from the + manifest into the `STATIC-DIR/emoji/PACK-NAME ### Options @@ -36,19 +36,23 @@ defmodule Mix.Tasks.Pleroma.Emoji do mix pleroma.emoji gen-pack PACK-URL - Creates a new manifest entry and a file list from the specified remote pack file. - Currently, only .zip archives are recognized as remote pack files and packs are therefore - assumed to be zip archives. This command is intended to run interactively and - will first ask you some basic questions about the pack, then download the remote - file and generate an MD5 signature for it, then generate an emoji file list for you. - - The manifest entry will either be written to a newly created `index.json` file or appended to the existing one, - *replacing* the old pack with the same name if it was in the file previously. - - The file list will be written to the file specified previously, *replacing* that file. - You _should_ check that the file list doesn't contain anything you don't need in the pack, that is, - anything that is not an emoji (the whole pack is downloaded, but only emoji files are extracted). - + Creates a new manifest entry and a file list from the specified + remote pack file. Currently, only .zip archives are recognized + as remote pack files and packs are therefore assumed to be zip + archives. This command is intended to run interactively and will + first ask you some basic questions about the pack, then download + the remote file and generate an MD5 signature for it, then + generate an emoji file list for you. + + The manifest entry will either be written to a newly created + `index.json` file or appended to the existing one, *replacing* + the old pack with the same name if it was in the file previously. + + The file list will be written to the file specified previously, + *replacing* that file. You _should_ check that the file list doesn't + contain anything you don't need in the pack, that is, anything that is + not an emoji (the whole pack is downloaded, but only emoji files + are extracted). """ @default_manifest Pleroma.Config.get!([:emoji, :default_manifest]) -- cgit v1.2.3 From 153f5375a6fa7ed3ae78a921acc87b1fb025aba9 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 21 Apr 2019 22:19:19 +0300 Subject: Replace MD5 with SHA256 for emoji pack verification --- lib/mix/tasks/pleroma/emoji.ex | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 92d62b6de..2754dd876 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -41,7 +41,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do as remote pack files and packs are therefore assumed to be zip archives. This command is intended to run interactively and will first ask you some basic questions about the pack, then download - the remote file and generate an MD5 signature for it, then + the remote file and generate an SHA256 checksum for it, then generate an emoji file list for you. The manifest entry will either be written to a newly created @@ -110,16 +110,16 @@ defmodule Mix.Tasks.Pleroma.Emoji do ) binary_archive = Tesla.get!(src_url).body - archive_md5 = :crypto.hash(:md5, binary_archive) |> Base.encode16() + archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16() - md5_status_text = ["MD5 of ", :bright, pack_name, :normal, " source file is ", :bright] + sha_status_text = ["SHA256 of ", :bright, pack_name, :normal, " source file is ", :bright] - if archive_md5 == String.upcase(pack["src_md5"]) do - IO.puts(IO.ANSI.format(md5_status_text ++ [:green, "OK"])) + if archive_sha == String.upcase(pack["src_sha256"]) do + IO.puts(IO.ANSI.format(sha_status_text ++ [:green, "OK"])) else - IO.puts(IO.ANSI.format(md5_status_text ++ [:red, "BAD"])) + IO.puts(IO.ANSI.format(sha_status_text ++ [:red, "BAD"])) - raise "Bad MD5 for #{pack_name}" + raise "Bad SHA256 for #{pack_name}" end # The url specified in files should be in the same directory @@ -211,12 +211,12 @@ defmodule Mix.Tasks.Pleroma.Emoji do default_exts end - IO.puts("Downloading the pack and generating MD5") + IO.puts("Downloading the pack and generating SHA256") binary_archive = Tesla.get!(src).body - archive_md5 = :crypto.hash(:md5, binary_archive) |> Base.encode16() + archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16() - IO.puts("MD5 is #{archive_md5}") + IO.puts("SHA256 is #{archive_sha}") pack_json = %{ name => %{ @@ -224,7 +224,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do homepage: homepage, description: description, src: src, - src_md5: archive_md5, + src_sha256: archive_sha, files: files_name } } -- cgit v1.2.3 From ba19ba8a88c206f1570abdd22aa09a45b48a8de5 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 21 Apr 2019 22:33:54 +0300 Subject: Formatting --- lib/pleroma/emoji.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 6f9bd3eda..071c7f6cd 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -132,12 +132,12 @@ defmodule Pleroma.Emoji do "No emoji.txt found for pack \"#{pack_name}\", assuming all .png files are emoji" ) - make_shortcode_to_file_map(pack_dir, [".png"]) |> - Enum.map(fn {shortcode, rel_file} -> - filename = Path.join("/emoji/#{pack_name}", rel_file) + make_shortcode_to_file_map(pack_dir, [".png"]) + |> Enum.map(fn {shortcode, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) - {shortcode, filename, [to_string(match_extra(@groups, filename))]} - end) + {shortcode, filename, [to_string(match_extra(@groups, filename))]} + end) end end -- cgit v1.2.3 From 3cefdc62e0dda75985221853fb123572d6fc068b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 21 Apr 2019 22:46:55 +0300 Subject: Fix emoji.txt / custom_emoji.txt / shortcode_globs handling --- lib/pleroma/emoji.ex | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 071c7f6cd..6390cce4c 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -101,22 +101,27 @@ defmodule Pleroma.Emoji do # Print the packs we've found Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}") - # compat thing for old custom emoji handling - shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || [] - emojis = - (Enum.flat_map( - packs, - fn pack -> load_pack(Path.join(emoji_dir_path, pack)) end - ) ++ - load_from_file("config/emoji.txt") ++ - load_from_file("config/custom_emoji.txt") ++ - load_from_globs(shortcode_globs)) - |> Enum.reject(fn value -> value == nil end) + Enum.flat_map( + packs, + fn pack -> load_pack(Path.join(emoji_dir_path, pack)) end + ) true = :ets.insert(@ets, emojis) end + # Compat thing for old custom emoji handling & default emoji, + # it should run even if there are no emoji packs + shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || [] + + emojis = + (load_from_file("config/emoji.txt") ++ + load_from_file("config/custom_emoji.txt") ++ + load_from_globs(shortcode_globs)) + |> Enum.reject(fn value -> value == nil end) + + true = :ets.insert(@ets, emojis) + :ok end -- cgit v1.2.3