A platform for community discussion. Free, open, simple.
DEV: Retroactive review and refactor of custom emoji bulk import/export (#41860)
Martin Brennanb736e72b1c5f
diff --git a/app/controllers/admin/emoji_controller.rb b/app/controllers/admin/emoji_controller.rbindex 26318720b21..ddda72faf03 100644--- a/app/controllers/admin/emoji_controller.rb+++ b/app/controllers/admin/emoji_controller.rb@@ -1,16 +1,8 @@# frozen_string_literal: true-require "csv"-class Admin::EmojiController < Admin::AdminControllerskip_before_action :check_xhr, only: [:export]- IMPORT_PREVIEW_TTL = 2.hours.to_i- IMPORT_MAX_ENTRIES = 1000- IMPORT_MAX_BYTES = 500.megabytes- IMPORT_MAX_COMPRESSION_RATIO = 20- SUPPORTED_EXTENSIONS = %w[png gif svg].freeze-def indexrender_serialized(Emoji.custom, EmojiSerializer, root: false)end@@ -60,281 +52,81 @@ class Admin::EmojiController < Admin::AdminControllerenddef export- names = Array(params[:names]).map(&:to_s).reject(&:blank?)- if names.empty?- return(+ CustomEmoji::Export.call(service_params) do |result|+ on_success do |archive:|+ send_data archive,+ type: "application/zip",+ disposition: "attachment",+ filename: "emojis.zip"+ end+ on_failed_contract dorender json: failed_json.merge(errors: [I18n.t("emoji.export.no_selection")]),status: :unprocessable_entity- )- end-- emojis = CustomEmoji.where(name: names).includes(:upload)- return render json: failed_json, status: :not_found if emojis.empty?-- temp_dir = Dir.mktmpdir("discourse_emoji_export_")-- begin- csv_rows = []-- emojis.each do |emoji|- filename = "#{emoji.name}.#{emoji.upload.extension}"- File.binwrite(File.join(temp_dir, filename), emoji.upload.content)- group = emoji.group.presence- group = nil if group == "default"- csv_rows << [emoji.name, group, filename]- end-- CSV.open(File.join(temp_dir, "emojis.csv"), "w") do |csv|- csv << %w[name group filename]- csv_rows.each { |row| csv << row }end-- zip_path = Compression::Zip.new.compress(File.dirname(temp_dir), File.basename(temp_dir))- send_data File.binread(zip_path),- type: "application/zip",- disposition: "attachment",- filename: "emojis.zip"- ensure- FileUtils.rm_rf(temp_dir)- File.delete(zip_path) if zip_path && File.exist?(zip_path)+ on_model_not_found(:emojis) { render json: failed_json, status: :not_found }endenddef import_preview- zip_file = params[:file]- if zip_file.blank?- return(- render json: failed_json.merge(errors: [I18n.t("emoji.import.missing_file")]),- status: :unprocessable_entity- )- end-- rows = []- upload_map = {}-hijack do- begin- Compression::SafeZipReader.open(- zip_file.tempfile.path,- max_entries: IMPORT_MAX_ENTRIES,- max_total_bytes: IMPORT_MAX_BYTES,- max_compression_ratio: IMPORT_MAX_COMPRESSION_RATIO,- ) do |reader|- csv_data = reader.read_entry("emojis.csv", max_bytes: 1.megabyte, required: true)-- csv_rows = CSV.parse(csv_data, headers: true)-- filenames_seen = {}- parsed_rows = []-- csv_rows.each_with_index do |row, idx|- name = row["name"].to_s.strip- group = row["group"].to_s.strip.downcase.presence- group = nil if group == "default"- filename = row["filename"].to_s.strip-- errors = validate_import_row(name, group, filename, filenames_seen)- filenames_seen[filename] = true-- parsed_rows << {- idx: idx,- name: name,- group: group,- filename: filename,- errors: errors,- }- end-- valid_names = parsed_rows.filter_map { |r| r[:name] if r[:errors].empty? }- existing_by_name = CustomEmoji.where(name: valid_names).includes(:upload).index_by(&:name)-- parsed_rows.each do |parsed|- idx = parsed[:idx]- name = parsed[:name]- group = parsed[:group]- filename = parsed[:filename]- errors = parsed[:errors]-- if errors.any?- rows << {- index: idx,- name: name,- group: group || "default",- filename: filename,- category: "invalid",- errors: errors,- }- next- end-- tmp_image = Tempfile.new(["emoji_import_", ".#{File.extname(filename).delete(".")}"])- tmp_image.binmode-- begin- bytes_written =- reader.stream_entry_to_file(- filename,- tmp_image,- max_bytes: SiteSetting.max_image_size_kb.kilobytes,- required: false,- )-- if bytes_written.nil?- rows << {- index: idx,- name: name,- group: group || "default",- filename: filename,- category: "invalid",- errors: [I18n.t("emoji.import.validation.missing_filename")],- }- next- end-- tmp_image.rewind-- upload =- UploadCreator.new(tmp_image, filename, type: "custom_emoji").create_for(- current_user.id,- )-- unless upload.persisted?- rows << {- index: idx,- name: name,- group: group || "default",- filename: filename,- category: "invalid",- errors: upload.errors.full_messages,- }- next- end-- upload.update_columns(retain_hours: 3)- upload_map[filename] = upload.id-- category, existing_url, existing_group =- classify_import_row(name, group, upload, existing_by_name[name])-- rows << {- index: idx,- name: name,- group: group || "default",- filename: filename,- category: category,- incoming_url: upload.url,- existing_url: existing_url,- existing_group: existing_group || "default",- }- ensure- tmp_image.close!- end- end+ CustomEmoji::PreviewImport.call(service_params) do |result|+ on_success do |token:, rows:|+ render json: {+ token:,+ rows:+ ActiveModel::ArraySerializer.new(+ rows,+ each_serializer: CustomEmoji::ImportRowSerializer,+ ).as_json,+ }+ end+ on_failed_contract do+ render json: failed_json.merge(errors: [I18n.t("emoji.import.missing_file")]),+ status: :unprocessable_entityend- rescue Compression::SafeZipReader::MissingEntryError- return(+ on_failed_policy(:manifest_not_empty) do+ render json: failed_json.merge(errors: [I18n.t("emoji.import.empty_manifest")]),+ status: :unprocessable_entity+ end+ on_exceptions(CSV::MalformedCSVError) do+ render json: failed_json.merge(errors: [I18n.t("emoji.import.invalid_csv")]),+ status: :unprocessable_entity+ end+ on_exceptions(Compression::SafeZipReader::MissingEntryError) dorender json: failed_json.merge(errors: [I18n.t("emoji.import.missing_csv")]),status: :unprocessable_entity- )- rescue Compression::SafeZipReader::TooManyEntriesError- return(+ end+ on_exceptions(Compression::SafeZipReader::TooManyEntriesError) dorender json: failed_json.merge(errors: [I18n.t("emoji.import.too_many_entries")]),status: :unprocessable_entity- )- rescue Compression::SafeZipReader::EntryTooLargeError- return(+ end+ on_exceptions(Compression::SafeZipReader::EntryTooLargeError) dorender json: failed_json.merge(errors: [I18n.t("emoji.import.file_too_large")]),status: :unprocessable_entity- )+ end+ on_exceptions(Compression::SafeZipReader::SuspiciousEntryError) do+ render json: failed_json.merge(errors: [I18n.t("emoji.import.suspicious_entry")]),+ status: :unprocessable_entity+ endend-- token = SecureRandom.hex- manifest = { rows: rows, upload_map: upload_map }- Discourse.redis.setex(import_preview_key(token), IMPORT_PREVIEW_TTL, manifest.to_json)-- render json: { token: token, rows: rows }endenddef import_confirm- token = params.require(:token)- resolutions = params[:resolutions] || {}-- manifest_json = Discourse.redis.get(import_preview_key(token))- if manifest_json.blank?- return(+ CustomEmoji::ConfirmImport.call(service_params) do |result|+ on_success { |report:| render json: report }+ on_failed_contract do |contract|+ render json: failed_json.merge(errors: contract.errors.full_messages),+ status: :unprocessable_entity+ end+ on_model_not_found(:rows) dorender json: failed_json.merge(errors: [I18n.t("emoji.import.session_expired")]),status: :unprocessable_entity- )- end-- manifest = JSON.parse(manifest_json, symbolize_names: true)- rows = manifest[:rows]- upload_map = manifest[:upload_map]-- created = 0- updated = 0- skipped = 0-- actionable_names =- rows- .map { |r| r.transform_keys(&:to_s) }- .reject { |r| %w[invalid identical].include?(r["category"]) }- .filter_map { |r| r["name"] if (resolutions[r["name"].to_s] || "incoming") != "keep" }- existing_by_name = CustomEmoji.where(name: actionable_names).index_by(&:name)-- ActiveRecord::Base.transaction do- rows.each do |row|- row = row.transform_keys(&:to_s)- name = row["name"]- group = row["group"].presence- filename = row["filename"]- category = row["category"]-- next if category == "invalid"-- if category == "identical"- skipped += 1- next- end-- resolution = resolutions[name.to_s] || "incoming"- next if resolution == "keep"-- upload_id = upload_map[filename.to_sym] || upload_map[filename.to_s]- upload = Upload.find_by(id: upload_id)- next if upload.nil?-- existing = existing_by_name[name]-- if existing- existing.update!(upload: upload, group: group)- else- custom_emoji =- CustomEmoji.new(name: name, upload: upload, group: group, user: current_user)- custom_emoji.save!- StaffActionLogger.new(current_user).log_custom_emoji_create(name, group: group)- created += 1- end-- if existing &&- (- category == "conflict_group" || category == "conflict_image" ||- category == "conflict_both"- )- StaffActionLogger.new(current_user).log_custom_emoji_create(name, group: group)- updated += 1- endend-- Emoji.clear_cache+ on_exceptions(ActiveRecord::RecordInvalid) do |exception|+ render json: failed_json.merge(errors: [exception.message]), status: :unprocessable_entity+ endend-- Discourse.redis.del(import_preview_key(token))-- render json: { created: created, updated: updated, skipped: skipped }- rescue ActiveRecord::RecordInvalid => e- render json: failed_json.merge(errors: [e.message]), status: :unprocessable_entityenddef destroy@@ -354,60 +146,4 @@ class Admin::EmojiController < Admin::AdminControllerrender json: success_jsonend-- private-- def import_preview_key(token)- "emoji_import_preview:#{current_user.id}:#{token}"- end-- def validate_import_row(name, group, filename, filenames_seen)- errors = []- errors << I18n.t("emoji.import.validation.missing_name") if name.blank?- errors << I18n.t("emoji.import.validation.missing_filename") if filename.blank?-- if name.present?- sanitized = Emoji.sanitize_emoji_name(name)- errors << I18n.t("emoji.import.validation.invalid_name") if sanitized.blank?- end-- if group.present? && group.length > 20- errors << I18n.t("emoji.import.validation.group_too_long")- end-- if filename.present?- ext = File.extname(filename).delete(".").downcase- if SUPPORTED_EXTENSIONS.exclude?(ext)- errors << I18n.t("emoji.import.validation.unsupported_extension", ext: ext)- end- errors << I18n.t("emoji.import.validation.duplicate_filename") if filenames_seen[filename]- end-- errors- end-- def classify_import_row(name, group, upload, existing = nil)- existing ||= CustomEmoji.find_by(name: name)- return "new", nil unless existing-- existing_url = existing.upload&.url- existing_group = existing.group.presence- existing_group = nil if existing_group == "default"- group_changed = existing_group != group- display_existing_group = existing_group || "default"- image_changed = existing.upload&.sha1 != upload.sha1-- category =- if group_changed && image_changed- "conflict_both"- elsif image_changed- "conflict_image"- elsif group_changed- "conflict_group"- else- "identical"- end-- [category, existing_url, display_existing_group]- endenddiff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rbindex 64e5440edb3..38049efe67d 100644--- a/app/models/custom_emoji.rb+++ b/app/models/custom_emoji.rb@@ -1,15 +1,24 @@# frozen_string_literal: trueclass CustomEmoji < ActiveRecord::Base+ DEFAULT_GROUP = "default"+ MAX_GROUP_LENGTH = 20+belongs_to :uploadbelongs_to :userhas_many :upload_references, as: :target, dependent: :destroyvalidates :name, presence: true, uniqueness: true+ validates :group, length: { maximum: MAX_GROUP_LENGTH }validates :upload_id, presence: truevalidates :user_id, presence: true+ def self.normalize_group(group)+ normalized = group.to_s.strip.downcase.presence+ normalized == DEFAULT_GROUP ? nil : normalized+ end+before_validation :set_default_user_id, on: :createafter_save dodiff --git a/app/serializers/custom_emoji/import_row_serializer.rb b/app/serializers/custom_emoji/import_row_serializer.rbnew file mode 100644index 00000000000..4ca32176c39--- /dev/null+++ b/app/serializers/custom_emoji/import_row_serializer.rb@@ -0,0 +1,37 @@+# frozen_string_literal: true++class CustomEmoji::ImportRowSerializer < ApplicationSerializer+ attributes :index,+ :name,+ :group,+ :filename,+ :category,+ :errors,+ :incoming_url,+ :existing_url,+ :existing_group++ def group+ object.display_group+ end++ def existing_group+ object.display_existing_group+ end++ def include_errors?+ object.invalid?+ end++ def include_incoming_url?+ object.incoming_url.present?+ end++ def include_existing_url?+ object.conflict? && object.existing_url.present?+ end++ def include_existing_group?+ object.conflict?+ end+enddiff --git a/app/services/custom_emoji/action/apply_import_rows.rb b/app/services/custom_emoji/action/apply_import_rows.rbnew file mode 100644index 00000000000..a629a456ab7--- /dev/null+++ b/app/services/custom_emoji/action/apply_import_rows.rb@@ -0,0 +1,65 @@+# frozen_string_literal: true++class CustomEmoji::Action::ApplyImportRows < Service::ActionBase+ KEEP_RESOLUTION = "keep"++ option :rows+ option :resolutions+ option :acting_user++ def call+ report = { created: 0, updated: 0, skipped: 0 }++ rows.each do |row|+ next if row.invalid?++ if row.identical?+ report[:skipped] += 1+ next+ end++ next if keep_existing?(row)++ upload = uploads[row.upload_id]+ next if upload.nil?++ if (existing = existing_emojis[row.name])+ existing.update!(upload:, group: row.group)+ report[:updated] += 1+ else+ CustomEmoji.create!(name: row.name, group: row.group, upload:, user: acting_user)+ report[:created] += 1+ end+ log_change(row)+ end++ report+ end++ private++ def existing_emojis+ @existing_emojis ||= CustomEmoji.where(name: rows_to_apply.map(&:name)).index_by(&:name)+ end++ def uploads+ @uploads ||= Upload.where(id: rows_to_apply.map(&:upload_id)).index_by(&:id)+ end++ def rows_to_apply+ @rows_to_apply ||= rows.reject { |row| row.invalid? || row.identical? || keep_existing?(row) }+ end++ def keep_existing?(row)+ resolutions_by_name[row.name] == KEEP_RESOLUTION+ end++ # resolution keys may arrive as strings or symbols depending on the caller+ def resolutions_by_name+ @resolutions_by_name ||= resolutions.to_h.transform_keys(&:to_s)+ end++ def log_change(row)+ StaffActionLogger.new(acting_user).log_custom_emoji_create(row.name, group: row.group)+ end+enddiff --git a/app/services/custom_emoji/action/build_export_archive.rb b/app/services/custom_emoji/action/build_export_archive.rbnew file mode 100644index 00000000000..9105dabb4a1--- /dev/null+++ b/app/services/custom_emoji/action/build_export_archive.rb@@ -0,0 +1,46 @@+# frozen_string_literal: true++require "csv"++class CustomEmoji::Action::BuildExportArchive < Service::ActionBase+ MANIFEST_FILENAME = "emojis.csv"+ MANIFEST_HEADERS = %w[name group filename].freeze++ option :emojis++ def call+ temp_dir = Dir.mktmpdir("discourse_emoji_export_")+ zip_path = nil++ begin+ write_images(temp_dir)+ write_manifest(temp_dir)+ zip_path = Compression::Zip.new.compress(File.dirname(temp_dir), File.basename(temp_dir))+ File.binread(zip_path)+ ensure+ FileUtils.rm_rf(temp_dir)+ File.delete(zip_path) if zip_path && File.exist?(zip_path)+ end+ end++ private++ def write_images(temp_dir)+ emojis.each do |emoji|+ File.binwrite(File.join(temp_dir, image_filename(emoji)), emoji.upload.content)+ end+ end++ def write_manifest(temp_dir)+ CSV.open(File.join(temp_dir, MANIFEST_FILENAME), "w") do |csv|+ csv << MANIFEST_HEADERS+ emojis.each do |emoji|+ csv << [emoji.name, CustomEmoji.normalize_group(emoji.group), image_filename(emoji)]+ end+ end+ end++ def image_filename(emoji)+ "#{emoji.name}.#{emoji.upload.extension}"+ end+enddiff --git a/app/services/custom_emoji/action/parse_import_manifest.rb b/app/services/custom_emoji/action/parse_import_manifest.rbnew file mode 100644index 00000000000..90ade5a51af--- /dev/null+++ b/app/services/custom_emoji/action/parse_import_manifest.rb@@ -0,0 +1,69 @@+# frozen_string_literal: true++require "csv"++class CustomEmoji::Action::ParseImportManifest < Service::ActionBase+ MANIFEST_FILENAME = "emojis.csv"+ MAX_MANIFEST_BYTES = 1.megabyte+ SUPPORTED_EXTENSIONS = %w[png gif svg].freeze++ option :reader++ def call+ filenames_seen = Set.new+ names_seen = Set.new++ parse_manifest.map.with_index do |csv_row, index|+ row =+ CustomEmoji::ImportRow.new(+ index:,+ name: Emoji.sanitize_emoji_name(csv_row["name"].to_s.strip),+ group: CustomEmoji.normalize_group(csv_row["group"]),+ filename: csv_row["filename"].to_s.strip,+ )+ validate_row(row, filenames_seen:, names_seen:)+ filenames_seen << row.filename+ names_seen << row.name+ row+ end+ end++ private++ def parse_manifest+ CSV.parse(+ reader.read_entry(MANIFEST_FILENAME, max_bytes: MAX_MANIFEST_BYTES, required: true),+ headers: true,+ )+ end++ def validate_row(row, filenames_seen:, names_seen:)+ errors = name_errors(row, names_seen) + filename_errors(row, filenames_seen)+ if row.group.present? && row.group.length > CustomEmoji::MAX_GROUP_LENGTH+ errors << translated_error("group_too_long")+ end+ row.mark_invalid(*errors) if errors.any?+ end++ def name_errors(row, names_seen)+ return [translated_error("missing_name")] if row.name.blank?+ return [translated_error("duplicate_name")] if names_seen.include?(row.name)+ []+ end++ def filename_errors(row, filenames_seen)+ return [translated_error("missing_filename")] if row.filename.blank?++ errors = []+ extension = File.extname(row.filename).delete(".").downcase+ if SUPPORTED_EXTENSIONS.exclude?(extension)+ errors << translated_error("unsupported_extension", ext: extension)+ end+ errors << translated_error("duplicate_filename") if filenames_seen.include?(row.filename)+ errors+ end++ def translated_error(key, **args)+ I18n.t("emoji.import.validation.#{key}", **args)+ end+enddiff --git a/app/services/custom_emoji/action/stage_import_row.rb b/app/services/custom_emoji/action/stage_import_row.rbnew file mode 100644index 00000000000..31dddf5759c--- /dev/null+++ b/app/services/custom_emoji/action/stage_import_row.rb@@ -0,0 +1,44 @@+# frozen_string_literal: true++class CustomEmoji::Action::StageImportRow < Service::ActionBase+ UPLOAD_RETAIN_HOURS = 3++ option :reader+ option :row+ option :acting_user+ option :existing_emoji, optional: true++ def call+ with_image_tempfile do |image|+ return row.mark_invalid(I18n.t("emoji.import.validation.missing_image")) if image.nil?++ upload = create_upload(image)+ return row.mark_invalid(*upload.errors.full_messages) if !upload.persisted?++ upload.update_columns(retain_hours: UPLOAD_RETAIN_HOURS)+ row.stage(upload:, existing_emoji:)+ end+ end++ private++ def with_image_tempfile+ tempfile = Tempfile.new(["emoji_import_", File.extname(row.filename)])+ tempfile.binmode+ bytes_written =+ reader.stream_entry_to_file(+ row.filename,+ tempfile,+ max_bytes: SiteSetting.max_image_size_kb.kilobytes,+ required: false,+ )+ tempfile.rewind+ yield(bytes_written && tempfile)+ ensure+ tempfile.close!+ end++ def create_upload(image)+ UploadCreator.new(image, row.filename, type: "custom_emoji").create_for(acting_user.id)+ end+enddiff --git a/app/services/custom_emoji/action/stage_import_rows.rb b/app/services/custom_emoji/action/stage_import_rows.rbnew file mode 100644index 00000000000..f6317051f24--- /dev/null+++ b/app/services/custom_emoji/action/stage_import_rows.rb@@ -0,0 +1,40 @@+# frozen_string_literal: true++class CustomEmoji::Action::StageImportRows < Service::ActionBase+ MAX_ENTRIES = 1000+ MAX_TOTAL_BYTES = 500.megabytes+ MAX_COMPRESSION_RATIO = 20++ option :zip_path+ option :acting_user++ def call+ Compression::SafeZipReader.open(+ zip_path,+ max_entries: MAX_ENTRIES,+ max_total_bytes: MAX_TOTAL_BYTES,+ max_compression_ratio: MAX_COMPRESSION_RATIO,+ ) do |reader|+ rows = CustomEmoji::Action::ParseImportManifest.call(reader:)+ existing_emojis = existing_emojis_for(rows)++ rows.map do |row|+ next row if row.invalid?++ CustomEmoji::Action::StageImportRow.call(+ reader:,+ row:,+ acting_user:,+ existing_emoji: existing_emojis[row.name],+ )+ end+ end+ end++ private++ def existing_emojis_for(rows)+ names = rows.reject(&:invalid?).map(&:name)+ CustomEmoji.where(name: names).includes(:upload).index_by(&:name)+ end+enddiff --git a/app/services/custom_emoji/confirm_import.rb b/app/services/custom_emoji/confirm_import.rbnew file mode 100644index 00000000000..fcd73c6e175--- /dev/null+++ b/app/services/custom_emoji/confirm_import.rb@@ -0,0 +1,40 @@+# frozen_string_literal: true++class CustomEmoji::ConfirmImport+ include Service::Base++ params do+ attribute :token, :string+ attribute :resolutions, default: -> { {} }++ validates :token, presence: true+ end++ model :rows, :fetch_preview+ try(ActiveRecord::RecordInvalid) { transaction { step :apply_rows } }++ step :clear_preview+ step :refresh_emoji_cache++ private++ def fetch_preview(params:, guardian:)+ CustomEmoji::ImportPreviewCache.new(guardian.user).fetch(params.token)+ end++ def apply_rows(rows:, params:, guardian:)+ context[:report] = CustomEmoji::Action::ApplyImportRows.call(+ rows:,+ resolutions: params.resolutions,+ acting_user: guardian.user,+ )+ end++ def clear_preview(params:, guardian:)+ CustomEmoji::ImportPreviewCache.new(guardian.user).delete(params.token)+ end++ def refresh_emoji_cache+ Emoji.clear_cache+ end+enddiff --git a/app/services/custom_emoji/export.rb b/app/services/custom_emoji/export.rbnew file mode 100644index 00000000000..9deeaa88fd8--- /dev/null+++ b/app/services/custom_emoji/export.rb@@ -0,0 +1,26 @@+# frozen_string_literal: true++class CustomEmoji::Export+ include Service::Base++ params do+ attribute :names, :array++ validates :names, presence: true++ before_validation { self.names = names.to_a.map(&:to_s).reject(&:blank?) }+ end++ model :emojis+ model :archive, :build_archive++ private++ def fetch_emojis(params:)+ CustomEmoji.where(name: params.names).includes(:upload)+ end++ def build_archive(emojis:)+ CustomEmoji::Action::BuildExportArchive.call(emojis:)+ end+enddiff --git a/app/services/custom_emoji/lib/import_preview_cache.rb b/app/services/custom_emoji/lib/import_preview_cache.rbnew file mode 100644index 00000000000..ec658b9d9a6--- /dev/null+++ b/app/services/custom_emoji/lib/import_preview_cache.rb@@ -0,0 +1,32 @@+# frozen_string_literal: true++class CustomEmoji::ImportPreviewCache+ TTL = 2.hours+ KEY_NAMESPACE = "emoji_import_preview"++ def initialize(user)+ @user = user+ end++ def store(rows)+ token = SecureRandom.hex+ Discourse.redis.setex(key(token), TTL.to_i, rows.map(&:to_h).to_json)+ token+ end++ def fetch(token)+ payload = Discourse.redis.get(key(token))+ return if payload.blank?+ JSON.parse(payload).map { CustomEmoji::ImportRow.from_h(it) }+ end++ def delete(token)+ Discourse.redis.del(key(token))+ end++ private++ def key(token)+ "#{KEY_NAMESPACE}:#{@user.id}:#{token}"+ end+enddiff --git a/app/services/custom_emoji/lib/import_row.rb b/app/services/custom_emoji/lib/import_row.rbnew file mode 100644index 00000000000..3b4c03f439c--- /dev/null+++ b/app/services/custom_emoji/lib/import_row.rb@@ -0,0 +1,140 @@+# frozen_string_literal: true++class CustomEmoji::ImportRow+ CATEGORY_NEW = "new"+ CATEGORY_IDENTICAL = "identical"+ CATEGORY_INVALID = "invalid"+ CATEGORY_CONFLICT_GROUP = "conflict_group"+ CATEGORY_CONFLICT_IMAGE = "conflict_image"+ CATEGORY_CONFLICT_BOTH = "conflict_both"+ CONFLICT_CATEGORIES = [+ CATEGORY_CONFLICT_GROUP,+ CATEGORY_CONFLICT_IMAGE,+ CATEGORY_CONFLICT_BOTH,+ ].freeze++ attr_reader :index,+ :name,+ :group,+ :filename,+ :errors,+ :category,+ :incoming_url,+ :existing_url,+ :existing_group,+ :upload_id++ def self.from_h(hash)+ hash = hash.deep_symbolize_keys+ new(+ index: hash[:index],+ name: hash[:name].to_s,+ group: CustomEmoji.normalize_group(hash[:group]),+ filename: hash[:filename].to_s,+ errors: Array(hash[:errors]),+ category: hash[:category],+ incoming_url: hash[:incoming_url],+ existing_url: hash[:existing_url],+ existing_group: CustomEmoji.normalize_group(hash[:existing_group]),+ upload_id: hash[:upload_id],+ )+ end++ def initialize(+ index:,+ name:,+ group:,+ filename:,+ errors: [],+ category: nil,+ incoming_url: nil,+ existing_url: nil,+ existing_group: nil,+ upload_id: nil+ )+ @index = index+ @name = name+ @group = group+ @filename = filename+ @errors = errors+ @category = category+ @incoming_url = incoming_url+ @existing_url = existing_url+ @existing_group = existing_group+ @upload_id = upload_id+ end++ def read_attribute_for_serialization(attribute)+ public_send(attribute)+ end++ def invalid?+ category == CATEGORY_INVALID+ end++ def identical?+ category == CATEGORY_IDENTICAL+ end++ def conflict?+ CONFLICT_CATEGORIES.include?(category)+ end++ def display_group+ group || CustomEmoji::DEFAULT_GROUP+ end++ def display_existing_group+ existing_group || CustomEmoji::DEFAULT_GROUP+ end++ def mark_invalid(*messages)+ @errors += messages+ @category = CATEGORY_INVALID+ self+ end++ def stage(upload:, existing_emoji:)+ @upload_id = upload.id+ @incoming_url = upload.url+ @category = classify(upload, existing_emoji)+ self+ end++ def to_h+ {+ index:,+ name:,+ group:,+ filename:,+ errors:,+ category:,+ incoming_url:,+ existing_url:,+ existing_group:,+ upload_id:,+ }+ end++ private++ def classify(upload, existing_emoji)+ return CATEGORY_NEW if existing_emoji.nil?++ @existing_url = existing_emoji.upload&.url+ @existing_group = CustomEmoji.normalize_group(existing_emoji.group)++ group_changed = existing_group != group+ image_changed = existing_emoji.upload&.sha1 != upload.sha1++ if group_changed && image_changed+ CATEGORY_CONFLICT_BOTH+ elsif image_changed+ CATEGORY_CONFLICT_IMAGE+ elsif group_changed+ CATEGORY_CONFLICT_GROUP+ else+ CATEGORY_IDENTICAL+ end+ end+enddiff --git a/app/services/custom_emoji/preview_import.rb b/app/services/custom_emoji/preview_import.rbnew file mode 100644index 00000000000..6aea3b05d15--- /dev/null+++ b/app/services/custom_emoji/preview_import.rb@@ -0,0 +1,45 @@+# frozen_string_literal: true++require "csv"++class CustomEmoji::PreviewImport+ include Service::Base++ EXPECTED_ARCHIVE_ERRORS = [+ CSV::MalformedCSVError,+ Compression::SafeZipReader::MissingEntryError,+ Compression::SafeZipReader::TooManyEntriesError,+ Compression::SafeZipReader::EntryTooLargeError,+ Compression::SafeZipReader::SuspiciousEntryError,+ ].freeze++ params do+ attribute :file++ validates :file, presence: true+ end++ # Has to be done this way rather than in a model step+ # otherwise the error doesn't bubble up correctly+ try(*EXPECTED_ARCHIVE_ERRORS) { step :stage_rows }++ policy :manifest_not_empty+ model :token, :store_preview++ private++ def stage_rows(params:, guardian:)+ context[:rows] = CustomEmoji::Action::StageImportRows.call(+ zip_path: params.file.tempfile.path,+ acting_user: guardian.user,+ )+ end++ def manifest_not_empty(rows:)+ rows.present?+ end++ def store_preview(rows:, guardian:)+ CustomEmoji::ImportPreviewCache.new(guardian.user).store(rows)+ end+enddiff --git a/config/application.rb b/config/application.rbindex adc6b2fb226..47937689c78 100644--- a/config/application.rb+++ b/config/application.rb@@ -113,6 +113,10 @@ module Discourseconfig.autoload_paths << "#{root}/lib/i18n"config.autoload_paths << "#{root}/lib/validators"+ # `lib` directories under a service namespace hold supporting classes+ # (value objects, caches, etc.) without adding a `Lib` constant.+ Rails.autoloaders.main.collapse("#{root}/app/services/*/lib")+# Only load the plugins named here, in the order given (default is alphabetical).# :all can be used as a placeholder for all plugins not explicitly named.# config.plugins = [ :exception_notification, :ssl_requirement, :all ]diff --git a/config/locales/server.en.yml b/config/locales/server.en.ymlindex a2afad95433..54371aabf57 100644--- a/config/locales/server.en.yml+++ b/config/locales/server.en.yml@@ -6368,13 +6368,17 @@ en:import:missing_file: "No ZIP file provided."missing_csv: "The ZIP must contain an emojis.csv file."+ invalid_csv: "The emojis.csv file could not be parsed."+ empty_manifest: "The emojis.csv file does not list any emojis."too_many_entries: "The ZIP contains too many entries (maximum 1000)."file_too_large: "One or more files in the ZIP are too large."+ suspicious_entry: "One or more files in the ZIP have a suspicious compression ratio."session_expired: "Import session has expired. Please start the import again."validation:missing_name: "Name is required."+ duplicate_name: "Name appears more than once in the CSV."missing_filename: "Filename is required."- invalid_name: "Name contains invalid characters."+ missing_image: "Image file is missing from the ZIP."group_too_long: "Group name must be 20 characters or fewer."unsupported_extension: "File extension .%{ext} is not supported (use png, gif, or svg)."duplicate_filename: "Filename appears more than once in the CSV."diff --git a/spec/models/custom_emoji_spec.rb b/spec/models/custom_emoji_spec.rbnew file mode 100644index 00000000000..f52aeb42c13--- /dev/null+++ b/spec/models/custom_emoji_spec.rb@@ -0,0 +1,27 @@+# frozen_string_literal: true++RSpec.describe CustomEmoji do+ describe "validations" do+ subject(:custom_emoji) { Fabricate.build(:custom_emoji) }++ it { is_expected.to validate_length_of(:group).is_at_most(described_class::MAX_GROUP_LENGTH) }+ end++ describe ".normalize_group" do+ it "returns nil for blank values" do+ expect(described_class.normalize_group(nil)).to be_nil+ expect(described_class.normalize_group("")).to be_nil+ expect(described_class.normalize_group(" ")).to be_nil+ end++ it "returns nil for the default group regardless of case" do+ expect(described_class.normalize_group("default")).to be_nil+ expect(described_class.normalize_group("Default")).to be_nil+ expect(described_class.normalize_group(" DEFAULT ")).to be_nil+ end++ it "strips and downcases any other group" do+ expect(described_class.normalize_group(" Fun ")).to eq("fun")+ end+ end+enddiff --git a/spec/requests/admin/emojis_controller_spec.rb b/spec/requests/admin/emojis_controller_spec.rbindex c7aef6558c9..f50ad4abf27 100644--- a/spec/requests/admin/emojis_controller_spec.rb+++ b/spec/requests/admin/emojis_controller_spec.rb@@ -406,6 +406,15 @@ RSpec.describe Admin::EmojiController doexpect(response.status).to eq(422)end+ it "returns 422 when the manifest only contains headers" do+ zip = build_emoji_zip("name,group,filename\n")++ post "/admin/config/emoji/import_preview.json", params: { file: zip }++ expect(response.status).to eq(422)+ expect(response.parsed_body["errors"]).to eq([I18n.t("emoji.import.empty_manifest")])+ end+it "stores a manifest in Redis" dozip =build_emoji_zip(@@ -442,9 +451,9 @@ RSpec.describe Admin::EmojiController dodescribe "#import_confirm" dolet(:image_path) { Rails.root.join("spec/fixtures/images/logo.png") }- def store_manifest(user, token, rows, upload_map)+ def store_manifest(user, token, rows)key = "emoji_import_preview:#{user.id}:#{token}"- Discourse.redis.setex(key, 2.hours.to_i, { rows: rows, upload_map: upload_map }.to_json)+ Discourse.redis.setex(key, 2.hours.to_i, rows.to_json)endcontext "when logged in as an admin" do@@ -461,8 +470,15 @@ RSpec.describe Admin::EmojiController dostore_manifest(admin,token,- [{ name: "confirm-new", group: "default", filename: "confirm-new.png", category: "new" }],- { "confirm-new.png" => staged_upload.id },+ [+ {+ name: "confirm-new",+ group: "default",+ filename: "confirm-new.png",+ category: "new",+ upload_id: staged_upload.id,+ },+ ],)expect do@@ -486,10 +502,21 @@ RSpec.describe Admin::EmojiController doadmin,token,[- { name: "rollback-ok", group: "default", filename: "rollback-ok.png", category: "new" },- { name: "", group: "default", filename: "bad.png", category: "new" },+ {+ name: "rollback-ok",+ group: "default",+ filename: "rollback-ok.png",+ category: "new",+ upload_id: staged_upload.id,+ },+ {+ name: "",+ group: "default",+ filename: "bad.png",+ category: "new",+ upload_id: staged_upload.id,+ },],- { "rollback-ok.png" => staged_upload.id, "bad.png" => staged_upload.id },)post "/admin/config/emoji/import_confirm.json", params: { token: token }@@ -511,7 +538,6 @@ RSpec.describe Admin::EmojiController docategory: "identical",},],- {},)post "/admin/config/emoji/import_confirm.json", params: { token: token }@@ -538,9 +564,9 @@ RSpec.describe Admin::EmojiController dogroup: "new-group",filename: "keep-me.png",category: "conflict_group",+ upload_id: staged_upload.id,},],- { "keep-me.png" => staged_upload.id },)post "/admin/config/emoji/import_confirm.json",diff --git a/spec/services/custom_emoji/action/apply_import_rows_spec.rb b/spec/services/custom_emoji/action/apply_import_rows_spec.rbnew file mode 100644index 00000000000..dfe050c681f--- /dev/null+++ b/spec/services/custom_emoji/action/apply_import_rows_spec.rb@@ -0,0 +1,155 @@+# frozen_string_literal: true++RSpec.describe CustomEmoji::Action::ApplyImportRows do+ describe "#call" do+ subject(:report) { described_class.call(rows:, resolutions:, acting_user:) }++ fab!(:acting_user, :admin)+ fab!(:staged_upload, :upload)++ let(:resolutions) { {} }++ def build_row(name:, category:, index: 0, group: nil, upload_id: staged_upload.id)+ CustomEmoji::ImportRow.new(+ index:,+ name:,+ group:,+ filename: "#{name}.png",+ category:,+ upload_id:,+ )+ end++ context "with an invalid row" do+ let(:rows) { [build_row(name: "broken", category: CustomEmoji::ImportRow::CATEGORY_INVALID)] }++ it "skips it without creating anything or counting it" do+ expect { report }.not_to change { CustomEmoji.count }+ expect(report).to eq(created: 0, updated: 0, skipped: 0)+ end+ end++ context "with an identical row" do+ let(:rows) do+ [build_row(name: "same-emoji", category: CustomEmoji::ImportRow::CATEGORY_IDENTICAL)]+ end++ it "counts it as skipped without creating anything" do+ expect { report }.not_to change { CustomEmoji.count }+ expect(report).to eq(created: 0, updated: 0, skipped: 1)+ end+ end++ context "with a conflict row resolved with keep" do+ fab!(:existing_emoji) { Fabricate(:custom_emoji, name: "keep-me", group: "old") }++ let(:resolutions) { { "keep-me" => "keep" } }+ let(:rows) do+ [+ build_row(+ name: "keep-me",+ group: "new-group",+ category: CustomEmoji::ImportRow::CATEGORY_CONFLICT_GROUP,+ ),+ ]+ end++ it "leaves the existing emoji untouched and counts nothing" do+ expect { report }.not_to change { existing_emoji.reload.attributes }+ expect(report).to eq(created: 0, updated: 0, skipped: 0)+ end++ context "when the resolution keys were symbolized" do+ let(:resolutions) { { "keep-me": "keep" } }++ it "still keeps the existing emoji" do+ expect { report }.not_to change { existing_emoji.reload.attributes }+ end+ end+ end++ context "with a row whose staged upload no longer exists" do+ let(:rows) do+ [build_row(name: "gone", category: CustomEmoji::ImportRow::CATEGORY_NEW, upload_id: 0)]+ end++ it "skips it without creating anything or counting it" do+ expect { report }.not_to change { CustomEmoji.count }+ expect(report).to eq(created: 0, updated: 0, skipped: 0)+ end+ end++ context "with a conflict row for an existing emoji" do+ fab!(:existing_emoji) { Fabricate(:custom_emoji, name: "clash", group: "old") }++ let(:rows) do+ [+ build_row(+ name: "clash",+ group: "fresh",+ category: CustomEmoji::ImportRow::CATEGORY_CONFLICT_BOTH,+ ),+ ]+ end++ it "updates the emoji with the incoming upload and group" do+ expect { report }.to change { existing_emoji.reload.upload_id }.to(staged_upload.id).and(+ change { existing_emoji.reload.group }.from("old").to("fresh"),+ )+ expect(report).to eq(created: 0, updated: 1, skipped: 0)+ end++ it "logs the change" do+ expect { report }.to change {+ UserHistory.where(action: UserHistory.actions[:custom_emoji_create]).count+ }.by(1)+ end+ end++ context "with a new row" do+ let(:rows) do+ [build_row(name: "brand-new", group: "fun", category: CustomEmoji::ImportRow::CATEGORY_NEW)]+ end++ it "creates the emoji for the acting user" do+ expect { report }.to change { CustomEmoji.count }.by(1)+ expect(CustomEmoji.find_by(name: "brand-new")).to have_attributes(+ group: "fun",+ upload_id: staged_upload.id,+ user_id: acting_user.id,+ )+ expect(report).to eq(created: 1, updated: 0, skipped: 0)+ end++ it "logs the change" do+ expect { report }.to change {+ UserHistory.where(action: UserHistory.actions[:custom_emoji_create]).count+ }.by(1)+ end+ end++ context "with a mix of new, conflicting and identical rows" do+ fab!(:existing_emoji) { Fabricate(:custom_emoji, name: "clash", group: "old") }++ let(:rows) do+ [+ build_row(name: "brand-new", index: 0, category: CustomEmoji::ImportRow::CATEGORY_NEW),+ build_row(+ name: "clash",+ index: 1,+ category: CustomEmoji::ImportRow::CATEGORY_CONFLICT_IMAGE,+ ),+ build_row(+ name: "same-emoji",+ index: 2,+ category: CustomEmoji::ImportRow::CATEGORY_IDENTICAL,+ ),+ ]+ end++ it "reports the totals per outcome" do+ expect(report).to eq(created: 1, updated: 1, skipped: 1)+ end+ end+ end+enddiff --git a/spec/services/custom_emoji/action/parse_import_manifest_spec.rb b/spec/services/custom_emoji/action/parse_import_manifest_spec.rbnew file mode 100644index 00000000000..7f0fe72bc79--- /dev/null+++ b/spec/services/custom_emoji/action/parse_import_manifest_spec.rb@@ -0,0 +1,108 @@+# frozen_string_literal: true++require "zip"++RSpec.describe CustomEmoji::Action::ParseImportManifest do+ describe "#call" do+ subject(:rows) { parse_manifest(csv_content) }++ def parse_manifest(csv_content)+ tmp = Tempfile.new(%w[manifest_ .zip])+ tmp.close++ Zip::File.open(tmp.path, create: true) do |zip|+ zip.get_output_stream("emojis.csv") { |entry| entry.write(csv_content) }+ end++ Compression::SafeZipReader.open(tmp.path) { |reader| described_class.call(reader:) }+ ensure+ tmp.unlink+ end++ context "with valid rows" do+ let(:csv_content) { "name,group,filename\nparty,Fun,party.png\nblob,,blob.gif\n" }++ it "returns one uncategorized row per CSV row with normalized attributes" do+ expect(rows.map { [it.name, it.group, it.filename, it.category] }).to eq(+ [["party", "fun", "party.png", nil], ["blob", nil, "blob.gif", nil]],+ )+ end+ end++ context "with names needing sanitization" do+ let(:csv_content) { "name,group,filename\nMy Emoji!,,my-emoji.png\n" }++ it "sanitizes the name" do+ expect(rows.first.name).to eq("my_emoji_")+ end+ end++ context "with a group matching the default group" do+ let(:csv_content) do+ "name,group,filename\none,Default,one.png\ntwo,default,two.png\nthree,,three.png\n"+ end++ it "normalizes the group to nil" do+ expect(rows.map(&:group)).to eq([nil, nil, nil])+ end+ end++ context "with a missing name" do+ let(:csv_content) { "name,group,filename\n,,unnamed.png\n" }++ it "marks the row invalid" do+ expect(rows.first).to be_invalid+ expect(rows.first.errors).to eq([I18n.t("emoji.import.validation.missing_name")])+ end+ end++ context "with two names identical after sanitization" do+ let(:csv_content) { "name,group,filename\nMy Emoji!,,first.png\nmy emoji?,,second.png\n" }++ it "marks the second row invalid" do+ expect(rows.first).not_to be_invalid+ expect(rows.second).to be_invalid+ expect(rows.second.errors).to eq([I18n.t("emoji.import.validation.duplicate_name")])+ end+ end++ context "with a missing filename" do+ let(:csv_content) { "name,group,filename\nno-file,,\n" }++ it "marks the row invalid" do+ expect(rows.first).to be_invalid+ expect(rows.first.errors).to eq([I18n.t("emoji.import.validation.missing_filename")])+ end+ end++ context "with an unsupported file extension" do+ let(:csv_content) { "name,group,filename\nbad-ext,,bad-ext.bmp\n" }++ it "marks the row invalid" do+ expect(rows.first).to be_invalid+ expect(rows.first.errors).to eq(+ [I18n.t("emoji.import.validation.unsupported_extension", ext: "bmp")],+ )+ end+ end++ context "with a duplicate filename" do+ let(:csv_content) { "name,group,filename\nfirst,,shared.png\nsecond,,shared.png\n" }++ it "marks the second row invalid" do+ expect(rows.first).not_to be_invalid+ expect(rows.second).to be_invalid+ expect(rows.second.errors).to eq([I18n.t("emoji.import.validation.duplicate_filename")])+ end+ end++ context "with a group longer than the maximum length" do+ let(:csv_content) { "name,group,filename\nlong-group,#{"g" * 21},long-group.png\n" }++ it "marks the row invalid" do+ expect(rows.first).to be_invalid+ expect(rows.first.errors).to eq([I18n.t("emoji.import.validation.group_too_long")])+ end+ end+ end+enddiff --git a/spec/services/custom_emoji/confirm_import_spec.rb b/spec/services/custom_emoji/confirm_import_spec.rbnew file mode 100644index 00000000000..fb106acf994--- /dev/null+++ b/spec/services/custom_emoji/confirm_import_spec.rb@@ -0,0 +1,141 @@+# frozen_string_literal: true++RSpec.describe CustomEmoji::ConfirmImport do+ describe described_class::Contract, type: :model do+ it { is_expected.to validate_presence_of(:token) }+ end++ describe ".call" do+ subject(:result) { described_class.call(params:, **dependencies) }++ fab!(:current_user, :admin)+ fab!(:staged_upload, :upload)+ fab!(:existing_emoji) { Fabricate(:custom_emoji, name: "conflict-emoji", group: "old") }++ let(:params) { { token:, resolutions: } }+ let(:dependencies) { { guardian: current_user.guardian } }+ let(:resolutions) { {} }+ let(:token) { CustomEmoji::ImportPreviewCache.new(current_user).store(rows) }+ let(:rows) do+ [+ CustomEmoji::ImportRow.new(+ index: 0,+ name: "confirm-new",+ group: "fun",+ filename: "confirm-new.png",+ category: CustomEmoji::ImportRow::CATEGORY_NEW,+ upload_id: staged_upload.id,+ ),+ CustomEmoji::ImportRow.new(+ index: 1,+ name: "conflict-emoji",+ group: nil,+ filename: "conflict-emoji.png",+ category: CustomEmoji::ImportRow::CATEGORY_CONFLICT_IMAGE,+ upload_id: staged_upload.id,+ ),+ CustomEmoji::ImportRow.new(+ index: 2,+ name: "same-emoji",+ group: nil,+ filename: "same-emoji.png",+ category: CustomEmoji::ImportRow::CATEGORY_IDENTICAL,+ ),+ ]+ end++ context "when contract is invalid" do+ let(:token) { "" }++ it { is_expected.to fail_a_contract }+ end++ context "when the token is unknown or expired" do+ let(:token) { "unknown-token" }++ it { is_expected.to fail_to_find_a_model(:rows) }+ end++ context "when a row fails to apply" do+ let(:rows) do+ [+ CustomEmoji::ImportRow.new(+ index: 0,+ name: "rollback-ok",+ group: nil,+ filename: "rollback-ok.png",+ category: CustomEmoji::ImportRow::CATEGORY_NEW,+ upload_id: staged_upload.id,+ ),+ CustomEmoji::ImportRow.new(+ index: 1,+ name: "",+ group: nil,+ filename: "bad.png",+ category: CustomEmoji::ImportRow::CATEGORY_NEW,+ upload_id: staged_upload.id,+ ),+ ]+ end++ it { is_expected.to fail_with_exception(ActiveRecord::RecordInvalid) }++ it "rolls back emojis created earlier in the batch" do+ expect { result }.not_to change { CustomEmoji.count }+ end++ it "keeps the staged manifest in Redis" do+ result+ expect(Discourse.redis.exists?("emoji_import_preview:#{current_user.id}:#{token}")).to eq(+ true,+ )+ end+ end++ context "when everything's ok" do+ it { is_expected.to run_successfully }++ it "reports created, updated and skipped counts" do+ expect(result[:report]).to eq(created: 1, updated: 1, skipped: 1)+ end++ it "creates the new emoji with its group" do+ expect { result }.to change { CustomEmoji.count }.by(1)+ expect(CustomEmoji.find_by(name: "confirm-new")).to have_attributes(+ group: "fun",+ upload_id: staged_upload.id,+ user_id: current_user.id,+ )+ end++ it "updates the conflicting emoji with the incoming upload and group" do+ expect { result }.to change { existing_emoji.reload.upload_id }.to(staged_upload.id).and(+ change { existing_emoji.reload.group }.from("old").to(nil),+ )+ end++ it "logs the applied changes" do+ expect { result }.to change {+ UserHistory.where(action: UserHistory.actions[:custom_emoji_create]).count+ }.by(2)+ end++ it "deletes the staged manifest from Redis" do+ result+ expect(Discourse.redis.exists?("emoji_import_preview:#{current_user.id}:#{token}")).to eq(+ false,+ )+ end+ end++ context "when a conflict is resolved with keep" do+ let(:resolutions) { { "conflict-emoji" => "keep" } }++ it { is_expected.to run_successfully }++ it "leaves the existing emoji untouched" do+ expect { result }.not_to change { existing_emoji.reload.attributes }+ end+ end+ end+enddiff --git a/spec/services/custom_emoji/export_spec.rb b/spec/services/custom_emoji/export_spec.rbnew file mode 100644index 00000000000..522a73ae982--- /dev/null+++ b/spec/services/custom_emoji/export_spec.rb@@ -0,0 +1,80 @@+# frozen_string_literal: true++require "zip"+require "csv"++RSpec.describe CustomEmoji::Export do+ describe described_class::Contract, type: :model do+ it { is_expected.to validate_presence_of(:names) }+ end++ describe ".call" do+ subject(:result) { described_class.call(params:, **dependencies) }++ fab!(:current_user, :admin)+ fab!(:png_upload) do+ UploadCreator.new(+ file_from_fixtures("logo.png"),+ "logo.png",+ type: "custom_emoji",+ ).create_for(Discourse.system_user.id)+ end+ fab!(:default_group_emoji) { Fabricate(:custom_emoji, name: "emoji-a", upload: png_upload) }+ fab!(:grouped_emoji) do+ Fabricate(:custom_emoji, name: "emoji-b", upload: png_upload, group: "fun")+ end++ let(:params) { { names: } }+ let(:dependencies) { { guardian: current_user.guardian } }+ let(:names) { %w[emoji-a emoji-b] }++ context "when contract is invalid" do+ let(:names) { ["", nil] }++ it { is_expected.to fail_a_contract }+ end++ context "when no emojis match the provided names" do+ let(:names) { ["nonexistent-emoji"] }++ it { is_expected.to fail_to_find_a_model(:emojis) }+ end++ context "when everything's ok" do+ it { is_expected.to run_successfully }++ it "builds a ZIP archive containing the manifest and one image per emoji" do+ with_archive do |zip|+ expect(zip.entries.map(&:name)).to contain_exactly(+ "emojis.csv",+ "emoji-a.png",+ "emoji-b.png",+ )+ end+ end++ it "writes a manifest row per emoji with a blank group for the default group" do+ with_archive do |zip|+ manifest = CSV.parse(zip.read("emojis.csv"), headers: true)++ expect(manifest.map { |row| row.to_h.slice("name", "group", "filename") }).to eq(+ [+ { "name" => "emoji-a", "group" => nil, "filename" => "emoji-a.png" },+ { "name" => "emoji-b", "group" => "fun", "filename" => "emoji-b.png" },+ ],+ )+ end+ end++ def with_archive+ Tempfile.create(%w[emoji_export_ .zip]) do |file|+ file.binmode+ file.write(result[:archive])+ file.rewind++ Zip::File.open(file.path) { |zip| yield(zip) }+ end+ end+ end+ end+enddiff --git a/spec/services/custom_emoji/import_row_spec.rb b/spec/services/custom_emoji/import_row_spec.rbnew file mode 100644index 00000000000..c8cd3b75948--- /dev/null+++ b/spec/services/custom_emoji/import_row_spec.rb@@ -0,0 +1,89 @@+# frozen_string_literal: true++RSpec.describe CustomEmoji::ImportRow do+ describe "#stage" do+ subject(:staged_row) { row.stage(upload:, existing_emoji:) }++ fab!(:upload)++ let(:row) { described_class.new(index: 0, name: "party", group: "fun", filename: "party.png") }++ context "when no emoji exists with that name" do+ let(:existing_emoji) { nil }++ it "categorizes the row as new and records the incoming upload" do+ expect(staged_row.category).to eq(described_class::CATEGORY_NEW)+ expect(staged_row.upload_id).to eq(upload.id)+ expect(staged_row.incoming_url).to eq(upload.url)+ end+ end++ context "when an emoji exists with the same image and group" do+ fab!(:existing_emoji) { Fabricate(:custom_emoji, name: "party", group: "fun") }++ let(:upload) { existing_emoji.upload }++ it "categorizes the row as identical" do+ expect(staged_row.category).to eq(described_class::CATEGORY_IDENTICAL)+ end+ end++ context "when an emoji exists with the same image but another group" do+ fab!(:existing_emoji) { Fabricate(:custom_emoji, name: "party", group: "old") }++ let(:upload) { existing_emoji.upload }++ it "categorizes the row as a group conflict and records the existing state" do+ expect(staged_row.category).to eq(described_class::CATEGORY_CONFLICT_GROUP)+ expect(staged_row.existing_group).to eq("old")+ expect(staged_row.existing_url).to eq(existing_emoji.upload.url)+ end+ end++ context "when an emoji exists with another image but the same group" do+ fab!(:existing_emoji) { Fabricate(:custom_emoji, name: "party", group: "fun") }++ it "categorizes the row as an image conflict" do+ expect(staged_row.category).to eq(described_class::CATEGORY_CONFLICT_IMAGE)+ end+ end++ context "when an emoji exists with another image and another group" do+ fab!(:existing_emoji) { Fabricate(:custom_emoji, name: "party", group: "old") }++ it "categorizes the row as a conflict on both" do+ expect(staged_row.category).to eq(described_class::CATEGORY_CONFLICT_BOTH)+ end+ end+ end++ describe "#mark_invalid" do+ it "accumulates errors and categorizes the row as invalid" do+ row = described_class.new(index: 0, name: "party", group: nil, filename: "party.png")++ row.mark_invalid("first error", "second error")++ expect(row).to be_invalid+ expect(row.errors).to eq(["first error", "second error"])+ end+ end++ describe ".from_h" do+ it "round-trips a staged row through its hash representation" do+ row =+ described_class.new(+ index: 3,+ name: "party",+ group: "fun",+ filename: "party.png",+ category: described_class::CATEGORY_CONFLICT_GROUP,+ incoming_url: "/incoming.png",+ existing_url: "/existing.png",+ existing_group: "old",+ upload_id: 42,+ )++ expect(described_class.from_h(row.to_h.as_json).to_h).to eq(row.to_h)+ end+ end+enddiff --git a/spec/services/custom_emoji/preview_import_spec.rb b/spec/services/custom_emoji/preview_import_spec.rbnew file mode 100644index 00000000000..c8864e57560--- /dev/null+++ b/spec/services/custom_emoji/preview_import_spec.rb@@ -0,0 +1,82 @@+# frozen_string_literal: true++require "zip"++RSpec.describe CustomEmoji::PreviewImport do+ describe described_class::Contract, type: :model do+ it { is_expected.to validate_presence_of(:file) }+ end++ describe ".call" do+ subject(:result) { described_class.call(params:, **dependencies) }++ fab!(:current_user, :admin)++ let(:params) { { file: } }+ let(:dependencies) { { guardian: current_user.guardian } }+ let(:image_path) { Rails.root.join("spec/fixtures/images/logo.png") }+ let(:csv_content) { "name,group,filename\npreview-emoji,,preview-emoji.png\n" }+ let(:images) { { "preview-emoji.png" => image_path } }+ let(:file) { build_emoji_zip(csv_content, images) }++ def build_emoji_zip(csv_content, images = {})+ tmp = Tempfile.new(%w[emoji_import_ .zip])+ tmp.close++ Zip::File.open(tmp.path, create: true) do |zip|+ zip.get_output_stream("emojis.csv") { |entry| entry.write(csv_content) } if csv_content+ images.each { |filename, path| zip.add(filename, path) }+ end++ Rack::Test::UploadedFile.new(tmp.path, "application/zip")+ end++ context "when contract is invalid" do+ let(:file) { nil }++ it { is_expected.to fail_a_contract }+ end++ context "when the ZIP has no emojis.csv entry" do+ let(:file) do+ tmp = Tempfile.new(%w[no_csv_ .zip])+ tmp.close+ Zip::File.open(tmp.path, create: true) do |zip|+ zip.get_output_stream("readme.txt") { |entry| entry.write("hi") }+ end+ Rack::Test::UploadedFile.new(tmp.path, "application/zip")+ end++ it { is_expected.to fail_with_exception(Compression::SafeZipReader::MissingEntryError) }+ end++ context "when the manifest only contains headers" do+ let(:csv_content) { "name,group,filename\n" }+ let(:images) { {} }++ it { is_expected.to fail_a_policy(:manifest_not_empty) }+ end++ context "when everything's ok" do+ it { is_expected.to run_successfully }++ it "returns a token and the staged rows with their categories" do+ expect(result[:token]).to be_present+ expect(result[:rows].map { |row| [row.name, row.category] }).to eq(+ [["preview-emoji", CustomEmoji::ImportRow::CATEGORY_NEW]],+ )+ end++ it "stores the staged rows in Redis under the user and token" do+ expect(+ Discourse.redis.exists?("emoji_import_preview:#{current_user.id}:#{result[:token]}"),+ ).to eq(true)+ end++ it "creates a staged upload retained for three hours" do+ expect { result }.to change { Upload.count }.by(1)+ expect(Upload.last.retain_hours).to eq(3)+ end+ end+ end+end