A platform for community discussion. Free, open, simple.
DEV: update GIFs migration script (#40741)
David Battersby60fb6af2d8f6
diff --git a/lib/tasks/migrate_discourse_gifs_to_core.rake b/lib/tasks/migrate_discourse_gifs_to_core.rakeindex 1b44b197256..186076b08fd 100644--- a/lib/tasks/migrate_discourse_gifs_to_core.rake+++ b/lib/tasks/migrate_discourse_gifs_to_core.rake@@ -3,8 +3,53 @@require "discourse_gifs"module DiscourseGifsMigration+ # Klipy expects locale in xx_YY form (ISO 639-1 language + ISO 3166-1 alpha-2+ # country). Giphy stores a bare language code, so each of its locale choices is+ # translated to a representative xx_YY. Where a language spans several countries+ # the predominant one is used — these picks are deliberate, not 1:1:+ # en→en_US, pt→pt_BR, es→es_ES, ar→ar_SA, bn→bn_BD, ms→ms_MY.+ # "iw" is Giphy's legacy code for Hebrew, normalised to the modern "he".+ GIPHY_LOCALES = {+ "ar" => "ar_SA",+ "bn" => "bn_BD",+ "cs" => "cs_CZ",+ "da" => "da_DK",+ "de" => "de_DE",+ "en" => "en_US",+ "es" => "es_ES",+ "fa" => "fa_IR",+ "fi" => "fi_FI",+ "fr" => "fr_FR",+ "hi" => "hi_IN",+ "hu" => "hu_HU",+ "id" => "id_ID",+ "it" => "it_IT",+ "iw" => "he_IL",+ "ja" => "ja_JP",+ "ko" => "ko_KR",+ "ms" => "ms_MY",+ "nl" => "nl_NL",+ "no" => "no_NO",+ "pl" => "pl_PL",+ "pt" => "pt_BR",+ "ro" => "ro_RO",+ "ru" => "ru_RU",+ "sv" => "sv_SE",+ "th" => "th_TH",+ "tl" => "tl_PH",+ "tr" => "tr_TR",+ "uk" => "uk_UA",+ "vi" => "vi_VN",+ "zh-CN" => "zh_CN",+ "zh-TW" => "zh_TW",+ }.freeze+# Mapping from each TC provider's theme settings to core site settings.# `value:` is an optional lookup table for non-1:1 translations.+ #+ # Tenor and Klipy already store locale in Klipy's xx_YY form, so their locale+ # carries across unchanged; Giphy's bare language code is translated via+ # GIPHY_LOCALES above.PROVIDER_MAPPINGS = {"giphy" => {"giphy_file_format" => {@@ -21,6 +66,7 @@ module DiscourseGifsMigration},"giphy_locale" => {name: "klipy_locale",+ value: GIPHY_LOCALES,},},"tenor" => {@@ -72,16 +118,54 @@ module DiscourseGifsMigration},}.freeze+ GIF_PROVIDER_DOMAINS = %w[giphy.com tenor.com].freeze++ KLIPY_MEDIA_HOSTS = %w[static.klipy.com static1.klipy.com static2.klipy.com].freeze++ # SGR foreground color codes for terminal output+ COLORS = { red: 31, green: 32, yellow: 33, blue: 34 }.freeze+module_function+ def paint(text, color = nil, bold: false)+ codes = []+ codes << 1 if bold+ codes << COLORS.fetch(color) if color+ return text if codes.empty?++ "\e[#{codes.join(";")}m#{text}\e[0m"+ end++ # A highlighted [db] label (blue, or red when the named db is missing).+ def db_label(db, missing: false)+ "\e[#{missing ? "1;101" : "1;104"}m[#{db}]\e[0m"+ end++ # indented bullet lines used throughout the migration output+ def item(text)+ puts " - #{text}"+ end++ def success(text)+ item(paint(text, :green))+ end++ def failure(text)+ item(paint(text, :red))+ end++ def status(text, color, bold: false)+ puts " #{paint(text, color, bold: bold)}"+ end+def find_componentsif ENV["RAILS_DB"].present?db = ENV["RAILS_DB"]if !RailsMultisite::ConnectionManagement.has_db?(db)default_db = RailsMultisite::ConnectionManagement::DEFAULT- puts "\e[31m✗ Database \e[1;101m[#{db}]\e[0m \e[31mnot found\e[0m"- puts "Using default database instead: \e[1;104m[#{default_db}]\e[0m\n\n"+ puts "#{paint("✗ Database", :red)} #{db_label(db, missing: true)} #{paint("not found", :red)}"+ puts "Using default database instead: #{db_label(default_db)}\n\n"db = default_dbend@@ -97,8 +181,8 @@ module DiscourseGifsMigrationenddef find_component_in_db(db)- puts "Accessing database: \e[1;104m[#{db}]\e[0m"- puts " Searching for #{DiscourseGifs::COMPONENT_NAME} theme component..."+ puts "Accessing database: #{db_label(db)}"+ puts "Searching for #{DiscourseGifs::COMPONENT_NAME} theme component..."themes =RemoteTheme@@ -107,27 +191,35 @@ module DiscourseGifsMigration.map(&:theme)if themes.length > 1- puts " \e[33mMultiple (#{themes.length}) #{DiscourseGifs::COMPONENT_NAME} components found:\e[0m"- themes.each { |t| puts " - #{t.name} (ID: #{t.id})" }- puts " \e[33mInstall a single instance before running this task.\e[0m"+ status(+ "Multiple (#{themes.length}) #{DiscourseGifs::COMPONENT_NAME} components found:",+ :yellow,+ )+ themes.each { |t| item("#{t.name} (ID: #{t.id})") }+ status("Install a single instance before running this task.", :yellow)return nilelsif themes.one?theme = themes.first- puts " \e[1;34m✓ Found: \e[1m#{theme.name} (ID: #{theme.id})\e[0m"+ status("✓ Found: #{theme.name} (ID: #{theme.id})", :blue, bold: true)return themeend- puts " \e[33m✗ Not found.\e[0m"+ status("✗ Not found.", :yellow)nilenddef migrate_component(theme, enable_gifs:)- puts "\n Migrating settings for \e[1m#{theme.name} (ID: #{theme.id})\e[0m..."+ puts "\n Migrating settings for #{paint("#{theme.name} (ID: #{theme.id})", bold: true)}..."- overrides = theme.theme_settings.each_with_object({}) { |ts, h| h[ts.name] = ts.value }- # TC's default api_provider is "giphy" — applies to any site that never picked one.- provider = overrides["api_provider"].presence || "giphy"- puts " Detected provider: \e[1m#{provider}\e[0m"+ # Read the TC's *effective* settings — overrides AND schema defaults. A+ # ThemeSetting row only exists once a value is changed from its default, so+ # reading theme_settings alone would miss every untouched setting (e.g. the+ # default provider and content rating) and silently fall back to core's+ # defaults instead of preserving the site's actual behaviour.+ values =+ theme.settings.each_with_object({}) { |(name, setting), h| h[name.to_s] = setting.value }+ provider = values["api_provider"].presence || "giphy"+ puts " Detected provider: #{paint(provider, bold: true)}"mapping = (PROVIDER_MAPPINGS[provider] || {}).merge(SHARED_MAPPINGS)@@ -135,26 +227,35 @@ module DiscourseGifsMigrationerrors = []mapping.each do |tc_name, target|- raw = overrides[tc_name]+ raw = values[tc_name]next if raw.blank?new_value = target[:value] ? (target[:value][raw] || raw) : rawbegin- SiteSetting.set_and_log(- target[:name],- new_value,- Discourse.system_user,- "Migrated from #{DiscourseGifs::COMPONENT_NAME} theme component",- )- puts " - \e[0;31m#{tc_name}: #{raw}\e[0m => \e[0;32m#{target[:name]}: #{new_value}\e[0m"+ # set_and_log returns nil when the value already matches, so we only+ # report/count settings that actually changed — reading effective+ # values means most settings already equal core's default.+ changed =+ SiteSetting.set_and_log(+ target[:name],+ new_value,+ Discourse.system_user,+ "Migrated from #{DiscourseGifs::COMPONENT_NAME} theme component",+ )+ next unless changed++ success("#{tc_name}: #{raw} => #{target[:name]}: #{new_value}")migrated += 1rescue StandardError => eerrors << e- puts " \e[31m- failed to migrate '#{tc_name}': \e[1m#{e.message}\e[0m"+ failure("failed to migrate '#{tc_name}': #{e.message}")endend+ errors.concat(migrate_disabled_image_download_domains)+ errors.concat(migrate_translation_overrides(theme))+if enable_gifsbeginSiteSetting.set_and_log(@@ -163,16 +264,66 @@ module DiscourseGifsMigrationDiscourse.system_user,"Migrated from #{DiscourseGifs::COMPONENT_NAME} theme component",)- puts " - \e[0;32menable_gifs: true\e[0m (auto-enabled per task argument)"+ success("enable_gifs: true (auto-enabled per task argument)")migrated += 1rescue StandardError => eerrors << e- puts " \e[31m- failed to enable enable_gifs: \e[1m#{e.message}\e[0m"+ failure("failed to enable enable_gifs: #{e.message}")endend- puts " \e[1;32m✓ Migrated #{migrated} setting#{"s" if migrated != 1}\e[0m"- puts " \e[1;31m#{errors.size} error#{"s" if errors.size != 1}\e[0m" if errors.any?+ status("✓ Migrated #{migrated} setting#{"s" if migrated != 1}", :green, bold: true)+ status("#{errors.size} error#{"s" if errors.size != 1}", :red, bold: true) if errors.any?+ end++ # Sites that blocked the old provider's gifs from being downloaded via+ # `disabled_image_download_domains` should keep blocking gifs after the switch+ # to Klipy. We only act when a known gif-provider host is already listed, so+ # we never touch the setting on sites that weren't blocking gifs+ def migrate_disabled_image_download_domains+ hosts =+ SiteSetting.disabled_image_download_domains.to_s.split("|").map(&:strip).reject(&:empty?)+ return [] unless hosts.any? { |host| gif_provider_host?(host) }++ missing = KLIPY_MEDIA_HOSTS - hosts+ return [] if missing.empty?++ SiteSetting.set_and_log(+ :disabled_image_download_domains,+ (hosts + missing).join("|"),+ Discourse.system_user,+ "Added Klipy media hosts so gifs stay blocked after migrating from " \+ "#{DiscourseGifs::COMPONENT_NAME} theme component",+ )+ success(+ "disabled_image_download_domains += #{missing.join(", ")} " \+ "(preserving existing gif download block)",+ )+ []+ rescue StandardError => e+ failure("failed to update disabled_image_download_domains: #{e.message}")+ [e]+ end++ def gif_provider_host?(host)+ GIF_PROVIDER_DOMAINS.any? { |domain| host == domain || host.end_with?(".#{domain}") }+ end++ def migrate_translation_overrides(theme)+ theme+ .theme_translation_overrides+ .each_with_object([]) do |override, errors|+ core_key = "js.#{override.translation_key.sub(/\Agif\./, "gifs.")}"+ next unless I18n.exists?(core_key, :en)++ begin+ TranslationOverride.upsert!(override.locale, core_key, override.value)+ success("#{override.translation_key} (#{override.locale}) => #{core_key}")+ rescue StandardError => e+ errors << e+ failure("failed to migrate translation '#{override.translation_key}': #{e.message}")+ end+ endendenddiff --git a/spec/tasks/migrate_discourse_gifs_to_core_spec.rb b/spec/tasks/migrate_discourse_gifs_to_core_spec.rbindex 24cea6b3f45..b7f7ba419e4 100644--- a/spec/tasks/migrate_discourse_gifs_to_core_spec.rb+++ b/spec/tasks/migrate_discourse_gifs_to_core_spec.rb@@ -6,20 +6,89 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" dosilence_warnings { Discourse::Application.load_tasks }end+ # Mirrors the real discourse-gifs settings.yml so theme.settings resolves the+ # same defaults the migration reads in production (e.g. api_provider: giphy,+ # giphy_content_rating: r).+ SETTINGS_YAML = <<~YAML+ api_provider:+ default: "giphy"+ type: "enum"+ choices: [giphy, tenor, klipy]+ giphy_api_key:+ type: string+ default: ""+ giphy_file_format:+ type: enum+ default: webp+ choices: [webp, gif]+ giphy_content_rating:+ type: enum+ default: r+ choices: [g, pg, pg-13, r]+ giphy_locale:+ type: enum+ default: en+ choices:+ [+ ar, bn, cs, da, de, en, es, fa, fi, fr, hi, hu, id, it, iw, ja, ko,+ ms, nl, no, pl, pt, ro, ru, sv, th, tl, tr, uk, vi, zh-CN, zh-TW,+ ]+ limit_infinite_search_results:+ type: bool+ default: false+ max_results_limit:+ type: integer+ min: 24+ default: 240+ tenor_api_key:+ type: string+ default: ""+ tenor_file_detail:+ type: enum+ default: mediumgif+ choices: [gif, mediumgif, tinygif, nanogif]+ tenor_content_filter:+ type: "enum"+ default: high+ choices: [high, medium, low, "off"]+ tenor_country:+ type: string+ default: "US"+ tenor_locale:+ type: string+ default: "en_US"+ klipy_api_key:+ type: string+ default: ""+ klipy_file_detail:+ type: enum+ default: webp+ choices: [webp, gif]+ klipy_content_filter:+ type: "enum"+ default: high+ choices: [high, medium, low, "off"]+ klipy_country:+ type: string+ default: "US"+ klipy_locale:+ type: string+ default: "en_US"+ YAML+fab!(:remote_theme) doRemoteTheme.create!(remote_url: "https://github.com/discourse/discourse-gifs")end- fab!(:component) { Fabricate(:theme, component: true, remote_theme: remote_theme) }+ fab!(:component) do+ Fabricate(:theme, component: true, remote_theme: remote_theme).tap do |theme|+ theme.set_field(target: :settings, name: "yaml", value: SETTINGS_YAML)+ theme.save!+ end+ enddef add_overrides(theme, overrides)- overrides.each do |name, value|- ThemeSetting.create!(- theme: theme,- name: name.to_s,- value: value.to_s,- data_type: ThemeSetting.types[:string],- )- end+ overrides.each { |name, value| theme.update_setting(name, value) }+ theme.save!theme.reloadend@@ -97,12 +166,21 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" doendend- it "passes the Giphy locale through unchanged" do- add_overrides(component, api_provider: "giphy", giphy_locale: "fr")+ it "translates the Giphy language code to a Klipy xx_YY locale", :aggregate_failures do+ {+ "fr" => "fr_FR",+ "ja" => "ja_JP",+ "zh-CN" => "zh_CN",+ "iw" => "he_IL",+ }.each do |giphy_locale, expected|+ component.theme_settings.destroy_all+ add_overrides(component, api_provider: "giphy", giphy_locale: giphy_locale)- run_migration(component)+ run_migration(component)- expect(SiteSetting.klipy_locale).to eq("fr")+ expect(SiteSetting.klipy_locale).to eq(expected),+ "expected giphy '#{giphy_locale}' to map to '#{expected}', got '#{SiteSetting.klipy_locale}'"+ endendit "does not migrate the Giphy API key into klipy_api_key" do@@ -135,7 +213,8 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" doendend- it "passes Tenor content filter, country and locale through unchanged", :aggregate_failures do+ it "passes Tenor content filter, country and locale (already xx_YY) through",+ :aggregate_failures doadd_overrides(component,api_provider: "tenor",@@ -185,7 +264,7 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" docontext "with shared settings that apply regardless of provider" doit "migrates limit_infinite_search_results" do- add_overrides(component, api_provider: "tenor", limit_infinite_search_results: "true")+ add_overrides(component, api_provider: "tenor", limit_infinite_search_results: true)run_migration(component)@@ -193,7 +272,7 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" doendit "migrates max_results_limit" do- add_overrides(component, api_provider: "giphy", max_results_limit: "96")+ add_overrides(component, api_provider: "giphy", max_results_limit: 96)run_migration(component)@@ -201,6 +280,15 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" doendend+ context "when the TC uses its default (untouched) settings" do+ it "migrates effective defaults, e.g. giphy's default 'r' rating to klipy 'low'" do+ # No ThemeSetting rows exist, so this only passes if defaults are read.+ run_migration(component)++ expect(SiteSetting.klipy_content_filter).to eq("low")+ end+ end+context "when api_provider is not set in theme settings" doit "defaults to giphy and applies giphy mappings" doadd_overrides(component, giphy_content_rating: "pg")@@ -211,6 +299,72 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" doendend+ context "with disabled_image_download_domains" do+ it "adds Klipy media hosts when a gif-provider host was blocked, keeping others" do+ SiteSetting.disabled_image_download_domains = "example.com|media.giphy.com"++ run_migration(component)++ expect(SiteSetting.disabled_image_download_domains.split("|")).to include(+ "example.com",+ "media.giphy.com",+ "static.klipy.com",+ "static1.klipy.com",+ "static2.klipy.com",+ )+ end++ it "does not touch the setting when no gif-provider host is blocked" do+ SiteSetting.disabled_image_download_domains = "example.com"++ run_migration(component)++ expect(SiteSetting.disabled_image_download_domains).to eq("example.com")+ end++ it "does not duplicate Klipy hosts on re-run" do+ SiteSetting.disabled_image_download_domains = "media.giphy.com"++ run_migration(component)+ run_migration(component)++ hosts = SiteSetting.disabled_image_download_domains.split("|")+ expect(hosts.count("static.klipy.com")).to eq(1)+ end+ end++ context "with theme translation overrides" do+ def add_translation_override(theme, key, value, locale: "en")+ ThemeTranslationOverride.create!(theme:, locale:, translation_key: key, value:)+ end++ it "migrates customised strings (per locale) to the matching core site text",+ :aggregate_failures do+ add_translation_override(component, "gif.composer_title", "Add a GIF")+ add_translation_override(component, "gif.modal_title", "Chercher des GIFs", locale: "fr")++ run_migration(component)++ expect(+ TranslationOverride.find_by(+ locale: "en",+ translation_key: "js.gifs.composer_title",+ )&.value,+ ).to eq("Add a GIF")+ expect(+ TranslationOverride.find_by(locale: "fr", translation_key: "js.gifs.modal_title")&.value,+ ).to eq("Chercher des GIFs")+ end++ it "skips overrides of keys with no core equivalent" do+ add_translation_override(component, "gif.query", "Keyword")++ run_migration(component)++ expect(TranslationOverride.where("translation_key LIKE 'js.gifs.%'")).to be_empty+ end+ end+context "with the enable_gifs keyword" doit "leaves enable_gifs untouched by default" doSiteSetting.enable_gifs = false