Skip to content

A platform for community discussion. Free, open, simple.

DEV: update GIFs migration script (#40741)

David Battersby60fb6af2d8f6
diff --git a/lib/tasks/migrate_discourse_gifs_to_core.rake b/lib/tasks/migrate_discourse_gifs_to_core.rake
index 1b44b197256..186076b08fd 100644
--- a/lib/tasks/migrate_discourse_gifs_to_core.rake
+++ b/lib/tasks/migrate_discourse_gifs_to_core.rake
@@ -3,8 +3,53 @@
require "discourse_gifs"
module DiscourseGifsMigration
+ # Klipy expects locale in xx_YY form (ISO 639-1 language + ISO 3166-1 alpha-2
+ # country). Giphy stores a bare language code, so each of its locale choices is
+ # translated to a representative xx_YY. Where a language spans several countries
+ # the predominant one is used — these picks are deliberate, not 1:1:
+ # en→en_US, pt→pt_BR, es→es_ES, ar→ar_SA, bn→bn_BD, ms→ms_MY.
+ # "iw" is Giphy's legacy code for Hebrew, normalised to the modern "he".
+ GIPHY_LOCALES = {
+ "ar" => "ar_SA",
+ "bn" => "bn_BD",
+ "cs" => "cs_CZ",
+ "da" => "da_DK",
+ "de" => "de_DE",
+ "en" => "en_US",
+ "es" => "es_ES",
+ "fa" => "fa_IR",
+ "fi" => "fi_FI",
+ "fr" => "fr_FR",
+ "hi" => "hi_IN",
+ "hu" => "hu_HU",
+ "id" => "id_ID",
+ "it" => "it_IT",
+ "iw" => "he_IL",
+ "ja" => "ja_JP",
+ "ko" => "ko_KR",
+ "ms" => "ms_MY",
+ "nl" => "nl_NL",
+ "no" => "no_NO",
+ "pl" => "pl_PL",
+ "pt" => "pt_BR",
+ "ro" => "ro_RO",
+ "ru" => "ru_RU",
+ "sv" => "sv_SE",
+ "th" => "th_TH",
+ "tl" => "tl_PH",
+ "tr" => "tr_TR",
+ "uk" => "uk_UA",
+ "vi" => "vi_VN",
+ "zh-CN" => "zh_CN",
+ "zh-TW" => "zh_TW",
+ }.freeze
+
# Mapping from each TC provider's theme settings to core site settings.
# `value:` is an optional lookup table for non-1:1 translations.
+ #
+ # Tenor and Klipy already store locale in Klipy's xx_YY form, so their locale
+ # carries across unchanged; Giphy's bare language code is translated via
+ # GIPHY_LOCALES above.
PROVIDER_MAPPINGS = {
"giphy" => {
"giphy_file_format" => {
@@ -21,6 +66,7 @@ module DiscourseGifsMigration
},
"giphy_locale" => {
name: "klipy_locale",
+ value: GIPHY_LOCALES,
},
},
"tenor" => {
@@ -72,16 +118,54 @@ module DiscourseGifsMigration
},
}.freeze
+ GIF_PROVIDER_DOMAINS = %w[giphy.com tenor.com].freeze
+
+ KLIPY_MEDIA_HOSTS = %w[static.klipy.com static1.klipy.com static2.klipy.com].freeze
+
+ # SGR foreground color codes for terminal output
+ COLORS = { red: 31, green: 32, yellow: 33, blue: 34 }.freeze
+
module_function
+ def paint(text, color = nil, bold: false)
+ codes = []
+ codes << 1 if bold
+ codes << COLORS.fetch(color) if color
+ return text if codes.empty?
+
+ "\e[#{codes.join(";")}m#{text}\e[0m"
+ end
+
+ # A highlighted [db] label (blue, or red when the named db is missing).
+ def db_label(db, missing: false)
+ "\e[#{missing ? "1;101" : "1;104"}m[#{db}]\e[0m"
+ end
+
+ # indented bullet lines used throughout the migration output
+ def item(text)
+ puts " - #{text}"
+ end
+
+ def success(text)
+ item(paint(text, :green))
+ end
+
+ def failure(text)
+ item(paint(text, :red))
+ end
+
+ def status(text, color, bold: false)
+ puts " #{paint(text, color, bold: bold)}"
+ end
+
def find_components
if ENV["RAILS_DB"].present?
db = ENV["RAILS_DB"]
if !RailsMultisite::ConnectionManagement.has_db?(db)
default_db = RailsMultisite::ConnectionManagement::DEFAULT
- puts "\e[31m✗ Database \e[1;101m[#{db}]\e[0m \e[31mnot found\e[0m"
- puts "Using default database instead: \e[1;104m[#{default_db}]\e[0m\n\n"
+ puts "#{paint("✗ Database", :red)} #{db_label(db, missing: true)} #{paint("not found", :red)}"
+ puts "Using default database instead: #{db_label(default_db)}\n\n"
db = default_db
end
@@ -97,8 +181,8 @@ module DiscourseGifsMigration
end
def find_component_in_db(db)
- puts "Accessing database: \e[1;104m[#{db}]\e[0m"
- puts " Searching for #{DiscourseGifs::COMPONENT_NAME} theme component..."
+ puts "Accessing database: #{db_label(db)}"
+ puts "Searching for #{DiscourseGifs::COMPONENT_NAME} theme component..."
themes =
RemoteTheme
@@ -107,27 +191,35 @@ module DiscourseGifsMigration
.map(&:theme)
if themes.length > 1
- puts " \e[33mMultiple (#{themes.length}) #{DiscourseGifs::COMPONENT_NAME} components found:\e[0m"
- themes.each { |t| puts " - #{t.name} (ID: #{t.id})" }
- puts " \e[33mInstall a single instance before running this task.\e[0m"
+ status(
+ "Multiple (#{themes.length}) #{DiscourseGifs::COMPONENT_NAME} components found:",
+ :yellow,
+ )
+ themes.each { |t| item("#{t.name} (ID: #{t.id})") }
+ status("Install a single instance before running this task.", :yellow)
return nil
elsif themes.one?
theme = themes.first
- puts " \e[1;34m✓ Found: \e[1m#{theme.name} (ID: #{theme.id})\e[0m"
+ status("✓ Found: #{theme.name} (ID: #{theme.id})", :blue, bold: true)
return theme
end
- puts " \e[33m✗ Not found.\e[0m"
+ status("✗ Not found.", :yellow)
nil
end
def migrate_component(theme, enable_gifs:)
- puts "\n Migrating settings for \e[1m#{theme.name} (ID: #{theme.id})\e[0m..."
+ puts "\n Migrating settings for #{paint("#{theme.name} (ID: #{theme.id})", bold: true)}..."
- overrides = theme.theme_settings.each_with_object({}) { |ts, h| h[ts.name] = ts.value }
- # TC's default api_provider is "giphy" — applies to any site that never picked one.
- provider = overrides["api_provider"].presence || "giphy"
- puts " Detected provider: \e[1m#{provider}\e[0m"
+ # Read the TC's *effective* settings — overrides AND schema defaults. A
+ # ThemeSetting row only exists once a value is changed from its default, so
+ # reading theme_settings alone would miss every untouched setting (e.g. the
+ # default provider and content rating) and silently fall back to core's
+ # defaults instead of preserving the site's actual behaviour.
+ values =
+ theme.settings.each_with_object({}) { |(name, setting), h| h[name.to_s] = setting.value }
+ provider = values["api_provider"].presence || "giphy"
+ puts " Detected provider: #{paint(provider, bold: true)}"
mapping = (PROVIDER_MAPPINGS[provider] || {}).merge(SHARED_MAPPINGS)
@@ -135,26 +227,35 @@ module DiscourseGifsMigration
errors = []
mapping.each do |tc_name, target|
- raw = overrides[tc_name]
+ raw = values[tc_name]
next if raw.blank?
new_value = target[:value] ? (target[:value][raw] || raw) : raw
begin
- SiteSetting.set_and_log(
- target[:name],
- new_value,
- Discourse.system_user,
- "Migrated from #{DiscourseGifs::COMPONENT_NAME} theme component",
- )
- puts " - \e[0;31m#{tc_name}: #{raw}\e[0m => \e[0;32m#{target[:name]}: #{new_value}\e[0m"
+ # set_and_log returns nil when the value already matches, so we only
+ # report/count settings that actually changed — reading effective
+ # values means most settings already equal core's default.
+ changed =
+ SiteSetting.set_and_log(
+ target[:name],
+ new_value,
+ Discourse.system_user,
+ "Migrated from #{DiscourseGifs::COMPONENT_NAME} theme component",
+ )
+ next unless changed
+
+ success("#{tc_name}: #{raw} => #{target[:name]}: #{new_value}")
migrated += 1
rescue StandardError => e
errors << e
- puts " \e[31m- failed to migrate '#{tc_name}': \e[1m#{e.message}\e[0m"
+ failure("failed to migrate '#{tc_name}': #{e.message}")
end
end
+ errors.concat(migrate_disabled_image_download_domains)
+ errors.concat(migrate_translation_overrides(theme))
+
if enable_gifs
begin
SiteSetting.set_and_log(
@@ -163,16 +264,66 @@ module DiscourseGifsMigration
Discourse.system_user,
"Migrated from #{DiscourseGifs::COMPONENT_NAME} theme component",
)
- puts " - \e[0;32menable_gifs: true\e[0m (auto-enabled per task argument)"
+ success("enable_gifs: true (auto-enabled per task argument)")
migrated += 1
rescue StandardError => e
errors << e
- puts " \e[31m- failed to enable enable_gifs: \e[1m#{e.message}\e[0m"
+ failure("failed to enable enable_gifs: #{e.message}")
end
end
- puts " \e[1;32m✓ Migrated #{migrated} setting#{"s" if migrated != 1}\e[0m"
- puts " \e[1;31m#{errors.size} error#{"s" if errors.size != 1}\e[0m" if errors.any?
+ status("✓ Migrated #{migrated} setting#{"s" if migrated != 1}", :green, bold: true)
+ status("#{errors.size} error#{"s" if errors.size != 1}", :red, bold: true) if errors.any?
+ end
+
+ # Sites that blocked the old provider's gifs from being downloaded via
+ # `disabled_image_download_domains` should keep blocking gifs after the switch
+ # to Klipy. We only act when a known gif-provider host is already listed, so
+ # we never touch the setting on sites that weren't blocking gifs
+ def migrate_disabled_image_download_domains
+ hosts =
+ SiteSetting.disabled_image_download_domains.to_s.split("|").map(&:strip).reject(&:empty?)
+ return [] unless hosts.any? { |host| gif_provider_host?(host) }
+
+ missing = KLIPY_MEDIA_HOSTS - hosts
+ return [] if missing.empty?
+
+ SiteSetting.set_and_log(
+ :disabled_image_download_domains,
+ (hosts + missing).join("|"),
+ Discourse.system_user,
+ "Added Klipy media hosts so gifs stay blocked after migrating from " \
+ "#{DiscourseGifs::COMPONENT_NAME} theme component",
+ )
+ success(
+ "disabled_image_download_domains += #{missing.join(", ")} " \
+ "(preserving existing gif download block)",
+ )
+ []
+ rescue StandardError => e
+ failure("failed to update disabled_image_download_domains: #{e.message}")
+ [e]
+ end
+
+ def gif_provider_host?(host)
+ GIF_PROVIDER_DOMAINS.any? { |domain| host == domain || host.end_with?(".#{domain}") }
+ end
+
+ def migrate_translation_overrides(theme)
+ theme
+ .theme_translation_overrides
+ .each_with_object([]) do |override, errors|
+ core_key = "js.#{override.translation_key.sub(/\Agif\./, "gifs.")}"
+ next unless I18n.exists?(core_key, :en)
+
+ begin
+ TranslationOverride.upsert!(override.locale, core_key, override.value)
+ success("#{override.translation_key} (#{override.locale}) => #{core_key}")
+ rescue StandardError => e
+ errors << e
+ failure("failed to migrate translation '#{override.translation_key}': #{e.message}")
+ end
+ end
end
end
diff --git a/spec/tasks/migrate_discourse_gifs_to_core_spec.rb b/spec/tasks/migrate_discourse_gifs_to_core_spec.rb
index 24cea6b3f45..b7f7ba419e4 100644
--- a/spec/tasks/migrate_discourse_gifs_to_core_spec.rb
+++ b/spec/tasks/migrate_discourse_gifs_to_core_spec.rb
@@ -6,20 +6,89 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" do
silence_warnings { Discourse::Application.load_tasks }
end
+ # Mirrors the real discourse-gifs settings.yml so theme.settings resolves the
+ # same defaults the migration reads in production (e.g. api_provider: giphy,
+ # giphy_content_rating: r).
+ SETTINGS_YAML = <<~YAML
+ api_provider:
+ default: "giphy"
+ type: "enum"
+ choices: [giphy, tenor, klipy]
+ giphy_api_key:
+ type: string
+ default: ""
+ giphy_file_format:
+ type: enum
+ default: webp
+ choices: [webp, gif]
+ giphy_content_rating:
+ type: enum
+ default: r
+ choices: [g, pg, pg-13, r]
+ giphy_locale:
+ type: enum
+ default: en
+ choices:
+ [
+ ar, bn, cs, da, de, en, es, fa, fi, fr, hi, hu, id, it, iw, ja, ko,
+ ms, nl, no, pl, pt, ro, ru, sv, th, tl, tr, uk, vi, zh-CN, zh-TW,
+ ]
+ limit_infinite_search_results:
+ type: bool
+ default: false
+ max_results_limit:
+ type: integer
+ min: 24
+ default: 240
+ tenor_api_key:
+ type: string
+ default: ""
+ tenor_file_detail:
+ type: enum
+ default: mediumgif
+ choices: [gif, mediumgif, tinygif, nanogif]
+ tenor_content_filter:
+ type: "enum"
+ default: high
+ choices: [high, medium, low, "off"]
+ tenor_country:
+ type: string
+ default: "US"
+ tenor_locale:
+ type: string
+ default: "en_US"
+ klipy_api_key:
+ type: string
+ default: ""
+ klipy_file_detail:
+ type: enum
+ default: webp
+ choices: [webp, gif]
+ klipy_content_filter:
+ type: "enum"
+ default: high
+ choices: [high, medium, low, "off"]
+ klipy_country:
+ type: string
+ default: "US"
+ klipy_locale:
+ type: string
+ default: "en_US"
+ YAML
+
fab!(:remote_theme) do
RemoteTheme.create!(remote_url: "https://github.com/discourse/discourse-gifs")
end
- fab!(:component) { Fabricate(:theme, component: true, remote_theme: remote_theme) }
+ fab!(:component) do
+ Fabricate(:theme, component: true, remote_theme: remote_theme).tap do |theme|
+ theme.set_field(target: :settings, name: "yaml", value: SETTINGS_YAML)
+ theme.save!
+ end
+ end
def add_overrides(theme, overrides)
- overrides.each do |name, value|
- ThemeSetting.create!(
- theme: theme,
- name: name.to_s,
- value: value.to_s,
- data_type: ThemeSetting.types[:string],
- )
- end
+ overrides.each { |name, value| theme.update_setting(name, value) }
+ theme.save!
theme.reload
end
@@ -97,12 +166,21 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" do
end
end
- it "passes the Giphy locale through unchanged" do
- add_overrides(component, api_provider: "giphy", giphy_locale: "fr")
+ it "translates the Giphy language code to a Klipy xx_YY locale", :aggregate_failures do
+ {
+ "fr" => "fr_FR",
+ "ja" => "ja_JP",
+ "zh-CN" => "zh_CN",
+ "iw" => "he_IL",
+ }.each do |giphy_locale, expected|
+ component.theme_settings.destroy_all
+ add_overrides(component, api_provider: "giphy", giphy_locale: giphy_locale)
- run_migration(component)
+ run_migration(component)
- expect(SiteSetting.klipy_locale).to eq("fr")
+ expect(SiteSetting.klipy_locale).to eq(expected),
+ "expected giphy '#{giphy_locale}' to map to '#{expected}', got '#{SiteSetting.klipy_locale}'"
+ end
end
it "does not migrate the Giphy API key into klipy_api_key" do
@@ -135,7 +213,8 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" do
end
end
- it "passes Tenor content filter, country and locale through unchanged", :aggregate_failures do
+ it "passes Tenor content filter, country and locale (already xx_YY) through",
+ :aggregate_failures do
add_overrides(
component,
api_provider: "tenor",
@@ -185,7 +264,7 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" do
context "with shared settings that apply regardless of provider" do
it "migrates limit_infinite_search_results" do
- add_overrides(component, api_provider: "tenor", limit_infinite_search_results: "true")
+ add_overrides(component, api_provider: "tenor", limit_infinite_search_results: true)
run_migration(component)
@@ -193,7 +272,7 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" do
end
it "migrates max_results_limit" do
- add_overrides(component, api_provider: "giphy", max_results_limit: "96")
+ add_overrides(component, api_provider: "giphy", max_results_limit: 96)
run_migration(component)
@@ -201,6 +280,15 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" do
end
end
+ context "when the TC uses its default (untouched) settings" do
+ it "migrates effective defaults, e.g. giphy's default 'r' rating to klipy 'low'" do
+ # No ThemeSetting rows exist, so this only passes if defaults are read.
+ run_migration(component)
+
+ expect(SiteSetting.klipy_content_filter).to eq("low")
+ end
+ end
+
context "when api_provider is not set in theme settings" do
it "defaults to giphy and applies giphy mappings" do
add_overrides(component, giphy_content_rating: "pg")
@@ -211,6 +299,72 @@ RSpec.describe "tasks/migrate_discourse_gifs_to_core" do
end
end
+ context "with disabled_image_download_domains" do
+ it "adds Klipy media hosts when a gif-provider host was blocked, keeping others" do
+ SiteSetting.disabled_image_download_domains = "example.com|media.giphy.com"
+
+ run_migration(component)
+
+ expect(SiteSetting.disabled_image_download_domains.split("|")).to include(
+ "example.com",
+ "media.giphy.com",
+ "static.klipy.com",
+ "static1.klipy.com",
+ "static2.klipy.com",
+ )
+ end
+
+ it "does not touch the setting when no gif-provider host is blocked" do
+ SiteSetting.disabled_image_download_domains = "example.com"
+
+ run_migration(component)
+
+ expect(SiteSetting.disabled_image_download_domains).to eq("example.com")
+ end
+
+ it "does not duplicate Klipy hosts on re-run" do
+ SiteSetting.disabled_image_download_domains = "media.giphy.com"
+
+ run_migration(component)
+ run_migration(component)
+
+ hosts = SiteSetting.disabled_image_download_domains.split("|")
+ expect(hosts.count("static.klipy.com")).to eq(1)
+ end
+ end
+
+ context "with theme translation overrides" do
+ def add_translation_override(theme, key, value, locale: "en")
+ ThemeTranslationOverride.create!(theme:, locale:, translation_key: key, value:)
+ end
+
+ it "migrates customised strings (per locale) to the matching core site text",
+ :aggregate_failures do
+ add_translation_override(component, "gif.composer_title", "Add a GIF")
+ add_translation_override(component, "gif.modal_title", "Chercher des GIFs", locale: "fr")
+
+ run_migration(component)
+
+ expect(
+ TranslationOverride.find_by(
+ locale: "en",
+ translation_key: "js.gifs.composer_title",
+ )&.value,
+ ).to eq("Add a GIF")
+ expect(
+ TranslationOverride.find_by(locale: "fr", translation_key: "js.gifs.modal_title")&.value,
+ ).to eq("Chercher des GIFs")
+ end
+
+ it "skips overrides of keys with no core equivalent" do
+ add_translation_override(component, "gif.query", "Keyword")
+
+ run_migration(component)
+
+ expect(TranslationOverride.where("translation_key LIKE 'js.gifs.%'")).to be_empty
+ end
+ end
+
context "with the enable_gifs keyword" do
it "leaves enable_gifs untouched by default" do
SiteSetting.enable_gifs = false