A platform for community discussion. Free, open, simple.
FIX: Retranslate category and tag localizations after updates (#42407)
Natalie Tay20be010b0357
diff --git a/plugins/discourse-ai/app/jobs/regular/localize_categories.rb b/plugins/discourse-ai/app/jobs/regular/localize_categories.rbindex d411cd45b26..c56fa00835e 100644--- a/plugins/discourse-ai/app/jobs/regular/localize_categories.rb+++ b/plugins/discourse-ai/app/jobs/regular/localize_categories.rb@@ -31,17 +31,15 @@ module Jobs.where.not(locale: nil).order(:id).limit(limit)+ categories = categories.where(id: args[:category_id]) if args[:category_id].present?return if categories.empty?- remaining_limit = limitlocales = DiscourseAi::Translation.locales+ force = args[:force] || falsecategories.each do |category|- break if remaining_limit <= 0-existing_locales = CategoryLocalization.where(category_id: category.id).pluck(:locale)- missing_locales = locales - existing_locales - [category.locale]- missing_locales.each do |locale|- break if remaining_limit <= 0+ target_locales = force ? locales : locales - existing_locales+ target_locales.each do |locale|next if LocaleNormalizer.is_same?(locale, category.locale)begin@@ -50,6 +48,7 @@ module Jobslocale,short_text_llm_model:,post_raw_llm_model:,+ fields: args[:fields],)rescue FinalDestination::SSRFDetector::LookupFailedError# do nothing, there are too many sporadic lookup failures@@ -57,8 +56,6 @@ module JobsDiscourseAi::Translation::VerboseLogger.log("Failed to translate category #{category.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",)- ensure- remaining_limit -= 1endenddiff --git a/plugins/discourse-ai/app/jobs/regular/localize_tags.rb b/plugins/discourse-ai/app/jobs/regular/localize_tags.rbindex 32e6c341779..adcb8002355 100644--- a/plugins/discourse-ai/app/jobs/regular/localize_tags.rb+++ b/plugins/discourse-ai/app/jobs/regular/localize_tags.rb@@ -25,6 +25,7 @@ module Jobstags =DiscourseAi::Translation::TagCandidates.get.where.not(locale: nil).order(:id).limit(limit)+ tags = tags.where(id: args[:tag_id]) if args[:tag_id].present?return if tags.empty?# we remove localizations in locales that match the tag's locale@@ -34,27 +35,27 @@ module Jobs.where("tag_localizations.locale = tags.locale").delete_all- remaining_limit = limitlocales = SiteSetting.content_localization_supported_locales.split("|")+ force = args[:force] || falsetags.each do |tag|- break if remaining_limit <= 0-existing_locales = TagLocalization.where(tag_id: tag.id).pluck(:locale)- missing_locales = locales - existing_locales - [tag.locale]- missing_locales.each do |locale|- break if remaining_limit <= 0+ target_locales = force ? locales : locales - existing_locales+ target_locales.each do |locale|next if LocaleNormalizer.is_same?(locale, tag.locale)begin- DiscourseAi::Translation::TagLocalizer.localize(tag, locale, short_text_llm_model:)+ DiscourseAi::Translation::TagLocalizer.localize(+ tag,+ locale,+ short_text_llm_model:,+ fields: args[:fields],+ )rescue FinalDestination::SSRFDetector::LookupFailedError# do nothing, there are too many sporadic lookup failuresrescue => eDiscourseAi::Translation::VerboseLogger.log("Failed to translate tag #{tag.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",)- ensure- remaining_limit -= 1endendenddiff --git a/plugins/discourse-ai/lib/translation/category_localizer.rb b/plugins/discourse-ai/lib/translation/category_localizer.rbindex 9a48657039f..4938991dc1f 100644--- a/plugins/discourse-ai/lib/translation/category_localizer.rb+++ b/plugins/discourse-ai/lib/translation/category_localizer.rb@@ -7,37 +7,40 @@ module DiscourseAicategory,target_locale = I18n.locale,short_text_llm_model: nil,- post_raw_llm_model: nil+ post_raw_llm_model: nil,+ fields: nil)return if category.blank? || target_locale.blank?target_locale = target_locale.to_s.sub("-", "_")-- translated_name =- ShortTextTranslator.new(- text: category.name,- target_locale:,- llm_model: short_text_llm_model,- ).translate- translated_description =- if category.description_excerpt.present?- PostRawTranslator.new(- text: category.description_excerpt,- target_locale:,- llm_model: post_raw_llm_model,- ).translate- else- ""- end-localization =CategoryLocalization.find_or_initialize_by(category_id: category.id,locale: target_locale,)+ fields = %w[name description] if fields.blank? || localization.new_record?++ if fields.include?("name")+ localization.name =+ ShortTextTranslator.new(+ text: category.name,+ target_locale:,+ llm_model: short_text_llm_model,+ ).translate+ end- localization.name = translated_name- localization.description = translated_description+ if fields.include?("description")+ localization.description =+ if category.description_excerpt.present?+ PostRawTranslator.new(+ text: category.description_excerpt,+ target_locale:,+ llm_model: post_raw_llm_model,+ ).translate+ else+ ""+ end+ endlocalization.save!localizationenddiff --git a/plugins/discourse-ai/lib/translation/entry_point.rb b/plugins/discourse-ai/lib/translation/entry_point.rbindex caa73beebde..5a3339b5043 100644--- a/plugins/discourse-ai/lib/translation/entry_point.rb+++ b/plugins/discourse-ai/lib/translation/entry_point.rb@@ -32,6 +32,39 @@ module DiscourseAiendend+ plugin.on(:category_updated) do |category|+ next unless category.is_a?(Category)++ changed_fields =+ %w[name description].select { |field| category.previous_changes.key?(field) }++ if DiscourseAi::Translation.enabled? && changed_fields.present?+ Jobs.enqueue(+ :localize_categories,+ category_id: category.id,+ fields: changed_fields,+ limit: 1,+ force: true,+ )+ end+ end++ plugin.on(:tag_updated) do |tag|+ next unless tag.is_a?(Tag)++ changed_fields = %w[name description].select { |field| tag.previous_changes.key?(field) }++ if DiscourseAi::Translation.enabled? && changed_fields.present?+ Jobs.enqueue(+ :localize_tags,+ tag_id: tag.id,+ fields: changed_fields,+ limit: 1,+ force: true,+ )+ end+ end+plugin.on(:site_setting_changed) do |name, _old_value, new_value|if name == :ai_translation_enabled && new_value &&SiteSetting.ai_translation_backfill_start_date.blank?diff --git a/plugins/discourse-ai/lib/translation/tag_localizer.rb b/plugins/discourse-ai/lib/translation/tag_localizer.rbindex ec23082c6b7..cf70d2dd7ec 100644--- a/plugins/discourse-ai/lib/translation/tag_localizer.rb+++ b/plugins/discourse-ai/lib/translation/tag_localizer.rb@@ -3,31 +3,32 @@module DiscourseAimodule Translationclass TagLocalizer- def self.localize(tag, target_locale = I18n.locale, short_text_llm_model: nil)+ def self.localize(tag, target_locale = I18n.locale, short_text_llm_model: nil, fields: nil)return if tag.blank? || target_locale.blank?target_locale = target_locale.to_s.sub("-", "_")+ localization = TagLocalization.find_or_initialize_by(tag_id: tag.id, locale: target_locale)+ fields = %w[name description] if fields.blank? || localization.new_record?- translated_name =- ShortTextTranslator.new(- text: tag.name,- target_locale:,- llm_model: short_text_llm_model,- ).translate-- translated_description =- if tag.description.present?+ if fields.include?("name")+ localization.name =ShortTextTranslator.new(- text: tag.description,+ text: tag.name,target_locale:,llm_model: short_text_llm_model,).translate- end-- localization = TagLocalization.find_or_initialize_by(tag_id: tag.id, locale: target_locale)+ end- localization.name = translated_name- localization.description = translated_description+ if fields.include?("description")+ localization.description =+ if tag.description.present?+ ShortTextTranslator.new(+ text: tag.description,+ target_locale:,+ llm_model: short_text_llm_model,+ ).translate+ end+ endlocalization.save!localizationenddiff --git a/plugins/discourse-ai/spec/jobs/regular/localize_categories_spec.rb b/plugins/discourse-ai/spec/jobs/regular/localize_categories_spec.rbindex 2ba536433f3..4888994f1b6 100644--- a/plugins/discourse-ai/spec/jobs/regular/localize_categories_spec.rb+++ b/plugins/discourse-ai/spec/jobs/regular/localize_categories_spec.rb@@ -133,6 +133,38 @@ describe Jobs::LocalizeCategories dojob.execute({ limit: 10 })end+ it "retranslates existing localizations for a requested category" do+ localize_all_categories("pt_BR", "zh_CN")+ category = Fabricate(:category, locale: "en", description: "Source description")+ localizations =+ %w[pt_BR zh_CN].map do |locale|+ Fabricate(+ :category_localization,+ category:,+ locale:,+ name: "Old name",+ description: "Old description",+ )+ end++ short_text_translator = instance_double(DiscourseAi::Translation::ShortTextTranslator)+ allow(DiscourseAi::Translation::ShortTextTranslator).to receive(:new).and_return(+ short_text_translator,+ )+ allow(short_text_translator).to receive(:translate).and_return("New name")+ post_raw_translator = instance_double(DiscourseAi::Translation::PostRawTranslator)+ allow(DiscourseAi::Translation::PostRawTranslator).to receive(:new).and_return(+ post_raw_translator,+ )+ allow(post_raw_translator).to receive(:translate).and_return("New description")++ job.execute({ limit: 1, category_id: category.id, fields: ["description"], force: true })++ expect(+ CategoryLocalization.where(id: localizations.map(&:id)).pluck(:name, :description),+ ).to contain_exactly(["Old name", "New description"], ["Old name", "New description"])+ end+it "handles translation errors gracefully" dolocalize_all_categories("pt", "zh_CN")diff --git a/plugins/discourse-ai/spec/jobs/regular/localize_tags_spec.rb b/plugins/discourse-ai/spec/jobs/regular/localize_tags_spec.rbindex a5f44034e20..2815deb7fad 100644--- a/plugins/discourse-ai/spec/jobs/regular/localize_tags_spec.rb+++ b/plugins/discourse-ai/spec/jobs/regular/localize_tags_spec.rb@@ -76,10 +76,10 @@ describe Jobs::LocalizeTags do.with(tag, "zh_CN", has_entries(short_text_llm_model: anything)).once- job.execute({ limit: 10 })+ job.execute({ limit: 1 })end- it "limits the number of localizations" do+ it "limits the number of tags" doSiteSetting.content_localization_supported_locales = "pt"6.times { Fabricate(:tag, locale: "en") }@@ -108,6 +108,55 @@ describe Jobs::LocalizeTags dojob.execute({ limit: 10 })end+ it "retranslates existing localizations for a requested tag" do+ localize_all_tags("pt_BR", "zh_CN")+ tag = Fabricate(:tag, locale: "en", description: "Source description")+ localizations =+ %w[pt_BR zh_CN].map do |locale|+ Fabricate(+ :tag_localization,+ tag:,+ locale:,+ name: "old-name",+ description: "Old description",+ )+ end++ allow(DiscourseAi::Translation::ShortTextTranslator).to receive(:new) do |text:, **|+ translated = text == tag.name ? "new-name" : "New description"+ instance_double(DiscourseAi::Translation::ShortTextTranslator, translate: translated)+ end++ job.execute({ limit: 1, tag_id: tag.id, fields: ["description"], force: true })++ expect(+ TagLocalization.where(id: localizations.map(&:id)).pluck(:name, :description),+ ).to contain_exactly(["old-name", "New description"], ["old-name", "New description"])+ end++ it "clears existing localized descriptions when the source description is blank" do+ tag = Fabricate(:tag, locale: "en", description: "")+ localizations =+ %w[pt_BR zh_CN].map do |locale|+ Fabricate(+ :tag_localization,+ tag:,+ locale:,+ name: "old-name",+ description: "Old description",+ )+ end++ allow(DiscourseAi::Translation::ShortTextTranslator).to receive(:new)++ job.execute({ limit: 1, tag_id: tag.id, fields: ["description"], force: true })++ expect(+ TagLocalization.where(id: localizations.map(&:id)).pluck(:name, :description),+ ).to contain_exactly(["old-name", nil], ["old-name", nil])+ expect(DiscourseAi::Translation::ShortTextTranslator).not_to have_received(:new)+ end+it "handles translation errors gracefully" dotag1 = Fabricate(:tag, name: "first-tag", locale: "en")diff --git a/plugins/discourse-ai/spec/lib/translation/entry_point_spec.rb b/plugins/discourse-ai/spec/lib/translation/entry_point_spec.rbindex bbaaa0f129e..6664283c42c 100644--- a/plugins/discourse-ai/spec/lib/translation/entry_point_spec.rb+++ b/plugins/discourse-ai/spec/lib/translation/entry_point_spec.rb@@ -166,4 +166,90 @@ describe DiscourseAi::Translation::EntryPoint doexpect(job_enqueued?(job: :detect_translate_topic)).to eq falseendend++ describe "upon category updated" do+ it "enqueues category retranslation when the description changes" do+ SiteSetting.content_localization_supported_locales = "de|fr"+ category = Fabricate(:category_with_definition, locale: "en")++ category.update!(description: "An updated category description")++ expect_job_enqueued(+ job: :localize_categories,+ args: {+ category_id: category.id,+ fields: ["description"],+ limit: 1,+ force: true,+ },+ )+ end++ it "enqueues category retranslation when the name changes" do+ category = Fabricate(:category, locale: "en")++ category.update!(name: "A new category name")++ expect_job_enqueued(+ job: :localize_categories,+ args: {+ category_id: category.id,+ fields: ["name"],+ limit: 1,+ force: true,+ },+ )+ end++ it "does not enqueue category retranslation when another field changes" do+ category = Fabricate(:category, locale: "en")++ category.update!(color: "FF0000")++ expect(job_enqueued?(job: :localize_categories)).to eq(false)+ end+ end++ describe "upon tag updated" do+ it "enqueues tag retranslation when the description changes" do+ SiteSetting.content_localization_supported_locales = "de|fr"+ tag = Fabricate(:tag, locale: "en")++ tag.update!(description: "An updated tag description")++ expect_job_enqueued(+ job: :localize_tags,+ args: {+ tag_id: tag.id,+ fields: ["description"],+ limit: 1,+ force: true,+ },+ )+ end++ it "enqueues tag retranslation when the name changes" do+ tag = Fabricate(:tag, locale: "en")++ tag.update!(name: "a-new-tag-name")++ expect_job_enqueued(+ job: :localize_tags,+ args: {+ tag_id: tag.id,+ fields: ["name"],+ limit: 1,+ force: true,+ },+ )+ end++ it "does not enqueue tag retranslation when another field changes" do+ tag = Fabricate(:tag, locale: "en")++ tag.update!(staff_topic_count: 1)++ expect(job_enqueued?(job: :localize_tags)).to eq(false)+ end+ endend