Skip to content

A platform for community discussion. Free, open, simple.

FIX: Retranslate category and tag localizations after updates (#42407)

Natalie Tay20be010b0357
diff --git a/plugins/discourse-ai/app/jobs/regular/localize_categories.rb b/plugins/discourse-ai/app/jobs/regular/localize_categories.rb
index d411cd45b26..c56fa00835e 100644
--- a/plugins/discourse-ai/app/jobs/regular/localize_categories.rb
+++ b/plugins/discourse-ai/app/jobs/regular/localize_categories.rb
@@ -31,17 +31,15 @@ module Jobs
.where.not(locale: nil)
.order(:id)
.limit(limit)
+ categories = categories.where(id: args[:category_id]) if args[:category_id].present?
return if categories.empty?
- remaining_limit = limit
locales = DiscourseAi::Translation.locales
+ force = args[:force] || false
categories.each do |category|
- break if remaining_limit <= 0
-
existing_locales = CategoryLocalization.where(category_id: category.id).pluck(:locale)
- missing_locales = locales - existing_locales - [category.locale]
- missing_locales.each do |locale|
- break if remaining_limit <= 0
+ target_locales = force ? locales : locales - existing_locales
+ target_locales.each do |locale|
next if LocaleNormalizer.is_same?(locale, category.locale)
begin
@@ -50,6 +48,7 @@ module Jobs
locale,
short_text_llm_model:,
post_raw_llm_model:,
+ fields: args[:fields],
)
rescue FinalDestination::SSRFDetector::LookupFailedError
# do nothing, there are too many sporadic lookup failures
@@ -57,8 +56,6 @@ module Jobs
DiscourseAi::Translation::VerboseLogger.log(
"Failed to translate category #{category.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
)
- ensure
- remaining_limit -= 1
end
end
diff --git a/plugins/discourse-ai/app/jobs/regular/localize_tags.rb b/plugins/discourse-ai/app/jobs/regular/localize_tags.rb
index 32e6c341779..adcb8002355 100644
--- a/plugins/discourse-ai/app/jobs/regular/localize_tags.rb
+++ b/plugins/discourse-ai/app/jobs/regular/localize_tags.rb
@@ -25,6 +25,7 @@ module Jobs
tags =
DiscourseAi::Translation::TagCandidates.get.where.not(locale: nil).order(:id).limit(limit)
+ tags = tags.where(id: args[:tag_id]) if args[:tag_id].present?
return if tags.empty?
# we remove localizations in locales that match the tag's locale
@@ -34,27 +35,27 @@ module Jobs
.where("tag_localizations.locale = tags.locale")
.delete_all
- remaining_limit = limit
locales = SiteSetting.content_localization_supported_locales.split("|")
+ force = args[:force] || false
tags.each do |tag|
- break if remaining_limit <= 0
-
existing_locales = TagLocalization.where(tag_id: tag.id).pluck(:locale)
- missing_locales = locales - existing_locales - [tag.locale]
- missing_locales.each do |locale|
- break if remaining_limit <= 0
+ target_locales = force ? locales : locales - existing_locales
+ target_locales.each do |locale|
next if LocaleNormalizer.is_same?(locale, tag.locale)
begin
- DiscourseAi::Translation::TagLocalizer.localize(tag, locale, short_text_llm_model:)
+ DiscourseAi::Translation::TagLocalizer.localize(
+ tag,
+ locale,
+ short_text_llm_model:,
+ fields: args[:fields],
+ )
rescue FinalDestination::SSRFDetector::LookupFailedError
# do nothing, there are too many sporadic lookup failures
rescue => e
DiscourseAi::Translation::VerboseLogger.log(
"Failed to translate tag #{tag.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
)
- ensure
- remaining_limit -= 1
end
end
end
diff --git a/plugins/discourse-ai/lib/translation/category_localizer.rb b/plugins/discourse-ai/lib/translation/category_localizer.rb
index 9a48657039f..4938991dc1f 100644
--- a/plugins/discourse-ai/lib/translation/category_localizer.rb
+++ b/plugins/discourse-ai/lib/translation/category_localizer.rb
@@ -7,37 +7,40 @@ module DiscourseAi
category,
target_locale = I18n.locale,
short_text_llm_model: nil,
- post_raw_llm_model: nil
+ post_raw_llm_model: nil,
+ fields: nil
)
return if category.blank? || target_locale.blank?
target_locale = target_locale.to_s.sub("-", "_")
-
- translated_name =
- ShortTextTranslator.new(
- text: category.name,
- target_locale:,
- llm_model: short_text_llm_model,
- ).translate
- translated_description =
- if category.description_excerpt.present?
- PostRawTranslator.new(
- text: category.description_excerpt,
- target_locale:,
- llm_model: post_raw_llm_model,
- ).translate
- else
- ""
- end
-
localization =
CategoryLocalization.find_or_initialize_by(
category_id: category.id,
locale: target_locale,
)
+ fields = %w[name description] if fields.blank? || localization.new_record?
+
+ if fields.include?("name")
+ localization.name =
+ ShortTextTranslator.new(
+ text: category.name,
+ target_locale:,
+ llm_model: short_text_llm_model,
+ ).translate
+ end
- localization.name = translated_name
- localization.description = translated_description
+ if fields.include?("description")
+ localization.description =
+ if category.description_excerpt.present?
+ PostRawTranslator.new(
+ text: category.description_excerpt,
+ target_locale:,
+ llm_model: post_raw_llm_model,
+ ).translate
+ else
+ ""
+ end
+ end
localization.save!
localization
end
diff --git a/plugins/discourse-ai/lib/translation/entry_point.rb b/plugins/discourse-ai/lib/translation/entry_point.rb
index caa73beebde..5a3339b5043 100644
--- a/plugins/discourse-ai/lib/translation/entry_point.rb
+++ b/plugins/discourse-ai/lib/translation/entry_point.rb
@@ -32,6 +32,39 @@ module DiscourseAi
end
end
+ plugin.on(:category_updated) do |category|
+ next unless category.is_a?(Category)
+
+ changed_fields =
+ %w[name description].select { |field| category.previous_changes.key?(field) }
+
+ if DiscourseAi::Translation.enabled? && changed_fields.present?
+ Jobs.enqueue(
+ :localize_categories,
+ category_id: category.id,
+ fields: changed_fields,
+ limit: 1,
+ force: true,
+ )
+ end
+ end
+
+ plugin.on(:tag_updated) do |tag|
+ next unless tag.is_a?(Tag)
+
+ changed_fields = %w[name description].select { |field| tag.previous_changes.key?(field) }
+
+ if DiscourseAi::Translation.enabled? && changed_fields.present?
+ Jobs.enqueue(
+ :localize_tags,
+ tag_id: tag.id,
+ fields: changed_fields,
+ limit: 1,
+ force: true,
+ )
+ end
+ end
+
plugin.on(:site_setting_changed) do |name, _old_value, new_value|
if name == :ai_translation_enabled && new_value &&
SiteSetting.ai_translation_backfill_start_date.blank?
diff --git a/plugins/discourse-ai/lib/translation/tag_localizer.rb b/plugins/discourse-ai/lib/translation/tag_localizer.rb
index ec23082c6b7..cf70d2dd7ec 100644
--- a/plugins/discourse-ai/lib/translation/tag_localizer.rb
+++ b/plugins/discourse-ai/lib/translation/tag_localizer.rb
@@ -3,31 +3,32 @@
module DiscourseAi
module Translation
class TagLocalizer
- def self.localize(tag, target_locale = I18n.locale, short_text_llm_model: nil)
+ def self.localize(tag, target_locale = I18n.locale, short_text_llm_model: nil, fields: nil)
return if tag.blank? || target_locale.blank?
target_locale = target_locale.to_s.sub("-", "_")
+ localization = TagLocalization.find_or_initialize_by(tag_id: tag.id, locale: target_locale)
+ fields = %w[name description] if fields.blank? || localization.new_record?
- translated_name =
- ShortTextTranslator.new(
- text: tag.name,
- target_locale:,
- llm_model: short_text_llm_model,
- ).translate
-
- translated_description =
- if tag.description.present?
+ if fields.include?("name")
+ localization.name =
ShortTextTranslator.new(
- text: tag.description,
+ text: tag.name,
target_locale:,
llm_model: short_text_llm_model,
).translate
- end
-
- localization = TagLocalization.find_or_initialize_by(tag_id: tag.id, locale: target_locale)
+ end
- localization.name = translated_name
- localization.description = translated_description
+ if fields.include?("description")
+ localization.description =
+ if tag.description.present?
+ ShortTextTranslator.new(
+ text: tag.description,
+ target_locale:,
+ llm_model: short_text_llm_model,
+ ).translate
+ end
+ end
localization.save!
localization
end
diff --git a/plugins/discourse-ai/spec/jobs/regular/localize_categories_spec.rb b/plugins/discourse-ai/spec/jobs/regular/localize_categories_spec.rb
index 2ba536433f3..4888994f1b6 100644
--- a/plugins/discourse-ai/spec/jobs/regular/localize_categories_spec.rb
+++ b/plugins/discourse-ai/spec/jobs/regular/localize_categories_spec.rb
@@ -133,6 +133,38 @@ describe Jobs::LocalizeCategories do
job.execute({ limit: 10 })
end
+ it "retranslates existing localizations for a requested category" do
+ localize_all_categories("pt_BR", "zh_CN")
+ category = Fabricate(:category, locale: "en", description: "Source description")
+ localizations =
+ %w[pt_BR zh_CN].map do |locale|
+ Fabricate(
+ :category_localization,
+ category:,
+ locale:,
+ name: "Old name",
+ description: "Old description",
+ )
+ end
+
+ short_text_translator = instance_double(DiscourseAi::Translation::ShortTextTranslator)
+ allow(DiscourseAi::Translation::ShortTextTranslator).to receive(:new).and_return(
+ short_text_translator,
+ )
+ allow(short_text_translator).to receive(:translate).and_return("New name")
+ post_raw_translator = instance_double(DiscourseAi::Translation::PostRawTranslator)
+ allow(DiscourseAi::Translation::PostRawTranslator).to receive(:new).and_return(
+ post_raw_translator,
+ )
+ allow(post_raw_translator).to receive(:translate).and_return("New description")
+
+ job.execute({ limit: 1, category_id: category.id, fields: ["description"], force: true })
+
+ expect(
+ CategoryLocalization.where(id: localizations.map(&:id)).pluck(:name, :description),
+ ).to contain_exactly(["Old name", "New description"], ["Old name", "New description"])
+ end
+
it "handles translation errors gracefully" do
localize_all_categories("pt", "zh_CN")
diff --git a/plugins/discourse-ai/spec/jobs/regular/localize_tags_spec.rb b/plugins/discourse-ai/spec/jobs/regular/localize_tags_spec.rb
index a5f44034e20..2815deb7fad 100644
--- a/plugins/discourse-ai/spec/jobs/regular/localize_tags_spec.rb
+++ b/plugins/discourse-ai/spec/jobs/regular/localize_tags_spec.rb
@@ -76,10 +76,10 @@ describe Jobs::LocalizeTags do
.with(tag, "zh_CN", has_entries(short_text_llm_model: anything))
.once
- job.execute({ limit: 10 })
+ job.execute({ limit: 1 })
end
- it "limits the number of localizations" do
+ it "limits the number of tags" do
SiteSetting.content_localization_supported_locales = "pt"
6.times { Fabricate(:tag, locale: "en") }
@@ -108,6 +108,55 @@ describe Jobs::LocalizeTags do
job.execute({ limit: 10 })
end
+ it "retranslates existing localizations for a requested tag" do
+ localize_all_tags("pt_BR", "zh_CN")
+ tag = Fabricate(:tag, locale: "en", description: "Source description")
+ localizations =
+ %w[pt_BR zh_CN].map do |locale|
+ Fabricate(
+ :tag_localization,
+ tag:,
+ locale:,
+ name: "old-name",
+ description: "Old description",
+ )
+ end
+
+ allow(DiscourseAi::Translation::ShortTextTranslator).to receive(:new) do |text:, **|
+ translated = text == tag.name ? "new-name" : "New description"
+ instance_double(DiscourseAi::Translation::ShortTextTranslator, translate: translated)
+ end
+
+ job.execute({ limit: 1, tag_id: tag.id, fields: ["description"], force: true })
+
+ expect(
+ TagLocalization.where(id: localizations.map(&:id)).pluck(:name, :description),
+ ).to contain_exactly(["old-name", "New description"], ["old-name", "New description"])
+ end
+
+ it "clears existing localized descriptions when the source description is blank" do
+ tag = Fabricate(:tag, locale: "en", description: "")
+ localizations =
+ %w[pt_BR zh_CN].map do |locale|
+ Fabricate(
+ :tag_localization,
+ tag:,
+ locale:,
+ name: "old-name",
+ description: "Old description",
+ )
+ end
+
+ allow(DiscourseAi::Translation::ShortTextTranslator).to receive(:new)
+
+ job.execute({ limit: 1, tag_id: tag.id, fields: ["description"], force: true })
+
+ expect(
+ TagLocalization.where(id: localizations.map(&:id)).pluck(:name, :description),
+ ).to contain_exactly(["old-name", nil], ["old-name", nil])
+ expect(DiscourseAi::Translation::ShortTextTranslator).not_to have_received(:new)
+ end
+
it "handles translation errors gracefully" do
tag1 = Fabricate(:tag, name: "first-tag", locale: "en")
diff --git a/plugins/discourse-ai/spec/lib/translation/entry_point_spec.rb b/plugins/discourse-ai/spec/lib/translation/entry_point_spec.rb
index bbaaa0f129e..6664283c42c 100644
--- a/plugins/discourse-ai/spec/lib/translation/entry_point_spec.rb
+++ b/plugins/discourse-ai/spec/lib/translation/entry_point_spec.rb
@@ -166,4 +166,90 @@ describe DiscourseAi::Translation::EntryPoint do
expect(job_enqueued?(job: :detect_translate_topic)).to eq false
end
end
+
+ describe "upon category updated" do
+ it "enqueues category retranslation when the description changes" do
+ SiteSetting.content_localization_supported_locales = "de|fr"
+ category = Fabricate(:category_with_definition, locale: "en")
+
+ category.update!(description: "An updated category description")
+
+ expect_job_enqueued(
+ job: :localize_categories,
+ args: {
+ category_id: category.id,
+ fields: ["description"],
+ limit: 1,
+ force: true,
+ },
+ )
+ end
+
+ it "enqueues category retranslation when the name changes" do
+ category = Fabricate(:category, locale: "en")
+
+ category.update!(name: "A new category name")
+
+ expect_job_enqueued(
+ job: :localize_categories,
+ args: {
+ category_id: category.id,
+ fields: ["name"],
+ limit: 1,
+ force: true,
+ },
+ )
+ end
+
+ it "does not enqueue category retranslation when another field changes" do
+ category = Fabricate(:category, locale: "en")
+
+ category.update!(color: "FF0000")
+
+ expect(job_enqueued?(job: :localize_categories)).to eq(false)
+ end
+ end
+
+ describe "upon tag updated" do
+ it "enqueues tag retranslation when the description changes" do
+ SiteSetting.content_localization_supported_locales = "de|fr"
+ tag = Fabricate(:tag, locale: "en")
+
+ tag.update!(description: "An updated tag description")
+
+ expect_job_enqueued(
+ job: :localize_tags,
+ args: {
+ tag_id: tag.id,
+ fields: ["description"],
+ limit: 1,
+ force: true,
+ },
+ )
+ end
+
+ it "enqueues tag retranslation when the name changes" do
+ tag = Fabricate(:tag, locale: "en")
+
+ tag.update!(name: "a-new-tag-name")
+
+ expect_job_enqueued(
+ job: :localize_tags,
+ args: {
+ tag_id: tag.id,
+ fields: ["name"],
+ limit: 1,
+ force: true,
+ },
+ )
+ end
+
+ it "does not enqueue tag retranslation when another field changes" do
+ tag = Fabricate(:tag, locale: "en")
+
+ tag.update!(staff_topic_count: 1)
+
+ expect(job_enqueued?(job: :localize_tags)).to eq(false)
+ end
+ end
end