Skip to content

A platform for community discussion. Free, open, simple.

PERF: Add index and optimisations for localization jobs (#39454)

Natalie Tayca634e256003
diff --git a/plugins/discourse-ai/app/jobs/regular/detect_translate_post.rb b/plugins/discourse-ai/app/jobs/regular/detect_translate_post.rb
index b6933871226..9f824540b1f 100644
--- a/plugins/discourse-ai/app/jobs/regular/detect_translate_post.rb
+++ b/plugins/discourse-ai/app/jobs/regular/detect_translate_post.rb
@@ -57,9 +57,17 @@ module Jobs
locales = DiscourseAi::Translation.locales
return if locales.blank?
+ existing_base_locales =
+ PostLocalization
+ .where(post_id: post.id)
+ .pluck(:locale)
+ .map { |l| l.split("_").first }
+ .to_set
+
locales.each do |locale|
next if LocaleNormalizer.is_same?(locale, detected_locale)
- exists = post.localizations.matching_locale(locale).exists?
+ base_locale = locale.split("_").first
+ exists = existing_base_locales.include?(base_locale)
has_quota = DiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, locale)
next if !force && exists && !has_quota
diff --git a/plugins/discourse-ai/app/jobs/regular/detect_translate_topic.rb b/plugins/discourse-ai/app/jobs/regular/detect_translate_topic.rb
index 48b727c67fc..47db78227c6 100644
--- a/plugins/discourse-ai/app/jobs/regular/detect_translate_topic.rb
+++ b/plugins/discourse-ai/app/jobs/regular/detect_translate_topic.rb
@@ -53,9 +53,17 @@ module Jobs
locales = DiscourseAi::Translation.locales
return if locales.blank?
+ existing_base_locales =
+ TopicLocalization
+ .where(topic_id: topic.id)
+ .pluck(:locale)
+ .map { |l| l.split("_").first }
+ .to_set
+
locales.each do |locale|
next if LocaleNormalizer.is_same?(locale, detected_locale)
- exists = topic.localizations.matching_locale(locale).exists?
+ base_locale = locale.split("_").first
+ exists = existing_base_locales.include?(base_locale)
has_quota = DiscourseAi::Translation::TopicLocalizer.has_relocalize_quota?(topic, locale)
next if !force && exists && !has_quota
diff --git a/plugins/discourse-ai/app/jobs/regular/localize_topics.rb b/plugins/discourse-ai/app/jobs/regular/localize_topics.rb
index bec97e8fdfa..630ef3fd2bf 100644
--- a/plugins/discourse-ai/app/jobs/regular/localize_topics.rb
+++ b/plugins/discourse-ai/app/jobs/regular/localize_topics.rb
@@ -27,40 +27,61 @@ module Jobs
locales = DiscourseAi::Translation.locales
return if locales.blank?
- locales.each do |locale|
- base_locale = locale.split("_").first
- topics =
- DiscourseAi::Translation::TopicCandidates
- .get
- .joins(
- "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale LIKE '#{base_locale}%'",
- )
- .where.not(locale: nil)
- .where("topics.locale NOT LIKE '#{base_locale}%'")
- .where("tl.id IS NULL")
- .order(updated_at: :desc)
- .limit(limit)
+ locale_pairs = locales.map { |l| [l.split("_").first, l] }
+
+ topics =
+ DiscourseAi::Translation::TopicCandidates
+ .get
+ .where.not(locale: nil)
+ .order(updated_at: :desc)
+ .limit(limit)
+
+ return if topics.empty?
+
+ existing =
+ TopicLocalization
+ .where(topic_id: topics.map(&:id))
+ .pluck(:topic_id, :locale)
+ .group_by(&:first)
+
+ existing_base_locales =
+ existing.transform_values { |pairs| pairs.map { |_, loc| loc.split("_").first }.to_set }
- next if topics.empty?
+ budget = limit
+ translated_counts = Hash.new(0)
+
+ topics.each do |topic|
+ break if budget <= 0
+ topic_base = topic.locale.split("_").first
+
+ locale_pairs.each do |base_locale, target_locale|
+ break if budget <= 0
+ next if topic_base == base_locale
+ next if existing_base_locales.dig(topic.id)&.include?(base_locale)
- topics.each do |topic|
begin
DiscourseAi::Translation::TopicLocalizer.localize(
topic,
- locale,
+ target_locale,
topic_title_llm_model:,
post_raw_llm_model:,
)
+ translated_counts[target_locale] += 1
+ budget -= 1
rescue FinalDestination::SSRFDetector::LookupFailedError
# do nothing, there are too many sporadic lookup failures
rescue => e
DiscourseAi::Translation::VerboseLogger.log(
- "Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+ "Failed to translate topic #{topic.id} to #{target_locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
)
end
end
+ end
- DiscourseAi::Translation::VerboseLogger.log("Translated #{topics.size} topics to #{locale}")
+ translated_counts.each do |target_locale, count|
+ DiscourseAi::Translation::VerboseLogger.log(
+ "Translated #{count} topics to #{target_locale}",
+ )
end
end
diff --git a/plugins/discourse-ai/db/migrate/20260422102523_add_locale_detection_candidate_index_to_posts.rb b/plugins/discourse-ai/db/migrate/20260422102523_add_locale_detection_candidate_index_to_posts.rb
new file mode 100644
index 00000000000..5ac73d5bb81
--- /dev/null
+++ b/plugins/discourse-ai/db/migrate/20260422102523_add_locale_detection_candidate_index_to_posts.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class AddLocaleDetectionCandidateIndexToPosts < ActiveRecord::Migration[8.0]
+ disable_ddl_transaction!
+
+ def change
+ remove_index :posts,
+ name: "index_posts_on_updated_at_for_locale_detection",
+ algorithm: :concurrently,
+ if_exists: true
+ add_index :posts,
+ :updated_at,
+ order: {
+ updated_at: :desc,
+ },
+ where: "deleted_at IS NULL AND user_id > 0 AND locale IS NULL",
+ name: "index_posts_on_updated_at_for_locale_detection",
+ algorithm: :concurrently
+ end
+end
diff --git a/plugins/discourse-ai/db/migrate/20260422130653_add_localization_indexes_to_topics.rb b/plugins/discourse-ai/db/migrate/20260422130653_add_localization_indexes_to_topics.rb
new file mode 100644
index 00000000000..4a5ca733c94
--- /dev/null
+++ b/plugins/discourse-ai/db/migrate/20260422130653_add_localization_indexes_to_topics.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class AddLocalizationIndexesToTopics < ActiveRecord::Migration[8.0]
+ disable_ddl_transaction!
+
+ def change
+ remove_index :topics,
+ name: "index_topics_on_updated_at_for_locale_detection",
+ algorithm: :concurrently,
+ if_exists: true
+ add_index :topics,
+ :updated_at,
+ order: {
+ updated_at: :desc,
+ },
+ where: "deleted_at IS NULL AND user_id > 0 AND locale IS NULL",
+ name: "index_topics_on_updated_at_for_locale_detection",
+ algorithm: :concurrently
+
+ remove_index :topics,
+ name: "index_topics_on_updated_at_for_localization",
+ algorithm: :concurrently,
+ if_exists: true
+ add_index :topics,
+ :updated_at,
+ order: {
+ updated_at: :desc,
+ },
+ where: "deleted_at IS NULL AND user_id > 0 AND locale IS NOT NULL",
+ name: "index_topics_on_updated_at_for_localization",
+ algorithm: :concurrently
+ end
+end
diff --git a/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb b/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb
index 492bd6904af..80867c9f523 100644
--- a/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb
+++ b/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb
@@ -187,6 +187,22 @@ describe Jobs::LocalizeTopics do
end
end
+ it "enforces a budget across all locales" do
+ SiteSetting.content_localization_supported_locales = "en|ja|de"
+ topic.update!(locale: "es")
+
+ call_count = 0
+ DiscourseAi::Translation::TopicLocalizer
+ .stubs(:localize)
+ .with do
+ call_count += 1
+ true
+ end
+
+ job.execute({ limit: 2 })
+ expect(call_count).to eq(2)
+ end
+
describe "with target categories" do
fab!(:target_category, :category)
fab!(:non_target_category, :category)