A platform for community discussion. Free, open, simple.
PERF: Add index and optimisations for localization jobs (#39454)
Natalie Tayca634e256003
diff --git a/plugins/discourse-ai/app/jobs/regular/detect_translate_post.rb b/plugins/discourse-ai/app/jobs/regular/detect_translate_post.rbindex b6933871226..9f824540b1f 100644--- a/plugins/discourse-ai/app/jobs/regular/detect_translate_post.rb+++ b/plugins/discourse-ai/app/jobs/regular/detect_translate_post.rb@@ -57,9 +57,17 @@ module Jobslocales = DiscourseAi::Translation.localesreturn if locales.blank?+ existing_base_locales =+ PostLocalization+ .where(post_id: post.id)+ .pluck(:locale)+ .map { |l| l.split("_").first }+ .to_set+locales.each do |locale|next if LocaleNormalizer.is_same?(locale, detected_locale)- exists = post.localizations.matching_locale(locale).exists?+ base_locale = locale.split("_").first+ exists = existing_base_locales.include?(base_locale)has_quota = DiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, locale)next if !force && exists && !has_quotadiff --git a/plugins/discourse-ai/app/jobs/regular/detect_translate_topic.rb b/plugins/discourse-ai/app/jobs/regular/detect_translate_topic.rbindex 48b727c67fc..47db78227c6 100644--- a/plugins/discourse-ai/app/jobs/regular/detect_translate_topic.rb+++ b/plugins/discourse-ai/app/jobs/regular/detect_translate_topic.rb@@ -53,9 +53,17 @@ module Jobslocales = DiscourseAi::Translation.localesreturn if locales.blank?+ existing_base_locales =+ TopicLocalization+ .where(topic_id: topic.id)+ .pluck(:locale)+ .map { |l| l.split("_").first }+ .to_set+locales.each do |locale|next if LocaleNormalizer.is_same?(locale, detected_locale)- exists = topic.localizations.matching_locale(locale).exists?+ base_locale = locale.split("_").first+ exists = existing_base_locales.include?(base_locale)has_quota = DiscourseAi::Translation::TopicLocalizer.has_relocalize_quota?(topic, locale)next if !force && exists && !has_quotadiff --git a/plugins/discourse-ai/app/jobs/regular/localize_topics.rb b/plugins/discourse-ai/app/jobs/regular/localize_topics.rbindex bec97e8fdfa..630ef3fd2bf 100644--- a/plugins/discourse-ai/app/jobs/regular/localize_topics.rb+++ b/plugins/discourse-ai/app/jobs/regular/localize_topics.rb@@ -27,40 +27,61 @@ module Jobslocales = DiscourseAi::Translation.localesreturn if locales.blank?- locales.each do |locale|- base_locale = locale.split("_").first- topics =- DiscourseAi::Translation::TopicCandidates- .get- .joins(- "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale LIKE '#{base_locale}%'",- )- .where.not(locale: nil)- .where("topics.locale NOT LIKE '#{base_locale}%'")- .where("tl.id IS NULL")- .order(updated_at: :desc)- .limit(limit)+ locale_pairs = locales.map { |l| [l.split("_").first, l] }++ topics =+ DiscourseAi::Translation::TopicCandidates+ .get+ .where.not(locale: nil)+ .order(updated_at: :desc)+ .limit(limit)++ return if topics.empty?++ existing =+ TopicLocalization+ .where(topic_id: topics.map(&:id))+ .pluck(:topic_id, :locale)+ .group_by(&:first)++ existing_base_locales =+ existing.transform_values { |pairs| pairs.map { |_, loc| loc.split("_").first }.to_set }- next if topics.empty?+ budget = limit+ translated_counts = Hash.new(0)++ topics.each do |topic|+ break if budget <= 0+ topic_base = topic.locale.split("_").first++ locale_pairs.each do |base_locale, target_locale|+ break if budget <= 0+ next if topic_base == base_locale+ next if existing_base_locales.dig(topic.id)&.include?(base_locale)- topics.each do |topic|beginDiscourseAi::Translation::TopicLocalizer.localize(topic,- locale,+ target_locale,topic_title_llm_model:,post_raw_llm_model:,)+ translated_counts[target_locale] += 1+ budget -= 1rescue FinalDestination::SSRFDetector::LookupFailedError# do nothing, there are too many sporadic lookup failuresrescue => eDiscourseAi::Translation::VerboseLogger.log(- "Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",+ "Failed to translate topic #{topic.id} to #{target_locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",)endend+ end- DiscourseAi::Translation::VerboseLogger.log("Translated #{topics.size} topics to #{locale}")+ translated_counts.each do |target_locale, count|+ DiscourseAi::Translation::VerboseLogger.log(+ "Translated #{count} topics to #{target_locale}",+ )endenddiff --git a/plugins/discourse-ai/db/migrate/20260422102523_add_locale_detection_candidate_index_to_posts.rb b/plugins/discourse-ai/db/migrate/20260422102523_add_locale_detection_candidate_index_to_posts.rbnew file mode 100644index 00000000000..5ac73d5bb81--- /dev/null+++ b/plugins/discourse-ai/db/migrate/20260422102523_add_locale_detection_candidate_index_to_posts.rb@@ -0,0 +1,20 @@+# frozen_string_literal: true++class AddLocaleDetectionCandidateIndexToPosts < ActiveRecord::Migration[8.0]+ disable_ddl_transaction!++ def change+ remove_index :posts,+ name: "index_posts_on_updated_at_for_locale_detection",+ algorithm: :concurrently,+ if_exists: true+ add_index :posts,+ :updated_at,+ order: {+ updated_at: :desc,+ },+ where: "deleted_at IS NULL AND user_id > 0 AND locale IS NULL",+ name: "index_posts_on_updated_at_for_locale_detection",+ algorithm: :concurrently+ end+enddiff --git a/plugins/discourse-ai/db/migrate/20260422130653_add_localization_indexes_to_topics.rb b/plugins/discourse-ai/db/migrate/20260422130653_add_localization_indexes_to_topics.rbnew file mode 100644index 00000000000..4a5ca733c94--- /dev/null+++ b/plugins/discourse-ai/db/migrate/20260422130653_add_localization_indexes_to_topics.rb@@ -0,0 +1,33 @@+# frozen_string_literal: true++class AddLocalizationIndexesToTopics < ActiveRecord::Migration[8.0]+ disable_ddl_transaction!++ def change+ remove_index :topics,+ name: "index_topics_on_updated_at_for_locale_detection",+ algorithm: :concurrently,+ if_exists: true+ add_index :topics,+ :updated_at,+ order: {+ updated_at: :desc,+ },+ where: "deleted_at IS NULL AND user_id > 0 AND locale IS NULL",+ name: "index_topics_on_updated_at_for_locale_detection",+ algorithm: :concurrently++ remove_index :topics,+ name: "index_topics_on_updated_at_for_localization",+ algorithm: :concurrently,+ if_exists: true+ add_index :topics,+ :updated_at,+ order: {+ updated_at: :desc,+ },+ where: "deleted_at IS NULL AND user_id > 0 AND locale IS NOT NULL",+ name: "index_topics_on_updated_at_for_localization",+ algorithm: :concurrently+ end+enddiff --git a/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb b/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rbindex 492bd6904af..80867c9f523 100644--- a/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb+++ b/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb@@ -187,6 +187,22 @@ describe Jobs::LocalizeTopics doendend+ it "enforces a budget across all locales" do+ SiteSetting.content_localization_supported_locales = "en|ja|de"+ topic.update!(locale: "es")++ call_count = 0+ DiscourseAi::Translation::TopicLocalizer+ .stubs(:localize)+ .with do+ call_count += 1+ true+ end++ job.execute({ limit: 2 })+ expect(call_count).to eq(2)+ end+describe "with target categories" dofab!(:target_category, :category)fab!(:non_target_category, :category)