Skip to content

A platform for community discussion. Free, open, simple.

PERF: Run candidate query once instead of per-locale in LocalizePosts (#39442)

Natalie Tay3e0cdc2afe32
diff --git a/plugins/discourse-ai/app/jobs/regular/localize_posts.rb b/plugins/discourse-ai/app/jobs/regular/localize_posts.rb
index e0c5958c901..813e895ac9f 100644
--- a/plugins/discourse-ai/app/jobs/regular/localize_posts.rb
+++ b/plugins/discourse-ai/app/jobs/regular/localize_posts.rb
@@ -27,40 +27,61 @@ module Jobs
locales = DiscourseAi::Translation.locales
return if locales.blank?
- locales.each do |locale|
- base_locale = locale.split("_").first
-
- posts =
- DiscourseAi::Translation::PostCandidates
- .get
- .joins(
- "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale LIKE '#{base_locale}%'",
- )
- .where.not(locale: nil)
- .where("posts.locale NOT LIKE '#{base_locale}%'")
- .where("pl.id IS NULL")
- .order(updated_at: :desc)
- .offset(offset)
- .limit(limit)
+ locale_pairs = locales.map { |l| [l.split("_").first, l] }
+
+ posts =
+ DiscourseAi::Translation::PostCandidates
+ .get
+ .where.not(locale: nil)
+ .order(updated_at: :desc)
+ .offset(offset)
+ .limit(limit)
+
+ return if posts.empty?
+
+ existing =
+ PostLocalization.where(post_id: posts.map(&:id)).pluck(:post_id, :locale).group_by(&:first)
+
+ existing_base_locales =
+ existing.transform_values { |pairs| pairs.map { |_, loc| loc.split("_").first }.to_set }
- next if posts.empty?
+ budget = limit
+ translated_counts = Hash.new(0)
+
+ posts.each do |post|
+ break if budget <= 0
+ post_base = post.locale.split("_").first
+
+ locale_pairs.each do |base_locale, target_locale|
+ break if budget <= 0
+ next if post_base == base_locale
+ next if existing_base_locales.dig(post.id)&.include?(base_locale)
- posts.each do |post|
Discourse.redis.expire(REDIS_KEY, 15.minutes.to_i)
- next unless DiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, locale)
+ unless DiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, target_locale)
+ next
+ end
begin
- DiscourseAi::Translation::PostLocalizer.localize(post, locale, llm_model: llm_model)
+ DiscourseAi::Translation::PostLocalizer.localize(
+ post,
+ target_locale,
+ llm_model: llm_model,
+ )
+ translated_counts[target_locale] += 1
+ budget -= 1
rescue FinalDestination::SSRFDetector::LookupFailedError
# do nothing, there are too many sporadic lookup failures
rescue => e
DiscourseAi::Translation::VerboseLogger.log(
- "Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+ "Failed to translate post #{post.id} to #{target_locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
)
end
end
+ end
- DiscourseAi::Translation::VerboseLogger.log("Translated #{posts.size} posts to #{locale}")
+ translated_counts.each do |target_locale, count|
+ DiscourseAi::Translation::VerboseLogger.log("Translated #{count} posts to #{target_locale}")
end
ensure
remaining = Discourse.redis.decr(REDIS_KEY)
diff --git a/plugins/discourse-ai/db/migrate/20260422062938_add_localization_candidate_index_to_posts.rb b/plugins/discourse-ai/db/migrate/20260422062938_add_localization_candidate_index_to_posts.rb
new file mode 100644
index 00000000000..b54fb05cce3
--- /dev/null
+++ b/plugins/discourse-ai/db/migrate/20260422062938_add_localization_candidate_index_to_posts.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class AddLocalizationCandidateIndexToPosts < ActiveRecord::Migration[8.0]
+ disable_ddl_transaction!
+
+ def change
+ remove_index :posts,
+ name: "index_posts_on_updated_at_for_localization",
+ algorithm: :concurrently,
+ if_exists: true
+ add_index :posts,
+ :updated_at,
+ order: {
+ updated_at: :desc,
+ },
+ where: "deleted_at IS NULL AND user_id > 0 AND locale IS NOT NULL",
+ name: "index_posts_on_updated_at_for_localization",
+ algorithm: :concurrently
+ end
+end