A platform for community discussion. Free, open, simple.
PERF: Run candidate query once instead of per-locale in LocalizePosts (#39442)
Natalie Tay3e0cdc2afe32
diff --git a/plugins/discourse-ai/app/jobs/regular/localize_posts.rb b/plugins/discourse-ai/app/jobs/regular/localize_posts.rbindex e0c5958c901..813e895ac9f 100644--- a/plugins/discourse-ai/app/jobs/regular/localize_posts.rb+++ b/plugins/discourse-ai/app/jobs/regular/localize_posts.rb@@ -27,40 +27,61 @@ module Jobslocales = DiscourseAi::Translation.localesreturn if locales.blank?- locales.each do |locale|- base_locale = locale.split("_").first-- posts =- DiscourseAi::Translation::PostCandidates- .get- .joins(- "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale LIKE '#{base_locale}%'",- )- .where.not(locale: nil)- .where("posts.locale NOT LIKE '#{base_locale}%'")- .where("pl.id IS NULL")- .order(updated_at: :desc)- .offset(offset)- .limit(limit)+ locale_pairs = locales.map { |l| [l.split("_").first, l] }++ posts =+ DiscourseAi::Translation::PostCandidates+ .get+ .where.not(locale: nil)+ .order(updated_at: :desc)+ .offset(offset)+ .limit(limit)++ return if posts.empty?++ existing =+ PostLocalization.where(post_id: posts.map(&:id)).pluck(:post_id, :locale).group_by(&:first)++ existing_base_locales =+ existing.transform_values { |pairs| pairs.map { |_, loc| loc.split("_").first }.to_set }- next if posts.empty?+ budget = limit+ translated_counts = Hash.new(0)++ posts.each do |post|+ break if budget <= 0+ post_base = post.locale.split("_").first++ locale_pairs.each do |base_locale, target_locale|+ break if budget <= 0+ next if post_base == base_locale+ next if existing_base_locales.dig(post.id)&.include?(base_locale)- posts.each do |post|Discourse.redis.expire(REDIS_KEY, 15.minutes.to_i)- next unless DiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, locale)+ unless DiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, target_locale)+ next+ endbegin- DiscourseAi::Translation::PostLocalizer.localize(post, locale, llm_model: llm_model)+ DiscourseAi::Translation::PostLocalizer.localize(+ post,+ target_locale,+ llm_model: llm_model,+ )+ translated_counts[target_locale] += 1+ budget -= 1rescue FinalDestination::SSRFDetector::LookupFailedError# do nothing, there are too many sporadic lookup failuresrescue => eDiscourseAi::Translation::VerboseLogger.log(- "Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",+ "Failed to translate post #{post.id} to #{target_locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",)endend+ end- DiscourseAi::Translation::VerboseLogger.log("Translated #{posts.size} posts to #{locale}")+ translated_counts.each do |target_locale, count|+ DiscourseAi::Translation::VerboseLogger.log("Translated #{count} posts to #{target_locale}")endensureremaining = Discourse.redis.decr(REDIS_KEY)diff --git a/plugins/discourse-ai/db/migrate/20260422062938_add_localization_candidate_index_to_posts.rb b/plugins/discourse-ai/db/migrate/20260422062938_add_localization_candidate_index_to_posts.rbnew file mode 100644index 00000000000..b54fb05cce3--- /dev/null+++ b/plugins/discourse-ai/db/migrate/20260422062938_add_localization_candidate_index_to_posts.rb@@ -0,0 +1,20 @@+# frozen_string_literal: true++class AddLocalizationCandidateIndexToPosts < ActiveRecord::Migration[8.0]+ disable_ddl_transaction!++ def change+ remove_index :posts,+ name: "index_posts_on_updated_at_for_localization",+ algorithm: :concurrently,+ if_exists: true+ add_index :posts,+ :updated_at,+ order: {+ updated_at: :desc,+ },+ where: "deleted_at IS NULL AND user_id > 0 AND locale IS NOT NULL",+ name: "index_posts_on_updated_at_for_localization",+ algorithm: :concurrently+ end+end