A platform for community discussion. Free, open, simple.
PERF: Fix localization backfills to pass (id, locale) pairs to workers (#39486)
Natalie Tay915999029899
diff --git a/plugins/discourse-ai/app/jobs/regular/localize_posts.rb b/plugins/discourse-ai/app/jobs/regular/localize_posts.rbindex 813e895ac9f..c359e50ff97 100644--- a/plugins/discourse-ai/app/jobs/regular/localize_posts.rb+++ b/plugins/discourse-ai/app/jobs/regular/localize_posts.rb@@ -7,10 +7,8 @@ module JobsREDIS_KEY = "discourse-ai:localize_posts:in_progress"def execute(args)- limit = args[:limit]- raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0-- offset = args[:offset].to_i+ pairs = args[:pairs]+ raise Discourse::InvalidParameters.new(:pairs) if pairs.blank?return if !DiscourseAi::Translation.backfill_enabled?@@ -24,64 +22,39 @@ module Jobsllm_model = find_llm_model_for_agent(SiteSetting.ai_translation_post_raw_translator_agent)return if llm_model.blank?- locales = DiscourseAi::Translation.locales- return if locales.blank?-- locale_pairs = locales.map { |l| [l.split("_").first, l] }-- posts =- DiscourseAi::Translation::PostCandidates- .get- .where.not(locale: nil)- .order(updated_at: :desc)- .offset(offset)- .limit(limit)-- return if posts.empty?-- existing =- PostLocalization.where(post_id: posts.map(&:id)).pluck(:post_id, :locale).group_by(&:first)+ post_ids = pairs.map(&:first).uniq+ posts_by_id = Post.where(id: post_ids).index_by(&:id)- existing_base_locales =- existing.transform_values { |pairs| pairs.map { |_, loc| loc.split("_").first }.to_set }+ translated = 0+ pairs.each do |post_id, target_locale|+ post = posts_by_id[post_id]+ next if post.nil?- budget = limit- translated_counts = Hash.new(0)-- posts.each do |post|- break if budget <= 0- post_base = post.locale.split("_").first-- locale_pairs.each do |base_locale, target_locale|- break if budget <= 0- next if post_base == base_locale- next if existing_base_locales.dig(post.id)&.include?(base_locale)-- Discourse.redis.expire(REDIS_KEY, 15.minutes.to_i)- unless DiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, target_locale)- next- end+ Discourse.redis.expire(REDIS_KEY, 15.minutes.to_i)+ unless DiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, target_locale)+ next+ end- begin- DiscourseAi::Translation::PostLocalizer.localize(- post,- target_locale,- llm_model: llm_model,- )- translated_counts[target_locale] += 1- budget -= 1- rescue FinalDestination::SSRFDetector::LookupFailedError- # do nothing, there are too many sporadic lookup failures- rescue => e- DiscourseAi::Translation::VerboseLogger.log(- "Failed to translate post #{post.id} to #{target_locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",- )- end+ begin+ DiscourseAi::Translation::PostLocalizer.localize(+ post,+ target_locale,+ llm_model: llm_model,+ )+ translated += 1+ rescue FinalDestination::SSRFDetector::LookupFailedError+ # do nothing, there are too many sporadic lookup failures+ rescue => e+ DiscourseAi::Translation::VerboseLogger.log(+ "Failed to translate post #{post.id} to #{target_locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",+ )endend- translated_counts.each do |target_locale, count|- DiscourseAi::Translation::VerboseLogger.log("Translated #{count} posts to #{target_locale}")+ if translated > 0+ DiscourseAi::Translation::VerboseLogger.log(+ "Translated #{translated}/#{pairs.size} post localizations: #{pairs.map { |id, loc| "#{id}:#{loc}" }.join(", ")}",+ )endensureremaining = Discourse.redis.decr(REDIS_KEY)diff --git a/plugins/discourse-ai/app/jobs/regular/localize_topics.rb b/plugins/discourse-ai/app/jobs/regular/localize_topics.rbindex 630ef3fd2bf..cd522602cb4 100644--- a/plugins/discourse-ai/app/jobs/regular/localize_topics.rb+++ b/plugins/discourse-ai/app/jobs/regular/localize_topics.rb@@ -6,8 +6,8 @@ module Jobssidekiq_options retry: falsedef execute(args)- limit = args[:limit]- raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0+ pairs = args[:pairs]+ raise Discourse::InvalidParameters.new(:pairs) if pairs.blank?return if !DiscourseAi::Translation.backfill_enabled?@@ -24,63 +24,34 @@ module Jobsfind_llm_model_for_agent(SiteSetting.ai_translation_post_raw_translator_agent)return if topic_title_llm_model.blank? && post_raw_llm_model.blank?- locales = DiscourseAi::Translation.locales- return if locales.blank?-- locale_pairs = locales.map { |l| [l.split("_").first, l] }-- topics =- DiscourseAi::Translation::TopicCandidates- .get- .where.not(locale: nil)- .order(updated_at: :desc)- .limit(limit)-- return if topics.empty?-- existing =- TopicLocalization- .where(topic_id: topics.map(&:id))- .pluck(:topic_id, :locale)- .group_by(&:first)-- existing_base_locales =- existing.transform_values { |pairs| pairs.map { |_, loc| loc.split("_").first }.to_set }-- budget = limit- translated_counts = Hash.new(0)-- topics.each do |topic|- break if budget <= 0- topic_base = topic.locale.split("_").first-- locale_pairs.each do |base_locale, target_locale|- break if budget <= 0- next if topic_base == base_locale- next if existing_base_locales.dig(topic.id)&.include?(base_locale)-- begin- DiscourseAi::Translation::TopicLocalizer.localize(- topic,- target_locale,- topic_title_llm_model:,- post_raw_llm_model:,- )- translated_counts[target_locale] += 1- budget -= 1- rescue FinalDestination::SSRFDetector::LookupFailedError- # do nothing, there are too many sporadic lookup failures- rescue => e- DiscourseAi::Translation::VerboseLogger.log(- "Failed to translate topic #{topic.id} to #{target_locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",- )- end+ topic_ids = pairs.map(&:first).uniq+ topics_by_id = Topic.where(id: topic_ids).index_by(&:id)++ translated = 0+ pairs.each do |topic_id, target_locale|+ topic = topics_by_id[topic_id]+ next if topic.nil?++ begin+ DiscourseAi::Translation::TopicLocalizer.localize(+ topic,+ target_locale,+ topic_title_llm_model:,+ post_raw_llm_model:,+ )+ translated += 1+ rescue FinalDestination::SSRFDetector::LookupFailedError+ # do nothing, there are too many sporadic lookup failures+ rescue => e+ DiscourseAi::Translation::VerboseLogger.log(+ "Failed to translate topic #{topic.id} to #{target_locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",+ )endend- translated_counts.each do |target_locale, count|+ if translated > 0DiscourseAi::Translation::VerboseLogger.log(- "Translated #{count} topics to #{target_locale}",+ "Translated #{translated}/#{pairs.size} topic localizations: #{pairs.map { |id, loc| "#{id}:#{loc}" }.join(", ")}",)endenddiff --git a/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb b/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rbindex 7418d97926c..780f3bb6197 100644--- a/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb+++ b/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb@@ -25,15 +25,16 @@ module Jobslimit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervalsreturn if limit == 0+ pairs = DiscourseAi::Translation::PostCandidates.needs_localization(limit: limit)+ return if pairs.empty?+parallel_jobs = SiteSetting.ai_translation_backfill_parallel_jobs- per_job_limit = (limit.to_f / parallel_jobs).ceil+ chunks = pairs.each_slice((pairs.size.to_f / parallel_jobs).ceil).to_a# Set counter with 15-min TTL safety net (if a job crashes without decrementing)- Discourse.redis.setex(REDIS_KEY, 15.minutes.to_i, parallel_jobs)+ Discourse.redis.setex(REDIS_KEY, 15.minutes.to_i, chunks.size)- parallel_jobs.times do |i|- Jobs.enqueue(:localize_posts, limit: per_job_limit, offset: i * per_job_limit)- end+ chunks.each { |chunk| Jobs.enqueue(:localize_posts, pairs: chunk) }endprivatediff --git a/plugins/discourse-ai/app/jobs/scheduled/topic_localization_backfill.rb b/plugins/discourse-ai/app/jobs/scheduled/topic_localization_backfill.rbindex 11a04cdb487..5ee8d6ed5d2 100644--- a/plugins/discourse-ai/app/jobs/scheduled/topic_localization_backfill.rb+++ b/plugins/discourse-ai/app/jobs/scheduled/topic_localization_backfill.rb@@ -23,7 +23,10 @@ module Jobslimit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervalsreturn if limit == 0- Jobs.enqueue(:localize_topics, limit:)+ pairs = DiscourseAi::Translation::TopicCandidates.needs_localization(limit: limit)+ return if pairs.empty?++ Jobs.enqueue(:localize_topics, pairs: pairs)endprivatediff --git a/plugins/discourse-ai/lib/translation/post_candidates.rb b/plugins/discourse-ai/lib/translation/post_candidates.rbindex 092502b669e..1d4f7089c96 100644--- a/plugins/discourse-ai/lib/translation/post_candidates.rb+++ b/plugins/discourse-ai/lib/translation/post_candidates.rb@@ -11,6 +11,34 @@ module DiscourseAicompletion_all_localesend+ def self.needs_localization(limit:)+ locales = DiscourseAi::Translation.locales+ return [] if locales.blank?++ locale_map = {}+ locales.each { |l| locale_map[l.split("_").first] ||= l }++ target_locale_values = locale_map.map { |base, full| "('#{base}', '#{full}')" }.join(", ")++ base_sql = get.where.not(locale: nil).to_sql++ sql = <<~SQL+ SELECT ep.id AS post_id, target.target_locale+ FROM (#{base_sql}) ep+ JOIN (VALUES #{target_locale_values}) AS target(base_locale, target_locale)+ ON target.base_locale != split_part(ep.locale, '_', 1)+ WHERE NOT EXISTS (+ SELECT 1 FROM post_localizations pl+ WHERE pl.post_id = ep.id+ AND split_part(pl.locale, '_', 1) = target.base_locale+ )+ ORDER BY ep.updated_at DESC, target.target_locale+ LIMIT #{limit.to_i}+ SQL++ DB.query(sql).map { |r| [r.post_id, r.target_locale] }+ end+private# all posts that are eligible for translation based on site settings,diff --git a/plugins/discourse-ai/lib/translation/topic_candidates.rb b/plugins/discourse-ai/lib/translation/topic_candidates.rbindex a15a7126d2e..0bb056239f1 100644--- a/plugins/discourse-ai/lib/translation/topic_candidates.rb+++ b/plugins/discourse-ai/lib/translation/topic_candidates.rb@@ -3,6 +3,34 @@module DiscourseAimodule Translationclass TopicCandidates < BaseCandidates+ def self.needs_localization(limit:)+ locales = DiscourseAi::Translation.locales+ return [] if locales.blank?++ locale_map = {}+ locales.each { |l| locale_map[l.split("_").first] ||= l }++ target_locale_values = locale_map.map { |base, full| "('#{base}', '#{full}')" }.join(", ")++ base_sql = get.where.not(locale: nil).to_sql++ sql = <<~SQL+ SELECT et.id AS topic_id, target.target_locale+ FROM (#{base_sql}) et+ JOIN (VALUES #{target_locale_values}) AS target(base_locale, target_locale)+ ON target.base_locale != split_part(et.locale, '_', 1)+ WHERE NOT EXISTS (+ SELECT 1 FROM topic_localizations tl+ WHERE tl.topic_id = et.id+ AND split_part(tl.locale, '_', 1) = target.base_locale+ )+ ORDER BY et.updated_at DESC, target.target_locale+ LIMIT #{limit.to_i}+ SQL++ DB.query(sql).map { |r| [r.topic_id, r.target_locale] }+ end+private# all topics that are eligible for translation based on site settings,diff --git a/plugins/discourse-ai/spec/jobs/regular/localize_posts_spec.rb b/plugins/discourse-ai/spec/jobs/regular/localize_posts_spec.rbindex ef7c7413a5e..eef132819db 100644--- a/plugins/discourse-ai/spec/jobs/regular/localize_posts_spec.rb+++ b/plugins/discourse-ai/spec/jobs/regular/localize_posts_spec.rb@@ -5,69 +5,70 @@ describe Jobs::LocalizePosts dofab!(:post)- let(:locales) { %w[en ja de] }-before doassign_fake_provider_to(:ai_default_llm_model)enable_current_pluginSiteSetting.ai_translation_enabled = true- SiteSetting.content_localization_supported_locales = locales.join("|")+ SiteSetting.content_localization_supported_locales = "en|ja|de"SiteSetting.ai_translation_backfill_hourly_rate = 100SiteSetting.ai_translation_backfill_max_age_days = 100- SiteSetting.ai_translation_target_categories = post.topic.category_id.to_sendit "does nothing when translator is disabled" doSiteSetting.discourse_ai_enabled = falseDiscourseAi::Translation::PostLocalizer.expects(:localize).never- job.execute({ limit: 10 })+ job.execute({ pairs: [[post.id, "ja"]] })endit "does nothing when ai_translation_enabled is disabled" doSiteSetting.ai_translation_enabled = falseDiscourseAi::Translation::PostLocalizer.expects(:localize).never- job.execute({ limit: 10 })+ job.execute({ pairs: [[post.id, "ja"]] })endit "does nothing when no target languages are configured" doSiteSetting.content_localization_supported_locales = ""DiscourseAi::Translation::PostLocalizer.expects(:localize).never- job.execute({ limit: 10 })+ job.execute({ pairs: [[post.id, "ja"]] })endit "does nothing when ai_translation_backfill_hourly_rate is 0" doSiteSetting.ai_translation_backfill_hourly_rate = 0DiscourseAi::Translation::PostLocalizer.expects(:localize).never- job.execute({ limit: 10 })+ job.execute({ pairs: [[post.id, "ja"]] })endit "skips translation when credits are unavailable" doDiscourseAi::Translation.expects(:credits_available_for_post_localization?).returns(false)DiscourseAi::Translation::PostLocalizer.expects(:localize).never- job.execute({ limit: 10 })+ job.execute({ pairs: [[post.id, "ja"]] })end- it "does nothing when there are no posts to translate" do- Post.destroy_all+ it "skips pairs where post is not found" doDiscourseAi::Translation::PostLocalizer.expects(:localize).never- job.execute({ limit: 10 })+ job.execute({ pairs: [[-1, "ja"]] })end- it "skips bot posts" do- post.update!(locale: "es", user: Discourse.system_user)- DiscourseAi::Translation::PostLocalizer.expects(:localize).never+ it "translates each pair it receives" do+ DiscourseAi::Translation::PostLocalizer+ .expects(:localize)+ .with(post, "en", has_entries(llm_model: anything))+ .once+ DiscourseAi::Translation::PostLocalizer+ .expects(:localize)+ .with(post, "ja", has_entries(llm_model: anything))+ .once- job.execute({ limit: 10 })+ job.execute({ pairs: [[post.id, "en"], [post.id, "ja"]] })endit "handles translation errors gracefully" do- post.update(locale: "es")DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "en", has_entries(llm_model: anything))@@ -76,243 +77,43 @@ describe Jobs::LocalizePosts do.expects(:localize).with(post, "ja", has_entries(llm_model: anything)).once- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(post, "de", has_entries(llm_model: anything))- .once- expect { job.execute({ limit: 10 }) }.not_to raise_error+ expect { job.execute({ pairs: [[post.id, "en"], [post.id, "ja"]] }) }.not_to raise_errorendit "logs a summary after translation" do- post.update(locale: "es")DiscourseAi::Translation::PostLocalizer.stubs(:localize)- DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Translated 1 posts to en"))- DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Translated 1 posts to ja"))- DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Translated 1 posts to de"))-- job.execute({ limit: 10 })- end-- context "for translation scenarios" do- it "scenario 1: skips post when locale is not set" do- DiscourseAi::Translation::PostLocalizer.expects(:localize).never-- job.execute({ limit: 10 })- end-- it "scenario 2: localizes post with locale 'es' when localizations for en/ja/de do not exist" do- post = Fabricate(:post, locale: "es", topic: self.post.topic)-- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(post, "en", has_entries(llm_model: anything))- .once- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(post, "ja", has_entries(llm_model: anything))- .once- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(post, "de", has_entries(llm_model: anything))- .once-- job.execute({ limit: 10 })- end-- it "scenario 3: localizes post with locale 'en' when ja/de localization do not exist" do- post = Fabricate(:post, locale: "en", topic: self.post.topic)-- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(post, "ja", has_entries(llm_model: anything))- .once- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(post, "de", has_entries(llm_model: anything))- .once- DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "en").never-- job.execute({ limit: 10 })- end-- it "scenario 4: skips post with locale 'en' if all localizations exist" do- post = Fabricate(:post, locale: "en", topic: self.post.topic)- Fabricate(:post_localization, post: post, locale: "ja")- Fabricate(:post_localization, post: post, locale: "de")-- DiscourseAi::Translation::PostLocalizer.expects(:localize).never-- job.execute({ limit: 10 })- end-- it "scenario 5: skips posts that already have localizations in similar language variant" do- post = Fabricate(:post, locale: "en", topic: self.post.topic)- Fabricate(:post_localization, post: post, locale: "ja_JP")- Fabricate(:post_localization, post: post, locale: "de_DE")-- DiscourseAi::Translation::PostLocalizer.expects(:localize).never-- job.execute({ limit: 10 })- end+ DiscourseAi::Translation::VerboseLogger.expects(:log).with(+ includes("Translated 2/2 post localizations"),+ )- it "scenario 6: skips posts with variant 'en_GB' when localizations for ja/de exist" do- post = Fabricate(:post, locale: "en_GB", topic: self.post.topic)- Fabricate(:post_localization, post: post, locale: "ja_JP")- Fabricate(:post_localization, post: post, locale: "de_DE")-- DiscourseAi::Translation::PostLocalizer.expects(:localize).never-- job.execute({ limit: 10 })- end+ job.execute({ pairs: [[post.id, "en"], [post.id, "ja"]] })endcontext "when relocalize quota is exhausted" do- before { post.update(locale: "es") }-it "skips localization for posts that have exceeded quota for a specific locale" doDiscourseAi::Translation::PostLocalizer::MAX_QUOTA_PER_DAY.times doDiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, "en")end- DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "en").never- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(post, "ja", has_entries(llm_model: anything))- .once- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(post, "de", has_entries(llm_model: anything))- .once-- job.execute({ limit: 10 })- end- end-- describe "with target categories" do- fab!(:target_category, :category)- fab!(:non_target_category, :category)- fab!(:target_topic) { Fabricate(:topic, category: target_category) }- fab!(:non_target_topic) { Fabricate(:topic, category: non_target_category) }- fab!(:target_post) { Fabricate(:post, topic: target_topic, locale: "es") }- fab!(:non_target_post) { Fabricate(:post, topic: non_target_topic, locale: "es") }-- fab!(:personal_pm_topic, :private_message_topic)- fab!(:personal_pm_post) { Fabricate(:post, topic: personal_pm_topic, locale: "es") }-- fab!(:group)- fab!(:group_pm_topic) { Fabricate(:group_private_message_topic, recipient_group: group) }- fab!(:group_pm_post) { Fabricate(:post, topic: group_pm_topic, locale: "es") }-- before do- SiteSetting.default_locale = "ja"- SiteSetting.content_localization_supported_locales = "ja"- SiteSetting.ai_translation_target_categories = target_category.id.to_s- end-- context "when pm_translation_scope is none" do- before { SiteSetting.ai_translation_personal_messages = "none" }-- it "only processes posts from target categories" do- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(target_post, "ja", has_entries(llm_model: anything))- .once-- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(non_target_post, any_parameters)- .never-- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(personal_pm_post, any_parameters)- .never- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(group_pm_post, any_parameters)- .never-- job.execute({ limit: 10 })- end- end-- context "when pm_translation_scope is group" do- before { SiteSetting.ai_translation_personal_messages = "group" }-- it "processes target posts and group PMs but not personal PMs" do- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(target_post, "ja", has_entries(llm_model: anything))- .once- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(non_target_post, any_parameters)- .never-- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(group_pm_post, "ja", has_entries(llm_model: anything))- .once-- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(personal_pm_post, any_parameters)- .never-- job.execute({ limit: 10 })- end- end- end-- describe "with max age limit" do- fab!(:old_post) { Fabricate(:post, locale: "es", created_at: 10.days.ago) }- fab!(:new_post) { Fabricate(:post, locale: "es", created_at: 2.days.ago) }-- before do- SiteSetting.default_locale = "ja"- SiteSetting.ai_translation_backfill_max_age_days = 5- SiteSetting.content_localization_supported_locales = "ja"- end-- it "only processes posts within the age limit" do- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(new_post, "ja", has_entries(llm_model: anything))- .once-DiscourseAi::Translation::PostLocalizer.expects(:localize)- .with(old_post, any_parameters)+ .with(post, "en", has_entries(llm_model: anything)).never-- job.execute({ limit: 10 })- end-- it "processes all posts when setting is large" do- SiteSetting.ai_translation_backfill_max_age_days = 1000-- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(new_post, "ja", has_entries(llm_model: anything))- .once-DiscourseAi::Translation::PostLocalizer.expects(:localize)- .with(old_post, "ja", has_entries(llm_model: anything))+ .with(post, "ja", has_entries(llm_model: anything)).once- job.execute({ limit: 10 })+ job.execute({ pairs: [[post.id, "en"], [post.id, "ja"]] })endenddescribe "LlmModel caching" doit "caches the LlmModel and reuses it for all posts in a batch" do- post_1 = Fabricate(:post, locale: "es", topic: post.topic)- post_2 = Fabricate(:post, locale: "es", topic: post.topic)- SiteSetting.content_localization_supported_locales = "ja"+ post_1 = Fabricate(:post, topic: post.topic)+ post_2 = Fabricate(:post, topic: post.topic)- # Track how many times LlmModel.find_by is calledfind_by_call_count = 0LlmModel.stubs(:find_by)@@ -322,15 +123,12 @@ describe Jobs::LocalizePosts doend.returns(LlmModel.last)- # Stub the actual localization to avoid real translation callsDiscourseAi::Translation::PostLocalizer.stubs(:localize)- job.execute({ limit: 10 })+ job.execute({ pairs: [[post_1.id, "ja"], [post_2.id, "ja"]] })- # Should call LlmModel.find_by only twice for the entire batch:# 1. Once in credits_available_for_post_localization? check# 2. Once in the job's find_llm_model_for_agent for caching- # Without caching, it would be called 2 + (number of posts) timesexpect(find_by_call_count).to eq(2)endenddiff --git a/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb b/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rbindex 80867c9f523..9fd7b9883be 100644--- a/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb+++ b/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb@@ -5,73 +5,63 @@ describe Jobs::LocalizeTopics dofab!(:topic)- let(:locales) { %w[en ja de] }-before doassign_fake_provider_to(:ai_default_llm_model)enable_current_pluginSiteSetting.ai_translation_enabled = true- SiteSetting.content_localization_supported_locales = locales.join("|")+ SiteSetting.content_localization_supported_locales = "en|ja|de"SiteSetting.ai_translation_backfill_hourly_rate = 100SiteSetting.ai_translation_backfill_max_age_days = 100- SiteSetting.ai_translation_target_categories = topic.category_id.to_sendit "does nothing when translator is disabled" doSiteSetting.discourse_ai_enabled = falseDiscourseAi::Translation::TopicLocalizer.expects(:localize).never- job.execute({ limit: 10 })+ job.execute({ pairs: [[topic.id, "ja"]] })endit "does nothing when ai_translation_enabled is disabled" doSiteSetting.ai_translation_enabled = falseDiscourseAi::Translation::TopicLocalizer.expects(:localize).never- job.execute({ limit: 10 })+ job.execute({ pairs: [[topic.id, "ja"]] })endit "does nothing when no target languages are configured" doSiteSetting.content_localization_supported_locales = ""DiscourseAi::Translation::TopicLocalizer.expects(:localize).never- job.execute({ limit: 10 })- end-- it "does nothing when there are no topics to translate" do- Topic.destroy_all- DiscourseAi::Translation::TopicLocalizer.expects(:localize).never-- job.execute({ limit: 10 })+ job.execute({ pairs: [[topic.id, "ja"]] })endit "skips translation when credits are unavailable" doDiscourseAi::Translation.expects(:credits_available_for_topic_localization?).returns(false)DiscourseAi::Translation::TopicLocalizer.expects(:localize).never- job.execute({ limit: 10 })+ job.execute({ pairs: [[topic.id, "ja"]] })end- it "skips topics that already have localizations" do- Topic.all.each do |topic|- Fabricate(:topic_localization, topic:, locale: "en")- Fabricate(:topic_localization, topic:, locale: "ja")- end+ it "skips pairs where topic is not found" doDiscourseAi::Translation::TopicLocalizer.expects(:localize).never- job.execute({ limit: 10 })+ job.execute({ pairs: [[-1, "ja"]] })end- it "skips bot topics" do- topic.update!(user: Discourse.system_user)- DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "en").never- DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "ja").never+ it "translates each pair it receives" do+ DiscourseAi::Translation::TopicLocalizer+ .expects(:localize)+ .with(topic, "en", has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything))+ .once+ DiscourseAi::Translation::TopicLocalizer+ .expects(:localize)+ .with(topic, "ja", has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything))+ .once- job.execute({ limit: 10 })+ job.execute({ pairs: [[topic.id, "en"], [topic.id, "ja"]] })endit "handles translation errors gracefully" do- topic.update(locale: "es")DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "en", has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything))@@ -80,314 +70,16 @@ describe Jobs::LocalizeTopics do.expects(:localize).with(topic, "ja", has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything)).once- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(topic, "de", has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything))- .once- expect { job.execute({ limit: 10 }) }.not_to raise_error+ expect { job.execute({ pairs: [[topic.id, "en"], [topic.id, "ja"]] }) }.not_to raise_errorendit "logs a summary after translation" do- topic.update(locale: "es")DiscourseAi::Translation::TopicLocalizer.stubs(:localize)DiscourseAi::Translation::VerboseLogger.expects(:log).with(- includes("Translated 1 topics to en"),+ includes("Translated 2/2 topic localizations"),)- DiscourseAi::Translation::VerboseLogger.expects(:log).with(- includes("Translated 1 topics to ja"),- )- DiscourseAi::Translation::VerboseLogger.expects(:log).with(- includes("Translated 1 topics to de"),- )-- job.execute({ limit: 10 })- end-- context "for translation scenarios" do- it "scenario 1: skips topic when locale is not set" do- DiscourseAi::Translation::TopicLocalizer.expects(:localize).never-- job.execute({ limit: 10 })- end-- it "scenario 2: returns topic with locale 'es' if localizations for en/ja/de do not exist" do- topic = Fabricate(:topic, locale: "es", category: self.topic.category)-- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- topic,- "en",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- topic,- "ja",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- topic,- "de",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once-- job.execute({ limit: 10 })- end-- it "scenario 3: returns topic with locale 'en' if ja/de localization does not exist" do- topic = Fabricate(:topic, locale: "en", category: self.topic.category)-- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- topic,- "ja",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- topic,- "de",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once- DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "en").never-- job.execute({ limit: 10 })- end-- it "scenario 4: skips topic with locale 'en' if all localizations exist" do- topic = Fabricate(:topic, locale: "en", category: self.topic.category)- Fabricate(:topic_localization, topic: topic, locale: "ja")- Fabricate(:topic_localization, topic: topic, locale: "de")-- DiscourseAi::Translation::TopicLocalizer.expects(:localize).never-- job.execute({ limit: 10 })- end-- it "scenario 5: skips topic that already have localizations in similar language variant" do- topic = Fabricate(:topic, locale: "en", category: self.topic.category)- Fabricate(:topic_localization, topic: topic, locale: "ja_JP")- Fabricate(:topic_localization, topic: topic, locale: "de_DE")-- DiscourseAi::Translation::TopicLocalizer.expects(:localize).never-- job.execute({ limit: 10 })- end- end-- it "enforces a budget across all locales" do- SiteSetting.content_localization_supported_locales = "en|ja|de"- topic.update!(locale: "es")-- call_count = 0- DiscourseAi::Translation::TopicLocalizer- .stubs(:localize)- .with do- call_count += 1- true- end-- job.execute({ limit: 2 })- expect(call_count).to eq(2)- end-- describe "with target categories" do- fab!(:target_category, :category)- fab!(:non_target_category, :category)- fab!(:target_topic) { Fabricate(:topic, category: target_category, locale: "es") }- fab!(:non_target_topic) { Fabricate(:topic, category: non_target_category, locale: "es") }-- fab!(:personal_pm_topic) { Fabricate(:private_message_topic, locale: "es") }-- fab!(:group_pm_topic) do- Fabricate(:group_private_message_topic, recipient_group: Fabricate(:group), locale: "es")- end-- before do- SiteSetting.default_locale = "ja"- SiteSetting.content_localization_supported_locales = "ja"- SiteSetting.ai_translation_target_categories = target_category.id.to_s- end-- context "when pm_translation_scope is none" do- before { SiteSetting.ai_translation_personal_messages = "none" }-- it "only processes topics from target categories" do- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- target_topic,- "ja",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once-- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(non_target_topic, any_parameters)- .never-- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(personal_pm_topic, any_parameters)- .never-- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(group_pm_topic, any_parameters)- .never-- job.execute({ limit: 10 })- end- end-- context "when pm_translation_scope is group" do- before { SiteSetting.ai_translation_personal_messages = "group" }-- it "processes target topics and group PMs but not personal PMs" do- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- target_topic,- "ja",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once-- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- group_pm_topic,- "ja",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once-- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(non_target_topic, any_parameters)- .never-- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(personal_pm_topic, any_parameters)- .never-- job.execute({ limit: 10 })- end- end- end-- describe "with max age limit" do- fab!(:old_topic) do- Fabricate(:topic, locale: "es", created_at: 10.days.ago, category: topic.category)- end- fab!(:new_topic) do- Fabricate(:topic, locale: "es", created_at: 2.days.ago, category: topic.category)- end-- before { SiteSetting.ai_translation_backfill_max_age_days = 5 }-- it "only processes topics within the age limit" do- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- new_topic,- "en",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- new_topic,- "ja",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- new_topic,- "de",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once-- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(old_topic, any_parameters)- .never-- job.execute({ limit: 10 })- end-- it "processes all topics when setting is more than the post age" do- SiteSetting.ai_translation_backfill_max_age_days = 100-- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- new_topic,- "en",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- new_topic,- "ja",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- new_topic,- "de",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once-- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- old_topic,- "en",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- old_topic,- "ja",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(- old_topic,- "de",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once- job.execute({ limit: 10 })- end+ job.execute({ pairs: [[topic.id, "en"], [topic.id, "ja"]] })endenddiff --git a/plugins/discourse-ai/spec/jobs/scheduled/post_localization_backfill_spec.rb b/plugins/discourse-ai/spec/jobs/scheduled/post_localization_backfill_spec.rbindex af25f50617c..41096ba3cc2 100644--- a/plugins/discourse-ai/spec/jobs/scheduled/post_localization_backfill_spec.rb+++ b/plugins/discourse-ai/spec/jobs/scheduled/post_localization_backfill_spec.rb@@ -1,12 +1,16 @@# frozen_string_literal: truedescribe Jobs::PostLocalizationBackfill do+ fab!(:post) { Fabricate(:post, locale: "es") }+before doenable_current_pluginassign_fake_provider_to(:ai_default_llm_model)SiteSetting.ai_translation_backfill_hourly_rate = 100+ SiteSetting.ai_translation_backfill_max_age_days = 100SiteSetting.content_localization_supported_locales = "en"SiteSetting.ai_translation_enabled = true+ SiteSetting.ai_translation_target_categories = post.topic.category_id.to_sendit "does not enqueue post translation when translator disabled" do@@ -42,12 +46,27 @@ describe Jobs::PostLocalizationBackfill doexpect_not_enqueued_with(job: :localize_posts)end- it "enqueues post translation with correct limit" do- SiteSetting.ai_translation_enabled = true- SiteSetting.ai_translation_backfill_hourly_rate = 100+ it "does not enqueue when all posts are fully translated" do+ Fabricate(:post_localization, post: post, locale: "en")++ described_class.new.execute({})++ expect_not_enqueued_with(job: :localize_posts)+ end++ it "enqueues post translation with pairs" do+ described_class.new.execute({})++ expect_job_enqueued(job: :localize_posts, args: { pairs: [[post.id, "en"]] })+ end++ it "distributes pairs across parallel jobs" do+ SiteSetting.ai_translation_backfill_parallel_jobs = 2+ post_2 = Fabricate(:post, locale: "es", topic: post.topic)described_class.new.execute({})- expect_job_enqueued(job: :localize_posts, args: { limit: 8 })+ expect_job_enqueued(job: :localize_posts, args: { pairs: [[post_2.id, "en"]] })+ expect_job_enqueued(job: :localize_posts, args: { pairs: [[post.id, "en"]] })endenddiff --git a/plugins/discourse-ai/spec/jobs/scheduled/topic_localization_backfill_spec.rb b/plugins/discourse-ai/spec/jobs/scheduled/topic_localization_backfill_spec.rbindex 42ca7d2cea8..a59267aa986 100644--- a/plugins/discourse-ai/spec/jobs/scheduled/topic_localization_backfill_spec.rb+++ b/plugins/discourse-ai/spec/jobs/scheduled/topic_localization_backfill_spec.rb@@ -1,12 +1,16 @@# frozen_string_literal: truedescribe Jobs::TopicLocalizationBackfill do+ fab!(:topic) { Fabricate(:topic, locale: "es") }+before doenable_current_pluginassign_fake_provider_to(:ai_default_llm_model)SiteSetting.ai_translation_backfill_hourly_rate = 100+ SiteSetting.ai_translation_backfill_max_age_days = 100SiteSetting.content_localization_supported_locales = "en"SiteSetting.ai_translation_enabled = true+ SiteSetting.ai_translation_target_categories = topic.category_id.to_sendit "does not enqueue topic translation when translator disabled" do@@ -33,11 +37,15 @@ describe Jobs::TopicLocalizationBackfill doexpect_not_enqueued_with(job: :localize_topics) { described_class.new.execute({}) }end- it "enqueues topic translation with correct limit" do- SiteSetting.ai_translation_backfill_hourly_rate = 100+ it "does not enqueue when all topics are fully translated" do+ Fabricate(:topic_localization, topic: topic, locale: "en")++ expect_not_enqueued_with(job: :localize_topics) { described_class.new.execute({}) }+ end+ it "enqueues topic translation with pairs" dodescribed_class.new.execute({})- expect_job_enqueued(job: :localize_topics, args: { limit: 8 })+ expect_job_enqueued(job: :localize_topics, args: { pairs: [[topic.id, "en"]] })endenddiff --git a/plugins/discourse-ai/spec/lib/translation/post_candidates_spec.rb b/plugins/discourse-ai/spec/lib/translation/post_candidates_spec.rbindex 2b67e1ee33b..c2c5836cb89 100644--- a/plugins/discourse-ai/spec/lib/translation/post_candidates_spec.rb+++ b/plugins/discourse-ai/spec/lib/translation/post_candidates_spec.rb@@ -101,6 +101,90 @@ describe DiscourseAi::Translation::PostCandidates doendend+ describe ".needs_localization" do+ fab!(:target_category, :category)++ before do+ SiteSetting.ai_translation_backfill_max_age_days = 100+ SiteSetting.content_localization_supported_locales = "en|ja|de"+ SiteSetting.ai_translation_target_categories = target_category.id.to_s+ SiteSetting.ai_translation_personal_messages = "none"+ end++ it "returns [post_id, target_locale] pairs for posts needing localization" do+ post = Fabricate(:post, locale: "es", topic: Fabricate(:topic, category: target_category))++ pairs = described_class.needs_localization(limit: 10)+ expect(pairs).to include([post.id, "en"])+ expect(pairs).to include([post.id, "ja"])+ expect(pairs).to include([post.id, "de"])+ end++ it "excludes posts without a detected locale" do+ Fabricate(:post, locale: nil, topic: Fabricate(:topic, category: target_category))++ pairs = described_class.needs_localization(limit: 10)+ expect(pairs).to be_empty+ end++ it "excludes fully translated posts" do+ post = Fabricate(:post, locale: "es", topic: Fabricate(:topic, category: target_category))+ Fabricate(:post_localization, post: post, locale: "en")+ Fabricate(:post_localization, post: post, locale: "ja")+ Fabricate(:post_localization, post: post, locale: "de")++ pairs = described_class.needs_localization(limit: 10)+ post_ids = pairs.map(&:first)+ expect(post_ids).not_to include(post.id)+ end++ it "returns only missing locale pairs for partially translated posts" do+ post = Fabricate(:post, locale: "es", topic: Fabricate(:topic, category: target_category))+ Fabricate(:post_localization, post: post, locale: "en")++ pairs = described_class.needs_localization(limit: 10)+ expect(pairs).not_to include([post.id, "en"])+ expect(pairs).to include([post.id, "ja"])+ expect(pairs).to include([post.id, "de"])+ end++ it "excludes posts whose locale matches all target base locales" do+ SiteSetting.content_localization_supported_locales = "en"+ post = Fabricate(:post, locale: "en", topic: Fabricate(:topic, category: target_category))++ pairs = described_class.needs_localization(limit: 10)+ post_ids = pairs.map(&:first)+ expect(post_ids).not_to include(post.id)+ end++ it "handles base-locale deduplication (ja_JP localization covers ja target)" do+ post = Fabricate(:post, locale: "es", topic: Fabricate(:topic, category: target_category))+ Fabricate(:post_localization, post: post, locale: "en")+ Fabricate(:post_localization, post: post, locale: "ja_JP")+ Fabricate(:post_localization, post: post, locale: "de_DE")++ pairs = described_class.needs_localization(limit: 10)+ post_ids = pairs.map(&:first)+ expect(post_ids).not_to include(post.id)+ end++ it "respects the limit parameter" do+ 3.times do+ Fabricate(:post, locale: "es", topic: Fabricate(:topic, category: target_category))+ end++ pairs = described_class.needs_localization(limit: 2)+ expect(pairs.size).to eq(2)+ end++ it "returns empty when no locales are configured" do+ SiteSetting.content_localization_supported_locales = ""++ pairs = described_class.needs_localization(limit: 10)+ expect(pairs).to be_empty+ end+ end+describe ".get_completion_all_locales" dofab!(:target_category, :category)diff --git a/plugins/discourse-ai/spec/lib/translation/topic_candidates_spec.rb b/plugins/discourse-ai/spec/lib/translation/topic_candidates_spec.rbindex 0cfb22c2703..670e566ca4d 100644--- a/plugins/discourse-ai/spec/lib/translation/topic_candidates_spec.rb+++ b/plugins/discourse-ai/spec/lib/translation/topic_candidates_spec.rb@@ -127,6 +127,79 @@ describe DiscourseAi::Translation::TopicCandidates doendend+ describe ".needs_localization" do+ fab!(:target_category, :category)++ before do+ SiteSetting.ai_translation_backfill_max_age_days = 100+ SiteSetting.content_localization_supported_locales = "en|ja|de"+ SiteSetting.ai_translation_target_categories = target_category.id.to_s+ SiteSetting.ai_translation_personal_messages = "none"+ end++ it "returns [topic_id, target_locale] pairs for topics needing localization" do+ topic = Fabricate(:topic, locale: "es", category: target_category)++ pairs = described_class.needs_localization(limit: 10)+ expect(pairs).to include([topic.id, "en"])+ expect(pairs).to include([topic.id, "ja"])+ expect(pairs).to include([topic.id, "de"])+ end++ it "excludes topics without a detected locale" do+ Fabricate(:topic, locale: nil, category: target_category)++ pairs = described_class.needs_localization(limit: 10)+ expect(pairs).to be_empty+ end++ it "excludes fully translated topics" do+ topic = Fabricate(:topic, locale: "es", category: target_category)+ Fabricate(:topic_localization, topic: topic, locale: "en")+ Fabricate(:topic_localization, topic: topic, locale: "ja")+ Fabricate(:topic_localization, topic: topic, locale: "de")++ pairs = described_class.needs_localization(limit: 10)+ topic_ids = pairs.map(&:first)+ expect(topic_ids).not_to include(topic.id)+ end++ it "returns only missing locale pairs for partially translated topics" do+ topic = Fabricate(:topic, locale: "es", category: target_category)+ Fabricate(:topic_localization, topic: topic, locale: "en")++ pairs = described_class.needs_localization(limit: 10)+ expect(pairs).not_to include([topic.id, "en"])+ expect(pairs).to include([topic.id, "ja"])+ expect(pairs).to include([topic.id, "de"])+ end++ it "handles base-locale deduplication (ja_JP localization covers ja target)" do+ topic = Fabricate(:topic, locale: "es", category: target_category)+ Fabricate(:topic_localization, topic: topic, locale: "en")+ Fabricate(:topic_localization, topic: topic, locale: "ja_JP")+ Fabricate(:topic_localization, topic: topic, locale: "de_DE")++ pairs = described_class.needs_localization(limit: 10)+ topic_ids = pairs.map(&:first)+ expect(topic_ids).not_to include(topic.id)+ end++ it "respects the limit parameter" do+ 3.times { Fabricate(:topic, locale: "es", category: target_category) }++ pairs = described_class.needs_localization(limit: 2)+ expect(pairs.size).to eq(2)+ end++ it "returns empty when no locales are configured" do+ SiteSetting.content_localization_supported_locales = ""++ pairs = described_class.needs_localization(limit: 10)+ expect(pairs).to be_empty+ end+ end+describe ".calculate_completion_per_locale" dobefore { SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|") }