Skip to content

A platform for community discussion. Free, open, simple.

PERF: Fix localization backfills to pass (id, locale) pairs to workers (#39486)

Natalie Tay915999029899
diff --git a/plugins/discourse-ai/app/jobs/regular/localize_posts.rb b/plugins/discourse-ai/app/jobs/regular/localize_posts.rb
index 813e895ac9f..c359e50ff97 100644
--- a/plugins/discourse-ai/app/jobs/regular/localize_posts.rb
+++ b/plugins/discourse-ai/app/jobs/regular/localize_posts.rb
@@ -7,10 +7,8 @@ module Jobs
REDIS_KEY = "discourse-ai:localize_posts:in_progress"
def execute(args)
- limit = args[:limit]
- raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
-
- offset = args[:offset].to_i
+ pairs = args[:pairs]
+ raise Discourse::InvalidParameters.new(:pairs) if pairs.blank?
return if !DiscourseAi::Translation.backfill_enabled?
@@ -24,64 +22,39 @@ module Jobs
llm_model = find_llm_model_for_agent(SiteSetting.ai_translation_post_raw_translator_agent)
return if llm_model.blank?
- locales = DiscourseAi::Translation.locales
- return if locales.blank?
-
- locale_pairs = locales.map { |l| [l.split("_").first, l] }
-
- posts =
- DiscourseAi::Translation::PostCandidates
- .get
- .where.not(locale: nil)
- .order(updated_at: :desc)
- .offset(offset)
- .limit(limit)
-
- return if posts.empty?
-
- existing =
- PostLocalization.where(post_id: posts.map(&:id)).pluck(:post_id, :locale).group_by(&:first)
+ post_ids = pairs.map(&:first).uniq
+ posts_by_id = Post.where(id: post_ids).index_by(&:id)
- existing_base_locales =
- existing.transform_values { |pairs| pairs.map { |_, loc| loc.split("_").first }.to_set }
+ translated = 0
+ pairs.each do |post_id, target_locale|
+ post = posts_by_id[post_id]
+ next if post.nil?
- budget = limit
- translated_counts = Hash.new(0)
-
- posts.each do |post|
- break if budget <= 0
- post_base = post.locale.split("_").first
-
- locale_pairs.each do |base_locale, target_locale|
- break if budget <= 0
- next if post_base == base_locale
- next if existing_base_locales.dig(post.id)&.include?(base_locale)
-
- Discourse.redis.expire(REDIS_KEY, 15.minutes.to_i)
- unless DiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, target_locale)
- next
- end
+ Discourse.redis.expire(REDIS_KEY, 15.minutes.to_i)
+ unless DiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, target_locale)
+ next
+ end
- begin
- DiscourseAi::Translation::PostLocalizer.localize(
- post,
- target_locale,
- llm_model: llm_model,
- )
- translated_counts[target_locale] += 1
- budget -= 1
- rescue FinalDestination::SSRFDetector::LookupFailedError
- # do nothing, there are too many sporadic lookup failures
- rescue => e
- DiscourseAi::Translation::VerboseLogger.log(
- "Failed to translate post #{post.id} to #{target_locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
- )
- end
+ begin
+ DiscourseAi::Translation::PostLocalizer.localize(
+ post,
+ target_locale,
+ llm_model: llm_model,
+ )
+ translated += 1
+ rescue FinalDestination::SSRFDetector::LookupFailedError
+ # do nothing, there are too many sporadic lookup failures
+ rescue => e
+ DiscourseAi::Translation::VerboseLogger.log(
+ "Failed to translate post #{post.id} to #{target_locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+ )
end
end
- translated_counts.each do |target_locale, count|
- DiscourseAi::Translation::VerboseLogger.log("Translated #{count} posts to #{target_locale}")
+ if translated > 0
+ DiscourseAi::Translation::VerboseLogger.log(
+ "Translated #{translated}/#{pairs.size} post localizations: #{pairs.map { |id, loc| "#{id}:#{loc}" }.join(", ")}",
+ )
end
ensure
remaining = Discourse.redis.decr(REDIS_KEY)
diff --git a/plugins/discourse-ai/app/jobs/regular/localize_topics.rb b/plugins/discourse-ai/app/jobs/regular/localize_topics.rb
index 630ef3fd2bf..cd522602cb4 100644
--- a/plugins/discourse-ai/app/jobs/regular/localize_topics.rb
+++ b/plugins/discourse-ai/app/jobs/regular/localize_topics.rb
@@ -6,8 +6,8 @@ module Jobs
sidekiq_options retry: false
def execute(args)
- limit = args[:limit]
- raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
+ pairs = args[:pairs]
+ raise Discourse::InvalidParameters.new(:pairs) if pairs.blank?
return if !DiscourseAi::Translation.backfill_enabled?
@@ -24,63 +24,34 @@ module Jobs
find_llm_model_for_agent(SiteSetting.ai_translation_post_raw_translator_agent)
return if topic_title_llm_model.blank? && post_raw_llm_model.blank?
- locales = DiscourseAi::Translation.locales
- return if locales.blank?
-
- locale_pairs = locales.map { |l| [l.split("_").first, l] }
-
- topics =
- DiscourseAi::Translation::TopicCandidates
- .get
- .where.not(locale: nil)
- .order(updated_at: :desc)
- .limit(limit)
-
- return if topics.empty?
-
- existing =
- TopicLocalization
- .where(topic_id: topics.map(&:id))
- .pluck(:topic_id, :locale)
- .group_by(&:first)
-
- existing_base_locales =
- existing.transform_values { |pairs| pairs.map { |_, loc| loc.split("_").first }.to_set }
-
- budget = limit
- translated_counts = Hash.new(0)
-
- topics.each do |topic|
- break if budget <= 0
- topic_base = topic.locale.split("_").first
-
- locale_pairs.each do |base_locale, target_locale|
- break if budget <= 0
- next if topic_base == base_locale
- next if existing_base_locales.dig(topic.id)&.include?(base_locale)
-
- begin
- DiscourseAi::Translation::TopicLocalizer.localize(
- topic,
- target_locale,
- topic_title_llm_model:,
- post_raw_llm_model:,
- )
- translated_counts[target_locale] += 1
- budget -= 1
- rescue FinalDestination::SSRFDetector::LookupFailedError
- # do nothing, there are too many sporadic lookup failures
- rescue => e
- DiscourseAi::Translation::VerboseLogger.log(
- "Failed to translate topic #{topic.id} to #{target_locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
- )
- end
+ topic_ids = pairs.map(&:first).uniq
+ topics_by_id = Topic.where(id: topic_ids).index_by(&:id)
+
+ translated = 0
+ pairs.each do |topic_id, target_locale|
+ topic = topics_by_id[topic_id]
+ next if topic.nil?
+
+ begin
+ DiscourseAi::Translation::TopicLocalizer.localize(
+ topic,
+ target_locale,
+ topic_title_llm_model:,
+ post_raw_llm_model:,
+ )
+ translated += 1
+ rescue FinalDestination::SSRFDetector::LookupFailedError
+ # do nothing, there are too many sporadic lookup failures
+ rescue => e
+ DiscourseAi::Translation::VerboseLogger.log(
+ "Failed to translate topic #{topic.id} to #{target_locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+ )
end
end
- translated_counts.each do |target_locale, count|
+ if translated > 0
DiscourseAi::Translation::VerboseLogger.log(
- "Translated #{count} topics to #{target_locale}",
+ "Translated #{translated}/#{pairs.size} topic localizations: #{pairs.map { |id, loc| "#{id}:#{loc}" }.join(", ")}",
)
end
end
diff --git a/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb b/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb
index 7418d97926c..780f3bb6197 100644
--- a/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb
+++ b/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb
@@ -25,15 +25,16 @@ module Jobs
limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
return if limit == 0
+ pairs = DiscourseAi::Translation::PostCandidates.needs_localization(limit: limit)
+ return if pairs.empty?
+
parallel_jobs = SiteSetting.ai_translation_backfill_parallel_jobs
- per_job_limit = (limit.to_f / parallel_jobs).ceil
+ chunks = pairs.each_slice((pairs.size.to_f / parallel_jobs).ceil).to_a
# Set counter with 15-min TTL safety net (if a job crashes without decrementing)
- Discourse.redis.setex(REDIS_KEY, 15.minutes.to_i, parallel_jobs)
+ Discourse.redis.setex(REDIS_KEY, 15.minutes.to_i, chunks.size)
- parallel_jobs.times do |i|
- Jobs.enqueue(:localize_posts, limit: per_job_limit, offset: i * per_job_limit)
- end
+ chunks.each { |chunk| Jobs.enqueue(:localize_posts, pairs: chunk) }
end
private
diff --git a/plugins/discourse-ai/app/jobs/scheduled/topic_localization_backfill.rb b/plugins/discourse-ai/app/jobs/scheduled/topic_localization_backfill.rb
index 11a04cdb487..5ee8d6ed5d2 100644
--- a/plugins/discourse-ai/app/jobs/scheduled/topic_localization_backfill.rb
+++ b/plugins/discourse-ai/app/jobs/scheduled/topic_localization_backfill.rb
@@ -23,7 +23,10 @@ module Jobs
limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
return if limit == 0
- Jobs.enqueue(:localize_topics, limit:)
+ pairs = DiscourseAi::Translation::TopicCandidates.needs_localization(limit: limit)
+ return if pairs.empty?
+
+ Jobs.enqueue(:localize_topics, pairs: pairs)
end
private
diff --git a/plugins/discourse-ai/lib/translation/post_candidates.rb b/plugins/discourse-ai/lib/translation/post_candidates.rb
index 092502b669e..1d4f7089c96 100644
--- a/plugins/discourse-ai/lib/translation/post_candidates.rb
+++ b/plugins/discourse-ai/lib/translation/post_candidates.rb
@@ -11,6 +11,34 @@ module DiscourseAi
completion_all_locales
end
+ def self.needs_localization(limit:)
+ locales = DiscourseAi::Translation.locales
+ return [] if locales.blank?
+
+ locale_map = {}
+ locales.each { |l| locale_map[l.split("_").first] ||= l }
+
+ target_locale_values = locale_map.map { |base, full| "('#{base}', '#{full}')" }.join(", ")
+
+ base_sql = get.where.not(locale: nil).to_sql
+
+ sql = <<~SQL
+ SELECT ep.id AS post_id, target.target_locale
+ FROM (#{base_sql}) ep
+ JOIN (VALUES #{target_locale_values}) AS target(base_locale, target_locale)
+ ON target.base_locale != split_part(ep.locale, '_', 1)
+ WHERE NOT EXISTS (
+ SELECT 1 FROM post_localizations pl
+ WHERE pl.post_id = ep.id
+ AND split_part(pl.locale, '_', 1) = target.base_locale
+ )
+ ORDER BY ep.updated_at DESC, target.target_locale
+ LIMIT #{limit.to_i}
+ SQL
+
+ DB.query(sql).map { |r| [r.post_id, r.target_locale] }
+ end
+
private
# all posts that are eligible for translation based on site settings,
diff --git a/plugins/discourse-ai/lib/translation/topic_candidates.rb b/plugins/discourse-ai/lib/translation/topic_candidates.rb
index a15a7126d2e..0bb056239f1 100644
--- a/plugins/discourse-ai/lib/translation/topic_candidates.rb
+++ b/plugins/discourse-ai/lib/translation/topic_candidates.rb
@@ -3,6 +3,34 @@
module DiscourseAi
module Translation
class TopicCandidates < BaseCandidates
+ def self.needs_localization(limit:)
+ locales = DiscourseAi::Translation.locales
+ return [] if locales.blank?
+
+ locale_map = {}
+ locales.each { |l| locale_map[l.split("_").first] ||= l }
+
+ target_locale_values = locale_map.map { |base, full| "('#{base}', '#{full}')" }.join(", ")
+
+ base_sql = get.where.not(locale: nil).to_sql
+
+ sql = <<~SQL
+ SELECT et.id AS topic_id, target.target_locale
+ FROM (#{base_sql}) et
+ JOIN (VALUES #{target_locale_values}) AS target(base_locale, target_locale)
+ ON target.base_locale != split_part(et.locale, '_', 1)
+ WHERE NOT EXISTS (
+ SELECT 1 FROM topic_localizations tl
+ WHERE tl.topic_id = et.id
+ AND split_part(tl.locale, '_', 1) = target.base_locale
+ )
+ ORDER BY et.updated_at DESC, target.target_locale
+ LIMIT #{limit.to_i}
+ SQL
+
+ DB.query(sql).map { |r| [r.topic_id, r.target_locale] }
+ end
+
private
# all topics that are eligible for translation based on site settings,
diff --git a/plugins/discourse-ai/spec/jobs/regular/localize_posts_spec.rb b/plugins/discourse-ai/spec/jobs/regular/localize_posts_spec.rb
index ef7c7413a5e..eef132819db 100644
--- a/plugins/discourse-ai/spec/jobs/regular/localize_posts_spec.rb
+++ b/plugins/discourse-ai/spec/jobs/regular/localize_posts_spec.rb
@@ -5,69 +5,70 @@ describe Jobs::LocalizePosts do
fab!(:post)
- let(:locales) { %w[en ja de] }
-
before do
assign_fake_provider_to(:ai_default_llm_model)
enable_current_plugin
SiteSetting.ai_translation_enabled = true
- SiteSetting.content_localization_supported_locales = locales.join("|")
+ SiteSetting.content_localization_supported_locales = "en|ja|de"
SiteSetting.ai_translation_backfill_hourly_rate = 100
SiteSetting.ai_translation_backfill_max_age_days = 100
- SiteSetting.ai_translation_target_categories = post.topic.category_id.to_s
end
it "does nothing when translator is disabled" do
SiteSetting.discourse_ai_enabled = false
DiscourseAi::Translation::PostLocalizer.expects(:localize).never
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[post.id, "ja"]] })
end
it "does nothing when ai_translation_enabled is disabled" do
SiteSetting.ai_translation_enabled = false
DiscourseAi::Translation::PostLocalizer.expects(:localize).never
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[post.id, "ja"]] })
end
it "does nothing when no target languages are configured" do
SiteSetting.content_localization_supported_locales = ""
DiscourseAi::Translation::PostLocalizer.expects(:localize).never
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[post.id, "ja"]] })
end
it "does nothing when ai_translation_backfill_hourly_rate is 0" do
SiteSetting.ai_translation_backfill_hourly_rate = 0
DiscourseAi::Translation::PostLocalizer.expects(:localize).never
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[post.id, "ja"]] })
end
it "skips translation when credits are unavailable" do
DiscourseAi::Translation.expects(:credits_available_for_post_localization?).returns(false)
DiscourseAi::Translation::PostLocalizer.expects(:localize).never
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[post.id, "ja"]] })
end
- it "does nothing when there are no posts to translate" do
- Post.destroy_all
+ it "skips pairs where post is not found" do
DiscourseAi::Translation::PostLocalizer.expects(:localize).never
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[-1, "ja"]] })
end
- it "skips bot posts" do
- post.update!(locale: "es", user: Discourse.system_user)
- DiscourseAi::Translation::PostLocalizer.expects(:localize).never
+ it "translates each pair it receives" do
+ DiscourseAi::Translation::PostLocalizer
+ .expects(:localize)
+ .with(post, "en", has_entries(llm_model: anything))
+ .once
+ DiscourseAi::Translation::PostLocalizer
+ .expects(:localize)
+ .with(post, "ja", has_entries(llm_model: anything))
+ .once
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[post.id, "en"], [post.id, "ja"]] })
end
it "handles translation errors gracefully" do
- post.update(locale: "es")
DiscourseAi::Translation::PostLocalizer
.expects(:localize)
.with(post, "en", has_entries(llm_model: anything))
@@ -76,243 +77,43 @@ describe Jobs::LocalizePosts do
.expects(:localize)
.with(post, "ja", has_entries(llm_model: anything))
.once
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(post, "de", has_entries(llm_model: anything))
- .once
- expect { job.execute({ limit: 10 }) }.not_to raise_error
+ expect { job.execute({ pairs: [[post.id, "en"], [post.id, "ja"]] }) }.not_to raise_error
end
it "logs a summary after translation" do
- post.update(locale: "es")
DiscourseAi::Translation::PostLocalizer.stubs(:localize)
- DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Translated 1 posts to en"))
- DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Translated 1 posts to ja"))
- DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Translated 1 posts to de"))
-
- job.execute({ limit: 10 })
- end
-
- context "for translation scenarios" do
- it "scenario 1: skips post when locale is not set" do
- DiscourseAi::Translation::PostLocalizer.expects(:localize).never
-
- job.execute({ limit: 10 })
- end
-
- it "scenario 2: localizes post with locale 'es' when localizations for en/ja/de do not exist" do
- post = Fabricate(:post, locale: "es", topic: self.post.topic)
-
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(post, "en", has_entries(llm_model: anything))
- .once
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(post, "ja", has_entries(llm_model: anything))
- .once
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(post, "de", has_entries(llm_model: anything))
- .once
-
- job.execute({ limit: 10 })
- end
-
- it "scenario 3: localizes post with locale 'en' when ja/de localization do not exist" do
- post = Fabricate(:post, locale: "en", topic: self.post.topic)
-
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(post, "ja", has_entries(llm_model: anything))
- .once
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(post, "de", has_entries(llm_model: anything))
- .once
- DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "en").never
-
- job.execute({ limit: 10 })
- end
-
- it "scenario 4: skips post with locale 'en' if all localizations exist" do
- post = Fabricate(:post, locale: "en", topic: self.post.topic)
- Fabricate(:post_localization, post: post, locale: "ja")
- Fabricate(:post_localization, post: post, locale: "de")
-
- DiscourseAi::Translation::PostLocalizer.expects(:localize).never
-
- job.execute({ limit: 10 })
- end
-
- it "scenario 5: skips posts that already have localizations in similar language variant" do
- post = Fabricate(:post, locale: "en", topic: self.post.topic)
- Fabricate(:post_localization, post: post, locale: "ja_JP")
- Fabricate(:post_localization, post: post, locale: "de_DE")
-
- DiscourseAi::Translation::PostLocalizer.expects(:localize).never
-
- job.execute({ limit: 10 })
- end
+ DiscourseAi::Translation::VerboseLogger.expects(:log).with(
+ includes("Translated 2/2 post localizations"),
+ )
- it "scenario 6: skips posts with variant 'en_GB' when localizations for ja/de exist" do
- post = Fabricate(:post, locale: "en_GB", topic: self.post.topic)
- Fabricate(:post_localization, post: post, locale: "ja_JP")
- Fabricate(:post_localization, post: post, locale: "de_DE")
-
- DiscourseAi::Translation::PostLocalizer.expects(:localize).never
-
- job.execute({ limit: 10 })
- end
+ job.execute({ pairs: [[post.id, "en"], [post.id, "ja"]] })
end
context "when relocalize quota is exhausted" do
- before { post.update(locale: "es") }
-
it "skips localization for posts that have exceeded quota for a specific locale" do
DiscourseAi::Translation::PostLocalizer::MAX_QUOTA_PER_DAY.times do
DiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, "en")
end
- DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "en").never
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(post, "ja", has_entries(llm_model: anything))
- .once
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(post, "de", has_entries(llm_model: anything))
- .once
-
- job.execute({ limit: 10 })
- end
- end
-
- describe "with target categories" do
- fab!(:target_category, :category)
- fab!(:non_target_category, :category)
- fab!(:target_topic) { Fabricate(:topic, category: target_category) }
- fab!(:non_target_topic) { Fabricate(:topic, category: non_target_category) }
- fab!(:target_post) { Fabricate(:post, topic: target_topic, locale: "es") }
- fab!(:non_target_post) { Fabricate(:post, topic: non_target_topic, locale: "es") }
-
- fab!(:personal_pm_topic, :private_message_topic)
- fab!(:personal_pm_post) { Fabricate(:post, topic: personal_pm_topic, locale: "es") }
-
- fab!(:group)
- fab!(:group_pm_topic) { Fabricate(:group_private_message_topic, recipient_group: group) }
- fab!(:group_pm_post) { Fabricate(:post, topic: group_pm_topic, locale: "es") }
-
- before do
- SiteSetting.default_locale = "ja"
- SiteSetting.content_localization_supported_locales = "ja"
- SiteSetting.ai_translation_target_categories = target_category.id.to_s
- end
-
- context "when pm_translation_scope is none" do
- before { SiteSetting.ai_translation_personal_messages = "none" }
-
- it "only processes posts from target categories" do
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(target_post, "ja", has_entries(llm_model: anything))
- .once
-
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(non_target_post, any_parameters)
- .never
-
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(personal_pm_post, any_parameters)
- .never
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(group_pm_post, any_parameters)
- .never
-
- job.execute({ limit: 10 })
- end
- end
-
- context "when pm_translation_scope is group" do
- before { SiteSetting.ai_translation_personal_messages = "group" }
-
- it "processes target posts and group PMs but not personal PMs" do
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(target_post, "ja", has_entries(llm_model: anything))
- .once
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(non_target_post, any_parameters)
- .never
-
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(group_pm_post, "ja", has_entries(llm_model: anything))
- .once
-
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(personal_pm_post, any_parameters)
- .never
-
- job.execute({ limit: 10 })
- end
- end
- end
-
- describe "with max age limit" do
- fab!(:old_post) { Fabricate(:post, locale: "es", created_at: 10.days.ago) }
- fab!(:new_post) { Fabricate(:post, locale: "es", created_at: 2.days.ago) }
-
- before do
- SiteSetting.default_locale = "ja"
- SiteSetting.ai_translation_backfill_max_age_days = 5
- SiteSetting.content_localization_supported_locales = "ja"
- end
-
- it "only processes posts within the age limit" do
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(new_post, "ja", has_entries(llm_model: anything))
- .once
-
DiscourseAi::Translation::PostLocalizer
.expects(:localize)
- .with(old_post, any_parameters)
+ .with(post, "en", has_entries(llm_model: anything))
.never
-
- job.execute({ limit: 10 })
- end
-
- it "processes all posts when setting is large" do
- SiteSetting.ai_translation_backfill_max_age_days = 1000
-
- DiscourseAi::Translation::PostLocalizer
- .expects(:localize)
- .with(new_post, "ja", has_entries(llm_model: anything))
- .once
-
DiscourseAi::Translation::PostLocalizer
.expects(:localize)
- .with(old_post, "ja", has_entries(llm_model: anything))
+ .with(post, "ja", has_entries(llm_model: anything))
.once
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[post.id, "en"], [post.id, "ja"]] })
end
end
describe "LlmModel caching" do
it "caches the LlmModel and reuses it for all posts in a batch" do
- post_1 = Fabricate(:post, locale: "es", topic: post.topic)
- post_2 = Fabricate(:post, locale: "es", topic: post.topic)
- SiteSetting.content_localization_supported_locales = "ja"
+ post_1 = Fabricate(:post, topic: post.topic)
+ post_2 = Fabricate(:post, topic: post.topic)
- # Track how many times LlmModel.find_by is called
find_by_call_count = 0
LlmModel
.stubs(:find_by)
@@ -322,15 +123,12 @@ describe Jobs::LocalizePosts do
end
.returns(LlmModel.last)
- # Stub the actual localization to avoid real translation calls
DiscourseAi::Translation::PostLocalizer.stubs(:localize)
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[post_1.id, "ja"], [post_2.id, "ja"]] })
- # Should call LlmModel.find_by only twice for the entire batch:
# 1. Once in credits_available_for_post_localization? check
# 2. Once in the job's find_llm_model_for_agent for caching
- # Without caching, it would be called 2 + (number of posts) times
expect(find_by_call_count).to eq(2)
end
end
diff --git a/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb b/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb
index 80867c9f523..9fd7b9883be 100644
--- a/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb
+++ b/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb
@@ -5,73 +5,63 @@ describe Jobs::LocalizeTopics do
fab!(:topic)
- let(:locales) { %w[en ja de] }
-
before do
assign_fake_provider_to(:ai_default_llm_model)
enable_current_plugin
SiteSetting.ai_translation_enabled = true
- SiteSetting.content_localization_supported_locales = locales.join("|")
+ SiteSetting.content_localization_supported_locales = "en|ja|de"
SiteSetting.ai_translation_backfill_hourly_rate = 100
SiteSetting.ai_translation_backfill_max_age_days = 100
- SiteSetting.ai_translation_target_categories = topic.category_id.to_s
end
it "does nothing when translator is disabled" do
SiteSetting.discourse_ai_enabled = false
DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[topic.id, "ja"]] })
end
it "does nothing when ai_translation_enabled is disabled" do
SiteSetting.ai_translation_enabled = false
DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[topic.id, "ja"]] })
end
it "does nothing when no target languages are configured" do
SiteSetting.content_localization_supported_locales = ""
DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
- job.execute({ limit: 10 })
- end
-
- it "does nothing when there are no topics to translate" do
- Topic.destroy_all
- DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
-
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[topic.id, "ja"]] })
end
it "skips translation when credits are unavailable" do
DiscourseAi::Translation.expects(:credits_available_for_topic_localization?).returns(false)
DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[topic.id, "ja"]] })
end
- it "skips topics that already have localizations" do
- Topic.all.each do |topic|
- Fabricate(:topic_localization, topic:, locale: "en")
- Fabricate(:topic_localization, topic:, locale: "ja")
- end
+ it "skips pairs where topic is not found" do
DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[-1, "ja"]] })
end
- it "skips bot topics" do
- topic.update!(user: Discourse.system_user)
- DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "en").never
- DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "ja").never
+ it "translates each pair it receives" do
+ DiscourseAi::Translation::TopicLocalizer
+ .expects(:localize)
+ .with(topic, "en", has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything))
+ .once
+ DiscourseAi::Translation::TopicLocalizer
+ .expects(:localize)
+ .with(topic, "ja", has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything))
+ .once
- job.execute({ limit: 10 })
+ job.execute({ pairs: [[topic.id, "en"], [topic.id, "ja"]] })
end
it "handles translation errors gracefully" do
- topic.update(locale: "es")
DiscourseAi::Translation::TopicLocalizer
.expects(:localize)
.with(topic, "en", has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything))
@@ -80,314 +70,16 @@ describe Jobs::LocalizeTopics do
.expects(:localize)
.with(topic, "ja", has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything))
.once
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(topic, "de", has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything))
- .once
- expect { job.execute({ limit: 10 }) }.not_to raise_error
+ expect { job.execute({ pairs: [[topic.id, "en"], [topic.id, "ja"]] }) }.not_to raise_error
end
it "logs a summary after translation" do
- topic.update(locale: "es")
DiscourseAi::Translation::TopicLocalizer.stubs(:localize)
DiscourseAi::Translation::VerboseLogger.expects(:log).with(
- includes("Translated 1 topics to en"),
+ includes("Translated 2/2 topic localizations"),
)
- DiscourseAi::Translation::VerboseLogger.expects(:log).with(
- includes("Translated 1 topics to ja"),
- )
- DiscourseAi::Translation::VerboseLogger.expects(:log).with(
- includes("Translated 1 topics to de"),
- )
-
- job.execute({ limit: 10 })
- end
-
- context "for translation scenarios" do
- it "scenario 1: skips topic when locale is not set" do
- DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
-
- job.execute({ limit: 10 })
- end
-
- it "scenario 2: returns topic with locale 'es' if localizations for en/ja/de do not exist" do
- topic = Fabricate(:topic, locale: "es", category: self.topic.category)
-
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- topic,
- "en",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- topic,
- "ja",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- topic,
- "de",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
-
- job.execute({ limit: 10 })
- end
-
- it "scenario 3: returns topic with locale 'en' if ja/de localization does not exist" do
- topic = Fabricate(:topic, locale: "en", category: self.topic.category)
-
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- topic,
- "ja",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- topic,
- "de",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
- DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "en").never
-
- job.execute({ limit: 10 })
- end
-
- it "scenario 4: skips topic with locale 'en' if all localizations exist" do
- topic = Fabricate(:topic, locale: "en", category: self.topic.category)
- Fabricate(:topic_localization, topic: topic, locale: "ja")
- Fabricate(:topic_localization, topic: topic, locale: "de")
-
- DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
-
- job.execute({ limit: 10 })
- end
-
- it "scenario 5: skips topic that already have localizations in similar language variant" do
- topic = Fabricate(:topic, locale: "en", category: self.topic.category)
- Fabricate(:topic_localization, topic: topic, locale: "ja_JP")
- Fabricate(:topic_localization, topic: topic, locale: "de_DE")
-
- DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
-
- job.execute({ limit: 10 })
- end
- end
-
- it "enforces a budget across all locales" do
- SiteSetting.content_localization_supported_locales = "en|ja|de"
- topic.update!(locale: "es")
-
- call_count = 0
- DiscourseAi::Translation::TopicLocalizer
- .stubs(:localize)
- .with do
- call_count += 1
- true
- end
-
- job.execute({ limit: 2 })
- expect(call_count).to eq(2)
- end
-
- describe "with target categories" do
- fab!(:target_category, :category)
- fab!(:non_target_category, :category)
- fab!(:target_topic) { Fabricate(:topic, category: target_category, locale: "es") }
- fab!(:non_target_topic) { Fabricate(:topic, category: non_target_category, locale: "es") }
-
- fab!(:personal_pm_topic) { Fabricate(:private_message_topic, locale: "es") }
-
- fab!(:group_pm_topic) do
- Fabricate(:group_private_message_topic, recipient_group: Fabricate(:group), locale: "es")
- end
-
- before do
- SiteSetting.default_locale = "ja"
- SiteSetting.content_localization_supported_locales = "ja"
- SiteSetting.ai_translation_target_categories = target_category.id.to_s
- end
-
- context "when pm_translation_scope is none" do
- before { SiteSetting.ai_translation_personal_messages = "none" }
-
- it "only processes topics from target categories" do
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- target_topic,
- "ja",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
-
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(non_target_topic, any_parameters)
- .never
-
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(personal_pm_topic, any_parameters)
- .never
-
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(group_pm_topic, any_parameters)
- .never
-
- job.execute({ limit: 10 })
- end
- end
-
- context "when pm_translation_scope is group" do
- before { SiteSetting.ai_translation_personal_messages = "group" }
-
- it "processes target topics and group PMs but not personal PMs" do
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- target_topic,
- "ja",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
-
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- group_pm_topic,
- "ja",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
-
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(non_target_topic, any_parameters)
- .never
-
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(personal_pm_topic, any_parameters)
- .never
-
- job.execute({ limit: 10 })
- end
- end
- end
-
- describe "with max age limit" do
- fab!(:old_topic) do
- Fabricate(:topic, locale: "es", created_at: 10.days.ago, category: topic.category)
- end
- fab!(:new_topic) do
- Fabricate(:topic, locale: "es", created_at: 2.days.ago, category: topic.category)
- end
-
- before { SiteSetting.ai_translation_backfill_max_age_days = 5 }
-
- it "only processes topics within the age limit" do
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- new_topic,
- "en",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- new_topic,
- "ja",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- new_topic,
- "de",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
-
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(old_topic, any_parameters)
- .never
-
- job.execute({ limit: 10 })
- end
-
- it "processes all topics when setting is more than the post age" do
- SiteSetting.ai_translation_backfill_max_age_days = 100
-
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- new_topic,
- "en",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- new_topic,
- "ja",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- new_topic,
- "de",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
-
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- old_topic,
- "en",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- old_topic,
- "ja",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
- DiscourseAi::Translation::TopicLocalizer
- .expects(:localize)
- .with(
- old_topic,
- "de",
- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),
- )
- .once
- job.execute({ limit: 10 })
- end
+ job.execute({ pairs: [[topic.id, "en"], [topic.id, "ja"]] })
end
end
diff --git a/plugins/discourse-ai/spec/jobs/scheduled/post_localization_backfill_spec.rb b/plugins/discourse-ai/spec/jobs/scheduled/post_localization_backfill_spec.rb
index af25f50617c..41096ba3cc2 100644
--- a/plugins/discourse-ai/spec/jobs/scheduled/post_localization_backfill_spec.rb
+++ b/plugins/discourse-ai/spec/jobs/scheduled/post_localization_backfill_spec.rb
@@ -1,12 +1,16 @@
# frozen_string_literal: true
describe Jobs::PostLocalizationBackfill do
+ fab!(:post) { Fabricate(:post, locale: "es") }
+
before do
enable_current_plugin
assign_fake_provider_to(:ai_default_llm_model)
SiteSetting.ai_translation_backfill_hourly_rate = 100
+ SiteSetting.ai_translation_backfill_max_age_days = 100
SiteSetting.content_localization_supported_locales = "en"
SiteSetting.ai_translation_enabled = true
+ SiteSetting.ai_translation_target_categories = post.topic.category_id.to_s
end
it "does not enqueue post translation when translator disabled" do
@@ -42,12 +46,27 @@ describe Jobs::PostLocalizationBackfill do
expect_not_enqueued_with(job: :localize_posts)
end
- it "enqueues post translation with correct limit" do
- SiteSetting.ai_translation_enabled = true
- SiteSetting.ai_translation_backfill_hourly_rate = 100
+ it "does not enqueue when all posts are fully translated" do
+ Fabricate(:post_localization, post: post, locale: "en")
+
+ described_class.new.execute({})
+
+ expect_not_enqueued_with(job: :localize_posts)
+ end
+
+ it "enqueues post translation with pairs" do
+ described_class.new.execute({})
+
+ expect_job_enqueued(job: :localize_posts, args: { pairs: [[post.id, "en"]] })
+ end
+
+ it "distributes pairs across parallel jobs" do
+ SiteSetting.ai_translation_backfill_parallel_jobs = 2
+ post_2 = Fabricate(:post, locale: "es", topic: post.topic)
described_class.new.execute({})
- expect_job_enqueued(job: :localize_posts, args: { limit: 8 })
+ expect_job_enqueued(job: :localize_posts, args: { pairs: [[post_2.id, "en"]] })
+ expect_job_enqueued(job: :localize_posts, args: { pairs: [[post.id, "en"]] })
end
end
diff --git a/plugins/discourse-ai/spec/jobs/scheduled/topic_localization_backfill_spec.rb b/plugins/discourse-ai/spec/jobs/scheduled/topic_localization_backfill_spec.rb
index 42ca7d2cea8..a59267aa986 100644
--- a/plugins/discourse-ai/spec/jobs/scheduled/topic_localization_backfill_spec.rb
+++ b/plugins/discourse-ai/spec/jobs/scheduled/topic_localization_backfill_spec.rb
@@ -1,12 +1,16 @@
# frozen_string_literal: true
describe Jobs::TopicLocalizationBackfill do
+ fab!(:topic) { Fabricate(:topic, locale: "es") }
+
before do
enable_current_plugin
assign_fake_provider_to(:ai_default_llm_model)
SiteSetting.ai_translation_backfill_hourly_rate = 100
+ SiteSetting.ai_translation_backfill_max_age_days = 100
SiteSetting.content_localization_supported_locales = "en"
SiteSetting.ai_translation_enabled = true
+ SiteSetting.ai_translation_target_categories = topic.category_id.to_s
end
it "does not enqueue topic translation when translator disabled" do
@@ -33,11 +37,15 @@ describe Jobs::TopicLocalizationBackfill do
expect_not_enqueued_with(job: :localize_topics) { described_class.new.execute({}) }
end
- it "enqueues topic translation with correct limit" do
- SiteSetting.ai_translation_backfill_hourly_rate = 100
+ it "does not enqueue when all topics are fully translated" do
+ Fabricate(:topic_localization, topic: topic, locale: "en")
+
+ expect_not_enqueued_with(job: :localize_topics) { described_class.new.execute({}) }
+ end
+ it "enqueues topic translation with pairs" do
described_class.new.execute({})
- expect_job_enqueued(job: :localize_topics, args: { limit: 8 })
+ expect_job_enqueued(job: :localize_topics, args: { pairs: [[topic.id, "en"]] })
end
end
diff --git a/plugins/discourse-ai/spec/lib/translation/post_candidates_spec.rb b/plugins/discourse-ai/spec/lib/translation/post_candidates_spec.rb
index 2b67e1ee33b..c2c5836cb89 100644
--- a/plugins/discourse-ai/spec/lib/translation/post_candidates_spec.rb
+++ b/plugins/discourse-ai/spec/lib/translation/post_candidates_spec.rb
@@ -101,6 +101,90 @@ describe DiscourseAi::Translation::PostCandidates do
end
end
+ describe ".needs_localization" do
+ fab!(:target_category, :category)
+
+ before do
+ SiteSetting.ai_translation_backfill_max_age_days = 100
+ SiteSetting.content_localization_supported_locales = "en|ja|de"
+ SiteSetting.ai_translation_target_categories = target_category.id.to_s
+ SiteSetting.ai_translation_personal_messages = "none"
+ end
+
+ it "returns [post_id, target_locale] pairs for posts needing localization" do
+ post = Fabricate(:post, locale: "es", topic: Fabricate(:topic, category: target_category))
+
+ pairs = described_class.needs_localization(limit: 10)
+ expect(pairs).to include([post.id, "en"])
+ expect(pairs).to include([post.id, "ja"])
+ expect(pairs).to include([post.id, "de"])
+ end
+
+ it "excludes posts without a detected locale" do
+ Fabricate(:post, locale: nil, topic: Fabricate(:topic, category: target_category))
+
+ pairs = described_class.needs_localization(limit: 10)
+ expect(pairs).to be_empty
+ end
+
+ it "excludes fully translated posts" do
+ post = Fabricate(:post, locale: "es", topic: Fabricate(:topic, category: target_category))
+ Fabricate(:post_localization, post: post, locale: "en")
+ Fabricate(:post_localization, post: post, locale: "ja")
+ Fabricate(:post_localization, post: post, locale: "de")
+
+ pairs = described_class.needs_localization(limit: 10)
+ post_ids = pairs.map(&:first)
+ expect(post_ids).not_to include(post.id)
+ end
+
+ it "returns only missing locale pairs for partially translated posts" do
+ post = Fabricate(:post, locale: "es", topic: Fabricate(:topic, category: target_category))
+ Fabricate(:post_localization, post: post, locale: "en")
+
+ pairs = described_class.needs_localization(limit: 10)
+ expect(pairs).not_to include([post.id, "en"])
+ expect(pairs).to include([post.id, "ja"])
+ expect(pairs).to include([post.id, "de"])
+ end
+
+ it "excludes posts whose locale matches all target base locales" do
+ SiteSetting.content_localization_supported_locales = "en"
+ post = Fabricate(:post, locale: "en", topic: Fabricate(:topic, category: target_category))
+
+ pairs = described_class.needs_localization(limit: 10)
+ post_ids = pairs.map(&:first)
+ expect(post_ids).not_to include(post.id)
+ end
+
+ it "handles base-locale deduplication (ja_JP localization covers ja target)" do
+ post = Fabricate(:post, locale: "es", topic: Fabricate(:topic, category: target_category))
+ Fabricate(:post_localization, post: post, locale: "en")
+ Fabricate(:post_localization, post: post, locale: "ja_JP")
+ Fabricate(:post_localization, post: post, locale: "de_DE")
+
+ pairs = described_class.needs_localization(limit: 10)
+ post_ids = pairs.map(&:first)
+ expect(post_ids).not_to include(post.id)
+ end
+
+ it "respects the limit parameter" do
+ 3.times do
+ Fabricate(:post, locale: "es", topic: Fabricate(:topic, category: target_category))
+ end
+
+ pairs = described_class.needs_localization(limit: 2)
+ expect(pairs.size).to eq(2)
+ end
+
+ it "returns empty when no locales are configured" do
+ SiteSetting.content_localization_supported_locales = ""
+
+ pairs = described_class.needs_localization(limit: 10)
+ expect(pairs).to be_empty
+ end
+ end
+
describe ".get_completion_all_locales" do
fab!(:target_category, :category)
diff --git a/plugins/discourse-ai/spec/lib/translation/topic_candidates_spec.rb b/plugins/discourse-ai/spec/lib/translation/topic_candidates_spec.rb
index 0cfb22c2703..670e566ca4d 100644
--- a/plugins/discourse-ai/spec/lib/translation/topic_candidates_spec.rb
+++ b/plugins/discourse-ai/spec/lib/translation/topic_candidates_spec.rb
@@ -127,6 +127,79 @@ describe DiscourseAi::Translation::TopicCandidates do
end
end
+ describe ".needs_localization" do
+ fab!(:target_category, :category)
+
+ before do
+ SiteSetting.ai_translation_backfill_max_age_days = 100
+ SiteSetting.content_localization_supported_locales = "en|ja|de"
+ SiteSetting.ai_translation_target_categories = target_category.id.to_s
+ SiteSetting.ai_translation_personal_messages = "none"
+ end
+
+ it "returns [topic_id, target_locale] pairs for topics needing localization" do
+ topic = Fabricate(:topic, locale: "es", category: target_category)
+
+ pairs = described_class.needs_localization(limit: 10)
+ expect(pairs).to include([topic.id, "en"])
+ expect(pairs).to include([topic.id, "ja"])
+ expect(pairs).to include([topic.id, "de"])
+ end
+
+ it "excludes topics without a detected locale" do
+ Fabricate(:topic, locale: nil, category: target_category)
+
+ pairs = described_class.needs_localization(limit: 10)
+ expect(pairs).to be_empty
+ end
+
+ it "excludes fully translated topics" do
+ topic = Fabricate(:topic, locale: "es", category: target_category)
+ Fabricate(:topic_localization, topic: topic, locale: "en")
+ Fabricate(:topic_localization, topic: topic, locale: "ja")
+ Fabricate(:topic_localization, topic: topic, locale: "de")
+
+ pairs = described_class.needs_localization(limit: 10)
+ topic_ids = pairs.map(&:first)
+ expect(topic_ids).not_to include(topic.id)
+ end
+
+ it "returns only missing locale pairs for partially translated topics" do
+ topic = Fabricate(:topic, locale: "es", category: target_category)
+ Fabricate(:topic_localization, topic: topic, locale: "en")
+
+ pairs = described_class.needs_localization(limit: 10)
+ expect(pairs).not_to include([topic.id, "en"])
+ expect(pairs).to include([topic.id, "ja"])
+ expect(pairs).to include([topic.id, "de"])
+ end
+
+ it "handles base-locale deduplication (ja_JP localization covers ja target)" do
+ topic = Fabricate(:topic, locale: "es", category: target_category)
+ Fabricate(:topic_localization, topic: topic, locale: "en")
+ Fabricate(:topic_localization, topic: topic, locale: "ja_JP")
+ Fabricate(:topic_localization, topic: topic, locale: "de_DE")
+
+ pairs = described_class.needs_localization(limit: 10)
+ topic_ids = pairs.map(&:first)
+ expect(topic_ids).not_to include(topic.id)
+ end
+
+ it "respects the limit parameter" do
+ 3.times { Fabricate(:topic, locale: "es", category: target_category) }
+
+ pairs = described_class.needs_localization(limit: 2)
+ expect(pairs.size).to eq(2)
+ end
+
+ it "returns empty when no locales are configured" do
+ SiteSetting.content_localization_supported_locales = ""
+
+ pairs = described_class.needs_localization(limit: 10)
+ expect(pairs).to be_empty
+ end
+ end
+
describe ".calculate_completion_per_locale" do
before { SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|") }