Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Add parallel post localization backfill jobs (#39134)

Rafael dos Santos Silvacb8eaf61d203
diff --git a/plugins/discourse-ai/app/jobs/regular/localize_posts.rb b/plugins/discourse-ai/app/jobs/regular/localize_posts.rb
index 36d98355920..e0c5958c901 100644
--- a/plugins/discourse-ai/app/jobs/regular/localize_posts.rb
+++ b/plugins/discourse-ai/app/jobs/regular/localize_posts.rb
@@ -2,13 +2,16 @@
module Jobs
class LocalizePosts < ::Jobs::Base
- cluster_concurrency 1
sidekiq_options retry: false
+ REDIS_KEY = "discourse-ai:localize_posts:in_progress"
+
def execute(args)
limit = args[:limit]
raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
+ offset = args[:offset].to_i
+
return if !DiscourseAi::Translation.backfill_enabled?
unless DiscourseAi::Translation.credits_available_for_post_localization?
@@ -37,11 +40,13 @@ module Jobs
.where("posts.locale NOT LIKE '#{base_locale}%'")
.where("pl.id IS NULL")
.order(updated_at: :desc)
+ .offset(offset)
.limit(limit)
next if posts.empty?
posts.each do |post|
+ Discourse.redis.expire(REDIS_KEY, 15.minutes.to_i)
next unless DiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, locale)
begin
@@ -57,6 +62,9 @@ module Jobs
DiscourseAi::Translation::VerboseLogger.log("Translated #{posts.size} posts to #{locale}")
end
+ ensure
+ remaining = Discourse.redis.decr(REDIS_KEY)
+ Discourse.redis.del(REDIS_KEY) if remaining <= 0
end
private
diff --git a/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb b/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb
index 83c75cb960d..7418d97926c 100644
--- a/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb
+++ b/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb
@@ -5,9 +5,14 @@ module Jobs
every 5.minutes
cluster_concurrency 1
+ REDIS_KEY = "discourse-ai:localize_posts:in_progress"
+
def execute(args)
return if !DiscourseAi::Translation.backfill_enabled?
+ # Skip if previous batch is still running
+ return if Discourse.redis.get(REDIS_KEY).to_i > 0
+
post_raw_llm = find_llm_model_for_agent(SiteSetting.ai_translation_post_raw_translator_agent)
if post_raw_llm && !LlmCreditAllocation.credits_available?(post_raw_llm)
@@ -20,7 +25,15 @@ module Jobs
limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
return if limit == 0
- Jobs.enqueue(:localize_posts, limit:)
+ parallel_jobs = SiteSetting.ai_translation_backfill_parallel_jobs
+ per_job_limit = (limit.to_f / parallel_jobs).ceil
+
+ # Set counter with 15-min TTL safety net (if a job crashes without decrementing)
+ Discourse.redis.setex(REDIS_KEY, 15.minutes.to_i, parallel_jobs)
+
+ parallel_jobs.times do |i|
+ Jobs.enqueue(:localize_posts, limit: per_job_limit, offset: i * per_job_limit)
+ end
end
private
diff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.yml
index 946f2a58f19..fcd8ebf6f52 100644
--- a/plugins/discourse-ai/config/locales/server.en.yml
+++ b/plugins/discourse-ai/config/locales/server.en.yml
@@ -147,6 +147,7 @@ en:
ai_translation_max_post_length: "The maximum number of characters for a post to be translated. Posts longer than this will not be translated."
ai_translation_backfill_max_age_days: "The maximum age of a post and topic to be translated. Posts and topics older than this will not be translated. 0 disables backfilling, but will not disable translation of new posts."
ai_translation_backfill_hourly_rate: "The number of posts and topics to translate per hour during backfill operations. Set to 0 to disable automatic backfilling of translations for existing content."
+ ai_translation_backfill_parallel_jobs: "The number of parallel post localization jobs to run per backfill cycle. Higher values increase throughput but consume more resources."
ai_usage_rollup_after_days: "Number of days to keep per-request AI usage stats before rolling them into daily aggregates. Set to 0 to disable rollups."
ai_audit_logs_purge_after_days: "Delete AI API audit logs older than this many days. Defaults to about 6 months. Set to 0 to keep indefinitely."
diff --git a/plugins/discourse-ai/config/settings.yml b/plugins/discourse-ai/config/settings.yml
index 64204eee190..a5e53a50fb8 100644
--- a/plugins/discourse-ai/config/settings.yml
+++ b/plugins/discourse-ai/config/settings.yml
@@ -483,6 +483,13 @@ discourse_ai:
client: false
area: "ai-features/translation"
+ ai_translation_backfill_parallel_jobs:
+ default: 1
+ min: 1
+ max: 10
+ hidden: true
+ area: "ai-features/translation"
+
ai_translation_verbose_logs:
default: false
client: false