A platform for community discussion. Free, open, simple.
FEATURE: Add parallel post localization backfill jobs (#39134)
Rafael dos Santos Silvacb8eaf61d203
diff --git a/plugins/discourse-ai/app/jobs/regular/localize_posts.rb b/plugins/discourse-ai/app/jobs/regular/localize_posts.rbindex 36d98355920..e0c5958c901 100644--- a/plugins/discourse-ai/app/jobs/regular/localize_posts.rb+++ b/plugins/discourse-ai/app/jobs/regular/localize_posts.rb@@ -2,13 +2,16 @@module Jobsclass LocalizePosts < ::Jobs::Base- cluster_concurrency 1sidekiq_options retry: false+ REDIS_KEY = "discourse-ai:localize_posts:in_progress"+def execute(args)limit = args[:limit]raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0+ offset = args[:offset].to_i+return if !DiscourseAi::Translation.backfill_enabled?unless DiscourseAi::Translation.credits_available_for_post_localization?@@ -37,11 +40,13 @@ module Jobs.where("posts.locale NOT LIKE '#{base_locale}%'").where("pl.id IS NULL").order(updated_at: :desc)+ .offset(offset).limit(limit)next if posts.empty?posts.each do |post|+ Discourse.redis.expire(REDIS_KEY, 15.minutes.to_i)next unless DiscourseAi::Translation::PostLocalizer.has_relocalize_quota?(post, locale)begin@@ -57,6 +62,9 @@ module JobsDiscourseAi::Translation::VerboseLogger.log("Translated #{posts.size} posts to #{locale}")end+ ensure+ remaining = Discourse.redis.decr(REDIS_KEY)+ Discourse.redis.del(REDIS_KEY) if remaining <= 0endprivatediff --git a/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb b/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rbindex 83c75cb960d..7418d97926c 100644--- a/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb+++ b/plugins/discourse-ai/app/jobs/scheduled/post_localization_backfill.rb@@ -5,9 +5,14 @@ module Jobsevery 5.minutescluster_concurrency 1+ REDIS_KEY = "discourse-ai:localize_posts:in_progress"+def execute(args)return if !DiscourseAi::Translation.backfill_enabled?+ # Skip if previous batch is still running+ return if Discourse.redis.get(REDIS_KEY).to_i > 0+post_raw_llm = find_llm_model_for_agent(SiteSetting.ai_translation_post_raw_translator_agent)if post_raw_llm && !LlmCreditAllocation.credits_available?(post_raw_llm)@@ -20,7 +25,15 @@ module Jobslimit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervalsreturn if limit == 0- Jobs.enqueue(:localize_posts, limit:)+ parallel_jobs = SiteSetting.ai_translation_backfill_parallel_jobs+ per_job_limit = (limit.to_f / parallel_jobs).ceil++ # Set counter with 15-min TTL safety net (if a job crashes without decrementing)+ Discourse.redis.setex(REDIS_KEY, 15.minutes.to_i, parallel_jobs)++ parallel_jobs.times do |i|+ Jobs.enqueue(:localize_posts, limit: per_job_limit, offset: i * per_job_limit)+ endendprivatediff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.ymlindex 946f2a58f19..fcd8ebf6f52 100644--- a/plugins/discourse-ai/config/locales/server.en.yml+++ b/plugins/discourse-ai/config/locales/server.en.yml@@ -147,6 +147,7 @@ en:ai_translation_max_post_length: "The maximum number of characters for a post to be translated. Posts longer than this will not be translated."ai_translation_backfill_max_age_days: "The maximum age of a post and topic to be translated. Posts and topics older than this will not be translated. 0 disables backfilling, but will not disable translation of new posts."ai_translation_backfill_hourly_rate: "The number of posts and topics to translate per hour during backfill operations. Set to 0 to disable automatic backfilling of translations for existing content."+ ai_translation_backfill_parallel_jobs: "The number of parallel post localization jobs to run per backfill cycle. Higher values increase throughput but consume more resources."ai_usage_rollup_after_days: "Number of days to keep per-request AI usage stats before rolling them into daily aggregates. Set to 0 to disable rollups."ai_audit_logs_purge_after_days: "Delete AI API audit logs older than this many days. Defaults to about 6 months. Set to 0 to keep indefinitely."diff --git a/plugins/discourse-ai/config/settings.yml b/plugins/discourse-ai/config/settings.ymlindex 64204eee190..a5e53a50fb8 100644--- a/plugins/discourse-ai/config/settings.yml+++ b/plugins/discourse-ai/config/settings.yml@@ -483,6 +483,13 @@ discourse_ai:client: falsearea: "ai-features/translation"+ ai_translation_backfill_parallel_jobs:+ default: 1+ min: 1+ max: 10+ hidden: true+ area: "ai-features/translation"+ai_translation_verbose_logs:default: falseclient: false