A platform for community discussion. Free, open, simple.
FEATURE: Expand AI translation settings to allow choosing specific categories instead of just 'all public content' (#39000)
Natalie Tay1f4d83405f2d
diff --git a/plugins/discourse-ai/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-features/edit.gjs b/plugins/discourse-ai/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-features/edit.gjsindex 8a46ac471cd..cb4c5044532 100644--- a/plugins/discourse-ai/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-features/edit.gjs+++ b/plugins/discourse-ai/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-features/edit.gjs@@ -50,7 +50,11 @@ export default <template>(if(eq setting.type "enum")"select"- (if (eq setting.type "list") "custom" "input")+ (if+ (eq setting.type "category_list")+ "custom"+ (if (eq setting.type "list") "custom" "input")+ )))}}diff --git a/plugins/discourse-ai/app/jobs/regular/detect_translate_post.rb b/plugins/discourse-ai/app/jobs/regular/detect_translate_post.rbindex b6d6942a244..b6933871226 100644--- a/plugins/discourse-ai/app/jobs/regular/detect_translate_post.rb+++ b/plugins/discourse-ai/app/jobs/regular/detect_translate_post.rb@@ -27,13 +27,19 @@ module Jobsif force# no restrictions- elsif SiteSetting.ai_translation_backfill_limit_to_public_content- return if topic.category&.read_restricted? || topic.archetype == Archetype.private_message- else- if topic.archetype == Archetype.private_message &&- !TopicAllowedGroup.exists?(topic_id: topic.id)+ elsif topic.archetype == Archetype.private_message+ case SiteSetting.ai_translation_personal_messages+ when "all"+ # allow+ when "group"+ return unless TopicAllowedGroup.exists?(topic_id: topic.id)+ elsereturnend+ else+ target_category_ids = SiteSetting.ai_translation_target_categories+ return if target_category_ids.blank?+ return if target_category_ids.split("|").map(&:to_i).exclude?(topic.category_id)end# the user may fill locale in manuallydiff --git a/plugins/discourse-ai/app/jobs/regular/detect_translate_topic.rb b/plugins/discourse-ai/app/jobs/regular/detect_translate_topic.rbindex 29d246adf15..48b727c67fc 100644--- a/plugins/discourse-ai/app/jobs/regular/detect_translate_topic.rb+++ b/plugins/discourse-ai/app/jobs/regular/detect_translate_topic.rb@@ -24,13 +24,19 @@ module Jobsif force# no restrictions- elsif SiteSetting.ai_translation_backfill_limit_to_public_content- return if topic.category&.read_restricted? || topic.archetype == Archetype.private_message- else- if topic.archetype == Archetype.private_message &&- !TopicAllowedGroup.exists?(topic_id: topic.id)+ elsif topic.archetype == Archetype.private_message+ case SiteSetting.ai_translation_personal_messages+ when "all"+ # allow+ when "group"+ return unless TopicAllowedGroup.exists?(topic_id: topic.id)+ elsereturnend+ else+ target_category_ids = SiteSetting.ai_translation_target_categories+ return if target_category_ids.blank?+ return if target_category_ids.split("|").map(&:to_i).exclude?(topic.category_id)endif (detected_locale = topic.locale).blank?diff --git a/plugins/discourse-ai/app/jobs/scheduled/categories_locale_detection_backfill.rb b/plugins/discourse-ai/app/jobs/scheduled/categories_locale_detection_backfill.rbindex ae271b06a79..7331b193fa0 100644--- a/plugins/discourse-ai/app/jobs/scheduled/categories_locale_detection_backfill.rb+++ b/plugins/discourse-ai/app/jobs/scheduled/categories_locale_detection_backfill.rb@@ -19,10 +19,12 @@ module Jobsreturnend- categories = Category.where(locale: nil)-- if SiteSetting.ai_translation_backfill_limit_to_public_content- categories = categories.where(read_restricted: false)+ target_category_ids = SiteSetting.ai_translation_target_categories+ if target_category_ids.present?+ categories =+ Category.where(locale: nil).where(id: target_category_ids.split("|").map(&:to_i))+ else+ returnendlimit = SiteSetting.ai_translation_backfill_hourly_ratediff --git a/plugins/discourse-ai/assets/javascripts/discourse/components/ai-feature-setting-field.gjs b/plugins/discourse-ai/assets/javascripts/discourse/components/ai-feature-setting-field.gjsindex de22d8ccd46..590e140863d 100644--- a/plugins/discourse-ai/assets/javascripts/discourse/components/ai-feature-setting-field.gjs+++ b/plugins/discourse-ai/assets/javascripts/discourse/components/ai-feature-setting-field.gjs@@ -1,11 +1,18 @@import Component from "@glimmer/component";+import { tracked } from "@glimmer/tracking";+import { action } from "@ember/object";+import didUpdate from "@ember/render-modifiers/modifiers/did-update";import { service } from "@ember/service";+import Category from "discourse/models/category";+import CategorySelector from "discourse/select-kit/components/category-selector";import ListSetting from "discourse/select-kit/components/list-setting";import { eq } from "discourse/truth-helpers";export default class AiFeatureSettingField extends Component {@service site;+ @tracked selectedCategories = [];+parseList = (value) => {return value?.toString().split("|").filter(Boolean) || [];};@@ -16,6 +23,13 @@ export default class AiFeatureSettingField extends Component {};};+ constructor() {+ super(...arguments);+ if (this.controlType === "category_list") {+ this.#loadCategories();+ }+ }+get groupChoices() {return (this.site.groups || []).map((g) => ({name: g.name,@@ -23,6 +37,28 @@ export default class AiFeatureSettingField extends Component {}));}+ async #loadCategories() {+ const ids = this.#categoryIds;+ if (ids.length) {+ this.selectedCategories = await Category.asyncFindByIds(ids);+ }+ }++ get #categoryIds() {+ return (this.args.field?.value || "").toString().split("|").filter(Boolean);+ }++ @action+ categoryValueChanged() {+ this.#loadCategories();+ }++ @action+ onChangeCategories(categories) {+ const value = (categories || []).map((c) => c.id).join("|");+ this.args.field.set(value);+ }+get hasEnumChoices() {return this.args.setting.valid_values?.length > 0;}@@ -61,6 +97,15 @@ export default class AiFeatureSettingField extends Component {</select.Option>{{/each}}</@Control>+ {{else if (eq this.controlType "category_list")}}+ <@Control>+ <div {{didUpdate this.categoryValueChanged @field.value}}>+ <CategorySelector+ @categories={{this.selectedCategories}}+ @onChange={{this.onChangeCategories}}+ />+ </div>+ </@Control>{{else if (eq this.controlType "group_list")}}<@Control><ListSettingdiff --git a/plugins/discourse-ai/assets/javascripts/discourse/lib/ai-feature-setting-groups.js b/plugins/discourse-ai/assets/javascripts/discourse/lib/ai-feature-setting-groups.jsindex 11c232ffd23..5729d4c15c1 100644--- a/plugins/discourse-ai/assets/javascripts/discourse/lib/ai-feature-setting-groups.js+++ b/plugins/discourse-ai/assets/javascripts/discourse/lib/ai-feature-setting-groups.js@@ -166,7 +166,8 @@ export const AI_FEATURE_SETTING_GROUPS = {"discourse_ai.features.translation.setting_groups.backfill_and_limits",settings: ["ai_translation_backfill_hourly_rate",- "ai_translation_backfill_limit_to_public_content",+ "ai_translation_target_categories",+ "ai_translation_personal_messages","ai_translation_include_bot_content","ai_translation_max_post_length","ai_translation_backfill_max_age_days",diff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.ymlindex e3f3a27b471..1123fb75b71 100644--- a/plugins/discourse-ai/config/locales/server.en.yml+++ b/plugins/discourse-ai/config/locales/server.en.yml@@ -141,7 +141,8 @@ en:ai_discord_allowed_guilds: "Discord guilds (servers) where the bot is allowed to search"ai_translation_enabled: "Enable the AI translation feature"- ai_translation_backfill_limit_to_public_content: "When enabled, only content in public categories will be translated. When disabled, content in group PMs and private categories will also be sent for translation."+ ai_translation_target_categories: "Only content in the selected categories will be translated. Subcategories must be added separately. Leave empty to disable translation of topics in categories."+ ai_translation_personal_messages: "Controls which personal messages are translated. 'none' disables PM translation. 'group' translates only group PMs. 'all' translates all PMs."ai_translation_include_bot_content: "When enabled, content written by bots (user IDs below 0) will also be sent for translation."ai_translation_max_post_length: "The maximum number of characters for a post to be translated. Posts longer than this will not be translated."ai_translation_backfill_max_age_days: "The maximum age of a post and topic to be translated. Posts and topics older than this will not be translated. 0 disables backfilling, but will not disable translation of new posts."diff --git a/plugins/discourse-ai/config/settings.yml b/plugins/discourse-ai/config/settings.ymlindex 05583d9cd40..64204eee190 100644--- a/plugins/discourse-ai/config/settings.yml+++ b/plugins/discourse-ai/config/settings.yml@@ -454,8 +454,18 @@ discourse_ai:max: 100000client: falsearea: "ai-features/translation"- ai_translation_backfill_limit_to_public_content:- default: true+ ai_translation_target_categories:+ type: category_list+ default: ""+ client: false+ area: "ai-features/translation"+ ai_translation_personal_messages:+ type: enum+ default: "none"+ choices:+ - "none"+ - "group"+ - "all"client: falsearea: "ai-features/translation"ai_translation_include_bot_content:diff --git a/plugins/discourse-ai/db/migrate/20260331024139_split_translation_public_content_setting.rb b/plugins/discourse-ai/db/migrate/20260331024139_split_translation_public_content_setting.rbnew file mode 100644index 00000000000..067849d7a24--- /dev/null+++ b/plugins/discourse-ai/db/migrate/20260331024139_split_translation_public_content_setting.rb@@ -0,0 +1,43 @@+# frozen_string_literal: true++class SplitTranslationPublicContentSetting < ActiveRecord::Migration[7.2]+ def up+ limit_to_public =+ DB.query_single(+ "SELECT value FROM site_settings WHERE name = 'ai_translation_backfill_limit_to_public_content' LIMIT 1",+ ).first++ enabled =+ DB.query_single(+ "SELECT value FROM site_settings WHERE name = 'ai_translation_enabled' LIMIT 1",+ ).first++ if limit_to_public == "f"+ all_category_ids = DB.query_single("SELECT id FROM categories")+ execute <<~SQL if all_category_ids.present?+ INSERT INTO site_settings (name, data_type, value, created_at, updated_at)+ VALUES ('ai_translation_target_categories', 20, '#{all_category_ids.join("|")}', NOW(), NOW())+ SQL++ execute <<~SQL+ INSERT INTO site_settings (name, data_type, value, created_at, updated_at)+ VALUES ('ai_translation_personal_messages', 7, 'all', NOW(), NOW())+ SQL+ elsif enabled == "t"+ public_category_ids =+ DB.query_single("SELECT id FROM categories WHERE read_restricted = false")+ execute <<~SQL if public_category_ids.present?+ INSERT INTO site_settings (name, data_type, value, created_at, updated_at)+ VALUES ('ai_translation_target_categories', 20, '#{public_category_ids.join("|")}', NOW(), NOW())+ SQL+ end++ execute <<~SQL+ DELETE FROM site_settings WHERE name = 'ai_translation_backfill_limit_to_public_content'+ SQL+ end++ def down+ raise ActiveRecord::IrreversibleMigration+ end+enddiff --git a/plugins/discourse-ai/lib/translation/category_candidates.rb b/plugins/discourse-ai/lib/translation/category_candidates.rbindex af1fc7bbda5..0980197f804 100644--- a/plugins/discourse-ai/lib/translation/category_candidates.rb+++ b/plugins/discourse-ai/lib/translation/category_candidates.rb@@ -8,11 +8,12 @@ module DiscourseAi# all categories that are eligible for translation based on site settings,# including those without locale detected yet.def self.get- categories = Category.all- if SiteSetting.ai_translation_backfill_limit_to_public_content- categories = categories.where(read_restricted: false)+ target_category_ids = SiteSetting.ai_translation_target_categories+ if target_category_ids.present?+ Category.where(id: target_category_ids.split("|").map(&:to_i))+ else+ Category.noneend- categoriesenddef self.calculate_completion_per_locale(locale)diff --git a/plugins/discourse-ai/lib/translation/post_candidates.rb b/plugins/discourse-ai/lib/translation/post_candidates.rbindex 1bcc36b7ce7..4a9606c5e78 100644--- a/plugins/discourse-ai/lib/translation/post_candidates.rb+++ b/plugins/discourse-ai/lib/translation/post_candidates.rb@@ -30,21 +30,36 @@ module DiscourseAiposts.where("posts.user_id > 0") unless SiteSetting.ai_translation_include_bot_contentposts = posts.joins(:topic)- if SiteSetting.ai_translation_backfill_limit_to_public_content- # exclude all PMs- # and only include posts from public categories++ target_category_ids = SiteSetting.ai_translation_target_categories+ pm_scope = SiteSetting.ai_translation_personal_messages++ # Category filter: include target categories + PMs (PMs filtered in next step)+ if target_category_ids.present?+ category_ids = target_category_ids.split("|").map(&:to_i)posts =- posts- .where.not(topics: { archetype: Archetype.private_message })- .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })+ posts.where(+ "topics.category_id IN (:cats) OR topics.archetype = :pm",+ cats: category_ids,+ pm: Archetype.private_message,+ )else- # all regular topics, and group PMs+ posts = posts.where(topics: { archetype: Archetype.private_message })+ end++ # PM scope filter+ case pm_scope+ when "group"posts =posts.where(- "topics.archetype != ? OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",- Archetype.private_message,+ "topics.archetype != :pm OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",+ pm: Archetype.private_message,)+ when "none", nil+ posts = posts.where.not(topics: { archetype: Archetype.private_message })end++ postsenddef self.completion_all_localesdiff --git a/plugins/discourse-ai/lib/translation/tag_candidates.rb b/plugins/discourse-ai/lib/translation/tag_candidates.rbindex 37d387a0de0..353b7677919 100644--- a/plugins/discourse-ai/lib/translation/tag_candidates.rb+++ b/plugins/discourse-ai/lib/translation/tag_candidates.rb@@ -8,27 +8,7 @@ module DiscourseAi# all tags that are eligible for translation based on site settings,# including those without locale detected yet.def self.get- tags = Tag.all- if SiteSetting.ai_translation_backfill_limit_to_public_content- tags = filter_to_public_tags(tags)- end- tags- end-- def self.filter_to_public_tags(tags)- # tags visible to everyone are:- # 1. not in any tag group, OR- # 2. in a tag group that grants permission to EVERYONE- everyone_group_id = Group::AUTO_GROUPS[:everyone]- tags.where(<<~SQL, everyone_group_id)- tags.id NOT IN (SELECT tag_id FROM tag_group_memberships)- OR tags.id IN (- SELECT tgm.tag_id- FROM tag_group_memberships tgm- INNER JOIN tag_group_permissions tgp ON tgp.tag_group_id = tgm.tag_group_id- WHERE tgp.group_id = ?- )- SQL+ Tag.allenddef self.calculate_completion_per_locale(locale)diff --git a/plugins/discourse-ai/lib/translation/topic_candidates.rb b/plugins/discourse-ai/lib/translation/topic_candidates.rbindex 67447826f5a..edf1dc736f8 100644--- a/plugins/discourse-ai/lib/translation/topic_candidates.rb+++ b/plugins/discourse-ai/lib/translation/topic_candidates.rb@@ -17,21 +17,35 @@ module DiscourseAitopics =topics.where("topics.user_id > 0") unless SiteSetting.ai_translation_include_bot_content- if SiteSetting.ai_translation_backfill_limit_to_public_content- # exclude all PMs- # and only include topics from public categories+ target_category_ids = SiteSetting.ai_translation_target_categories+ pm_scope = SiteSetting.ai_translation_personal_messages++ # Category filter: include target categories + PMs (PMs filtered in next step)+ if target_category_ids.present?+ category_ids = target_category_ids.split("|").map(&:to_i)topics =- topics- .where.not(archetype: Archetype.private_message)- .where(category_id: Category.where(read_restricted: false).select(:id))+ topics.where(+ "topics.category_id IN (:cats) OR topics.archetype = :pm",+ cats: category_ids,+ pm: Archetype.private_message,+ )else- # all regular topics, and group PMs+ topics = topics.where(archetype: Archetype.private_message)+ end++ # PM scope filter+ case pm_scope+ when "group"topics =topics.where(- "topics.archetype != ? OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",- Archetype.private_message,+ "topics.archetype != :pm OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",+ pm: Archetype.private_message,)+ when "none", nil+ topics = topics.where.not(archetype: Archetype.private_message)end++ topicsenddef self.calculate_completion_per_locale(locale)diff --git a/plugins/discourse-ai/spec/db/migrate/20260331024139_split_translation_public_content_setting_spec.rb b/plugins/discourse-ai/spec/db/migrate/20260331024139_split_translation_public_content_setting_spec.rbnew file mode 100644index 00000000000..ff995759b7d--- /dev/null+++ b/plugins/discourse-ai/spec/db/migrate/20260331024139_split_translation_public_content_setting_spec.rb@@ -0,0 +1,103 @@+# frozen_string_literal: true++require Rails.root.join(+ "plugins/discourse-ai/db/migrate/20260331024139_split_translation_public_content_setting",+ )++RSpec.describe SplitTranslationPublicContentSetting do+ subject(:migration) { described_class.new }++ let(:connection) { ActiveRecord::Base.connection }++ def store_setting(name, val, data_type: 3)+ connection.execute <<~SQL+ INSERT INTO site_settings(name, data_type, value, created_at, updated_at)+ VALUES ('#{name}', #{data_type}, '#{val}', NOW(), NOW())+ SQL+ end++ def setting_value(name)+ DB.query_single("SELECT value FROM site_settings WHERE name = '#{name}'").first+ end++ def setting_exists?(name)+ DB.query_single("SELECT COUNT(*) FROM site_settings WHERE name = '#{name}'").first > 0+ end++ fab!(:public_category) { Fabricate(:category, read_restricted: false) }+ fab!(:private_category) { Fabricate(:category, read_restricted: true) }++ before { enable_current_plugin }++ after do+ connection.execute(+ "DELETE FROM site_settings WHERE name IN ('ai_translation_target_categories', 'ai_translation_personal_messages', 'ai_translation_backfill_limit_to_public_content', 'ai_translation_enabled')",+ )+ end++ describe "#up" do+ context "when old setting was false and translation is enabled" do+ before do+ store_setting("ai_translation_enabled", "t")+ store_setting("ai_translation_backfill_limit_to_public_content", "f")+ end++ it "sets target_categories to all categories and private_messages to all" do+ migration.up++ category_ids = setting_value("ai_translation_target_categories").split("|").map(&:to_i)+ expect(category_ids).to include(public_category.id, private_category.id)+ expect(setting_value("ai_translation_personal_messages")).to eq("all")+ expect(setting_exists?("ai_translation_backfill_limit_to_public_content")).to eq(false)+ end+ end++ context "when old setting was false and translation is disabled" do+ before { store_setting("ai_translation_backfill_limit_to_public_content", "f") }++ it "still migrates settings and deletes the old setting" do+ migration.up++ category_ids = setting_value("ai_translation_target_categories").split("|").map(&:to_i)+ expect(category_ids).to include(public_category.id, private_category.id)+ expect(setting_value("ai_translation_personal_messages")).to eq("all")+ expect(setting_exists?("ai_translation_backfill_limit_to_public_content")).to eq(false)+ end+ end++ context "when old setting was default (absent) and translation is enabled" do+ before { store_setting("ai_translation_enabled", "t") }++ it "sets target_categories to public categories only" do+ migration.up++ category_ids = setting_value("ai_translation_target_categories").split("|").map(&:to_i)+ expect(category_ids).to include(public_category.id)+ expect(category_ids).not_to include(private_category.id)+ expect(setting_value("ai_translation_personal_messages")).to be_nil+ expect(setting_exists?("ai_translation_backfill_limit_to_public_content")).to eq(false)+ end+ end++ context "when translation was never used" do+ it "inserts nothing and deletes the old setting if present" do+ store_setting("ai_translation_backfill_limit_to_public_content", "t")++ migration.up++ expect(setting_value("ai_translation_target_categories")).to be_nil+ expect(setting_value("ai_translation_personal_messages")).to be_nil+ expect(setting_exists?("ai_translation_backfill_limit_to_public_content")).to eq(false)+ end+ end++ context "when no settings exist at all" do+ it "does nothing" do+ migration.up++ expect(setting_value("ai_translation_target_categories")).to be_nil+ expect(setting_value("ai_translation_personal_messages")).to be_nil+ end+ end+ end+enddiff --git a/plugins/discourse-ai/spec/jobs/regular/detect_translate_post_spec.rb b/plugins/discourse-ai/spec/jobs/regular/detect_translate_post_spec.rbindex cfca233dfc6..145d1f38641 100644--- a/plugins/discourse-ai/spec/jobs/regular/detect_translate_post_spec.rb+++ b/plugins/discourse-ai/spec/jobs/regular/detect_translate_post_spec.rb@@ -18,6 +18,7 @@ describe Jobs::DetectTranslatePost doenable_current_pluginSiteSetting.ai_translation_enabled = trueSiteSetting.content_localization_supported_locales = locales.join("|")+ SiteSetting.ai_translation_target_categories = post.topic.category_id.to_sendit "does nothing when translator is disabled" do@@ -143,10 +144,13 @@ describe Jobs::DetectTranslatePost doexpect { job.execute({ post_id: post.id }) }.not_to raise_errorend- describe "with public content and PM limitations" do- fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) }- fab!(:private_topic) { Fabricate(:topic, category: private_category) }- fab!(:private_post) { Fabricate(:post, topic: private_topic) }+ describe "with target categories and PM scope" do+ fab!(:target_category, :category)+ fab!(:non_target_category, :category)+ fab!(:target_topic) { Fabricate(:topic, category: target_category) }+ fab!(:non_target_topic) { Fabricate(:topic, category: non_target_category) }+ fab!(:target_post) { Fabricate(:post, topic: target_topic) }+ fab!(:non_target_post) { Fabricate(:post, topic: non_target_topic) }fab!(:personal_pm_topic, :private_message_topic)fab!(:personal_pm_post) { Fabricate(:post, topic: personal_pm_topic) }@@ -156,49 +160,43 @@ describe Jobs::DetectTranslatePost doendfab!(:group_pm_post) { Fabricate(:post, topic: group_pm_topic) }- context "when ai_translation_backfill_limit_to_public_content is true" do- before { SiteSetting.ai_translation_backfill_limit_to_public_content = true }+ before { SiteSetting.ai_translation_target_categories = target_category.id.to_s }- it "skips posts from restricted categories and PMs" do- DiscourseAi::Translation::PostLocaleDetector- .expects(:detect_locale)- .with(private_post)- .never- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(private_post, any_parameters)- .never- job.execute({ post_id: private_post.id })+ it "skips posts not in target categories" do+ DiscourseAi::Translation::PostLocaleDetector+ .expects(:detect_locale)+ .with(non_target_post)+ .never+ job.execute({ post_id: non_target_post.id })+ end+ it "processes posts in target categories" do+ DiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(target_post).once+ job.execute({ post_id: target_post.id })+ end++ context "when pm_translation_scope is none" do+ before { SiteSetting.ai_translation_personal_messages = "none" }++ it "skips all PMs" doDiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(personal_pm_post).never- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(personal_pm_post, any_parameters)- .neverjob.execute({ post_id: personal_pm_post.id })DiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(group_pm_post).never- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(group_pm_post, any_parameters)- .neverjob.execute({ post_id: group_pm_post.id })endend- context "when ai_translation_backfill_limit_to_public_content is false" do- before { SiteSetting.ai_translation_backfill_limit_to_public_content = false }-- it "processes posts from private categories and group PMs but skips personal PMs" do- DiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(private_post).once- job.execute({ post_id: private_post.id })+ context "when pm_translation_scope is group" do+ before { SiteSetting.ai_translation_personal_messages = "group" }+ it "processes group PMs but skips personal PMs" doDiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(group_pm_post)@@ -209,31 +207,45 @@ describe Jobs::DetectTranslatePost do.expects(:detect_locale).with(personal_pm_post).never- DiscourseAi::Translation::PostLocalizer- .expects(:localize)- .with(personal_pm_post, any_parameters)- .neverjob.execute({ post_id: personal_pm_post.id })end+ end- describe "force arg" do- it "processes private content when force is true" do- DiscourseAi::Translation::PostLocaleDetector- .expects(:detect_locale)- .with(group_pm_post)- .once+ context "when pm_translation_scope is all" do+ before { SiteSetting.ai_translation_personal_messages = "all" }- job.execute({ post_id: group_pm_post.id, force: true })- end+ it "processes all PMs" do+ DiscourseAi::Translation::PostLocaleDetector+ .expects(:detect_locale)+ .with(group_pm_post)+ .once+ job.execute({ post_id: group_pm_post.id })- it "processes PM content when force is true" do- DiscourseAi::Translation::PostLocaleDetector- .expects(:detect_locale)- .with(personal_pm_post)- .once+ DiscourseAi::Translation::PostLocaleDetector+ .expects(:detect_locale)+ .with(personal_pm_post)+ .once+ job.execute({ post_id: personal_pm_post.id })+ end+ end++ describe "force arg" do+ it "processes private content when force is true" do+ DiscourseAi::Translation::PostLocaleDetector+ .expects(:detect_locale)+ .with(group_pm_post)+ .once++ job.execute({ post_id: group_pm_post.id, force: true })+ end++ it "processes PM content when force is true" do+ DiscourseAi::Translation::PostLocaleDetector+ .expects(:detect_locale)+ .with(personal_pm_post)+ .once- job.execute({ post_id: personal_pm_post.id, force: true })- end+ job.execute({ post_id: personal_pm_post.id, force: true })endendenddiff --git a/plugins/discourse-ai/spec/jobs/regular/detect_translate_topic_spec.rb b/plugins/discourse-ai/spec/jobs/regular/detect_translate_topic_spec.rbindex 55995ecd956..06a2c8da6cb 100644--- a/plugins/discourse-ai/spec/jobs/regular/detect_translate_topic_spec.rb+++ b/plugins/discourse-ai/spec/jobs/regular/detect_translate_topic_spec.rb@@ -12,6 +12,7 @@ describe Jobs::DetectTranslateTopic doenable_current_pluginSiteSetting.ai_translation_enabled = trueSiteSetting.content_localization_supported_locales = locales.join("|")+ SiteSetting.ai_translation_target_categories = topic.category_id.to_sendit "does nothing when translator is disabled" do@@ -138,9 +139,11 @@ describe Jobs::DetectTranslateTopic doexpect { job.execute({ topic_id: topic.id }) }.not_to raise_errorend- describe "with public content and PM limitations" do- fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) }- fab!(:private_topic) { Fabricate(:topic, category: private_category) }+ describe "with target categories and PM scope" do+ fab!(:target_category, :category)+ fab!(:non_target_category, :category)+ fab!(:target_topic) { Fabricate(:topic, category: target_category) }+ fab!(:non_target_topic) { Fabricate(:topic, category: non_target_category) }fab!(:personal_pm_topic, :private_message_topic)@@ -148,54 +151,71 @@ describe Jobs::DetectTranslateTopic doFabricate(:group_private_message_topic, recipient_group: Fabricate(:group))end- context "when ai_translation_backfill_limit_to_public_content is true" do- before { SiteSetting.ai_translation_backfill_limit_to_public_content = true }+ before { SiteSetting.ai_translation_target_categories = target_category.id.to_s }- it "skips topics from restricted categories and PMs" do- DiscourseAi::Translation::TopicLocaleDetector- .expects(:detect_locale)- .with(private_topic)- .never- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(private_topic, any_parameters)- .never- job.execute({ topic_id: private_topic.id })+ it "skips topics not in target categories" do+ DiscourseAi::Translation::TopicLocaleDetector+ .expects(:detect_locale)+ .with(non_target_topic)+ .never++ job.execute({ topic_id: non_target_topic.id })+ end++ it "processes topics in target categories" do+ DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(target_topic).once++ job.execute({ topic_id: target_topic.id })+ end++ it "skips topics when target_categories is empty" do+ SiteSetting.ai_translation_target_categories = ""++ DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(target_topic).never++ job.execute({ topic_id: target_topic.id })+ end++ context "when pm_translation_scope is none" do+ before { SiteSetting.ai_translation_personal_messages = "none" }- # Skip personal PMs+ it "skips all PMs" doDiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(personal_pm_topic).never- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(personal_pm_topic, any_parameters)- .neverjob.execute({ topic_id: personal_pm_topic.id })DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(group_pm_topic).never- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(group_pm_topic, any_parameters)- .never-job.execute({ topic_id: group_pm_topic.id })endend- context "when ai_translation_backfill_limit_to_public_content is false" do- before { SiteSetting.ai_translation_backfill_limit_to_public_content = false }+ context "when pm_translation_scope is group" do+ before { SiteSetting.ai_translation_personal_messages = "group" }- it "processes topics from private categories and group PMs but skips personal PMs" do+ it "processes group PMs but skips personal PMs" doDiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale)- .with(private_topic)+ .with(group_pm_topic).once- job.execute({ topic_id: private_topic.id })+ job.execute({ topic_id: group_pm_topic.id })++ DiscourseAi::Translation::TopicLocaleDetector+ .expects(:detect_locale)+ .with(personal_pm_topic)+ .never+ job.execute({ topic_id: personal_pm_topic.id })+ end+ end++ context "when pm_translation_scope is all" do+ before { SiteSetting.ai_translation_personal_messages = "all" }+ it "processes all PMs" doDiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(group_pm_topic)@@ -205,11 +225,7 @@ describe Jobs::DetectTranslateTopic doDiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(personal_pm_topic)- .never- DiscourseAi::Translation::TopicLocalizer- .expects(:localize)- .with(personal_pm_topic, any_parameters)- .never+ .oncejob.execute({ topic_id: personal_pm_topic.id })endend@@ -235,6 +251,8 @@ describe Jobs::DetectTranslateTopic doendit "publishes a MessageBus event to update the topic" do+ SiteSetting.ai_translation_personal_messages = "all"+allow(DiscourseAi::Translation::TopicLocaleDetector).to receive(:detect_locale).with(group_pm_topic,).and_return("en")diff --git a/plugins/discourse-ai/spec/jobs/regular/localize_categories_spec.rb b/plugins/discourse-ai/spec/jobs/regular/localize_categories_spec.rbindex 34f44b360ec..f7769e9b9a6 100644--- a/plugins/discourse-ai/spec/jobs/regular/localize_categories_spec.rb+++ b/plugins/discourse-ai/spec/jobs/regular/localize_categories_spec.rb@@ -66,6 +66,7 @@ describe Jobs::LocalizeCategories doendit "translates categories to the configured locales" do+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")Category.update_all(locale: "en")number_of_categories = Category.count@@ -94,6 +95,7 @@ describe Jobs::LocalizeCategories doSiteSetting.content_localization_supported_locales = "pt"6.times { Fabricate(:category) }+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")Category.update_all(locale: "en")DiscourseAi::Translation::CategoryLocalizer@@ -109,6 +111,7 @@ describe Jobs::LocalizeCategories doendit "skips categories that already have localizations" do+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")localize_all_categories("pt", "zh_CN")DiscourseAi::Translation::CategoryLocalizer@@ -132,9 +135,13 @@ describe Jobs::LocalizeCategories doendit "handles translation errors gracefully" do+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")localize_all_categories("pt", "zh_CN")category1 = Fabricate(:category, name: "First", description: "First description", locale: "en")+ SiteSetting.ai_translation_target_categories =+ (SiteSetting.ai_translation_target_categories.split("|") + [category1.id.to_s]).join("|")+DiscourseAi::Translation::CategoryLocalizer.expects(:localize).with(@@ -156,25 +163,25 @@ describe Jobs::LocalizeCategories doexpect { job.execute({ limit: 10 }) }.not_to raise_errorend- it "skips read-restricted categories when configured" do- SiteSetting.ai_translation_backfill_limit_to_public_content = true-- category1 = Fabricate(:category, name: "Public Category", read_restricted: false, locale: "en")- category2 = Fabricate(:category, name: "Private Category", read_restricted: true, locale: "en")+ it "only translates categories in target_categories" do+ target = Fabricate(:category, locale: "en")+ non_target = Fabricate(:category, locale: "en")+ SiteSetting.ai_translation_target_categories = target.id.to_sDiscourseAi::Translation::CategoryLocalizer.expects(:localize)- .with(category1, any_parameters)+ .with(target, any_parameters).twiceDiscourseAi::Translation::CategoryLocalizer.expects(:localize)- .with(category2, any_parameters)+ .with(non_target, any_parameters).neverjob.execute({ limit: 10 })endit "skips creating localizations in the same language as the category's locale" do+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")Category.update_all(locale: "pt")DiscourseAi::Translation::CategoryLocalizer@@ -198,6 +205,7 @@ describe Jobs::LocalizeCategories doendit "deletes existing localizations that match the category's locale" do+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")# update all categories to portugueseCategory.update_all(locale: "pt")@@ -211,6 +219,7 @@ describe Jobs::LocalizeCategories doit "doesn't process categories with nil locale" do# Add a category with nil localenil_locale_category = Fabricate(:category, name: "No Locale", locale: nil)+ SiteSetting.ai_translation_target_categories = nil_locale_category.id.to_s# Make sure our query for categories with non-null locales excludes itDiscourseAi::Translation::CategoryLocalizerdiff --git a/plugins/discourse-ai/spec/jobs/regular/localize_posts_spec.rb b/plugins/discourse-ai/spec/jobs/regular/localize_posts_spec.rbindex 11769e16547..ef7c7413a5e 100644--- a/plugins/discourse-ai/spec/jobs/regular/localize_posts_spec.rb+++ b/plugins/discourse-ai/spec/jobs/regular/localize_posts_spec.rb@@ -14,6 +14,7 @@ describe Jobs::LocalizePosts doSiteSetting.content_localization_supported_locales = locales.join("|")SiteSetting.ai_translation_backfill_hourly_rate = 100SiteSetting.ai_translation_backfill_max_age_days = 100+ SiteSetting.ai_translation_target_categories = post.topic.category_id.to_sendit "does nothing when translator is disabled" do@@ -101,7 +102,7 @@ describe Jobs::LocalizePosts doendit "scenario 2: localizes post with locale 'es' when localizations for en/ja/de do not exist" do- post = Fabricate(:post, locale: "es")+ post = Fabricate(:post, locale: "es", topic: self.post.topic)DiscourseAi::Translation::PostLocalizer.expects(:localize)@@ -120,7 +121,7 @@ describe Jobs::LocalizePosts doendit "scenario 3: localizes post with locale 'en' when ja/de localization do not exist" do- post = Fabricate(:post, locale: "en")+ post = Fabricate(:post, locale: "en", topic: self.post.topic)DiscourseAi::Translation::PostLocalizer.expects(:localize)@@ -136,7 +137,7 @@ describe Jobs::LocalizePosts doendit "scenario 4: skips post with locale 'en' if all localizations exist" do- post = Fabricate(:post, locale: "en")+ post = Fabricate(:post, locale: "en", topic: self.post.topic)Fabricate(:post_localization, post: post, locale: "ja")Fabricate(:post_localization, post: post, locale: "de")@@ -146,7 +147,7 @@ describe Jobs::LocalizePosts doendit "scenario 5: skips posts that already have localizations in similar language variant" do- post = Fabricate(:post, locale: "en")+ post = Fabricate(:post, locale: "en", topic: self.post.topic)Fabricate(:post_localization, post: post, locale: "ja_JP")Fabricate(:post_localization, post: post, locale: "de_DE")@@ -156,7 +157,7 @@ describe Jobs::LocalizePosts doendit "scenario 6: skips posts with variant 'en_GB' when localizations for ja/de exist" do- post = Fabricate(:post, locale: "en_GB")+ post = Fabricate(:post, locale: "en_GB", topic: self.post.topic)Fabricate(:post_localization, post: post, locale: "ja_JP")Fabricate(:post_localization, post: post, locale: "de_DE")@@ -188,12 +189,13 @@ describe Jobs::LocalizePosts doendend- describe "with public content limitation" do- fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) }- fab!(:private_topic) { Fabricate(:topic, category: private_category) }- fab!(:private_post) { Fabricate(:post, topic: private_topic, locale: "es") }-- fab!(:public_post) { Fabricate(:post, locale: "es") }+ describe "with target categories" do+ fab!(:target_category, :category)+ fab!(:non_target_category, :category)+ fab!(:target_topic) { Fabricate(:topic, category: target_category) }+ fab!(:non_target_topic) { Fabricate(:topic, category: non_target_category) }+ fab!(:target_post) { Fabricate(:post, topic: target_topic, locale: "es") }+ fab!(:non_target_post) { Fabricate(:post, topic: non_target_topic, locale: "es") }fab!(:personal_pm_topic, :private_message_topic)fab!(:personal_pm_post) { Fabricate(:post, topic: personal_pm_topic, locale: "es") }@@ -205,20 +207,21 @@ describe Jobs::LocalizePosts dobefore doSiteSetting.default_locale = "ja"SiteSetting.content_localization_supported_locales = "ja"+ SiteSetting.ai_translation_target_categories = target_category.id.to_send- context "when ai_translation_backfill_limit_to_public_content is true" do- before { SiteSetting.ai_translation_backfill_limit_to_public_content = true }+ context "when pm_translation_scope is none" do+ before { SiteSetting.ai_translation_personal_messages = "none" }- it "only processes posts from public categories" do+ it "only processes posts from target categories" doDiscourseAi::Translation::PostLocalizer.expects(:localize)- .with(public_post, "ja", has_entries(llm_model: anything))+ .with(target_post, "ja", has_entries(llm_model: anything)).onceDiscourseAi::Translation::PostLocalizer.expects(:localize)- .with(private_post, any_parameters)+ .with(non_target_post, any_parameters).neverDiscourseAi::Translation::PostLocalizer@@ -234,18 +237,18 @@ describe Jobs::LocalizePosts doendend- context "when ai_translation_backfill_limit_to_public_content is false" do- before { SiteSetting.ai_translation_backfill_limit_to_public_content = false }+ context "when pm_translation_scope is group" do+ before { SiteSetting.ai_translation_personal_messages = "group" }- it "processes public posts and group PMs but not personal PMs" do+ it "processes target posts and group PMs but not personal PMs" doDiscourseAi::Translation::PostLocalizer.expects(:localize)- .with(public_post, "ja", has_entries(llm_model: anything))+ .with(target_post, "ja", has_entries(llm_model: anything)).onceDiscourseAi::Translation::PostLocalizer.expects(:localize)- .with(private_post, "ja", has_entries(llm_model: anything))- .once+ .with(non_target_post, any_parameters)+ .neverDiscourseAi::Translation::PostLocalizer.expects(:localize)@@ -305,8 +308,8 @@ describe Jobs::LocalizePosts dodescribe "LlmModel caching" doit "caches the LlmModel and reuses it for all posts in a batch" do- post_1 = Fabricate(:post, locale: "es")- post_2 = Fabricate(:post, locale: "es")+ post_1 = Fabricate(:post, locale: "es", topic: post.topic)+ post_2 = Fabricate(:post, locale: "es", topic: post.topic)SiteSetting.content_localization_supported_locales = "ja"# Track how many times LlmModel.find_by is calleddiff --git a/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb b/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rbindex ff30a1e187b..492bd6904af 100644--- a/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb+++ b/plugins/discourse-ai/spec/jobs/regular/localize_topics_spec.rb@@ -14,6 +14,7 @@ describe Jobs::LocalizeTopics doSiteSetting.content_localization_supported_locales = locales.join("|")SiteSetting.ai_translation_backfill_hourly_rate = 100SiteSetting.ai_translation_backfill_max_age_days = 100+ SiteSetting.ai_translation_target_categories = topic.category_id.to_sendit "does nothing when translator is disabled" do@@ -111,7 +112,7 @@ describe Jobs::LocalizeTopics doendit "scenario 2: returns topic with locale 'es' if localizations for en/ja/de do not exist" do- topic = Fabricate(:topic, locale: "es")+ topic = Fabricate(:topic, locale: "es", category: self.topic.category)DiscourseAi::Translation::TopicLocalizer.expects(:localize)@@ -142,7 +143,7 @@ describe Jobs::LocalizeTopics doendit "scenario 3: returns topic with locale 'en' if ja/de localization does not exist" do- topic = Fabricate(:topic, locale: "en")+ topic = Fabricate(:topic, locale: "en", category: self.topic.category)DiscourseAi::Translation::TopicLocalizer.expects(:localize)@@ -166,7 +167,7 @@ describe Jobs::LocalizeTopics doendit "scenario 4: skips topic with locale 'en' if all localizations exist" do- topic = Fabricate(:topic, locale: "en")+ topic = Fabricate(:topic, locale: "en", category: self.topic.category)Fabricate(:topic_localization, topic: topic, locale: "ja")Fabricate(:topic_localization, topic: topic, locale: "de")@@ -176,7 +177,7 @@ describe Jobs::LocalizeTopics doendit "scenario 5: skips topic that already have localizations in similar language variant" do- topic = Fabricate(:topic, locale: "en")+ topic = Fabricate(:topic, locale: "en", category: self.topic.category)Fabricate(:topic_localization, topic: topic, locale: "ja_JP")Fabricate(:topic_localization, topic: topic, locale: "de_DE")@@ -186,10 +187,11 @@ describe Jobs::LocalizeTopics doendend- describe "with public content limitation" do- fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) }- fab!(:private_topic) { Fabricate(:topic, category: private_category, locale: "es") }- fab!(:public_topic) { Fabricate(:topic, locale: "es") }+ describe "with target categories" do+ fab!(:target_category, :category)+ fab!(:non_target_category, :category)+ fab!(:target_topic) { Fabricate(:topic, category: target_category, locale: "es") }+ fab!(:non_target_topic) { Fabricate(:topic, category: non_target_category, locale: "es") }fab!(:personal_pm_topic) { Fabricate(:private_message_topic, locale: "es") }@@ -200,16 +202,17 @@ describe Jobs::LocalizeTopics dobefore doSiteSetting.default_locale = "ja"SiteSetting.content_localization_supported_locales = "ja"+ SiteSetting.ai_translation_target_categories = target_category.id.to_send- context "when ai_translation_backfill_limit_to_public_content is true" do- before { SiteSetting.ai_translation_backfill_limit_to_public_content = true }+ context "when pm_translation_scope is none" do+ before { SiteSetting.ai_translation_personal_messages = "none" }- it "only processes topics from public categories" do+ it "only processes topics from target categories" doDiscourseAi::Translation::TopicLocalizer.expects(:localize).with(- public_topic,+ target_topic,"ja",has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),)@@ -217,7 +220,7 @@ describe Jobs::LocalizeTopics doDiscourseAi::Translation::TopicLocalizer.expects(:localize)- .with(private_topic, any_parameters)+ .with(non_target_topic, any_parameters).neverDiscourseAi::Translation::TopicLocalizer@@ -234,14 +237,14 @@ describe Jobs::LocalizeTopics doendend- context "when ai_translation_backfill_limit_to_public_content is false" do- before { SiteSetting.ai_translation_backfill_limit_to_public_content = false }+ context "when pm_translation_scope is group" do+ before { SiteSetting.ai_translation_personal_messages = "group" }- it "processes public topics, private topics and group PMs but not personal PMs" do+ it "processes target topics and group PMs but not personal PMs" doDiscourseAi::Translation::TopicLocalizer.expects(:localize).with(- public_topic,+ target_topic,"ja",has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),)@@ -250,7 +253,7 @@ describe Jobs::LocalizeTopics doDiscourseAi::Translation::TopicLocalizer.expects(:localize).with(- private_topic,+ group_pm_topic,"ja",has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),)@@ -258,12 +261,8 @@ describe Jobs::LocalizeTopics doDiscourseAi::Translation::TopicLocalizer.expects(:localize)- .with(- group_pm_topic,- "ja",- has_entries(topic_title_llm_model: anything, post_raw_llm_model: anything),- )- .once+ .with(non_target_topic, any_parameters)+ .neverDiscourseAi::Translation::TopicLocalizer.expects(:localize)@@ -276,8 +275,12 @@ describe Jobs::LocalizeTopics doenddescribe "with max age limit" do- fab!(:old_topic) { Fabricate(:topic, locale: "es", created_at: 10.days.ago) }- fab!(:new_topic) { Fabricate(:topic, locale: "es", created_at: 2.days.ago) }+ fab!(:old_topic) do+ Fabricate(:topic, locale: "es", created_at: 10.days.ago, category: topic.category)+ end+ fab!(:new_topic) do+ Fabricate(:topic, locale: "es", created_at: 2.days.ago, category: topic.category)+ endbefore { SiteSetting.ai_translation_backfill_max_age_days = 5 }diff --git a/plugins/discourse-ai/spec/jobs/scheduled/categories_locale_detection_backfill_spec.rb b/plugins/discourse-ai/spec/jobs/scheduled/categories_locale_detection_backfill_spec.rbindex 04a6dd43df1..c9b708497a4 100644--- a/plugins/discourse-ai/spec/jobs/scheduled/categories_locale_detection_backfill_spec.rb+++ b/plugins/discourse-ai/spec/jobs/scheduled/categories_locale_detection_backfill_spec.rb@@ -11,6 +11,7 @@ xdescribe Jobs::CategoriesLocaleDetectionBackfill doSiteSetting.ai_translation_enabled = trueSiteSetting.ai_translation_backfill_hourly_rate = 100SiteSetting.content_localization_supported_locales = "en"+ SiteSetting.ai_translation_target_categories = category.id.to_sendit "does nothing when AI is disabled" do@@ -41,20 +42,24 @@ xdescribe Jobs::CategoriesLocaleDetectionBackfill dojob.execute({})end- it "detects locale for categories with nil locale" do- DiscourseAi::Translation::CategoryLocaleDetector- .expects(:detect_locale)- .with(is_a(Category))- .times(Category.count)+ it "does nothing when target_categories is empty" do+ SiteSetting.ai_translation_target_categories = ""+ DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).never++ job.execute({})+ end++ it "detects locale for categories with nil locale in target categories" do+ non_target = Fabricate(:category, locale: nil)++ DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(category).once++ DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(non_target).neverjob.execute({})endit "handles detection errors gracefully" do- DiscourseAi::Translation::CategoryLocaleDetector- .expects(:detect_locale)- .with(is_a(Category))- .at_least_onceDiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(category)@@ -67,51 +72,16 @@ xdescribe Jobs::CategoriesLocaleDetectionBackfill doit "logs a summary after running" doDiscourseAi::Translation::CategoryLocaleDetector.stubs(:detect_locale)DiscourseAi::Translation::VerboseLogger.expects(:log).with(- includes("Detected #{Category.count} category locales"),+ includes("Detected 1 category locales"),)job.execute({})end- describe "with public content limitation" do- fab!(:private_category) { Fabricate(:private_category, group: Group[:staff], locale: nil) }-- before do- # catch-all for other categories- DiscourseAi::Translation::CategoryLocaleDetector- .expects(:detect_locale)- .with(is_a(Category))- .at_least_once-- SiteSetting.ai_translation_backfill_limit_to_public_content = true- end-- it "only processes public categories" do- DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(category).once- DiscourseAi::Translation::CategoryLocaleDetector- .expects(:detect_locale)- .with(private_category)- .never-- job.execute({})- end-- it "processes all categories when setting is disabled" do- SiteSetting.ai_translation_backfill_limit_to_public_content = false-- DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(category).once- DiscourseAi::Translation::CategoryLocaleDetector- .expects(:detect_locale)- .with(private_category)- .once-- job.execute({})- end- end-it "limits processing to the backfill rate" doSiteSetting.ai_translation_backfill_hourly_rate = 1- Fabricate(:category, locale: nil)+ extra = Fabricate(:category, locale: nil)+ SiteSetting.ai_translation_target_categories = "#{category.id}|#{extra.id}"DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).oncediff --git a/plugins/discourse-ai/spec/jobs/scheduled/posts_locale_detection_backfill_spec.rb b/plugins/discourse-ai/spec/jobs/scheduled/posts_locale_detection_backfill_spec.rbindex 9b7c03df892..1115db44a8f 100644--- a/plugins/discourse-ai/spec/jobs/scheduled/posts_locale_detection_backfill_spec.rb+++ b/plugins/discourse-ai/spec/jobs/scheduled/posts_locale_detection_backfill_spec.rb@@ -16,6 +16,7 @@ describe Jobs::PostsLocaleDetectionBackfill doSiteSetting.ai_translation_enabled = trueSiteSetting.ai_translation_backfill_hourly_rate = 100SiteSetting.content_localization_supported_locales = "en"+ SiteSetting.ai_translation_target_categories = post.topic.category_id.to_sendit "does nothing when translator is disabled" do@@ -45,8 +46,8 @@ describe Jobs::PostsLocaleDetectionBackfill doendit "detects most recently updated posts first" do- post_2 = Fabricate(:post, locale: nil)- post_3 = Fabricate(:post, locale: nil)+ post_2 = Fabricate(:post, locale: nil, topic: post.topic)+ post_3 = Fabricate(:post, locale: nil, topic: post.topic)post.update!(updated_at: 3.days.ago)post_2.update!(updated_at: 2.days.ago)@@ -100,10 +101,13 @@ describe Jobs::PostsLocaleDetectionBackfill dojob.execute({})end- describe "with public content limitation" do- fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) }- fab!(:private_cat_topic) { Fabricate(:topic, category: private_category) }- fab!(:private_cat_post) { Fabricate(:post, topic: private_cat_topic, locale: nil) }+ describe "with target categories" do+ fab!(:target_category, :category)+ fab!(:non_target_category, :category)+ fab!(:target_topic) { Fabricate(:topic, category: target_category) }+ fab!(:non_target_topic) { Fabricate(:topic, category: non_target_category) }+ fab!(:target_post) { Fabricate(:post, topic: target_topic, locale: nil) }+ fab!(:non_target_post) { Fabricate(:post, topic: non_target_topic, locale: nil) }fab!(:group)fab!(:group_pm_topic) { Fabricate(:private_message_topic, allowed_groups: [group]) }@@ -112,13 +116,16 @@ describe Jobs::PostsLocaleDetectionBackfill dofab!(:pm_topic, :private_message_topic)fab!(:pm_post) { Fabricate(:post, topic: pm_topic, locale: nil) }- before { SiteSetting.ai_translation_backfill_limit_to_public_content = true }+ before do+ SiteSetting.ai_translation_target_categories = target_category.id.to_s+ SiteSetting.ai_translation_personal_messages = "none"+ end- it "only processes posts from public categories" do- DiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(post).once+ it "only processes posts from target categories" do+ DiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(target_post).onceDiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale)- .with(private_cat_post)+ .with(non_target_post).neverDiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(group_pm_post).neverDiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(pm_post).never@@ -126,14 +133,14 @@ describe Jobs::PostsLocaleDetectionBackfill dojob.execute({})end- it "processes all public content and group PMs and private categories when setting is disabled" do- SiteSetting.ai_translation_backfill_limit_to_public_content = false+ it "processes target posts and group PMs when pm_translation_scope is group" do+ SiteSetting.ai_translation_personal_messages = "group"- DiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(post).once+ DiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(target_post).onceDiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale)- .with(private_cat_post)- .once+ .with(non_target_post)+ .neverDiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(group_pm_post).onceDiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(pm_post).neverdiff --git a/plugins/discourse-ai/spec/jobs/scheduled/tags_locale_detection_backfill_spec.rb b/plugins/discourse-ai/spec/jobs/scheduled/tags_locale_detection_backfill_spec.rbindex 30a4d563182..f5f19098804 100644--- a/plugins/discourse-ai/spec/jobs/scheduled/tags_locale_detection_backfill_spec.rb+++ b/plugins/discourse-ai/spec/jobs/scheduled/tags_locale_detection_backfill_spec.rb@@ -68,50 +68,22 @@ describe Jobs::TagsLocaleDetectionBackfill dojob.execute({})end- describe "with public content limitation" do- fab!(:tag_group)- fab!(:restricted_group, :group)-- before do- SiteSetting.ai_translation_backfill_limit_to_public_content = true- Tag.update_all(locale: "en")- end-- it "processes tags not in any tag group" do- standalone_tag = Fabricate(:tag, locale: nil)- DiscourseAi::Translation::TagLocaleDetector.expects(:detect_locale).with(standalone_tag).once-- job.execute({})- end-- it "processes tags in tag groups visible to everyone" do- public_tag = Fabricate(:tag, locale: nil)- tag_group.tags << public_tag- TagGroupPermission.create!(- tag_group: tag_group,- group_id: Group::AUTO_GROUPS[:everyone],- permission_type: TagGroupPermission.permission_types[:full],- )-- DiscourseAi::Translation::TagLocaleDetector.expects(:detect_locale).with(public_tag).once-- job.execute({})- end-- it "skips tags in tag groups not visible to everyone" do- restricted_tag = Fabricate(:tag, locale: nil)- tag_group.tags << restricted_tag- TagGroupPermission.where(tag_group: tag_group).destroy_all- TagGroupPermission.create!(- tag_group: tag_group,- group_id: restricted_group.id,- permission_type: TagGroupPermission.permission_types[:full],- )-- DiscourseAi::Translation::TagLocaleDetector.expects(:detect_locale).never-- job.execute({})- end+ it "includes all tags regardless of tag group restrictions" do+ Tag.update_all(locale: "en")++ restricted_tag = Fabricate(:tag, locale: nil)+ tag_group = Fabricate(:tag_group, tags: [restricted_tag])+ TagGroupPermission.where(tag_group: tag_group).destroy_all+ restricted_group = Fabricate(:group)+ TagGroupPermission.create!(+ tag_group: tag_group,+ group_id: restricted_group.id,+ permission_type: TagGroupPermission.permission_types[:full],+ )++ DiscourseAi::Translation::TagLocaleDetector.expects(:detect_locale).with(restricted_tag).once++ job.execute({})endit "limits processing to the backfill rate" dodiff --git a/plugins/discourse-ai/spec/jobs/scheduled/topics_locale_detection_backfill_spec.rb b/plugins/discourse-ai/spec/jobs/scheduled/topics_locale_detection_backfill_spec.rbindex 0dce0daf8a6..8735fb77195 100644--- a/plugins/discourse-ai/spec/jobs/scheduled/topics_locale_detection_backfill_spec.rb+++ b/plugins/discourse-ai/spec/jobs/scheduled/topics_locale_detection_backfill_spec.rb@@ -16,6 +16,7 @@ describe Jobs::TopicsLocaleDetectionBackfill doSiteSetting.ai_translation_enabled = trueSiteSetting.ai_translation_backfill_hourly_rate = 100SiteSetting.content_localization_supported_locales = "en"+ SiteSetting.ai_translation_target_categories = topic.category_id.to_sendit "does nothing when translator is disabled" do@@ -45,8 +46,8 @@ describe Jobs::TopicsLocaleDetectionBackfill doendit "detects most recently updated topics first" do- topic_2 = Fabricate(:topic, locale: nil)- topic_3 = Fabricate(:topic, locale: nil)+ topic_2 = Fabricate(:topic, locale: nil, category: topic.category)+ topic_3 = Fabricate(:topic, locale: nil, category: topic.category)topic.update!(updated_at: 3.days.ago)topic_2.update!(updated_at: 2.days.ago)@@ -85,29 +86,27 @@ describe Jobs::TopicsLocaleDetectionBackfill dojob.execute({ limit: 10 })end- describe "with public content limitation" do- fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) }- fab!(:private_cat_topic) { Fabricate(:topic, category: private_category, locale: nil) }+ describe "with target categories" do+ fab!(:target_category, :category)+ fab!(:non_target_category, :category)+ fab!(:target_topic) { Fabricate(:topic, category: target_category, locale: nil) }+ fab!(:non_target_topic) { Fabricate(:topic, category: non_target_category, locale: nil) }fab!(:group)fab!(:group_pm_topic) { Fabricate(:private_message_topic, allowed_groups: [group]) }fab!(:pm_topic, :private_message_topic)- fab!(:public_topic) { Fabricate(:topic, locale: nil) }-before do- DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).at_least_once-- SiteSetting.ai_translation_backfill_limit_to_public_content = true+ SiteSetting.ai_translation_target_categories = target_category.id.to_s+ SiteSetting.ai_translation_personal_messages = "none"end- it "only processes topics from public categories" do- DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(public_topic).once-+ it "only processes topics from target categories" do+ DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(target_topic).onceDiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale)- .with(private_cat_topic)+ .with(non_target_topic).neverDiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale)@@ -118,18 +117,18 @@ describe Jobs::TopicsLocaleDetectionBackfill dojob.execute({ limit: 10 })end- it "processes public category topics, group PMs, and private category topics when setting is disabled" do- SiteSetting.ai_translation_backfill_limit_to_public_content = false+ it "processes target category topics and group PMs when pm_translation_scope is group" do+ SiteSetting.ai_translation_personal_messages = "group"- DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(public_topic).once+ DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(target_topic).onceDiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(group_pm_topic).onceDiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale)- .with(private_cat_topic)- .once+ .with(non_target_topic)+ .neverDiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(pm_topic).neverjob.execute({ limit: 10 })@@ -137,8 +136,12 @@ describe Jobs::TopicsLocaleDetectionBackfill doenddescribe "with max age limit" do- fab!(:old_topic) { Fabricate(:topic, locale: nil, created_at: 10.days.ago) }- fab!(:new_topic) { Fabricate(:topic, locale: nil, created_at: 2.days.ago) }+ fab!(:old_topic) do+ Fabricate(:topic, locale: nil, created_at: 10.days.ago, category: topic.category)+ end+ fab!(:new_topic) do+ Fabricate(:topic, locale: nil, created_at: 2.days.ago, category: topic.category)+ endbefore doDiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).at_least_oncediff --git a/plugins/discourse-ai/spec/lib/translation/category_candidates_spec.rb b/plugins/discourse-ai/spec/lib/translation/category_candidates_spec.rbindex e48296e30d9..b747348319b 100644--- a/plugins/discourse-ai/spec/lib/translation/category_candidates_spec.rb+++ b/plugins/discourse-ai/spec/lib/translation/category_candidates_spec.rb@@ -2,117 +2,88 @@describe DiscourseAi::Translation::CategoryCandidates dodescribe ".get" do- it "returns all categories" do- expect(DiscourseAi::Translation::CategoryCandidates.get.count).to eq(Category.count)+ it "returns only target categories when target_categories is set" do+ target = Fabricate(:category)+ non_target = Fabricate(:category)+ SiteSetting.ai_translation_target_categories = target.id.to_s++ categories = DiscourseAi::Translation::CategoryCandidates.get+ expect(categories).to include(target)+ expect(categories).not_to include(non_target)end- it "filters out read restricted categories if ai_translation_backfill_limit_to_public_content is enabled" do- SiteSetting.ai_translation_backfill_limit_to_public_content = true- restricted_category = Fabricate(:category, read_restricted: true)- public_category = Fabricate(:category, read_restricted: false)+ it "returns no categories when target_categories is empty" do+ Fabricate(:category)- categories = DiscourseAi::Translation::CategoryCandidates.get- expect(categories).not_to include(restricted_category)- expect(categories).to include(public_category)+ SiteSetting.ai_translation_target_categories = ""+ expect(DiscourseAi::Translation::CategoryCandidates.get.count).to eq(0)endenddescribe ".calculate_completion_per_locale" do+ fab!(:target_category, :category)++ before { SiteSetting.ai_translation_target_categories = target_category.id.to_s }+context "when (scenario A) completion determined by category's locale" doit "returns done = total if all categories are in the locale" dolocale = "pt_BR"- Fabricate(:category, locale:)- Category.update_all(locale: locale)- Fabricate(:category, locale: "pt") # pt counts as pt_BR+ target_category.update!(locale: locale)completion =DiscourseAi::Translation::CategoryCandidates.calculate_completion_per_locale(locale)- expect(completion).to eq({ done: Category.count, total: Category.count })+ expect(completion).to eq({ done: 1, total: 1 })endit "returns correct done and total if some categories are in the locale" dolocale = "pt_BR"- Category.update_all(locale: "ar") # not portuguese-- Fabricate(:category, locale:)- Fabricate(:category, locale: "ar") # not portuguese+ target2 = Fabricate(:category, locale: "ar")+ SiteSetting.ai_translation_target_categories = "#{target_category.id}|#{target2.id}"+ target_category.update!(locale: locale)completion =DiscourseAi::Translation::CategoryCandidates.calculate_completion_per_locale(locale)- expect(completion).to eq({ done: 1, total: Category.count })+ expect(completion).to eq({ done: 1, total: 2 })endendcontext "when (scenario B) completion determined by category localizations" doit "returns done = total if all categories have a localization in the locale" dolocale = "pt_BR"-- Fabricate(:category, locale: "en")- Category.all.each do |category|- category.update(locale: "en")- Fabricate(:category_localization, category:, locale:)- end- CategoryLocalization.order("RANDOM()").first.update(locale: "pt") # pt counts as pt_BR+ target_category.update!(locale: "en")+ Fabricate(:category_localization, category: target_category, locale:)completion =DiscourseAi::Translation::CategoryCandidates.calculate_completion_per_locale(locale)- expect(completion).to eq({ done: Category.count, total: Category.count })+ expect(completion).to eq({ done: 1, total: 1 })endit "returns correct done and total if some categories have a localization in the locale" dolocale = "es"- category1 = Fabricate(:category, locale: "en")- category2 = Fabricate(:category, locale: "fr")- Fabricate(:category_localization, category: category1, locale:)- Fabricate(:category_localization, category: category2, locale: "ar") # not the target locale+ target2 = Fabricate(:category, locale: "fr")+ SiteSetting.ai_translation_target_categories = "#{target_category.id}|#{target2.id}"+ target_category.update!(locale: "en")+ Fabricate(:category_localization, category: target_category, locale:)+ Fabricate(:category_localization, category: target2, locale: "ar")completion =DiscourseAi::Translation::CategoryCandidates.calculate_completion_per_locale(locale)- categories_with_locale = Category.where.not(locale: nil).count- expect(completion).to eq({ done: 1, total: categories_with_locale })+ expect(completion).to eq({ done: 1, total: 2 })endend- it "returns the correct done and total based on (scenario A & B) `category.locale` and `CategoryLocalization` in the specified locale" do- locale = "pt_BR"-- Category.update_all(locale: "en")-- # translated candidates- Fabricate(:category, locale:)- category2 = Fabricate(:category, locale: "en")- Fabricate(:category_localization, category: category2, locale:)-- # untranslated candidate- category3 = Fabricate(:category, locale: "fr")- Fabricate(:category_localization, category: category3, locale: "zh_CN")-- # not a candidate as it is read restricted- SiteSetting.ai_translation_backfill_limit_to_public_content = true- category4 = Fabricate(:category, read_restricted: true, locale: "de")- Fabricate(:category_localization, category: category4, locale:)-- completion =- DiscourseAi::Translation::CategoryCandidates.calculate_completion_per_locale(locale)- translated_candidates = 2 # category1 + category2- total_candidates = Category.count - 1 # excluding the read restricted category- expect(completion).to eq({ done: translated_candidates, total: total_candidates })- end-it "does not allow done to exceed total when category.locale and category_localization both exist" dolocale = "pt_BR"- Category.update_all(locale:)- category = Fabricate(:category, locale:)- Fabricate(:category_localization, category:, locale:)+ target_category.update!(locale:)+ Fabricate(:category_localization, category: target_category, locale:)completion =DiscourseAi::Translation::CategoryCandidates.calculate_completion_per_locale(locale)- expect(completion).to eq({ done: Category.count, total: Category.count })+ expect(completion).to eq({ done: 1, total: 1 })end- it "returns nil - nil for done and total when no categories are present" do- SiteSetting.ai_translation_backfill_limit_to_public_content = false- Category.destroy_all+ it "returns 0 for done and total when no categories match" do+ SiteSetting.ai_translation_target_categories = ""completion =DiscourseAi::Translation::CategoryCandidates.calculate_completion_per_locale("pt")diff --git a/plugins/discourse-ai/spec/lib/translation/post_candidates_spec.rb b/plugins/discourse-ai/spec/lib/translation/post_candidates_spec.rbindex 8cedfe2f2d0..2b67e1ee33b 100644--- a/plugins/discourse-ai/spec/lib/translation/post_candidates_spec.rb+++ b/plugins/discourse-ai/spec/lib/translation/post_candidates_spec.rb@@ -1,9 +1,12 @@# frozen_string_literal: truedescribe DiscourseAi::Translation::PostCandidates do+ before { SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|") }+describe ".get" doit "does not return bot posts" dopost = Fabricate(:post, user: Discourse.system_user)+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")expect(DiscourseAi::Translation::PostCandidates.get).not_to include(post)end@@ -13,6 +16,7 @@ describe DiscourseAi::Translation::PostCandidates doSiteSetting.ai_translation_include_bot_content = truebot_post = Fabricate(:post, user: Discourse.system_user)regular_post = Fabricate(:post)+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")posts = DiscourseAi::Translation::PostCandidates.getexpect(posts).to include(bot_post)@@ -26,12 +30,14 @@ describe DiscourseAi::Translation::PostCandidates do:post,created_at: SiteSetting.ai_translation_backfill_max_age_days.days.ago - 1.day,)+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")expect(DiscourseAi::Translation::PostCandidates.get).not_to include(post)endit "does not return deleted posts" dopost = Fabricate(:post, deleted_at: Time.now)+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")expect(DiscourseAi::Translation::PostCandidates.get).not_to include(post)end@@ -40,13 +46,16 @@ describe DiscourseAi::Translation::PostCandidates doSiteSetting.ai_translation_max_post_length = 100short_post = Fabricate(:post, raw: "This is a short post that fits within the limit.")long_post = Fabricate(:post, raw: "a" * 50 + " This is a long post. " + "b" * 50)+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")posts = DiscourseAi::Translation::PostCandidates.getexpect(posts).to include(short_post)expect(posts).not_to include(long_post)end- describe "SiteSetting.ai_translation_backfill_limit_to_public_content" do+ describe "category and PM filtering" do+ fab!(:target_category, :category)+ fab!(:non_target_category, :category)fab!(:pm_post) { Fabricate(:post, topic: Fabricate(:private_message_topic)) }fab!(:group_pm_post) doFabricate(@@ -54,38 +63,52 @@ describe DiscourseAi::Translation::PostCandidates dotopic: Fabricate(:private_message_topic, allowed_groups: [Fabricate(:group)]),)end- fab!(:public_post) do- Fabricate(- :post,- topic: Fabricate(:topic, category: Fabricate(:category, read_restricted: false)),- )+ fab!(:target_post) { Fabricate(:post, topic: Fabricate(:topic, category: target_category)) }+ fab!(:non_target_post) do+ Fabricate(:post, topic: Fabricate(:topic, category: non_target_category))end- it "excludes PMs and only includes posts from public categories" do- SiteSetting.ai_translation_backfill_limit_to_public_content = true+ it "only includes posts from target categories" do+ SiteSetting.ai_translation_target_categories = target_category.id.to_s+ SiteSetting.ai_translation_personal_messages = "none"posts = DiscourseAi::Translation::PostCandidates.get+ expect(posts).to include(target_post)+ expect(posts).not_to include(non_target_post)expect(posts).not_to include(pm_post)expect(posts).not_to include(group_pm_post)- expect(posts).to include(public_post)end- it "includes all regular posts and group PMs but not personal PMs" do- SiteSetting.ai_translation_backfill_limit_to_public_content = false+ it "includes group PMs but not personal PMs when pm_translation_scope is group" do+ SiteSetting.ai_translation_target_categories = target_category.id.to_s+ SiteSetting.ai_translation_personal_messages = "group"posts = DiscourseAi::Translation::PostCandidates.get+ expect(posts).to include(target_post)expect(posts).not_to include(pm_post)expect(posts).to include(group_pm_post)- expect(posts).to include(public_post)+ end++ it "includes all PMs when pm_translation_scope is all" do+ SiteSetting.ai_translation_target_categories = target_category.id.to_s+ SiteSetting.ai_translation_personal_messages = "all"++ posts = DiscourseAi::Translation::PostCandidates.get+ expect(posts).to include(target_post)+ expect(posts).to include(pm_post)+ expect(posts).to include(group_pm_post)endendenddescribe ".get_completion_all_locales" do+ fab!(:target_category, :category)+before doSiteSetting.content_localization_supported_locales = "en_GB|pt|es"SiteSetting.ai_translation_backfill_max_age_days = 30- SiteSetting.ai_translation_backfill_limit_to_public_content = false+ SiteSetting.ai_translation_target_categories = target_category.id.to_s+ SiteSetting.ai_translation_personal_messages = "group"endit "returns empty state when no posts exist" do@@ -100,10 +123,11 @@ describe DiscourseAi::Translation::PostCandidates doendit "returns progress grouped by base locale (of en_GB) and correct totals" do- post1 = Fabricate(:post, locale: "en_GB")- post2 = Fabricate(:post, locale: "fr")- post3 = Fabricate(:post, locale: "es")- post_without_locale = Fabricate(:post, locale: nil) # not eligible for translation+ post1 = Fabricate(:post, locale: "en_GB", topic: Fabricate(:topic, category: target_category))+ post2 = Fabricate(:post, locale: "fr", topic: Fabricate(:topic, category: target_category))+ post3 = Fabricate(:post, locale: "es", topic: Fabricate(:topic, category: target_category))+ post_without_locale =+ Fabricate(:post, locale: nil, topic: Fabricate(:topic, category: target_category))# add an en_GB localization to a non-en base postPostLocalization.create!(@@ -146,9 +170,20 @@ describe DiscourseAi::Translation::PostCandidates doit "excludes posts longer than ai_translation_max_post_length from totals" doSiteSetting.ai_translation_max_post_length = 100- short_post = Fabricate(:post, locale: "en_GB", raw: "This is a short post that fits.")+ short_post =+ Fabricate(+ :post,+ locale: "en_GB",+ raw: "This is a short post that fits.",+ topic: Fabricate(:topic, category: target_category),+ )long_post =- Fabricate(:post, locale: "fr", raw: "a" * 50 + " This is a long post. " + "b" * 50)+ Fabricate(+ :post,+ locale: "fr",+ raw: "a" * 50 + " This is a long post. " + "b" * 50,+ topic: Fabricate(:topic, category: target_category),+ )result = DiscourseAi::Translation::PostCandidates.get_completion_all_localesexpect(result[:total]).to eq(1)diff --git a/plugins/discourse-ai/spec/lib/translation/tag_candidates_spec.rb b/plugins/discourse-ai/spec/lib/translation/tag_candidates_spec.rbindex 20a2d4b59a1..9b45f93a861 100644--- a/plugins/discourse-ai/spec/lib/translation/tag_candidates_spec.rb+++ b/plugins/discourse-ai/spec/lib/translation/tag_candidates_spec.rb@@ -7,45 +7,19 @@ describe DiscourseAi::Translation::TagCandidates doexpect(DiscourseAi::Translation::TagCandidates.get.count).to eq(Tag.count)end- context "when ai_translation_backfill_limit_to_public_content is enabled" do- before { SiteSetting.ai_translation_backfill_limit_to_public_content = true }-- it "includes tags not in any tag group" do- tag = Fabricate(:tag)-- tags = DiscourseAi::Translation::TagCandidates.get- expect(tags).to include(tag)- end-- it "includes tags in tag groups visible to everyone" do- tag = Fabricate(:tag)- tag_group = Fabricate(:tag_group, tags: [tag])- TagGroupPermission.create!(- tag_group: tag_group,- group_id: Group::AUTO_GROUPS[:everyone],- permission_type: TagGroupPermission.permission_types[:full],- )-- tags = DiscourseAi::Translation::TagCandidates.get- expect(tags).to include(tag)- end-- it "excludes tags in tag groups not visible to everyone" do- tag = Fabricate(:tag)- tag_group = Fabricate(:tag_group, tags: [tag])- # remove default everyone permission if exists- TagGroupPermission.where(tag_group: tag_group).destroy_all- # add restricted permission- restricted_group = Fabricate(:group)- TagGroupPermission.create!(- tag_group: tag_group,- group_id: restricted_group.id,- permission_type: TagGroupPermission.permission_types[:full],- )-- tags = DiscourseAi::Translation::TagCandidates.get- expect(tags).not_to include(tag)- end+ it "includes tags in restricted tag groups" do+ tag = Fabricate(:tag)+ tag_group = Fabricate(:tag_group, tags: [tag])+ TagGroupPermission.where(tag_group: tag_group).destroy_all+ restricted_group = Fabricate(:group)+ TagGroupPermission.create!(+ tag_group: tag_group,+ group_id: restricted_group.id,+ permission_type: TagGroupPermission.permission_types[:full],+ )++ tags = DiscourseAi::Translation::TagCandidates.get+ expect(tags).to include(tag)endenddiff --git a/plugins/discourse-ai/spec/lib/translation/topic_candidates_spec.rb b/plugins/discourse-ai/spec/lib/translation/topic_candidates_spec.rbindex b05f25eb59e..1cadd181121 100644--- a/plugins/discourse-ai/spec/lib/translation/topic_candidates_spec.rb+++ b/plugins/discourse-ai/spec/lib/translation/topic_candidates_spec.rb@@ -1,9 +1,12 @@# frozen_string_literal: truedescribe DiscourseAi::Translation::TopicCandidates do+ before { SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|") }+describe ".get" doit "does not return bot topics" dotopic = Fabricate(:topic, user: Discourse.system_user)+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")expect(DiscourseAi::Translation::TopicCandidates.get).not_to include(topic)end@@ -13,6 +16,7 @@ describe DiscourseAi::Translation::TopicCandidates doSiteSetting.ai_translation_include_bot_content = truebot_topic = Fabricate(:topic, user: Discourse.system_user)regular_topic = Fabricate(:topic)+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")topics = DiscourseAi::Translation::TopicCandidates.getexpect(topics).to include(bot_topic)@@ -26,50 +30,87 @@ describe DiscourseAi::Translation::TopicCandidates do:topic,created_at: SiteSetting.ai_translation_backfill_max_age_days.days.ago - 1.day,)+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")expect(DiscourseAi::Translation::TopicCandidates.get).not_to include(topic)endit "does not return deleted topics" dotopic = Fabricate(:topic, deleted_at: Time.now)+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")expect(DiscourseAi::Translation::TopicCandidates.get).not_to include(topic)end- describe "SiteSetting.ai_translation_backfill_limit_to_public_content" do+ describe "category and PM filtering" do+ fab!(:target_category, :category)+ fab!(:non_target_category, :category)fab!(:pm, :private_message_topic)fab!(:group_pm) { Fabricate(:private_message_topic, allowed_groups: [Fabricate(:group)]) }- fab!(:public_topic) do- Fabricate(:topic, category: Fabricate(:category, read_restricted: false))+ fab!(:target_topic) { Fabricate(:topic, category: target_category) }+ fab!(:non_target_topic) { Fabricate(:topic, category: non_target_category) }++ it "only includes topics from target categories when target_categories is set" do+ SiteSetting.ai_translation_target_categories = target_category.id.to_s++ topics = DiscourseAi::Translation::TopicCandidates.get+ expect(topics).to include(target_topic)+ expect(topics).not_to include(non_target_topic)+ end++ it "returns no regular topics when target_categories is empty" do+ SiteSetting.ai_translation_target_categories = ""+ SiteSetting.ai_translation_personal_messages = "none"++ topics = DiscourseAi::Translation::TopicCandidates.get+ expect(topics).not_to include(target_topic)+ expect(topics).not_to include(non_target_topic)+ expect(topics).not_to include(pm)+ expect(topics).not_to include(group_pm)end- it "excludes PMs and only includes topics from public categories" do- SiteSetting.ai_translation_backfill_limit_to_public_content = true+ it "excludes all PMs when pm_translation_scope is none" do+ SiteSetting.ai_translation_target_categories = target_category.id.to_s+ SiteSetting.ai_translation_personal_messages = "none"topics = DiscourseAi::Translation::TopicCandidates.get+ expect(topics).to include(target_topic)expect(topics).not_to include(pm)expect(topics).not_to include(group_pm)- expect(topics).to include(public_topic)end- it "includes all regular topics and group PMs but not personal PMs" do- SiteSetting.ai_translation_backfill_limit_to_public_content = false+ it "includes group PMs but not personal PMs when pm_translation_scope is group" do+ SiteSetting.ai_translation_target_categories = target_category.id.to_s+ SiteSetting.ai_translation_personal_messages = "group"topics = DiscourseAi::Translation::TopicCandidates.get+ expect(topics).to include(target_topic)expect(topics).not_to include(pm)expect(topics).to include(group_pm)- expect(topics).to include(public_topic)+ end++ it "includes all PMs when pm_translation_scope is all" do+ SiteSetting.ai_translation_target_categories = target_category.id.to_s+ SiteSetting.ai_translation_personal_messages = "all"++ topics = DiscourseAi::Translation::TopicCandidates.get+ expect(topics).to include(target_topic)+ expect(topics).to include(pm)+ expect(topics).to include(group_pm)endendenddescribe ".calculate_completion_per_locale" do+ before { SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|") }+context "when (scenario A) 'done' determined by topic's locale" doit "returns total = done if all topics are in the locale" dolocale = "pt_BR"Fabricate(:topic, locale:)Topic.update_all(locale: locale)Fabricate(:topic, locale: "pt")+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")completion =DiscourseAi::Translation::TopicCandidates.calculate_completion_per_locale(locale)@@ -80,6 +121,7 @@ describe DiscourseAi::Translation::TopicCandidates dolocale = "es"Fabricate(:topic, locale:)Fabricate(:topic, locale: "not_es")+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")completion =DiscourseAi::Translation::TopicCandidates.calculate_completion_per_locale(locale)@@ -96,6 +138,7 @@ describe DiscourseAi::Translation::TopicCandidates doFabricate(:topic_localization, topic:, locale:)endTopicLocalization.order("RANDOM()").first.update(locale: "pt")+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")completion =DiscourseAi::Translation::TopicCandidates.calculate_completion_per_locale(locale)@@ -108,6 +151,7 @@ describe DiscourseAi::Translation::TopicCandidates dotopic2 = Fabricate(:topic, locale: "fr")Fabricate(:topic_localization, topic: topic1, locale:)Fabricate(:topic_localization, topic: topic2, locale: "not_es")+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")completion =DiscourseAi::Translation::TopicCandidates.calculate_completion_per_locale(locale)@@ -131,6 +175,7 @@ describe DiscourseAi::Translation::TopicCandidates do# not a candidate as it is a bot topictopic3 = Fabricate(:topic, user: Discourse.system_user, locale: "de")Fabricate(:topic_localization, topic: topic3, locale:)+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")completion = DiscourseAi::Translation::TopicCandidates.calculate_completion_per_locale(locale)translated_candidates = 2 # topic1 + topic2@@ -142,6 +187,7 @@ describe DiscourseAi::Translation::TopicCandidates dolocale = "es"topic = Fabricate(:topic, locale:)Fabricate(:topic_localization, topic:, locale:)+ SiteSetting.ai_translation_target_categories = Category.pluck(:id).join("|")completion = DiscourseAi::Translation::TopicCandidates.calculate_completion_per_locale(locale)expect(completion).to eq({ done: 1, total: 1 })diff --git a/plugins/discourse-ai/spec/requests/admin/ai_translations_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/ai_translations_controller_spec.rbindex 7331fc6390a..2bab41351d7 100644--- a/plugins/discourse-ai/spec/requests/admin/ai_translations_controller_spec.rb+++ b/plugins/discourse-ai/spec/requests/admin/ai_translations_controller_spec.rb@@ -11,21 +11,31 @@ describe DiscourseAi::Admin::AiTranslationsController dodescribe "#show" docontext "when logged in as admin" do+ fab!(:target_category, :category)+before dosign_in(admin)SiteSetting.discourse_ai_enabled = trueSiteSetting.ai_translation_enabled = trueSiteSetting.content_localization_supported_locales = "en|fr|es"+ SiteSetting.ai_translation_target_categories = target_category.id.to_sendit "returns translation progress data" doSiteSetting.ai_translation_backfill_max_age_days = 30- SiteSetting.ai_translation_backfill_limit_to_public_content = false+ SiteSetting.ai_translation_personal_messages = "group"SiteSetting.ai_translation_backfill_hourly_rate = 100- english_posts = Fabricate.times(14, :post, locale: "en")- french_post = Fabricate(:post, locale: "fr")- Fabricate.times(4, :post)+ english_posts =+ Fabricate.times(+ 14,+ :post,+ locale: "en",+ topic: Fabricate(:topic, category: target_category),+ )+ french_post =+ Fabricate(:post, locale: "fr", topic: Fabricate(:topic, category: target_category))+ Fabricate.times(4, :post, topic: Fabricate(:topic, category: target_category))PostLocalization.create!(post: french_post,@@ -60,12 +70,30 @@ describe DiscourseAi::Admin::AiTranslationsController doit "shows only posts requiring translation for all locales (consistent behavior)" doSiteSetting.ai_translation_backfill_max_age_days = 30- SiteSetting.ai_translation_backfill_limit_to_public_content = false+ SiteSetting.ai_translation_personal_messages = "group"SiteSetting.default_locale = "en"- english_posts = Fabricate.times(100, :post, locale: "en")- french_posts = Fabricate.times(10, :post, locale: "fr")- spanish_posts = Fabricate.times(5, :post, locale: "es")+ english_posts =+ Fabricate.times(+ 100,+ :post,+ locale: "en",+ topic: Fabricate(:topic, category: target_category),+ )+ french_posts =+ Fabricate.times(+ 10,+ :post,+ locale: "fr",+ topic: Fabricate(:topic, category: target_category),+ )+ spanish_posts =+ Fabricate.times(+ 5,+ :post,+ locale: "es",+ topic: Fabricate(:topic, category: target_category),+ )french_posts.take(8)