A platform for community discussion. Free, open, simple.
FEATURE: Allow detecting spam with a decision model (#45413)
Rafael dos Santos Silvaefafad060b70
diff --git a/db/structure.sql b/db/structure.sqlindex 4f1198034c4..6a9c77eae45 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -917,10 +917,11 @@ CREATE TABLE public.ai_moderation_settings (id bigint NOT NULL,setting_type public.ai_moderation_setting_type NOT NULL,data jsonb DEFAULT '{}'::jsonb,- llm_model_id bigint NOT NULL,+ llm_model_id bigint,created_at timestamp(6) without time zone NOT NULL,updated_at timestamp(6) without time zone NOT NULL,- ai_agent_id bigint DEFAULT '-31'::integer NOT NULL+ ai_agent_id bigint DEFAULT '-31'::integer NOT NULL,+ decision_model_id bigint);@@ -1038,7 +1039,7 @@ ALTER SEQUENCE public.ai_secrets_id_seq OWNED BY public.ai_secrets.id;CREATE TABLE public.ai_spam_logs (id bigint NOT NULL,post_id bigint NOT NULL,- llm_model_id bigint NOT NULL,+ llm_model_id bigint,ai_api_audit_log_id bigint,reviewable_id bigint,is_spam boolean NOT NULL,@@ -1046,7 +1047,9 @@ CREATE TABLE public.ai_spam_logs (created_at timestamp(6) without time zone NOT NULL,updated_at timestamp(6) without time zone NOT NULL,error character varying(3000),- reason text+ reason text,+ decision_model_id bigint,+ spam_probability double precision);@@ -26520,6 +26523,7 @@ SET search_path TO "$user", public;INSERT INTO "schema_migrations" (version) VALUES('20261102174233'),('20261030215104'),+('20261030183551'),('20261030180958'),('20261029021723'),('20261028093149'),diff --git a/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-spam.gjs b/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-spam.gjsindex 52414ff00f5..a8f3652518e 100644--- a/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-spam.gjs+++ b/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-spam.gjs@@ -1,6 +1,6 @@import Component from "@glimmer/component";import { tracked } from "@glimmer/tracking";-import { fn } from "@ember/helper";+import { fn, hash } from "@ember/helper";import { on } from "@ember/modifier";import { action } from "@ember/object";import { LinkTo } from "@ember/routing";@@ -36,6 +36,7 @@ export default class AiSpam extends Component {};@tracked isEnabled = false;@tracked selectedLLM = null;+ @tracked selectedDecisionModelId = null;@tracked selectedAgentId = null;@tracked customInstructions = "";@autoTrackedArray errors;@@ -59,6 +60,10 @@ export default class AiSpam extends Component {return this.args.model?.available_llms || [];}+ get availableDecisionModels() {+ return this.args.model?.available_decision_models || [];+ }+get availableAgents() {return this.args.model?.available_agents || [];}@@ -150,13 +155,12 @@ export default class AiSpam extends Component {const model = this.args.model;this.isEnabled = model.is_enabled;+ this.selectedDecisionModelId = model.decision_model_id;if (model.llm_id) {this.selectedLLM = model.llm_id;- } else {- if (this.availableLLMs.length) {- this.selectedLLM = this.availableLLMs[0].id;- this.autoSelectedLLM = true;- }+ } else if (!model.decision_model_id && this.availableLLMs.length) {+ this.selectedLLM = this.availableLLMs[0].id;+ this.autoSelectedLLM = true;}this.customInstructions = model.custom_instructions;this.stats = model.stats;@@ -192,6 +196,11 @@ export default class AiSpam extends Component {this.selectedLLM = value;}+ @action+ async updateDecisionModel(value) {+ this.selectedDecisionModelId = value;+ }+@actionasync updateAgent(value) {this.selectedAgentId = value;@@ -203,7 +212,8 @@ export default class AiSpam extends Component {await ajax("/admin/plugins/discourse-ai/ai-spam.json", {type: "PUT",data: {- llm_model_id: this.llmId,+ llm_model_id: this.llmId ?? "",+ decision_model_id: this.selectedDecisionModelId ?? "",custom_instructions: this.customInstructions,ai_agent_id: this.selectedAgentId,},@@ -223,6 +233,7 @@ export default class AiSpam extends Component {model: {customInstructions: this.customInstructions,llmId: this.llmId,+ decisionModelId: this.selectedDecisionModelId,},});}@@ -273,6 +284,7 @@ export default class AiSpam extends Component {class="ai-spam__llm-selector"@content={{this.availableLLMs}}@onChange={{this.updateLLM}}+ @options={{hash none="discourse_ai.spam.no_llm_selected"}}@value={{this.selectedLLM}}/>{{else}}@@ -284,17 +296,40 @@ export default class AiSpam extends Component {{{/if}}</div>- <div class="ai-spam__agent">- <label class="ai-spam__agent-label">{{i18n- "discourse_ai.spam.select_agent"- }}</label>- <ComboBox- class="ai-spam__agent-selector"- @content={{this.availableAgents}}- @onChange={{this.updateAgent}}- @value={{this.selectedAgentId}}- />- </div>+ {{#if this.availableDecisionModels.length}}+ <div class="ai-spam__decision-model">+ <label class="ai-spam__decision-model-label">+ {{i18n "discourse_ai.spam.select_decision_model"}}+ <DTooltip+ @content={{i18n "discourse_ai.spam.decision_model_help"}}+ @icon="circle-question"+ />+ </label>+ <ComboBox+ class="ai-spam__decision-model-selector"+ @content={{this.availableDecisionModels}}+ @onChange={{this.updateDecisionModel}}+ @options={{hash+ none="discourse_ai.spam.no_decision_model_selected"+ }}+ @value={{this.selectedDecisionModelId}}+ />+ </div>+ {{/if}}++ {{#if this.selectedLLM}}+ <div class="ai-spam__agent">+ <label class="ai-spam__agent-label">{{i18n+ "discourse_ai.spam.select_agent"+ }}</label>+ <ComboBox+ class="ai-spam__agent-selector"+ @content={{this.availableAgents}}+ @onChange={{this.updateAgent}}+ @value={{this.selectedAgentId}}+ />+ </div>+ {{/if}}<div class="ai-spam__instructions"><label class="ai-spam__instructions-label">diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_decision_models_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_decision_models_controller.rbindex 3dc55254c3e..d28086a3740 100644--- a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_decision_models_controller.rb+++ b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_decision_models_controller.rb@@ -77,6 +77,14 @@ module DiscourseAidef destroydecision_model = DecisionModel.find(params[:id])return render_seeded_error if decision_model.seeded?+ if AiModerationSetting.exists?(decision_model_id: decision_model.id)+ return(+ render_json_error(+ I18n.t("discourse_ai.decision_models.delete_failed_spam"),+ status: 409,+ )+ )+ enddetails =decision_modeldiff --git a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_spam_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_spam_controller.rbindex 86cad4449f4..7c773fbf6d1 100644--- a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_spam_controller.rb+++ b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_spam_controller.rb@@ -15,12 +15,16 @@ module DiscourseAiinitial_data = {custom_instructions: initial_settings&.data&.dig("custom_instructions"),llm_model_id: initial_settings&.llm_model_id,+ decision_model_id: initial_settings&.decision_model_id,ai_agent_id: initial_settings&.ai_agent_id,}updated_params = {}if allowed_params.key?(:llm_model_id)- updated_params[:llm_model_id] = allowed_params[:llm_model_id]+ updated_params[:llm_model_id] = allowed_params[:llm_model_id].presence+ end+ if allowed_params.key?(:decision_model_id)+ updated_params[:decision_model_id] = allowed_params[:decision_model_id].presenceendai_agent_id = allowed_params[:ai_agent_id]@@ -45,11 +49,9 @@ module DiscourseAiif updated_params.present?# not using upsert cause we will not get the correct validation errors- if AiModerationSetting.spam- AiModerationSetting.spam.update!(updated_params)- else- AiModerationSetting.create!(updated_params.merge(setting_type: :spam))- end+ settings = AiModerationSetting.spam || AiModerationSetting.new(setting_type: :spam)+ settings.assign_attributes(updated_params)+ return render_json_error(settings) if !settings.savelog_ai_spam_update(initial_data, allowed_params)end@@ -57,12 +59,9 @@ module DiscourseAiis_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])if allowed_params.key?(:is_enabled)- if is_enabled && !AiModerationSetting.spam&.llm_model_id+ if is_enabled && !AiModerationSetting.spamreturn(- render_json_error(- I18n.t("discourse_ai.llm.configuration.must_select_model"),- status: 422,- )+ render_json_error(I18n.t("discourse_ai.spam_detection.model_missing"), status: 422))end@@ -92,9 +91,30 @@ module DiscourseAiraise Discourse::NotFound if !post- if params[:llm_id].present? && !LlmModel.exists?(id: params[:llm_id])+ settings = AiModerationSetting.spam+ llm_model = params.key?(:llm_id) ? nil : settings&.llm_model+ if params[:llm_id].present?+ llm_model = LlmModel.find_by(id: params[:llm_id])+ if !llm_model+ return(+ render_json_error(+ I18n.t("discourse_ai.llm.configuration.invalid_llm_id"),+ status: 422,+ )+ )+ end+ end+ decision_model =+ if params.key?(:decision_model_id)+ if params[:decision_model_id].present?+ DecisionModel.find_by(id: params[:decision_model_id])+ end+ else+ settings&.decision_model+ end+ if !llm_model && !decision_modelreturn(- render_json_error(I18n.t("discourse_ai.llm.configuration.invalid_llm_id"), status: 422)+ render_json_error(I18n.t("discourse_ai.spam_detection.model_missing"), status: 422))end@@ -102,10 +122,13 @@ module DiscourseAiDiscourseAi::AiModeration::SpamScanner.test_post(post,custom_instructions: params[:custom_instructions],- llm_id: params[:llm_id],+ llm_model:,+ decision_model:,)render json: result+ rescue DiscourseAi::Decisions::Error => e+ render_json_error(e.message, status: 422)enddef fix_errors@@ -148,6 +171,19 @@ module DiscourseAichanges_to_log[:llm_model_id] = "#{old_model_name} → #{new_model_name}"end+ initial_decision_model_id = initial_data[:decision_model_id]+ if params.key?(:decision_model_id) &&+ initial_decision_model_id.to_s != params[:decision_model_id].to_s+ old_model_name =+ DecisionModel.find_by(id: initial_decision_model_id)&.display_name ||+ initial_decision_model_id+ new_model_name =+ DecisionModel.find_by(id: params[:decision_model_id])&.display_name ||+ params[:decision_model_id]++ changes_to_log[:decision_model_id] = "#{old_model_name} → #{new_model_name}"+ end+initial_custom_instructions = initial_data[:custom_instructions]if params.key?(:custom_instructions) &&initial_custom_instructions != params[:custom_instructions]@@ -172,7 +208,13 @@ module DiscourseAienddef allowed_params- params.permit(:is_enabled, :llm_model_id, :custom_instructions, :ai_agent_id)+ params.permit(+ :is_enabled,+ :llm_model_id,+ :decision_model_id,+ :custom_instructions,+ :ai_agent_id,+ )enddef spam_configdiff --git a/plugins/discourse-ai/app/models/ai_moderation_setting.rb b/plugins/discourse-ai/app/models/ai_moderation_setting.rbindex ffc04bf6a28..b51b133a754 100644--- a/plugins/discourse-ai/app/models/ai_moderation_setting.rb+++ b/plugins/discourse-ai/app/models/ai_moderation_setting.rb@@ -1,12 +1,13 @@# frozen_string_literal: trueclass AiModerationSetting < ActiveRecord::Base- belongs_to :llm_model+ belongs_to :llm_model, optional: true+ belongs_to :decision_model, optional: truebelongs_to :ai_agent, class_name: "AiAgent", optional: true- validates :llm_model_id, presence: truevalidates :setting_type, presence: truevalidates :setting_type, uniqueness: true+ validate :model_presentdef self.spamfind_by(setting_type: :spam)@@ -15,19 +16,28 @@ class AiModerationSetting < ActiveRecord::Basedef custom_instructionsdata&.[]("custom_instructions")end++ private++ def model_present+ if llm_model_id.blank? && decision_model_id.blank?+ errors.add(:base, I18n.t("discourse_ai.spam_detection.model_missing"))+ end+ endend# == Schema Information## Table name: ai_moderation_settings#-# id :bigint not null, primary key-# data :jsonb-# setting_type :enum not null-# created_at :datetime not null-# updated_at :datetime not null-# ai_agent_id :bigint default(-31), not null-# llm_model_id :bigint not null+# id :bigint not null, primary key+# data :jsonb+# setting_type :enum not null+# created_at :datetime not null+# updated_at :datetime not null+# ai_agent_id :bigint default(-31), not null+# decision_model_id :bigint+# llm_model_id :bigint## Indexes#diff --git a/plugins/discourse-ai/app/models/ai_spam_log.rb b/plugins/discourse-ai/app/models/ai_spam_log.rbindex 09743a43085..f36528c3060 100644--- a/plugins/discourse-ai/app/models/ai_spam_log.rb+++ b/plugins/discourse-ai/app/models/ai_spam_log.rb@@ -1,7 +1,8 @@# frozen_string_literal: trueclass AiSpamLog < ActiveRecord::Basebelongs_to :post- belongs_to :llm_model+ belongs_to :llm_model, optional: true+ belongs_to :decision_model, optional: truebelongs_to :ai_api_audit_logbelongs_to :reviewableend@@ -15,10 +16,12 @@ end# is_spam :boolean not null# payload :string(20000) default(""), not null# reason :text+# spam_probability :float# created_at :datetime not null# updated_at :datetime not null# ai_api_audit_log_id :bigint-# llm_model_id :bigint not null+# decision_model_id :bigint+# llm_model_id :bigint# post_id :bigint not null# reviewable_id :bigint#diff --git a/plugins/discourse-ai/app/serializers/ai_spam_serializer.rb b/plugins/discourse-ai/app/serializers/ai_spam_serializer.rbindex d04a596ab2a..1d6962abd28 100644--- a/plugins/discourse-ai/app/serializers/ai_spam_serializer.rb+++ b/plugins/discourse-ai/app/serializers/ai_spam_serializer.rb@@ -3,6 +3,8 @@class AiSpamSerializer < ApplicationSerializerattributes :is_enabled,:llm_id,+ :decision_model_id,+ :available_decision_models,:custom_instructions,:available_llms,:stats,@@ -20,6 +22,16 @@ class AiSpamSerializer < ApplicationSerializersettings&.llm_model&.idend+ def decision_model_id+ settings&.decision_model_id+ end++ def available_decision_models+ DiscourseAi::Configuration::DecisionModelEnumerator.values.map do |hash|+ { id: hash[:value], name: hash[:name] }+ end+ end+def ai_agent_idsettings&.ai_agent_id || settings&.ai_agent&.id ||DiscourseAi::Agents::Agent.system_agents[DiscourseAi::Agents::SpamDetector]diff --git a/plugins/discourse-ai/assets/javascripts/discourse/components/modal/spam-test-modal.gjs b/plugins/discourse-ai/assets/javascripts/discourse/components/modal/spam-test-modal.gjsindex 4b8e460fae9..0bdc13a3237 100644--- a/plugins/discourse-ai/assets/javascripts/discourse/components/modal/spam-test-modal.gjs+++ b/plugins/discourse-ai/assets/javascripts/discourse/components/modal/spam-test-modal.gjs@@ -18,6 +18,8 @@ export default class SpamTestModal extends Component {@tracked isSpam;@tracked reason;@tracked llmName;+ @tracked decisionModelName;+ @tracked spamProbability;@tracked systemPrompt;@tracked sentMessage;@tracked scanHistory;@@ -33,7 +35,8 @@ export default class SpamTestModal extends Component {data: {post_url: this.postUrl,custom_instructions: this.args.model.customInstructions,- llm_id: this.args.model.llmId,+ llm_id: this.args.model.llmId ?? "",+ decision_model_id: this.args.model.decisionModelId ?? "",},});@@ -44,6 +47,11 @@ export default class SpamTestModal extends Component {: i18n("discourse_ai.spam.test_modal.not_spam");this.reason = response.reason;this.llmName = response.llm_name;+ this.decisionModelName = response.decision_model_name;+ this.spamProbability =+ response.spam_probability == null+ ? null+ : `${Math.round(response.spam_probability * 100)}%`;this.systemPrompt = response.system_prompt;this.sentMessage = response.sent_message;this.scanHistory = response.scan_history;@@ -109,6 +117,32 @@ export default class SpamTestModal extends Component {</div>{{/if}}+ {{#if this.decisionModelName}}+ <div+ class="spam-test-modal__info-box spam-test-modal__info-box--decision-model"+ >+ <h4 class="spam-test-modal__info-box-title">{{i18n+ "discourse_ai.spam.test_modal.decision_model"+ }}</h4>+ <div class="spam-test-modal__info-box-content">+ {{this.decisionModelName}}+ </div>+ </div>+ {{/if}}++ {{#if this.spamProbability}}+ <div+ class="spam-test-modal__info-box spam-test-modal__info-box--probability"+ >+ <h4 class="spam-test-modal__info-box-title">{{i18n+ "discourse_ai.spam.test_modal.spam_probability"+ }}</h4>+ <div class="spam-test-modal__info-box-content">+ {{this.spamProbability}}+ </div>+ </div>+ {{/if}}+{{#if this.scanHistory}}<divclass="spam-test-modal__info-box spam-test-modal__info-box--history"diff --git a/plugins/discourse-ai/assets/stylesheets/modules/llms/common/spam.scss b/plugins/discourse-ai/assets/stylesheets/modules/llms/common/spam.scssindex 75a7f9262a1..85ac833e01b 100644--- a/plugins/discourse-ai/assets/stylesheets/modules/llms/common/spam.scss+++ b/plugins/discourse-ai/assets/stylesheets/modules/llms/common/spam.scss@@ -24,6 +24,7 @@&__toggle,&__llm,+ &__decision-model,&__agent,&__instructions {margin-bottom: 1em;@@ -31,6 +32,7 @@&__toggle-label,&__llm-label,+ &__decision-model-label,&__agent-label,&__instructions-label {display: block;diff --git a/plugins/discourse-ai/config/locales/client.en.yml b/plugins/discourse-ai/config/locales/client.en.ymlindex 35d77f721ad..7401cf4c696 100644--- a/plugins/discourse-ai/config/locales/client.en.yml+++ b/plugins/discourse-ai/config/locales/client.en.yml@@ -712,6 +712,10 @@ en:short_title: "Spam"title: "Configure spam handling"select_llm: "Select LLM"+ no_llm_selected: "None"+ select_decision_model: "Select decision model"+ no_decision_model_selected: "None"+ decision_model_help: "A decision model rates every post on its own when it is confident and leaves unsure posts to the LLM. Without an LLM, the decision model decides every post."select_agent: "Select agent"custom_instructions: "Custom instructions"custom_instructions_help: "Custom instructions specific to your site to help guide the AI in identifying spam, e.g. 'Be more aggressive about scanning posts not in English'."@@ -736,6 +740,8 @@ en:result: "Result"reason: "Reasoning"llm: "LLM"+ decision_model: "Decision model"+ spam_probability: "Spam probability"scan_history: "Scan History"sent_message: "Sent Message"system_prompt: "System Prompt"diff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.ymlindex ea16388271d..84866cdaf90 100644--- a/plugins/discourse-ai/config/locales/server.en.yml+++ b/plugins/discourse-ai/config/locales/server.en.yml@@ -509,6 +509,9 @@ en:bot_user_update_failed: "Failed to update the spam scanning bot user"configuration_missing: "The AI spam detection configuration is missing. Add configuration in the 'Admin > Plugins > Discourse AI > Spam' before enabling."logging_subject: "Spam detection"+ model_missing: "Select an LLM or a decision model for spam detection."+ decision_reason: "The decision model rated this post %{probability}% likely to be spam."+ decision_refused: "The decision model did not answer, so the post was not flagged."ai_bot:sources:@@ -1428,6 +1431,7 @@ en:duplicate_name: "must have unique names (case-insensitive)"secret_not_found: "The selected credential does not exist"invalid_url: "must be an http:// or https:// URL, or an srv:// URL without a port"+ delete_failed_spam: "This decision model is used for spam detection. Choose another model in the spam settings first."test_refused: "The decision model declined to answer the test question."test_image_not_read: "The decision model did not recognise the test image. Untick \"Supports image input\" if it cannot read images."credit_limit_exceeded: "%{model_name} has used all of today's AI credits. Credits reset in %{reset_time}."diff --git a/plugins/discourse-ai/db/migrate/20261030183551_add_decision_model_to_ai_spam_detection.rb b/plugins/discourse-ai/db/migrate/20261030183551_add_decision_model_to_ai_spam_detection.rbnew file mode 100644index 00000000000..46a1583fe7f--- /dev/null+++ b/plugins/discourse-ai/db/migrate/20261030183551_add_decision_model_to_ai_spam_detection.rb@@ -0,0 +1,11 @@+# frozen_string_literal: true+class AddDecisionModelToAiSpamDetection < ActiveRecord::Migration[8.1]+ def change+ add_column :ai_moderation_settings, :decision_model_id, :bigint+ change_column_null :ai_moderation_settings, :llm_model_id, true++ add_column :ai_spam_logs, :decision_model_id, :bigint+ add_column :ai_spam_logs, :spam_probability, :float+ change_column_null :ai_spam_logs, :llm_model_id, true+ end+enddiff --git a/plugins/discourse-ai/lib/ai_moderation/spam_scanner.rb b/plugins/discourse-ai/lib/ai_moderation/spam_scanner.rbindex 4b7d4b3c781..e6b259212a5 100644--- a/plugins/discourse-ai/lib/ai_moderation/spam_scanner.rb+++ b/plugins/discourse-ai/lib/ai_moderation/spam_scanner.rb@@ -10,6 +10,21 @@ module DiscourseAiSHOULD_SCAN_POST_CUSTOM_FIELD = "discourse_ai_should_scan_post"+ DECISION_FEATURE_NAME = "spam_detection"+ SPAM_QUESTION_NAME = "spam"+ SPAM_QUESTION_INSTRUCTIONS = <<~TEXT.squish+ Is this forum post spam? Answer yes for unsolicited promotion or advertising, link+ dropping, SEO or bot-generated content, gibberish, or replies unrelated to the thread.+ Answer no for genuine questions, answers, feedback and discussion, even from new users or+ non-native speakers.+ TEXT+ # A decision model settles posts outside these bounds; the LLM decides the rest.+ CLEARLY_NOT_SPAM_PROBABILITY = 0.05+ CLEARLY_SPAM_PROBABILITY = 0.9+ # Without an LLM to settle unsure posts, one cutoff decides them all.+ DECISION_ONLY_SPAM_PROBABILITY = 0.5+ MAX_DECISION_IMAGE_PIXELS = 1_600_000+def self.new_post(post)return if !enabled?return if !should_scan_post?(post)@@ -147,33 +162,55 @@ module DiscourseAidistance >= MINIMUM_EDIT_DIFFERENCEend- def self.test_post(post, custom_instructions: nil, llm_id: nil)+ def self.test_post(+ post,+ custom_instructions: nil,+ llm_model: AiModerationSetting.spam&.llm_model,+ decision_model: AiModerationSetting.spam&.decision_model+ )settings = AiModerationSetting.spam- custom_instructions = custom_instructions || settings.custom_instructions.presence+ custom_instructions =+ staff_instructions(custom_instructions || settings&.custom_instructions.presence)target_msg =build_target_content_msg(post,post.topic || Topic.with_deleted.find_by(id: post.topic_id),)- custom_insts = custom_instructions || settings.custom_instructions.presence- if custom_insts.present?- custom_insts =- "\n\nAdditional site-specific instructions provided by Staff:\n#{custom_insts}"++ verdict = {}+ if decision_model+ verdict =+ decision_verdict(+ decision_model,+ target_msg,+ custom_instructions,+ decides_alone: llm_model.nil?,+ ) || {}end- ctx =- build_bot_context(- feature_name: "spam_detection_test",- messages: [target_msg],- custom_instructions: custom_insts,- )- bot = build_scanner_bot(settings: settings, llm_id: llm_id)+ system_prompt = decision_instructions(custom_instructions) if decision_model+ if verdict[:is_spam].nil?+ ctx =+ build_bot_context(+ feature_name: "spam_detection_test",+ messages: [target_msg],+ custom_instructions: custom_instructions,+ )+ bot = build_scanner_bot(settings: settings, llm_model: llm_model)++ structured_output = nil+ llm_args = { feature_context: { post_id: post.id } }+ bot.reply(ctx, llm_args: llm_args) do |partial, _, type|+ structured_output = partial if type == :structured_output+ end- structured_output = nil- llm_args = { feature_context: { post_id: post.id } }- bot.reply(ctx, llm_args: llm_args) do |partial, _, type|- structured_output = partial if type == :structured_output+ verdict.merge!(+ is_spam: is_spam?(structured_output),+ reason: extract_reason(structured_output),+ llm_model: bot.model,+ )+ system_prompt = bot.agent.craft_prompt(ctx, llm: bot.model.to_llm).system_message_textendhistory = nil@@ -188,26 +225,14 @@ module DiscourseAihistory << "\n"end- used_model = bot.model- spam_agent = bot.agent- used_prompt = spam_agent.craft_prompt(ctx, llm: used_model.to_llm).system_message_text-- text_content =- if target_msg[:content].is_a?(Array)- target_msg[:content].first- else- target_msg[:content]- end-- is_spam = is_spam?(structured_output)- reasoning = extract_reason(structured_output)-{- is_spam: is_spam,- reason: reasoning,- llm_name: used_model.name,- system_prompt: used_prompt,- sent_message: text_content,+ is_spam: verdict[:is_spam],+ reason: verdict[:reason],+ llm_name: verdict[:llm_model]&.name,+ decision_model_name: decision_model&.display_name,+ spam_probability: verdict[:spam_probability],+ system_prompt: system_prompt,+ sent_message: text_content(target_msg),scan_history: history,post_url: post.url,}@@ -222,55 +247,43 @@ module DiscourseAidef self.perform_scan!(post, triggering_user_id: nil)return if !enabled?settings = AiModerationSetting.spam- return if !settings || !settings.llm_model-- agent = settings.ai_agent- return if !agent+ return if !settings+ decision_model = settings.decision_model+ return if !decision_model && !llm_configured?(settings)target_msg = build_target_content_msg(post)- custom_instructions = settings.custom_instructions.presence- if custom_instructions.present?- custom_instructions =- "\n\nAdditional site-specific instructions provided by Staff:\n#{custom_instructions}"- end-- ctx =- build_bot_context(- messages: [target_msg],- custom_instructions: custom_instructions,- user: flagging_user,- )- bot = build_scanner_bot(settings: settings, user: flagging_user)- structured_output = nil+ custom_instructions = staff_instructions(settings.custom_instructions.presence)begin- llm_args = { feature_context: { post_id: post.id } }- bot.reply(ctx, llm_args: llm_args) do |partial, _, type|- structured_output = partial if type == :structured_output+ verdict = {}+ if decision_model+ verdict =+ decision_verdict(+ decision_model,+ target_msg,+ custom_instructions,+ decides_alone: !llm_configured?(settings),+ ) || {}+ end+ if verdict[:is_spam].nil?+ verdict.merge!(llm_verdict(settings, target_msg, custom_instructions, post:))end- is_spam = is_spam?(structured_output)- reason = extract_reason(structured_output)-- log = AiApiAuditLog.order(id: :desc).where(feature_name: "spam_detection").first- text_content =- if target_msg[:content].is_a?(Array)- target_msg[:content].first- else- target_msg[:content]- endAiSpamLog.transaction dolog =AiSpamLog.create!(post: post,- llm_model: settings.llm_model,- ai_api_audit_log: log,- is_spam: is_spam,- payload: text_content,- reason: reason,+ payload: text_content(target_msg),+ ai_api_audit_log_id: verdict[:audit_log_id],+ **verdict.slice(:is_spam, :reason, :spam_probability, :llm_model, :decision_model),)- handle_spam(post, log, triggering_user_id: triggering_user_id) if is_spam+ handle_spam(post, log, triggering_user_id: triggering_user_id) if log.is_spamend+ rescue DiscourseAi::Decisions::CreditLimitExceeded+ # with no LLM to fall back on, the post goes unscanned until credits reset+ Rails.logger.warn(+ "Discourse AI: Spam scan skipped for post #{post.id}, the decision model is out of credits",+ )rescue StandardError => e# we need retries otherwise stuff will not be handledDiscourse.warn_exception(@@ -313,17 +326,131 @@ module DiscourseAidef self.build_scanner_bot(settings:,- use_structured_output: true,- llm_id: nil,+ llm_model: settings.llm_model,user: Discourse.system_user)- agent = settings.ai_agent.class_instance&.new-- llm_model = llm_id ? LlmModel.find(llm_id) : settings.llm_model+ ai_agent =+ settings&.ai_agent ||+ AiAgent.find_by_id_from_cache(+ DiscourseAi::Agents::Agent.system_agents[DiscourseAi::Agents::SpamDetector],+ )+ agent = ai_agent.class_instance&.newDiscourseAi::Agents::Bot.as(user, agent: agent, model: llm_model)end+ def self.llm_configured?(settings)+ settings.llm_model.present? && settings.ai_agent.present?+ end++ def self.staff_instructions(custom_instructions)+ return if custom_instructions.blank?++ "\n\nAdditional site-specific instructions provided by Staff:\n#{custom_instructions}"+ end++ def self.decision_instructions(custom_instructions)+ "#{SPAM_QUESTION_INSTRUCTIONS}#{custom_instructions}"+ end++ def self.text_content(target_msg)+ content = target_msg[:content]+ content.is_a?(Array) ? content.first : content+ end++ # @return [Hash, nil] the verdict, with a nil :is_spam when the LLM should decide; nil+ # when the decision model could not answer and the LLM should decide+ # @raise [DiscourseAi::Decisions::Error] when the decision model decides alone+ def self.decision_verdict(decision_model, target_msg, custom_instructions, decides_alone:)+ result =+ decision_model.decide(+ text_content(target_msg),+ questions: [+ {+ name: SPAM_QUESTION_NAME,+ type: "yes_no",+ instructions: decision_instructions(custom_instructions),+ },+ ],+ feature_name: DECISION_FEATURE_NAME,+ user: Discourse.system_user,+ images: decision_images(decision_model, target_msg),+ )+ verdict = { decision_model:, audit_log_id: result.audit_log_id }++ answer = result[SPAM_QUESTION_NAME]+ if answer.refusal?+ return if !decides_alone+ return(+ verdict.merge(+ is_spam: false,+ reason: I18n.t("discourse_ai.spam_detection.decision_refused"),+ )+ )+ end++ probability = answer.probability+ is_spam =+ if decides_alone+ probability >= DECISION_ONLY_SPAM_PROBABILITY+ elsif probability >= CLEARLY_SPAM_PROBABILITY+ true+ elsif probability <= CLEARLY_NOT_SPAM_PROBABILITY+ false+ end+ reason =+ if !is_spam.nil?+ I18n.t(+ "discourse_ai.spam_detection.decision_reason",+ probability: (probability * 100).round,+ )+ end++ verdict.merge(is_spam:, reason:, spam_probability: probability)+ rescue DiscourseAi::Decisions::Error => e+ raise if decides_alone++ Rails.logger.warn("Discourse AI: Spam decision model failed, the LLM decides: #{e.class}")+ nil+ end++ # Images only add evidence, so a post whose images cannot be prepared is judged on its text.+ def self.decision_images(decision_model, target_msg)+ return [] if !decision_model.vision_enabled || !target_msg[:content].is_a?(Array)++ upload_ids = target_msg[:content].filter_map { |part| part[:upload_id] if part.is_a?(Hash) }+ DiscourseAi::Completions::UploadEncoder+ .encode(upload_ids:, max_pixels: MAX_DECISION_IMAGE_PIXELS)+ .map { |image| "data:#{image[:mime_type]};base64,#{image[:base64]}" }+ rescue StandardError => e+ Rails.logger.warn("Discourse AI: Could not prepare images for spam detection: #{e.message}")+ []+ end++ def self.llm_verdict(settings, target_msg, custom_instructions, post:)+ ctx =+ build_bot_context(+ messages: [target_msg],+ custom_instructions: custom_instructions,+ user: flagging_user,+ )+ bot = build_scanner_bot(settings: settings, user: flagging_user)+ structured_output = nil++ llm_args = { feature_context: { post_id: post.id } }+ bot.reply(ctx, llm_args: llm_args) do |partial, _, type|+ structured_output = partial if type == :structured_output+ end++ {+ is_spam: is_spam?(structured_output),+ reason: extract_reason(structured_output),+ llm_model: settings.llm_model,+ audit_log_id:+ AiApiAuditLog.order(id: :desc).where(feature_name: "spam_detection").pick(:id),+ }+ end+def self.is_spam?(structured_output)structured_output.present? && structured_output.read_buffered_property(:spam)enddiff --git a/plugins/discourse-ai/lib/configuration/llm_enumerator.rb b/plugins/discourse-ai/lib/configuration/llm_enumerator.rbindex 0b5c9ddcba4..d8cab6726e1 100644--- a/plugins/discourse-ai/lib/configuration/llm_enumerator.rb+++ b/plugins/discourse-ai/lib/configuration/llm_enumerator.rb@@ -86,8 +86,8 @@ module DiscourseAi}end- if SiteSetting.ai_spam_detection_enabled && AiModerationSetting.spam.present?- model_id = AiModerationSetting.spam[:llm_model_id]+ if SiteSetting.ai_spam_detection_enabled &&+ (model_id = AiModerationSetting.spam&.llm_model_id).present?rval[model_id] << { type: :ai_spam, id: DiscourseAi::Configuration::Module::SPAM_ID }enddiff --git a/plugins/discourse-ai/spec/lib/modules/ai_moderation/spam_scanner_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_moderation/spam_scanner_spec.rbindex a823a698a52..49e358b9665 100644--- a/plugins/discourse-ai/spec/lib/modules/ai_moderation/spam_scanner_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/ai_moderation/spam_scanner_spec.rb@@ -170,6 +170,78 @@ RSpec.describe DiscourseAi::AiModeration::SpamScanner doendend+ describe ".perform_scan! with a decision model" do+ fab!(:decision_model) { Fabricate(:decision_model, id: -1) }++ before { spam_setting.update!(decision_model:) }++ def scan_with(probability, llm_responses: [])+ DiscourseAi::Decisions.with_canned_answers(spam: probability) do |canned|+ DiscourseAi::Completions::Llm.with_prepared_responses(llm_responses) do |_, _, prompts|+ described_class.perform_scan!(post, triggering_user_id: post.user_id)+ [canned.calls, prompts]+ end+ end+ end++ it "flags confident spam without asking the LLM" do+ decision_calls, prompts = scan_with(0.95)++ log = AiSpamLog.find_by!(post:)+ expect([log.is_spam, log.spam_probability, log.decision_model_id, log.llm_model_id]).to eq(+ [true, 0.95, decision_model.id, nil],+ )+ expect(log.reviewable).to be_present+ expect(post.user.reload).to be_silenced+ expect(prompts).to be_empty+ expect(decision_calls.first[:questions].first.instructions).to include("test instructions")+ end++ it "clears confident non-spam without asking the LLM" do+ _, prompts = scan_with(0.01)++ expect(AiSpamLog.find_by!(post:).is_spam).to eq(false)+ expect(prompts).to be_empty+ end++ it "lets the LLM decide unsure posts and keeps the decision model's probability" do+ scan_with(0.5, llm_responses: [{ spam: true, reason: "llm reason" }])++ log = AiSpamLog.find_by!(post:)+ expect([log.is_spam, log.reason, log.spam_probability, log.llm_model_id]).to eq(+ [true, "llm reason", 0.5, llm_model.id],+ )+ end++ it "lets the LLM decide when the decision model fails" do+ stub_request(:post, decision_model.url).to_return(status: 400, body: "{}")++ DiscourseAi::Completions::Llm.with_prepared_responses([{ spam: false }]) do+ described_class.perform_scan!(post)+ end++ expect(AiSpamLog.find_by!(post:).llm_model_id).to eq(llm_model.id)+ end++ it "decides every post at one cutoff without an LLM" do+ spam_setting.update!(llm_model: nil)++ scan_with(0.6)+ expect(AiSpamLog.find_by!(post:).is_spam).to eq(true)++ AiSpamLog.delete_all+ scan_with(0.4)+ expect(AiSpamLog.find_by!(post:).is_spam).to eq(false)+ end++ it "skips the scan without an LLM once the decision model is out of credits" do+ spam_setting.update!(llm_model: nil)+ Fabricate(:decision_model_credit_allocation, decision_model:, daily_used: 10_000)++ expect { described_class.perform_scan!(post) }.not_to change { AiSpamLog.count }+ end+ end+describe ".scanned_max_times?" doit "returns true when post has been scanned 3 times" do3.times do@@ -406,6 +478,17 @@ RSpec.describe DiscourseAi::AiModeration::SpamScanner doexpect(result[:is_spam]).to eq(false)end+ it "tests with the spam detector agent before any spam settings are saved" do+ spam_setting.destroy!++ result =+ DiscourseAi::Completions::Llm.with_prepared_responses([{ spam: true }]) do+ described_class.test_post(post, llm_model:, decision_model: nil)+ end++ expect(result[:is_spam]).to eq(true)+ end+it "correctly handles spam scanning" doexpect(described_class.flagging_user.id).not_to eq(Discourse.system_user.id)diff --git a/plugins/discourse-ai/spec/requests/admin/ai_decision_models_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/ai_decision_models_controller_spec.rbindex 24feb6dff6d..c23db3aaff8 100644--- a/plugins/discourse-ai/spec/requests/admin/ai_decision_models_controller_spec.rb+++ b/plugins/discourse-ai/spec/requests/admin/ai_decision_models_controller_spec.rb@@ -297,6 +297,15 @@ RSpec.describe DiscourseAi::Admin::AiDecisionModelsController do).count,).to eq(1)end++ it "refuses to delete a decision model used for spam detection" do+ AiModerationSetting.create!(setting_type: :spam, decision_model:)++ delete "/admin/plugins/discourse-ai/ai-decision-models/#{decision_model.id}.json"++ expect(response.status).to eq(409)+ expect(DecisionModel.exists?(decision_model.id)).to eq(true)+ endenddescribe "#test" dodiff --git a/plugins/discourse-ai/spec/requests/admin/ai_spam_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/ai_spam_controller_spec.rbindex 87fefa96716..96ca467dd39 100644--- a/plugins/discourse-ai/spec/requests/admin/ai_spam_controller_spec.rb+++ b/plugins/discourse-ai/spec/requests/admin/ai_spam_controller_spec.rb@@ -70,6 +70,25 @@ RSpec.describe DiscourseAi::Admin::AiSpamController doexpect(response.status).to eq(422)end+ it "can enable spam detection with only a decision model" do+ decision_model = Fabricate(:decision_model)++ put "/admin/plugins/discourse-ai/ai-spam.json",+ params: {+ is_enabled: true,+ llm_model_id: "",+ decision_model_id: decision_model.id,+ }++ expect(response.status).to eq(200)+ expect(SiteSetting.ai_spam_detection_enabled).to eq(true)+ expect(AiModerationSetting.spam.slice(:llm_model_id, :decision_model_id)).to eq(+ "llm_model_id" => nil,+ "decision_model_id" => decision_model.id,+ )+ expect(response.parsed_body["decision_model_id"]).to eq(decision_model.id)+ end+context "when spam detection was already set" dofab!(:setting) doAiModerationSetting.create(@@ -277,6 +296,28 @@ RSpec.describe DiscourseAi::Admin::AiSpamController doexpect(parsed["reason"]).to eq("because apples")end+ it "tests with only a decision model" do+ decision_model = Fabricate(:decision_model)++ DiscourseAi::Decisions.with_canned_answers(spam: 0.8) do+ post "/admin/plugins/discourse-ai/ai-spam/test.json",+ params: {+ post_url: spam_post.url,+ llm_id: "",+ decision_model_id: decision_model.id,+ }+ end++ expect(response.status).to eq(200)+ expect(+ response.parsed_body.slice("is_spam", "spam_probability", "decision_model_name"),+ ).to eq(+ "is_spam" => true,+ "spam_probability" => 0.8,+ "decision_model_name" => decision_model.display_name,+ )+ end+it "rejects llm_id that is not a valid LlmModel" dopost "/admin/plugins/discourse-ai/ai-spam/test.json",params: {diff --git a/plugins/discourse-ai/spec/system/ai_moderation/ai_spam_spec.rb b/plugins/discourse-ai/spec/system/ai_moderation/ai_spam_spec.rbindex 5ce1cc0a7f4..9543118becb 100644--- a/plugins/discourse-ai/spec/system/ai_moderation/ai_spam_spec.rb+++ b/plugins/discourse-ai/spec/system/ai_moderation/ai_spam_spec.rb@@ -18,7 +18,7 @@ RSpec.describe "AI Spam Configuration" dotoggle.toggledialog = PageObjects::Components::Dialog.new- expect(dialog).to have_content(I18n.t("discourse_ai.llm.configuration.must_select_model"))+ expect(dialog).to have_content(I18n.t("discourse_ai.spam_detection.model_missing"))dialog.click_okexpect(toggle.unchecked?).to eq(true)@@ -49,5 +49,27 @@ RSpec.describe "AI Spam Configuration" doI18n.t("js.discourse_ai.llms.usage.ai_spam"),)end++ it "can detect spam with only a decision model" do+ decision_model = Fabricate(:decision_model)+ visit "/admin/plugins/discourse-ai/ai-spam"++ PageObjects::Components::SelectKit.new(+ ".ai-spam__decision-model-selector",+ ).select_row_by_value(decision_model.id)+ llm_selector = PageObjects::Components::SelectKit.new(".ai-spam__llm-selector")+ llm_selector.expand+ llm_selector.select_row_by_index(0)+ expect(page).to have_no_css(".ai-spam__agent-selector")++ find(".ai-spam__instructions-save").click+ expect(PageObjects::Components::Toasts.new).to have_content(+ I18n.t("js.discourse_ai.spam.settings_saved"),+ )+ expect(AiModerationSetting.spam.slice(:llm_model_id, :decision_model_id)).to eq(+ "llm_model_id" => nil,+ "decision_model_id" => decision_model.id,+ )+ endendend