Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Allow detecting spam with a decision model (#45413)

Rafael dos Santos Silvaefafad060b70
diff --git a/db/structure.sql b/db/structure.sql
index 4f1198034c4..6a9c77eae45 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -917,10 +917,11 @@ CREATE TABLE public.ai_moderation_settings (
id bigint NOT NULL,
setting_type public.ai_moderation_setting_type NOT NULL,
data jsonb DEFAULT '{}'::jsonb,
- llm_model_id bigint NOT NULL,
+ llm_model_id bigint,
created_at timestamp(6) without time zone NOT NULL,
updated_at timestamp(6) without time zone NOT NULL,
- ai_agent_id bigint DEFAULT '-31'::integer NOT NULL
+ ai_agent_id bigint DEFAULT '-31'::integer NOT NULL,
+ decision_model_id bigint
);
@@ -1038,7 +1039,7 @@ ALTER SEQUENCE public.ai_secrets_id_seq OWNED BY public.ai_secrets.id;
CREATE TABLE public.ai_spam_logs (
id bigint NOT NULL,
post_id bigint NOT NULL,
- llm_model_id bigint NOT NULL,
+ llm_model_id bigint,
ai_api_audit_log_id bigint,
reviewable_id bigint,
is_spam boolean NOT NULL,
@@ -1046,7 +1047,9 @@ CREATE TABLE public.ai_spam_logs (
created_at timestamp(6) without time zone NOT NULL,
updated_at timestamp(6) without time zone NOT NULL,
error character varying(3000),
- reason text
+ reason text,
+ decision_model_id bigint,
+ spam_probability double precision
);
@@ -26520,6 +26523,7 @@ SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
('20261102174233'),
('20261030215104'),
+('20261030183551'),
('20261030180958'),
('20261029021723'),
('20261028093149'),
diff --git a/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-spam.gjs b/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-spam.gjs
index 52414ff00f5..a8f3652518e 100644
--- a/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-spam.gjs
+++ b/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-spam.gjs
@@ -1,6 +1,6 @@
import Component from "@glimmer/component";
import { tracked } from "@glimmer/tracking";
-import { fn } from "@ember/helper";
+import { fn, hash } from "@ember/helper";
import { on } from "@ember/modifier";
import { action } from "@ember/object";
import { LinkTo } from "@ember/routing";
@@ -36,6 +36,7 @@ export default class AiSpam extends Component {
};
@tracked isEnabled = false;
@tracked selectedLLM = null;
+ @tracked selectedDecisionModelId = null;
@tracked selectedAgentId = null;
@tracked customInstructions = "";
@autoTrackedArray errors;
@@ -59,6 +60,10 @@ export default class AiSpam extends Component {
return this.args.model?.available_llms || [];
}
+ get availableDecisionModels() {
+ return this.args.model?.available_decision_models || [];
+ }
+
get availableAgents() {
return this.args.model?.available_agents || [];
}
@@ -150,13 +155,12 @@ export default class AiSpam extends Component {
const model = this.args.model;
this.isEnabled = model.is_enabled;
+ this.selectedDecisionModelId = model.decision_model_id;
if (model.llm_id) {
this.selectedLLM = model.llm_id;
- } else {
- if (this.availableLLMs.length) {
- this.selectedLLM = this.availableLLMs[0].id;
- this.autoSelectedLLM = true;
- }
+ } else if (!model.decision_model_id && this.availableLLMs.length) {
+ this.selectedLLM = this.availableLLMs[0].id;
+ this.autoSelectedLLM = true;
}
this.customInstructions = model.custom_instructions;
this.stats = model.stats;
@@ -192,6 +196,11 @@ export default class AiSpam extends Component {
this.selectedLLM = value;
}
+ @action
+ async updateDecisionModel(value) {
+ this.selectedDecisionModelId = value;
+ }
+
@action
async updateAgent(value) {
this.selectedAgentId = value;
@@ -203,7 +212,8 @@ export default class AiSpam extends Component {
await ajax("/admin/plugins/discourse-ai/ai-spam.json", {
type: "PUT",
data: {
- llm_model_id: this.llmId,
+ llm_model_id: this.llmId ?? "",
+ decision_model_id: this.selectedDecisionModelId ?? "",
custom_instructions: this.customInstructions,
ai_agent_id: this.selectedAgentId,
},
@@ -223,6 +233,7 @@ export default class AiSpam extends Component {
model: {
customInstructions: this.customInstructions,
llmId: this.llmId,
+ decisionModelId: this.selectedDecisionModelId,
},
});
}
@@ -273,6 +284,7 @@ export default class AiSpam extends Component {
class="ai-spam__llm-selector"
@content={{this.availableLLMs}}
@onChange={{this.updateLLM}}
+ @options={{hash none="discourse_ai.spam.no_llm_selected"}}
@value={{this.selectedLLM}}
/>
{{else}}
@@ -284,17 +296,40 @@ export default class AiSpam extends Component {
{{/if}}
</div>
- <div class="ai-spam__agent">
- <label class="ai-spam__agent-label">{{i18n
- "discourse_ai.spam.select_agent"
- }}</label>
- <ComboBox
- class="ai-spam__agent-selector"
- @content={{this.availableAgents}}
- @onChange={{this.updateAgent}}
- @value={{this.selectedAgentId}}
- />
- </div>
+ {{#if this.availableDecisionModels.length}}
+ <div class="ai-spam__decision-model">
+ <label class="ai-spam__decision-model-label">
+ {{i18n "discourse_ai.spam.select_decision_model"}}
+ <DTooltip
+ @content={{i18n "discourse_ai.spam.decision_model_help"}}
+ @icon="circle-question"
+ />
+ </label>
+ <ComboBox
+ class="ai-spam__decision-model-selector"
+ @content={{this.availableDecisionModels}}
+ @onChange={{this.updateDecisionModel}}
+ @options={{hash
+ none="discourse_ai.spam.no_decision_model_selected"
+ }}
+ @value={{this.selectedDecisionModelId}}
+ />
+ </div>
+ {{/if}}
+
+ {{#if this.selectedLLM}}
+ <div class="ai-spam__agent">
+ <label class="ai-spam__agent-label">{{i18n
+ "discourse_ai.spam.select_agent"
+ }}</label>
+ <ComboBox
+ class="ai-spam__agent-selector"
+ @content={{this.availableAgents}}
+ @onChange={{this.updateAgent}}
+ @value={{this.selectedAgentId}}
+ />
+ </div>
+ {{/if}}
<div class="ai-spam__instructions">
<label class="ai-spam__instructions-label">
diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_decision_models_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_decision_models_controller.rb
index 3dc55254c3e..d28086a3740 100644
--- a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_decision_models_controller.rb
+++ b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_decision_models_controller.rb
@@ -77,6 +77,14 @@ module DiscourseAi
def destroy
decision_model = DecisionModel.find(params[:id])
return render_seeded_error if decision_model.seeded?
+ if AiModerationSetting.exists?(decision_model_id: decision_model.id)
+ return(
+ render_json_error(
+ I18n.t("discourse_ai.decision_models.delete_failed_spam"),
+ status: 409,
+ )
+ )
+ end
details =
decision_model
diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_spam_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_spam_controller.rb
index 86cad4449f4..7c773fbf6d1 100644
--- a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_spam_controller.rb
+++ b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_spam_controller.rb
@@ -15,12 +15,16 @@ module DiscourseAi
initial_data = {
custom_instructions: initial_settings&.data&.dig("custom_instructions"),
llm_model_id: initial_settings&.llm_model_id,
+ decision_model_id: initial_settings&.decision_model_id,
ai_agent_id: initial_settings&.ai_agent_id,
}
updated_params = {}
if allowed_params.key?(:llm_model_id)
- updated_params[:llm_model_id] = allowed_params[:llm_model_id]
+ updated_params[:llm_model_id] = allowed_params[:llm_model_id].presence
+ end
+ if allowed_params.key?(:decision_model_id)
+ updated_params[:decision_model_id] = allowed_params[:decision_model_id].presence
end
ai_agent_id = allowed_params[:ai_agent_id]
@@ -45,11 +49,9 @@ module DiscourseAi
if updated_params.present?
# not using upsert cause we will not get the correct validation errors
- if AiModerationSetting.spam
- AiModerationSetting.spam.update!(updated_params)
- else
- AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
- end
+ settings = AiModerationSetting.spam || AiModerationSetting.new(setting_type: :spam)
+ settings.assign_attributes(updated_params)
+ return render_json_error(settings) if !settings.save
log_ai_spam_update(initial_data, allowed_params)
end
@@ -57,12 +59,9 @@ module DiscourseAi
is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
if allowed_params.key?(:is_enabled)
- if is_enabled && !AiModerationSetting.spam&.llm_model_id
+ if is_enabled && !AiModerationSetting.spam
return(
- render_json_error(
- I18n.t("discourse_ai.llm.configuration.must_select_model"),
- status: 422,
- )
+ render_json_error(I18n.t("discourse_ai.spam_detection.model_missing"), status: 422)
)
end
@@ -92,9 +91,30 @@ module DiscourseAi
raise Discourse::NotFound if !post
- if params[:llm_id].present? && !LlmModel.exists?(id: params[:llm_id])
+ settings = AiModerationSetting.spam
+ llm_model = params.key?(:llm_id) ? nil : settings&.llm_model
+ if params[:llm_id].present?
+ llm_model = LlmModel.find_by(id: params[:llm_id])
+ if !llm_model
+ return(
+ render_json_error(
+ I18n.t("discourse_ai.llm.configuration.invalid_llm_id"),
+ status: 422,
+ )
+ )
+ end
+ end
+ decision_model =
+ if params.key?(:decision_model_id)
+ if params[:decision_model_id].present?
+ DecisionModel.find_by(id: params[:decision_model_id])
+ end
+ else
+ settings&.decision_model
+ end
+ if !llm_model && !decision_model
return(
- render_json_error(I18n.t("discourse_ai.llm.configuration.invalid_llm_id"), status: 422)
+ render_json_error(I18n.t("discourse_ai.spam_detection.model_missing"), status: 422)
)
end
@@ -102,10 +122,13 @@ module DiscourseAi
DiscourseAi::AiModeration::SpamScanner.test_post(
post,
custom_instructions: params[:custom_instructions],
- llm_id: params[:llm_id],
+ llm_model:,
+ decision_model:,
)
render json: result
+ rescue DiscourseAi::Decisions::Error => e
+ render_json_error(e.message, status: 422)
end
def fix_errors
@@ -148,6 +171,19 @@ module DiscourseAi
changes_to_log[:llm_model_id] = "#{old_model_name} → #{new_model_name}"
end
+ initial_decision_model_id = initial_data[:decision_model_id]
+ if params.key?(:decision_model_id) &&
+ initial_decision_model_id.to_s != params[:decision_model_id].to_s
+ old_model_name =
+ DecisionModel.find_by(id: initial_decision_model_id)&.display_name ||
+ initial_decision_model_id
+ new_model_name =
+ DecisionModel.find_by(id: params[:decision_model_id])&.display_name ||
+ params[:decision_model_id]
+
+ changes_to_log[:decision_model_id] = "#{old_model_name} → #{new_model_name}"
+ end
+
initial_custom_instructions = initial_data[:custom_instructions]
if params.key?(:custom_instructions) &&
initial_custom_instructions != params[:custom_instructions]
@@ -172,7 +208,13 @@ module DiscourseAi
end
def allowed_params
- params.permit(:is_enabled, :llm_model_id, :custom_instructions, :ai_agent_id)
+ params.permit(
+ :is_enabled,
+ :llm_model_id,
+ :decision_model_id,
+ :custom_instructions,
+ :ai_agent_id,
+ )
end
def spam_config
diff --git a/plugins/discourse-ai/app/models/ai_moderation_setting.rb b/plugins/discourse-ai/app/models/ai_moderation_setting.rb
index ffc04bf6a28..b51b133a754 100644
--- a/plugins/discourse-ai/app/models/ai_moderation_setting.rb
+++ b/plugins/discourse-ai/app/models/ai_moderation_setting.rb
@@ -1,12 +1,13 @@
# frozen_string_literal: true
class AiModerationSetting < ActiveRecord::Base
- belongs_to :llm_model
+ belongs_to :llm_model, optional: true
+ belongs_to :decision_model, optional: true
belongs_to :ai_agent, class_name: "AiAgent", optional: true
- validates :llm_model_id, presence: true
validates :setting_type, presence: true
validates :setting_type, uniqueness: true
+ validate :model_present
def self.spam
find_by(setting_type: :spam)
@@ -15,19 +16,28 @@ class AiModerationSetting < ActiveRecord::Base
def custom_instructions
data&.[]("custom_instructions")
end
+
+ private
+
+ def model_present
+ if llm_model_id.blank? && decision_model_id.blank?
+ errors.add(:base, I18n.t("discourse_ai.spam_detection.model_missing"))
+ end
+ end
end
# == Schema Information
#
# Table name: ai_moderation_settings
#
-# id :bigint not null, primary key
-# data :jsonb
-# setting_type :enum not null
-# created_at :datetime not null
-# updated_at :datetime not null
-# ai_agent_id :bigint default(-31), not null
-# llm_model_id :bigint not null
+# id :bigint not null, primary key
+# data :jsonb
+# setting_type :enum not null
+# created_at :datetime not null
+# updated_at :datetime not null
+# ai_agent_id :bigint default(-31), not null
+# decision_model_id :bigint
+# llm_model_id :bigint
#
# Indexes
#
diff --git a/plugins/discourse-ai/app/models/ai_spam_log.rb b/plugins/discourse-ai/app/models/ai_spam_log.rb
index 09743a43085..f36528c3060 100644
--- a/plugins/discourse-ai/app/models/ai_spam_log.rb
+++ b/plugins/discourse-ai/app/models/ai_spam_log.rb
@@ -1,7 +1,8 @@
# frozen_string_literal: true
class AiSpamLog < ActiveRecord::Base
belongs_to :post
- belongs_to :llm_model
+ belongs_to :llm_model, optional: true
+ belongs_to :decision_model, optional: true
belongs_to :ai_api_audit_log
belongs_to :reviewable
end
@@ -15,10 +16,12 @@ end
# is_spam :boolean not null
# payload :string(20000) default(""), not null
# reason :text
+# spam_probability :float
# created_at :datetime not null
# updated_at :datetime not null
# ai_api_audit_log_id :bigint
-# llm_model_id :bigint not null
+# decision_model_id :bigint
+# llm_model_id :bigint
# post_id :bigint not null
# reviewable_id :bigint
#
diff --git a/plugins/discourse-ai/app/serializers/ai_spam_serializer.rb b/plugins/discourse-ai/app/serializers/ai_spam_serializer.rb
index d04a596ab2a..1d6962abd28 100644
--- a/plugins/discourse-ai/app/serializers/ai_spam_serializer.rb
+++ b/plugins/discourse-ai/app/serializers/ai_spam_serializer.rb
@@ -3,6 +3,8 @@
class AiSpamSerializer < ApplicationSerializer
attributes :is_enabled,
:llm_id,
+ :decision_model_id,
+ :available_decision_models,
:custom_instructions,
:available_llms,
:stats,
@@ -20,6 +22,16 @@ class AiSpamSerializer < ApplicationSerializer
settings&.llm_model&.id
end
+ def decision_model_id
+ settings&.decision_model_id
+ end
+
+ def available_decision_models
+ DiscourseAi::Configuration::DecisionModelEnumerator.values.map do |hash|
+ { id: hash[:value], name: hash[:name] }
+ end
+ end
+
def ai_agent_id
settings&.ai_agent_id || settings&.ai_agent&.id ||
DiscourseAi::Agents::Agent.system_agents[DiscourseAi::Agents::SpamDetector]
diff --git a/plugins/discourse-ai/assets/javascripts/discourse/components/modal/spam-test-modal.gjs b/plugins/discourse-ai/assets/javascripts/discourse/components/modal/spam-test-modal.gjs
index 4b8e460fae9..0bdc13a3237 100644
--- a/plugins/discourse-ai/assets/javascripts/discourse/components/modal/spam-test-modal.gjs
+++ b/plugins/discourse-ai/assets/javascripts/discourse/components/modal/spam-test-modal.gjs
@@ -18,6 +18,8 @@ export default class SpamTestModal extends Component {
@tracked isSpam;
@tracked reason;
@tracked llmName;
+ @tracked decisionModelName;
+ @tracked spamProbability;
@tracked systemPrompt;
@tracked sentMessage;
@tracked scanHistory;
@@ -33,7 +35,8 @@ export default class SpamTestModal extends Component {
data: {
post_url: this.postUrl,
custom_instructions: this.args.model.customInstructions,
- llm_id: this.args.model.llmId,
+ llm_id: this.args.model.llmId ?? "",
+ decision_model_id: this.args.model.decisionModelId ?? "",
},
}
);
@@ -44,6 +47,11 @@ export default class SpamTestModal extends Component {
: i18n("discourse_ai.spam.test_modal.not_spam");
this.reason = response.reason;
this.llmName = response.llm_name;
+ this.decisionModelName = response.decision_model_name;
+ this.spamProbability =
+ response.spam_probability == null
+ ? null
+ : `${Math.round(response.spam_probability * 100)}%`;
this.systemPrompt = response.system_prompt;
this.sentMessage = response.sent_message;
this.scanHistory = response.scan_history;
@@ -109,6 +117,32 @@ export default class SpamTestModal extends Component {
</div>
{{/if}}
+ {{#if this.decisionModelName}}
+ <div
+ class="spam-test-modal__info-box spam-test-modal__info-box--decision-model"
+ >
+ <h4 class="spam-test-modal__info-box-title">{{i18n
+ "discourse_ai.spam.test_modal.decision_model"
+ }}</h4>
+ <div class="spam-test-modal__info-box-content">
+ {{this.decisionModelName}}
+ </div>
+ </div>
+ {{/if}}
+
+ {{#if this.spamProbability}}
+ <div
+ class="spam-test-modal__info-box spam-test-modal__info-box--probability"
+ >
+ <h4 class="spam-test-modal__info-box-title">{{i18n
+ "discourse_ai.spam.test_modal.spam_probability"
+ }}</h4>
+ <div class="spam-test-modal__info-box-content">
+ {{this.spamProbability}}
+ </div>
+ </div>
+ {{/if}}
+
{{#if this.scanHistory}}
<div
class="spam-test-modal__info-box spam-test-modal__info-box--history"
diff --git a/plugins/discourse-ai/assets/stylesheets/modules/llms/common/spam.scss b/plugins/discourse-ai/assets/stylesheets/modules/llms/common/spam.scss
index 75a7f9262a1..85ac833e01b 100644
--- a/plugins/discourse-ai/assets/stylesheets/modules/llms/common/spam.scss
+++ b/plugins/discourse-ai/assets/stylesheets/modules/llms/common/spam.scss
@@ -24,6 +24,7 @@
&__toggle,
&__llm,
+ &__decision-model,
&__agent,
&__instructions {
margin-bottom: 1em;
@@ -31,6 +32,7 @@
&__toggle-label,
&__llm-label,
+ &__decision-model-label,
&__agent-label,
&__instructions-label {
display: block;
diff --git a/plugins/discourse-ai/config/locales/client.en.yml b/plugins/discourse-ai/config/locales/client.en.yml
index 35d77f721ad..7401cf4c696 100644
--- a/plugins/discourse-ai/config/locales/client.en.yml
+++ b/plugins/discourse-ai/config/locales/client.en.yml
@@ -712,6 +712,10 @@ en:
short_title: "Spam"
title: "Configure spam handling"
select_llm: "Select LLM"
+ no_llm_selected: "None"
+ select_decision_model: "Select decision model"
+ no_decision_model_selected: "None"
+ decision_model_help: "A decision model rates every post on its own when it is confident and leaves unsure posts to the LLM. Without an LLM, the decision model decides every post."
select_agent: "Select agent"
custom_instructions: "Custom instructions"
custom_instructions_help: "Custom instructions specific to your site to help guide the AI in identifying spam, e.g. 'Be more aggressive about scanning posts not in English'."
@@ -736,6 +740,8 @@ en:
result: "Result"
reason: "Reasoning"
llm: "LLM"
+ decision_model: "Decision model"
+ spam_probability: "Spam probability"
scan_history: "Scan History"
sent_message: "Sent Message"
system_prompt: "System Prompt"
diff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.yml
index ea16388271d..84866cdaf90 100644
--- a/plugins/discourse-ai/config/locales/server.en.yml
+++ b/plugins/discourse-ai/config/locales/server.en.yml
@@ -509,6 +509,9 @@ en:
bot_user_update_failed: "Failed to update the spam scanning bot user"
configuration_missing: "The AI spam detection configuration is missing. Add configuration in the 'Admin > Plugins > Discourse AI > Spam' before enabling."
logging_subject: "Spam detection"
+ model_missing: "Select an LLM or a decision model for spam detection."
+ decision_reason: "The decision model rated this post %{probability}% likely to be spam."
+ decision_refused: "The decision model did not answer, so the post was not flagged."
ai_bot:
sources:
@@ -1428,6 +1431,7 @@ en:
duplicate_name: "must have unique names (case-insensitive)"
secret_not_found: "The selected credential does not exist"
invalid_url: "must be an http:// or https:// URL, or an srv:// URL without a port"
+ delete_failed_spam: "This decision model is used for spam detection. Choose another model in the spam settings first."
test_refused: "The decision model declined to answer the test question."
test_image_not_read: "The decision model did not recognise the test image. Untick \"Supports image input\" if it cannot read images."
credit_limit_exceeded: "%{model_name} has used all of today's AI credits. Credits reset in %{reset_time}."
diff --git a/plugins/discourse-ai/db/migrate/20261030183551_add_decision_model_to_ai_spam_detection.rb b/plugins/discourse-ai/db/migrate/20261030183551_add_decision_model_to_ai_spam_detection.rb
new file mode 100644
index 00000000000..46a1583fe7f
--- /dev/null
+++ b/plugins/discourse-ai/db/migrate/20261030183551_add_decision_model_to_ai_spam_detection.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+class AddDecisionModelToAiSpamDetection < ActiveRecord::Migration[8.1]
+ def change
+ add_column :ai_moderation_settings, :decision_model_id, :bigint
+ change_column_null :ai_moderation_settings, :llm_model_id, true
+
+ add_column :ai_spam_logs, :decision_model_id, :bigint
+ add_column :ai_spam_logs, :spam_probability, :float
+ change_column_null :ai_spam_logs, :llm_model_id, true
+ end
+end
diff --git a/plugins/discourse-ai/lib/ai_moderation/spam_scanner.rb b/plugins/discourse-ai/lib/ai_moderation/spam_scanner.rb
index 4b7d4b3c781..e6b259212a5 100644
--- a/plugins/discourse-ai/lib/ai_moderation/spam_scanner.rb
+++ b/plugins/discourse-ai/lib/ai_moderation/spam_scanner.rb
@@ -10,6 +10,21 @@ module DiscourseAi
SHOULD_SCAN_POST_CUSTOM_FIELD = "discourse_ai_should_scan_post"
+ DECISION_FEATURE_NAME = "spam_detection"
+ SPAM_QUESTION_NAME = "spam"
+ SPAM_QUESTION_INSTRUCTIONS = <<~TEXT.squish
+ Is this forum post spam? Answer yes for unsolicited promotion or advertising, link
+ dropping, SEO or bot-generated content, gibberish, or replies unrelated to the thread.
+ Answer no for genuine questions, answers, feedback and discussion, even from new users or
+ non-native speakers.
+ TEXT
+ # A decision model settles posts outside these bounds; the LLM decides the rest.
+ CLEARLY_NOT_SPAM_PROBABILITY = 0.05
+ CLEARLY_SPAM_PROBABILITY = 0.9
+ # Without an LLM to settle unsure posts, one cutoff decides them all.
+ DECISION_ONLY_SPAM_PROBABILITY = 0.5
+ MAX_DECISION_IMAGE_PIXELS = 1_600_000
+
def self.new_post(post)
return if !enabled?
return if !should_scan_post?(post)
@@ -147,33 +162,55 @@ module DiscourseAi
distance >= MINIMUM_EDIT_DIFFERENCE
end
- def self.test_post(post, custom_instructions: nil, llm_id: nil)
+ def self.test_post(
+ post,
+ custom_instructions: nil,
+ llm_model: AiModerationSetting.spam&.llm_model,
+ decision_model: AiModerationSetting.spam&.decision_model
+ )
settings = AiModerationSetting.spam
- custom_instructions = custom_instructions || settings.custom_instructions.presence
+ custom_instructions =
+ staff_instructions(custom_instructions || settings&.custom_instructions.presence)
target_msg =
build_target_content_msg(
post,
post.topic || Topic.with_deleted.find_by(id: post.topic_id),
)
- custom_insts = custom_instructions || settings.custom_instructions.presence
- if custom_insts.present?
- custom_insts =
- "\n\nAdditional site-specific instructions provided by Staff:\n#{custom_insts}"
+
+ verdict = {}
+ if decision_model
+ verdict =
+ decision_verdict(
+ decision_model,
+ target_msg,
+ custom_instructions,
+ decides_alone: llm_model.nil?,
+ ) || {}
end
- ctx =
- build_bot_context(
- feature_name: "spam_detection_test",
- messages: [target_msg],
- custom_instructions: custom_insts,
- )
- bot = build_scanner_bot(settings: settings, llm_id: llm_id)
+ system_prompt = decision_instructions(custom_instructions) if decision_model
+ if verdict[:is_spam].nil?
+ ctx =
+ build_bot_context(
+ feature_name: "spam_detection_test",
+ messages: [target_msg],
+ custom_instructions: custom_instructions,
+ )
+ bot = build_scanner_bot(settings: settings, llm_model: llm_model)
+
+ structured_output = nil
+ llm_args = { feature_context: { post_id: post.id } }
+ bot.reply(ctx, llm_args: llm_args) do |partial, _, type|
+ structured_output = partial if type == :structured_output
+ end
- structured_output = nil
- llm_args = { feature_context: { post_id: post.id } }
- bot.reply(ctx, llm_args: llm_args) do |partial, _, type|
- structured_output = partial if type == :structured_output
+ verdict.merge!(
+ is_spam: is_spam?(structured_output),
+ reason: extract_reason(structured_output),
+ llm_model: bot.model,
+ )
+ system_prompt = bot.agent.craft_prompt(ctx, llm: bot.model.to_llm).system_message_text
end
history = nil
@@ -188,26 +225,14 @@ module DiscourseAi
history << "\n"
end
- used_model = bot.model
- spam_agent = bot.agent
- used_prompt = spam_agent.craft_prompt(ctx, llm: used_model.to_llm).system_message_text
-
- text_content =
- if target_msg[:content].is_a?(Array)
- target_msg[:content].first
- else
- target_msg[:content]
- end
-
- is_spam = is_spam?(structured_output)
- reasoning = extract_reason(structured_output)
-
{
- is_spam: is_spam,
- reason: reasoning,
- llm_name: used_model.name,
- system_prompt: used_prompt,
- sent_message: text_content,
+ is_spam: verdict[:is_spam],
+ reason: verdict[:reason],
+ llm_name: verdict[:llm_model]&.name,
+ decision_model_name: decision_model&.display_name,
+ spam_probability: verdict[:spam_probability],
+ system_prompt: system_prompt,
+ sent_message: text_content(target_msg),
scan_history: history,
post_url: post.url,
}
@@ -222,55 +247,43 @@ module DiscourseAi
def self.perform_scan!(post, triggering_user_id: nil)
return if !enabled?
settings = AiModerationSetting.spam
- return if !settings || !settings.llm_model
-
- agent = settings.ai_agent
- return if !agent
+ return if !settings
+ decision_model = settings.decision_model
+ return if !decision_model && !llm_configured?(settings)
target_msg = build_target_content_msg(post)
- custom_instructions = settings.custom_instructions.presence
- if custom_instructions.present?
- custom_instructions =
- "\n\nAdditional site-specific instructions provided by Staff:\n#{custom_instructions}"
- end
-
- ctx =
- build_bot_context(
- messages: [target_msg],
- custom_instructions: custom_instructions,
- user: flagging_user,
- )
- bot = build_scanner_bot(settings: settings, user: flagging_user)
- structured_output = nil
+ custom_instructions = staff_instructions(settings.custom_instructions.presence)
begin
- llm_args = { feature_context: { post_id: post.id } }
- bot.reply(ctx, llm_args: llm_args) do |partial, _, type|
- structured_output = partial if type == :structured_output
+ verdict = {}
+ if decision_model
+ verdict =
+ decision_verdict(
+ decision_model,
+ target_msg,
+ custom_instructions,
+ decides_alone: !llm_configured?(settings),
+ ) || {}
+ end
+ if verdict[:is_spam].nil?
+ verdict.merge!(llm_verdict(settings, target_msg, custom_instructions, post:))
end
- is_spam = is_spam?(structured_output)
- reason = extract_reason(structured_output)
-
- log = AiApiAuditLog.order(id: :desc).where(feature_name: "spam_detection").first
- text_content =
- if target_msg[:content].is_a?(Array)
- target_msg[:content].first
- else
- target_msg[:content]
- end
AiSpamLog.transaction do
log =
AiSpamLog.create!(
post: post,
- llm_model: settings.llm_model,
- ai_api_audit_log: log,
- is_spam: is_spam,
- payload: text_content,
- reason: reason,
+ payload: text_content(target_msg),
+ ai_api_audit_log_id: verdict[:audit_log_id],
+ **verdict.slice(:is_spam, :reason, :spam_probability, :llm_model, :decision_model),
)
- handle_spam(post, log, triggering_user_id: triggering_user_id) if is_spam
+ handle_spam(post, log, triggering_user_id: triggering_user_id) if log.is_spam
end
+ rescue DiscourseAi::Decisions::CreditLimitExceeded
+ # with no LLM to fall back on, the post goes unscanned until credits reset
+ Rails.logger.warn(
+ "Discourse AI: Spam scan skipped for post #{post.id}, the decision model is out of credits",
+ )
rescue StandardError => e
# we need retries otherwise stuff will not be handled
Discourse.warn_exception(
@@ -313,17 +326,131 @@ module DiscourseAi
def self.build_scanner_bot(
settings:,
- use_structured_output: true,
- llm_id: nil,
+ llm_model: settings.llm_model,
user: Discourse.system_user
)
- agent = settings.ai_agent.class_instance&.new
-
- llm_model = llm_id ? LlmModel.find(llm_id) : settings.llm_model
+ ai_agent =
+ settings&.ai_agent ||
+ AiAgent.find_by_id_from_cache(
+ DiscourseAi::Agents::Agent.system_agents[DiscourseAi::Agents::SpamDetector],
+ )
+ agent = ai_agent.class_instance&.new
DiscourseAi::Agents::Bot.as(user, agent: agent, model: llm_model)
end
+ def self.llm_configured?(settings)
+ settings.llm_model.present? && settings.ai_agent.present?
+ end
+
+ def self.staff_instructions(custom_instructions)
+ return if custom_instructions.blank?
+
+ "\n\nAdditional site-specific instructions provided by Staff:\n#{custom_instructions}"
+ end
+
+ def self.decision_instructions(custom_instructions)
+ "#{SPAM_QUESTION_INSTRUCTIONS}#{custom_instructions}"
+ end
+
+ def self.text_content(target_msg)
+ content = target_msg[:content]
+ content.is_a?(Array) ? content.first : content
+ end
+
+ # @return [Hash, nil] the verdict, with a nil :is_spam when the LLM should decide; nil
+ # when the decision model could not answer and the LLM should decide
+ # @raise [DiscourseAi::Decisions::Error] when the decision model decides alone
+ def self.decision_verdict(decision_model, target_msg, custom_instructions, decides_alone:)
+ result =
+ decision_model.decide(
+ text_content(target_msg),
+ questions: [
+ {
+ name: SPAM_QUESTION_NAME,
+ type: "yes_no",
+ instructions: decision_instructions(custom_instructions),
+ },
+ ],
+ feature_name: DECISION_FEATURE_NAME,
+ user: Discourse.system_user,
+ images: decision_images(decision_model, target_msg),
+ )
+ verdict = { decision_model:, audit_log_id: result.audit_log_id }
+
+ answer = result[SPAM_QUESTION_NAME]
+ if answer.refusal?
+ return if !decides_alone
+ return(
+ verdict.merge(
+ is_spam: false,
+ reason: I18n.t("discourse_ai.spam_detection.decision_refused"),
+ )
+ )
+ end
+
+ probability = answer.probability
+ is_spam =
+ if decides_alone
+ probability >= DECISION_ONLY_SPAM_PROBABILITY
+ elsif probability >= CLEARLY_SPAM_PROBABILITY
+ true
+ elsif probability <= CLEARLY_NOT_SPAM_PROBABILITY
+ false
+ end
+ reason =
+ if !is_spam.nil?
+ I18n.t(
+ "discourse_ai.spam_detection.decision_reason",
+ probability: (probability * 100).round,
+ )
+ end
+
+ verdict.merge(is_spam:, reason:, spam_probability: probability)
+ rescue DiscourseAi::Decisions::Error => e
+ raise if decides_alone
+
+ Rails.logger.warn("Discourse AI: Spam decision model failed, the LLM decides: #{e.class}")
+ nil
+ end
+
+ # Images only add evidence, so a post whose images cannot be prepared is judged on its text.
+ def self.decision_images(decision_model, target_msg)
+ return [] if !decision_model.vision_enabled || !target_msg[:content].is_a?(Array)
+
+ upload_ids = target_msg[:content].filter_map { |part| part[:upload_id] if part.is_a?(Hash) }
+ DiscourseAi::Completions::UploadEncoder
+ .encode(upload_ids:, max_pixels: MAX_DECISION_IMAGE_PIXELS)
+ .map { |image| "data:#{image[:mime_type]};base64,#{image[:base64]}" }
+ rescue StandardError => e
+ Rails.logger.warn("Discourse AI: Could not prepare images for spam detection: #{e.message}")
+ []
+ end
+
+ def self.llm_verdict(settings, target_msg, custom_instructions, post:)
+ ctx =
+ build_bot_context(
+ messages: [target_msg],
+ custom_instructions: custom_instructions,
+ user: flagging_user,
+ )
+ bot = build_scanner_bot(settings: settings, user: flagging_user)
+ structured_output = nil
+
+ llm_args = { feature_context: { post_id: post.id } }
+ bot.reply(ctx, llm_args: llm_args) do |partial, _, type|
+ structured_output = partial if type == :structured_output
+ end
+
+ {
+ is_spam: is_spam?(structured_output),
+ reason: extract_reason(structured_output),
+ llm_model: settings.llm_model,
+ audit_log_id:
+ AiApiAuditLog.order(id: :desc).where(feature_name: "spam_detection").pick(:id),
+ }
+ end
+
def self.is_spam?(structured_output)
structured_output.present? && structured_output.read_buffered_property(:spam)
end
diff --git a/plugins/discourse-ai/lib/configuration/llm_enumerator.rb b/plugins/discourse-ai/lib/configuration/llm_enumerator.rb
index 0b5c9ddcba4..d8cab6726e1 100644
--- a/plugins/discourse-ai/lib/configuration/llm_enumerator.rb
+++ b/plugins/discourse-ai/lib/configuration/llm_enumerator.rb
@@ -86,8 +86,8 @@ module DiscourseAi
}
end
- if SiteSetting.ai_spam_detection_enabled && AiModerationSetting.spam.present?
- model_id = AiModerationSetting.spam[:llm_model_id]
+ if SiteSetting.ai_spam_detection_enabled &&
+ (model_id = AiModerationSetting.spam&.llm_model_id).present?
rval[model_id] << { type: :ai_spam, id: DiscourseAi::Configuration::Module::SPAM_ID }
end
diff --git a/plugins/discourse-ai/spec/lib/modules/ai_moderation/spam_scanner_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_moderation/spam_scanner_spec.rb
index a823a698a52..49e358b9665 100644
--- a/plugins/discourse-ai/spec/lib/modules/ai_moderation/spam_scanner_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/ai_moderation/spam_scanner_spec.rb
@@ -170,6 +170,78 @@ RSpec.describe DiscourseAi::AiModeration::SpamScanner do
end
end
+ describe ".perform_scan! with a decision model" do
+ fab!(:decision_model) { Fabricate(:decision_model, id: -1) }
+
+ before { spam_setting.update!(decision_model:) }
+
+ def scan_with(probability, llm_responses: [])
+ DiscourseAi::Decisions.with_canned_answers(spam: probability) do |canned|
+ DiscourseAi::Completions::Llm.with_prepared_responses(llm_responses) do |_, _, prompts|
+ described_class.perform_scan!(post, triggering_user_id: post.user_id)
+ [canned.calls, prompts]
+ end
+ end
+ end
+
+ it "flags confident spam without asking the LLM" do
+ decision_calls, prompts = scan_with(0.95)
+
+ log = AiSpamLog.find_by!(post:)
+ expect([log.is_spam, log.spam_probability, log.decision_model_id, log.llm_model_id]).to eq(
+ [true, 0.95, decision_model.id, nil],
+ )
+ expect(log.reviewable).to be_present
+ expect(post.user.reload).to be_silenced
+ expect(prompts).to be_empty
+ expect(decision_calls.first[:questions].first.instructions).to include("test instructions")
+ end
+
+ it "clears confident non-spam without asking the LLM" do
+ _, prompts = scan_with(0.01)
+
+ expect(AiSpamLog.find_by!(post:).is_spam).to eq(false)
+ expect(prompts).to be_empty
+ end
+
+ it "lets the LLM decide unsure posts and keeps the decision model's probability" do
+ scan_with(0.5, llm_responses: [{ spam: true, reason: "llm reason" }])
+
+ log = AiSpamLog.find_by!(post:)
+ expect([log.is_spam, log.reason, log.spam_probability, log.llm_model_id]).to eq(
+ [true, "llm reason", 0.5, llm_model.id],
+ )
+ end
+
+ it "lets the LLM decide when the decision model fails" do
+ stub_request(:post, decision_model.url).to_return(status: 400, body: "{}")
+
+ DiscourseAi::Completions::Llm.with_prepared_responses([{ spam: false }]) do
+ described_class.perform_scan!(post)
+ end
+
+ expect(AiSpamLog.find_by!(post:).llm_model_id).to eq(llm_model.id)
+ end
+
+ it "decides every post at one cutoff without an LLM" do
+ spam_setting.update!(llm_model: nil)
+
+ scan_with(0.6)
+ expect(AiSpamLog.find_by!(post:).is_spam).to eq(true)
+
+ AiSpamLog.delete_all
+ scan_with(0.4)
+ expect(AiSpamLog.find_by!(post:).is_spam).to eq(false)
+ end
+
+ it "skips the scan without an LLM once the decision model is out of credits" do
+ spam_setting.update!(llm_model: nil)
+ Fabricate(:decision_model_credit_allocation, decision_model:, daily_used: 10_000)
+
+ expect { described_class.perform_scan!(post) }.not_to change { AiSpamLog.count }
+ end
+ end
+
describe ".scanned_max_times?" do
it "returns true when post has been scanned 3 times" do
3.times do
@@ -406,6 +478,17 @@ RSpec.describe DiscourseAi::AiModeration::SpamScanner do
expect(result[:is_spam]).to eq(false)
end
+ it "tests with the spam detector agent before any spam settings are saved" do
+ spam_setting.destroy!
+
+ result =
+ DiscourseAi::Completions::Llm.with_prepared_responses([{ spam: true }]) do
+ described_class.test_post(post, llm_model:, decision_model: nil)
+ end
+
+ expect(result[:is_spam]).to eq(true)
+ end
+
it "correctly handles spam scanning" do
expect(described_class.flagging_user.id).not_to eq(Discourse.system_user.id)
diff --git a/plugins/discourse-ai/spec/requests/admin/ai_decision_models_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/ai_decision_models_controller_spec.rb
index 24feb6dff6d..c23db3aaff8 100644
--- a/plugins/discourse-ai/spec/requests/admin/ai_decision_models_controller_spec.rb
+++ b/plugins/discourse-ai/spec/requests/admin/ai_decision_models_controller_spec.rb
@@ -297,6 +297,15 @@ RSpec.describe DiscourseAi::Admin::AiDecisionModelsController do
).count,
).to eq(1)
end
+
+ it "refuses to delete a decision model used for spam detection" do
+ AiModerationSetting.create!(setting_type: :spam, decision_model:)
+
+ delete "/admin/plugins/discourse-ai/ai-decision-models/#{decision_model.id}.json"
+
+ expect(response.status).to eq(409)
+ expect(DecisionModel.exists?(decision_model.id)).to eq(true)
+ end
end
describe "#test" do
diff --git a/plugins/discourse-ai/spec/requests/admin/ai_spam_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/ai_spam_controller_spec.rb
index 87fefa96716..96ca467dd39 100644
--- a/plugins/discourse-ai/spec/requests/admin/ai_spam_controller_spec.rb
+++ b/plugins/discourse-ai/spec/requests/admin/ai_spam_controller_spec.rb
@@ -70,6 +70,25 @@ RSpec.describe DiscourseAi::Admin::AiSpamController do
expect(response.status).to eq(422)
end
+ it "can enable spam detection with only a decision model" do
+ decision_model = Fabricate(:decision_model)
+
+ put "/admin/plugins/discourse-ai/ai-spam.json",
+ params: {
+ is_enabled: true,
+ llm_model_id: "",
+ decision_model_id: decision_model.id,
+ }
+
+ expect(response.status).to eq(200)
+ expect(SiteSetting.ai_spam_detection_enabled).to eq(true)
+ expect(AiModerationSetting.spam.slice(:llm_model_id, :decision_model_id)).to eq(
+ "llm_model_id" => nil,
+ "decision_model_id" => decision_model.id,
+ )
+ expect(response.parsed_body["decision_model_id"]).to eq(decision_model.id)
+ end
+
context "when spam detection was already set" do
fab!(:setting) do
AiModerationSetting.create(
@@ -277,6 +296,28 @@ RSpec.describe DiscourseAi::Admin::AiSpamController do
expect(parsed["reason"]).to eq("because apples")
end
+ it "tests with only a decision model" do
+ decision_model = Fabricate(:decision_model)
+
+ DiscourseAi::Decisions.with_canned_answers(spam: 0.8) do
+ post "/admin/plugins/discourse-ai/ai-spam/test.json",
+ params: {
+ post_url: spam_post.url,
+ llm_id: "",
+ decision_model_id: decision_model.id,
+ }
+ end
+
+ expect(response.status).to eq(200)
+ expect(
+ response.parsed_body.slice("is_spam", "spam_probability", "decision_model_name"),
+ ).to eq(
+ "is_spam" => true,
+ "spam_probability" => 0.8,
+ "decision_model_name" => decision_model.display_name,
+ )
+ end
+
it "rejects llm_id that is not a valid LlmModel" do
post "/admin/plugins/discourse-ai/ai-spam/test.json",
params: {
diff --git a/plugins/discourse-ai/spec/system/ai_moderation/ai_spam_spec.rb b/plugins/discourse-ai/spec/system/ai_moderation/ai_spam_spec.rb
index 5ce1cc0a7f4..9543118becb 100644
--- a/plugins/discourse-ai/spec/system/ai_moderation/ai_spam_spec.rb
+++ b/plugins/discourse-ai/spec/system/ai_moderation/ai_spam_spec.rb
@@ -18,7 +18,7 @@ RSpec.describe "AI Spam Configuration" do
toggle.toggle
dialog = PageObjects::Components::Dialog.new
- expect(dialog).to have_content(I18n.t("discourse_ai.llm.configuration.must_select_model"))
+ expect(dialog).to have_content(I18n.t("discourse_ai.spam_detection.model_missing"))
dialog.click_ok
expect(toggle.unchecked?).to eq(true)
@@ -49,5 +49,27 @@ RSpec.describe "AI Spam Configuration" do
I18n.t("js.discourse_ai.llms.usage.ai_spam"),
)
end
+
+ it "can detect spam with only a decision model" do
+ decision_model = Fabricate(:decision_model)
+ visit "/admin/plugins/discourse-ai/ai-spam"
+
+ PageObjects::Components::SelectKit.new(
+ ".ai-spam__decision-model-selector",
+ ).select_row_by_value(decision_model.id)
+ llm_selector = PageObjects::Components::SelectKit.new(".ai-spam__llm-selector")
+ llm_selector.expand
+ llm_selector.select_row_by_index(0)
+ expect(page).to have_no_css(".ai-spam__agent-selector")
+
+ find(".ai-spam__instructions-save").click
+ expect(PageObjects::Components::Toasts.new).to have_content(
+ I18n.t("js.discourse_ai.spam.settings_saved"),
+ )
+ expect(AiModerationSetting.spam.slice(:llm_model_id, :decision_model_id)).to eq(
+ "llm_model_id" => nil,
+ "decision_model_id" => decision_model.id,
+ )
+ end
end
end