Skip to content

A platform for community discussion. Free, open, simple.

FIX: Count AI spam flags as automated detection (#44621)

Alan Guo Xiang Tand2d399cd2211
diff --git a/plugins/discourse-ai/plugin.rb b/plugins/discourse-ai/plugin.rb
index c27b2e5f4e8..94b7f82bbad 100644
--- a/plugins/discourse-ai/plugin.rb
+++ b/plugins/discourse-ai/plugin.rb
@@ -148,13 +148,20 @@ after_initialize do
register_problem_check ProblemCheck::AiImageCaptionAgent
register_modifier(:reviewable_automated_detection) do |automated, reviewable|
+ scores = reviewable.reviewable_scores
+
automated ||
- reviewable
- .reviewable_scores
+ scores
.where(
"context LIKE ?",
"#{ReviewableScore.sanitize_sql_like(DiscourseAi::Automation::TRIAGE_AUTOMATION_SCORE_CONTEXT_PREFIX)}%",
)
+ .or(
+ scores.where(
+ user_id: SiteSetting.ai_spam_detection_user_id.presence,
+ reviewable_score_type: ReviewableScore.types[:spam],
+ ),
+ )
.exists?
end
diff --git a/plugins/discourse-ai/spec/models/reviewable_flagged_post_spec.rb b/plugins/discourse-ai/spec/models/reviewable_flagged_post_spec.rb
index a29fc35014d..236c2cb8925 100644
--- a/plugins/discourse-ai/spec/models/reviewable_flagged_post_spec.rb
+++ b/plugins/discourse-ai/spec/models/reviewable_flagged_post_spec.rb
@@ -8,6 +8,61 @@ RSpec.describe ReviewableFlaggedPost do
SiteSetting.dsa_reporting_enabled = true
end
+ it "records automatic detection when a moderator deletes spam detected by the spam bot" do
+ AiModerationSetting.create!(setting_type: :spam, llm_model: Fabricate(:llm_model))
+ SiteSetting.ai_spam_detection_enabled = true
+ post = Fabricate(:post, user: Fabricate(:user, trust_level: TrustLevel[0]))
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ [{ spam: true, reason: "spam detected" }],
+ ) { DiscourseAi::AiModeration::SpamScanner.perform_scan(post) }
+ reviewable = AiSpamLog.find_by!(post: post).reviewable
+ SiteSetting.ai_spam_detection_enabled = false
+
+ reviewable.perform(Fabricate(:admin), :delete_and_agree)
+
+ statement = DsaStatementOfRecord.find_by!(reviewable_id: reviewable.id)
+ expect(statement.payload).to include(
+ "automated_detection" => "Yes",
+ "automated_decision" => "AUTOMATED_DECISION_NOT_AUTOMATED",
+ )
+ end
+
+ it "preserves manual detection for system spam flags when no scanner bot is configured" do
+ SiteSetting.ai_spam_detection_user_id = ""
+ reviewable = Fabricate(:reviewable_flagged_post)
+ reviewable.add_score(Discourse.system_user, ReviewableScore.types[:spam])
+
+ reviewable.perform(Fabricate(:admin), :delete_and_agree)
+
+ statement = DsaStatementOfRecord.find_by!(reviewable_id: reviewable.id)
+ expect(statement.payload["automated_detection"]).to eq("No")
+ end
+
+ it "records manual detection when a moderator deletes a post flagged as spam by a human" do
+ DiscourseAi::AiModeration::SpamScanner.ensure_flagging_user!
+ reviewable = Fabricate(:reviewable_flagged_post)
+ reviewable.add_score(Fabricate(:user), ReviewableScore.types[:spam])
+
+ reviewable.perform(Fabricate(:admin), :delete_and_agree)
+
+ statement = DsaStatementOfRecord.find_by!(reviewable_id: reviewable.id)
+ expect(statement.payload["automated_detection"]).to eq("No")
+ end
+
+ it "records manual detection for a nonspam flag from the configured spam flagger" do
+ DiscourseAi::AiModeration::SpamScanner.ensure_flagging_user!
+ reviewable = Fabricate(:reviewable_flagged_post)
+ reviewable.add_score(
+ DiscourseAi::AiModeration::SpamScanner.flagging_user,
+ ReviewableScore.types[:off_topic],
+ )
+
+ reviewable.perform(Fabricate(:admin), :delete_and_agree)
+
+ statement = DsaStatementOfRecord.find_by!(reviewable_id: reviewable.id)
+ expect(statement.payload["automated_detection"]).to eq("No")
+ end
+
it "records automatic detection when a moderator deletes a post queued for review by AI triage" do
reviewable = Fabricate(:reviewable_flagged_post)
post = reviewable.target