Skip to content

A platform for community discussion. Free, open, simple.

FIX: Wiki edits can silence the original author through AI spam scans (#39842)

Sam7d89b9d46a6b
diff --git a/plugins/discourse-ai/app/jobs/regular/ai_spam_scan.rb b/plugins/discourse-ai/app/jobs/regular/ai_spam_scan.rb
index 483e56ec696..fd857f1f11c 100644
--- a/plugins/discourse-ai/app/jobs/regular/ai_spam_scan.rb
+++ b/plugins/discourse-ai/app/jobs/regular/ai_spam_scan.rb
@@ -7,7 +7,10 @@ module Jobs
post = Post.find_by(id: args[:post_id])
return if !post
- DiscourseAi::AiModeration::SpamScanner.perform_scan(post)
+ DiscourseAi::AiModeration::SpamScanner.perform_scan(
+ post,
+ triggering_user_id: args[:triggering_user_id],
+ )
end
end
end
diff --git a/plugins/discourse-ai/lib/ai_moderation/spam_scanner.rb b/plugins/discourse-ai/lib/ai_moderation/spam_scanner.rb
index e341e902997..6489f32367f 100644
--- a/plugins/discourse-ai/lib/ai_moderation/spam_scanner.rb
+++ b/plugins/discourse-ai/lib/ai_moderation/spam_scanner.rb
@@ -74,14 +74,15 @@ module DiscourseAi
editor = post.last_editor
return if editor && (editor.staff? || editor.bot?)
+ scan_args = { post_id: post.id, triggering_user_id: post.last_editor_id || post.user_id }
last_scan = AiSpamLog.where(post_id: post.id).order(created_at: :desc).first
if last_scan && last_scan.created_at > EDIT_DELAY_MINUTES.minutes.ago
delay_minutes =
((last_scan.created_at + EDIT_DELAY_MINUTES.minutes) - Time.current).to_i / 60
- Jobs.enqueue_in(delay_minutes.minutes, :ai_spam_scan, post_id: post.id)
+ Jobs.enqueue_in(delay_minutes.minutes, :ai_spam_scan, scan_args)
else
- Jobs.enqueue(:ai_spam_scan, post_id: post.id)
+ Jobs.enqueue(:ai_spam_scan, scan_args)
end
end
@@ -209,13 +210,13 @@ module DiscourseAi
}
end
- def self.perform_scan(post)
+ def self.perform_scan(post, triggering_user_id: nil)
return if !should_scan_post?(post)
- perform_scan!(post)
+ perform_scan!(post, triggering_user_id: triggering_user_id)
end
- def self.perform_scan!(post)
+ def self.perform_scan!(post, triggering_user_id: nil)
return if !enabled?
settings = AiModerationSetting.spam
return if !settings || !settings.llm_model
@@ -265,7 +266,7 @@ module DiscourseAi
payload: text_content,
reason: reason,
)
- handle_spam(post, log) if is_spam
+ handle_spam(post, log, triggering_user_id: triggering_user_id) if is_spam
end
rescue StandardError => e
# we need retries otherwise stuff will not be handled
@@ -409,7 +410,7 @@ module DiscourseAi
nil
end
- def self.handle_spam(post, log)
+ def self.handle_spam(post, log, triggering_user_id: nil)
url = "#{Discourse.base_url}/admin/plugins/discourse-ai/ai-spam"
reason = I18n.t("discourse_ai.spam_detection.flag_reason", url: url)
@@ -430,20 +431,22 @@ module DiscourseAi
if result.success?
log.update!(reviewable: result.reviewable)
- reason = I18n.t("discourse_ai.spam_detection.silence_reason", url: url)
- silencer =
- UserSilencer.new(
- post.user,
- flagging_user,
- message: :too_many_spam_flags,
- post_id: post.id,
- reason: reason,
- keep_posts: true,
- )
- silencer.silence
-
- # silencer will not hide tl1 posts, so we do this here
- hide_post(post)
+ if triggering_user_id.present? && triggering_user_id.to_i == post.user_id
+ reason = I18n.t("discourse_ai.spam_detection.silence_reason", url: url)
+ silencer =
+ UserSilencer.new(
+ post.user,
+ flagging_user,
+ message: :too_many_spam_flags,
+ post_id: post.id,
+ reason: reason,
+ keep_posts: true,
+ )
+ silencer.silence
+
+ # silencer will not hide tl1 posts, so we do this here
+ hide_post(post)
+ end
else
log.update!(
error:
diff --git a/plugins/discourse-ai/spec/lib/modules/ai_moderation/spam_scanner_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_moderation/spam_scanner_spec.rb
index fd39a6764ab..857417880d5 100644
--- a/plugins/discourse-ai/spec/lib/modules/ai_moderation/spam_scanner_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/ai_moderation/spam_scanner_spec.rb
@@ -237,6 +237,25 @@ RSpec.describe DiscourseAi::AiModeration::SpamScanner do
}.to change(Jobs::AiSpamScan.jobs, :size).by(1)
end
+ it "passes the editor id in the job args without persisting it on the post" do
+ editor = Fabricate(:user, trust_level: TrustLevel[0])
+ post.update!(last_editor_id: editor.id)
+ PostRevision.create!(
+ post: post,
+ user: editor,
+ modifications: {
+ raw: ["old content", "completely new content"],
+ },
+ )
+
+ described_class.edited_post(post)
+ described_class.after_cooked_post(post)
+
+ job_args = Jobs::AiSpamScan.jobs.last["args"].first
+ expect(post.reload.custom_fields).not_to have_key("discourse_ai_spam_scan_triggering_user_id")
+ expect(job_args["triggering_user_id"]).to eq(editor.id)
+ end
+
it "does nothing when staff is the last revisor" do
expect {
PostRevisor.new(post).revise!(moderator, title: "#{post.topic.title} spam spam")
@@ -400,6 +419,29 @@ RSpec.describe DiscourseAi::AiModeration::SpamScanner do
expect(post.user.reload.silenced?).to eq(false)
end
+ it "does not silence the author when another user adds spam to a wiki post" do
+ wiki_editor = Fabricate(:user, trust_level: TrustLevel[0])
+ wiki_post = Fabricate(:post, user: user, wiki: true, raw: "Helpful community wiki content")
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ [{ spam: true, reason: "spam detected" }],
+ ) do
+ PostRevisor.new(wiki_post).revise!(
+ wiki_editor,
+ raw: "Buy questionable products from this completely different wiki content",
+ )
+ wiki_post.rebake!
+ end
+
+ log = AiSpamLog.find_by(post: wiki_post)
+
+ expect(log.reviewable).to be_present
+ expect(wiki_post.user.reload).not_to be_silenced
+ expect(
+ UserHistory.where(action: UserHistory.actions[:silence_user], target_user_id: user.id),
+ ).to be_blank
+ end
+
it "does not silence the user or hide the post when a flag cannot be created" do
post = post_with_uploaded_image
Fabricate(