Skip to content

A platform for community discussion. Free, open, simple.

FIX: Avoid duplicate AI triage reviewables for spam flags (#41043)

Krzysztof Kotlarek3709009232e1
diff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.yml
index 3fe654e7481..0563b342440 100644
--- a/plugins/discourse-ai/config/locales/server.en.yml
+++ b/plugins/discourse-ai/config/locales/server.en.yml
@@ -31,8 +31,11 @@ en:
title: Triage posts using AI
description: "Triage posts using a large language model"
flagged_post: |
- <div>Response from the model:</div>
- <p>%{llm_response}</p>
+ %{score_reason}
+ %{response}
+ flagged_post_response: |
+ <p>Response from the model: %{llm_response}</p>
+ flagged_post_score_reason: |
<b>Triggered by the <a href="%{base_path}/admin/plugins/automation/%{automation_id}">%{automation_name}</a> rule.</b>
notify_author_pm:
subject: "Your post was temporarily removed for review"
@@ -752,6 +755,7 @@ en:
already_flagged: "Post is already flagged for review."
flagged: "Post flagged for review."
reason: "AI bot flagged this post.\n\nReason: %{reason}"
+ score_reason: "AI bot flagged this post."
close_topic:
errors:
diff --git a/plugins/discourse-ai/lib/agents/tools/flag_post.rb b/plugins/discourse-ai/lib/agents/tools/flag_post.rb
index d7de03086a1..b6fc659c2e7 100644
--- a/plugins/discourse-ai/lib/agents/tools/flag_post.rb
+++ b/plugins/discourse-ai/lib/agents/tools/flag_post.rb
@@ -74,13 +74,14 @@ module DiscourseAi
end
flag_success = true
- if %w[spam spam_silence].include?(flag_type)
+ if spam_flag?
result =
PostActionCreator.new(
Discourse.system_user,
post,
PostActionType.types[:spam],
- message: flag_reason,
+ message: spam_post_action_message,
+ reason: spam_score_reason,
queue_for_review: true,
).perform
flag_success = result.success?
@@ -153,18 +154,35 @@ module DiscourseAi
end
def flag_reason
- if feature_context[:automation_id].present? && feature_context[:automation_name].present?
- I18n.t(
- "discourse_automation.scriptables.llm_triage.flagged_post",
- base_path: feature_context[:base_path] || Discourse.base_path,
- llm_response:
- ERB::Util.html_escape(feature_context[:llm_response].presence || reason),
- automation_id: feature_context[:automation_id].to_s,
- automation_name: ERB::Util.html_escape(feature_context[:automation_name].to_s),
- )
- else
- I18n.t("discourse_ai.ai_bot.flag_post.reason", reason: reason)
- end
+ DiscourseAi::Automation.flag_post_reason(
+ reason: reason,
+ llm_response: feature_context[:llm_response],
+ automation_id: feature_context[:automation_id],
+ automation_name: feature_context[:automation_name],
+ base_path: feature_context[:base_path] || Discourse.base_path,
+ )
+ end
+
+ def spam_score_reason
+ DiscourseAi::Automation.spam_score_reason(
+ automation_id: feature_context[:automation_id],
+ automation_name: feature_context[:automation_name],
+ base_path: feature_context[:base_path] || Discourse.base_path,
+ )
+ end
+
+ def spam_post_action_message
+ DiscourseAi::Automation.spam_post_action_message(
+ reason: reason,
+ llm_response: feature_context[:llm_response],
+ automation_id: feature_context[:automation_id],
+ automation_name: feature_context[:automation_name],
+ base_path: feature_context[:base_path] || Discourse.base_path,
+ )
+ end
+
+ def spam_flag?
+ %w[spam spam_silence].include?(flag_type)
end
def feature_context
diff --git a/plugins/discourse-ai/lib/automation.rb b/plugins/discourse-ai/lib/automation.rb
index 267c811adff..9b8689c79a4 100644
--- a/plugins/discourse-ai/lib/automation.rb
+++ b/plugins/discourse-ai/lib/automation.rb
@@ -6,6 +6,73 @@ module DiscourseAi
%w[spam spam_silence]
end
+ def self.flag_post_reason(
+ reason:,
+ llm_response: nil,
+ automation_id: nil,
+ automation_name: nil,
+ base_path: Discourse.base_path
+ )
+ if automation_context?(automation_id, automation_name)
+ I18n.t(
+ "discourse_automation.scriptables.llm_triage.flagged_post",
+ score_reason:
+ spam_score_reason(
+ automation_id: automation_id,
+ automation_name: automation_name,
+ base_path: base_path,
+ ),
+ response: flag_post_response(reason: reason, llm_response: llm_response),
+ )
+ else
+ I18n.t("discourse_ai.ai_bot.flag_post.reason", reason: reason)
+ end
+ end
+
+ def self.spam_score_reason(
+ automation_id: nil,
+ automation_name: nil,
+ base_path: Discourse.base_path
+ )
+ if automation_context?(automation_id, automation_name)
+ I18n.t(
+ "discourse_automation.scriptables.llm_triage.flagged_post_score_reason",
+ base_path: base_path,
+ automation_id: automation_id.to_s,
+ automation_name: ERB::Util.html_escape(automation_name.to_s),
+ )
+ else
+ I18n.t("discourse_ai.ai_bot.flag_post.score_reason")
+ end
+ end
+
+ def self.spam_post_action_message(
+ reason:,
+ llm_response: nil,
+ automation_id: nil,
+ automation_name: nil,
+ base_path: Discourse.base_path
+ )
+ if automation_context?(automation_id, automation_name)
+ flag_post_response(reason: reason, llm_response: llm_response)
+ else
+ flag_post_reason(
+ reason: reason,
+ llm_response: llm_response,
+ automation_id: automation_id,
+ automation_name: automation_name,
+ base_path: base_path,
+ )
+ end
+ end
+
+ def self.flag_post_response(reason:, llm_response: nil)
+ I18n.t(
+ "discourse_automation.scriptables.llm_triage.flagged_post_response",
+ llm_response: ERB::Util.html_escape(llm_response.presence || reason),
+ )
+ end
+
def self.flag_types
[
{ id: "review", translated_name: I18n.t("discourse_automation.ai.flag_types.review") },
@@ -65,5 +132,10 @@ module DiscourseAi
phash
end
end
+
+ def self.automation_context?(automation_id, automation_name)
+ automation_id.present? && automation_name.present?
+ end
+ private_class_method :automation_context?, :flag_post_response
end
end
diff --git a/plugins/discourse-ai/lib/automation/llm_triage.rb b/plugins/discourse-ai/lib/automation/llm_triage.rb
index 067c0da570a..ef0f6a44ed4 100644
--- a/plugins/discourse-ai/lib/automation/llm_triage.rb
+++ b/plugins/discourse-ai/lib/automation/llm_triage.rb
@@ -36,6 +36,24 @@ module DiscourseAi
end
end
+ def self.promote_post_reviewable_to_flagged!(post)
+ reviewable = ReviewablePost.pending.find_by(target: post)
+ return if reviewable.blank?
+ return if ReviewableFlaggedPost.exists?(target: post)
+
+ reviewable.update!(
+ type: ReviewableFlaggedPost.name,
+ potential_spam: true,
+ reviewable_by_moderator: true,
+ payload: {
+ targets_topic: false,
+ },
+ )
+ rescue ActiveRecord::RecordNotUnique
+ raise if !ReviewableFlaggedPost.exists?(target: post)
+ end
+ private_class_method :promote_post_reviewable_to_flagged!
+
def self.handle(
post:,
triage_agent_id:,
@@ -197,22 +215,36 @@ module DiscourseAi
already_flagged = flagged_by_another_triage_rule?(post)
score_reason =
- I18n.t(
- "discourse_automation.scriptables.llm_triage.flagged_post",
- base_path: Discourse.base_path,
- llm_response: ERB::Util.html_escape(result),
+ DiscourseAi::Automation.flag_post_reason(
+ reason: result,
automation_id: automation&.id.to_s,
- automation_name: ERB::Util.html_escape(automation&.name.to_s),
+ automation_name: automation&.name,
)
if !flagged_by_tool
if flag_type == :spam || flag_type == :spam_silence
+ spam_score_reason =
+ DiscourseAi::Automation.spam_score_reason(
+ automation_id: automation&.id.to_s,
+ automation_name: automation&.name,
+ )
+
+ spam_post_action_message =
+ DiscourseAi::Automation.spam_post_action_message(
+ reason: result,
+ automation_id: automation&.id.to_s,
+ automation_name: automation&.name,
+ )
+
+ promote_post_reviewable_to_flagged!(post)
+
result =
PostActionCreator.new(
Discourse.system_user,
post,
PostActionType.types[:spam],
- message: score_reason,
+ message: spam_post_action_message,
+ reason: spam_score_reason,
queue_for_review: true,
).perform
diff --git a/plugins/discourse-ai/spec/lib/discourse_automation/llm_triage_spec.rb b/plugins/discourse-ai/spec/lib/discourse_automation/llm_triage_spec.rb
index f50568c1168..49222a8f14d 100644
--- a/plugins/discourse-ai/spec/lib/discourse_automation/llm_triage_spec.rb
+++ b/plugins/discourse-ai/spec/lib/discourse_automation/llm_triage_spec.rb
@@ -276,8 +276,69 @@ describe DiscourseAi::Automation::LlmTriage do
reviewable = ReviewableFlaggedPost.find_by(target: post)
scores = reviewable.reviewable_scores.select { |rs| rs.user == Discourse.system_user }
+ conversation_excerpt = scores.first.reviewable_conversation.conversation_posts.first.excerpt
- expect(scores.size).to eq(1)
+ aggregate_failures do
+ expect(scores.size).to eq(1)
+ expect(scores.first.reason).to include("Triggered by")
+ expect(scores.first.reason).not_to include("Response from the model:")
+ expect(conversation_excerpt).to include("Response from the model: bad")
+ end
+ end
+
+ it "adds spam scores to an existing flagged reviewable" do
+ reviewable_post =
+ ReviewablePost.needs_review!(
+ target: post,
+ created_by: Discourse.system_user,
+ reviewable_by_moderator: true,
+ )
+ reviewable_post.add_score(
+ Discourse.system_user,
+ ReviewableScore.types[:needs_approval],
+ reason: "Existing review flag",
+ force_review: true,
+ )
+
+ flagged_reviewable =
+ ReviewableFlaggedPost.needs_review!(
+ target: post,
+ created_by: Fabricate(:user),
+ reviewable_by_moderator: true,
+ )
+ flagged_reviewable.add_score(
+ flagged_reviewable.created_by,
+ ReviewableScore.types[:spam],
+ reason: "Existing spam flag",
+ force_review: true,
+ )
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
+ add_automation_field("flag_type", "spam")
+
+ automation.running_in_background!
+ automation.trigger!({ "post" => post })
+ end
+
+ flagged_reviewable.reload
+ system_spam_scores =
+ flagged_reviewable.reviewable_scores.select do |score|
+ score.user == Discourse.system_user &&
+ score.reviewable_score_type == ReviewableScore.types[:spam]
+ end
+ conversation_excerpt =
+ system_spam_scores.first.reviewable_conversation.conversation_posts.first.excerpt
+
+ aggregate_failures do
+ expect(ReviewablePost.pending.where(target: post)).to contain_exactly(reviewable_post)
+ expect(ReviewableFlaggedPost.pending.where(target: post)).to contain_exactly(
+ flagged_reviewable,
+ )
+ expect(system_spam_scores.size).to eq(1)
+ expect(system_spam_scores.first.reason).to include("Triggered by")
+ expect(system_spam_scores.first.reason).not_to include("Response from the model:")
+ expect(conversation_excerpt).to include("Response from the model: bad")
+ end
end
it "flags the post once when not using spam-based types (review_* types)" do
@@ -292,7 +353,13 @@ describe DiscourseAi::Automation::LlmTriage do
reviewable = ReviewablePost.find_by(target: post)
scores = reviewable.reviewable_scores.select { |rs| rs.user == Discourse.system_user }
- expect(scores.size).to eq(1)
+ score_reason = scores.first.reason
+
+ aggregate_failures do
+ expect(scores.size).to eq(1)
+ expect(score_reason.index("Triggered by")).to be <
+ score_reason.index("Response from the model")
+ end
end
it "makes the reviewable visible to moderators when using review flag types" do
diff --git a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb
index 55f59cfd137..760d739451b 100644
--- a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb
@@ -93,7 +93,9 @@ describe DiscourseAi::Automation::LlmTriage do
expect(reviewable.target_id).to eq(post.id)
expect(reviewable.target_type).to eq("Post")
- expect(reviewable.reviewable_scores.first.reason).to include("bad")
+ expect(reviewable.reviewable_scores.first.reason).to eq(
+ I18n.t("discourse_ai.ai_bot.flag_post.reason", reason: "bad"),
+ )
end
it "flags via tool call when the agent invokes flag_post" do
@@ -139,6 +141,43 @@ describe DiscourseAi::Automation::LlmTriage do
expect(post.reload).to be_hidden
expect(post.topic.reload.visible).to eq(false)
+ expect(ReviewableFlaggedPost.last.reviewable_scores.first.reason).to eq(
+ I18n.t("discourse_ai.ai_bot.flag_post.score_reason"),
+ )
+ end
+
+ it "keeps one reviewable when spam follows review" do
+ DiscourseAi::Completions::Llm.with_prepared_responses(%w[bad bad]) do
+ triage(
+ post: post,
+ triage_agent_id: ai_agent.id,
+ search_for_text: "bad",
+ flag_post: true,
+ flag_type: :review,
+ automation: nil,
+ )
+
+ triage(
+ post: post,
+ triage_agent_id: ai_agent.id,
+ search_for_text: "bad",
+ flag_post: true,
+ flag_type: :spam,
+ automation: nil,
+ )
+ end
+
+ reviewables = Reviewable.pending.where(target: post)
+ reviewable = reviewables.first
+
+ aggregate_failures do
+ expect(reviewables.size).to eq(1)
+ expect(reviewable).to be_a(ReviewableFlaggedPost)
+ expect(reviewable.reviewable_scores.map(&:reviewable_score_type)).to contain_exactly(
+ ReviewableScore.types[:needs_approval],
+ ReviewableScore.types[:spam],
+ )
+ end
end
it "can handle spam+silence flags" do
@@ -423,6 +462,7 @@ describe DiscourseAi::Automation::LlmTriage do
end
it "includes the base path in the flagged post message" do
+ automation = Fabricate(:automation, script: "llm_triage")
allow(Discourse).to receive(:base_path).and_return("http://test.host")
DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
@@ -431,7 +471,7 @@ describe DiscourseAi::Automation::LlmTriage do
triage_agent_id: ai_agent.id,
search_for_text: "bad",
flag_post: true,
- automation: nil,
+ automation: automation,
)
end