A platform for community discussion. Free, open, simple.
FIX: Avoid duplicate AI triage reviewables for spam flags (#41043)
Krzysztof Kotlarek3709009232e1
diff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.ymlindex 3fe654e7481..0563b342440 100644--- a/plugins/discourse-ai/config/locales/server.en.yml+++ b/plugins/discourse-ai/config/locales/server.en.yml@@ -31,8 +31,11 @@ en:title: Triage posts using AIdescription: "Triage posts using a large language model"flagged_post: |- <div>Response from the model:</div>- <p>%{llm_response}</p>+ %{score_reason}+ %{response}+ flagged_post_response: |+ <p>Response from the model: %{llm_response}</p>+ flagged_post_score_reason: |<b>Triggered by the <a href="%{base_path}/admin/plugins/automation/%{automation_id}">%{automation_name}</a> rule.</b>notify_author_pm:subject: "Your post was temporarily removed for review"@@ -752,6 +755,7 @@ en:already_flagged: "Post is already flagged for review."flagged: "Post flagged for review."reason: "AI bot flagged this post.\n\nReason: %{reason}"+ score_reason: "AI bot flagged this post."close_topic:errors:diff --git a/plugins/discourse-ai/lib/agents/tools/flag_post.rb b/plugins/discourse-ai/lib/agents/tools/flag_post.rbindex d7de03086a1..b6fc659c2e7 100644--- a/plugins/discourse-ai/lib/agents/tools/flag_post.rb+++ b/plugins/discourse-ai/lib/agents/tools/flag_post.rb@@ -74,13 +74,14 @@ module DiscourseAiendflag_success = true- if %w[spam spam_silence].include?(flag_type)+ if spam_flag?result =PostActionCreator.new(Discourse.system_user,post,PostActionType.types[:spam],- message: flag_reason,+ message: spam_post_action_message,+ reason: spam_score_reason,queue_for_review: true,).performflag_success = result.success?@@ -153,18 +154,35 @@ module DiscourseAienddef flag_reason- if feature_context[:automation_id].present? && feature_context[:automation_name].present?- I18n.t(- "discourse_automation.scriptables.llm_triage.flagged_post",- base_path: feature_context[:base_path] || Discourse.base_path,- llm_response:- ERB::Util.html_escape(feature_context[:llm_response].presence || reason),- automation_id: feature_context[:automation_id].to_s,- automation_name: ERB::Util.html_escape(feature_context[:automation_name].to_s),- )- else- I18n.t("discourse_ai.ai_bot.flag_post.reason", reason: reason)- end+ DiscourseAi::Automation.flag_post_reason(+ reason: reason,+ llm_response: feature_context[:llm_response],+ automation_id: feature_context[:automation_id],+ automation_name: feature_context[:automation_name],+ base_path: feature_context[:base_path] || Discourse.base_path,+ )+ end++ def spam_score_reason+ DiscourseAi::Automation.spam_score_reason(+ automation_id: feature_context[:automation_id],+ automation_name: feature_context[:automation_name],+ base_path: feature_context[:base_path] || Discourse.base_path,+ )+ end++ def spam_post_action_message+ DiscourseAi::Automation.spam_post_action_message(+ reason: reason,+ llm_response: feature_context[:llm_response],+ automation_id: feature_context[:automation_id],+ automation_name: feature_context[:automation_name],+ base_path: feature_context[:base_path] || Discourse.base_path,+ )+ end++ def spam_flag?+ %w[spam spam_silence].include?(flag_type)enddef feature_contextdiff --git a/plugins/discourse-ai/lib/automation.rb b/plugins/discourse-ai/lib/automation.rbindex 267c811adff..9b8689c79a4 100644--- a/plugins/discourse-ai/lib/automation.rb+++ b/plugins/discourse-ai/lib/automation.rb@@ -6,6 +6,73 @@ module DiscourseAi%w[spam spam_silence]end+ def self.flag_post_reason(+ reason:,+ llm_response: nil,+ automation_id: nil,+ automation_name: nil,+ base_path: Discourse.base_path+ )+ if automation_context?(automation_id, automation_name)+ I18n.t(+ "discourse_automation.scriptables.llm_triage.flagged_post",+ score_reason:+ spam_score_reason(+ automation_id: automation_id,+ automation_name: automation_name,+ base_path: base_path,+ ),+ response: flag_post_response(reason: reason, llm_response: llm_response),+ )+ else+ I18n.t("discourse_ai.ai_bot.flag_post.reason", reason: reason)+ end+ end++ def self.spam_score_reason(+ automation_id: nil,+ automation_name: nil,+ base_path: Discourse.base_path+ )+ if automation_context?(automation_id, automation_name)+ I18n.t(+ "discourse_automation.scriptables.llm_triage.flagged_post_score_reason",+ base_path: base_path,+ automation_id: automation_id.to_s,+ automation_name: ERB::Util.html_escape(automation_name.to_s),+ )+ else+ I18n.t("discourse_ai.ai_bot.flag_post.score_reason")+ end+ end++ def self.spam_post_action_message(+ reason:,+ llm_response: nil,+ automation_id: nil,+ automation_name: nil,+ base_path: Discourse.base_path+ )+ if automation_context?(automation_id, automation_name)+ flag_post_response(reason: reason, llm_response: llm_response)+ else+ flag_post_reason(+ reason: reason,+ llm_response: llm_response,+ automation_id: automation_id,+ automation_name: automation_name,+ base_path: base_path,+ )+ end+ end++ def self.flag_post_response(reason:, llm_response: nil)+ I18n.t(+ "discourse_automation.scriptables.llm_triage.flagged_post_response",+ llm_response: ERB::Util.html_escape(llm_response.presence || reason),+ )+ end+def self.flag_types[{ id: "review", translated_name: I18n.t("discourse_automation.ai.flag_types.review") },@@ -65,5 +132,10 @@ module DiscourseAiphashendend++ def self.automation_context?(automation_id, automation_name)+ automation_id.present? && automation_name.present?+ end+ private_class_method :automation_context?, :flag_post_responseendenddiff --git a/plugins/discourse-ai/lib/automation/llm_triage.rb b/plugins/discourse-ai/lib/automation/llm_triage.rbindex 067c0da570a..ef0f6a44ed4 100644--- a/plugins/discourse-ai/lib/automation/llm_triage.rb+++ b/plugins/discourse-ai/lib/automation/llm_triage.rb@@ -36,6 +36,24 @@ module DiscourseAiendend+ def self.promote_post_reviewable_to_flagged!(post)+ reviewable = ReviewablePost.pending.find_by(target: post)+ return if reviewable.blank?+ return if ReviewableFlaggedPost.exists?(target: post)++ reviewable.update!(+ type: ReviewableFlaggedPost.name,+ potential_spam: true,+ reviewable_by_moderator: true,+ payload: {+ targets_topic: false,+ },+ )+ rescue ActiveRecord::RecordNotUnique+ raise if !ReviewableFlaggedPost.exists?(target: post)+ end+ private_class_method :promote_post_reviewable_to_flagged!+def self.handle(post:,triage_agent_id:,@@ -197,22 +215,36 @@ module DiscourseAialready_flagged = flagged_by_another_triage_rule?(post)score_reason =- I18n.t(- "discourse_automation.scriptables.llm_triage.flagged_post",- base_path: Discourse.base_path,- llm_response: ERB::Util.html_escape(result),+ DiscourseAi::Automation.flag_post_reason(+ reason: result,automation_id: automation&.id.to_s,- automation_name: ERB::Util.html_escape(automation&.name.to_s),+ automation_name: automation&.name,)if !flagged_by_toolif flag_type == :spam || flag_type == :spam_silence+ spam_score_reason =+ DiscourseAi::Automation.spam_score_reason(+ automation_id: automation&.id.to_s,+ automation_name: automation&.name,+ )++ spam_post_action_message =+ DiscourseAi::Automation.spam_post_action_message(+ reason: result,+ automation_id: automation&.id.to_s,+ automation_name: automation&.name,+ )++ promote_post_reviewable_to_flagged!(post)+result =PostActionCreator.new(Discourse.system_user,post,PostActionType.types[:spam],- message: score_reason,+ message: spam_post_action_message,+ reason: spam_score_reason,queue_for_review: true,).performdiff --git a/plugins/discourse-ai/spec/lib/discourse_automation/llm_triage_spec.rb b/plugins/discourse-ai/spec/lib/discourse_automation/llm_triage_spec.rbindex f50568c1168..49222a8f14d 100644--- a/plugins/discourse-ai/spec/lib/discourse_automation/llm_triage_spec.rb+++ b/plugins/discourse-ai/spec/lib/discourse_automation/llm_triage_spec.rb@@ -276,8 +276,69 @@ describe DiscourseAi::Automation::LlmTriage doreviewable = ReviewableFlaggedPost.find_by(target: post)scores = reviewable.reviewable_scores.select { |rs| rs.user == Discourse.system_user }+ conversation_excerpt = scores.first.reviewable_conversation.conversation_posts.first.excerpt- expect(scores.size).to eq(1)+ aggregate_failures do+ expect(scores.size).to eq(1)+ expect(scores.first.reason).to include("Triggered by")+ expect(scores.first.reason).not_to include("Response from the model:")+ expect(conversation_excerpt).to include("Response from the model: bad")+ end+ end++ it "adds spam scores to an existing flagged reviewable" do+ reviewable_post =+ ReviewablePost.needs_review!(+ target: post,+ created_by: Discourse.system_user,+ reviewable_by_moderator: true,+ )+ reviewable_post.add_score(+ Discourse.system_user,+ ReviewableScore.types[:needs_approval],+ reason: "Existing review flag",+ force_review: true,+ )++ flagged_reviewable =+ ReviewableFlaggedPost.needs_review!(+ target: post,+ created_by: Fabricate(:user),+ reviewable_by_moderator: true,+ )+ flagged_reviewable.add_score(+ flagged_reviewable.created_by,+ ReviewableScore.types[:spam],+ reason: "Existing spam flag",+ force_review: true,+ )++ DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do+ add_automation_field("flag_type", "spam")++ automation.running_in_background!+ automation.trigger!({ "post" => post })+ end++ flagged_reviewable.reload+ system_spam_scores =+ flagged_reviewable.reviewable_scores.select do |score|+ score.user == Discourse.system_user &&+ score.reviewable_score_type == ReviewableScore.types[:spam]+ end+ conversation_excerpt =+ system_spam_scores.first.reviewable_conversation.conversation_posts.first.excerpt++ aggregate_failures do+ expect(ReviewablePost.pending.where(target: post)).to contain_exactly(reviewable_post)+ expect(ReviewableFlaggedPost.pending.where(target: post)).to contain_exactly(+ flagged_reviewable,+ )+ expect(system_spam_scores.size).to eq(1)+ expect(system_spam_scores.first.reason).to include("Triggered by")+ expect(system_spam_scores.first.reason).not_to include("Response from the model:")+ expect(conversation_excerpt).to include("Response from the model: bad")+ endendit "flags the post once when not using spam-based types (review_* types)" do@@ -292,7 +353,13 @@ describe DiscourseAi::Automation::LlmTriage doreviewable = ReviewablePost.find_by(target: post)scores = reviewable.reviewable_scores.select { |rs| rs.user == Discourse.system_user }- expect(scores.size).to eq(1)+ score_reason = scores.first.reason++ aggregate_failures do+ expect(scores.size).to eq(1)+ expect(score_reason.index("Triggered by")).to be <+ score_reason.index("Response from the model")+ endendit "makes the reviewable visible to moderators when using review flag types" dodiff --git a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rbindex 55f59cfd137..760d739451b 100644--- a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb@@ -93,7 +93,9 @@ describe DiscourseAi::Automation::LlmTriage doexpect(reviewable.target_id).to eq(post.id)expect(reviewable.target_type).to eq("Post")- expect(reviewable.reviewable_scores.first.reason).to include("bad")+ expect(reviewable.reviewable_scores.first.reason).to eq(+ I18n.t("discourse_ai.ai_bot.flag_post.reason", reason: "bad"),+ )endit "flags via tool call when the agent invokes flag_post" do@@ -139,6 +141,43 @@ describe DiscourseAi::Automation::LlmTriage doexpect(post.reload).to be_hiddenexpect(post.topic.reload.visible).to eq(false)+ expect(ReviewableFlaggedPost.last.reviewable_scores.first.reason).to eq(+ I18n.t("discourse_ai.ai_bot.flag_post.score_reason"),+ )+ end++ it "keeps one reviewable when spam follows review" do+ DiscourseAi::Completions::Llm.with_prepared_responses(%w[bad bad]) do+ triage(+ post: post,+ triage_agent_id: ai_agent.id,+ search_for_text: "bad",+ flag_post: true,+ flag_type: :review,+ automation: nil,+ )++ triage(+ post: post,+ triage_agent_id: ai_agent.id,+ search_for_text: "bad",+ flag_post: true,+ flag_type: :spam,+ automation: nil,+ )+ end++ reviewables = Reviewable.pending.where(target: post)+ reviewable = reviewables.first++ aggregate_failures do+ expect(reviewables.size).to eq(1)+ expect(reviewable).to be_a(ReviewableFlaggedPost)+ expect(reviewable.reviewable_scores.map(&:reviewable_score_type)).to contain_exactly(+ ReviewableScore.types[:needs_approval],+ ReviewableScore.types[:spam],+ )+ endendit "can handle spam+silence flags" do@@ -423,6 +462,7 @@ describe DiscourseAi::Automation::LlmTriage doendit "includes the base path in the flagged post message" do+ automation = Fabricate(:automation, script: "llm_triage")allow(Discourse).to receive(:base_path).and_return("http://test.host")DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do@@ -431,7 +471,7 @@ describe DiscourseAi::Automation::LlmTriage dotriage_agent_id: ai_agent.id,search_for_text: "bad",flag_post: true,- automation: nil,+ automation: automation,)end