Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Add AI triage filters to the review queue (#42861)

Gabriel Grubbac78c141f869a
diff --git a/app/controllers/reviewables_controller.rb b/app/controllers/reviewables_controller.rb
index 46a104d4502..b5f65a864ed 100644
--- a/app/controllers/reviewables_controller.rb
+++ b/app/controllers/reviewables_controller.rb
@@ -16,7 +16,9 @@ class ReviewablesController < ApplicationController
offset = params[:offset].to_i
if params[:type].present?
- raise Discourse::InvalidParameters.new(:type) unless Reviewable.valid_type?(params[:type])
+ unless Reviewable.valid_filter_type?(params[:type])
+ raise Discourse::InvalidParameters.new(:type)
+ end
end
status = (params[:status] || "pending").to_sym
@@ -74,13 +76,17 @@ class ReviewablesController < ApplicationController
filters.merge(
total_rows_reviewables: total_rows,
types: meta_types,
- reviewable_types: Reviewable.types,
+ reviewable_types:
+ Reviewable.types + Reviewable.custom_filter_type_options.map { |option| option[:id] },
unknown_reviewable_types_and_sources: Reviewable.unknown_types_and_sources,
score_types:
ReviewableScore
.types
.filter { |k, v| k != :notify_user }
- .map { |k, v| { id: v, name: ReviewableScore.type_title(k) } },
+ .map { |k, v| { id: v, name: ReviewableScore.type_title(k) } }
+ .concat(
+ Reviewable.custom_reason_filter_options.map { |option| option.slice(:id, :name) },
+ ),
reviewable_count: current_user.reviewable_count,
unseen_reviewable_count: Reviewable.unseen_reviewable_count(current_user),
),
diff --git a/app/models/reviewable.rb b/app/models/reviewable.rb
index 4f6ff634336..68c3ac0c258 100644
--- a/app/models/reviewable.rb
+++ b/app/models/reviewable.rb
@@ -80,6 +80,10 @@ class Reviewable < ActiveRecord::Base
type.to_s.safe_constantize.in?(types)
end
+ def self.valid_filter_type?(type)
+ valid_type?(type) || custom_filter_type_options.any? { |option| option[:id].to_s == type.to_s }
+ end
+
def self.types
[ReviewableFlaggedPost, ReviewableQueuedPost, ReviewableUser, ReviewablePost]
end
@@ -106,12 +110,40 @@ class Reviewable < ActiveRecord::Base
@reviewable_filters ||= []
end
- def self.add_custom_filter(new_filter)
+ def self.custom_filter_type_options
+ (@reviewable_filter_type_options || []) | DiscoursePluginRegistry.reviewable_filter_type_options
+ end
+
+ def self.custom_reason_filter_options
+ registrations =
+ (@reviewable_reason_filter_options || []) |
+ DiscoursePluginRegistry.reviewable_filter_reason_registrations
+
+ registrations.flat_map do |registration|
+ options = registration[:options]
+ options = options.call if options.respond_to?(:call)
+
+ Array(options).map { |option| option.merge(filter: registration[:filter]) }
+ end
+ end
+
+ def self.add_custom_filter(new_filter, type_filter: nil, reason_filters: nil)
custom_filters << new_filter
+ if type_filter
+ (@reviewable_filter_type_options ||= []) << type_filter.merge(filter: new_filter.first)
+ end
+ if reason_filters
+ (@reviewable_reason_filter_options ||= []) << {
+ filter: new_filter.first,
+ options: reason_filters,
+ }
+ end
end
def self.clear_custom_filters!
@reviewable_filters = []
+ @reviewable_filter_type_options = []
+ @reviewable_reason_filter_options = []
end
def set_type_source
@@ -559,7 +591,14 @@ class Reviewable < ActiveRecord::Base
result = viewable_by(user, order: order, preload: preload)
result = by_status(result, status)
result = result.where(id: ids) if ids
- result = result.where("reviewables.type = ?", Reviewable.sti_class_for(type).sti_name) if type
+ if type
+ custom_type = custom_filter_type_options.find { |option| option[:id].to_s == type.to_s }
+ if custom_type
+ result = apply_custom_filter(result, custom_type[:filter], custom_type[:value])
+ else
+ result = result.where("reviewables.type = ?", Reviewable.sti_class_for(type).sti_name)
+ end
+ end
result = result.where("reviewables.category_id = ?", category_id) if category_id
result = result.where("reviewables.topic_id = ?", topic_id) if topic_id
result = result.where("reviewables.created_at >= ?", from_date) if from_date
@@ -577,13 +616,19 @@ class Reviewable < ActiveRecord::Base
end
if score_type
- score_type = score_type.to_i
- result = result.where(<<~SQL, score_type: score_type)
- EXISTS(
- SELECT 1 FROM reviewable_scores
- WHERE reviewable_scores.reviewable_id = reviewables.id AND reviewable_scores.reviewable_score_type = :score_type
- )
- SQL
+ custom_score_type =
+ custom_reason_filter_options.find { |option| option[:id].to_s == score_type.to_s }
+ if custom_score_type
+ result = apply_custom_filter(result, custom_score_type[:filter], custom_score_type[:value])
+ else
+ score_type = score_type.to_i
+ result = result.where(<<~SQL, score_type: score_type)
+ EXISTS(
+ SELECT 1 FROM reviewable_scores
+ WHERE reviewable_scores.reviewable_id = reviewables.id AND reviewable_scores.reviewable_score_type = :score_type
+ )
+ SQL
+ end
end
if reviewed_by
@@ -626,7 +671,7 @@ class Reviewable < ActiveRecord::Base
filter_query = filter.last
next(memo) unless additional_filters[key]
- filter_query.call(result, additional_filters[key])
+ filter_query.call(memo, additional_filters[key])
end
end
@@ -653,6 +698,12 @@ class Reviewable < ActiveRecord::Base
result
end
+ def self.apply_custom_filter(result, key, value)
+ filter = custom_filters.find { |registered_filter| registered_filter.first == key }
+ filter ? filter.last.call(result, value) : result
+ end
+ private_class_method :apply_custom_filter
+
def self.unseen_list_for(user, preload: true, limit: nil)
results = list_for(user, preload: preload, limit: limit, include_claimed_by_others: false)
if user.last_seen_reviewable_id
diff --git a/lib/discourse_plugin_registry.rb b/lib/discourse_plugin_registry.rb
index 278cc6d4568..2f2f7301184 100644
--- a/lib/discourse_plugin_registry.rb
+++ b/lib/discourse_plugin_registry.rb
@@ -104,6 +104,8 @@ class DiscoursePluginRegistry
define_filtered_register :permitted_bulk_action_parameters
define_filtered_register :reviewable_params
define_filtered_register :reviewable_score_links
+ define_filtered_register :reviewable_filter_type_options
+ define_filtered_register :reviewable_filter_reason_registrations
define_filtered_register :presence_channel_prefixes
diff --git a/lib/plugin/instance.rb b/lib/plugin/instance.rb
index f3998fa66bc..d9fd4eae0c6 100644
--- a/lib/plugin/instance.rb
+++ b/lib/plugin/instance.rb
@@ -1009,8 +1009,25 @@ class Plugin::Instance
# Receives an array with two elements:
# 1. A symbol that represents the name of the value to filter.
# 2. A Proc that takes the existing ActiveRecord::Relation and the value received from the front-end.
- def add_custom_reviewable_filter(filter)
+ #
+ # type_filter accepts an id and filter value for the Type control. reason_filters accepts an
+ # array or callable returning ids, names, and filter values for the Reason control.
+ def add_custom_reviewable_filter(filter, type_filter: nil, reason_filters: nil)
reloadable_patch { Reviewable.add_custom_filter(filter) }
+
+ if type_filter
+ DiscoursePluginRegistry.register_reviewable_filter_type_option(
+ type_filter.merge(filter: filter.first),
+ self,
+ )
+ end
+
+ if reason_filters
+ DiscoursePluginRegistry.register_reviewable_filter_reason_registration(
+ { filter: filter.first, options: reason_filters },
+ self,
+ )
+ end
end
# Register a new API key scope.
diff --git a/plugins/discourse-ai/config/locales/client.en.yml b/plugins/discourse-ai/config/locales/client.en.yml
index fe03cce096d..aa59b108e36 100644
--- a/plugins/discourse-ai/config/locales/client.en.yml
+++ b/plugins/discourse-ai/config/locales/client.en.yml
@@ -1654,6 +1654,8 @@ en:
rejected_tool_action:
title: "Tool rejected"
types:
+ discourse_ai_triage:
+ title: "AI triage"
reviewable_ai_post:
title: "AI-Flagged post"
reviewable_ai_chat_message:
diff --git a/plugins/discourse-ai/lib/agents/tools/flag_post.rb b/plugins/discourse-ai/lib/agents/tools/flag_post.rb
index b6fc659c2e7..c7799513704 100644
--- a/plugins/discourse-ai/lib/agents/tools/flag_post.rb
+++ b/plugins/discourse-ai/lib/agents/tools/flag_post.rb
@@ -82,6 +82,7 @@ module DiscourseAi
PostActionType.types[:spam],
message: spam_post_action_message,
reason: spam_score_reason,
+ context: automation_score_context,
queue_for_review: true,
).perform
flag_success = result.success?
@@ -106,6 +107,7 @@ module DiscourseAi
Discourse.system_user,
ReviewableScore.types[:needs_approval],
reason: flag_reason,
+ context: automation_score_context,
force_review: true,
)
@@ -163,6 +165,10 @@ module DiscourseAi
)
end
+ def automation_score_context
+ DiscourseAi::Automation.triage_automation_score_context(feature_context[:automation_id])
+ end
+
def spam_score_reason
DiscourseAi::Automation.spam_score_reason(
automation_id: feature_context[:automation_id],
diff --git a/plugins/discourse-ai/lib/automation.rb b/plugins/discourse-ai/lib/automation.rb
index cd30a31f833..c3e6d199443 100644
--- a/plugins/discourse-ai/lib/automation.rb
+++ b/plugins/discourse-ai/lib/automation.rb
@@ -2,10 +2,16 @@
module DiscourseAi
module Automation
+ TRIAGE_AUTOMATION_SCORE_CONTEXT_PREFIX = "discourse_ai:triage_automation:"
+
def self.spam_based_flag_types
%w[spam spam_silence]
end
+ def self.triage_automation_score_context(automation_id)
+ "#{TRIAGE_AUTOMATION_SCORE_CONTEXT_PREFIX}#{automation_id}" if automation_id.present?
+ end
+
def self.flag_post_reason(
reason:,
llm_response: nil,
diff --git a/plugins/discourse-ai/lib/automation/llm_triage.rb b/plugins/discourse-ai/lib/automation/llm_triage.rb
index afd8a3d7fcf..a07aa665892 100644
--- a/plugins/discourse-ai/lib/automation/llm_triage.rb
+++ b/plugins/discourse-ai/lib/automation/llm_triage.rb
@@ -239,6 +239,9 @@ module DiscourseAi
automation_name: automation&.name,
)
+ automation_score_context =
+ DiscourseAi::Automation.triage_automation_score_context(automation&.id)
+
if !flagged_by_tool
if flag_type == :spam || flag_type == :spam_silence
spam_score_reason =
@@ -263,6 +266,7 @@ module DiscourseAi
PostActionType.types[:spam],
message: spam_post_action_message,
reason: spam_score_reason,
+ context: automation_score_context,
queue_for_review: true,
).perform
@@ -282,6 +286,7 @@ module DiscourseAi
Discourse.system_user,
ReviewableScore.types[:needs_approval],
reason: score_reason,
+ context: automation_score_context,
force_review: true,
)
diff --git a/plugins/discourse-ai/plugin.rb b/plugins/discourse-ai/plugin.rb
index d2cf1698f7d..16aeb43a692 100644
--- a/plugins/discourse-ai/plugin.rb
+++ b/plugins/discourse-ai/plugin.rb
@@ -251,4 +251,39 @@ after_initialize do
add_model_callback(DiscourseAutomation::Automation, :after_save) do
DiscourseAi::Configuration::Feature.feature_cache.flush!
end
+
+ add_custom_reviewable_filter(
+ [
+ :ai_triage_automation_id,
+ Proc.new do |results, value|
+ context = "#{DiscourseAi::Automation::TRIAGE_AUTOMATION_SCORE_CONTEXT_PREFIX}%"
+ if value != :all
+ automation_id = value.is_a?(Integer) ? value : value.to_s[/\A\d+\z/]&.to_i
+ next results if !automation_id || automation_id <= 0
+
+ context = DiscourseAi::Automation.triage_automation_score_context(automation_id)
+ end
+
+ results.where(<<~SQL, context:)
+ EXISTS (
+ SELECT 1
+ FROM reviewable_scores
+ WHERE reviewable_scores.reviewable_id = reviewables.id
+ AND reviewable_scores.context LIKE :context
+ )
+ SQL
+ end,
+ ],
+ type_filter: {
+ id: "discourse_ai:triage",
+ value: :all,
+ },
+ reason_filters: -> do
+ DiscourseAutomation::Automation
+ .where(script: %w[llm_triage llm_agent_triage])
+ .order(:name)
+ .pluck(:id, :name)
+ .map { |id, name| { id: "ai_triage_automation:#{id}", name:, value: id } }
+ end,
+ )
end
diff --git a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb
index 6883b9e3472..8555f4dd209 100644
--- a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb
@@ -98,6 +98,69 @@ describe DiscourseAi::Automation::LlmTriage do
expect(reviewable.reviewable_scores.first.reason).to eq(
I18n.t("discourse_ai.ai_bot.flag_post.reason", reason: "bad"),
)
+ expect(reviewable.reviewable_scores.first.context).to be_nil
+ end
+
+ it "records the automation on the reviewable score for review flags" do
+ automation = Fabricate(:automation, script: "llm_triage")
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
+ triage(
+ post: post,
+ triage_agent_id: ai_agent.id,
+ search_for_text: "bad",
+ flag_post: true,
+ automation: automation,
+ )
+ end
+
+ score = ReviewablePost.last.reviewable_scores.first
+ expect(score.context).to eq("discourse_ai:triage_automation:#{automation.id}")
+ end
+
+ it "records the automation on the reviewable score for spam flags" do
+ automation = Fabricate(:automation, script: "llm_triage")
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
+ triage(
+ post: post,
+ triage_agent_id: ai_agent.id,
+ search_for_text: "bad",
+ flag_post: true,
+ flag_type: :spam,
+ automation: automation,
+ )
+ end
+
+ score = ReviewableFlaggedPost.last.reviewable_scores.first
+ expect(score.context).to eq("discourse_ai:triage_automation:#{automation.id}")
+ end
+
+ it "records the automation on the reviewable score when flagged via the flag_post tool" do
+ automation = Fabricate(:automation, script: "llm_triage")
+ ai_agent.update!(tools: ["FlagPost"])
+ tool_call =
+ DiscourseAi::Completions::ToolCall.new(
+ name: "flag_post",
+ parameters: {
+ flag_post: true,
+ reason: "Looks unsafe",
+ },
+ id: "tool_call_1",
+ )
+
+ DiscourseAi::Completions::Llm.with_prepared_responses([tool_call, "all good"]) do
+ triage(
+ post: post,
+ triage_agent_id: ai_agent.id,
+ search_for_text: "bad",
+ flag_post: true,
+ automation: automation,
+ )
+ end
+
+ score = ReviewablePost.last.reviewable_scores.first
+ expect(score.context).to eq("discourse_ai:triage_automation:#{automation.id}")
end
it "flags via tool call when the agent invokes flag_post" do
diff --git a/plugins/discourse-ai/spec/requests/admin/reviewable_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/reviewable_controller_spec.rb
index d6385f50ea0..2c81e53e21c 100644
--- a/plugins/discourse-ai/spec/requests/admin/reviewable_controller_spec.rb
+++ b/plugins/discourse-ai/spec/requests/admin/reviewable_controller_spec.rb
@@ -47,4 +47,77 @@ RSpec.describe ReviewablesController do
json = JSON.parse(response.body)
expect(json["reviewables"].length).to eq(2)
end
+
+ describe "AI triage filters" do
+ fab!(:automation) { Fabricate(:automation, name: "Politics check", script: "llm_triage") }
+ fab!(:agent_automation) do
+ Fabricate(:automation, name: "Sensitive content check", script: "llm_agent_triage")
+ end
+
+ before do
+ reviewable.add_score(
+ Discourse.system_user,
+ ReviewableScore.types[:needs_approval],
+ reason: "llm_triage",
+ context: DiscourseAi::Automation.triage_automation_score_context(automation.id),
+ force_review: true,
+ )
+ end
+
+ it "adds AI triage to Type and automation names to Reason" do
+ sign_in(admin)
+
+ get "/review.json"
+
+ expect(response.status).to eq(200)
+ expect(response.parsed_body.dig("meta", "reviewable_types")).to include("discourse_ai:triage")
+ expect(response.parsed_body.dig("meta", "score_types")).to include(
+ { "id" => "ai_triage_automation:#{automation.id}", "name" => automation.name },
+ { "id" => "ai_triage_automation:#{agent_automation.id}", "name" => agent_automation.name },
+ )
+ end
+
+ it "hides AI triage filters when the plugin is disabled on the current site" do
+ SiteSetting.discourse_ai_enabled = false
+ sign_in(admin)
+
+ get "/review.json"
+
+ expect(response.status).to eq(200)
+ expect(response.parsed_body.dig("meta", "reviewable_types")).not_to include(
+ "discourse_ai:triage",
+ )
+ expect(response.parsed_body.dig("meta", "score_types")).not_to include(
+ { "id" => "ai_triage_automation:#{automation.id}", "name" => automation.name },
+ )
+ end
+
+ it "filters by AI triage Type and automation Reason" do
+ sign_in(admin)
+
+ get "/review.json?type=discourse_ai:triage"
+ expect(response.status).to eq(200)
+ ids = response.parsed_body["reviewables"].map { |json| json["id"] }
+ expect(ids).to contain_exactly(reviewable.id)
+
+ get "/review.json?type=discourse_ai:triage&score_type=ai_triage_automation:#{automation.id}"
+ expect(response.status).to eq(200)
+ ids = response.parsed_body["reviewables"].map { |json| json["id"] }
+ expect(ids).to contain_exactly(reviewable.id)
+
+ get "/review.json?score_type=ai_triage_automation:#{automation.id + 1}"
+ expect(response.status).to eq(200)
+ expect(response.parsed_body["reviewables"]).to be_empty
+ end
+
+ it "ignores malformed filter values instead of failing" do
+ sign_in(admin)
+
+ ["true", "[1]", "{\"x\":1}", "\"1 OR 1=1 --\"", "0", "1.9", "\"junk\""].each do |malformed|
+ get "/review.json?additional_filters={\"ai_triage_automation_id\":#{CGI.escape(malformed)}}"
+ expect(response.status).to eq(200)
+ expect(response.parsed_body["reviewables"].length).to eq(2)
+ end
+ end
+ end
end
diff --git a/spec/models/reviewable_spec.rb b/spec/models/reviewable_spec.rb
index b889225c23a..ab7a5fa3490 100644
--- a/spec/models/reviewable_spec.rb
+++ b/spec/models/reviewable_spec.rb
@@ -812,9 +812,10 @@ RSpec.describe Reviewable, type: :model do
after { Reviewable.clear_custom_filters! }
it "correctly add a new filter" do
- Reviewable.add_custom_filter([:assigned_to, Proc.new { |results, value| results }])
+ custom_filter = [:assigned_to, Proc.new { |results, value| results }]
+ Reviewable.add_custom_filter(custom_filter)
- expect(Reviewable.custom_filters.size).to eq(1)
+ expect(Reviewable.custom_filters).to include(custom_filter)
end
it "applies the custom filter" do
@@ -831,6 +832,65 @@ RSpec.describe Reviewable, type: :model do
expect(results.first).to eq first_reviewable
end
+ it "exposes a custom filter through the Type and Reason filters" do
+ admin = Fabricate(:admin)
+ type_reviewable = Fabricate(:reviewable)
+ reason_reviewable = Fabricate(:reviewable)
+
+ Reviewable.add_custom_filter(
+ [:source, Proc.new { |results, value| results.where(id: value) }],
+ type_filter: {
+ id: "custom_source",
+ value: type_reviewable.id,
+ },
+ reason_filters: -> do
+ [{ id: "custom_source:reason", name: "Custom reason", value: reason_reviewable.id }]
+ end,
+ )
+
+ expect(Reviewable.valid_type?("custom_source")).to eq(false)
+ expect(Reviewable.valid_filter_type?("custom_source")).to eq(true)
+ expect(Reviewable.custom_filter_type_options).to include(
+ { id: "custom_source", value: type_reviewable.id, filter: :source },
+ )
+ expect(Reviewable.custom_reason_filter_options).to include(
+ {
+ id: "custom_source:reason",
+ name: "Custom reason",
+ value: reason_reviewable.id,
+ filter: :source,
+ },
+ )
+ expect(Reviewable.list_for(admin, type: "custom_source")).to contain_exactly(type_reviewable)
+ expect(Reviewable.list_for(admin, score_type: "custom_source:reason")).to contain_exactly(
+ reason_reviewable,
+ )
+ end
+
+ it "composes multiple custom filters" do
+ admin = Fabricate(:admin)
+ first_reviewable = Fabricate(:reviewable)
+ second_reviewable = Fabricate(:reviewable)
+
+ Reviewable.add_custom_filter(
+ [:first_id, Proc.new { |results, value| results.where(id: value) }],
+ )
+ Reviewable.add_custom_filter(
+ [:second_id, Proc.new { |results, value| results.where(id: value) }],
+ )
+
+ results =
+ Reviewable.list_for(
+ admin,
+ additional_filters: {
+ first_id: first_reviewable.id,
+ second_id: second_reviewable.id,
+ },
+ )
+
+ expect(results).to be_empty
+ end
+
context "when listing for a moderator with a custom filter that joins tables with same named columns" do
it "should not error" do
first_reviewable = Fabricate(:reviewable)