A platform for community discussion. Free, open, simple.
FEATURE: Add AI triage filters to the review queue (#42861)
Gabriel Grubbac78c141f869a
diff --git a/app/controllers/reviewables_controller.rb b/app/controllers/reviewables_controller.rbindex 46a104d4502..b5f65a864ed 100644--- a/app/controllers/reviewables_controller.rb+++ b/app/controllers/reviewables_controller.rb@@ -16,7 +16,9 @@ class ReviewablesController < ApplicationControlleroffset = params[:offset].to_iif params[:type].present?- raise Discourse::InvalidParameters.new(:type) unless Reviewable.valid_type?(params[:type])+ unless Reviewable.valid_filter_type?(params[:type])+ raise Discourse::InvalidParameters.new(:type)+ endendstatus = (params[:status] || "pending").to_sym@@ -74,13 +76,17 @@ class ReviewablesController < ApplicationControllerfilters.merge(total_rows_reviewables: total_rows,types: meta_types,- reviewable_types: Reviewable.types,+ reviewable_types:+ Reviewable.types + Reviewable.custom_filter_type_options.map { |option| option[:id] },unknown_reviewable_types_and_sources: Reviewable.unknown_types_and_sources,score_types:ReviewableScore.types.filter { |k, v| k != :notify_user }- .map { |k, v| { id: v, name: ReviewableScore.type_title(k) } },+ .map { |k, v| { id: v, name: ReviewableScore.type_title(k) } }+ .concat(+ Reviewable.custom_reason_filter_options.map { |option| option.slice(:id, :name) },+ ),reviewable_count: current_user.reviewable_count,unseen_reviewable_count: Reviewable.unseen_reviewable_count(current_user),),diff --git a/app/models/reviewable.rb b/app/models/reviewable.rbindex 4f6ff634336..68c3ac0c258 100644--- a/app/models/reviewable.rb+++ b/app/models/reviewable.rb@@ -80,6 +80,10 @@ class Reviewable < ActiveRecord::Basetype.to_s.safe_constantize.in?(types)end+ def self.valid_filter_type?(type)+ valid_type?(type) || custom_filter_type_options.any? { |option| option[:id].to_s == type.to_s }+ end+def self.types[ReviewableFlaggedPost, ReviewableQueuedPost, ReviewableUser, ReviewablePost]end@@ -106,12 +110,40 @@ class Reviewable < ActiveRecord::Base@reviewable_filters ||= []end- def self.add_custom_filter(new_filter)+ def self.custom_filter_type_options+ (@reviewable_filter_type_options || []) | DiscoursePluginRegistry.reviewable_filter_type_options+ end++ def self.custom_reason_filter_options+ registrations =+ (@reviewable_reason_filter_options || []) |+ DiscoursePluginRegistry.reviewable_filter_reason_registrations++ registrations.flat_map do |registration|+ options = registration[:options]+ options = options.call if options.respond_to?(:call)++ Array(options).map { |option| option.merge(filter: registration[:filter]) }+ end+ end++ def self.add_custom_filter(new_filter, type_filter: nil, reason_filters: nil)custom_filters << new_filter+ if type_filter+ (@reviewable_filter_type_options ||= []) << type_filter.merge(filter: new_filter.first)+ end+ if reason_filters+ (@reviewable_reason_filter_options ||= []) << {+ filter: new_filter.first,+ options: reason_filters,+ }+ endenddef self.clear_custom_filters!@reviewable_filters = []+ @reviewable_filter_type_options = []+ @reviewable_reason_filter_options = []enddef set_type_source@@ -559,7 +591,14 @@ class Reviewable < ActiveRecord::Baseresult = viewable_by(user, order: order, preload: preload)result = by_status(result, status)result = result.where(id: ids) if ids- result = result.where("reviewables.type = ?", Reviewable.sti_class_for(type).sti_name) if type+ if type+ custom_type = custom_filter_type_options.find { |option| option[:id].to_s == type.to_s }+ if custom_type+ result = apply_custom_filter(result, custom_type[:filter], custom_type[:value])+ else+ result = result.where("reviewables.type = ?", Reviewable.sti_class_for(type).sti_name)+ end+ endresult = result.where("reviewables.category_id = ?", category_id) if category_idresult = result.where("reviewables.topic_id = ?", topic_id) if topic_idresult = result.where("reviewables.created_at >= ?", from_date) if from_date@@ -577,13 +616,19 @@ class Reviewable < ActiveRecord::Baseendif score_type- score_type = score_type.to_i- result = result.where(<<~SQL, score_type: score_type)- EXISTS(- SELECT 1 FROM reviewable_scores- WHERE reviewable_scores.reviewable_id = reviewables.id AND reviewable_scores.reviewable_score_type = :score_type- )- SQL+ custom_score_type =+ custom_reason_filter_options.find { |option| option[:id].to_s == score_type.to_s }+ if custom_score_type+ result = apply_custom_filter(result, custom_score_type[:filter], custom_score_type[:value])+ else+ score_type = score_type.to_i+ result = result.where(<<~SQL, score_type: score_type)+ EXISTS(+ SELECT 1 FROM reviewable_scores+ WHERE reviewable_scores.reviewable_id = reviewables.id AND reviewable_scores.reviewable_score_type = :score_type+ )+ SQL+ endendif reviewed_by@@ -626,7 +671,7 @@ class Reviewable < ActiveRecord::Basefilter_query = filter.lastnext(memo) unless additional_filters[key]- filter_query.call(result, additional_filters[key])+ filter_query.call(memo, additional_filters[key])endend@@ -653,6 +698,12 @@ class Reviewable < ActiveRecord::Baseresultend+ def self.apply_custom_filter(result, key, value)+ filter = custom_filters.find { |registered_filter| registered_filter.first == key }+ filter ? filter.last.call(result, value) : result+ end+ private_class_method :apply_custom_filter+def self.unseen_list_for(user, preload: true, limit: nil)results = list_for(user, preload: preload, limit: limit, include_claimed_by_others: false)if user.last_seen_reviewable_iddiff --git a/lib/discourse_plugin_registry.rb b/lib/discourse_plugin_registry.rbindex 278cc6d4568..2f2f7301184 100644--- a/lib/discourse_plugin_registry.rb+++ b/lib/discourse_plugin_registry.rb@@ -104,6 +104,8 @@ class DiscoursePluginRegistrydefine_filtered_register :permitted_bulk_action_parametersdefine_filtered_register :reviewable_paramsdefine_filtered_register :reviewable_score_links+ define_filtered_register :reviewable_filter_type_options+ define_filtered_register :reviewable_filter_reason_registrationsdefine_filtered_register :presence_channel_prefixesdiff --git a/lib/plugin/instance.rb b/lib/plugin/instance.rbindex f3998fa66bc..d9fd4eae0c6 100644--- a/lib/plugin/instance.rb+++ b/lib/plugin/instance.rb@@ -1009,8 +1009,25 @@ class Plugin::Instance# Receives an array with two elements:# 1. A symbol that represents the name of the value to filter.# 2. A Proc that takes the existing ActiveRecord::Relation and the value received from the front-end.- def add_custom_reviewable_filter(filter)+ #+ # type_filter accepts an id and filter value for the Type control. reason_filters accepts an+ # array or callable returning ids, names, and filter values for the Reason control.+ def add_custom_reviewable_filter(filter, type_filter: nil, reason_filters: nil)reloadable_patch { Reviewable.add_custom_filter(filter) }++ if type_filter+ DiscoursePluginRegistry.register_reviewable_filter_type_option(+ type_filter.merge(filter: filter.first),+ self,+ )+ end++ if reason_filters+ DiscoursePluginRegistry.register_reviewable_filter_reason_registration(+ { filter: filter.first, options: reason_filters },+ self,+ )+ endend# Register a new API key scope.diff --git a/plugins/discourse-ai/config/locales/client.en.yml b/plugins/discourse-ai/config/locales/client.en.ymlindex fe03cce096d..aa59b108e36 100644--- a/plugins/discourse-ai/config/locales/client.en.yml+++ b/plugins/discourse-ai/config/locales/client.en.yml@@ -1654,6 +1654,8 @@ en:rejected_tool_action:title: "Tool rejected"types:+ discourse_ai_triage:+ title: "AI triage"reviewable_ai_post:title: "AI-Flagged post"reviewable_ai_chat_message:diff --git a/plugins/discourse-ai/lib/agents/tools/flag_post.rb b/plugins/discourse-ai/lib/agents/tools/flag_post.rbindex b6fc659c2e7..c7799513704 100644--- a/plugins/discourse-ai/lib/agents/tools/flag_post.rb+++ b/plugins/discourse-ai/lib/agents/tools/flag_post.rb@@ -82,6 +82,7 @@ module DiscourseAiPostActionType.types[:spam],message: spam_post_action_message,reason: spam_score_reason,+ context: automation_score_context,queue_for_review: true,).performflag_success = result.success?@@ -106,6 +107,7 @@ module DiscourseAiDiscourse.system_user,ReviewableScore.types[:needs_approval],reason: flag_reason,+ context: automation_score_context,force_review: true,)@@ -163,6 +165,10 @@ module DiscourseAi)end+ def automation_score_context+ DiscourseAi::Automation.triage_automation_score_context(feature_context[:automation_id])+ end+def spam_score_reasonDiscourseAi::Automation.spam_score_reason(automation_id: feature_context[:automation_id],diff --git a/plugins/discourse-ai/lib/automation.rb b/plugins/discourse-ai/lib/automation.rbindex cd30a31f833..c3e6d199443 100644--- a/plugins/discourse-ai/lib/automation.rb+++ b/plugins/discourse-ai/lib/automation.rb@@ -2,10 +2,16 @@module DiscourseAimodule Automation+ TRIAGE_AUTOMATION_SCORE_CONTEXT_PREFIX = "discourse_ai:triage_automation:"+def self.spam_based_flag_types%w[spam spam_silence]end+ def self.triage_automation_score_context(automation_id)+ "#{TRIAGE_AUTOMATION_SCORE_CONTEXT_PREFIX}#{automation_id}" if automation_id.present?+ end+def self.flag_post_reason(reason:,llm_response: nil,diff --git a/plugins/discourse-ai/lib/automation/llm_triage.rb b/plugins/discourse-ai/lib/automation/llm_triage.rbindex afd8a3d7fcf..a07aa665892 100644--- a/plugins/discourse-ai/lib/automation/llm_triage.rb+++ b/plugins/discourse-ai/lib/automation/llm_triage.rb@@ -239,6 +239,9 @@ module DiscourseAiautomation_name: automation&.name,)+ automation_score_context =+ DiscourseAi::Automation.triage_automation_score_context(automation&.id)+if !flagged_by_toolif flag_type == :spam || flag_type == :spam_silencespam_score_reason =@@ -263,6 +266,7 @@ module DiscourseAiPostActionType.types[:spam],message: spam_post_action_message,reason: spam_score_reason,+ context: automation_score_context,queue_for_review: true,).perform@@ -282,6 +286,7 @@ module DiscourseAiDiscourse.system_user,ReviewableScore.types[:needs_approval],reason: score_reason,+ context: automation_score_context,force_review: true,)diff --git a/plugins/discourse-ai/plugin.rb b/plugins/discourse-ai/plugin.rbindex d2cf1698f7d..16aeb43a692 100644--- a/plugins/discourse-ai/plugin.rb+++ b/plugins/discourse-ai/plugin.rb@@ -251,4 +251,39 @@ after_initialize doadd_model_callback(DiscourseAutomation::Automation, :after_save) doDiscourseAi::Configuration::Feature.feature_cache.flush!end++ add_custom_reviewable_filter(+ [+ :ai_triage_automation_id,+ Proc.new do |results, value|+ context = "#{DiscourseAi::Automation::TRIAGE_AUTOMATION_SCORE_CONTEXT_PREFIX}%"+ if value != :all+ automation_id = value.is_a?(Integer) ? value : value.to_s[/\A\d+\z/]&.to_i+ next results if !automation_id || automation_id <= 0++ context = DiscourseAi::Automation.triage_automation_score_context(automation_id)+ end++ results.where(<<~SQL, context:)+ EXISTS (+ SELECT 1+ FROM reviewable_scores+ WHERE reviewable_scores.reviewable_id = reviewables.id+ AND reviewable_scores.context LIKE :context+ )+ SQL+ end,+ ],+ type_filter: {+ id: "discourse_ai:triage",+ value: :all,+ },+ reason_filters: -> do+ DiscourseAutomation::Automation+ .where(script: %w[llm_triage llm_agent_triage])+ .order(:name)+ .pluck(:id, :name)+ .map { |id, name| { id: "ai_triage_automation:#{id}", name:, value: id } }+ end,+ )enddiff --git a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rbindex 6883b9e3472..8555f4dd209 100644--- a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb@@ -98,6 +98,69 @@ describe DiscourseAi::Automation::LlmTriage doexpect(reviewable.reviewable_scores.first.reason).to eq(I18n.t("discourse_ai.ai_bot.flag_post.reason", reason: "bad"),)+ expect(reviewable.reviewable_scores.first.context).to be_nil+ end++ it "records the automation on the reviewable score for review flags" do+ automation = Fabricate(:automation, script: "llm_triage")++ DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do+ triage(+ post: post,+ triage_agent_id: ai_agent.id,+ search_for_text: "bad",+ flag_post: true,+ automation: automation,+ )+ end++ score = ReviewablePost.last.reviewable_scores.first+ expect(score.context).to eq("discourse_ai:triage_automation:#{automation.id}")+ end++ it "records the automation on the reviewable score for spam flags" do+ automation = Fabricate(:automation, script: "llm_triage")++ DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do+ triage(+ post: post,+ triage_agent_id: ai_agent.id,+ search_for_text: "bad",+ flag_post: true,+ flag_type: :spam,+ automation: automation,+ )+ end++ score = ReviewableFlaggedPost.last.reviewable_scores.first+ expect(score.context).to eq("discourse_ai:triage_automation:#{automation.id}")+ end++ it "records the automation on the reviewable score when flagged via the flag_post tool" do+ automation = Fabricate(:automation, script: "llm_triage")+ ai_agent.update!(tools: ["FlagPost"])+ tool_call =+ DiscourseAi::Completions::ToolCall.new(+ name: "flag_post",+ parameters: {+ flag_post: true,+ reason: "Looks unsafe",+ },+ id: "tool_call_1",+ )++ DiscourseAi::Completions::Llm.with_prepared_responses([tool_call, "all good"]) do+ triage(+ post: post,+ triage_agent_id: ai_agent.id,+ search_for_text: "bad",+ flag_post: true,+ automation: automation,+ )+ end++ score = ReviewablePost.last.reviewable_scores.first+ expect(score.context).to eq("discourse_ai:triage_automation:#{automation.id}")endit "flags via tool call when the agent invokes flag_post" dodiff --git a/plugins/discourse-ai/spec/requests/admin/reviewable_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/reviewable_controller_spec.rbindex d6385f50ea0..2c81e53e21c 100644--- a/plugins/discourse-ai/spec/requests/admin/reviewable_controller_spec.rb+++ b/plugins/discourse-ai/spec/requests/admin/reviewable_controller_spec.rb@@ -47,4 +47,77 @@ RSpec.describe ReviewablesController dojson = JSON.parse(response.body)expect(json["reviewables"].length).to eq(2)end++ describe "AI triage filters" do+ fab!(:automation) { Fabricate(:automation, name: "Politics check", script: "llm_triage") }+ fab!(:agent_automation) do+ Fabricate(:automation, name: "Sensitive content check", script: "llm_agent_triage")+ end++ before do+ reviewable.add_score(+ Discourse.system_user,+ ReviewableScore.types[:needs_approval],+ reason: "llm_triage",+ context: DiscourseAi::Automation.triage_automation_score_context(automation.id),+ force_review: true,+ )+ end++ it "adds AI triage to Type and automation names to Reason" do+ sign_in(admin)++ get "/review.json"++ expect(response.status).to eq(200)+ expect(response.parsed_body.dig("meta", "reviewable_types")).to include("discourse_ai:triage")+ expect(response.parsed_body.dig("meta", "score_types")).to include(+ { "id" => "ai_triage_automation:#{automation.id}", "name" => automation.name },+ { "id" => "ai_triage_automation:#{agent_automation.id}", "name" => agent_automation.name },+ )+ end++ it "hides AI triage filters when the plugin is disabled on the current site" do+ SiteSetting.discourse_ai_enabled = false+ sign_in(admin)++ get "/review.json"++ expect(response.status).to eq(200)+ expect(response.parsed_body.dig("meta", "reviewable_types")).not_to include(+ "discourse_ai:triage",+ )+ expect(response.parsed_body.dig("meta", "score_types")).not_to include(+ { "id" => "ai_triage_automation:#{automation.id}", "name" => automation.name },+ )+ end++ it "filters by AI triage Type and automation Reason" do+ sign_in(admin)++ get "/review.json?type=discourse_ai:triage"+ expect(response.status).to eq(200)+ ids = response.parsed_body["reviewables"].map { |json| json["id"] }+ expect(ids).to contain_exactly(reviewable.id)++ get "/review.json?type=discourse_ai:triage&score_type=ai_triage_automation:#{automation.id}"+ expect(response.status).to eq(200)+ ids = response.parsed_body["reviewables"].map { |json| json["id"] }+ expect(ids).to contain_exactly(reviewable.id)++ get "/review.json?score_type=ai_triage_automation:#{automation.id + 1}"+ expect(response.status).to eq(200)+ expect(response.parsed_body["reviewables"]).to be_empty+ end++ it "ignores malformed filter values instead of failing" do+ sign_in(admin)++ ["true", "[1]", "{\"x\":1}", "\"1 OR 1=1 --\"", "0", "1.9", "\"junk\""].each do |malformed|+ get "/review.json?additional_filters={\"ai_triage_automation_id\":#{CGI.escape(malformed)}}"+ expect(response.status).to eq(200)+ expect(response.parsed_body["reviewables"].length).to eq(2)+ end+ end+ endenddiff --git a/spec/models/reviewable_spec.rb b/spec/models/reviewable_spec.rbindex b889225c23a..ab7a5fa3490 100644--- a/spec/models/reviewable_spec.rb+++ b/spec/models/reviewable_spec.rb@@ -812,9 +812,10 @@ RSpec.describe Reviewable, type: :model doafter { Reviewable.clear_custom_filters! }it "correctly add a new filter" do- Reviewable.add_custom_filter([:assigned_to, Proc.new { |results, value| results }])+ custom_filter = [:assigned_to, Proc.new { |results, value| results }]+ Reviewable.add_custom_filter(custom_filter)- expect(Reviewable.custom_filters.size).to eq(1)+ expect(Reviewable.custom_filters).to include(custom_filter)endit "applies the custom filter" do@@ -831,6 +832,65 @@ RSpec.describe Reviewable, type: :model doexpect(results.first).to eq first_reviewableend+ it "exposes a custom filter through the Type and Reason filters" do+ admin = Fabricate(:admin)+ type_reviewable = Fabricate(:reviewable)+ reason_reviewable = Fabricate(:reviewable)++ Reviewable.add_custom_filter(+ [:source, Proc.new { |results, value| results.where(id: value) }],+ type_filter: {+ id: "custom_source",+ value: type_reviewable.id,+ },+ reason_filters: -> do+ [{ id: "custom_source:reason", name: "Custom reason", value: reason_reviewable.id }]+ end,+ )++ expect(Reviewable.valid_type?("custom_source")).to eq(false)+ expect(Reviewable.valid_filter_type?("custom_source")).to eq(true)+ expect(Reviewable.custom_filter_type_options).to include(+ { id: "custom_source", value: type_reviewable.id, filter: :source },+ )+ expect(Reviewable.custom_reason_filter_options).to include(+ {+ id: "custom_source:reason",+ name: "Custom reason",+ value: reason_reviewable.id,+ filter: :source,+ },+ )+ expect(Reviewable.list_for(admin, type: "custom_source")).to contain_exactly(type_reviewable)+ expect(Reviewable.list_for(admin, score_type: "custom_source:reason")).to contain_exactly(+ reason_reviewable,+ )+ end++ it "composes multiple custom filters" do+ admin = Fabricate(:admin)+ first_reviewable = Fabricate(:reviewable)+ second_reviewable = Fabricate(:reviewable)++ Reviewable.add_custom_filter(+ [:first_id, Proc.new { |results, value| results.where(id: value) }],+ )+ Reviewable.add_custom_filter(+ [:second_id, Proc.new { |results, value| results.where(id: value) }],+ )++ results =+ Reviewable.list_for(+ admin,+ additional_filters: {+ first_id: first_reviewable.id,+ second_id: second_reviewable.id,+ },+ )++ expect(results).to be_empty+ end+context "when listing for a moderator with a custom filter that joins tables with same named columns" doit "should not error" dofirst_reviewable = Fabricate(:reviewable)