A platform for community discussion. Free, open, simple.
FEATURE: Expose an agent's structured output as workflow fields
Régis Hanol8018b367a53f
diff --git a/plugins/discourse-ai/admin/assets/javascripts/admin/lib/workflows/output-schemas/ai-agent.js b/plugins/discourse-ai/admin/assets/javascripts/admin/lib/workflows/output-schemas/ai-agent.jsnew file mode 100644index 00000000000..73d3943e04c--- /dev/null+++ b/plugins/discourse-ai/admin/assets/javascripts/admin/lib/workflows/output-schemas/ai-agent.js@@ -0,0 +1,32 @@+const DRAFT_URI = "https://json-schema.org/draft/2020-12/schema";++function schemaTypeFor(field) {+ if (field.type !== "array") {+ return { type: field.type };+ }++ const schema = {+ type: "array",+ items: { type: field.array_type || "string" },+ };+ if (Number.isInteger(field.max_items) && field.max_items >= 0) {+ schema.maxItems = field.max_items;+ }++ return schema;+}++export default function aiAgentOutputSchemas(configuration = {}) {+ const properties = { result: { type: "string" } };+ const fields = Array.isArray(configuration.agent_response_format)+ ? configuration.agent_response_format+ : [];++ for (const field of fields) {+ if (field?.key) {+ properties[field.key] = schemaTypeFor(field);+ }+ }++ return [{ $schema: DRAFT_URI, type: "object", properties }];+}diff --git a/plugins/discourse-ai/discourse_workflows/nodes/ai_agent/v1.rb b/plugins/discourse-ai/discourse_workflows/nodes/ai_agent/v1.rbindex fdc2ae0549c..870ae291713 100644--- a/plugins/discourse-ai/discourse_workflows/nodes/ai_agent/v1.rb+++ b/plugins/discourse-ai/discourse_workflows/nodes/ai_agent/v1.rb@@ -7,6 +7,7 @@ if defined?(DiscourseWorkflows)class V1 < DiscourseWorkflows::NodeTypeRUN_ONCE_FOR_ALL_ITEMS = "runOnceForAllItems"RUN_ONCE_FOR_EACH_ITEM = "runOnceForEachItem"+ RESULT_PROPERTY = { "result" => { "type" => "string" } }.freezedescription(name: "action:ai_agent",@@ -19,6 +20,7 @@ if defined?(DiscourseWorkflows)available: -> { SiteSetting.discourse_ai_enabled },unavailable_reason_key: "discourse_workflows.node_unavailable.requires_ai",i18n_prefix: "discourse_ai.discourse_workflows",+ output_schema_resolver: "ai-agent",capabilities: {run_scope: {parameter: "mode",@@ -28,19 +30,7 @@ if defined?(DiscourseWorkflows)},},},- output_contracts: [- {- schema: {- "$schema" => DiscourseWorkflows::Schema::DRAFT_URI,- "type" => "object",- "properties" => {- "result" => {- "type" => "string",- },- },- },- },- ],+ output_contracts: [{ schema: DiscourseWorkflows::Schema.document(RESULT_PROPERTY) }],properties: {agent_id: {type: :integer,@@ -64,6 +54,7 @@ if defined?(DiscourseWorkflows)agent_name: "name",agent_force_default_llm: "force_default_llm",agent_resolved_llm_name: "resolved_llm_name",+ agent_response_format: "response_format",},},},@@ -73,6 +64,13 @@ if defined?(DiscourseWorkflows)hidden: true,},},+ agent_response_format: {+ type: :array,+ default: [],+ ui: {+ hidden: true,+ },+ },agent_force_default_llm: {type: :boolean,default: false,@@ -152,6 +150,14 @@ if defined?(DiscourseWorkflows)},)+ def self.output_schemas(configuration = {}, input_schemas: [])+ fields =+ Array((configuration || {}).deep_stringify_keys["agent_response_format"]).grep(Hash)+ properties = DiscourseAi::Agents::Bot.json_schema_properties(fields).deep_stringify_keys++ [DiscourseWorkflows::Schema.document(RESULT_PROPERTY.merge(properties))]+ end+def self.group_definition{ icon: "robot", label_key: "discourse_workflows.add_node.categories.ai", order: 40 }end@@ -170,16 +176,16 @@ if defined?(DiscourseWorkflows)::AiAgent.where(enabled: true).order(:name)- .pluck(:id, :name, :default_llm_id, :force_default_llm)+ .pluck(:id, :name, :default_llm_id, :force_default_llm, :response_format)site_default_llm_id = SiteSetting.ai_default_llm_model.presence&.to_i- llm_model_ids = agents.map { |_id, _name, default_llm_id, _force| default_llm_id }+ llm_model_ids = agents.map { |_id, _name, default_llm_id, *| default_llm_id }llm_model_ids << site_default_llm_idllm_models_by_id = ::LlmModel.where(id: llm_model_ids.compact.uniq).index_by(&:id)default_llm = llm_models_by_id[site_default_llm_id]- agents.map do |id, name, default_llm_id, force_default_llm|+ agents.map do |id, name, default_llm_id, force_default_llm, response_format|configured_llm = llm_models_by_id[default_llm_id]resolved_llm = force_default_llm ? configured_llm : configured_llm || default_llm{@@ -189,6 +195,7 @@ if defined?(DiscourseWorkflows)force_default_llm: force_default_llm,resolved_llm_id: resolved_llm&.id,resolved_llm_name: resolved_llm&.display_name,+ response_format: response_format || [],}endend@@ -223,7 +230,7 @@ if defined?(DiscourseWorkflows)runner(exec_ctx, item_index),)- wrap({ "result" => result }, paired_item: exec_ctx.paired_item_for(item))+ wrap(result, paired_item: exec_ctx.paired_item_for(item))end[items]@@ -235,7 +242,7 @@ if defined?(DiscourseWorkflows)result = run_agent(agent, item_config(exec_ctx, 0), exec_ctx.log, runner(exec_ctx, 0))wrap(- { "result" => result },+ result,paired_item: exec_ctx.input_items.map { |item| exec_ctx.paired_item_for(item) },)end@@ -401,17 +408,18 @@ if defined?(DiscourseWorkflows)execution_context: execution_context,)- result = collect_reply(agent[:bot], bot_context, log)+ result, structured = collect_reply(agent[:bot], bot_context, log)skipped =execution_context.upload_skips.map { |skip| error_text("upload_skipped", **skip) }raise_uploads_not_sent!(agent_record, skipped) if skipped.any?- result+ { "result" => result }.merge(structured_fields(structured, agent_record))enddef collect_reply(bot, bot_context, log)result = +""+ structured = niltool_calls = 0bot.reply(bot_context) do |partial, _, type|@@ -419,6 +427,7 @@ if defined?(DiscourseWorkflows)tool_calls += 1log.info("Tool call: #{partial}") if partial.is_a?(String)elsif type == :structured_output+ structured = partialresult = partial.to_s.dupelsif type.blank?result << partial@@ -428,7 +437,18 @@ if defined?(DiscourseWorkflows)log.info("Tool calls: #{tool_calls}") if tool_calls > 0log.info("Result length: #{result.size} chars")- result+ [result, structured]+ end++ def structured_fields(structured, agent_record)+ return {} if structured.nil?++ Array(agent_record.response_format)+ .grep(Hash)+ .to_h do |field|+ key = field["key"].to_s+ [key, structured.read_buffered_property(key.to_sym)]+ endendendenddiff --git a/plugins/discourse-ai/lib/agents/bot.rb b/plugins/discourse-ai/lib/agents/bot.rbindex 3cc70b5342d..f2c971e42db 100644--- a/plugins/discourse-ai/lib/agents/bot.rb+++ b/plugins/discourse-ai/lib/agents/bot.rb@@ -77,6 +77,25 @@ module DiscourseAinew(bot_user, agent, model)end+ def self.json_schema_properties(response_format)+ response_format+ .to_a+ .reduce({}) do |memo, format|+ next memo if !format.is_a?(Hash) || format["key"].blank?++ type_desc = { type: format["type"] }++ if format["type"] == "array"+ type_desc[:items] = { type: format["array_type"] || "string" }+ max_items = format["max_items"]+ type_desc[:maxItems] = max_items if max_items.is_a?(Integer) && max_items >= 0+ end++ memo[format["key"].to_sym] = type_desc+ memo+ end+ end+def initialize(bot_user, agent, model = nil)@bot_user = bot_user@agent = agent@@ -946,21 +965,7 @@ module DiscourseAienddef build_json_schema(response_format)- properties =- response_format- .to_a- .reduce({}) do |memo, format|- type_desc = { type: format["type"] }-- if format["type"] == "array"- type_desc[:items] = { type: format["array_type"] || "string" }- max_items = format["max_items"]- type_desc[:maxItems] = max_items if max_items.is_a?(Integer) && max_items >= 0- end-- memo[format["key"].to_sym] = type_desc- memo- end+ properties = self.class.json_schema_properties(response_format){type: "json_schema",diff --git a/plugins/discourse-ai/spec/lib/discourse_workflows/nodes/ai_agent_spec.rb b/plugins/discourse-ai/spec/lib/discourse_workflows/nodes/ai_agent_spec.rbindex 08979020af1..2f802b1458f 100644--- a/plugins/discourse-ai/spec/lib/discourse_workflows/nodes/ai_agent_spec.rb+++ b/plugins/discourse-ai/spec/lib/discourse_workflows/nodes/ai_agent_spec.rb@@ -69,6 +69,7 @@ RSpec.describe DiscourseWorkflows::Nodes::AiAgent::V1 doforce_default_llm: false,resolved_llm_id: llm_model.id,resolved_llm_name: llm_model.display_name,+ response_format: [],},)end@@ -366,6 +367,78 @@ RSpec.describe DiscourseWorkflows::Nodes::AiAgent::V1 doexpect(prompts).to eq([["Review the document", { upload_id: upload.id }]])end+ describe "structured output" do+ let(:response_format) do+ [+ { "key" => "verdict", "type" => "string" },+ { "key" => "confident", "type" => "boolean" },+ { "key" => "reasons", "type" => "array", "array_type" => "string" },+ ]+ end++ before do+ agent.update!(response_format: response_format)+ allow(bot).to receive(:reply) do |_bot_context, &block|+ structured =+ DiscourseAi::Completions::StructuredOutput.new(+ DiscourseAi::Agents::Bot.json_schema_properties(response_format),+ )+ structured << { verdict: "reject", confident: true, reasons: %w[nsfw] }.to_json+ structured.finish+ block.call(structured, nil, :structured_output)+ end+ end++ it "emits each response format field next to the raw result" do+ item = execute_node(configuration: { "agent_id" => agent.id, "prompt" => "Check" })++ expect(item).to eq(+ "result" => { verdict: "reject", confident: true, reasons: %w[nsfw] }.to_json,+ "verdict" => "reject",+ "confident" => true,+ "reasons" => %w[nsfw],+ )+ expect(item).to match_node_output_schema(+ described_class,+ configuration: {+ "agent_response_format" => response_format,+ },+ )+ end++ it "declares the picked agent's response format fields in the output schema" do+ schema = described_class.output_schemas({ "agent_response_format" => response_format }).first++ expect(schema["properties"]).to eq(+ "result" => {+ "type" => "string",+ },+ "verdict" => {+ "type" => "string",+ },+ "confident" => {+ "type" => "boolean",+ },+ "reasons" => {+ "type" => "array",+ "items" => {+ "type" => "string",+ },+ },+ )+ expect(+ described_class.output_schemas({ "agent_response_format" => "" }).first["properties"].keys,+ ).to eq(["result"])+ expect(+ described_class.output_schemas(+ { "agent_response_format" => [{ "type" => "string" }] },+ ).first[+ "properties"+ ].keys,+ ).to eq(["result"])+ end+ end+it "uses an empty string when the optional prompt is blank" doitems = execute_node_output(configuration: { "agent_id" => agent.id }).firstdiff --git a/plugins/discourse-ai/test/javascripts/unit/lib/ai-agent-output-schemas-test.js b/plugins/discourse-ai/test/javascripts/unit/lib/ai-agent-output-schemas-test.jsnew file mode 100644index 00000000000..2b4f7b4d271--- /dev/null+++ b/plugins/discourse-ai/test/javascripts/unit/lib/ai-agent-output-schemas-test.js@@ -0,0 +1,35 @@+import { module, test } from "qunit";+import aiAgentOutputSchemas from "discourse/plugins/discourse-ai/admin/lib/workflows/output-schemas/ai-agent";++module("Unit | lib | discourse-ai | AI agent output schemas", function () {+ test("declares the response format fields next to the raw result", function (assert) {+ const [schema] = aiAgentOutputSchemas({+ agent_response_format: [+ { key: "verdict", type: "string" },+ { key: "confident", type: "boolean" },+ { key: "reasons", type: "array", array_type: "string", max_items: 2 },+ ],+ });++ assert.deepEqual(schema.properties, {+ result: { type: "string" },+ verdict: { type: "string" },+ confident: { type: "boolean" },+ reasons: { type: "array", items: { type: "string" }, maxItems: 2 },+ });+ });++ test("declares only the raw result without usable fields", function (assert) {+ for (const configuration of [+ undefined,+ { agent_response_format: "" },+ { agent_response_format: [{ type: "string" }] },+ ]) {+ assert.deepEqual(+ Object.keys(aiAgentOutputSchemas(configuration)[0].properties),+ ["result"],+ `only result for ${JSON.stringify(configuration)}`+ );+ }+ });+});diff --git a/plugins/discourse-workflows/app/services/discourse_workflows/workflow/action/apply_patch.rb b/plugins/discourse-workflows/app/services/discourse_workflows/workflow/action/apply_patch.rbindex 330fdcbad59..fb1d611c4e9 100644--- a/plugins/discourse-workflows/app/services/discourse_workflows/workflow/action/apply_patch.rb+++ b/plugins/discourse-workflows/app/services/discourse_workflows/workflow/action/apply_patch.rb@@ -281,7 +281,13 @@ module DiscourseWorkflowsraise PatchError, "#{node_label(node)} references disabled AI agent #{agent.name.inspect}"end- parameters["agent_name"] ||= agent.name+ ai_agent_parameters(agent).each do |key, value|+ parameters[key] = value if parameters[key].blank?+ end+ end++ def ai_agent_parameters(agent)+ { "agent_name" => agent.name, "agent_response_format" => agent.response_format }enddef ai_agent_ref(value)@@ -367,7 +373,7 @@ module DiscourseWorkflowsnext if agent.blank?parameters["agent_id"] = agent.id- parameters["agent_name"] = agent.name+ parameters.merge!(ai_agent_parameters(agent))endenddiff --git a/plugins/discourse-workflows/spec/services/discourse_workflows/workflow/action/apply_patch_spec.rb b/plugins/discourse-workflows/spec/services/discourse_workflows/workflow/action/apply_patch_spec.rbindex 63e0bf85e94..64b75fd27b1 100644--- a/plugins/discourse-workflows/spec/services/discourse_workflows/workflow/action/apply_patch_spec.rb+++ b/plugins/discourse-workflows/spec/services/discourse_workflows/workflow/action/apply_patch_spec.rb@@ -199,6 +199,39 @@ RSpec.describe DiscourseWorkflows::Workflow::Action::ApplyPatch doexpect(node.dig("parameters", "agent_name")).to eq("Workflow summary agent")end+ it "mirrors an existing agent onto the node it is referenced from", :aggregate_failures do+ response_format = [{ "key" => "verdict", "type" => "string" }]+ agent = Fabricate(:ai_agent, name: "Workflow triage agent", response_format: response_format)++ result =+ described_class.call(+ workflow: workflow,+ operations: [+ {+ op: "add_node",+ client_id: "triage-post",+ node: {+ type: "action:ai_agent",+ name: "Triage post",+ parameters: {+ agent_id: agent.id,+ prompt: "={{ $json.post.raw }}",+ },+ },+ },+ ],+ persist: true,+ user: admin,+ )+ node = workflow.reload.nodes.find { |workflow_node| workflow_node["type"] == "action:ai_agent" }++ expect(result).to include(valid: true, errors: [])+ expect(node["parameters"]).to include(+ "agent_name" => "Workflow triage agent",+ "agent_response_format" => response_format,+ )+ end+it "rejects unknown AI agent references" doresult =described_class.call(