Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Expose an agent's structured output as workflow fields

Régis Hanol8018b367a53f
diff --git a/plugins/discourse-ai/admin/assets/javascripts/admin/lib/workflows/output-schemas/ai-agent.js b/plugins/discourse-ai/admin/assets/javascripts/admin/lib/workflows/output-schemas/ai-agent.js
new file mode 100644
index 00000000000..73d3943e04c
--- /dev/null
+++ b/plugins/discourse-ai/admin/assets/javascripts/admin/lib/workflows/output-schemas/ai-agent.js
@@ -0,0 +1,32 @@
+const DRAFT_URI = "https://json-schema.org/draft/2020-12/schema";
+
+function schemaTypeFor(field) {
+ if (field.type !== "array") {
+ return { type: field.type };
+ }
+
+ const schema = {
+ type: "array",
+ items: { type: field.array_type || "string" },
+ };
+ if (Number.isInteger(field.max_items) && field.max_items >= 0) {
+ schema.maxItems = field.max_items;
+ }
+
+ return schema;
+}
+
+export default function aiAgentOutputSchemas(configuration = {}) {
+ const properties = { result: { type: "string" } };
+ const fields = Array.isArray(configuration.agent_response_format)
+ ? configuration.agent_response_format
+ : [];
+
+ for (const field of fields) {
+ if (field?.key) {
+ properties[field.key] = schemaTypeFor(field);
+ }
+ }
+
+ return [{ $schema: DRAFT_URI, type: "object", properties }];
+}
diff --git a/plugins/discourse-ai/discourse_workflows/nodes/ai_agent/v1.rb b/plugins/discourse-ai/discourse_workflows/nodes/ai_agent/v1.rb
index fdc2ae0549c..870ae291713 100644
--- a/plugins/discourse-ai/discourse_workflows/nodes/ai_agent/v1.rb
+++ b/plugins/discourse-ai/discourse_workflows/nodes/ai_agent/v1.rb
@@ -7,6 +7,7 @@ if defined?(DiscourseWorkflows)
class V1 < DiscourseWorkflows::NodeType
RUN_ONCE_FOR_ALL_ITEMS = "runOnceForAllItems"
RUN_ONCE_FOR_EACH_ITEM = "runOnceForEachItem"
+ RESULT_PROPERTY = { "result" => { "type" => "string" } }.freeze
description(
name: "action:ai_agent",
@@ -19,6 +20,7 @@ if defined?(DiscourseWorkflows)
available: -> { SiteSetting.discourse_ai_enabled },
unavailable_reason_key: "discourse_workflows.node_unavailable.requires_ai",
i18n_prefix: "discourse_ai.discourse_workflows",
+ output_schema_resolver: "ai-agent",
capabilities: {
run_scope: {
parameter: "mode",
@@ -28,19 +30,7 @@ if defined?(DiscourseWorkflows)
},
},
},
- output_contracts: [
- {
- schema: {
- "$schema" => DiscourseWorkflows::Schema::DRAFT_URI,
- "type" => "object",
- "properties" => {
- "result" => {
- "type" => "string",
- },
- },
- },
- },
- ],
+ output_contracts: [{ schema: DiscourseWorkflows::Schema.document(RESULT_PROPERTY) }],
properties: {
agent_id: {
type: :integer,
@@ -64,6 +54,7 @@ if defined?(DiscourseWorkflows)
agent_name: "name",
agent_force_default_llm: "force_default_llm",
agent_resolved_llm_name: "resolved_llm_name",
+ agent_response_format: "response_format",
},
},
},
@@ -73,6 +64,13 @@ if defined?(DiscourseWorkflows)
hidden: true,
},
},
+ agent_response_format: {
+ type: :array,
+ default: [],
+ ui: {
+ hidden: true,
+ },
+ },
agent_force_default_llm: {
type: :boolean,
default: false,
@@ -152,6 +150,14 @@ if defined?(DiscourseWorkflows)
},
)
+ def self.output_schemas(configuration = {}, input_schemas: [])
+ fields =
+ Array((configuration || {}).deep_stringify_keys["agent_response_format"]).grep(Hash)
+ properties = DiscourseAi::Agents::Bot.json_schema_properties(fields).deep_stringify_keys
+
+ [DiscourseWorkflows::Schema.document(RESULT_PROPERTY.merge(properties))]
+ end
+
def self.group_definition
{ icon: "robot", label_key: "discourse_workflows.add_node.categories.ai", order: 40 }
end
@@ -170,16 +176,16 @@ if defined?(DiscourseWorkflows)
::AiAgent
.where(enabled: true)
.order(:name)
- .pluck(:id, :name, :default_llm_id, :force_default_llm)
+ .pluck(:id, :name, :default_llm_id, :force_default_llm, :response_format)
site_default_llm_id = SiteSetting.ai_default_llm_model.presence&.to_i
- llm_model_ids = agents.map { |_id, _name, default_llm_id, _force| default_llm_id }
+ llm_model_ids = agents.map { |_id, _name, default_llm_id, *| default_llm_id }
llm_model_ids << site_default_llm_id
llm_models_by_id = ::LlmModel.where(id: llm_model_ids.compact.uniq).index_by(&:id)
default_llm = llm_models_by_id[site_default_llm_id]
- agents.map do |id, name, default_llm_id, force_default_llm|
+ agents.map do |id, name, default_llm_id, force_default_llm, response_format|
configured_llm = llm_models_by_id[default_llm_id]
resolved_llm = force_default_llm ? configured_llm : configured_llm || default_llm
{
@@ -189,6 +195,7 @@ if defined?(DiscourseWorkflows)
force_default_llm: force_default_llm,
resolved_llm_id: resolved_llm&.id,
resolved_llm_name: resolved_llm&.display_name,
+ response_format: response_format || [],
}
end
end
@@ -223,7 +230,7 @@ if defined?(DiscourseWorkflows)
runner(exec_ctx, item_index),
)
- wrap({ "result" => result }, paired_item: exec_ctx.paired_item_for(item))
+ wrap(result, paired_item: exec_ctx.paired_item_for(item))
end
[items]
@@ -235,7 +242,7 @@ if defined?(DiscourseWorkflows)
result = run_agent(agent, item_config(exec_ctx, 0), exec_ctx.log, runner(exec_ctx, 0))
wrap(
- { "result" => result },
+ result,
paired_item: exec_ctx.input_items.map { |item| exec_ctx.paired_item_for(item) },
)
end
@@ -401,17 +408,18 @@ if defined?(DiscourseWorkflows)
execution_context: execution_context,
)
- result = collect_reply(agent[:bot], bot_context, log)
+ result, structured = collect_reply(agent[:bot], bot_context, log)
skipped =
execution_context.upload_skips.map { |skip| error_text("upload_skipped", **skip) }
raise_uploads_not_sent!(agent_record, skipped) if skipped.any?
- result
+ { "result" => result }.merge(structured_fields(structured, agent_record))
end
def collect_reply(bot, bot_context, log)
result = +""
+ structured = nil
tool_calls = 0
bot.reply(bot_context) do |partial, _, type|
@@ -419,6 +427,7 @@ if defined?(DiscourseWorkflows)
tool_calls += 1
log.info("Tool call: #{partial}") if partial.is_a?(String)
elsif type == :structured_output
+ structured = partial
result = partial.to_s.dup
elsif type.blank?
result << partial
@@ -428,7 +437,18 @@ if defined?(DiscourseWorkflows)
log.info("Tool calls: #{tool_calls}") if tool_calls > 0
log.info("Result length: #{result.size} chars")
- result
+ [result, structured]
+ end
+
+ def structured_fields(structured, agent_record)
+ return {} if structured.nil?
+
+ Array(agent_record.response_format)
+ .grep(Hash)
+ .to_h do |field|
+ key = field["key"].to_s
+ [key, structured.read_buffered_property(key.to_sym)]
+ end
end
end
end
diff --git a/plugins/discourse-ai/lib/agents/bot.rb b/plugins/discourse-ai/lib/agents/bot.rb
index 3cc70b5342d..f2c971e42db 100644
--- a/plugins/discourse-ai/lib/agents/bot.rb
+++ b/plugins/discourse-ai/lib/agents/bot.rb
@@ -77,6 +77,25 @@ module DiscourseAi
new(bot_user, agent, model)
end
+ def self.json_schema_properties(response_format)
+ response_format
+ .to_a
+ .reduce({}) do |memo, format|
+ next memo if !format.is_a?(Hash) || format["key"].blank?
+
+ type_desc = { type: format["type"] }
+
+ if format["type"] == "array"
+ type_desc[:items] = { type: format["array_type"] || "string" }
+ max_items = format["max_items"]
+ type_desc[:maxItems] = max_items if max_items.is_a?(Integer) && max_items >= 0
+ end
+
+ memo[format["key"].to_sym] = type_desc
+ memo
+ end
+ end
+
def initialize(bot_user, agent, model = nil)
@bot_user = bot_user
@agent = agent
@@ -946,21 +965,7 @@ module DiscourseAi
end
def build_json_schema(response_format)
- properties =
- response_format
- .to_a
- .reduce({}) do |memo, format|
- type_desc = { type: format["type"] }
-
- if format["type"] == "array"
- type_desc[:items] = { type: format["array_type"] || "string" }
- max_items = format["max_items"]
- type_desc[:maxItems] = max_items if max_items.is_a?(Integer) && max_items >= 0
- end
-
- memo[format["key"].to_sym] = type_desc
- memo
- end
+ properties = self.class.json_schema_properties(response_format)
{
type: "json_schema",
diff --git a/plugins/discourse-ai/spec/lib/discourse_workflows/nodes/ai_agent_spec.rb b/plugins/discourse-ai/spec/lib/discourse_workflows/nodes/ai_agent_spec.rb
index 08979020af1..2f802b1458f 100644
--- a/plugins/discourse-ai/spec/lib/discourse_workflows/nodes/ai_agent_spec.rb
+++ b/plugins/discourse-ai/spec/lib/discourse_workflows/nodes/ai_agent_spec.rb
@@ -69,6 +69,7 @@ RSpec.describe DiscourseWorkflows::Nodes::AiAgent::V1 do
force_default_llm: false,
resolved_llm_id: llm_model.id,
resolved_llm_name: llm_model.display_name,
+ response_format: [],
},
)
end
@@ -366,6 +367,78 @@ RSpec.describe DiscourseWorkflows::Nodes::AiAgent::V1 do
expect(prompts).to eq([["Review the document", { upload_id: upload.id }]])
end
+ describe "structured output" do
+ let(:response_format) do
+ [
+ { "key" => "verdict", "type" => "string" },
+ { "key" => "confident", "type" => "boolean" },
+ { "key" => "reasons", "type" => "array", "array_type" => "string" },
+ ]
+ end
+
+ before do
+ agent.update!(response_format: response_format)
+ allow(bot).to receive(:reply) do |_bot_context, &block|
+ structured =
+ DiscourseAi::Completions::StructuredOutput.new(
+ DiscourseAi::Agents::Bot.json_schema_properties(response_format),
+ )
+ structured << { verdict: "reject", confident: true, reasons: %w[nsfw] }.to_json
+ structured.finish
+ block.call(structured, nil, :structured_output)
+ end
+ end
+
+ it "emits each response format field next to the raw result" do
+ item = execute_node(configuration: { "agent_id" => agent.id, "prompt" => "Check" })
+
+ expect(item).to eq(
+ "result" => { verdict: "reject", confident: true, reasons: %w[nsfw] }.to_json,
+ "verdict" => "reject",
+ "confident" => true,
+ "reasons" => %w[nsfw],
+ )
+ expect(item).to match_node_output_schema(
+ described_class,
+ configuration: {
+ "agent_response_format" => response_format,
+ },
+ )
+ end
+
+ it "declares the picked agent's response format fields in the output schema" do
+ schema = described_class.output_schemas({ "agent_response_format" => response_format }).first
+
+ expect(schema["properties"]).to eq(
+ "result" => {
+ "type" => "string",
+ },
+ "verdict" => {
+ "type" => "string",
+ },
+ "confident" => {
+ "type" => "boolean",
+ },
+ "reasons" => {
+ "type" => "array",
+ "items" => {
+ "type" => "string",
+ },
+ },
+ )
+ expect(
+ described_class.output_schemas({ "agent_response_format" => "" }).first["properties"].keys,
+ ).to eq(["result"])
+ expect(
+ described_class.output_schemas(
+ { "agent_response_format" => [{ "type" => "string" }] },
+ ).first[
+ "properties"
+ ].keys,
+ ).to eq(["result"])
+ end
+ end
+
it "uses an empty string when the optional prompt is blank" do
items = execute_node_output(configuration: { "agent_id" => agent.id }).first
diff --git a/plugins/discourse-ai/test/javascripts/unit/lib/ai-agent-output-schemas-test.js b/plugins/discourse-ai/test/javascripts/unit/lib/ai-agent-output-schemas-test.js
new file mode 100644
index 00000000000..2b4f7b4d271
--- /dev/null
+++ b/plugins/discourse-ai/test/javascripts/unit/lib/ai-agent-output-schemas-test.js
@@ -0,0 +1,35 @@
+import { module, test } from "qunit";
+import aiAgentOutputSchemas from "discourse/plugins/discourse-ai/admin/lib/workflows/output-schemas/ai-agent";
+
+module("Unit | lib | discourse-ai | AI agent output schemas", function () {
+ test("declares the response format fields next to the raw result", function (assert) {
+ const [schema] = aiAgentOutputSchemas({
+ agent_response_format: [
+ { key: "verdict", type: "string" },
+ { key: "confident", type: "boolean" },
+ { key: "reasons", type: "array", array_type: "string", max_items: 2 },
+ ],
+ });
+
+ assert.deepEqual(schema.properties, {
+ result: { type: "string" },
+ verdict: { type: "string" },
+ confident: { type: "boolean" },
+ reasons: { type: "array", items: { type: "string" }, maxItems: 2 },
+ });
+ });
+
+ test("declares only the raw result without usable fields", function (assert) {
+ for (const configuration of [
+ undefined,
+ { agent_response_format: "" },
+ { agent_response_format: [{ type: "string" }] },
+ ]) {
+ assert.deepEqual(
+ Object.keys(aiAgentOutputSchemas(configuration)[0].properties),
+ ["result"],
+ `only result for ${JSON.stringify(configuration)}`
+ );
+ }
+ });
+});
diff --git a/plugins/discourse-workflows/app/services/discourse_workflows/workflow/action/apply_patch.rb b/plugins/discourse-workflows/app/services/discourse_workflows/workflow/action/apply_patch.rb
index 330fdcbad59..fb1d611c4e9 100644
--- a/plugins/discourse-workflows/app/services/discourse_workflows/workflow/action/apply_patch.rb
+++ b/plugins/discourse-workflows/app/services/discourse_workflows/workflow/action/apply_patch.rb
@@ -281,7 +281,13 @@ module DiscourseWorkflows
raise PatchError, "#{node_label(node)} references disabled AI agent #{agent.name.inspect}"
end
- parameters["agent_name"] ||= agent.name
+ ai_agent_parameters(agent).each do |key, value|
+ parameters[key] = value if parameters[key].blank?
+ end
+ end
+
+ def ai_agent_parameters(agent)
+ { "agent_name" => agent.name, "agent_response_format" => agent.response_format }
end
def ai_agent_ref(value)
@@ -367,7 +373,7 @@ module DiscourseWorkflows
next if agent.blank?
parameters["agent_id"] = agent.id
- parameters["agent_name"] = agent.name
+ parameters.merge!(ai_agent_parameters(agent))
end
end
diff --git a/plugins/discourse-workflows/spec/services/discourse_workflows/workflow/action/apply_patch_spec.rb b/plugins/discourse-workflows/spec/services/discourse_workflows/workflow/action/apply_patch_spec.rb
index 63e0bf85e94..64b75fd27b1 100644
--- a/plugins/discourse-workflows/spec/services/discourse_workflows/workflow/action/apply_patch_spec.rb
+++ b/plugins/discourse-workflows/spec/services/discourse_workflows/workflow/action/apply_patch_spec.rb
@@ -199,6 +199,39 @@ RSpec.describe DiscourseWorkflows::Workflow::Action::ApplyPatch do
expect(node.dig("parameters", "agent_name")).to eq("Workflow summary agent")
end
+ it "mirrors an existing agent onto the node it is referenced from", :aggregate_failures do
+ response_format = [{ "key" => "verdict", "type" => "string" }]
+ agent = Fabricate(:ai_agent, name: "Workflow triage agent", response_format: response_format)
+
+ result =
+ described_class.call(
+ workflow: workflow,
+ operations: [
+ {
+ op: "add_node",
+ client_id: "triage-post",
+ node: {
+ type: "action:ai_agent",
+ name: "Triage post",
+ parameters: {
+ agent_id: agent.id,
+ prompt: "={{ $json.post.raw }}",
+ },
+ },
+ },
+ ],
+ persist: true,
+ user: admin,
+ )
+ node = workflow.reload.nodes.find { |workflow_node| workflow_node["type"] == "action:ai_agent" }
+
+ expect(result).to include(valid: true, errors: [])
+ expect(node["parameters"]).to include(
+ "agent_name" => "Workflow triage agent",
+ "agent_response_format" => response_format,
+ )
+ end
+
it "rejects unknown AI agent references" do
result =
described_class.call(