Skip to content

A platform for community discussion. Free, open, simple.

FIX: Data explorer agent reliability for schema and plurality (#40152)

Natalie Tayfbcbdc46d816
diff --git a/plugins/discourse-ai/evals/lib/runners/data_explorer.rb b/plugins/discourse-ai/evals/lib/runners/data_explorer.rb
index ea978c171c3..a625da11fd7 100644
--- a/plugins/discourse-ai/evals/lib/runners/data_explorer.rb
+++ b/plugins/discourse-ai/evals/lib/runners/data_explorer.rb
@@ -6,6 +6,8 @@ module DiscourseAi
module Evals
module Runners
class DataExplorer < Base
+ STRUCTURED_KEYS = %i[name description sql].freeze
+
def self.can_handle?(full_feature_name)
full_feature_name&.start_with?("data_explorer:")
end
@@ -24,15 +26,35 @@ module DiscourseAi
)
bot = DiscourseAi::Agents::Bot.as(user, agent: agent, model: llm)
- sql =
- capture_structured_response(
- bot,
- context,
- schema_key: "sql",
- execution_context: execution_context,
- )
+ captured = capture_structured_fields(bot, context, execution_context:)
+
+ sql = captured[:sql].to_s.strip
+ metadata = {
+ feature: feature_name,
+ name: captured[:name].to_s.strip,
+ description: captured[:description].to_s.strip,
+ }
+
+ wrap_result(sql, metadata)
+ end
+
+ private
+
+ def capture_structured_fields(bot, context, execution_context:)
+ buffers = STRUCTURED_KEYS.index_with { +"" }
+
+ bot.reply(context, execution_context:) do |partial, _, type|
+ if type == :structured_output
+ STRUCTURED_KEYS.each do |key|
+ chunk = partial.read_buffered_property(key)
+ buffers[key] << chunk.to_s if chunk
+ end
+ elsif type.blank?
+ buffers[:sql] << partial.to_s
+ end
+ end
- wrap_result(sql.strip, { feature: feature_name })
+ buffers
end
end
end
diff --git a/plugins/discourse-ai/evals/lib/workbench.rb b/plugins/discourse-ai/evals/lib/workbench.rb
index ce6069cb855..708042b9500 100644
--- a/plugins/discourse-ai/evals/lib/workbench.rb
+++ b/plugins/discourse-ai/evals/lib/workbench.rb
@@ -296,7 +296,13 @@ module DiscourseAi
metadata = entry.is_a?(Hash) ? entry[:metadata] : nil
classification =
- classify_result(eval_case, raw_value, skip_judge: skip_judge, execution_context:)
+ classify_result(
+ eval_case,
+ raw_value,
+ metadata: metadata,
+ skip_judge: skip_judge,
+ execution_context: execution_context,
+ )
classification[:metadata] = metadata if metadata.present?
@@ -467,7 +473,13 @@ module DiscourseAi
DiscourseAi::Evals::Judge.new(eval_case: eval_case, judge_llm: judge_llm)
end
- def classify_result(eval_case, result, skip_judge: false, execution_context: nil)
+ def classify_result(
+ eval_case,
+ result,
+ metadata: nil,
+ skip_judge: false,
+ execution_context: nil
+ )
if eval_case.expected_output
if result == eval_case.expected_output
{ result: :pass }
@@ -487,12 +499,22 @@ module DiscourseAi
elsif eval_case.expected_tool_call
classify_tool_call(eval_case.expected_tool_call, result)
elsif eval_case.judge && !skip_judge
- judge_result(eval_case, result, execution_context:)
+ judge_result(eval_case, judge_input(result, metadata), execution_context:)
else
{ result: :pass }
end
end
+ def judge_input(result, metadata)
+ return result if metadata.blank? || !metadata.is_a?(Hash)
+
+ keys = %i[name description]
+ extras = metadata.slice(*keys).compact
+ return result if extras.empty?
+
+ { result: result }.merge(extras)
+ end
+
def classify_tool_call(expected_tool_call, result)
tool_call = result
tool_call = result.find { |r| r.is_a?(DiscourseAi::Completions::ToolCall) } if result.is_a?(
diff --git a/plugins/discourse-ai/lib/agents/tools/db_schema.rb b/plugins/discourse-ai/lib/agents/tools/db_schema.rb
index 30ee491955f..c5356c5fa25 100644
--- a/plugins/discourse-ai/lib/agents/tools/db_schema.rb
+++ b/plugins/discourse-ai/lib/agents/tools/db_schema.rb
@@ -29,26 +29,42 @@ module DiscourseAi
end
def invoke
- tables_arr = tables.split(",").map(&:strip)
+ tables_arr = tables.split(",").map(&:strip).reject(&:empty?)
+ return { schema_info: "", tables: tables } if tables_arr.empty?
- table_info = {}
- DB
- .query(<<~SQL, tables_arr)
- select table_name, column_name, data_type from information_schema.columns
+ rows = DB.query(<<~SQL, tables_arr)
+ select table_name, column_name, data_type, is_nullable, character_maximum_length
+ from information_schema.columns
where table_schema = 'public'
and table_name in (?)
- order by table_name
+ order by table_name, ordinal_position
SQL
- .each do |row|
- (table_info[row.table_name] ||= []) << "#{row.column_name} #{row.data_type}"
+
+ by_table = rows.group_by(&:table_name)
+ missing_tables = tables_arr - by_table.keys
+
+ sections =
+ tables_arr.filter_map do |table_name|
+ cols = by_table[table_name]
+ next unless cols
+
+ lines =
+ cols.map do |row|
+ attrs = [simplify_type(row.data_type, row.character_maximum_length)]
+ attrs << "PK" if row.column_name == "id"
+ if (fkey = fkey_target(table_name, row.column_name))
+ attrs << "FK → #{fkey}"
+ end
+ attrs << "null" if row.is_nullable == "YES"
+ " #{row.column_name} #{attrs.join(", ")}"
+ end
+
+ "TABLE #{table_name}\n#{lines.join("\n")}"
end
- schema_info =
- table_info
- .map { |table_name, columns| "#{table_name}(#{columns.join(",")})" }
- .join("\n")
+ sections << "TABLES NOT FOUND: #{missing_tables.join(", ")}" if missing_tables.any?
- { schema_info: schema_info, tables: tables }
+ { schema_info: sections.join("\n\n"), tables: tables }
end
protected
@@ -56,6 +72,34 @@ module DiscourseAi
def description_args
{ tables: tables }
end
+
+ private
+
+ def simplify_type(type, max_len)
+ case type
+ when "character varying"
+ max_len ? "varchar(#{max_len})" : "varchar"
+ when "timestamp without time zone"
+ "timestamp"
+ when "double precision"
+ "double"
+ else
+ type
+ end
+ end
+
+ def fkey_target(table_name, column_name)
+ return nil if column_name == "id"
+ return nil unless column_name.end_with?("_id")
+
+ if defined?(DiscourseDataExplorer::DataExplorer) &&
+ DiscourseDataExplorer::DataExplorer.respond_to?(:fkey_info)
+ target = DiscourseDataExplorer::DataExplorer.fkey_info(table_name, column_name)
+ return target.to_s if target
+ end
+
+ nil
+ end
end
end
end
diff --git a/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rb b/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rb
new file mode 100644
index 00000000000..f34fa902ceb
--- /dev/null
+++ b/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "../../../evals/lib/runners/data_explorer"
+require_relative "../support/runner_helper"
+
+RSpec.describe DiscourseAi::Evals::Runners::DataExplorer do
+ fab!(:llm, :fake_model)
+ let(:execution_context) { DiscourseAi::Completions::ExecutionContext.new }
+ let(:runner) { described_class.new("query_generation") }
+ let(:eval_case) { OpenStruct.new(args: { input: "Show me signups by month" }) }
+
+ def stub_structured_response(name:, description:, sql:)
+ payload = { name: name, description: description, sql: sql }
+ structured =
+ instance_double(DiscourseAi::Completions::StructuredOutput).tap do |double|
+ allow(double).to receive(:read_buffered_property) { |key| payload[key] }
+ end
+
+ stub_runner_bot { |blk| blk.call(structured, nil, :structured_output) }
+ end
+
+ describe "#run" do
+ it "captures name, description, and sql separately" do
+ stub_structured_response(
+ name: "Signups by month",
+ description: "Counts user signups grouped by month.",
+ sql: "SELECT date_trunc('month', created_at) AS month FROM users GROUP BY month",
+ )
+
+ result = runner.run(eval_case, llm, execution_context: execution_context)
+
+ expect(result[:raw]).to include("SELECT date_trunc")
+ expect(result[:metadata][:name]).to eq("Signups by month")
+ expect(result[:metadata][:description]).to eq("Counts user signups grouped by month.")
+ expect(result[:metadata][:feature]).to eq("query_generation")
+ end
+ end
+end
diff --git a/plugins/discourse-ai/spec/evals/workbench_spec.rb b/plugins/discourse-ai/spec/evals/workbench_spec.rb
index 9b366bdf612..c41636cb2cf 100644
--- a/plugins/discourse-ai/spec/evals/workbench_spec.rb
+++ b/plugins/discourse-ai/spec/evals/workbench_spec.rb
@@ -305,4 +305,18 @@ RSpec.describe DiscourseAi::Evals::Workbench do
expect(result[:message]).to include("LLM Rating below threshold")
end
end
+
+ describe "#judge_input" do
+ it "forwards `name` and `description` from metadata so the judge sees them" do
+ input =
+ workbench.send(:judge_input, "SELECT 1", { name: "My Query", description: "Counts users" })
+
+ expect(input).to eq(result: "SELECT 1", name: "My Query", description: "Counts users")
+ end
+
+ it "returns the raw result when there is no metadata" do
+ expect(workbench.send(:judge_input, "SELECT 1", nil)).to eq("SELECT 1")
+ expect(workbench.send(:judge_input, "SELECT 1", {})).to eq("SELECT 1")
+ end
+ end
end
diff --git a/plugins/discourse-ai/spec/lib/agents/tools/db_schema_spec.rb b/plugins/discourse-ai/spec/lib/agents/tools/db_schema_spec.rb
index 91c8965b2f4..e271dfb469a 100644
--- a/plugins/discourse-ai/spec/lib/agents/tools/db_schema_spec.rb
+++ b/plugins/discourse-ai/spec/lib/agents/tools/db_schema_spec.rb
@@ -1,4 +1,4 @@
-#frozen_string_literal: true
+# frozen_string_literal: true
RSpec.describe DiscourseAi::Agents::Tools::DbSchema do
fab!(:llm_model)
@@ -10,16 +10,60 @@ RSpec.describe DiscourseAi::Agents::Tools::DbSchema do
SiteSetting.ai_bot_enabled = true
end
- describe "#process" do
- it "returns rich schema for tables" do
- result = described_class.new({ tables: "posts,topics" }, bot_user: bot_user, llm: llm).invoke
+ def invoke(tables)
+ described_class.new({ tables: tables }, bot_user: bot_user, llm: llm).invoke
+ end
+
+ describe "#invoke" do
+ it "returns one column per line with table headers" do
+ info = invoke("posts,topics")[:schema_info]
+
+ expect(info).to match(/^TABLE posts$/)
+ expect(info).to match(/^TABLE topics$/)
+ expect(info).to match(/^ id integer, PK$/)
+ end
+
+ it "marks foreign-key columns using Discourse's curated map" do
+ info = invoke("topic_tags,topics")[:schema_info]
+
+ expect(info).to match(/^ topic_id integer, FK → topics/)
+ expect(info).to match(/^ user_id integer, FK → users/)
+ expect(info).to match(/^ last_post_user_id integer, FK → users/) # via *_user_id rule
+ end
+
+ it "does not invent FK targets for non-FK _id columns" do
+ info = invoke("topics")[:schema_info]
+
+ # Previous heuristic produced misleading targets like "FK → externals" / "FK → featured_user1s".
+ # Now we only mark FKs the Discourse map actually knows about.
+ expect(info).not_to include("FK → externals")
+ expect(info).not_to include("FK → featured_user1s")
+ end
+
+ it "simplifies long postgres type names" do
+ info = invoke("topics")[:schema_info]
+
+ expect(info).to include("varchar")
+ expect(info).to include("timestamp")
+ expect(info).not_to include("character varying")
+ expect(info).not_to include("timestamp without time zone")
+ end
+
+ it "marks nullable columns" do
+ info = invoke("topics")[:schema_info]
- expect(result[:schema_info]).to include("raw text")
- expect(result[:schema_info]).to include("views integer")
- expect(result[:schema_info]).to include("posts")
- expect(result[:schema_info]).to include("topics")
+ expect(info).to match(/^ deleted_at timestamp, null$/)
+ end
+
+ it "reports missing tables instead of silently dropping them" do
+ info = invoke("topics,not_a_real_table")[:schema_info]
+
+ expect(info).to include("TABLE topics")
+ expect(info).to include("TABLES NOT FOUND: not_a_real_table")
+ end
- expect(result[:tables]).to eq("posts,topics")
+ it "returns the requested tables string" do
+ expect(invoke("posts,topics")[:tables]).to eq("posts,topics")
end
end
end
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/ai_query_generator.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/ai_query_generator.rb
index 5377dbac65f..467dc28e29d 100644
--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/ai_query_generator.rb
+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/ai_query_generator.rb
@@ -63,23 +63,30 @@ module DiscourseDataExplorer
-- int :num = 3
-- bigint :big = 12345678912345
-- boolean :flag
- -- null boolean :opt_flag = #null
+ -- null boolean :opt_flag
-- string :name = default value
- -- date :date = 14 jul 2015
- -- time :time = 5:02 pm
- -- datetime :datetime = 14 jul 2015 5:02 pm
+ -- date :start_date = #{(Date.today - 30).strftime("%Y-%m-%d")}
+ -- date :end_date = #{Date.today.strftime("%Y-%m-%d")}
+ -- time :time = 17:02
+ -- datetime :datetime = #{Date.today.strftime("%Y-%m-%d")} 17:02
-- double :ratio = 3.1415
- -- user_id :user = system
+ -- user_id :user
-- post_id :post
-- topic_id :topic
-- int_list :ids = 1,2,3
-- string_list :names = a,b,c
- -- category_id :category = meta
- -- group_id :group = admins
- -- user_list :users = system,discobot
+ -- category_id :category
+ -- group_id :group
+ -- user_list :users
-- current_user_id :me
- Prefix with "null" for optional params.
- Today is #{Date.today.strftime("%Y-%m-%d")}. Date param defaults MUST be real dates like #{Date.today.strftime("%Y-%m-%d")}, NEVER natural language like "today" or "3 months ago".
+
+ ### Parameter rules
+ - For category_id, group_id, user_id, user_list, post_id, topic_id, badge_id: OMIT the default value entirely (or use `null` prefix).
+ - Optional params: prefix the type with "null" and OMIT the default. The "null" prefix is the optional marker; do NOT use `= #null` as a default value (e.g. write `-- null category_id :category`, NOT `-- category_id :category = #null`).
+ - Date param defaults MUST be real ISO dates in YYYY-MM-DD form (today is #{Date.today.strftime("%Y-%m-%d")}). NEVER use natural-language defaults like "today", "yesterday", "3 months ago", or "14 jul 2015". For a date range, name params `:start_date` and `:end_date`.
+ - Plural nouns → list-style param TYPES. Apply this rule independently to EACH plural noun in the prompt; don't skip one. Only `int_list`, `string_list`, `user_list`, `group_list` accept multiple values — single-value types like `category_id`, `user_id`, `topic_id` do NOT. Example: "Get topics for selected categories and tags." → `-- null int_list :category_ids` AND `-- null string_list :tag_names` (BOTH list types).
+ - `ANY(:param)` and `:param IN (...)` REQUIRE :param to be a list-type (`int_list`, `string_list`, etc.). With single-value types (`category_id`, `user_id`, etc.), use `column = :param` instead. Mismatching the param type with array usage is a runtime error ("op ANY/ALL (array) requires array on right side").
+ - First-person prompts ("my posts", "queries about me", "topics I've replied to") MUST use `current_user_id` — this auto-injects the requester's user id at run time. Do NOT use `user_id :user = system` for "my" queries.
## Discourse domain knowledge
- topics.archetype: 'regular' = topics, 'private_message' = PMs. Default to 'regular'.