A platform for community discussion. Free, open, simple.
FIX: Data explorer agent reliability for schema and plurality (#40152)
Natalie Tayfbcbdc46d816
diff --git a/plugins/discourse-ai/evals/lib/runners/data_explorer.rb b/plugins/discourse-ai/evals/lib/runners/data_explorer.rbindex ea978c171c3..a625da11fd7 100644--- a/plugins/discourse-ai/evals/lib/runners/data_explorer.rb+++ b/plugins/discourse-ai/evals/lib/runners/data_explorer.rb@@ -6,6 +6,8 @@ module DiscourseAimodule Evalsmodule Runnersclass DataExplorer < Base+ STRUCTURED_KEYS = %i[name description sql].freeze+def self.can_handle?(full_feature_name)full_feature_name&.start_with?("data_explorer:")end@@ -24,15 +26,35 @@ module DiscourseAi)bot = DiscourseAi::Agents::Bot.as(user, agent: agent, model: llm)- sql =- capture_structured_response(- bot,- context,- schema_key: "sql",- execution_context: execution_context,- )+ captured = capture_structured_fields(bot, context, execution_context:)++ sql = captured[:sql].to_s.strip+ metadata = {+ feature: feature_name,+ name: captured[:name].to_s.strip,+ description: captured[:description].to_s.strip,+ }++ wrap_result(sql, metadata)+ end++ private++ def capture_structured_fields(bot, context, execution_context:)+ buffers = STRUCTURED_KEYS.index_with { +"" }++ bot.reply(context, execution_context:) do |partial, _, type|+ if type == :structured_output+ STRUCTURED_KEYS.each do |key|+ chunk = partial.read_buffered_property(key)+ buffers[key] << chunk.to_s if chunk+ end+ elsif type.blank?+ buffers[:sql] << partial.to_s+ end+ end- wrap_result(sql.strip, { feature: feature_name })+ buffersendendenddiff --git a/plugins/discourse-ai/evals/lib/workbench.rb b/plugins/discourse-ai/evals/lib/workbench.rbindex ce6069cb855..708042b9500 100644--- a/plugins/discourse-ai/evals/lib/workbench.rb+++ b/plugins/discourse-ai/evals/lib/workbench.rb@@ -296,7 +296,13 @@ module DiscourseAimetadata = entry.is_a?(Hash) ? entry[:metadata] : nilclassification =- classify_result(eval_case, raw_value, skip_judge: skip_judge, execution_context:)+ classify_result(+ eval_case,+ raw_value,+ metadata: metadata,+ skip_judge: skip_judge,+ execution_context: execution_context,+ )classification[:metadata] = metadata if metadata.present?@@ -467,7 +473,13 @@ module DiscourseAiDiscourseAi::Evals::Judge.new(eval_case: eval_case, judge_llm: judge_llm)end- def classify_result(eval_case, result, skip_judge: false, execution_context: nil)+ def classify_result(+ eval_case,+ result,+ metadata: nil,+ skip_judge: false,+ execution_context: nil+ )if eval_case.expected_outputif result == eval_case.expected_output{ result: :pass }@@ -487,12 +499,22 @@ module DiscourseAielsif eval_case.expected_tool_callclassify_tool_call(eval_case.expected_tool_call, result)elsif eval_case.judge && !skip_judge- judge_result(eval_case, result, execution_context:)+ judge_result(eval_case, judge_input(result, metadata), execution_context:)else{ result: :pass }endend+ def judge_input(result, metadata)+ return result if metadata.blank? || !metadata.is_a?(Hash)++ keys = %i[name description]+ extras = metadata.slice(*keys).compact+ return result if extras.empty?++ { result: result }.merge(extras)+ end+def classify_tool_call(expected_tool_call, result)tool_call = resulttool_call = result.find { |r| r.is_a?(DiscourseAi::Completions::ToolCall) } if result.is_a?(diff --git a/plugins/discourse-ai/lib/agents/tools/db_schema.rb b/plugins/discourse-ai/lib/agents/tools/db_schema.rbindex 30ee491955f..c5356c5fa25 100644--- a/plugins/discourse-ai/lib/agents/tools/db_schema.rb+++ b/plugins/discourse-ai/lib/agents/tools/db_schema.rb@@ -29,26 +29,42 @@ module DiscourseAienddef invoke- tables_arr = tables.split(",").map(&:strip)+ tables_arr = tables.split(",").map(&:strip).reject(&:empty?)+ return { schema_info: "", tables: tables } if tables_arr.empty?- table_info = {}- DB- .query(<<~SQL, tables_arr)- select table_name, column_name, data_type from information_schema.columns+ rows = DB.query(<<~SQL, tables_arr)+ select table_name, column_name, data_type, is_nullable, character_maximum_length+ from information_schema.columnswhere table_schema = 'public'and table_name in (?)- order by table_name+ order by table_name, ordinal_positionSQL- .each do |row|- (table_info[row.table_name] ||= []) << "#{row.column_name} #{row.data_type}"++ by_table = rows.group_by(&:table_name)+ missing_tables = tables_arr - by_table.keys++ sections =+ tables_arr.filter_map do |table_name|+ cols = by_table[table_name]+ next unless cols++ lines =+ cols.map do |row|+ attrs = [simplify_type(row.data_type, row.character_maximum_length)]+ attrs << "PK" if row.column_name == "id"+ if (fkey = fkey_target(table_name, row.column_name))+ attrs << "FK → #{fkey}"+ end+ attrs << "null" if row.is_nullable == "YES"+ " #{row.column_name} #{attrs.join(", ")}"+ end++ "TABLE #{table_name}\n#{lines.join("\n")}"end- schema_info =- table_info- .map { |table_name, columns| "#{table_name}(#{columns.join(",")})" }- .join("\n")+ sections << "TABLES NOT FOUND: #{missing_tables.join(", ")}" if missing_tables.any?- { schema_info: schema_info, tables: tables }+ { schema_info: sections.join("\n\n"), tables: tables }endprotected@@ -56,6 +72,34 @@ module DiscourseAidef description_args{ tables: tables }end++ private++ def simplify_type(type, max_len)+ case type+ when "character varying"+ max_len ? "varchar(#{max_len})" : "varchar"+ when "timestamp without time zone"+ "timestamp"+ when "double precision"+ "double"+ else+ type+ end+ end++ def fkey_target(table_name, column_name)+ return nil if column_name == "id"+ return nil unless column_name.end_with?("_id")++ if defined?(DiscourseDataExplorer::DataExplorer) &&+ DiscourseDataExplorer::DataExplorer.respond_to?(:fkey_info)+ target = DiscourseDataExplorer::DataExplorer.fkey_info(table_name, column_name)+ return target.to_s if target+ end++ nil+ endendendenddiff --git a/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rb b/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rbnew file mode 100644index 00000000000..f34fa902ceb--- /dev/null+++ b/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rb@@ -0,0 +1,38 @@+# frozen_string_literal: true++require_relative "../../../evals/lib/runners/data_explorer"+require_relative "../support/runner_helper"++RSpec.describe DiscourseAi::Evals::Runners::DataExplorer do+ fab!(:llm, :fake_model)+ let(:execution_context) { DiscourseAi::Completions::ExecutionContext.new }+ let(:runner) { described_class.new("query_generation") }+ let(:eval_case) { OpenStruct.new(args: { input: "Show me signups by month" }) }++ def stub_structured_response(name:, description:, sql:)+ payload = { name: name, description: description, sql: sql }+ structured =+ instance_double(DiscourseAi::Completions::StructuredOutput).tap do |double|+ allow(double).to receive(:read_buffered_property) { |key| payload[key] }+ end++ stub_runner_bot { |blk| blk.call(structured, nil, :structured_output) }+ end++ describe "#run" do+ it "captures name, description, and sql separately" do+ stub_structured_response(+ name: "Signups by month",+ description: "Counts user signups grouped by month.",+ sql: "SELECT date_trunc('month', created_at) AS month FROM users GROUP BY month",+ )++ result = runner.run(eval_case, llm, execution_context: execution_context)++ expect(result[:raw]).to include("SELECT date_trunc")+ expect(result[:metadata][:name]).to eq("Signups by month")+ expect(result[:metadata][:description]).to eq("Counts user signups grouped by month.")+ expect(result[:metadata][:feature]).to eq("query_generation")+ end+ end+enddiff --git a/plugins/discourse-ai/spec/evals/workbench_spec.rb b/plugins/discourse-ai/spec/evals/workbench_spec.rbindex 9b366bdf612..c41636cb2cf 100644--- a/plugins/discourse-ai/spec/evals/workbench_spec.rb+++ b/plugins/discourse-ai/spec/evals/workbench_spec.rb@@ -305,4 +305,18 @@ RSpec.describe DiscourseAi::Evals::Workbench doexpect(result[:message]).to include("LLM Rating below threshold")endend++ describe "#judge_input" do+ it "forwards `name` and `description` from metadata so the judge sees them" do+ input =+ workbench.send(:judge_input, "SELECT 1", { name: "My Query", description: "Counts users" })++ expect(input).to eq(result: "SELECT 1", name: "My Query", description: "Counts users")+ end++ it "returns the raw result when there is no metadata" do+ expect(workbench.send(:judge_input, "SELECT 1", nil)).to eq("SELECT 1")+ expect(workbench.send(:judge_input, "SELECT 1", {})).to eq("SELECT 1")+ end+ endenddiff --git a/plugins/discourse-ai/spec/lib/agents/tools/db_schema_spec.rb b/plugins/discourse-ai/spec/lib/agents/tools/db_schema_spec.rbindex 91c8965b2f4..e271dfb469a 100644--- a/plugins/discourse-ai/spec/lib/agents/tools/db_schema_spec.rb+++ b/plugins/discourse-ai/spec/lib/agents/tools/db_schema_spec.rb@@ -1,4 +1,4 @@-#frozen_string_literal: true+# frozen_string_literal: trueRSpec.describe DiscourseAi::Agents::Tools::DbSchema dofab!(:llm_model)@@ -10,16 +10,60 @@ RSpec.describe DiscourseAi::Agents::Tools::DbSchema doSiteSetting.ai_bot_enabled = trueend- describe "#process" do- it "returns rich schema for tables" do- result = described_class.new({ tables: "posts,topics" }, bot_user: bot_user, llm: llm).invoke+ def invoke(tables)+ described_class.new({ tables: tables }, bot_user: bot_user, llm: llm).invoke+ end++ describe "#invoke" do+ it "returns one column per line with table headers" do+ info = invoke("posts,topics")[:schema_info]++ expect(info).to match(/^TABLE posts$/)+ expect(info).to match(/^TABLE topics$/)+ expect(info).to match(/^ id integer, PK$/)+ end++ it "marks foreign-key columns using Discourse's curated map" do+ info = invoke("topic_tags,topics")[:schema_info]++ expect(info).to match(/^ topic_id integer, FK → topics/)+ expect(info).to match(/^ user_id integer, FK → users/)+ expect(info).to match(/^ last_post_user_id integer, FK → users/) # via *_user_id rule+ end++ it "does not invent FK targets for non-FK _id columns" do+ info = invoke("topics")[:schema_info]++ # Previous heuristic produced misleading targets like "FK → externals" / "FK → featured_user1s".+ # Now we only mark FKs the Discourse map actually knows about.+ expect(info).not_to include("FK → externals")+ expect(info).not_to include("FK → featured_user1s")+ end++ it "simplifies long postgres type names" do+ info = invoke("topics")[:schema_info]++ expect(info).to include("varchar")+ expect(info).to include("timestamp")+ expect(info).not_to include("character varying")+ expect(info).not_to include("timestamp without time zone")+ end++ it "marks nullable columns" do+ info = invoke("topics")[:schema_info]- expect(result[:schema_info]).to include("raw text")- expect(result[:schema_info]).to include("views integer")- expect(result[:schema_info]).to include("posts")- expect(result[:schema_info]).to include("topics")+ expect(info).to match(/^ deleted_at timestamp, null$/)+ end++ it "reports missing tables instead of silently dropping them" do+ info = invoke("topics,not_a_real_table")[:schema_info]++ expect(info).to include("TABLE topics")+ expect(info).to include("TABLES NOT FOUND: not_a_real_table")+ end- expect(result[:tables]).to eq("posts,topics")+ it "returns the requested tables string" do+ expect(invoke("posts,topics")[:tables]).to eq("posts,topics")endendenddiff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/ai_query_generator.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/ai_query_generator.rbindex 5377dbac65f..467dc28e29d 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/ai_query_generator.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/ai_query_generator.rb@@ -63,23 +63,30 @@ module DiscourseDataExplorer-- int :num = 3-- bigint :big = 12345678912345-- boolean :flag- -- null boolean :opt_flag = #null+ -- null boolean :opt_flag-- string :name = default value- -- date :date = 14 jul 2015- -- time :time = 5:02 pm- -- datetime :datetime = 14 jul 2015 5:02 pm+ -- date :start_date = #{(Date.today - 30).strftime("%Y-%m-%d")}+ -- date :end_date = #{Date.today.strftime("%Y-%m-%d")}+ -- time :time = 17:02+ -- datetime :datetime = #{Date.today.strftime("%Y-%m-%d")} 17:02-- double :ratio = 3.1415- -- user_id :user = system+ -- user_id :user-- post_id :post-- topic_id :topic-- int_list :ids = 1,2,3-- string_list :names = a,b,c- -- category_id :category = meta- -- group_id :group = admins- -- user_list :users = system,discobot+ -- category_id :category+ -- group_id :group+ -- user_list :users-- current_user_id :me- Prefix with "null" for optional params.- Today is #{Date.today.strftime("%Y-%m-%d")}. Date param defaults MUST be real dates like #{Date.today.strftime("%Y-%m-%d")}, NEVER natural language like "today" or "3 months ago".++ ### Parameter rules+ - For category_id, group_id, user_id, user_list, post_id, topic_id, badge_id: OMIT the default value entirely (or use `null` prefix).+ - Optional params: prefix the type with "null" and OMIT the default. The "null" prefix is the optional marker; do NOT use `= #null` as a default value (e.g. write `-- null category_id :category`, NOT `-- category_id :category = #null`).+ - Date param defaults MUST be real ISO dates in YYYY-MM-DD form (today is #{Date.today.strftime("%Y-%m-%d")}). NEVER use natural-language defaults like "today", "yesterday", "3 months ago", or "14 jul 2015". For a date range, name params `:start_date` and `:end_date`.+ - Plural nouns → list-style param TYPES. Apply this rule independently to EACH plural noun in the prompt; don't skip one. Only `int_list`, `string_list`, `user_list`, `group_list` accept multiple values — single-value types like `category_id`, `user_id`, `topic_id` do NOT. Example: "Get topics for selected categories and tags." → `-- null int_list :category_ids` AND `-- null string_list :tag_names` (BOTH list types).+ - `ANY(:param)` and `:param IN (...)` REQUIRE :param to be a list-type (`int_list`, `string_list`, etc.). With single-value types (`category_id`, `user_id`, etc.), use `column = :param` instead. Mismatching the param type with array usage is a runtime error ("op ANY/ALL (array) requires array on right side").+ - First-person prompts ("my posts", "queries about me", "topics I've replied to") MUST use `current_user_id` — this auto-injects the requester's user id at run time. Do NOT use `user_id :user = system` for "my" queries.## Discourse domain knowledge- topics.archetype: 'regular' = topics, 'private_message' = PMs. Default to 'regular'.