3D engine that AI agents drive
Classify agent bench runs with 0 scene calls and keep infra out of pass@1 (#954)
Nguyen Xuan An9282c71db7a4
diff --git a/bench/agent-modeling/README.md b/bench/agent-modeling/README.mdindex ef6b18fd..2114ee5a 100644--- a/bench/agent-modeling/README.md+++ b/bench/agent-modeling/README.md@@ -99,6 +99,12 @@ It prints a table with pass@1 (average success over all runs), pass^k (share of`results/bench-<label>.json` holds that summary and every run's record. `--smoke` runs the two smoke tasks once to prove thepipeline, `--tasks a,b` picks tasks, and `--resume` (same label, models and `--runs`) continues a stopped bench.+A run that makes no scene call is classified in its record (`zeroCalls`, see `zero-calls.mjs`). `infra` causes+(`start-up-failure`, `mcp-server-failed`, `timeout`, `toolsearch-empty`: claude -p or the MCP server never got going, or+ToolSearch never showed the 67js tools) are retried once, and a run that still ends that way is marked `infra`, counted in the+`infra` column and left out of pass@1. `model` causes (`toolsearch-then-quit` after ToolSearch showed the tools, `refusal`,+`no-tool-use`) stay counted as failures and are listed under the table.+To compare a branch with the baseline, run the same command with another label on the branch, then:```sh@@ -106,7 +112,8 @@ node bench/agent-modeling/compare-bench.mjs bench/agent-modeling/results/bench-b```It prints the totals side by side per model and the tasks whose pass count changed, and exits with 1 when a model's pass@1-drops by more than `--tolerance` (0.05). Use the same models, `--runs` and backend on both sides. A task that moves by one+drops by more than `--tolerance` (0.05) or when infra runs pass `--infra-bound` (0.1) of a model's runs, because its pass@1+then rests on too few runs. The infra count prints next to pass@1. Use the same models, `--runs` and backend on both sides. A task that moves by onerun out of three is noise; read the totals and the tasks that moved by two runs or more.### Several runs and several modelsdiff --git a/bench/agent-modeling/bench-summary.mjs b/bench/agent-modeling/bench-summary.mjsindex 183c2322..a972b72b 100644--- a/bench/agent-modeling/bench-summary.mjs+++ b/bench/agent-modeling/bench-summary.mjs@@ -3,6 +3,9 @@// pass@1 average success rate over every run (the chance one try works)// pass^k share of tasks that passed on all k runs (the chance an agent can be relied on)// pass@k share of tasks that passed on at least one of the k runs+// A run flagged `infra` (claude -p or the MCP server never started, see zero-calls.mjs) says nothing about the schemas:+// it is counted in infraRuns, kept out of every other number, and reported apart. A run with 0 scene calls that is the+// model's doing (quit after ToolSearch, refusal) stays scored and is counted by cause in modelZeroCalls.import { kindOf } from './suite.mjs';const mean = (values, digits = 1) => (values.length ? Number((values.reduce((total, value) => total + value, 0) / values.length).toFixed(digits)) : null);@@ -16,15 +19,25 @@ function groupBy(records, keyOf) {return groups;}-/** Numbers for a group of records that belong to tasks run `runs` times each. */-export function benchTotals(records, runs) {+const countBy = (records, causeOf) => Object.fromEntries([...groupBy(records, causeOf)].map(([cause, own]) => [cause, own.length]));++/** Numbers for a group of records that belong to tasks run `runs` times each. Infra runs are counted, not scored. */+export function benchTotals(allRecords, runs) {+ const infraRecords = allRecords.filter(record => record.infra);+ const records = allRecords.filter(record => !record.infra);const perTask = [...groupBy(records, record => record.task).values()];- const everyRun = perTask.filter(own => own.length >= runs && own.every(record => record.pass));+ const attemptsByTask = groupBy(allRecords, record => record.task);+ // A task needs every attempt present (scored or infra); infra attempts neither help nor hurt pass^k.+ const everyRun = perTask.filter(own => attemptsByTask.get(own[0].task).length >= runs && own.every(record => record.pass));const anyRun = perTask.filter(own => own.some(record => record.pass));const share = count => (perTask.length ? Number((count / perTask.length).toFixed(3)) : null);return {tasks: perTask.length,runs: records.length,+ infraRuns: infraRecords.length,+ infraShare: allRecords.length ? Number((infraRecords.length / allRecords.length).toFixed(3)) : 0,+ infraByCause: countBy(infraRecords, record => record.zeroCalls?.cause ?? 'unknown'),+ modelZeroCalls: countBy(records.filter(record => record.zeroCalls), record => record.zeroCalls.cause),passAt1: records.length ? Number((records.filter(record => record.pass).length / records.length).toFixed(3)) : null,passPowK: share(everyRun.length),passAtK: share(anyRun.length),@@ -41,16 +54,20 @@ export function benchSummary(byModel, runs) {return Object.fromEntries(Object.entries(byModel).map(([model, records]) => [model, {...benchTotals(records, runs),kinds: Object.fromEntries([...groupBy(records, record => kindOf[record.task] ?? 'other')].map(([kind, own]) => [kind, benchTotals(own, runs)])),- taskResults: Object.fromEntries([...groupBy(records, record => record.task)].map(([task, own]) => [task, { passed: own.filter(record => record.pass).length, runs: own.length }])),+ taskResults: Object.fromEntries([...groupBy(records, record => record.task)].map(([task, own]) => {+ const scored = own.filter(record => !record.infra);+ const infraRuns = own.length - scored.length;+ return [task, { passed: scored.filter(record => record.pass).length, runs: scored.length, ...(infraRuns ? { infra: infraRuns } : {}) }];+ })),}]));}const percent = value => (value === null ? '?' : `${Math.round(value * 100)}%`);export function formatSummaryTable(summary, runs) {- const header = ['model', 'tasks', 'runs', 'pass@1', `pass^${runs}`, `any of ${runs}`, 'calls/run', 'tokens in/run', 'tokens out/run', 'seconds/run'];+ const header = ['model', 'tasks', 'runs', 'infra', 'pass@1', `pass^${runs}`, `any of ${runs}`, 'calls/run', 'tokens in/run', 'tokens out/run', 'seconds/run'];const rows = Object.entries(summary).map(([model, totals]) => [- model, totals.tasks, totals.runs, percent(totals.passAt1), percent(totals.passPowK), percent(totals.passAtK),+ model, totals.tasks, totals.runs, totals.infraRuns ?? 0, percent(totals.passAt1), percent(totals.passPowK), percent(totals.passAtK),totals.callsPerRun, totals.inputTokensPerRun, totals.outputTokensPerRun, totals.secondsPerRun,].map(String));const widths = header.map((title, column) => Math.max(title.length, ...rows.map(row => row[column].length)));@@ -60,3 +77,13 @@ export function formatSummaryTable(summary, runs) {]);return [line(header), ...rows.map(line), '', ...kindLines].join('\n');}++const causeList = counts => Object.entries(counts ?? {}).map(([cause, count]) => `${cause} ${count}`).join(', ');++/** Lines that name the runs with 0 scene calls: infra ones (left out of pass@1) and model ones (still counted). */+export function zeroCallSummary(totals) {+ const lines = [];+ if (totals.infraRuns) lines.push(`infra runs (not in pass@1): ${totals.infraRuns} (${causeList(totals.infraByCause)})`);+ if (Object.keys(totals.modelZeroCalls ?? {}).length) lines.push(`model runs with 0 scene calls (counted as failures): ${causeList(totals.modelZeroCalls)}`);+ return lines;+}diff --git a/bench/agent-modeling/compare-bench.mjs b/bench/agent-modeling/compare-bench.mjsindex 74e4a8e3..a285391d 100644--- a/bench/agent-modeling/compare-bench.mjs+++ b/bench/agent-modeling/compare-bench.mjs@@ -3,25 +3,37 @@// node bench/agent-modeling/compare-bench.mjs <baseline bench file> <candidate bench file> [--tolerance 0.05]//// Prints pass@1, pass^k, calls and tokens per run side by side, then the tasks whose pass count changed. Exits with 1-// when a model's pass@1 drops by more than the tolerance (default 0.05). With k runs per task, a swing of one run on+// when a model's pass@1 drops by more than the tolerance (default 0.05) or when infra runs pass --infra-bound (default 0.1) of its runs. With k runs per task, a swing of one run on// one task is noise: judge a branch by the totals and by tasks that moved by two runs or more.import { readFile } from 'node:fs/promises';import { fileURLToPath } from 'node:url';import { parseArgs } from 'node:util';+import { zeroCallSummary } from './bench-summary.mjs';const percent = value => (value === null || value === undefined ? '?' : `${Math.round(value * 100)}%`);const signed = value => (value > 0 ? `+${value}` : String(value));/** Differences per model: totals side by side and the tasks whose passes changed. */-export function compareBench(baseline, candidate, tolerance = 0.05) {+export const DEFAULT_INFRA_BOUND = 0.1;++/** Differences per model. Infra runs (no scene call because claude -p or the MCP server failed) are printed next to+ * pass@1 and are not in it; a model whose infra share passes `infraBound` in either file is flagged INFRA, since its+ * pass@1 then rests on too few runs, and that makes the comparison fail like a regression does. */+export function compareBench(baseline, candidate, tolerance = 0.05, infraBound = DEFAULT_INFRA_BOUND) {const lines = [];let regressed = false;+ let tooMuchInfra = false;for (const model of Object.keys(baseline.summary).filter(name => name in candidate.summary)) {const before = baseline.summary[model];const after = candidate.summary[model];const drop = Number(((before.passAt1 ?? 0) - (after.passAt1 ?? 0)).toFixed(3));if (drop > tolerance) regressed = true;- lines.push(`${model}: pass@1 ${percent(before.passAt1)} -> ${percent(after.passAt1)}${drop > tolerance ? ' REGRESSION' : ''}, pass^k ${percent(before.passPowK)} -> ${percent(after.passPowK)}, calls/run ${before.callsPerRun} -> ${after.callsPerRun}, tokens in/run ${before.inputTokensPerRun} -> ${after.inputTokensPerRun}, tokens out/run ${before.outputTokensPerRun} -> ${after.outputTokensPerRun}`);+ const overBound = (before.infraShare ?? 0) > infraBound || (after.infraShare ?? 0) > infraBound;+ if (overBound) tooMuchInfra = true;+ lines.push(`${model}: infra ${before.infraRuns ?? 0} -> ${after.infraRuns ?? 0} runs${overBound ? ` INFRA over ${percent(infraBound)} of runs` : ''}; pass@1 ${percent(before.passAt1)} -> ${percent(after.passAt1)}${drop > tolerance ? ' REGRESSION' : ''}, pass^k ${percent(before.passPowK)} -> ${percent(after.passPowK)}, calls/run ${before.callsPerRun} -> ${after.callsPerRun}, tokens in/run ${before.inputTokensPerRun} -> ${after.inputTokensPerRun}, tokens out/run ${before.outputTokensPerRun} -> ${after.outputTokensPerRun}`);+ for (const [name, totals] of [['baseline', before], ['candidate', after]]) {+ for (const text of zeroCallSummary(totals)) lines.push(` ${name} ${text}`);+ }const tasks = new Set([...Object.keys(before.taskResults), ...Object.keys(after.taskResults)]);for (const task of tasks) {const passedBefore = before.taskResults[task]?.passed ?? 0;@@ -29,14 +41,14 @@ export function compareBench(baseline, candidate, tolerance = 0.05) {if (passedBefore !== passedAfter) lines.push(` ${task}: ${passedBefore}/${before.taskResults[task]?.runs ?? 0} -> ${passedAfter}/${after.taskResults[task]?.runs ?? 0} (${signed(passedAfter - passedBefore)})`);}}- return { regressed, text: lines.join('\n') || 'No model appears in both files.' };+ return { regressed, tooMuchInfra, text: lines.join('\n') || 'No model appears in both files.' };}if (process.argv[1] === fileURLToPath(import.meta.url)) {- const { values, positionals } = parseArgs({ allowPositionals: true, options: { tolerance: { type: 'string', default: '0.05' } } });+ const { values, positionals } = parseArgs({ allowPositionals: true, options: { tolerance: { type: 'string', default: '0.05' }, 'infra-bound': { type: 'string', default: String(DEFAULT_INFRA_BOUND) } } });const [baselinePath, candidatePath] = positionals;const [baseline, candidate] = await Promise.all([baselinePath, candidatePath].map(async path => JSON.parse(await readFile(path, 'utf8'))));- const result = compareBench(baseline, candidate, Number(values.tolerance));+ const result = compareBench(baseline, candidate, Number(values.tolerance), Number(values['infra-bound']));console.log(result.text);- process.exitCode = result.regressed ? 1 : 0;+ process.exitCode = result.regressed || result.tooMuchInfra ? 1 : 0;}diff --git a/bench/agent-modeling/run.mjs b/bench/agent-modeling/run.mjsindex 095b39f3..8aca40f2 100644--- a/bench/agent-modeling/run.mjs+++ b/bench/agent-modeling/run.mjs@@ -24,6 +24,7 @@ import { parseArgs } from 'node:util';import { removeLocalPaths, totalsByModel, totalsOf } from './aggregate.mjs';import { gradeRecipe, recipeFeatures } from './check.mjs';import { scoreLookSafely } from './look.mjs';+import { classifyZeroCalls } from './zero-calls.mjs';import { PROMPT_PREAMBLE, TASKS, taskById, taskPrompt } from './tasks.mjs';const repository = resolve(import.meta.dirname, '../..');@@ -211,6 +212,17 @@ async function runTask(task, attempt, systemPromptAddition) {}}+// An infra failure (claude -p or the MCP server never got going) says nothing about the schemas, so the run is tried+// once more. The retry's record replaces the first; it keeps the first cause in retriedAfter. A second infra failure+// stays marked infra and is left out of pass@1.+async function runTaskWithInfraRetry(task, attempt, systemPromptAddition) {+ const first = await runTask(task, attempt, systemPromptAddition);+ if (!first.infra) return first;+ console.log(`${task.id.padEnd(16)} #${attempt} infra ${first.zeroCalls.cause}: retrying once`);+ const second = await runTask(task, attempt, systemPromptAddition, '-retry');+ return { ...second, retriedAfter: first.zeroCalls.cause };+}+if (!['on', 'off'].includes(options.branches)) throw new RangeError('--branches must be on or off');if (!['on', 'off'].includes(options.relations)) throw new RangeError('--relations must be on or off');const tasks = options.tasks ? options.tasks.split(',').map(taskById) : TASKS;@@ -250,9 +262,10 @@ for (let attempt = 1; attempt <= attempts; attempt++) {}for (const { task, attempt } of pending) {- const record = await runTask(task, attempt, systemPromptAddition);+ const record = await runTaskWithInfraRetry(task, attempt, systemPromptAddition);records.push(record);- console.log(`${record.task.padEnd(16)} #${attempt} ${record.pass ? 'PASS' : 'FAIL'} ${record.passed}/${record.total} calls ${record.toolCalls} (errors ${record.toolErrors}) $${record.costUsd ?? '?'} ${record.wallSeconds}s${record.failed.length ? ` ${record.failed.join(' | ')}` : ''}`);+ const label = record.infra ? `INFRA ${record.zeroCalls.cause}` : record.pass ? 'PASS' : `FAIL${record.zeroCalls ? ` ${record.zeroCalls.cause}` : ''}`;+ console.log(`${record.task.padEnd(16)} #${attempt} ${label} ${record.passed}/${record.total} calls ${record.toolCalls} (errors ${record.toolErrors}) $${record.costUsd ?? '?'} ${record.wallSeconds}s${record.failed.length ? ` ${record.failed.join(' | ')}` : ''}`);// Saved after every task, so a run that gets killed keeps what it already paid for.await writeFile(summaryPath, `${JSON.stringify(summarize(), null, 2)}\n`);}diff --git a/bench/agent-modeling/zero-calls.mjs b/bench/agent-modeling/zero-calls.mjsnew file mode 100644index 00000000..61f48813--- /dev/null+++ b/bench/agent-modeling/zero-calls.mjs@@ -0,0 +1,79 @@+// Why a bench run made no scene tool calls. A run with 0 scene calls is either the machine's fault (infra: claude -p or+// the MCP server never got going) or the model's (it quit after ToolSearch, refused, or answered without tools).+// Infra runs are retried once and kept out of pass@1; model runs stay counted and carry a label.+//+// classifyZeroCalls(output, { exitCode, errors, toolCalls }) returns null when the run made scene calls, else+// { kind: 'infra' | 'model', cause, detail }.+// infra start-up-failure no init event, or no result event, or a non-zero exit without a result+// infra mcp-server-failed the init event lists the 67js server as not connected+// infra timeout the run was killed by the timeout (no result, exit signal)+// infra toolsearch-empty ToolSearch ran but returned no 67js tool, so the model never saw the tools+// model toolsearch-then-quit ToolSearch returned the 67js tools and the model ended without calling one+// model refusal the model declined (stop reason refusal, or a refusal in the result text)+// model no-tool-use the model answered without ToolSearch or any tool call+const MCP_SERVER_NAME = '67js';+const REFUSAL_PATTERN = /\b(i can(?:'|’)?t (?:help|assist|do)|i(?:'|’)m (?:unable|not able) to|i (?:cannot|won(?:'|’)t)|i must decline)\b/i;++function textOf(content) {+ if (typeof content === 'string') return content;+ return (content ?? []).filter(block => block.type === 'text').map(block => block.text).join('');+}++/** Parses the stream-json lines into the facts the classifier needs. */+export function readRunFacts(output) {+ const facts = { initialized: false, serverStatus: null, toolSearchCalls: 0, toolSearchSawSceneTools: false, result: null };+ const toolSearchIds = new Set();+ for (const line of String(output ?? '').split('\n')) {+ if (!line.trim().startsWith('{')) continue;+ let event;+ try {+ event = JSON.parse(line);+ } catch {+ continue;+ }+ if (event.type === 'system' && event.subtype === 'init') {+ facts.initialized = true;+ const server = (event.mcp_servers ?? []).find(candidate => candidate.name === MCP_SERVER_NAME);+ facts.serverStatus = server ? server.status : 'missing';+ }+ if (event.type === 'assistant') {+ for (const block of event.message?.content ?? []) {+ if (block.type === 'tool_use' && block.name === 'ToolSearch') {+ facts.toolSearchCalls++;+ toolSearchIds.add(block.id);+ }+ }+ }+ if (event.type === 'user') {+ for (const block of event.message?.content ?? []) {+ if (block.type === 'tool_result' && toolSearchIds.has(block.tool_use_id) && !block.is_error) {+ if (textOf(block.content).includes(`mcp__${MCP_SERVER_NAME}__`)) facts.toolSearchSawSceneTools = true;+ }+ }+ }+ if (event.type === 'result') facts.result = event;+ }+ return facts;+}++export function classifyZeroCalls(output, { exitCode = 0, errors = '', toolCalls = 0 } = {}) {+ if (toolCalls > 0) return null;+ const facts = readRunFacts(output);+ const stderrNote = String(errors ?? '').trim().split('\n')[0].slice(0, 120);+ if (!facts.initialized) return { kind: 'infra', cause: 'start-up-failure', detail: stderrNote || `no init event, exit ${exitCode}` };+ if (facts.serverStatus !== 'connected') return { kind: 'infra', cause: 'mcp-server-failed', detail: `67js server ${facts.serverStatus}` };+ if (!facts.result) {+ return exitCode === null || exitCode === undefined+ ? { kind: 'infra', cause: 'timeout', detail: 'killed before a result' }+ : { kind: 'infra', cause: 'start-up-failure', detail: stderrNote || `no result event, exit ${exitCode}` };+ }+ const resultText = String(facts.result.result ?? '');+ if (facts.result.stop_reason === 'refusal' || (facts.toolSearchCalls === 0 && REFUSAL_PATTERN.test(resultText))) {+ return { kind: 'model', cause: 'refusal', detail: resultText.slice(0, 120) };+ }+ if (facts.toolSearchCalls > 0 && !facts.toolSearchSawSceneTools) {+ return { kind: 'infra', cause: 'toolsearch-empty', detail: `${facts.toolSearchCalls} ToolSearch calls returned no 67js tool` };+ }+ if (facts.toolSearchCalls > 0) return { kind: 'model', cause: 'toolsearch-then-quit', detail: `${facts.toolSearchCalls} ToolSearch calls, then no scene call` };+ return { kind: 'model', cause: 'no-tool-use', detail: resultText.slice(0, 120) };+}diff --git a/tests/agent-bench.test.js b/tests/agent-bench.test.jsindex ae75e1e8..315443a9 100644--- a/tests/agent-bench.test.js+++ b/tests/agent-bench.test.js@@ -1,6 +1,8 @@import assert from 'node:assert/strict';+import { readFileSync } from 'node:fs';import test from 'node:test';import { benchSummary, benchTotals } from '../bench/agent-modeling/bench-summary.mjs';+import { classifyZeroCalls } from '../bench/agent-modeling/zero-calls.mjs';import { compareBench } from '../bench/agent-modeling/compare-bench.mjs';import { gradeRecipe } from '../bench/agent-modeling/check.mjs';import { REFERENCES } from '../bench/agent-modeling/references.mjs';@@ -70,3 +72,50 @@ test('the summary groups by model and kind, and the comparison flags a pass@1 drassert.match(result.text, /crate: 2\/2 -> 1\/2 \(-1\)/);assert.equal(compareBench(baseline, baseline).regressed, false);});++const readFixture = name => readFileSync(new URL(`./fixtures/agent-bench/${name}.jsonl`, import.meta.url), 'utf8');++test('a run with 0 scene calls is classified by cause: infra for start-up, server and empty ToolSearch, model for the rest', () => {+ const classify = (name, options = {}) => classifyZeroCalls(readFixture(name), { exitCode: 0, toolCalls: 0, ...options });+ assert.equal(classify('with-scene-calls', { toolCalls: 1 }), null);+ assert.deepEqual(classify('no-init', { exitCode: 1, errors: 'spawn failed\nmore' }), { kind: 'infra', cause: 'start-up-failure', detail: 'spawn failed' });+ assert.equal(classify('mcp-server-failed').cause, 'mcp-server-failed');+ assert.equal(classify('mcp-server-failed').kind, 'infra');+ assert.equal(classify('no-result', { exitCode: 1 }).cause, 'start-up-failure');+ assert.equal(classify('no-result', { exitCode: null }).cause, 'timeout');+ assert.equal(classify('toolsearch-empty').cause, 'toolsearch-empty');+ assert.equal(classify('toolsearch-empty').kind, 'infra');+});++test('a ToolSearch that showed the 67js tools and then quit, a refusal and a no-tool answer are model failures', () => {+ const classify = name => classifyZeroCalls(readFixture(name), { exitCode: 0, toolCalls: 0 });+ assert.deepEqual([classify('toolsearch-then-quit').kind, classify('toolsearch-then-quit').cause], ['model', 'toolsearch-then-quit']);+ assert.deepEqual([classify('refusal').kind, classify('refusal').cause], ['model', 'refusal']);+ assert.deepEqual([classify('no-tool-use').kind, classify('no-tool-use').cause], ['model', 'no-tool-use']);+});++test('infra runs stay out of pass@1 and are reported by cause; model zero-call runs stay counted and are labelled', () => {+ const infraRecord = cause => record('crate', false, { toolCalls: 0, infra: true, zeroCalls: { kind: 'infra', cause } });+ const quitRecord = record('bolt', false, { toolCalls: 0, infra: false, zeroCalls: { kind: 'model', cause: 'toolsearch-then-quit' } });+ const totals = benchTotals([record('crate', true), infraRecord('mcp-server-failed'), infraRecord('timeout'), record('bolt', true), quitRecord], 2);+ assert.equal(totals.runs, 3);+ assert.equal(totals.passAt1, 0.667);+ assert.equal(totals.infraRuns, 2);+ assert.deepEqual(totals.infraByCause, { 'mcp-server-failed': 1, timeout: 1 });+ assert.deepEqual(totals.modelZeroCalls, { 'toolsearch-then-quit': 1 });+ assert.equal(totals.infraShare, 0.4);+ assert.equal(totals.passPowK, 0.5);+ assert.equal(benchSummary({ haiku: [record('crate', true), infraRecord('timeout')] }, 2).haiku.taskResults.crate.infra, 1);+});++test('the comparison prints infra counts next to pass@1 and flags infra above the bound', () => {+ const infraRecord = record('crate', false, { toolCalls: 0, infra: true, zeroCalls: { kind: 'infra', cause: 'timeout' } });+ const clean = { summary: benchSummary({ haiku: [record('crate', true), record('crate', true), record('bolt', true), record('bolt', true)] }, 2) };+ const noisy = { summary: benchSummary({ haiku: [record('crate', true), infraRecord, record('bolt', true), record('bolt', true)] }, 2) };+ const result = compareBench(clean, noisy);+ assert.equal(result.regressed, false);+ assert.equal(result.tooMuchInfra, true);+ assert.match(result.text, /haiku: infra 0 -> 1 runs {2}INFRA over 10% of runs; pass@1 100% -> 100%/);+ assert.match(result.text, /candidate infra runs \(not in pass@1\): 1 \(timeout 1\)/);+ assert.equal(compareBench(clean, noisy, 0.05, 0.5).tooMuchInfra, false);+});diff --git a/tests/fixtures/agent-bench/mcp-server-failed.jsonl b/tests/fixtures/agent-bench/mcp-server-failed.jsonlnew file mode 100644index 00000000..4a407e5c--- /dev/null+++ b/tests/fixtures/agent-bench/mcp-server-failed.jsonl@@ -0,0 +1,3 @@+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"failed"}]}+{"type":"assistant","message":{"content":[{"type":"text","text":"No scene tools."}]}}+{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"No scene tools.","usage":{"output_tokens":900}}diff --git a/tests/fixtures/agent-bench/no-init.jsonl b/tests/fixtures/agent-bench/no-init.jsonlnew file mode 100644index 00000000..e69de29bdiff --git a/tests/fixtures/agent-bench/no-result.jsonl b/tests/fixtures/agent-bench/no-result.jsonlnew file mode 100644index 00000000..1fd5d6cd--- /dev/null+++ b/tests/fixtures/agent-bench/no-result.jsonl@@ -0,0 +1 @@+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"connected"}]}diff --git a/tests/fixtures/agent-bench/no-tool-use.jsonl b/tests/fixtures/agent-bench/no-tool-use.jsonlnew file mode 100644index 00000000..b41d75be--- /dev/null+++ b/tests/fixtures/agent-bench/no-tool-use.jsonl@@ -0,0 +1,3 @@+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"connected"}]}+{"type":"assistant","message":{"content":[{"type":"text","text":"Here is a description of a crate."}]}}+{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"Here is a description of a crate.","usage":{"output_tokens":900}}diff --git a/tests/fixtures/agent-bench/refusal.jsonl b/tests/fixtures/agent-bench/refusal.jsonlnew file mode 100644index 00000000..c3ae8b9d--- /dev/null+++ b/tests/fixtures/agent-bench/refusal.jsonl@@ -0,0 +1,3 @@+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"connected"}]}+{"type":"assistant","message":{"content":[{"type":"text","text":"I can't help with that request."}]}}+{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"I can't help with that request.","usage":{"output_tokens":900},"stop_reason":"refusal"}diff --git a/tests/fixtures/agent-bench/toolsearch-empty.jsonl b/tests/fixtures/agent-bench/toolsearch-empty.jsonlnew file mode 100644index 00000000..07828d45--- /dev/null+++ b/tests/fixtures/agent-bench/toolsearch-empty.jsonl@@ -0,0 +1,5 @@+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"connected"}]}+{"type":"assistant","message":{"content":[{"type":"tool_use","id":"t1","name":"ToolSearch","input":{"query":"select:mcp__67js__scene_create"}}]}}+{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1","content":[{"type":"text","text":"No matching deferred tools found"}]}]}}+{"type":"assistant","message":{"content":[{"type":"text","text":"The tools are not available."}]}}+{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"The tools are not available.","usage":{"output_tokens":900}}diff --git a/tests/fixtures/agent-bench/toolsearch-then-quit.jsonl b/tests/fixtures/agent-bench/toolsearch-then-quit.jsonlnew file mode 100644index 00000000..79147f6b--- /dev/null+++ b/tests/fixtures/agent-bench/toolsearch-then-quit.jsonl@@ -0,0 +1,5 @@+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"connected"}]}+{"type":"assistant","message":{"content":[{"type":"tool_use","id":"t1","name":"ToolSearch","input":{"query":"select:mcp__67js__scene_create"}}]}}+{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1","content":[{"type":"text","text":"mcp__67js__scene_create\nmcp__67js__scene_add_part\nmcp__67js__scene_export_recipe"}]}]}}+{"type":"assistant","message":{"content":[{"type":"text","text":"I will plan the crate."}]}}+{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"I will plan the crate.","usage":{"output_tokens":900}}diff --git a/tests/fixtures/agent-bench/with-scene-calls.jsonl b/tests/fixtures/agent-bench/with-scene-calls.jsonlnew file mode 100644index 00000000..04de2df4--- /dev/null+++ b/tests/fixtures/agent-bench/with-scene-calls.jsonl@@ -0,0 +1,5 @@+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"connected"}]}+{"type":"assistant","message":{"content":[{"type":"tool_use","id":"t1","name":"ToolSearch","input":{"query":"select:mcp__67js__scene_create"}}]}}+{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1","content":[{"type":"text","text":"mcp__67js__scene_create\nmcp__67js__scene_add_part\nmcp__67js__scene_export_recipe"}]}]}}+{"type":"assistant","message":{"content":[{"type":"tool_use","id":"c1","name":"mcp__67js__scene_create","input":{}}]}}+{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"done","usage":{"output_tokens":900}}