Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Classify agent bench runs with 0 scene calls and keep infra out of pass@1 (#954)

Nguyen Xuan An9282c71db7a4
diff --git a/bench/agent-modeling/README.md b/bench/agent-modeling/README.md
index ef6b18fd..2114ee5a 100644
--- a/bench/agent-modeling/README.md
+++ b/bench/agent-modeling/README.md
@@ -99,6 +99,12 @@ It prints a table with pass@1 (average success over all runs), pass^k (share of
`results/bench-<label>.json` holds that summary and every run's record. `--smoke` runs the two smoke tasks once to prove the
pipeline, `--tasks a,b` picks tasks, and `--resume` (same label, models and `--runs`) continues a stopped bench.
+A run that makes no scene call is classified in its record (`zeroCalls`, see `zero-calls.mjs`). `infra` causes
+(`start-up-failure`, `mcp-server-failed`, `timeout`, `toolsearch-empty`: claude -p or the MCP server never got going, or
+ToolSearch never showed the 67js tools) are retried once, and a run that still ends that way is marked `infra`, counted in the
+`infra` column and left out of pass@1. `model` causes (`toolsearch-then-quit` after ToolSearch showed the tools, `refusal`,
+`no-tool-use`) stay counted as failures and are listed under the table.
+
To compare a branch with the baseline, run the same command with another label on the branch, then:
```sh
@@ -106,7 +112,8 @@ node bench/agent-modeling/compare-bench.mjs bench/agent-modeling/results/bench-b
```
It prints the totals side by side per model and the tasks whose pass count changed, and exits with 1 when a model's pass@1
-drops by more than `--tolerance` (0.05). Use the same models, `--runs` and backend on both sides. A task that moves by one
+drops by more than `--tolerance` (0.05) or when infra runs pass `--infra-bound` (0.1) of a model's runs, because its pass@1
+then rests on too few runs. The infra count prints next to pass@1. Use the same models, `--runs` and backend on both sides. A task that moves by one
run out of three is noise; read the totals and the tasks that moved by two runs or more.
### Several runs and several models
diff --git a/bench/agent-modeling/bench-summary.mjs b/bench/agent-modeling/bench-summary.mjs
index 183c2322..a972b72b 100644
--- a/bench/agent-modeling/bench-summary.mjs
+++ b/bench/agent-modeling/bench-summary.mjs
@@ -3,6 +3,9 @@
// pass@1 average success rate over every run (the chance one try works)
// pass^k share of tasks that passed on all k runs (the chance an agent can be relied on)
// pass@k share of tasks that passed on at least one of the k runs
+// A run flagged `infra` (claude -p or the MCP server never started, see zero-calls.mjs) says nothing about the schemas:
+// it is counted in infraRuns, kept out of every other number, and reported apart. A run with 0 scene calls that is the
+// model's doing (quit after ToolSearch, refusal) stays scored and is counted by cause in modelZeroCalls.
import { kindOf } from './suite.mjs';
const mean = (values, digits = 1) => (values.length ? Number((values.reduce((total, value) => total + value, 0) / values.length).toFixed(digits)) : null);
@@ -16,15 +19,25 @@ function groupBy(records, keyOf) {
return groups;
}
-/** Numbers for a group of records that belong to tasks run `runs` times each. */
-export function benchTotals(records, runs) {
+const countBy = (records, causeOf) => Object.fromEntries([...groupBy(records, causeOf)].map(([cause, own]) => [cause, own.length]));
+
+/** Numbers for a group of records that belong to tasks run `runs` times each. Infra runs are counted, not scored. */
+export function benchTotals(allRecords, runs) {
+ const infraRecords = allRecords.filter(record => record.infra);
+ const records = allRecords.filter(record => !record.infra);
const perTask = [...groupBy(records, record => record.task).values()];
- const everyRun = perTask.filter(own => own.length >= runs && own.every(record => record.pass));
+ const attemptsByTask = groupBy(allRecords, record => record.task);
+ // A task needs every attempt present (scored or infra); infra attempts neither help nor hurt pass^k.
+ const everyRun = perTask.filter(own => attemptsByTask.get(own[0].task).length >= runs && own.every(record => record.pass));
const anyRun = perTask.filter(own => own.some(record => record.pass));
const share = count => (perTask.length ? Number((count / perTask.length).toFixed(3)) : null);
return {
tasks: perTask.length,
runs: records.length,
+ infraRuns: infraRecords.length,
+ infraShare: allRecords.length ? Number((infraRecords.length / allRecords.length).toFixed(3)) : 0,
+ infraByCause: countBy(infraRecords, record => record.zeroCalls?.cause ?? 'unknown'),
+ modelZeroCalls: countBy(records.filter(record => record.zeroCalls), record => record.zeroCalls.cause),
passAt1: records.length ? Number((records.filter(record => record.pass).length / records.length).toFixed(3)) : null,
passPowK: share(everyRun.length),
passAtK: share(anyRun.length),
@@ -41,16 +54,20 @@ export function benchSummary(byModel, runs) {
return Object.fromEntries(Object.entries(byModel).map(([model, records]) => [model, {
...benchTotals(records, runs),
kinds: Object.fromEntries([...groupBy(records, record => kindOf[record.task] ?? 'other')].map(([kind, own]) => [kind, benchTotals(own, runs)])),
- taskResults: Object.fromEntries([...groupBy(records, record => record.task)].map(([task, own]) => [task, { passed: own.filter(record => record.pass).length, runs: own.length }])),
+ taskResults: Object.fromEntries([...groupBy(records, record => record.task)].map(([task, own]) => {
+ const scored = own.filter(record => !record.infra);
+ const infraRuns = own.length - scored.length;
+ return [task, { passed: scored.filter(record => record.pass).length, runs: scored.length, ...(infraRuns ? { infra: infraRuns } : {}) }];
+ })),
}]));
}
const percent = value => (value === null ? '?' : `${Math.round(value * 100)}%`);
export function formatSummaryTable(summary, runs) {
- const header = ['model', 'tasks', 'runs', 'pass@1', `pass^${runs}`, `any of ${runs}`, 'calls/run', 'tokens in/run', 'tokens out/run', 'seconds/run'];
+ const header = ['model', 'tasks', 'runs', 'infra', 'pass@1', `pass^${runs}`, `any of ${runs}`, 'calls/run', 'tokens in/run', 'tokens out/run', 'seconds/run'];
const rows = Object.entries(summary).map(([model, totals]) => [
- model, totals.tasks, totals.runs, percent(totals.passAt1), percent(totals.passPowK), percent(totals.passAtK),
+ model, totals.tasks, totals.runs, totals.infraRuns ?? 0, percent(totals.passAt1), percent(totals.passPowK), percent(totals.passAtK),
totals.callsPerRun, totals.inputTokensPerRun, totals.outputTokensPerRun, totals.secondsPerRun,
].map(String));
const widths = header.map((title, column) => Math.max(title.length, ...rows.map(row => row[column].length)));
@@ -60,3 +77,13 @@ export function formatSummaryTable(summary, runs) {
]);
return [line(header), ...rows.map(line), '', ...kindLines].join('\n');
}
+
+const causeList = counts => Object.entries(counts ?? {}).map(([cause, count]) => `${cause} ${count}`).join(', ');
+
+/** Lines that name the runs with 0 scene calls: infra ones (left out of pass@1) and model ones (still counted). */
+export function zeroCallSummary(totals) {
+ const lines = [];
+ if (totals.infraRuns) lines.push(`infra runs (not in pass@1): ${totals.infraRuns} (${causeList(totals.infraByCause)})`);
+ if (Object.keys(totals.modelZeroCalls ?? {}).length) lines.push(`model runs with 0 scene calls (counted as failures): ${causeList(totals.modelZeroCalls)}`);
+ return lines;
+}
diff --git a/bench/agent-modeling/compare-bench.mjs b/bench/agent-modeling/compare-bench.mjs
index 74e4a8e3..a285391d 100644
--- a/bench/agent-modeling/compare-bench.mjs
+++ b/bench/agent-modeling/compare-bench.mjs
@@ -3,25 +3,37 @@
// node bench/agent-modeling/compare-bench.mjs <baseline bench file> <candidate bench file> [--tolerance 0.05]
//
// Prints pass@1, pass^k, calls and tokens per run side by side, then the tasks whose pass count changed. Exits with 1
-// when a model's pass@1 drops by more than the tolerance (default 0.05). With k runs per task, a swing of one run on
+// when a model's pass@1 drops by more than the tolerance (default 0.05) or when infra runs pass --infra-bound (default 0.1) of its runs. With k runs per task, a swing of one run on
// one task is noise: judge a branch by the totals and by tasks that moved by two runs or more.
import { readFile } from 'node:fs/promises';
import { fileURLToPath } from 'node:url';
import { parseArgs } from 'node:util';
+import { zeroCallSummary } from './bench-summary.mjs';
const percent = value => (value === null || value === undefined ? '?' : `${Math.round(value * 100)}%`);
const signed = value => (value > 0 ? `+${value}` : String(value));
/** Differences per model: totals side by side and the tasks whose passes changed. */
-export function compareBench(baseline, candidate, tolerance = 0.05) {
+export const DEFAULT_INFRA_BOUND = 0.1;
+
+/** Differences per model. Infra runs (no scene call because claude -p or the MCP server failed) are printed next to
+ * pass@1 and are not in it; a model whose infra share passes `infraBound` in either file is flagged INFRA, since its
+ * pass@1 then rests on too few runs, and that makes the comparison fail like a regression does. */
+export function compareBench(baseline, candidate, tolerance = 0.05, infraBound = DEFAULT_INFRA_BOUND) {
const lines = [];
let regressed = false;
+ let tooMuchInfra = false;
for (const model of Object.keys(baseline.summary).filter(name => name in candidate.summary)) {
const before = baseline.summary[model];
const after = candidate.summary[model];
const drop = Number(((before.passAt1 ?? 0) - (after.passAt1 ?? 0)).toFixed(3));
if (drop > tolerance) regressed = true;
- lines.push(`${model}: pass@1 ${percent(before.passAt1)} -> ${percent(after.passAt1)}${drop > tolerance ? ' REGRESSION' : ''}, pass^k ${percent(before.passPowK)} -> ${percent(after.passPowK)}, calls/run ${before.callsPerRun} -> ${after.callsPerRun}, tokens in/run ${before.inputTokensPerRun} -> ${after.inputTokensPerRun}, tokens out/run ${before.outputTokensPerRun} -> ${after.outputTokensPerRun}`);
+ const overBound = (before.infraShare ?? 0) > infraBound || (after.infraShare ?? 0) > infraBound;
+ if (overBound) tooMuchInfra = true;
+ lines.push(`${model}: infra ${before.infraRuns ?? 0} -> ${after.infraRuns ?? 0} runs${overBound ? ` INFRA over ${percent(infraBound)} of runs` : ''}; pass@1 ${percent(before.passAt1)} -> ${percent(after.passAt1)}${drop > tolerance ? ' REGRESSION' : ''}, pass^k ${percent(before.passPowK)} -> ${percent(after.passPowK)}, calls/run ${before.callsPerRun} -> ${after.callsPerRun}, tokens in/run ${before.inputTokensPerRun} -> ${after.inputTokensPerRun}, tokens out/run ${before.outputTokensPerRun} -> ${after.outputTokensPerRun}`);
+ for (const [name, totals] of [['baseline', before], ['candidate', after]]) {
+ for (const text of zeroCallSummary(totals)) lines.push(` ${name} ${text}`);
+ }
const tasks = new Set([...Object.keys(before.taskResults), ...Object.keys(after.taskResults)]);
for (const task of tasks) {
const passedBefore = before.taskResults[task]?.passed ?? 0;
@@ -29,14 +41,14 @@ export function compareBench(baseline, candidate, tolerance = 0.05) {
if (passedBefore !== passedAfter) lines.push(` ${task}: ${passedBefore}/${before.taskResults[task]?.runs ?? 0} -> ${passedAfter}/${after.taskResults[task]?.runs ?? 0} (${signed(passedAfter - passedBefore)})`);
}
}
- return { regressed, text: lines.join('\n') || 'No model appears in both files.' };
+ return { regressed, tooMuchInfra, text: lines.join('\n') || 'No model appears in both files.' };
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
- const { values, positionals } = parseArgs({ allowPositionals: true, options: { tolerance: { type: 'string', default: '0.05' } } });
+ const { values, positionals } = parseArgs({ allowPositionals: true, options: { tolerance: { type: 'string', default: '0.05' }, 'infra-bound': { type: 'string', default: String(DEFAULT_INFRA_BOUND) } } });
const [baselinePath, candidatePath] = positionals;
const [baseline, candidate] = await Promise.all([baselinePath, candidatePath].map(async path => JSON.parse(await readFile(path, 'utf8'))));
- const result = compareBench(baseline, candidate, Number(values.tolerance));
+ const result = compareBench(baseline, candidate, Number(values.tolerance), Number(values['infra-bound']));
console.log(result.text);
- process.exitCode = result.regressed ? 1 : 0;
+ process.exitCode = result.regressed || result.tooMuchInfra ? 1 : 0;
}
diff --git a/bench/agent-modeling/run.mjs b/bench/agent-modeling/run.mjs
index 095b39f3..8aca40f2 100644
--- a/bench/agent-modeling/run.mjs
+++ b/bench/agent-modeling/run.mjs
@@ -24,6 +24,7 @@ import { parseArgs } from 'node:util';
import { removeLocalPaths, totalsByModel, totalsOf } from './aggregate.mjs';
import { gradeRecipe, recipeFeatures } from './check.mjs';
import { scoreLookSafely } from './look.mjs';
+import { classifyZeroCalls } from './zero-calls.mjs';
import { PROMPT_PREAMBLE, TASKS, taskById, taskPrompt } from './tasks.mjs';
const repository = resolve(import.meta.dirname, '../..');
@@ -211,6 +212,17 @@ async function runTask(task, attempt, systemPromptAddition) {
}
}
+// An infra failure (claude -p or the MCP server never got going) says nothing about the schemas, so the run is tried
+// once more. The retry's record replaces the first; it keeps the first cause in retriedAfter. A second infra failure
+// stays marked infra and is left out of pass@1.
+async function runTaskWithInfraRetry(task, attempt, systemPromptAddition) {
+ const first = await runTask(task, attempt, systemPromptAddition);
+ if (!first.infra) return first;
+ console.log(`${task.id.padEnd(16)} #${attempt} infra ${first.zeroCalls.cause}: retrying once`);
+ const second = await runTask(task, attempt, systemPromptAddition, '-retry');
+ return { ...second, retriedAfter: first.zeroCalls.cause };
+}
+
if (!['on', 'off'].includes(options.branches)) throw new RangeError('--branches must be on or off');
if (!['on', 'off'].includes(options.relations)) throw new RangeError('--relations must be on or off');
const tasks = options.tasks ? options.tasks.split(',').map(taskById) : TASKS;
@@ -250,9 +262,10 @@ for (let attempt = 1; attempt <= attempts; attempt++) {
}
for (const { task, attempt } of pending) {
- const record = await runTask(task, attempt, systemPromptAddition);
+ const record = await runTaskWithInfraRetry(task, attempt, systemPromptAddition);
records.push(record);
- console.log(`${record.task.padEnd(16)} #${attempt} ${record.pass ? 'PASS' : 'FAIL'} ${record.passed}/${record.total} calls ${record.toolCalls} (errors ${record.toolErrors}) $${record.costUsd ?? '?'} ${record.wallSeconds}s${record.failed.length ? ` ${record.failed.join(' | ')}` : ''}`);
+ const label = record.infra ? `INFRA ${record.zeroCalls.cause}` : record.pass ? 'PASS' : `FAIL${record.zeroCalls ? ` ${record.zeroCalls.cause}` : ''}`;
+ console.log(`${record.task.padEnd(16)} #${attempt} ${label} ${record.passed}/${record.total} calls ${record.toolCalls} (errors ${record.toolErrors}) $${record.costUsd ?? '?'} ${record.wallSeconds}s${record.failed.length ? ` ${record.failed.join(' | ')}` : ''}`);
// Saved after every task, so a run that gets killed keeps what it already paid for.
await writeFile(summaryPath, `${JSON.stringify(summarize(), null, 2)}\n`);
}
diff --git a/bench/agent-modeling/zero-calls.mjs b/bench/agent-modeling/zero-calls.mjs
new file mode 100644
index 00000000..61f48813
--- /dev/null
+++ b/bench/agent-modeling/zero-calls.mjs
@@ -0,0 +1,79 @@
+// Why a bench run made no scene tool calls. A run with 0 scene calls is either the machine's fault (infra: claude -p or
+// the MCP server never got going) or the model's (it quit after ToolSearch, refused, or answered without tools).
+// Infra runs are retried once and kept out of pass@1; model runs stay counted and carry a label.
+//
+// classifyZeroCalls(output, { exitCode, errors, toolCalls }) returns null when the run made scene calls, else
+// { kind: 'infra' | 'model', cause, detail }.
+// infra start-up-failure no init event, or no result event, or a non-zero exit without a result
+// infra mcp-server-failed the init event lists the 67js server as not connected
+// infra timeout the run was killed by the timeout (no result, exit signal)
+// infra toolsearch-empty ToolSearch ran but returned no 67js tool, so the model never saw the tools
+// model toolsearch-then-quit ToolSearch returned the 67js tools and the model ended without calling one
+// model refusal the model declined (stop reason refusal, or a refusal in the result text)
+// model no-tool-use the model answered without ToolSearch or any tool call
+const MCP_SERVER_NAME = '67js';
+const REFUSAL_PATTERN = /\b(i can(?:'|’)?t (?:help|assist|do)|i(?:'|’)m (?:unable|not able) to|i (?:cannot|won(?:'|’)t)|i must decline)\b/i;
+
+function textOf(content) {
+ if (typeof content === 'string') return content;
+ return (content ?? []).filter(block => block.type === 'text').map(block => block.text).join('');
+}
+
+/** Parses the stream-json lines into the facts the classifier needs. */
+export function readRunFacts(output) {
+ const facts = { initialized: false, serverStatus: null, toolSearchCalls: 0, toolSearchSawSceneTools: false, result: null };
+ const toolSearchIds = new Set();
+ for (const line of String(output ?? '').split('\n')) {
+ if (!line.trim().startsWith('{')) continue;
+ let event;
+ try {
+ event = JSON.parse(line);
+ } catch {
+ continue;
+ }
+ if (event.type === 'system' && event.subtype === 'init') {
+ facts.initialized = true;
+ const server = (event.mcp_servers ?? []).find(candidate => candidate.name === MCP_SERVER_NAME);
+ facts.serverStatus = server ? server.status : 'missing';
+ }
+ if (event.type === 'assistant') {
+ for (const block of event.message?.content ?? []) {
+ if (block.type === 'tool_use' && block.name === 'ToolSearch') {
+ facts.toolSearchCalls++;
+ toolSearchIds.add(block.id);
+ }
+ }
+ }
+ if (event.type === 'user') {
+ for (const block of event.message?.content ?? []) {
+ if (block.type === 'tool_result' && toolSearchIds.has(block.tool_use_id) && !block.is_error) {
+ if (textOf(block.content).includes(`mcp__${MCP_SERVER_NAME}__`)) facts.toolSearchSawSceneTools = true;
+ }
+ }
+ }
+ if (event.type === 'result') facts.result = event;
+ }
+ return facts;
+}
+
+export function classifyZeroCalls(output, { exitCode = 0, errors = '', toolCalls = 0 } = {}) {
+ if (toolCalls > 0) return null;
+ const facts = readRunFacts(output);
+ const stderrNote = String(errors ?? '').trim().split('\n')[0].slice(0, 120);
+ if (!facts.initialized) return { kind: 'infra', cause: 'start-up-failure', detail: stderrNote || `no init event, exit ${exitCode}` };
+ if (facts.serverStatus !== 'connected') return { kind: 'infra', cause: 'mcp-server-failed', detail: `67js server ${facts.serverStatus}` };
+ if (!facts.result) {
+ return exitCode === null || exitCode === undefined
+ ? { kind: 'infra', cause: 'timeout', detail: 'killed before a result' }
+ : { kind: 'infra', cause: 'start-up-failure', detail: stderrNote || `no result event, exit ${exitCode}` };
+ }
+ const resultText = String(facts.result.result ?? '');
+ if (facts.result.stop_reason === 'refusal' || (facts.toolSearchCalls === 0 && REFUSAL_PATTERN.test(resultText))) {
+ return { kind: 'model', cause: 'refusal', detail: resultText.slice(0, 120) };
+ }
+ if (facts.toolSearchCalls > 0 && !facts.toolSearchSawSceneTools) {
+ return { kind: 'infra', cause: 'toolsearch-empty', detail: `${facts.toolSearchCalls} ToolSearch calls returned no 67js tool` };
+ }
+ if (facts.toolSearchCalls > 0) return { kind: 'model', cause: 'toolsearch-then-quit', detail: `${facts.toolSearchCalls} ToolSearch calls, then no scene call` };
+ return { kind: 'model', cause: 'no-tool-use', detail: resultText.slice(0, 120) };
+}
diff --git a/tests/agent-bench.test.js b/tests/agent-bench.test.js
index ae75e1e8..315443a9 100644
--- a/tests/agent-bench.test.js
+++ b/tests/agent-bench.test.js
@@ -1,6 +1,8 @@
import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
import test from 'node:test';
import { benchSummary, benchTotals } from '../bench/agent-modeling/bench-summary.mjs';
+import { classifyZeroCalls } from '../bench/agent-modeling/zero-calls.mjs';
import { compareBench } from '../bench/agent-modeling/compare-bench.mjs';
import { gradeRecipe } from '../bench/agent-modeling/check.mjs';
import { REFERENCES } from '../bench/agent-modeling/references.mjs';
@@ -70,3 +72,50 @@ test('the summary groups by model and kind, and the comparison flags a pass@1 dr
assert.match(result.text, /crate: 2\/2 -> 1\/2 \(-1\)/);
assert.equal(compareBench(baseline, baseline).regressed, false);
});
+
+const readFixture = name => readFileSync(new URL(`./fixtures/agent-bench/${name}.jsonl`, import.meta.url), 'utf8');
+
+test('a run with 0 scene calls is classified by cause: infra for start-up, server and empty ToolSearch, model for the rest', () => {
+ const classify = (name, options = {}) => classifyZeroCalls(readFixture(name), { exitCode: 0, toolCalls: 0, ...options });
+ assert.equal(classify('with-scene-calls', { toolCalls: 1 }), null);
+ assert.deepEqual(classify('no-init', { exitCode: 1, errors: 'spawn failed\nmore' }), { kind: 'infra', cause: 'start-up-failure', detail: 'spawn failed' });
+ assert.equal(classify('mcp-server-failed').cause, 'mcp-server-failed');
+ assert.equal(classify('mcp-server-failed').kind, 'infra');
+ assert.equal(classify('no-result', { exitCode: 1 }).cause, 'start-up-failure');
+ assert.equal(classify('no-result', { exitCode: null }).cause, 'timeout');
+ assert.equal(classify('toolsearch-empty').cause, 'toolsearch-empty');
+ assert.equal(classify('toolsearch-empty').kind, 'infra');
+});
+
+test('a ToolSearch that showed the 67js tools and then quit, a refusal and a no-tool answer are model failures', () => {
+ const classify = name => classifyZeroCalls(readFixture(name), { exitCode: 0, toolCalls: 0 });
+ assert.deepEqual([classify('toolsearch-then-quit').kind, classify('toolsearch-then-quit').cause], ['model', 'toolsearch-then-quit']);
+ assert.deepEqual([classify('refusal').kind, classify('refusal').cause], ['model', 'refusal']);
+ assert.deepEqual([classify('no-tool-use').kind, classify('no-tool-use').cause], ['model', 'no-tool-use']);
+});
+
+test('infra runs stay out of pass@1 and are reported by cause; model zero-call runs stay counted and are labelled', () => {
+ const infraRecord = cause => record('crate', false, { toolCalls: 0, infra: true, zeroCalls: { kind: 'infra', cause } });
+ const quitRecord = record('bolt', false, { toolCalls: 0, infra: false, zeroCalls: { kind: 'model', cause: 'toolsearch-then-quit' } });
+ const totals = benchTotals([record('crate', true), infraRecord('mcp-server-failed'), infraRecord('timeout'), record('bolt', true), quitRecord], 2);
+ assert.equal(totals.runs, 3);
+ assert.equal(totals.passAt1, 0.667);
+ assert.equal(totals.infraRuns, 2);
+ assert.deepEqual(totals.infraByCause, { 'mcp-server-failed': 1, timeout: 1 });
+ assert.deepEqual(totals.modelZeroCalls, { 'toolsearch-then-quit': 1 });
+ assert.equal(totals.infraShare, 0.4);
+ assert.equal(totals.passPowK, 0.5);
+ assert.equal(benchSummary({ haiku: [record('crate', true), infraRecord('timeout')] }, 2).haiku.taskResults.crate.infra, 1);
+});
+
+test('the comparison prints infra counts next to pass@1 and flags infra above the bound', () => {
+ const infraRecord = record('crate', false, { toolCalls: 0, infra: true, zeroCalls: { kind: 'infra', cause: 'timeout' } });
+ const clean = { summary: benchSummary({ haiku: [record('crate', true), record('crate', true), record('bolt', true), record('bolt', true)] }, 2) };
+ const noisy = { summary: benchSummary({ haiku: [record('crate', true), infraRecord, record('bolt', true), record('bolt', true)] }, 2) };
+ const result = compareBench(clean, noisy);
+ assert.equal(result.regressed, false);
+ assert.equal(result.tooMuchInfra, true);
+ assert.match(result.text, /haiku: infra 0 -> 1 runs {2}INFRA over 10% of runs; pass@1 100% -> 100%/);
+ assert.match(result.text, /candidate infra runs \(not in pass@1\): 1 \(timeout 1\)/);
+ assert.equal(compareBench(clean, noisy, 0.05, 0.5).tooMuchInfra, false);
+});
diff --git a/tests/fixtures/agent-bench/mcp-server-failed.jsonl b/tests/fixtures/agent-bench/mcp-server-failed.jsonl
new file mode 100644
index 00000000..4a407e5c
--- /dev/null
+++ b/tests/fixtures/agent-bench/mcp-server-failed.jsonl
@@ -0,0 +1,3 @@
+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"failed"}]}
+{"type":"assistant","message":{"content":[{"type":"text","text":"No scene tools."}]}}
+{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"No scene tools.","usage":{"output_tokens":900}}
diff --git a/tests/fixtures/agent-bench/no-init.jsonl b/tests/fixtures/agent-bench/no-init.jsonl
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/fixtures/agent-bench/no-result.jsonl b/tests/fixtures/agent-bench/no-result.jsonl
new file mode 100644
index 00000000..1fd5d6cd
--- /dev/null
+++ b/tests/fixtures/agent-bench/no-result.jsonl
@@ -0,0 +1 @@
+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"connected"}]}
diff --git a/tests/fixtures/agent-bench/no-tool-use.jsonl b/tests/fixtures/agent-bench/no-tool-use.jsonl
new file mode 100644
index 00000000..b41d75be
--- /dev/null
+++ b/tests/fixtures/agent-bench/no-tool-use.jsonl
@@ -0,0 +1,3 @@
+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"connected"}]}
+{"type":"assistant","message":{"content":[{"type":"text","text":"Here is a description of a crate."}]}}
+{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"Here is a description of a crate.","usage":{"output_tokens":900}}
diff --git a/tests/fixtures/agent-bench/refusal.jsonl b/tests/fixtures/agent-bench/refusal.jsonl
new file mode 100644
index 00000000..c3ae8b9d
--- /dev/null
+++ b/tests/fixtures/agent-bench/refusal.jsonl
@@ -0,0 +1,3 @@
+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"connected"}]}
+{"type":"assistant","message":{"content":[{"type":"text","text":"I can't help with that request."}]}}
+{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"I can't help with that request.","usage":{"output_tokens":900},"stop_reason":"refusal"}
diff --git a/tests/fixtures/agent-bench/toolsearch-empty.jsonl b/tests/fixtures/agent-bench/toolsearch-empty.jsonl
new file mode 100644
index 00000000..07828d45
--- /dev/null
+++ b/tests/fixtures/agent-bench/toolsearch-empty.jsonl
@@ -0,0 +1,5 @@
+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"connected"}]}
+{"type":"assistant","message":{"content":[{"type":"tool_use","id":"t1","name":"ToolSearch","input":{"query":"select:mcp__67js__scene_create"}}]}}
+{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1","content":[{"type":"text","text":"No matching deferred tools found"}]}]}}
+{"type":"assistant","message":{"content":[{"type":"text","text":"The tools are not available."}]}}
+{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"The tools are not available.","usage":{"output_tokens":900}}
diff --git a/tests/fixtures/agent-bench/toolsearch-then-quit.jsonl b/tests/fixtures/agent-bench/toolsearch-then-quit.jsonl
new file mode 100644
index 00000000..79147f6b
--- /dev/null
+++ b/tests/fixtures/agent-bench/toolsearch-then-quit.jsonl
@@ -0,0 +1,5 @@
+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"connected"}]}
+{"type":"assistant","message":{"content":[{"type":"tool_use","id":"t1","name":"ToolSearch","input":{"query":"select:mcp__67js__scene_create"}}]}}
+{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1","content":[{"type":"text","text":"mcp__67js__scene_create\nmcp__67js__scene_add_part\nmcp__67js__scene_export_recipe"}]}]}}
+{"type":"assistant","message":{"content":[{"type":"text","text":"I will plan the crate."}]}}
+{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"I will plan the crate.","usage":{"output_tokens":900}}
diff --git a/tests/fixtures/agent-bench/with-scene-calls.jsonl b/tests/fixtures/agent-bench/with-scene-calls.jsonl
new file mode 100644
index 00000000..04de2df4
--- /dev/null
+++ b/tests/fixtures/agent-bench/with-scene-calls.jsonl
@@ -0,0 +1,5 @@
+{"type":"system","subtype":"init","model":"claude-haiku-4-5","tools":["ToolSearch"],"mcp_servers":[{"name":"67js","status":"connected"}]}
+{"type":"assistant","message":{"content":[{"type":"tool_use","id":"t1","name":"ToolSearch","input":{"query":"select:mcp__67js__scene_create"}}]}}
+{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1","content":[{"type":"text","text":"mcp__67js__scene_create\nmcp__67js__scene_add_part\nmcp__67js__scene_export_recipe"}]}]}}
+{"type":"assistant","message":{"content":[{"type":"tool_use","id":"c1","name":"mcp__67js__scene_create","input":{}}]}}
+{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"done","usage":{"output_tokens":900}}