3D engine that AI agents drive
Keep the bench cost cap outside the model's view (#962)
Nguyen Xuan An72bf2ef2ecf3
diff --git a/bench/agent-modeling/README.md b/bench/agent-modeling/README.mdindex ef6b18fd..7aaac231 100644--- a/bench/agent-modeling/README.md+++ b/bench/agent-modeling/README.md@@ -67,7 +67,7 @@ node bench/agent-modeling/run.mjs --claude <path to claude> --model sonnet --labEach task gets a fresh non-interactive Claude Code session in an empty temporary folder. The session has only the67js MCP server, its built-in file and shell tools are disabled, and it runs under a per-task budget (`--budget`, in-USD, default 3) and timeout (`--timeout`, in seconds, default 1200). Pick tasks with `--tasks crate,bolt`. Raw+USD, default 3) and timeout (`--timeout`, in seconds, default 1200). The budget is an estimate the bench adds up from streamed token usage and enforces itself in `run-guard.mjs`; it is never passed to claude as `--max-budget-usd`, because Claude Code shows that cap to the model and Haiku quit when it read the remainder as spent (#962). Pick tasks with `--tasks crate,bolt`. Rawtranscripts and exported recipes go to `artifacts/agent-modeling/<run>/`. The summary, with the model ID reported bythe session, goes to `results/<run>.json`. Grade one recipe by hand with`node bench/agent-modeling/check.mjs <task> <recipe.json>`.diff --git a/bench/agent-modeling/blender.mjs b/bench/agent-modeling/blender.mjsindex d7932d76..d1ca59be 100644--- a/bench/agent-modeling/blender.mjs+++ b/bench/agent-modeling/blender.mjs@@ -17,6 +17,7 @@ import { tmpdir } from 'node:os';import { join, resolve } from 'node:path';import { fileURLToPath } from 'node:url';import { parseArgs } from 'node:util';+import { createRunGuard } from './run-guard.mjs';import { removeLocalPaths, totalsByModel, totalsOf } from './aggregate.mjs';import { checkPng } from './png-check.mjs';import { gradeGeometries, readTranscript, scoreSilhouetteOfGeometries } from './threejs.mjs';@@ -89,14 +90,14 @@ function runAgent(options, prompt, workingDirectory, configPath) {'--mcp-config', configPath, '--strict-mcp-config','--allowedTools', `mcp__${MCP_SERVER_NAME}`,'--disallowedTools', ...BUILT_IN_TOOLS_DENIED,- '--permission-mode', 'dontAsk', '--setting-sources', 'project', '--no-session-persistence', '--max-budget-usd', options.budget,+ '--permission-mode', 'dontAsk', '--setting-sources', 'project', '--no-session-persistence',], { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });let output = '';- child.stdout.on('data', chunk => { output += chunk; });+ const guard = createRunGuard({ child, model: options.model, budgetUsd: options.budget, timeoutSeconds: options.timeout });+ child.stdout.on('data', chunk => { output += chunk; guard.feed(chunk); });child.stderr.on('data', () => {});- const timer = setTimeout(() => child.kill(), Number(options.timeout) * 1000);child.on('close', code => {- clearTimeout(timer);+ guard.stop();resolvePromise({ output, code, wallMs: Math.round(performance.now() - started) });});});diff --git a/bench/agent-modeling/llms-only.mjs b/bench/agent-modeling/llms-only.mjsindex 3109a743..b1a01346 100644--- a/bench/agent-modeling/llms-only.mjs+++ b/bench/agent-modeling/llms-only.mjs@@ -14,6 +14,7 @@ import { join, resolve } from 'node:path';import { parseArgs } from 'node:util';import { removeLocalPaths, totalsByModel } from './aggregate.mjs';import { checkPng } from './png-check.mjs';+import { createRunGuard } from './run-guard.mjs';const repository = resolve(import.meta.dirname, '../..');const TASK_ID = 'llms-only-png';@@ -63,7 +64,6 @@ function claudeArguments() {'--permission-mode', 'dontAsk','--setting-sources', 'project','--no-session-persistence',- '--max-budget-usd', options.budget,];}@@ -87,15 +87,16 @@ function runAgent(workingDirectory, settingsPath) {const child = spawn(executable, argumentsList, { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });let output = '';let errors = '';+ const guard = createRunGuard({ child, model: options.model, budgetUsd: options.budget, timeoutSeconds: options.timeout });child.stdout.on('data', chunk => {output += chunk;+ guard.feed(chunk);});child.stderr.on('data', chunk => {errors += chunk;});- const timer = setTimeout(() => child.kill(), Number(options.timeout) * 1000);child.on('close', code => {- clearTimeout(timer);+ guard.stop();resolvePromise({ output, errors, code, wallMs: Math.round(performance.now() - started) });});});diff --git a/bench/agent-modeling/results/bench-guard-962-haiku.json b/bench/agent-modeling/results/bench-guard-962-haiku.jsonnew file mode 100644index 00000000..4c4bc95e--- /dev/null+++ b/bench/agent-modeling/results/bench-guard-962-haiku.json@@ -0,0 +1,224 @@+{+ "run": "bench-guard-962-haiku",+ "model": "claude-haiku-5-5",+ "date": "2026-11-12T08:02:22.448Z",+ "skill": false,+ "branches": "on",+ "relations": "on",+ "runsPerTask": 3,+ "passed": 2,+ "tasks": 3,+ "toolCalls": 29,+ "costUsd": 0.04,+ "byModel": {+ "claude-haiku-5-5": {+ "runs": 3,+ "passed": 2,+ "successRate": 0.667,+ "toolCalls": 29,+ "toolErrors": 0,+ "inputTokens": 1231838,+ "outputTokens": 10409,+ "costUsd": 0.04,+ "tasks": {+ "crate": {+ "runs": 1,+ "passed": 1,+ "successRate": 1,+ "toolCalls": 17,+ "toolErrors": 0,+ "inputTokens": 602344,+ "outputTokens": 5221,+ "costUsd": 0.02+ },+ "bolt": {+ "runs": 1,+ "passed": 0,+ "successRate": 0,+ "toolCalls": 6,+ "toolErrors": 0,+ "inputTokens": 339360,+ "outputTokens": 2843,+ "costUsd": 0.01+ },+ "table": {+ "runs": 1,+ "passed": 1,+ "successRate": 1,+ "toolCalls": 6,+ "toolErrors": 0,+ "inputTokens": 290134,+ "outputTokens": 2345,+ "costUsd": 0.01+ }+ }+ }+ },+ "records": [+ {+ "task": "crate",+ "attempt": 1,+ "pass": true,+ "passed": 8,+ "total": 8,+ "failed": [],+ "model": "claude-haiku-5-5",+ "toolCalls": 17,+ "toolErrors": 0,+ "toolsByName": {+ "ToolSearch": 1,+ "scene_capabilities": 12,+ "scene_create": 1,+ "mesh_inspect": 1,+ "scene_look": 1,+ "scene_export_recipe": 1,+ "scene_dispose": 1+ },+ "branchCalls": 0,+ "relationCalls": 0,+ "pathQueries": 0,+ "features": {+ "geometryTypes": [+ "mesh",+ "box"+ ],+ "meshOperations": [+ "inset_individual"+ ],+ "motionIk": [],+ "motionTracks": 0+ },+ "look": {+ "silhouette": {+ "views": {+ "front": 1,+ "side": 1,+ "three-quarter": 1+ },+ "mean": 1+ },+ "images": [+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/crate-front.png",+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/crate-side.png",+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/crate-three-quarter.png"+ ]+ },+ "turns": 19,+ "costUsd": 0.01712702,+ "inputTokens": 602344,+ "outputTokens": 5221,+ "wallSeconds": 66,+ "exitCode": 0,+ "stopped": "success"+ },+ {+ "task": "bolt",+ "attempt": 1,+ "pass": false,+ "passed": 5,+ "total": 7,+ "failed": [+ "extent: 0.1155, 0.1",+ "volume: 0.0005103 m3"+ ],+ "model": "claude-haiku-5-5",+ "toolCalls": 6,+ "toolErrors": 0,+ "toolsByName": {+ "ToolSearch": 1,+ "scene_capabilities": 1,+ "scene_create": 1,+ "mesh_inspect": 1,+ "scene_inspect": 1,+ "scene_export_recipe": 1,+ "scene_dispose": 1+ },+ "branchCalls": 0,+ "relationCalls": 0,+ "pathQueries": 0,+ "features": {+ "geometryTypes": [+ "cylinder"+ ],+ "meshOperations": [],+ "motionIk": [],+ "motionTracks": 0+ },+ "look": {+ "silhouette": {+ "views": {+ "front": 0.5016,+ "side": 0.4984,+ "three-quarter": 0.4377+ },+ "mean": 0.4792+ },+ "images": [+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/bolt-front.png",+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/bolt-side.png",+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/bolt-three-quarter.png"+ ]+ },+ "turns": 8,+ "costUsd": 0.011502080000000001,+ "inputTokens": 339360,+ "outputTokens": 2843,+ "wallSeconds": 67,+ "exitCode": 0,+ "stopped": "success"+ },+ {+ "task": "table",+ "attempt": 1,+ "pass": true,+ "passed": 6,+ "total": 6,+ "failed": [],+ "model": "claude-haiku-5-5",+ "toolCalls": 6,+ "toolErrors": 0,+ "toolsByName": {+ "ToolSearch": 1,+ "scene_create": 1,+ "mesh_inspect": 1,+ "scene_inspect": 1,+ "scene_look": 1,+ "scene_export_recipe": 1,+ "scene_dispose": 1+ },+ "branchCalls": 0,+ "relationCalls": 0,+ "pathQueries": 0,+ "features": {+ "geometryTypes": [+ "box"+ ],+ "meshOperations": [],+ "motionIk": [],+ "motionTracks": 0+ },+ "look": {+ "silhouette": {+ "views": {+ "front": 1,+ "side": 0.1684,+ "three-quarter": 0.6073+ },+ "mean": 0.5919+ },+ "images": [+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/table-front.png",+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/table-side.png",+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/table-three-quarter.png"+ ]+ },+ "turns": 8,+ "costUsd": 0.01081859,+ "inputTokens": 290134,+ "outputTokens": 2345,+ "wallSeconds": 42,+ "exitCode": 0,+ "stopped": "success"+ }+ ]+}diff --git a/bench/agent-modeling/run-guard.mjs b/bench/agent-modeling/run-guard.mjsnew file mode 100644index 00000000..f1c43e3c--- /dev/null+++ b/bench/agent-modeling/run-guard.mjs@@ -0,0 +1,85 @@+// Runaway guard for `claude -p` bench runs.+//+// The bench used to pass `--max-budget-usd` to claude. Claude Code shows that budget to the model as a+// running "remaining" figure, and Haiku read a small remainder as a reason to stop before it started (#962).+// The cap now lives here, outside the model's view: the bench adds up the token cost of the streamed+// messages itself and kills the run when the estimate passes the cap, or when the time limit is reached.++// Dollars per million tokens, deliberately on the high side so the guard fires early rather than late.+const RATES_PER_MILLION_TOKENS = {+ haiku: { input: 1, output: 5 },+ sonnet: { input: 3, output: 15 },+ opus: { input: 5, output: 25 },+};+const CACHE_WRITE_MULTIPLIER = 1.25;+const CACHE_READ_MULTIPLIER = 0.1;++export function ratesForModel(model) {+ const name = String(model ?? '').toLowerCase();+ if (name.includes('haiku')) return RATES_PER_MILLION_TOKENS.haiku;+ if (name.includes('sonnet')) return RATES_PER_MILLION_TOKENS.sonnet;+ return RATES_PER_MILLION_TOKENS.opus;+}++export function estimateUsageCostUsd(usage, model) {+ const rates = ratesForModel(model);+ const inputTokens = usage?.input_tokens ?? 0;+ const outputTokens = usage?.output_tokens ?? 0;+ const cacheWriteTokens = usage?.cache_creation_input_tokens ?? 0;+ const cacheReadTokens = usage?.cache_read_input_tokens ?? 0;+ const inputTokenEquivalents = inputTokens + cacheWriteTokens * CACHE_WRITE_MULTIPLIER + cacheReadTokens * CACHE_READ_MULTIPLIER;+ return (inputTokenEquivalents * rates.input + outputTokens * rates.output) / 1e6;+}++/**+ * Watches a claude child process. Feed it each stdout chunk; it kills the child when the estimated cost+ * passes `budgetUsd` or when `timeoutSeconds` runs out. Call `stop()` once the child has closed.+ */+export function createRunGuard({ child, model, budgetUsd, timeoutSeconds }) {+ const costByMessageId = new Map();+ let pendingText = '';+ let trippedReason = null;++ const trip = reason => {+ if (trippedReason) return;+ trippedReason = reason;+ child.kill();+ };++ const timer = setTimeout(() => trip('timeout'), Number(timeoutSeconds) * 1000);++ const totalCostUsd = () => [...costByMessageId.values()].reduce((sum, cost) => sum + cost, 0);++ const readLine = line => {+ if (!line.startsWith('{')) return;+ let event;+ try {+ event = JSON.parse(line);+ } catch {+ return;+ }+ if (event.type !== 'assistant' || !event.message?.usage) return;+ // One message is streamed as several events that repeat its usage, so each message id counts once.+ const messageId = event.message.id ?? `line-${costByMessageId.size}`;+ costByMessageId.set(messageId, estimateUsageCostUsd(event.message.usage, event.message.model ?? model));+ if (totalCostUsd() > Number(budgetUsd)) trip('budget');+ };++ return {+ feed(chunk) {+ pendingText += chunk;+ const lines = pendingText.split('\n');+ pendingText = lines.pop();+ for (const line of lines) readLine(line.trim());+ },+ stop() {+ clearTimeout(timer);+ },+ get reason() {+ return trippedReason;+ },+ get estimatedCostUsd() {+ return totalCostUsd();+ },+ };+}diff --git a/bench/agent-modeling/run.mjs b/bench/agent-modeling/run.mjsindex 095b39f3..ce8c498e 100644--- a/bench/agent-modeling/run.mjs+++ b/bench/agent-modeling/run.mjs@@ -21,6 +21,7 @@ import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promiimport { tmpdir } from 'node:os';import { join, resolve } from 'node:path';import { parseArgs } from 'node:util';+import { createRunGuard } from './run-guard.mjs';import { removeLocalPaths, totalsByModel, totalsOf } from './aggregate.mjs';import { gradeRecipe, recipeFeatures } from './check.mjs';import { scoreLookSafely } from './look.mjs';@@ -78,7 +79,6 @@ function runClaude(prompt, workingDirectory, configPath, systemPromptAddition) {'--permission-mode', 'dontAsk','--setting-sources', 'project','--no-session-persistence',- '--max-budget-usd', options.budget,...(systemPromptAddition ? ['--append-system-prompt', systemPromptAddition] : []),];return new Promise(resolvePromise => {@@ -86,16 +86,17 @@ function runClaude(prompt, workingDirectory, configPath, systemPromptAddition) {const child = spawn(options.claude, argumentsList, { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });let output = '';let errors = '';+ const guard = createRunGuard({ child, model: options.model, budgetUsd: options.budget, timeoutSeconds: options.timeout });child.stdout.on('data', chunk => {output += chunk;+ guard.feed(chunk);});child.stderr.on('data', chunk => {errors += chunk;});- const timer = setTimeout(() => child.kill(), Number(options.timeout) * 1000);child.on('close', code => {- clearTimeout(timer);- resolvePromise({ output, errors, code, wallMs: Math.round(performance.now() - started) });+ guard.stop();+ resolvePromise({ output, errors, code, guardReason: guard.reason, wallMs: Math.round(performance.now() - started) });});});}@@ -204,7 +205,7 @@ async function runTask(task, attempt, systemPromptAddition) {outputTokens: usage.output_tokens ?? 0,wallSeconds: Math.round(run.wallMs / 1000),exitCode: run.code,- stopped: transcript.result?.subtype ?? 'no result',+ stopped: transcript.result?.subtype ?? (run.guardReason ? `guard-${run.guardReason}` : 'no result'),};} finally {await removeWorkingDirectory(workingDirectory);diff --git a/bench/agent-modeling/threejs.mjs b/bench/agent-modeling/threejs.mjsindex f7586e58..ff0a376c 100644--- a/bench/agent-modeling/threejs.mjs+++ b/bench/agent-modeling/threejs.mjs@@ -17,6 +17,7 @@ import { tmpdir } from 'node:os';import { join, resolve } from 'node:path';import { fileURLToPath } from 'node:url';import { parseArgs } from 'node:util';+import { createRunGuard } from './run-guard.mjs';import { removeLocalPaths, totalsByModel, totalsOf } from './aggregate.mjs';import { checkOne } from './check.mjs';import { camerasForTask, worldGeometriesOf } from './look.mjs';@@ -201,15 +202,15 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {'-p', prompt, '--model', options.model, '--output-format', 'stream-json', '--verbose', '--strict-mcp-config','--allowedTools', 'Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep','--disallowedTools', ...BUILT_IN_TOOLS_DENIED,- '--permission-mode', 'dontAsk', '--setting-sources', 'project', '--no-session-persistence', '--max-budget-usd', options.budget,+ '--permission-mode', 'dontAsk', '--setting-sources', 'project', '--no-session-persistence',], { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });let output = '';let errors = '';- child.stdout.on('data', chunk => { output += chunk; });+ const guard = createRunGuard({ child, model: options.model, budgetUsd: options.budget, timeoutSeconds: options.timeout });+ child.stdout.on('data', chunk => { output += chunk; guard.feed(chunk); });child.stderr.on('data', chunk => { errors += chunk; });- const timer = setTimeout(() => child.kill(), Number(options.timeout) * 1000);child.on('close', code => {- clearTimeout(timer);+ guard.stop();resolvePromise({ output, errors, code, wallMs: Math.round(performance.now() - started) });});});diff --git a/changes/962-bench-budget-cap.md b/changes/962-bench-budget-cap.mdnew file mode 100644index 00000000..9e8c3293--- /dev/null+++ b/changes/962-bench-budget-cap.md@@ -0,0 +1 @@+- The agent bench no longer passes `--max-budget-usd` to `claude -p`, which Haiku read as a nearly spent budget and quit on; `bench/agent-modeling/run-guard.mjs` now adds up streamed token cost and kills a runaway run by cost or time (#962).diff --git a/tests/agent-bench.test.js b/tests/agent-bench.test.jsindex ae75e1e8..14e02de0 100644--- a/tests/agent-bench.test.js+++ b/tests/agent-bench.test.js@@ -70,3 +70,29 @@ test('the summary groups by model and kind, and the comparison flags a pass@1 drassert.match(result.text, /crate: 2\/2 -> 1\/2 \(-1\)/);assert.equal(compareBench(baseline, baseline).regressed, false);});++test('the run guard counts each streamed message once and kills a run over its budget', async () => {+ const { createRunGuard, estimateUsageCostUsd } = await import('../bench/agent-modeling/run-guard.mjs');+ assert.equal(estimateUsageCostUsd({ input_tokens: 1e6, output_tokens: 1e6 }, 'claude-haiku-4-5'), 6);+ assert.equal(estimateUsageCostUsd({ cache_read_input_tokens: 1e6 }, 'sonnet'), 0.3);+ let kills = 0;+ const child = { kill: () => { kills++; } };+ const guard = createRunGuard({ child, model: 'haiku', budgetUsd: 0.9, timeoutSeconds: 60 });+ const event = JSON.stringify({ type: 'assistant', message: { id: 'a', usage: { output_tokens: 100000 } } });+ guard.feed(`${event}\n${event}\n{"type":"assis`);+ assert.equal(kills, 0, 'the same message id repeated is one message');+ assert.equal(guard.estimatedCostUsd, 0.5);+ guard.feed(`tant","message":{"id":"b","usage":{"output_tokens":100000}}}\n`);+ assert.equal(kills, 1);+ assert.equal(guard.reason, 'budget');+ guard.stop();+});++test('the run guard kills a run that reaches its time limit', async () => {+ const { createRunGuard } = await import('../bench/agent-modeling/run-guard.mjs');+ let kills = 0;+ const guard = createRunGuard({ child: { kill: () => { kills++; } }, model: 'haiku', budgetUsd: 1, timeoutSeconds: 0.01 });+ await new Promise(resolveWait => setTimeout(resolveWait, 60));+ assert.equal(kills, 1);+ assert.equal(guard.reason, 'timeout');+});