Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Keep the bench cost cap outside the model's view (#962)

Nguyen Xuan An72bf2ef2ecf3
diff --git a/bench/agent-modeling/README.md b/bench/agent-modeling/README.md
index ef6b18fd..7aaac231 100644
--- a/bench/agent-modeling/README.md
+++ b/bench/agent-modeling/README.md
@@ -67,7 +67,7 @@ node bench/agent-modeling/run.mjs --claude <path to claude> --model sonnet --lab
Each task gets a fresh non-interactive Claude Code session in an empty temporary folder. The session has only the
67js MCP server, its built-in file and shell tools are disabled, and it runs under a per-task budget (`--budget`, in
-USD, default 3) and timeout (`--timeout`, in seconds, default 1200). Pick tasks with `--tasks crate,bolt`. Raw
+USD, default 3) and timeout (`--timeout`, in seconds, default 1200). The budget is an estimate the bench adds up from streamed token usage and enforces itself in `run-guard.mjs`; it is never passed to claude as `--max-budget-usd`, because Claude Code shows that cap to the model and Haiku quit when it read the remainder as spent (#962). Pick tasks with `--tasks crate,bolt`. Raw
transcripts and exported recipes go to `artifacts/agent-modeling/<run>/`. The summary, with the model ID reported by
the session, goes to `results/<run>.json`. Grade one recipe by hand with
`node bench/agent-modeling/check.mjs <task> <recipe.json>`.
diff --git a/bench/agent-modeling/blender.mjs b/bench/agent-modeling/blender.mjs
index d7932d76..d1ca59be 100644
--- a/bench/agent-modeling/blender.mjs
+++ b/bench/agent-modeling/blender.mjs
@@ -17,6 +17,7 @@ import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { parseArgs } from 'node:util';
+import { createRunGuard } from './run-guard.mjs';
import { removeLocalPaths, totalsByModel, totalsOf } from './aggregate.mjs';
import { checkPng } from './png-check.mjs';
import { gradeGeometries, readTranscript, scoreSilhouetteOfGeometries } from './threejs.mjs';
@@ -89,14 +90,14 @@ function runAgent(options, prompt, workingDirectory, configPath) {
'--mcp-config', configPath, '--strict-mcp-config',
'--allowedTools', `mcp__${MCP_SERVER_NAME}`,
'--disallowedTools', ...BUILT_IN_TOOLS_DENIED,
- '--permission-mode', 'dontAsk', '--setting-sources', 'project', '--no-session-persistence', '--max-budget-usd', options.budget,
+ '--permission-mode', 'dontAsk', '--setting-sources', 'project', '--no-session-persistence',
], { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let output = '';
- child.stdout.on('data', chunk => { output += chunk; });
+ const guard = createRunGuard({ child, model: options.model, budgetUsd: options.budget, timeoutSeconds: options.timeout });
+ child.stdout.on('data', chunk => { output += chunk; guard.feed(chunk); });
child.stderr.on('data', () => {});
- const timer = setTimeout(() => child.kill(), Number(options.timeout) * 1000);
child.on('close', code => {
- clearTimeout(timer);
+ guard.stop();
resolvePromise({ output, code, wallMs: Math.round(performance.now() - started) });
});
});
diff --git a/bench/agent-modeling/llms-only.mjs b/bench/agent-modeling/llms-only.mjs
index 3109a743..b1a01346 100644
--- a/bench/agent-modeling/llms-only.mjs
+++ b/bench/agent-modeling/llms-only.mjs
@@ -14,6 +14,7 @@ import { join, resolve } from 'node:path';
import { parseArgs } from 'node:util';
import { removeLocalPaths, totalsByModel } from './aggregate.mjs';
import { checkPng } from './png-check.mjs';
+import { createRunGuard } from './run-guard.mjs';
const repository = resolve(import.meta.dirname, '../..');
const TASK_ID = 'llms-only-png';
@@ -63,7 +64,6 @@ function claudeArguments() {
'--permission-mode', 'dontAsk',
'--setting-sources', 'project',
'--no-session-persistence',
- '--max-budget-usd', options.budget,
];
}
@@ -87,15 +87,16 @@ function runAgent(workingDirectory, settingsPath) {
const child = spawn(executable, argumentsList, { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let output = '';
let errors = '';
+ const guard = createRunGuard({ child, model: options.model, budgetUsd: options.budget, timeoutSeconds: options.timeout });
child.stdout.on('data', chunk => {
output += chunk;
+ guard.feed(chunk);
});
child.stderr.on('data', chunk => {
errors += chunk;
});
- const timer = setTimeout(() => child.kill(), Number(options.timeout) * 1000);
child.on('close', code => {
- clearTimeout(timer);
+ guard.stop();
resolvePromise({ output, errors, code, wallMs: Math.round(performance.now() - started) });
});
});
diff --git a/bench/agent-modeling/results/bench-guard-962-haiku.json b/bench/agent-modeling/results/bench-guard-962-haiku.json
new file mode 100644
index 00000000..4c4bc95e
--- /dev/null
+++ b/bench/agent-modeling/results/bench-guard-962-haiku.json
@@ -0,0 +1,224 @@
+{
+ "run": "bench-guard-962-haiku",
+ "model": "claude-haiku-5-5",
+ "date": "2026-11-12T08:02:22.448Z",
+ "skill": false,
+ "branches": "on",
+ "relations": "on",
+ "runsPerTask": 3,
+ "passed": 2,
+ "tasks": 3,
+ "toolCalls": 29,
+ "costUsd": 0.04,
+ "byModel": {
+ "claude-haiku-5-5": {
+ "runs": 3,
+ "passed": 2,
+ "successRate": 0.667,
+ "toolCalls": 29,
+ "toolErrors": 0,
+ "inputTokens": 1231838,
+ "outputTokens": 10409,
+ "costUsd": 0.04,
+ "tasks": {
+ "crate": {
+ "runs": 1,
+ "passed": 1,
+ "successRate": 1,
+ "toolCalls": 17,
+ "toolErrors": 0,
+ "inputTokens": 602344,
+ "outputTokens": 5221,
+ "costUsd": 0.02
+ },
+ "bolt": {
+ "runs": 1,
+ "passed": 0,
+ "successRate": 0,
+ "toolCalls": 6,
+ "toolErrors": 0,
+ "inputTokens": 339360,
+ "outputTokens": 2843,
+ "costUsd": 0.01
+ },
+ "table": {
+ "runs": 1,
+ "passed": 1,
+ "successRate": 1,
+ "toolCalls": 6,
+ "toolErrors": 0,
+ "inputTokens": 290134,
+ "outputTokens": 2345,
+ "costUsd": 0.01
+ }
+ }
+ }
+ },
+ "records": [
+ {
+ "task": "crate",
+ "attempt": 1,
+ "pass": true,
+ "passed": 8,
+ "total": 8,
+ "failed": [],
+ "model": "claude-haiku-5-5",
+ "toolCalls": 17,
+ "toolErrors": 0,
+ "toolsByName": {
+ "ToolSearch": 1,
+ "scene_capabilities": 12,
+ "scene_create": 1,
+ "mesh_inspect": 1,
+ "scene_look": 1,
+ "scene_export_recipe": 1,
+ "scene_dispose": 1
+ },
+ "branchCalls": 0,
+ "relationCalls": 0,
+ "pathQueries": 0,
+ "features": {
+ "geometryTypes": [
+ "mesh",
+ "box"
+ ],
+ "meshOperations": [
+ "inset_individual"
+ ],
+ "motionIk": [],
+ "motionTracks": 0
+ },
+ "look": {
+ "silhouette": {
+ "views": {
+ "front": 1,
+ "side": 1,
+ "three-quarter": 1
+ },
+ "mean": 1
+ },
+ "images": [
+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/crate-front.png",
+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/crate-side.png",
+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/crate-three-quarter.png"
+ ]
+ },
+ "turns": 19,
+ "costUsd": 0.01712702,
+ "inputTokens": 602344,
+ "outputTokens": 5221,
+ "wallSeconds": 66,
+ "exitCode": 0,
+ "stopped": "success"
+ },
+ {
+ "task": "bolt",
+ "attempt": 1,
+ "pass": false,
+ "passed": 5,
+ "total": 7,
+ "failed": [
+ "extent: 0.1155, 0.1",
+ "volume: 0.0005103 m3"
+ ],
+ "model": "claude-haiku-5-5",
+ "toolCalls": 6,
+ "toolErrors": 0,
+ "toolsByName": {
+ "ToolSearch": 1,
+ "scene_capabilities": 1,
+ "scene_create": 1,
+ "mesh_inspect": 1,
+ "scene_inspect": 1,
+ "scene_export_recipe": 1,
+ "scene_dispose": 1
+ },
+ "branchCalls": 0,
+ "relationCalls": 0,
+ "pathQueries": 0,
+ "features": {
+ "geometryTypes": [
+ "cylinder"
+ ],
+ "meshOperations": [],
+ "motionIk": [],
+ "motionTracks": 0
+ },
+ "look": {
+ "silhouette": {
+ "views": {
+ "front": 0.5016,
+ "side": 0.4984,
+ "three-quarter": 0.4377
+ },
+ "mean": 0.4792
+ },
+ "images": [
+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/bolt-front.png",
+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/bolt-side.png",
+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/bolt-three-quarter.png"
+ ]
+ },
+ "turns": 8,
+ "costUsd": 0.011502080000000001,
+ "inputTokens": 339360,
+ "outputTokens": 2843,
+ "wallSeconds": 67,
+ "exitCode": 0,
+ "stopped": "success"
+ },
+ {
+ "task": "table",
+ "attempt": 1,
+ "pass": true,
+ "passed": 6,
+ "total": 6,
+ "failed": [],
+ "model": "claude-haiku-5-5",
+ "toolCalls": 6,
+ "toolErrors": 0,
+ "toolsByName": {
+ "ToolSearch": 1,
+ "scene_create": 1,
+ "mesh_inspect": 1,
+ "scene_inspect": 1,
+ "scene_look": 1,
+ "scene_export_recipe": 1,
+ "scene_dispose": 1
+ },
+ "branchCalls": 0,
+ "relationCalls": 0,
+ "pathQueries": 0,
+ "features": {
+ "geometryTypes": [
+ "box"
+ ],
+ "meshOperations": [],
+ "motionIk": [],
+ "motionTracks": 0
+ },
+ "look": {
+ "silhouette": {
+ "views": {
+ "front": 1,
+ "side": 0.1684,
+ "three-quarter": 0.6073
+ },
+ "mean": 0.5919
+ },
+ "images": [
+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/table-front.png",
+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/table-side.png",
+ "artifacts/agent-modeling/bench-guard-962-haiku/look/1/table-three-quarter.png"
+ ]
+ },
+ "turns": 8,
+ "costUsd": 0.01081859,
+ "inputTokens": 290134,
+ "outputTokens": 2345,
+ "wallSeconds": 42,
+ "exitCode": 0,
+ "stopped": "success"
+ }
+ ]
+}
diff --git a/bench/agent-modeling/run-guard.mjs b/bench/agent-modeling/run-guard.mjs
new file mode 100644
index 00000000..f1c43e3c
--- /dev/null
+++ b/bench/agent-modeling/run-guard.mjs
@@ -0,0 +1,85 @@
+// Runaway guard for `claude -p` bench runs.
+//
+// The bench used to pass `--max-budget-usd` to claude. Claude Code shows that budget to the model as a
+// running "remaining" figure, and Haiku read a small remainder as a reason to stop before it started (#962).
+// The cap now lives here, outside the model's view: the bench adds up the token cost of the streamed
+// messages itself and kills the run when the estimate passes the cap, or when the time limit is reached.
+
+// Dollars per million tokens, deliberately on the high side so the guard fires early rather than late.
+const RATES_PER_MILLION_TOKENS = {
+ haiku: { input: 1, output: 5 },
+ sonnet: { input: 3, output: 15 },
+ opus: { input: 5, output: 25 },
+};
+const CACHE_WRITE_MULTIPLIER = 1.25;
+const CACHE_READ_MULTIPLIER = 0.1;
+
+export function ratesForModel(model) {
+ const name = String(model ?? '').toLowerCase();
+ if (name.includes('haiku')) return RATES_PER_MILLION_TOKENS.haiku;
+ if (name.includes('sonnet')) return RATES_PER_MILLION_TOKENS.sonnet;
+ return RATES_PER_MILLION_TOKENS.opus;
+}
+
+export function estimateUsageCostUsd(usage, model) {
+ const rates = ratesForModel(model);
+ const inputTokens = usage?.input_tokens ?? 0;
+ const outputTokens = usage?.output_tokens ?? 0;
+ const cacheWriteTokens = usage?.cache_creation_input_tokens ?? 0;
+ const cacheReadTokens = usage?.cache_read_input_tokens ?? 0;
+ const inputTokenEquivalents = inputTokens + cacheWriteTokens * CACHE_WRITE_MULTIPLIER + cacheReadTokens * CACHE_READ_MULTIPLIER;
+ return (inputTokenEquivalents * rates.input + outputTokens * rates.output) / 1e6;
+}
+
+/**
+ * Watches a claude child process. Feed it each stdout chunk; it kills the child when the estimated cost
+ * passes `budgetUsd` or when `timeoutSeconds` runs out. Call `stop()` once the child has closed.
+ */
+export function createRunGuard({ child, model, budgetUsd, timeoutSeconds }) {
+ const costByMessageId = new Map();
+ let pendingText = '';
+ let trippedReason = null;
+
+ const trip = reason => {
+ if (trippedReason) return;
+ trippedReason = reason;
+ child.kill();
+ };
+
+ const timer = setTimeout(() => trip('timeout'), Number(timeoutSeconds) * 1000);
+
+ const totalCostUsd = () => [...costByMessageId.values()].reduce((sum, cost) => sum + cost, 0);
+
+ const readLine = line => {
+ if (!line.startsWith('{')) return;
+ let event;
+ try {
+ event = JSON.parse(line);
+ } catch {
+ return;
+ }
+ if (event.type !== 'assistant' || !event.message?.usage) return;
+ // One message is streamed as several events that repeat its usage, so each message id counts once.
+ const messageId = event.message.id ?? `line-${costByMessageId.size}`;
+ costByMessageId.set(messageId, estimateUsageCostUsd(event.message.usage, event.message.model ?? model));
+ if (totalCostUsd() > Number(budgetUsd)) trip('budget');
+ };
+
+ return {
+ feed(chunk) {
+ pendingText += chunk;
+ const lines = pendingText.split('\n');
+ pendingText = lines.pop();
+ for (const line of lines) readLine(line.trim());
+ },
+ stop() {
+ clearTimeout(timer);
+ },
+ get reason() {
+ return trippedReason;
+ },
+ get estimatedCostUsd() {
+ return totalCostUsd();
+ },
+ };
+}
diff --git a/bench/agent-modeling/run.mjs b/bench/agent-modeling/run.mjs
index 095b39f3..ce8c498e 100644
--- a/bench/agent-modeling/run.mjs
+++ b/bench/agent-modeling/run.mjs
@@ -21,6 +21,7 @@ import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promi
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { parseArgs } from 'node:util';
+import { createRunGuard } from './run-guard.mjs';
import { removeLocalPaths, totalsByModel, totalsOf } from './aggregate.mjs';
import { gradeRecipe, recipeFeatures } from './check.mjs';
import { scoreLookSafely } from './look.mjs';
@@ -78,7 +79,6 @@ function runClaude(prompt, workingDirectory, configPath, systemPromptAddition) {
'--permission-mode', 'dontAsk',
'--setting-sources', 'project',
'--no-session-persistence',
- '--max-budget-usd', options.budget,
...(systemPromptAddition ? ['--append-system-prompt', systemPromptAddition] : []),
];
return new Promise(resolvePromise => {
@@ -86,16 +86,17 @@ function runClaude(prompt, workingDirectory, configPath, systemPromptAddition) {
const child = spawn(options.claude, argumentsList, { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let output = '';
let errors = '';
+ const guard = createRunGuard({ child, model: options.model, budgetUsd: options.budget, timeoutSeconds: options.timeout });
child.stdout.on('data', chunk => {
output += chunk;
+ guard.feed(chunk);
});
child.stderr.on('data', chunk => {
errors += chunk;
});
- const timer = setTimeout(() => child.kill(), Number(options.timeout) * 1000);
child.on('close', code => {
- clearTimeout(timer);
- resolvePromise({ output, errors, code, wallMs: Math.round(performance.now() - started) });
+ guard.stop();
+ resolvePromise({ output, errors, code, guardReason: guard.reason, wallMs: Math.round(performance.now() - started) });
});
});
}
@@ -204,7 +205,7 @@ async function runTask(task, attempt, systemPromptAddition) {
outputTokens: usage.output_tokens ?? 0,
wallSeconds: Math.round(run.wallMs / 1000),
exitCode: run.code,
- stopped: transcript.result?.subtype ?? 'no result',
+ stopped: transcript.result?.subtype ?? (run.guardReason ? `guard-${run.guardReason}` : 'no result'),
};
} finally {
await removeWorkingDirectory(workingDirectory);
diff --git a/bench/agent-modeling/threejs.mjs b/bench/agent-modeling/threejs.mjs
index f7586e58..ff0a376c 100644
--- a/bench/agent-modeling/threejs.mjs
+++ b/bench/agent-modeling/threejs.mjs
@@ -17,6 +17,7 @@ import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { parseArgs } from 'node:util';
+import { createRunGuard } from './run-guard.mjs';
import { removeLocalPaths, totalsByModel, totalsOf } from './aggregate.mjs';
import { checkOne } from './check.mjs';
import { camerasForTask, worldGeometriesOf } from './look.mjs';
@@ -201,15 +202,15 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
'-p', prompt, '--model', options.model, '--output-format', 'stream-json', '--verbose', '--strict-mcp-config',
'--allowedTools', 'Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep',
'--disallowedTools', ...BUILT_IN_TOOLS_DENIED,
- '--permission-mode', 'dontAsk', '--setting-sources', 'project', '--no-session-persistence', '--max-budget-usd', options.budget,
+ '--permission-mode', 'dontAsk', '--setting-sources', 'project', '--no-session-persistence',
], { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let output = '';
let errors = '';
- child.stdout.on('data', chunk => { output += chunk; });
+ const guard = createRunGuard({ child, model: options.model, budgetUsd: options.budget, timeoutSeconds: options.timeout });
+ child.stdout.on('data', chunk => { output += chunk; guard.feed(chunk); });
child.stderr.on('data', chunk => { errors += chunk; });
- const timer = setTimeout(() => child.kill(), Number(options.timeout) * 1000);
child.on('close', code => {
- clearTimeout(timer);
+ guard.stop();
resolvePromise({ output, errors, code, wallMs: Math.round(performance.now() - started) });
});
});
diff --git a/changes/962-bench-budget-cap.md b/changes/962-bench-budget-cap.md
new file mode 100644
index 00000000..9e8c3293
--- /dev/null
+++ b/changes/962-bench-budget-cap.md
@@ -0,0 +1 @@
+- The agent bench no longer passes `--max-budget-usd` to `claude -p`, which Haiku read as a nearly spent budget and quit on; `bench/agent-modeling/run-guard.mjs` now adds up streamed token cost and kills a runaway run by cost or time (#962).
diff --git a/tests/agent-bench.test.js b/tests/agent-bench.test.js
index ae75e1e8..14e02de0 100644
--- a/tests/agent-bench.test.js
+++ b/tests/agent-bench.test.js
@@ -70,3 +70,29 @@ test('the summary groups by model and kind, and the comparison flags a pass@1 dr
assert.match(result.text, /crate: 2\/2 -> 1\/2 \(-1\)/);
assert.equal(compareBench(baseline, baseline).regressed, false);
});
+
+test('the run guard counts each streamed message once and kills a run over its budget', async () => {
+ const { createRunGuard, estimateUsageCostUsd } = await import('../bench/agent-modeling/run-guard.mjs');
+ assert.equal(estimateUsageCostUsd({ input_tokens: 1e6, output_tokens: 1e6 }, 'claude-haiku-4-5'), 6);
+ assert.equal(estimateUsageCostUsd({ cache_read_input_tokens: 1e6 }, 'sonnet'), 0.3);
+ let kills = 0;
+ const child = { kill: () => { kills++; } };
+ const guard = createRunGuard({ child, model: 'haiku', budgetUsd: 0.9, timeoutSeconds: 60 });
+ const event = JSON.stringify({ type: 'assistant', message: { id: 'a', usage: { output_tokens: 100000 } } });
+ guard.feed(`${event}\n${event}\n{"type":"assis`);
+ assert.equal(kills, 0, 'the same message id repeated is one message');
+ assert.equal(guard.estimatedCostUsd, 0.5);
+ guard.feed(`tant","message":{"id":"b","usage":{"output_tokens":100000}}}\n`);
+ assert.equal(kills, 1);
+ assert.equal(guard.reason, 'budget');
+ guard.stop();
+});
+
+test('the run guard kills a run that reaches its time limit', async () => {
+ const { createRunGuard } = await import('../bench/agent-modeling/run-guard.mjs');
+ let kills = 0;
+ const guard = createRunGuard({ child: { kill: () => { kills++; } }, model: 'haiku', budgetUsd: 1, timeoutSeconds: 0.01 });
+ await new Promise(resolveWait => setTimeout(resolveWait, 60));
+ assert.equal(kills, 1);
+ assert.equal(guard.reason, 'timeout');
+});