Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Merge origin/main (#962)

Nguyen Xuan An80d4e32739b8
diff --cc tests/agent-bench.test.js
index 14e02de0,315443a9..5b0519a8
--- a/tests/agent-bench.test.js
+++ b/tests/agent-bench.test.js
@@@ -71,28 -73,49 +73,74 @@@ test('the summary groups by model and k
assert.equal(compareBench(baseline, baseline).regressed, false);
});
+test('the run guard counts each streamed message once and kills a run over its budget', async () => {
+ const { createRunGuard, estimateUsageCostUsd } = await import('../bench/agent-modeling/run-guard.mjs');
+ assert.equal(estimateUsageCostUsd({ input_tokens: 1e6, output_tokens: 1e6 }, 'claude-haiku-4-5'), 6);
+ assert.equal(estimateUsageCostUsd({ cache_read_input_tokens: 1e6 }, 'sonnet'), 0.3);
+ let kills = 0;
+ const child = { kill: () => { kills++; } };
+ const guard = createRunGuard({ child, model: 'haiku', budgetUsd: 0.9, timeoutSeconds: 60 });
+ const event = JSON.stringify({ type: 'assistant', message: { id: 'a', usage: { output_tokens: 100000 } } });
+ guard.feed(`${event}\n${event}\n{"type":"assis`);
+ assert.equal(kills, 0, 'the same message id repeated is one message');
+ assert.equal(guard.estimatedCostUsd, 0.5);
+ guard.feed(`tant","message":{"id":"b","usage":{"output_tokens":100000}}}\n`);
+ assert.equal(kills, 1);
+ assert.equal(guard.reason, 'budget');
+ guard.stop();
+});
+
+test('the run guard kills a run that reaches its time limit', async () => {
+ const { createRunGuard } = await import('../bench/agent-modeling/run-guard.mjs');
+ let kills = 0;
+ const guard = createRunGuard({ child: { kill: () => { kills++; } }, model: 'haiku', budgetUsd: 1, timeoutSeconds: 0.01 });
+ await new Promise(resolveWait => setTimeout(resolveWait, 60));
+ assert.equal(kills, 1);
+ assert.equal(guard.reason, 'timeout');
++
+ const readFixture = name => readFileSync(new URL(`./fixtures/agent-bench/${name}.jsonl`, import.meta.url), 'utf8');
+
+ test('a run with 0 scene calls is classified by cause: infra for start-up, server and empty ToolSearch, model for the rest', () => {
+ const classify = (name, options = {}) => classifyZeroCalls(readFixture(name), { exitCode: 0, toolCalls: 0, ...options });
+ assert.equal(classify('with-scene-calls', { toolCalls: 1 }), null);
+ assert.deepEqual(classify('no-init', { exitCode: 1, errors: 'spawn failed\nmore' }), { kind: 'infra', cause: 'start-up-failure', detail: 'spawn failed' });
+ assert.equal(classify('mcp-server-failed').cause, 'mcp-server-failed');
+ assert.equal(classify('mcp-server-failed').kind, 'infra');
+ assert.equal(classify('no-result', { exitCode: 1 }).cause, 'start-up-failure');
+ assert.equal(classify('no-result', { exitCode: null }).cause, 'timeout');
+ assert.equal(classify('toolsearch-empty').cause, 'toolsearch-empty');
+ assert.equal(classify('toolsearch-empty').kind, 'infra');
+ });
+
+ test('a ToolSearch that showed the 67js tools and then quit, a refusal and a no-tool answer are model failures', () => {
+ const classify = name => classifyZeroCalls(readFixture(name), { exitCode: 0, toolCalls: 0 });
+ assert.deepEqual([classify('toolsearch-then-quit').kind, classify('toolsearch-then-quit').cause], ['model', 'toolsearch-then-quit']);
+ assert.deepEqual([classify('refusal').kind, classify('refusal').cause], ['model', 'refusal']);
+ assert.deepEqual([classify('no-tool-use').kind, classify('no-tool-use').cause], ['model', 'no-tool-use']);
+ });
+
+ test('infra runs stay out of pass@1 and are reported by cause; model zero-call runs stay counted and are labelled', () => {
+ const infraRecord = cause => record('crate', false, { toolCalls: 0, infra: true, zeroCalls: { kind: 'infra', cause } });
+ const quitRecord = record('bolt', false, { toolCalls: 0, infra: false, zeroCalls: { kind: 'model', cause: 'toolsearch-then-quit' } });
+ const totals = benchTotals([record('crate', true), infraRecord('mcp-server-failed'), infraRecord('timeout'), record('bolt', true), quitRecord], 2);
+ assert.equal(totals.runs, 3);
+ assert.equal(totals.passAt1, 0.667);
+ assert.equal(totals.infraRuns, 2);
+ assert.deepEqual(totals.infraByCause, { 'mcp-server-failed': 1, timeout: 1 });
+ assert.deepEqual(totals.modelZeroCalls, { 'toolsearch-then-quit': 1 });
+ assert.equal(totals.infraShare, 0.4);
+ assert.equal(totals.passPowK, 0.5);
+ assert.equal(benchSummary({ haiku: [record('crate', true), infraRecord('timeout')] }, 2).haiku.taskResults.crate.infra, 1);
+ });
+
+ test('the comparison prints infra counts next to pass@1 and flags infra above the bound', () => {
+ const infraRecord = record('crate', false, { toolCalls: 0, infra: true, zeroCalls: { kind: 'infra', cause: 'timeout' } });
+ const clean = { summary: benchSummary({ haiku: [record('crate', true), record('crate', true), record('bolt', true), record('bolt', true)] }, 2) };
+ const noisy = { summary: benchSummary({ haiku: [record('crate', true), infraRecord, record('bolt', true), record('bolt', true)] }, 2) };
+ const result = compareBench(clean, noisy);
+ assert.equal(result.regressed, false);
+ assert.equal(result.tooMuchInfra, true);
+ assert.match(result.text, /haiku: infra 0 -> 1 runs {2}INFRA over 10% of runs; pass@1 100% -> 100%/);
+ assert.match(result.text, /candidate infra runs \(not in pass@1\): 1 \(timeout 1\)/);
+ assert.equal(compareBench(clean, noisy, 0.05, 0.5).tooMuchInfra, false);
});