Skip to content

3D engine that AI agents drive

11.2 KB
Raw Blame
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import test from 'node:test';
import { benchSummary, benchTotals } from '../bench/agent-modeling/bench-summary.mjs';
import { classifyZeroCalls } from '../bench/agent-modeling/zero-calls.mjs';
import { compareBench } from '../bench/agent-modeling/compare-bench.mjs';
import { gradeRecipe } from '../bench/agent-modeling/check.mjs';
import { REFERENCES } from '../bench/agent-modeling/references.mjs';
import { BENCH_TASKS, SMOKE_TASKS } from '../bench/agent-modeling/suite.mjs';
import { taskById } from '../bench/agent-modeling/tasks.mjs';

const failedKinds = grade => grade.results.filter(result => !result.pass).map(result => result.label ?? result.kind);

test('the bench has 20 to 33 tasks, each known to the suite, covering every kind of work', () => {
  assert.ok(BENCH_TASKS.length >= 20 && BENCH_TASKS.length <= 33, `${BENCH_TASKS.length} tasks`);
  assert.equal(new Set(BENCH_TASKS.map(task => task.id)).size, BENCH_TASKS.length);
  for (const { id } of BENCH_TASKS) assert.ok(taskById(id).checks.length > 0, id);
  for (const id of SMOKE_TASKS) assert.ok(BENCH_TASKS.some(task => task.id === id), id);
  const kinds = new Set(BENCH_TASKS.map(task => task.kind));
  for (const kind of ['modeling', 'scene', 'look', 'light', 'camera', 'animation', 'film']) assert.ok(kinds.has(kind), kind);
});

test('the camera, light, look and film tasks reject plausible wrong answers', async () => {
  const wrong = (id, change) => {
    const recipe = structuredClone(REFERENCES[id]);
    change(recipe);
    return gradeRecipe(taskById(id), recipe);
  };
  assert.deepEqual(failedKinds(await wrong('camera-closeup', recipe => { recipe.camera.eye = [6, 4, 8]; })), ['cameraAims']);
  assert.deepEqual(failedKinds(await wrong('camera-closeup', recipe => { recipe.camera.target = [3, 1, 0]; })), ['cameraAims']);
  assert.deepEqual(failedKinds(await wrong('camera-orbit', recipe => { recipe.motion.tracks[0].keyframes.length = 4; })), ['cameraOrbit']);
  assert.deepEqual(failedKinds(await wrong('camera-orbit', recipe => { delete recipe.motion; })), ['cameraOrbit']);
  assert.deepEqual(failedKinds(await wrong('light-spot-shadow', recipe => { recipe.environment.spotLights[0].castShadow = false; })), ['one shadow spot above, aimed at the statue']);
  assert.deepEqual(failedKinds(await wrong('light-spot-shadow', recipe => { recipe.environment.spotLights[0].direction = [1, 0, 0]; })), ['one shadow spot above, aimed at the statue']);
  assert.deepEqual(failedKinds(await wrong('light-key-fill', recipe => { recipe.environment.pointLights[0].color = [1, 0.5, 0.3]; })), ['cool point fill on the right']);
  assert.deepEqual(failedKinds(await wrong('look-metal', recipe => { recipe.materials[1].metallic = 0; })), ['polished gold on the statue']);
  assert.deepEqual(failedKinds(await wrong('look-fog', recipe => { recipe.environment.fogDensity = 0.001; })), ['thin fog']);
  assert.deepEqual(failedKinds(await wrong('sunset-timelapse', recipe => { recipe.motion.tracks[1].keyframes[1].value = [1, 1, 1]; })), ['sun colour gets warmer']);
  assert.deepEqual(failedKinds(await wrong('film-reveal', recipe => { recipe.motion.tracks[0].keyframes[1].value = [0, 3, 9]; })), ['cameraDolly']);
  assert.deepEqual(failedKinds(await wrong('film-reveal', recipe => { recipe.motion.tracks.pop(); })), ['exposure fades up']);
  assert.deepEqual(failedKinds(await wrong('ask-user-lasso', recipe => { recipe.materials.find(material => material.id === 'bronze').color = [0.45, 0.3, 0.2]; })), ['the lassoed statue is blue']);
  assert.deepEqual(failedKinds(await wrong('ask-user-lasso', recipe => { recipe.nodes.find(node => node.id === 'plinth').material = 'stone'; })), ['the lassoed plinth is blue']);
  assert.deepEqual(failedKinds(await wrong('ask-user-pin-shared', recipe => { recipe.nodes.find(node => node.id === 'plinth').material = 'stone'; })), ['the pinned plinth is red']);
  // The shared stone material: reddening it reddens the floor too, which the pin did not cover.
  assert.deepEqual(failedKinds(await wrong('ask-user-pin-shared', recipe => {
    recipe.materials.find(material => material.id === 'stone').color = [0.85, 0.12, 0.1];
    recipe.nodes.find(node => node.id === 'plinth').material = 'stone';
  })), ['the floor sharing its material is not recoloured']);
  // The shared stone material: colouring it blue colours the floor too, which the lasso did not cover.
  assert.deepEqual(failedKinds(await wrong('ask-user-lasso', recipe => {
    recipe.materials.find(material => material.id === 'stone').color = [0.1, 0.2, 0.85];
    recipe.nodes.find(node => node.id === 'plinth').material = 'stone';
  })), ['the floor outside the lasso is not recoloured']);
});

const record = (task, pass, extra = {}) => ({ task, pass, toolCalls: 10, toolErrors: 1, inputTokens: 1000, outputTokens: 100, wallSeconds: 30, ...extra });

test('pass@1 is the share of runs that pass, pass^k needs every run of a task to pass', () => {
  const records = [
    record('crate', true), record('crate', true), record('crate', true),
    record('bolt', true), record('bolt', false), record('bolt', true),
    record('mug', false), record('mug', false), record('mug', false),
  ];
  const totals = benchTotals(records, 3);
  assert.equal(totals.passAt1, 0.556);
  assert.equal(totals.passPowK, 0.333);
  assert.equal(totals.passAtK, 0.667);
  assert.equal(totals.callsPerRun, 10);
  assert.equal(totals.secondsPerRun, 30);
});

test('a task with fewer runs than k does not count toward pass^k', () => {
  assert.equal(benchTotals([record('crate', true), record('crate', true)], 3).passPowK, 0);
});

test('the summary groups by model and kind, and the comparison flags a pass@1 drop and the tasks that moved', () => {
  const baselineRecords = { haiku: [record('crate', true), record('crate', true), record('look-fog', true), record('look-fog', false)] };
  const candidateRecords = { haiku: [record('crate', true), record('crate', false), record('look-fog', false), record('look-fog', false)] };
  const baseline = { summary: benchSummary(baselineRecords, 2) };
  const candidate = { summary: benchSummary(candidateRecords, 2) };
  assert.equal(baseline.summary.haiku.kinds.look.passAt1, 0.5);
  const result = compareBench(baseline, candidate);
  assert.equal(result.regressed, true);
  assert.match(result.text, /REGRESSION/);
  assert.match(result.text, /crate: 2\/2 -> 1\/2 \(-1\)/);
  assert.equal(compareBench(baseline, baseline).regressed, false);
});

test('the run guard counts each streamed message once and kills a run over its budget', async () => {
  const { createRunGuard, estimateUsageCostUsd } = await import('../bench/agent-modeling/run-guard.mjs');
  assert.equal(estimateUsageCostUsd({ input_tokens: 1e6, output_tokens: 1e6 }, 'claude-haiku-5-5'), 6);
  assert.equal(estimateUsageCostUsd({ cache_read_input_tokens: 1e6 }, 'sonnet'), 0.3);
  let kills = 0;
  const child = { kill: () => { kills++; } };
  const guard = createRunGuard({ child, model: 'haiku', budgetUsd: 0.9, timeoutSeconds: 60 });
  const event = JSON.stringify({ type: 'assistant', message: { id: 'a', usage: { output_tokens: 100000 } } });
  guard.feed(`${event}\n${event}\n{"type":"assis`);
  assert.equal(kills, 0, 'the same message id repeated is one message');
  assert.equal(guard.estimatedCostUsd, 0.5);
  guard.feed(`tant","message":{"id":"b","usage":{"output_tokens":100000}}}\n`);
  assert.equal(kills, 1);
  assert.equal(guard.reason, 'budget');
  guard.stop();
});

test('the run guard kills a run that reaches its time limit', async () => {
  const { createRunGuard } = await import('../bench/agent-modeling/run-guard.mjs');
  let kills = 0;
  const guard = createRunGuard({ child: { kill: () => { kills++; } }, model: 'haiku', budgetUsd: 1, timeoutSeconds: 0.01 });
  await new Promise(resolveWait => setTimeout(resolveWait, 60));
  assert.equal(kills, 1);
  assert.equal(guard.reason, 'timeout');
});

const readFixture = name => readFileSync(new URL(`./fixtures/agent-bench/${name}.jsonl`, import.meta.url), 'utf8');

test('a run with 0 scene calls is classified by cause: infra for start-up, server and empty ToolSearch, model for the rest', () => {
  const classify = (name, options = {}) => classifyZeroCalls(readFixture(name), { exitCode: 0, toolCalls: 0, ...options });
  assert.equal(classify('with-scene-calls', { toolCalls: 1 }), null);
  assert.deepEqual(classify('no-init', { exitCode: 1, errors: 'spawn failed\nmore' }), { kind: 'infra', cause: 'start-up-failure', detail: 'spawn failed' });
  assert.equal(classify('mcp-server-failed').cause, 'mcp-server-failed');
  assert.equal(classify('mcp-server-failed').kind, 'infra');
  assert.equal(classify('no-result', { exitCode: 1 }).cause, 'start-up-failure');
  assert.equal(classify('no-result', { exitCode: null }).cause, 'timeout');
  assert.equal(classify('toolsearch-empty').cause, 'toolsearch-empty');
  assert.equal(classify('toolsearch-empty').kind, 'infra');
});

test('a ToolSearch that showed the Crucible tools and then quit, a refusal and a no-tool answer are model failures', () => {
  const classify = name => classifyZeroCalls(readFixture(name), { exitCode: 0, toolCalls: 0 });
  assert.deepEqual([classify('toolsearch-then-quit').kind, classify('toolsearch-then-quit').cause], ['model', 'toolsearch-then-quit']);
  assert.deepEqual([classify('refusal').kind, classify('refusal').cause], ['model', 'refusal']);
  assert.deepEqual([classify('no-tool-use').kind, classify('no-tool-use').cause], ['model', 'no-tool-use']);
});

test('infra runs stay out of pass@1 and are reported by cause; model zero-call runs stay counted and are labelled', () => {
  const infraRecord = cause => record('crate', false, { toolCalls: 0, infra: true, zeroCalls: { kind: 'infra', cause } });
  const quitRecord = record('bolt', false, { toolCalls: 0, infra: false, zeroCalls: { kind: 'model', cause: 'toolsearch-then-quit' } });
  const totals = benchTotals([record('crate', true), infraRecord('mcp-server-failed'), infraRecord('timeout'), record('bolt', true), quitRecord], 2);
  assert.equal(totals.runs, 3);
  assert.equal(totals.passAt1, 0.667);
  assert.equal(totals.infraRuns, 2);
  assert.deepEqual(totals.infraByCause, { 'mcp-server-failed': 1, timeout: 1 });
  assert.deepEqual(totals.modelZeroCalls, { 'toolsearch-then-quit': 1 });
  assert.equal(totals.infraShare, 0.4);
  assert.equal(totals.passPowK, 0.5);
  assert.equal(benchSummary({ haiku: [record('crate', true), infraRecord('timeout')] }, 2).haiku.taskResults.crate.infra, 1);
});

test('the comparison prints infra counts next to pass@1 and flags infra above the bound', () => {
  const infraRecord = record('crate', false, { toolCalls: 0, infra: true, zeroCalls: { kind: 'infra', cause: 'timeout' } });
  const clean = { summary: benchSummary({ haiku: [record('crate', true), record('crate', true), record('bolt', true), record('bolt', true)] }, 2) };
  const noisy = { summary: benchSummary({ haiku: [record('crate', true), infraRecord, record('bolt', true), record('bolt', true)] }, 2) };
  const result = compareBench(clean, noisy);
  assert.equal(result.regressed, false);
  assert.equal(result.tooMuchInfra, true);
  assert.match(result.text, /haiku: infra 0 -> 1 runs {2}INFRA over 10% of runs; pass@1 100% -> 100%/);
  assert.match(result.text, /candidate infra runs \(not in pass@1\): 1 \(timeout 1\)/);
  assert.equal(compareBench(clean, noisy, 0.05, 0.5).tooMuchInfra, false);
});