Skip to content

3D engine that AI agents drive

16.0 KB
Raw Blame
// The three.js side of the agent friendliness comparison (issue #31): the same task prompts and model as run.mjs, but the
// agent has no MCP. It gets an empty folder with `three` installed from npm and writes code.
//
// node bench/agent-modeling/threejs.mjs --claude <claude executable> [--model sonnet] [--tasks crate,bolt] [--runs 1]
//   [--budget 3] [--timeout 1200] [--label threejs]
//
// The agent writes build.mjs, which builds the object with three.js, writes geometry.json (the world-space triangles of
// every mesh) and renders scene.png with headless Chrome. The same task checks and the same silhouette look score as
// the Crucible runs grade geometry.json, so the two systems are measured on the same shape. A run passes when every check
// passes and scene.png is a real picture.
//
// Raw transcripts go to artifacts/agent-modeling/<run>/ (ignored by Git); the summary goes to
// bench/agent-modeling/results/<run>.json.
import { execFileSync, spawn } from 'node:child_process';
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile, cp } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { parseArgs } from 'node:util';
import { createRunGuard } from './run-guard.mjs';
import { SHELL_ARM_DENIED_TOOLS, SHELL_ARM_PROCESS_RULE, assertShellArmAllowed } from './shell-arm-safety.mjs';
import { removeLocalPaths, totalsByModel, totalsOf } from './aggregate.mjs';
import { checkOne } from './check.mjs';
import { countFixRounds, isCodeRender, isCodeWrite } from './fix-rounds.mjs';
import { camerasForTask, worldGeometriesOf } from './look.mjs';
import { countTips, lineHits, sectionLoops, signedVolume } from './measure.mjs';
import { checkPng } from './png-check.mjs';
import { REFERENCES } from './references.mjs';
import { intersectionOverUnion, rasterizeSilhouette } from './silhouette.mjs';
import { taskById } from './tasks.mjs';

const repository = resolve(import.meta.dirname, '../..');
const BUILT_IN_TOOLS_DENIED = ['WebFetch', 'WebSearch', 'Agent', 'Task', 'NotebookEdit'];
const WELD_SCALE = 1e5;

export const THREEJS_PREAMBLE = `You are modeling with three.js code. The folder you are in has three installed from npm (node_modules/three). Build the
object described below as a closed, watertight solid, in metres with Y up. Rest the object on the ground (lowest point at
Y = 0), centred on X = 0 and Z = 0.
Write a Node.js script named build.mjs that does three things and run it:
1. Builds the object with three.js.
2. Writes geometry.json: a JSON array with one entry { "positions": [...], "indices": [...] } per mesh, with every vertex in
   world space (apply each mesh's world matrix) and indices as a flat triangle list.
3. Renders the scene to scene.png, 512 by 512 pixels, with headless Google Chrome (installed; use --use-angle=swiftshader
   --enable-unsafe-swiftshader --enable-webgl --ignore-gpu-blocklist so it needs no GPU). There is no internet access.
The files build.mjs, geometry.json and scene.png are what gets graded. When they exist, reply with one line saying so.
${SHELL_ARM_PROCESS_RULE}`;

/** Measures an array of { positions, indices } meshes like measure.mjs does for a recipe, welding shared corners first. */
export function measureGeometries(rawGeometries) {
  const geometries = rawGeometries.map(({ positions, indices }) => ({
    positions: Float32Array.from(positions),
    indices: Uint32Array.from(indices ?? Array.from({ length: positions.length / 3 }, (_, index) => index)),
  }));
  const bounds = { min: [Infinity, Infinity, Infinity], max: [-Infinity, -Infinity, -Infinity] };
  let signed = 0;
  let triangles = 0;
  let vertexCount = 0;
  let edgeCount = 0;
  let faceCount = 0;
  let components = 0;
  let closedComponents = 0;
  for (const geometry of geometries) {
    const weldedIndex = new Map();
    const welded = [];
    for (let vertex = 0; vertex < geometry.positions.length / 3; vertex++) {
      const point = [0, 1, 2].map(axis => geometry.positions[vertex * 3 + axis]);
      for (let axis = 0; axis < 3; axis++) {
        bounds.min[axis] = Math.min(bounds.min[axis], point[axis]);
        bounds.max[axis] = Math.max(bounds.max[axis], point[axis]);
      }
      const key = point.map(value => Math.round(value * WELD_SCALE)).join(',');
      if (!weldedIndex.has(key)) weldedIndex.set(key, weldedIndex.size);
      welded.push(weldedIndex.get(key));
    }
    signed += signedVolume(geometry);
    triangles += geometry.indices.length / 3;
    const edgeUses = new Map();
    const parent = Array.from({ length: weldedIndex.size }, (_, index) => index);
    const find = vertex => (parent[vertex] === vertex ? vertex : (parent[vertex] = find(parent[vertex])));
    let faces = 0;
    for (let triangle = 0; triangle < geometry.indices.length; triangle += 3) {
      const corners = [0, 1, 2].map(corner => welded[geometry.indices[triangle + corner]]);
      if (new Set(corners).size < 3) continue;
      faces++;
      for (let corner = 0; corner < 3; corner++) {
        const [from, to] = [corners[corner], corners[(corner + 1) % 3]];
        const key = from < to ? `${from}-${to}` : `${to}-${from}`;
        edgeUses.set(key, (edgeUses.get(key) ?? 0) + 1);
        parent[find(from)] = find(to);
      }
    }
    const roots = new Set(Array.from({ length: weldedIndex.size }, (_, vertex) => find(vertex)));
    components += roots.size;
    const closed = [...edgeUses.values()].every(uses => uses === 2);
    if (closed) closedComponents += roots.size;
    vertexCount += weldedIndex.size;
    edgeCount += edgeUses.size;
    faceCount += faces;
  }
  const allClosed = closedComponents === components;
  const volume = Math.abs(signed);
  return {
    parts: components,
    closedParts: closedComponents,
    lintProblems: allClosed ? 0 : components - closedComponents,
    problemKinds: allClosed ? [] : ['open or non-manifold edges'],
    holes: allClosed ? (2 * components - (vertexCount - edgeCount + faceCount)) / 2 : null,
    volume,
    triangles,
    bounds,
    size: [0, 1, 2].map(axis => bounds.max[axis] - bounds.min[axis]),
    tips: countTips(geometries, bounds),
    section: (axis, at) => sectionLoops(geometries, axis, at),
    lineHits: (axis, point) => lineHits(geometries, axis, point),
  };
}

/** Same shape as gradeRecipe: { pass, passed, total, results }. Tasks with checks this module cannot measure fail loudly. */
export function gradeGeometries(task, rawGeometries) {
  const unsupported = task.checks.filter(check => !['lint', 'parts', 'size', 'extent', 'ground', 'holes', 'volume', 'section', 'through', 'tips', 'triangles'].includes(check.kind));
  if (unsupported.length) throw new RangeError(`threejs.mjs cannot grade ${unsupported.map(check => check.kind).join(', ')}`);
  const measures = measureGeometries(rawGeometries);
  const results = task.checks.map(check => ({ kind: check.kind, ...(check.label ? { label: check.label } : {}), ...checkOne(check, measures) }));
  const passed = results.filter(result => result.pass).length;
  return { pass: passed === results.length, passed, total: results.length, results };
}

/** The look score of the suite (silhouette overlap from the three fixed cameras) for loose geometries. */
export async function scoreSilhouetteOfGeometries(taskId, rawGeometries) {
  const views = await camerasForTask(taskId);
  const reference = await worldGeometriesOf(REFERENCES[taskId]);
  const candidate = rawGeometries.map(({ positions, indices }) => ({ positions: Float32Array.from(positions), indices: Uint32Array.from(indices) }));
  const overlap = {};
  for (const { name, camera } of views) {
    overlap[name] = Number(intersectionOverUnion(rasterizeSilhouette(candidate, camera), rasterizeSilhouette(reference, camera)).toFixed(4));
  }
  const values = Object.values(overlap);
  return { views: overlap, mean: Number((values.reduce((sum, value) => sum + value, 0) / values.length).toFixed(4)) };
}

export function readTranscript(output) {
  const summary = { model: null, toolCalls: 0, toolErrors: 0, toolsByName: {}, fixRounds: 0, result: null };
  const steps = [];
  for (const line of output.split('\n')) {
    if (!line.trim().startsWith('{')) continue;
    let event;
    try {
      event = JSON.parse(line);
    } catch {
      continue;
    }
    if (event.type === 'system' && event.subtype === 'init') summary.model = event.model;
    if (event.type === 'assistant') {
      for (const block of event.message?.content ?? []) {
        if (block.type !== 'tool_use' || block.name === 'ToolSearch') continue;
        summary.toolCalls++;
        summary.toolsByName[block.name] = (summary.toolsByName[block.name] ?? 0) + 1;
        steps.push({ name: block.name, input: block.input });
      }
    }
    if (event.type === 'user') {
      for (const block of event.message?.content ?? []) {
        if (block.type === 'tool_result' && block.is_error) summary.toolErrors++;
      }
    }
    if (event.type === 'result') summary.result = event;
  }
  summary.fixRounds = countFixRounds(steps, { isRender: isCodeRender, isWrite: isCodeWrite });
  return summary;
}

async function removeFolder(folder) {
  try {
    await rm(folder, { recursive: true, force: true, maxRetries: 10, retryDelay: 500 });
  } catch (error) {
    console.warn(`Left a temporary folder behind: ${error.code}`);
  }
}

if (process.argv[1] === fileURLToPath(import.meta.url)) {
  const { values: options } = parseArgs({
    options: {
      claude: { type: 'string', default: process.env.CLAUDE_EXECUTABLE ?? 'claude' },
      model: { type: 'string', default: 'sonnet' },
      tasks: { type: 'string', default: 'crate,bolt,window,table,fence,bracket' },
      runs: { type: 'string', default: '1' },
      budget: { type: 'string', default: '3' },
      timeout: { type: 'string', default: '1200' },
      label: { type: 'string', default: 'threejs' },
    },
  });
  assertShellArmAllowed();
  const npmExecutable = process.platform === 'win32' ? 'npm.cmd' : 'npm';
  const runName = `${new Date().toISOString().slice(0, 16).replace(/[:T]/g, '-')}-${options.model}-${options.label}`;
  const rawDirectory = join(repository, 'artifacts', 'agent-modeling', runName);
  const resultsDirectory = join(repository, 'bench', 'agent-modeling', 'results');
  await mkdir(rawDirectory, { recursive: true });
  await mkdir(resultsDirectory, { recursive: true });

  // One install of three from npm, copied into each attempt's fresh folder.
  const template = await mkdtemp(join(tmpdir(), 'crucible-threejs-template-'));
  await writeFile(join(template, 'package.json'), JSON.stringify({ name: 'threejs-task', private: true, type: 'module' }));
  execFileSync(npmExecutable, ['install', 'three', '--ignore-scripts', '--no-audit', '--no-fund'], { cwd: template, stdio: 'ignore', shell: process.platform === 'win32' });
  const threeVersion = JSON.parse(await readFile(join(template, 'node_modules', 'three', 'package.json'), 'utf8')).version;

  const runAgent = (prompt, workingDirectory) => new Promise(resolvePromise => {
    const started = performance.now();
    const child = spawn(options.claude, [
      '-p', prompt, '--model', options.model, '--output-format', 'stream-json', '--verbose', '--strict-mcp-config',
      '--allowedTools', 'Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep',
      '--disallowedTools', ...BUILT_IN_TOOLS_DENIED, ...SHELL_ARM_DENIED_TOOLS,
      '--permission-mode', 'dontAsk', '--setting-sources', 'project', '--no-session-persistence',
    ], { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
    let output = '';
    let errors = '';
    const guard = createRunGuard({ child, model: options.model, budgetUsd: options.budget, timeoutSeconds: options.timeout });
    child.stdout.on('data', chunk => { output += chunk; guard.feed(chunk); });
    child.stderr.on('data', chunk => { errors += chunk; });
    child.on('close', code => {
      guard.stop();
      resolvePromise({ output, errors, code, wallMs: Math.round(performance.now() - started) });
    });
  });

  const records = [];
  const summarize = () => ({
    run: runName,
    system: 'three.js by code',
    threeVersion,
    model: records.find(record => record.model)?.model ?? options.model,
    date: new Date().toISOString(),
    runsPerTask: Number(options.runs),
    passed: records.filter(record => record.pass).length,
    tasks: records.length,
    toolCalls: records.reduce((sum, record) => sum + record.toolCalls, 0),
    byModel: totalsByModel(records),
    records,
  });
  try {
    for (let attempt = 1; attempt <= Number(options.runs); attempt++) {
      for (const task of options.tasks.split(',').map(taskById)) {
        const workingDirectory = await mkdtemp(join(tmpdir(), 'crucible-threejs-task-'));
        try {
          await cp(template, workingDirectory, { recursive: true });
          const run = await runAgent(`${THREEJS_PREAMBLE}\n\n${task.prompt.replace(/\n/g, ' ')}`, workingDirectory);
          const transcript = readTranscript(run.output);
          await writeFile(join(rawDirectory, `${task.id}-${attempt}.jsonl`), run.output);
          let geometries = null;
          let grade = { pass: false, passed: 0, total: task.checks.length, results: [{ kind: 'geometry', pass: false, detail: 'no geometry.json' }] };
          try {
            geometries = JSON.parse(await readFile(join(workingDirectory, 'geometry.json'), 'utf8'));
            grade = gradeGeometries(task, geometries);
          } catch (error) {
            grade.results[0].detail = `geometry.json unreadable: ${error.message.split('\n')[0]}`;
          }
          const png = await checkPng(join(workingDirectory, 'scene.png'));
          if (geometries) {
            await cp(join(workingDirectory, 'scene.png'), join(rawDirectory, `${task.id}-${attempt}.png`)).catch(() => {});
            await cp(join(workingDirectory, 'build.mjs'), join(rawDirectory, `${task.id}-${attempt}.build.mjs`)).catch(() => {});
          }
          let silhouette = null;
          if (geometries) silhouette = await scoreSilhouetteOfGeometries(task.id, geometries).catch(error => ({ error: error.message.split('\n')[0] }));
          const usage = transcript.result?.usage ?? {};
          const failed = grade.results.filter(result => !result.pass).map(result => `${result.label ?? result.kind}: ${result.detail}`);
          if (!png.pass) failed.push(`scene.png: ${png.detail}`);
          const record = {
            task: task.id,
            attempt,
            pass: grade.pass && png.pass,
            passed: grade.passed,
            total: grade.total,
            pngPass: png.pass,
            failed: failed.map(removeLocalPaths),
            model: transcript.model,
            toolCalls: transcript.toolCalls,
            toolErrors: transcript.toolErrors,
            fixRounds: transcript.fixRounds,
            toolsByName: transcript.toolsByName,
            look: silhouette ? { silhouette } : null,
            turns: transcript.result?.num_turns ?? null,
            costUsd: null,
            inputTokens: (usage.input_tokens ?? 0) + (usage.cache_read_input_tokens ?? 0) + (usage.cache_creation_input_tokens ?? 0),
            outputTokens: usage.output_tokens ?? 0,
            wallSeconds: Math.round(run.wallMs / 1000),
            exitCode: run.code,
            stopped: transcript.result?.subtype ?? 'no result',
            filesWritten: (await readdir(workingDirectory)).filter(name => !['node_modules', 'package-lock.json'].includes(name)),
          };
          records.push(record);
          console.log(`${record.task.padEnd(10)} #${attempt} ${record.pass ? 'PASS' : 'FAIL'} ${record.passed}/${record.total} png ${png.pass}  calls ${record.toolCalls} (errors ${record.toolErrors})  look ${silhouette?.mean ?? '-'}  ${record.wallSeconds}s${failed.length ? `  ${failed.join(' | ')}` : ''}`);
          await writeFile(join(resultsDirectory, `${runName}.json`), `${JSON.stringify(summarize(), null, 2)}\n`);
        } finally {
          await removeFolder(workingDirectory);
        }
      }
    }
  } finally {
    await removeFolder(template);
  }
  console.log(`${records.filter(record => record.pass).length}/${records.length} passed, ${totalsOf(records).toolCalls} tool calls. Summary: bench/agent-modeling/results/${runName}.json`);
}