3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
Add --prompt-note to the agent bench to compare prompt wording (#986)
Nguyen Xuan Anc19dda970b47
diff --git a/bench/agent-modeling/README.md b/bench/agent-modeling/README.mdindex 4db6ad75..34b3a18b 100644--- a/bench/agent-modeling/README.md+++ b/bench/agent-modeling/README.md@@ -544,3 +544,5 @@ The official Blender MCP server (Blender Lab, v1.0.3, add-on and server) driving- Wall time is not a fair figure. Measured in the transcripts: `crate` hit Claude Code's 300 s no-response abort on its render call while the files were still written; the other four runs with an error and about 600 s show the same wall time. Inferred: the render is a CPU Cycles render of an x86 Blender under emulation, which a native Blender would finish much faster; `fence`, whose render stayed under the limit, took 38 s. Treat the 517 s mean as environment cost, not as Blender MCP latency.- Every run but `fence` spent one call on a failed first attempt (a Python traceback or the timeout) and then repaired it. The tool takes raw `bpy` code, so the agent writes the mesh code, the Z-up to Y-up conversion and the export itself, as in the three.js run; unlike three.js it needs no browser for the picture.- Limits: 1 run per cell, one model, tasks built around Crucible. The server's own screenshot and render tools were never used; the agents rendered from their own code. The container has outbound network access (the model's code could reach the internet); only the scratch folder is shared with the host.++`--prompt-note <text>` (bench.mjs and run.mjs, `prompt-note.mjs`) appends one line of up to 300 characters to every task prompt and records it as `promptNote` in the run and bench files, so two runs on the same tasks compare prompt wording, for example the plain prompt against `--prompt-note "use 67js, you may write it like three.js"` (#986). The note only adds words: the run keeps the MCP arm and its tools, and gives the agent no shell.diff --git a/bench/agent-modeling/bench.mjs b/bench/agent-modeling/bench.mjsindex eb45d07e..2188ae6e 100644--- a/bench/agent-modeling/bench.mjs+++ b/bench/agent-modeling/bench.mjs@@ -3,7 +3,7 @@// Runs use the Claude Code subscription, so the summary has tokens, calls and time and never dollars.//// node bench/agent-modeling/bench.mjs --claude <claude executable> --models haiku,sonnet,opus --runs 3 --label baseline-// [--tasks crate,look-fog] [--kinds look,light] [--topics relations,explosion] [--dry-run] [--suggest <base ref>] [--smoke] [--skill-mode none|legacy|references] [--findings off] [--backend gpu|cpu] [--budget 3] [--timeout 1200] [--resume]+// [--tasks crate,look-fog] [--kinds look,light] [--topics relations,explosion] [--dry-run] [--suggest <base ref>] [--smoke] [--skill-mode none|legacy|references] [--findings off] [--backend gpu|cpu] [--budget 3] [--timeout 1200] [--resume] [--prompt-note <text>]//// --tasks, --kinds and --topics add to each other (a union of the tasks named, of the kinds and of the tasks tagged with a topic);// with none of them the bench runs every task in suite.mjs. --tasks also takes the tasks scenes.mjs defines that the default run@@ -12,6 +12,7 @@// ref (a full bench for the MCP server file or SKILL.md, a subset for a reference file under skills/ and a subset otherwise; a skill change adds --skill-mode references) and stops. Bench what changed (subsets.mjs); the full// bench runs for those shared changes and once before a release cut.//+// --prompt-note <text> appends one line to every task prompt (prompt-note.mjs) and records it, to compare prompt wording.// --smoke runs the two smoke tasks once. --resume continues the same label (same models, same --runs): attempts already// recorded are skipped. The summary goes to bench/agent-modeling/results/bench-<label>.json and holds the per-run records// of every model, so `compare-bench.mjs` can diff it against another bench file.@@ -23,6 +24,7 @@ import { benchSummary, formatSummaryTable } from './bench-summary.mjs';import { BENCH_TASKS, OPTIONAL_TASK_IDS, SMOKE_TASKS } from './suite.mjs';import { selectTasks, skillModeAdvice, subsetCommand, suggestSubset } from './subsets.mjs';import { resolveSkillMode } from './skill-prompt.mjs';+import { resolvePromptNote } from './prompt-note.mjs';const resultsDirectory = join(resolve(import.meta.dirname), 'results');@@ -44,6 +46,7 @@ const { values: options } = parseArgs({timeout: { type: 'string', default: '1200' },resume: { type: 'boolean', default: false },findings: { type: 'string', default: 'on' },+ 'prompt-note': { type: 'string' },},});@@ -60,10 +63,12 @@ if (options.suggest) {process.exit(0);}const skillMode = resolveSkillMode({ skillMode: options['skill-mode'] });+const promptNote = resolvePromptNote(options['prompt-note']);const hasSelection = options.tasks || options.kinds || options.topics;const taskIds = options.smoke && !hasSelection ? SMOKE_TASKS : selectTasks(options, BENCH_TASKS, OPTIONAL_TASK_IDS);if (options['dry-run']) {console.log(`${taskIds.length} of ${BENCH_TASKS.length} tasks: ${taskIds.join(', ')}`);+ if (promptNote) console.log(`Prompt note: ${promptNote}`);process.exit(0);}const models = options.models.split(',');@@ -77,6 +82,7 @@ function runModel(model) {'--backend', options.backend, '--budget', options.budget, '--timeout', options.timeout,'--findings', options.findings, '--skill-mode', skillMode, '--label', benchName, '--run-name', runName,...(options.resume ? ['--resume', runName] : []),+ ...(promptNote ? ['--prompt-note', promptNote] : []),];return new Promise((resolvePromise, reject) => {const child = spawn(process.execPath, argumentsList, { stdio: 'inherit', windowsHide: true });@@ -99,5 +105,5 @@ for (const model of models) {const summary = benchSummary(recordsByModel, runs);const benchFile = join(resultsDirectory, `bench-${benchName}.json`);-await writeFile(benchFile, `${JSON.stringify({ bench: benchName, date: new Date().toISOString(), runsPerTask: runs, tasks: taskIds, backend: options.backend, skillMode, summary, recordsByModel }, null, 2)}\n`);+await writeFile(benchFile, `${JSON.stringify({ bench: benchName, date: new Date().toISOString(), runsPerTask: runs, tasks: taskIds, backend: options.backend, skillMode, promptNote, summary, recordsByModel }, null, 2)}\n`);console.log(`\n${formatSummaryTable(summary, runs)}\n\nBench file: bench/agent-modeling/results/bench-${benchName}.json`);diff --git a/bench/agent-modeling/prompt-note.mjs b/bench/agent-modeling/prompt-note.mjsnew file mode 100644index 00000000..7d03c3bb--- /dev/null+++ b/bench/agent-modeling/prompt-note.mjs@@ -0,0 +1,21 @@+// A short note appended to every task prompt of a bench run (bench.mjs and run.mjs --prompt-note), so two runs can compare+// prompt wording on the same tasks, for example the plain prompt against "use 67js, you may write it like three.js" (#986).+// The note only adds words to the prompt: the run keeps the MCP arm and its tools, and the note is recorded in the result file.++export const PROMPT_NOTE_MAX_LENGTH = 300;++/** The validated note, or null when none was passed. Throws for an empty, multi-line or overlong note. */+export function resolvePromptNote(note) {+ if (note === undefined) return null;+ const trimmed = String(note).trim();+ if (trimmed.length === 0) throw new RangeError('--prompt-note must not be empty. Leave the flag out for the plain prompt.');+ if (/[\u0000-\u001f\u007f]/.test(trimmed)) throw new RangeError('--prompt-note must be one line of plain text, without line breaks or control characters.');+ if (trimmed.length > PROMPT_NOTE_MAX_LENGTH) throw new RangeError(`--prompt-note must be at most ${PROMPT_NOTE_MAX_LENGTH} characters, got ${trimmed.length}. Shorten the note.`);+ return trimmed;+}++/** The task prompt with the note appended as its last paragraph; the prompt unchanged when there is no note. */+export function withPromptNote(prompt, note) {+ if (note === null) return prompt;+ return `${prompt}\n\n${note}`;+}diff --git a/bench/agent-modeling/run.mjs b/bench/agent-modeling/run.mjsindex 6ad43c04..1cf83bca 100644--- a/bench/agent-modeling/run.mjs+++ b/bench/agent-modeling/run.mjs@@ -28,6 +28,7 @@ import { parseArgs } from 'node:util';import { createRunGuard } from './run-guard.mjs';import { claudeArguments, writeSkillPromptFile } from './claude-arguments.mjs';import { resolveSkillMode, skillPromptFor } from './skill-prompt.mjs';+import { resolvePromptNote, withPromptNote } from './prompt-note.mjs';import { removeLocalPaths, totalsByModel, totalsOf } from './aggregate.mjs';import { gradeRecipe, recipeFeatures } from './check.mjs';import { countFixRounds, isMcpRender, isMcpWrite } from './fix-rounds.mjs';@@ -58,10 +59,12 @@ const { values: options } = parseArgs({resume: { type: 'string' },'run-name': { type: 'string' },backend: { type: 'string', default: 'gpu' },+ 'prompt-note': { type: 'string' },},});const skillMode = resolveSkillMode({ skillMode: options['skill-mode'], skill: options.skill });+const promptNote = resolvePromptNote(options['prompt-note']);const runName = options.resume ?? options['run-name'] ?? `${new Date().toISOString().slice(0, 16).replace(/[:T]/g, '-')}-${options.model}-${options.label}`;const rawDirectory = join(repository, 'artifacts', 'agent-modeling', runName);const resultsDirectory = join(repository, 'bench', 'agent-modeling', 'results');@@ -180,7 +183,7 @@ async function runTask(task, attempt) {}await writeFile(configPath, JSON.stringify(mcpConfig(assetDirectory, task)));const text = taskPrompt(task);- const prompt = task.ownPreamble ? text : `${PROMPT_PREAMBLE}\n\n${text}`;+ const prompt = withPromptNote(task.ownPreamble ? text : `${PROMPT_PREAMBLE}\n\n${text}`, promptNote);const skillPromptPath = await writeSkillPromptFile(workingDirectory, await skillPromptFor(skillMode, task, repository));const run = await runClaude(prompt, workingDirectory, configPath, skillPromptPath);const transcript = readTranscript(run.output);@@ -253,6 +256,7 @@ function summarize() {date: new Date().toISOString(),skill: skillMode !== 'none',skillMode,+ promptNote,branches: options.branches,relations: options.relations,findings: options.findings,diff --git a/changes/986.md b/changes/986.mdindex 4a35c066..2d15993d 100644--- a/changes/986.md+++ b/changes/986.md@@ -1 +1,2 @@- The three.js addon test pins the addon at 10 or more snippets that each draw a frame, and its source under 40,000 bytes, so growth shows up in review (#986)+- The agent bench takes `--prompt-note <text>`, one line appended to every task prompt and recorded in the result files, to compare prompt wording on the same tasks (#986)diff --git a/tests/agent-bench-prompt-note.test.js b/tests/agent-bench-prompt-note.test.jsnew file mode 100644index 00000000..3f6ece40--- /dev/null+++ b/tests/agent-bench-prompt-note.test.js@@ -0,0 +1,43 @@+import test from 'node:test';+import assert from 'node:assert/strict';+import { spawnSync } from 'node:child_process';+import { join } from 'node:path';+import { PROMPT_NOTE_MAX_LENGTH, resolvePromptNote, withPromptNote } from '../bench/agent-modeling/prompt-note.mjs';++const repository = new URL('..', import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, '$1');+const benchDirectory = join(repository, 'bench', 'agent-modeling');+const threeJsNote = 'use 67js, you may write it like three.js';++function runScript(name, argumentsList) {+ return spawnSync(process.execPath, [join(benchDirectory, name), ...argumentsList], { encoding: 'utf8', windowsHide: true });+}++test('no note leaves the prompt as it was, and a note becomes its last paragraph', () => {+ assert.equal(resolvePromptNote(undefined), null);+ assert.equal(withPromptNote('Build a crate.', null), 'Build a crate.');+ assert.equal(withPromptNote('Build a crate.', resolvePromptNote(` ${threeJsNote} `)), `Build a crate.\n\n${threeJsNote}`);+});++test('an empty, multi-line or overlong note is refused with the fix', () => {+ assert.throws(() => resolvePromptNote(' '), /must not be empty/);+ assert.throws(() => resolvePromptNote('first line\nsecond line'), /one line of plain text/);+ assert.throws(() => resolvePromptNote('x'.repeat(PROMPT_NOTE_MAX_LENGTH + 1)), new RegExp(`at most ${PROMPT_NOTE_MAX_LENGTH} characters`));+ assert.equal(resolvePromptNote('x'.repeat(PROMPT_NOTE_MAX_LENGTH)).length, PROMPT_NOTE_MAX_LENGTH);+});++test('bench.mjs --dry-run shows the note it would append', () => {+ const result = runScript('bench.mjs', ['--dry-run', '--tasks', 'crate', '--prompt-note', threeJsNote]);+ assert.equal(result.status, 0, result.stderr);+ assert.match(result.stdout, /1 of \d+ tasks: crate/);+ assert.match(result.stdout, new RegExp(`Prompt note: ${threeJsNote}`));+});++test('bench.mjs and run.mjs refuse a bad note before any agent starts', () => {+ const overlong = 'x'.repeat(PROMPT_NOTE_MAX_LENGTH + 1);+ const bench = runScript('bench.mjs', ['--dry-run', '--tasks', 'crate', '--prompt-note', overlong]);+ assert.notEqual(bench.status, 0);+ assert.match(bench.stderr, /--prompt-note must be at most/);+ const run = runScript('run.mjs', ['--claude', 'no-such-claude-executable', '--tasks', 'crate', '--prompt-note', '']);+ assert.notEqual(run.status, 0);+ assert.match(run.stderr, /--prompt-note must not be empty/);+});