3D engine that AI agents drive
Bench: run the no-MCP task through agy with a session-only allow rule (issue #69)
Nguyen Xuan Anfb59a729175d
diff --git a/bench/agent-modeling/README.md b/bench/agent-modeling/README.mdindex ee760092..49f447fa 100644--- a/bench/agent-modeling/README.md+++ b/bench/agent-modeling/README.md@@ -393,4 +393,4 @@ Not run:### Antigravity CLI (2026-11-07)-`llms-only.mjs --agent agy --agy <agy executable> --model <model>` drives the Antigravity CLI, which offers Gemini 4 Argon, Gemini 3.8 Flash and GPT-OSS 120B. A first run of 2 attempts per model passed 0 of 6, but not on the docs: headless mode denies every shell command it cannot prompt for, so the agents read `llms.txt` and the package, wrote `render.mjs` (Gemini 4 Argon), and were refused `node render.mjs`. Those results are not kept. A valid run needs a session-only allow rule for commands, passed with `--settings`, which An has to approve; the runner records any web tool call as `usedWeb` and fails it, since the CLI cannot switch its web tools off.+`llms-only.mjs --agent agy --agy <agy executable> --model <model>` drives the Antigravity CLI, which offers Gemini 4 Argon, Gemini 3.8 Flash and GPT-OSS 120B. Headless mode denies every shell command, so the runner passes a session-only settings file (`--settings`, never saved) with one rule, `permissions.allow: ["command(*)"]`, and removes it after the run. The CLI cannot switch its web tools off, so the runner records any web tool call as `usedWeb` and fails it. Run `npm run build:lib` first: the pack skips scripts, and the runner stops when `build/` is missing. With 2 attempts per model (`results/2026-11-07-agy.json`): Gemini 4 Argon 2 of 2, Gemini 3.8 Flash 2 of 2, GPT-OSS 120B 0 of 2 (it searched the web in every attempt despite the prompt, then timed out waiting for the page). One Gemini 4 Argon PNG was 768 px, which the check accepts because it requires a minimum side only. The MCP suite is not run through this CLI: `agy` reads MCP servers only from its global config, and `--settings` rejects `mcpServers`.diff --git a/bench/agent-modeling/llms-only.mjs b/bench/agent-modeling/llms-only.mjsindex 2cdd9e06..3109a743 100644--- a/bench/agent-modeling/llms-only.mjs+++ b/bench/agent-modeling/llms-only.mjs@@ -7,6 +7,7 @@// Each attempt gets a fresh folder with the package installed from a packed archive, a copy of public/llms.txt and// nothing else. The check opens scene.png and requires a real image with content (png-check.mjs).import { spawn, execFileSync } from 'node:child_process';+import { existsSync } from 'node:fs';import { copyFile, mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises';import { tmpdir } from 'node:os';import { join, resolve } from 'node:path';@@ -41,6 +42,9 @@ const resultsDirectory = join(repository, 'bench', 'agent-modeling', 'results');const npmExecutable = process.platform === 'win32' ? 'npm.cmd' : 'npm';function packArchive(destination) {+ // The pack runs with --ignore-scripts, so prepack never builds: without build/ the archive has no library+ // and the agent cannot import anything (found in the first valid agy run, 2026-11-07).+ if (!existsSync(join(repository, 'build', '67.module.js'))) throw new Error('build/67.module.js is missing; run npm run build:lib first');const output = execFileSync(npmExecutable, ['pack', '--ignore-scripts', '--json', '--pack-destination', destination], { cwd: repository, encoding: 'utf8', shell: process.platform === 'win32' });const packed = JSON.parse(output);return join(destination, (Array.isArray(packed) ? packed[0] : Object.values(packed)[0]).filename);@@ -63,18 +67,21 @@ function claudeArguments() {];}-function agyArguments() {+// Headless agy auto-denies shell commands, so each run gets a session-only settings file with one allow rule.+// --settings is never saved to agy's own config, and the run's own folder is the working directory.+function agyArguments(settingsPath) {return ['-p', PROMPT,'--model', options.model,+ '--settings', settingsPath,'--output-format', 'stream-json','--print-timeout', `${options.timeout}s`,];}-function runAgent(workingDirectory) {+function runAgent(workingDirectory, settingsPath) {const executable = options.agent === 'agy' ? options.agy : options.claude;- const argumentsList = options.agent === 'agy' ? agyArguments() : claudeArguments();+ const argumentsList = options.agent === 'agy' ? agyArguments(settingsPath) : claudeArguments();return new Promise(resolvePromise => {const started = performance.now();const child = spawn(executable, argumentsList, { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });@@ -168,7 +175,9 @@ async function runAttempt(archivePath, attempt) {await writeFile(join(workingDirectory, 'package.json'), JSON.stringify({ name: 'llms-only-task', private: true, type: 'module' }));execFileSync(npmExecutable, ['install', archivePath, '--ignore-scripts', '--no-audit', '--no-fund'], { cwd: workingDirectory, stdio: 'ignore', shell: process.platform === 'win32' });await copyFile(join(repository, 'public', 'llms.txt'), join(workingDirectory, 'llms.txt'));- const run = await runAgent(workingDirectory);+ const settingsPath = `${workingDirectory}-agy-settings.json`;+ await writeFile(settingsPath, JSON.stringify({ permissions: { allow: ['command(*)'] } }));+ const run = await runAgent(workingDirectory, settingsPath).finally(() => rm(settingsPath, { force: true }));const transcript = options.agent === 'agy' ? readAgyTranscript(run.output) : readTranscript(run.output);const pngGrade = await checkPng(join(workingDirectory, 'scene.png'));const grade = transcript.usedWeb ? { pass: false, detail: `used a web tool; ${pngGrade.detail}` } : pngGrade;diff --git a/bench/agent-modeling/results/2026-11-07-agy.json b/bench/agent-modeling/results/2026-11-07-agy.jsonnew file mode 100644index 00000000..17380d10--- /dev/null+++ b/bench/agent-modeling/results/2026-11-07-agy.json@@ -0,0 +1,206 @@+{+ "run": "2026-11-07-agy",+ "date": "2026-11-07T16:19:43.731Z",+ "client": "Antigravity CLI (agy), headless, --settings with permissions.allow command(*), no MCP",+ "task": "llms-only-png",+ "runsPerModel": 2,+ "note": "MCP suite not run: agy only reads MCP servers from its global config, and --settings rejects mcpServers. A 768x768 PNG from gemini-4-argon-medium attempt 1 passed because the check requires a minimum side only. gpt-oss-120b-medium ignored the no-internet instruction and called search_web in every attempt.",+ "byModel": {+ "gemini-3.8-flash-medium": {+ "passed": 2,+ "runs": 2+ },+ "gemini-4-argon-medium": {+ "passed": 2,+ "runs": 2+ },+ "gpt-oss-120b-medium": {+ "passed": 0,+ "runs": 2+ }+ },+ "records": [+ {+ "task": "llms-only-png",+ "attempt": 1,+ "pass": true,+ "detail": "512x512, 15973 colours, 84% differ from the corner",+ "filesWritten": [+ "llms.txt",+ "package.json",+ "render.mjs",+ "scene.png"+ ],+ "model": "gemini-3.8-flash-medium",+ "agent": "agy",+ "usedWeb": false,+ "toolCalls": 106,+ "toolErrors": 1,+ "toolsByName": {+ "run_command": 77,+ "read_file": 3,+ "write_file": 4,+ "manage_task": 22+ },+ "turns": 1,+ "costUsd": null,+ "inputTokens": 507011,+ "outputTokens": 59018,+ "wallSeconds": 547,+ "stopped": "SUCCESS",+ "requestedModel": "gemini-3.8-flash-medium"+ },+ {+ "task": "llms-only-png",+ "attempt": 2,+ "pass": true,+ "detail": "512x512, 12231 colours, 98% differ from the corner",+ "filesWritten": [+ "llms.txt",+ "package.json",+ "render.mjs",+ "scene.png"+ ],+ "model": "gemini-3.8-flash-medium",+ "agent": "agy",+ "usedWeb": false,+ "toolCalls": 38,+ "toolErrors": 2,+ "toolsByName": {+ "read_file": 5,+ "run_command": 26,+ "manage_task": 4,+ "write_file": 2,+ "schedule": 1+ },+ "turns": 1,+ "costUsd": null,+ "inputTokens": 202678,+ "outputTokens": 21615,+ "wallSeconds": 255,+ "stopped": "SUCCESS",+ "requestedModel": "gemini-3.8-flash-medium"+ },+ {+ "task": "llms-only-png",+ "attempt": 1,+ "pass": true,+ "detail": "768x768, 20570 colours, 74% differ from the corner",+ "filesWritten": [+ "llms.txt",+ "package.json",+ "render.mjs",+ "scene.png"+ ],+ "model": "gemini-4-argon-medium",+ "agent": "agy",+ "usedWeb": false,+ "toolCalls": 22,+ "toolErrors": 1,+ "toolsByName": {+ "read_file": 14,+ "write_file": 3,+ "run_command": 3,+ "schedule": 2+ },+ "turns": 1,+ "costUsd": null,+ "inputTokens": 75531,+ "outputTokens": 8769,+ "wallSeconds": 275,+ "stopped": "SUCCESS",+ "requestedModel": "gemini-4-argon-medium"+ },+ {+ "task": "llms-only-png",+ "attempt": 2,+ "pass": true,+ "detail": "512x512, 14257 colours, 68% differ from the corner",+ "filesWritten": [+ "llms.txt",+ "package.json",+ "render.mjs",+ "scene.png"+ ],+ "model": "gemini-4-argon-medium",+ "agent": "agy",+ "usedWeb": false,+ "toolCalls": 36,+ "toolErrors": 4,+ "toolsByName": {+ "read_file": 20,+ "run_command": 7,+ "write_file": 5,+ "manage_task": 4+ },+ "turns": 1,+ "costUsd": null,+ "inputTokens": 148779,+ "outputTokens": 16480,+ "wallSeconds": 499,+ "stopped": "SUCCESS",+ "requestedModel": "gemini-4-argon-medium"+ },+ {+ "task": "llms-only-png",+ "attempt": 1,+ "pass": false,+ "detail": "used a web tool; the file does not exist",+ "filesWritten": [+ "llms.txt",+ "package.json",+ "render.mjs"+ ],+ "model": "gpt-oss-120b-medium",+ "agent": "agy",+ "usedWeb": true,+ "toolCalls": 23,+ "toolErrors": 1,+ "toolsByName": {+ "read_file": 5,+ "search_web": 4,+ "run_command": 5,+ "write_file": 1,+ "manage_task": 5,+ "edit_file": 3+ },+ "turns": null,+ "costUsd": null,+ "inputTokens": 0,+ "outputTokens": 0,+ "wallSeconds": 900,+ "stopped": "no result",+ "requestedModel": "gpt-oss-120b-medium"+ },+ {+ "task": "llms-only-png",+ "attempt": 2,+ "pass": false,+ "detail": "used a web tool; the file does not exist",+ "filesWritten": [+ "llms.txt",+ "package.json",+ "render.mjs"+ ],+ "model": "gpt-oss-120b-medium",+ "agent": "agy",+ "usedWeb": true,+ "toolCalls": 40,+ "toolErrors": 3,+ "toolsByName": {+ "read_file": 8,+ "search_web": 6,+ "run_command": 19,+ "write_file": 2,+ "edit_file": 5+ },+ "turns": 1,+ "costUsd": null,+ "inputTokens": 92186,+ "outputTokens": 21354,+ "wallSeconds": 349,+ "stopped": "SUCCESS",+ "requestedModel": "gpt-oss-120b-medium"+ }+ ]+}diff --git a/changes/69.md b/changes/69.mdindex 397ce376..5468d151 100644--- a/changes/69.md+++ b/changes/69.md@@ -1 +1 @@-- Record how other clients and Claude models use the docs and the MCP server (issue #69). A 5-task sample ran on haiku and sonnet through Claude Code, a no-MCP task gives an agent only `llms.txt` and the installed package (`bench/agent-modeling/llms-only.mjs`), and the results are in `bench/agent-modeling/results/2026-11-04-clients-models.json`. The guide and `llms.txt` now say that the render script assumes the `67js` alias install and what to change after a plain `npm install @codepawlhq/67js`; both sonnet runs first tried `node_modules/67js` and failed.+- Run the no-MCP task through the Antigravity CLI with a session-only command allow rule (`llms-only.mjs --agent agy`), and record the Gemini 4 Argon, Gemini 3.8 Flash and GPT-OSS 120B results (issue #69).