3D engine that AI agents drive
Let the llms-only task drive the Antigravity CLI for non-Anthropic models
Nguyen Xuan An10b3beaa5f2c
diff --git a/bench/agent-modeling/README.md b/bench/agent-modeling/README.mdindex 9409f635..ee760092 100644--- a/bench/agent-modeling/README.md+++ b/bench/agent-modeling/README.md@@ -390,3 +390,7 @@ Not run:| Gemini CLI | Needs a Google sign-in or API key, which this setup does not have || VS Code | Interactive client behind a Copilot subscription that is not set up here || Claude Desktop | A GUI app that cannot be driven without computer use |++### Antigravity CLI (2026-11-07)++`llms-only.mjs --agent agy --agy <agy executable> --model <model>` drives the Antigravity CLI, which offers Gemini 4 Argon, Gemini 3.8 Flash and GPT-OSS 120B. A first run of 2 attempts per model passed 0 of 6, but not on the docs: headless mode denies every shell command it cannot prompt for, so the agents read `llms.txt` and the package, wrote `render.mjs` (Gemini 4 Argon), and were refused `node render.mjs`. Those results are not kept. A valid run needs a session-only allow rule for commands, passed with `--settings`, which An has to approve; the runner records any web tool call as `usedWeb` and fails it, since the CLI cannot switch its web tools off.diff --git a/bench/agent-modeling/llms-only.mjs b/bench/agent-modeling/llms-only.mjsindex c25d2d1e..2cdd9e06 100644--- a/bench/agent-modeling/llms-only.mjs+++ b/bench/agent-modeling/llms-only.mjs@@ -1,6 +1,9 @@// A coding task with no MCP: the agent has `llms.txt` and the installed npm package, and must write a script that// renders a scene to a PNG. This measures the path a coding agent takes when it reads the docs instead of calling tools.// node bench/agent-modeling/llms-only.mjs --claude <claude executable> [--model sonnet] [--runs 2] [--budget 1.5] [--timeout 900] [--label llms-only]+// node bench/agent-modeling/llms-only.mjs --agent agy --agy <agy executable> --model gemini-3.8-flash-medium [--runs 2]+// --agent agy drives the Antigravity CLI (Google and other non-Anthropic models). It cannot switch off its web tools,+// so a run that calls one is recorded with usedWeb and fails the task.// Each attempt gets a fresh folder with the package installed from a packed archive, a copy of public/llms.txt and// nothing else. The check opens scene.png and requires a real image with content (png-check.mjs).import { spawn, execFileSync } from 'node:child_process';@@ -21,7 +24,9 @@ Google Chrome is installed. When scene.png exists, reply with one line saying soconst { values: options } = parseArgs({options: {+ agent: { type: 'string', default: 'claude' },claude: { type: 'string', default: process.env.CLAUDE_EXECUTABLE ?? 'claude' },+ agy: { type: 'string', default: process.env.AGY_EXECUTABLE ?? 'agy' },model: { type: 'string', default: 'sonnet' },runs: { type: 'string', default: '2' },budget: { type: 'string', default: '1.5' },@@ -41,8 +46,10 @@ function packArchive(destination) {return join(destination, (Array.isArray(packed) ? packed[0] : Object.values(packed)[0]).filename);}-function runClaude(workingDirectory) {- const argumentsList = [+const AGY_WEB_TOOLS = new Set(['search_web', 'read_url_content', 'open_browser_url', 'read_browser_page']);++function claudeArguments() {+ return ['-p', PROMPT,'--model', options.model,'--output-format', 'stream-json', '--verbose',@@ -54,9 +61,23 @@ function runClaude(workingDirectory) {'--no-session-persistence','--max-budget-usd', options.budget,];+}++function agyArguments() {+ return [+ '-p', PROMPT,+ '--model', options.model,+ '--output-format', 'stream-json',+ '--print-timeout', `${options.timeout}s`,+ ];+}++function runAgent(workingDirectory) {+ const executable = options.agent === 'agy' ? options.agy : options.claude;+ const argumentsList = options.agent === 'agy' ? agyArguments() : claudeArguments();return new Promise(resolvePromise => {const started = performance.now();- const child = spawn(options.claude, argumentsList, { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });+ const child = spawn(executable, argumentsList, { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });let output = '';let errors = '';child.stdout.on('data', chunk => {@@ -73,6 +94,38 @@ function runClaude(workingDirectory) {});}+// The Antigravity CLI writes init, step_update and result events; its tool steps carry tool_name.+function readAgyTranscript(output) {+ const summary = { model: null, toolCalls: 0, toolErrors: 0, toolsByName: {}, result: null, usedWeb: false };+ for (const line of output.split('\n')) {+ if (!line.trim().startsWith('{')) continue;+ let event;+ try {+ event = JSON.parse(line);+ } catch {+ continue;+ }+ if (event.event === 'init') summary.model = event.init?.model ?? null;+ const step = event.step_update;+ if (step?.step_type === 'tool' && step.state === 'DONE') {+ summary.toolCalls++;+ summary.toolsByName[step.tool_name] = (summary.toolsByName[step.tool_name] ?? 0) + 1;+ if (AGY_WEB_TOOLS.has(step.tool_name)) summary.usedWeb = true;+ }+ if (step?.step_type === 'tool' && step.state === 'ERROR') summary.toolErrors++;+ if (event.event === 'result') {+ const usage = event.result?.usage ?? {};+ summary.result = {+ num_turns: event.result?.num_turns ?? null,+ total_cost_usd: null,+ subtype: event.result?.status ?? 'no result',+ usage: { input_tokens: usage.input_tokens ?? 0, output_tokens: (usage.output_tokens ?? 0) + (usage.thinking_tokens ?? 0) },+ };+ }+ }+ return summary;+}+function readTranscript(output) {const summary = { model: null, toolCalls: 0, toolErrors: 0, toolsByName: {}, result: null };for (const line of output.split('\n')) {@@ -115,9 +168,10 @@ async function runAttempt(archivePath, attempt) {await writeFile(join(workingDirectory, 'package.json'), JSON.stringify({ name: 'llms-only-task', private: true, type: 'module' }));execFileSync(npmExecutable, ['install', archivePath, '--ignore-scripts', '--no-audit', '--no-fund'], { cwd: workingDirectory, stdio: 'ignore', shell: process.platform === 'win32' });await copyFile(join(repository, 'public', 'llms.txt'), join(workingDirectory, 'llms.txt'));- const run = await runClaude(workingDirectory);- const transcript = readTranscript(run.output);- const grade = await checkPng(join(workingDirectory, 'scene.png'));+ const run = await runAgent(workingDirectory);+ const transcript = options.agent === 'agy' ? readAgyTranscript(run.output) : readTranscript(run.output);+ const pngGrade = await checkPng(join(workingDirectory, 'scene.png'));+ const grade = transcript.usedWeb ? { pass: false, detail: `used a web tool; ${pngGrade.detail}` } : pngGrade;await writeFile(join(rawDirectory, `${TASK_ID}-${attempt}.jsonl`), run.output);const filesWritten = (await readdir(workingDirectory)).filter(name => !['node_modules', 'package-lock.json'].includes(name));const usage = transcript.result?.usage ?? {};@@ -128,6 +182,8 @@ async function runAttempt(archivePath, attempt) {detail: removeLocalPaths(grade.detail),filesWritten,model: transcript.model,+ agent: options.agent,+ usedWeb: transcript.usedWeb ?? false,toolCalls: transcript.toolCalls,toolErrors: transcript.toolErrors,toolsByName: transcript.toolsByName,