Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Let the llms-only task drive the Antigravity CLI for non-Anthropic models

Nguyen Xuan An10b3beaa5f2c
diff --git a/bench/agent-modeling/README.md b/bench/agent-modeling/README.md
index 9409f635..ee760092 100644
--- a/bench/agent-modeling/README.md
+++ b/bench/agent-modeling/README.md
@@ -390,3 +390,7 @@ Not run:
| Gemini CLI | Needs a Google sign-in or API key, which this setup does not have |
| VS Code | Interactive client behind a Copilot subscription that is not set up here |
| Claude Desktop | A GUI app that cannot be driven without computer use |
+
+### Antigravity CLI (2026-11-07)
+
+`llms-only.mjs --agent agy --agy <agy executable> --model <model>` drives the Antigravity CLI, which offers Gemini 4 Argon, Gemini 3.8 Flash and GPT-OSS 120B. A first run of 2 attempts per model passed 0 of 6, but not on the docs: headless mode denies every shell command it cannot prompt for, so the agents read `llms.txt` and the package, wrote `render.mjs` (Gemini 4 Argon), and were refused `node render.mjs`. Those results are not kept. A valid run needs a session-only allow rule for commands, passed with `--settings`, which An has to approve; the runner records any web tool call as `usedWeb` and fails it, since the CLI cannot switch its web tools off.
diff --git a/bench/agent-modeling/llms-only.mjs b/bench/agent-modeling/llms-only.mjs
index c25d2d1e..2cdd9e06 100644
--- a/bench/agent-modeling/llms-only.mjs
+++ b/bench/agent-modeling/llms-only.mjs
@@ -1,6 +1,9 @@
// A coding task with no MCP: the agent has `llms.txt` and the installed npm package, and must write a script that
// renders a scene to a PNG. This measures the path a coding agent takes when it reads the docs instead of calling tools.
// node bench/agent-modeling/llms-only.mjs --claude <claude executable> [--model sonnet] [--runs 2] [--budget 1.5] [--timeout 900] [--label llms-only]
+// node bench/agent-modeling/llms-only.mjs --agent agy --agy <agy executable> --model gemini-3.8-flash-medium [--runs 2]
+// --agent agy drives the Antigravity CLI (Google and other non-Anthropic models). It cannot switch off its web tools,
+// so a run that calls one is recorded with usedWeb and fails the task.
// Each attempt gets a fresh folder with the package installed from a packed archive, a copy of public/llms.txt and
// nothing else. The check opens scene.png and requires a real image with content (png-check.mjs).
import { spawn, execFileSync } from 'node:child_process';
@@ -21,7 +24,9 @@ Google Chrome is installed. When scene.png exists, reply with one line saying so
const { values: options } = parseArgs({
options: {
+ agent: { type: 'string', default: 'claude' },
claude: { type: 'string', default: process.env.CLAUDE_EXECUTABLE ?? 'claude' },
+ agy: { type: 'string', default: process.env.AGY_EXECUTABLE ?? 'agy' },
model: { type: 'string', default: 'sonnet' },
runs: { type: 'string', default: '2' },
budget: { type: 'string', default: '1.5' },
@@ -41,8 +46,10 @@ function packArchive(destination) {
return join(destination, (Array.isArray(packed) ? packed[0] : Object.values(packed)[0]).filename);
}
-function runClaude(workingDirectory) {
- const argumentsList = [
+const AGY_WEB_TOOLS = new Set(['search_web', 'read_url_content', 'open_browser_url', 'read_browser_page']);
+
+function claudeArguments() {
+ return [
'-p', PROMPT,
'--model', options.model,
'--output-format', 'stream-json', '--verbose',
@@ -54,9 +61,23 @@ function runClaude(workingDirectory) {
'--no-session-persistence',
'--max-budget-usd', options.budget,
];
+}
+
+function agyArguments() {
+ return [
+ '-p', PROMPT,
+ '--model', options.model,
+ '--output-format', 'stream-json',
+ '--print-timeout', `${options.timeout}s`,
+ ];
+}
+
+function runAgent(workingDirectory) {
+ const executable = options.agent === 'agy' ? options.agy : options.claude;
+ const argumentsList = options.agent === 'agy' ? agyArguments() : claudeArguments();
return new Promise(resolvePromise => {
const started = performance.now();
- const child = spawn(options.claude, argumentsList, { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
+ const child = spawn(executable, argumentsList, { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let output = '';
let errors = '';
child.stdout.on('data', chunk => {
@@ -73,6 +94,38 @@ function runClaude(workingDirectory) {
});
}
+// The Antigravity CLI writes init, step_update and result events; its tool steps carry tool_name.
+function readAgyTranscript(output) {
+ const summary = { model: null, toolCalls: 0, toolErrors: 0, toolsByName: {}, result: null, usedWeb: false };
+ for (const line of output.split('\n')) {
+ if (!line.trim().startsWith('{')) continue;
+ let event;
+ try {
+ event = JSON.parse(line);
+ } catch {
+ continue;
+ }
+ if (event.event === 'init') summary.model = event.init?.model ?? null;
+ const step = event.step_update;
+ if (step?.step_type === 'tool' && step.state === 'DONE') {
+ summary.toolCalls++;
+ summary.toolsByName[step.tool_name] = (summary.toolsByName[step.tool_name] ?? 0) + 1;
+ if (AGY_WEB_TOOLS.has(step.tool_name)) summary.usedWeb = true;
+ }
+ if (step?.step_type === 'tool' && step.state === 'ERROR') summary.toolErrors++;
+ if (event.event === 'result') {
+ const usage = event.result?.usage ?? {};
+ summary.result = {
+ num_turns: event.result?.num_turns ?? null,
+ total_cost_usd: null,
+ subtype: event.result?.status ?? 'no result',
+ usage: { input_tokens: usage.input_tokens ?? 0, output_tokens: (usage.output_tokens ?? 0) + (usage.thinking_tokens ?? 0) },
+ };
+ }
+ }
+ return summary;
+}
+
function readTranscript(output) {
const summary = { model: null, toolCalls: 0, toolErrors: 0, toolsByName: {}, result: null };
for (const line of output.split('\n')) {
@@ -115,9 +168,10 @@ async function runAttempt(archivePath, attempt) {
await writeFile(join(workingDirectory, 'package.json'), JSON.stringify({ name: 'llms-only-task', private: true, type: 'module' }));
execFileSync(npmExecutable, ['install', archivePath, '--ignore-scripts', '--no-audit', '--no-fund'], { cwd: workingDirectory, stdio: 'ignore', shell: process.platform === 'win32' });
await copyFile(join(repository, 'public', 'llms.txt'), join(workingDirectory, 'llms.txt'));
- const run = await runClaude(workingDirectory);
- const transcript = readTranscript(run.output);
- const grade = await checkPng(join(workingDirectory, 'scene.png'));
+ const run = await runAgent(workingDirectory);
+ const transcript = options.agent === 'agy' ? readAgyTranscript(run.output) : readTranscript(run.output);
+ const pngGrade = await checkPng(join(workingDirectory, 'scene.png'));
+ const grade = transcript.usedWeb ? { pass: false, detail: `used a web tool; ${pngGrade.detail}` } : pngGrade;
await writeFile(join(rawDirectory, `${TASK_ID}-${attempt}.jsonl`), run.output);
const filesWritten = (await readdir(workingDirectory)).filter(name => !['node_modules', 'package-lock.json'].includes(name));
const usage = transcript.result?.usage ?? {};
@@ -128,6 +182,8 @@ async function runAttempt(archivePath, attempt) {
detail: removeLocalPaths(grade.detail),
filesWritten,
model: transcript.model,
+ agent: options.agent,
+ usedWeb: transcript.usedWeb ?? false,
toolCalls: transcript.toolCalls,
toolErrors: transcript.toolErrors,
toolsByName: transcript.toolsByName,