Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Bench: run the no-MCP task through agy with a session-only allow rule (issue #69)

Nguyen Xuan Anfb59a729175d
diff --git a/bench/agent-modeling/README.md b/bench/agent-modeling/README.md
index ee760092..49f447fa 100644
--- a/bench/agent-modeling/README.md
+++ b/bench/agent-modeling/README.md
@@ -393,4 +393,4 @@ Not run:
### Antigravity CLI (2026-11-07)
-`llms-only.mjs --agent agy --agy <agy executable> --model <model>` drives the Antigravity CLI, which offers Gemini 4 Argon, Gemini 3.8 Flash and GPT-OSS 120B. A first run of 2 attempts per model passed 0 of 6, but not on the docs: headless mode denies every shell command it cannot prompt for, so the agents read `llms.txt` and the package, wrote `render.mjs` (Gemini 4 Argon), and were refused `node render.mjs`. Those results are not kept. A valid run needs a session-only allow rule for commands, passed with `--settings`, which An has to approve; the runner records any web tool call as `usedWeb` and fails it, since the CLI cannot switch its web tools off.
+`llms-only.mjs --agent agy --agy <agy executable> --model <model>` drives the Antigravity CLI, which offers Gemini 4 Argon, Gemini 3.8 Flash and GPT-OSS 120B. Headless mode denies every shell command, so the runner passes a session-only settings file (`--settings`, never saved) with one rule, `permissions.allow: ["command(*)"]`, and removes it after the run. The CLI cannot switch its web tools off, so the runner records any web tool call as `usedWeb` and fails it. Run `npm run build:lib` first: the pack skips scripts, and the runner stops when `build/` is missing. With 2 attempts per model (`results/2026-11-07-agy.json`): Gemini 4 Argon 2 of 2, Gemini 3.8 Flash 2 of 2, GPT-OSS 120B 0 of 2 (it searched the web in every attempt despite the prompt, then timed out waiting for the page). One Gemini 4 Argon PNG was 768 px, which the check accepts because it requires a minimum side only. The MCP suite is not run through this CLI: `agy` reads MCP servers only from its global config, and `--settings` rejects `mcpServers`.
diff --git a/bench/agent-modeling/llms-only.mjs b/bench/agent-modeling/llms-only.mjs
index 2cdd9e06..3109a743 100644
--- a/bench/agent-modeling/llms-only.mjs
+++ b/bench/agent-modeling/llms-only.mjs
@@ -7,6 +7,7 @@
// Each attempt gets a fresh folder with the package installed from a packed archive, a copy of public/llms.txt and
// nothing else. The check opens scene.png and requires a real image with content (png-check.mjs).
import { spawn, execFileSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
import { copyFile, mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
@@ -41,6 +42,9 @@ const resultsDirectory = join(repository, 'bench', 'agent-modeling', 'results');
const npmExecutable = process.platform === 'win32' ? 'npm.cmd' : 'npm';
function packArchive(destination) {
+ // The pack runs with --ignore-scripts, so prepack never builds: without build/ the archive has no library
+ // and the agent cannot import anything (found in the first valid agy run, 2026-11-07).
+ if (!existsSync(join(repository, 'build', '67.module.js'))) throw new Error('build/67.module.js is missing; run npm run build:lib first');
const output = execFileSync(npmExecutable, ['pack', '--ignore-scripts', '--json', '--pack-destination', destination], { cwd: repository, encoding: 'utf8', shell: process.platform === 'win32' });
const packed = JSON.parse(output);
return join(destination, (Array.isArray(packed) ? packed[0] : Object.values(packed)[0]).filename);
@@ -63,18 +67,21 @@ function claudeArguments() {
];
}
-function agyArguments() {
+// Headless agy auto-denies shell commands, so each run gets a session-only settings file with one allow rule.
+// --settings is never saved to agy's own config, and the run's own folder is the working directory.
+function agyArguments(settingsPath) {
return [
'-p', PROMPT,
'--model', options.model,
+ '--settings', settingsPath,
'--output-format', 'stream-json',
'--print-timeout', `${options.timeout}s`,
];
}
-function runAgent(workingDirectory) {
+function runAgent(workingDirectory, settingsPath) {
const executable = options.agent === 'agy' ? options.agy : options.claude;
- const argumentsList = options.agent === 'agy' ? agyArguments() : claudeArguments();
+ const argumentsList = options.agent === 'agy' ? agyArguments(settingsPath) : claudeArguments();
return new Promise(resolvePromise => {
const started = performance.now();
const child = spawn(executable, argumentsList, { cwd: workingDirectory, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
@@ -168,7 +175,9 @@ async function runAttempt(archivePath, attempt) {
await writeFile(join(workingDirectory, 'package.json'), JSON.stringify({ name: 'llms-only-task', private: true, type: 'module' }));
execFileSync(npmExecutable, ['install', archivePath, '--ignore-scripts', '--no-audit', '--no-fund'], { cwd: workingDirectory, stdio: 'ignore', shell: process.platform === 'win32' });
await copyFile(join(repository, 'public', 'llms.txt'), join(workingDirectory, 'llms.txt'));
- const run = await runAgent(workingDirectory);
+ const settingsPath = `${workingDirectory}-agy-settings.json`;
+ await writeFile(settingsPath, JSON.stringify({ permissions: { allow: ['command(*)'] } }));
+ const run = await runAgent(workingDirectory, settingsPath).finally(() => rm(settingsPath, { force: true }));
const transcript = options.agent === 'agy' ? readAgyTranscript(run.output) : readTranscript(run.output);
const pngGrade = await checkPng(join(workingDirectory, 'scene.png'));
const grade = transcript.usedWeb ? { pass: false, detail: `used a web tool; ${pngGrade.detail}` } : pngGrade;
diff --git a/bench/agent-modeling/results/2026-11-07-agy.json b/bench/agent-modeling/results/2026-11-07-agy.json
new file mode 100644
index 00000000..17380d10
--- /dev/null
+++ b/bench/agent-modeling/results/2026-11-07-agy.json
@@ -0,0 +1,206 @@
+{
+ "run": "2026-11-07-agy",
+ "date": "2026-11-07T16:19:43.731Z",
+ "client": "Antigravity CLI (agy), headless, --settings with permissions.allow command(*), no MCP",
+ "task": "llms-only-png",
+ "runsPerModel": 2,
+ "note": "MCP suite not run: agy only reads MCP servers from its global config, and --settings rejects mcpServers. A 768x768 PNG from gemini-4-argon-medium attempt 1 passed because the check requires a minimum side only. gpt-oss-120b-medium ignored the no-internet instruction and called search_web in every attempt.",
+ "byModel": {
+ "gemini-3.8-flash-medium": {
+ "passed": 2,
+ "runs": 2
+ },
+ "gemini-4-argon-medium": {
+ "passed": 2,
+ "runs": 2
+ },
+ "gpt-oss-120b-medium": {
+ "passed": 0,
+ "runs": 2
+ }
+ },
+ "records": [
+ {
+ "task": "llms-only-png",
+ "attempt": 1,
+ "pass": true,
+ "detail": "512x512, 15973 colours, 84% differ from the corner",
+ "filesWritten": [
+ "llms.txt",
+ "package.json",
+ "render.mjs",
+ "scene.png"
+ ],
+ "model": "gemini-3.8-flash-medium",
+ "agent": "agy",
+ "usedWeb": false,
+ "toolCalls": 106,
+ "toolErrors": 1,
+ "toolsByName": {
+ "run_command": 77,
+ "read_file": 3,
+ "write_file": 4,
+ "manage_task": 22
+ },
+ "turns": 1,
+ "costUsd": null,
+ "inputTokens": 507011,
+ "outputTokens": 59018,
+ "wallSeconds": 547,
+ "stopped": "SUCCESS",
+ "requestedModel": "gemini-3.8-flash-medium"
+ },
+ {
+ "task": "llms-only-png",
+ "attempt": 2,
+ "pass": true,
+ "detail": "512x512, 12231 colours, 98% differ from the corner",
+ "filesWritten": [
+ "llms.txt",
+ "package.json",
+ "render.mjs",
+ "scene.png"
+ ],
+ "model": "gemini-3.8-flash-medium",
+ "agent": "agy",
+ "usedWeb": false,
+ "toolCalls": 38,
+ "toolErrors": 2,
+ "toolsByName": {
+ "read_file": 5,
+ "run_command": 26,
+ "manage_task": 4,
+ "write_file": 2,
+ "schedule": 1
+ },
+ "turns": 1,
+ "costUsd": null,
+ "inputTokens": 202678,
+ "outputTokens": 21615,
+ "wallSeconds": 255,
+ "stopped": "SUCCESS",
+ "requestedModel": "gemini-3.8-flash-medium"
+ },
+ {
+ "task": "llms-only-png",
+ "attempt": 1,
+ "pass": true,
+ "detail": "768x768, 20570 colours, 74% differ from the corner",
+ "filesWritten": [
+ "llms.txt",
+ "package.json",
+ "render.mjs",
+ "scene.png"
+ ],
+ "model": "gemini-4-argon-medium",
+ "agent": "agy",
+ "usedWeb": false,
+ "toolCalls": 22,
+ "toolErrors": 1,
+ "toolsByName": {
+ "read_file": 14,
+ "write_file": 3,
+ "run_command": 3,
+ "schedule": 2
+ },
+ "turns": 1,
+ "costUsd": null,
+ "inputTokens": 75531,
+ "outputTokens": 8769,
+ "wallSeconds": 275,
+ "stopped": "SUCCESS",
+ "requestedModel": "gemini-4-argon-medium"
+ },
+ {
+ "task": "llms-only-png",
+ "attempt": 2,
+ "pass": true,
+ "detail": "512x512, 14257 colours, 68% differ from the corner",
+ "filesWritten": [
+ "llms.txt",
+ "package.json",
+ "render.mjs",
+ "scene.png"
+ ],
+ "model": "gemini-4-argon-medium",
+ "agent": "agy",
+ "usedWeb": false,
+ "toolCalls": 36,
+ "toolErrors": 4,
+ "toolsByName": {
+ "read_file": 20,
+ "run_command": 7,
+ "write_file": 5,
+ "manage_task": 4
+ },
+ "turns": 1,
+ "costUsd": null,
+ "inputTokens": 148779,
+ "outputTokens": 16480,
+ "wallSeconds": 499,
+ "stopped": "SUCCESS",
+ "requestedModel": "gemini-4-argon-medium"
+ },
+ {
+ "task": "llms-only-png",
+ "attempt": 1,
+ "pass": false,
+ "detail": "used a web tool; the file does not exist",
+ "filesWritten": [
+ "llms.txt",
+ "package.json",
+ "render.mjs"
+ ],
+ "model": "gpt-oss-120b-medium",
+ "agent": "agy",
+ "usedWeb": true,
+ "toolCalls": 23,
+ "toolErrors": 1,
+ "toolsByName": {
+ "read_file": 5,
+ "search_web": 4,
+ "run_command": 5,
+ "write_file": 1,
+ "manage_task": 5,
+ "edit_file": 3
+ },
+ "turns": null,
+ "costUsd": null,
+ "inputTokens": 0,
+ "outputTokens": 0,
+ "wallSeconds": 900,
+ "stopped": "no result",
+ "requestedModel": "gpt-oss-120b-medium"
+ },
+ {
+ "task": "llms-only-png",
+ "attempt": 2,
+ "pass": false,
+ "detail": "used a web tool; the file does not exist",
+ "filesWritten": [
+ "llms.txt",
+ "package.json",
+ "render.mjs"
+ ],
+ "model": "gpt-oss-120b-medium",
+ "agent": "agy",
+ "usedWeb": true,
+ "toolCalls": 40,
+ "toolErrors": 3,
+ "toolsByName": {
+ "read_file": 8,
+ "search_web": 6,
+ "run_command": 19,
+ "write_file": 2,
+ "edit_file": 5
+ },
+ "turns": 1,
+ "costUsd": null,
+ "inputTokens": 92186,
+ "outputTokens": 21354,
+ "wallSeconds": 349,
+ "stopped": "SUCCESS",
+ "requestedModel": "gpt-oss-120b-medium"
+ }
+ ]
+}
diff --git a/changes/69.md b/changes/69.md
index 397ce376..5468d151 100644
--- a/changes/69.md
+++ b/changes/69.md
@@ -1 +1 @@
-- Record how other clients and Claude models use the docs and the MCP server (issue #69). A 5-task sample ran on haiku and sonnet through Claude Code, a no-MCP task gives an agent only `llms.txt` and the installed package (`bench/agent-modeling/llms-only.mjs`), and the results are in `bench/agent-modeling/results/2026-11-04-clients-models.json`. The guide and `llms.txt` now say that the render script assumes the `67js` alias install and what to change after a plain `npm install @codepawlhq/67js`; both sonnet runs first tried `node_modules/67js` and failed.
+- Run the no-MCP task through the Antigravity CLI with a session-only command allow rule (`llms-only.mjs --agent agy`), and record the Gemini 4 Argon, Gemini 3.8 Flash and GPT-OSS 120B results (issue #69).