The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
/**
* workspace — the folder the test agent is created in, laid out like a person's real project:
* the notes the file steps read and edit, an MCP server registered for the engine
* (`e2e_calc`), and the log it writes to. The scenario (`smokeChecks.ts`) names them; the probe
* reads the files and the log.
*
* <cwd>/notes/info-N.txt read: a token made fresh for this run
* <cwd>/notes/todo-N.txt edit: `status: pending`
* <cwd>/tools/calc-mcp.mjs the MCP server
* <cwd>/.mcp.json claude: project-level MCP server
* ~/.codex/config.toml codex: registered with `codex mcp add` — see below
* <cwd>/.e2e/calc-mcp.log one line per MCP tools/call
*
* The two engines differ, and both shapes here were measured against the installed CLIs rather than
* assumed:
*
* * claude reads `<cwd>/.mcp.json` — `claude mcp list` in the workspace lists `e2e_calc` and a
* headless run really reaches the server. The agent runs in the app's default mode (`auto`), so
* the project's `.claude/settings.json` allows the steps' tools the way a person who answered
* "don't ask again" has them (`allowClaudeProjectTools`).
* * codex has NO project-level MCP config. A `<cwd>/.codex/config.toml` is never read (from inside
* such a workspace, `codex mcp list` does not list the server), so registration goes through
* codex's own `codex mcp add`, which writes `~/.codex/config.toml`, with its tools always allowed
* (`approveCodexMcpTools`) — and `removeCodexMcp` takes the entry out again when the run is over.
*/
import { mkdirSync, readFileSync, writeFileSync, existsSync, renameSync, realpathSync } from 'node:fs'
import { execFileSync } from 'node:child_process'
import { randomInt } from 'node:crypto'
import { homedir } from 'node:os'
import { join } from 'node:path'
import type { LogKind } from './smokeChecks.js'
// Written from this string, not copied from beside this file: the published CLI is ONE bundled
// `cli.js` (build-bundle.mjs), and files next to the source never reach an installed machine. The
// original under `workspace/` is the readable copy; the spec pins them equal.
export const CALC_MCP_MJS = `#!/usr/bin/env node
/**
* e2e_calc — the smallest MCP server that can prove "the tool still uses MCP after a switch".
*
* stdio transport, newline-delimited JSON-RPC, two tools: add(a, b) and sub(a, b). Every call is
* appended to the log file given as argv[2] (\`<ts> <tool> <a> <b> = <result>\`), so the e2e has hard
* evidence the model went THROUGH the server rather than doing the arithmetic in its head.
* No SDK on purpose: nothing to install in the scratch workspace the test agent runs in.
*/
import { appendFileSync } from 'node:fs'
import { createInterface } from 'node:readline'
const logFile = process.argv[2]
const TOOLS = [
{ name: 'add', description: 'Add two numbers: a + b', inputSchema: { type: 'object', properties: { a: { type: 'number' }, b: { type: 'number' } }, required: ['a', 'b'] } },
{ name: 'sub', description: 'Subtract two numbers: a - b', inputSchema: { type: 'object', properties: { a: { type: 'number' }, b: { type: 'number' } }, required: ['a', 'b'] } },
]
function send(msg) {
process.stdout.write(JSON.stringify(msg) + '\\n')
}
function log(line) {
if (logFile) appendFileSync(logFile, \`\${new Date().toISOString()} \${line}\\n\`)
}
const rl = createInterface({ input: process.stdin })
rl.on('line', (line) => {
if (!line.trim()) return
let req
try { req = JSON.parse(line) } catch { return }
const { id, method, params } = req
if (id === undefined) return // notification (notifications/initialized, …)
switch (method) {
case 'initialize':
send({ jsonrpc: '2.0', id, result: { protocolVersion: params?.protocolVersion ?? '2024-11-05', capabilities: { tools: {} }, serverInfo: { name: 'e2e_calc', version: '1.0.0' } } })
return
case 'ping':
send({ jsonrpc: '2.0', id, result: {} })
return
case 'tools/list':
send({ jsonrpc: '2.0', id, result: { tools: TOOLS } })
return
case 'tools/call': {
const name = params?.name
const a = Number(params?.arguments?.a)
const b = Number(params?.arguments?.b)
if ((name !== 'add' && name !== 'sub') || Number.isNaN(a) || Number.isNaN(b)) {
send({ jsonrpc: '2.0', id, result: { content: [{ type: 'text', text: \`error: unknown tool or bad arguments (\${name})\` }], isError: true } })
return
}
const result = name === 'add' ? a + b : a - b
log(\`\${name} \${a} \${b} = \${result}\`)
send({ jsonrpc: '2.0', id, result: { content: [{ type: 'text', text: String(result) }] } })
return
}
default:
send({ jsonrpc: '2.0', id, error: { code: -32601, message: \`method not found: \${method}\` } })
}
})
`
export const MCP_SERVER_NAME = 'e2e_calc'
/** The tools `CALC_MCP_MJS` serves. */
export const CALC_MCP_TOOLS = ['add', 'sub'] as const
/**
* What a person using claude in this project has long since answered "Yes, and don't ask again"
* to: the file tools, reading a file through the shell, the project's MCP server and
* the harness's web search on a grid — kept by claude in the project's `.claude/settings.json`
* (`permissions.allow`), with the `.mcp.json` server approved in the same file
* (`enabledMcpjsonServers`). Measured on claude 2.1.274 + Haiku in `auto`: without it a shell step
* stopped on "This command requires approval".
*/
export function allowClaudeProjectTools(cwd: string): void {
mkdirSync(join(cwd, '.claude'), { recursive: true })
const settings = {
permissions: {
allow: [
'Read', 'Write', 'Edit',
'Bash(cat:*)',
`mcp__${MCP_SERVER_NAME}`,
'mcp__harness',
],
},
enabledMcpjsonServers: [MCP_SERVER_NAME],
}
writeFileSync(join(cwd, '.claude', 'settings.json'), JSON.stringify(settings, null, 2) + '\n')
}
export function logPath(cwd: string, _kind: LogKind): string {
return join(cwd, '.e2e', 'calc-mcp.log')
}
export interface WorkspaceLayout {
mcpServer: string
/** Where this engine's MCP registration ended up, or null when there is none. */
mcpConfig: string | null
/** How it got there — or, when `mcpConfig` is null, why the MCP steps cannot pass. */
mcpNote: string
}
/**
* Register the workspace's MCP server with codex, through codex's own command.
*
* `remove` first so a stale entry from an interrupted run (pointing at a workspace that no longer
* exists) cannot survive as a server codex fails to start. Both calls are best-effort: a codex too
* old for `mcp add` must leave a run without MCP rather than without a run, and the caller reports
* that in the trace.
*/
export function registerCodexMcp(mcpServer: string, mcpLog: string, bin = 'codex'): { ok: true } | { ok: false; detail: string } {
const run = (args: string[]): string => execFileSync(bin, args, { encoding: 'utf8', timeout: 60_000, stdio: ['ignore', 'pipe', 'pipe'] })
try { run(['mcp', 'remove', MCP_SERVER_NAME]) } catch { /* not registered — the normal case */ }
try {
run(['mcp', 'add', MCP_SERVER_NAME, '--', 'node', mcpServer, mcpLog])
} catch (err) {
const e = err as { stderr?: string; message?: string }
return { ok: false, detail: (e.stderr || e.message || String(err)).trim().split('\n')[0] }
}
// Registered is not the same as visible: ask codex what it sees, so a run never starts believing
// in a server the engine will not load.
try {
if (!run(['mcp', 'list']).includes(MCP_SERVER_NAME)) return { ok: false, detail: `codex mcp add reported success but 'codex mcp list' does not show ${MCP_SERVER_NAME}` }
} catch { /* `mcp list` can fail on its own (a broken unrelated server); the add is what matters */ }
return { ok: true }
}
/**
* Mark the server's tools as always allowed, the entry codex writes itself when a person answers a
* tool's approval with "always allow" — how every MCP tool on a working developer's machine ends up
* (`[mcp_servers.<server>.tools.<tool>] approval_mode = "approve"`). Without it a tool with no
* `readOnlyHint` needs approval on every call (codex-rs/core/src/mcp_tool_call.rs
* `requires_mcp_tool_approval`), and in `auto` that approval is codex's reviewer, which asks for a
* model named `codex-auto-review` — one only OpenAI serves: on the grid it answered 503 and codex
* refused the call (openai/codex#24879). Nested under the server's table, so `codex mcp remove`
* takes it out with the server.
*/
export function approveCodexMcpTools(configPath: string, tools: readonly string[]): void {
const lines = tools.map((t) => `\n[mcp_servers.${MCP_SERVER_NAME}.tools.${t}]\napproval_mode = "approve"\n`).join('')
writeFileSync(configPath, lines, { flag: 'a' })
}
/** Take the entry back out — the run's own cleanup, so the next run starts from a clean config. */
export function removeCodexMcp(bin = 'codex'): boolean {
try {
execFileSync(bin, ['mcp', 'remove', MCP_SERVER_NAME], { encoding: 'utf8', timeout: 60_000, stdio: ['ignore', 'pipe', 'pipe'] })
return true
} catch {
return false
}
}
/**
* Lay the workspace out for `engine`. Idempotent. Returns what was written, for the trace.
*
* `codexBin` exists so a test can watch the registration without running the real `codex mcp add`:
* that command writes the developer's own `~/.codex/config.toml`, and a unit test has no business
* leaving an entry there pointing at a temp folder it is about to delete.
*/
export function prepareWorkspace(cwd: string, engine: string, opts: { codexBin?: string; codexConfig?: string } = {}): WorkspaceLayout {
mkdirSync(join(cwd, 'tools'), { recursive: true })
mkdirSync(join(cwd, '.e2e'), { recursive: true })
writeFileSync(logPath(cwd, 'mcp'), '', { flag: 'a' })
// read / write / edit (smokeChecks.ts): one set of files per side, so the grid side can never pass
// (named `info-N.txt`, not `secret-N.txt`: gpt-6-luna refused outright to read a file called
// "secret" — "I can't provide the contents of a file named notes/secret-1.txt" — and a test that
// provokes a refusal measures the refusal, not the switch)
// on what the first side left. The token is made HERE, fresh for every run, and written nowhere
// but its file — not in a prompt, not in a log — so the only way to name it is to read it.
mkdirSync(join(cwd, 'notes'), { recursive: true })
mkdirSync(join(cwd, 'out'), { recursive: true })
for (const n of [1, 2]) {
writeFileSync(join(cwd, 'notes', `info-${n}.txt`), `${freshToken()}\n`)
writeFileSync(join(cwd, 'notes', `todo-${n}.txt`), `# todo ${n}\nstatus: pending\n`)
}
// The MCP server runs from the workspace too, so the engine's config points at a path that exists
// on every machine the e2e runs on.
const mcpServer = join(cwd, 'tools', 'calc-mcp.mjs')
writeFileSync(mcpServer, CALC_MCP_MJS)
const mcpLog = logPath(cwd, 'mcp')
let mcpConfig: string | null = null
let mcpNote = `${engine} has no MCP registration here: the MCP steps cannot pass`
if (engine === 'codex') {
const registered = registerCodexMcp(mcpServer, mcpLog, opts.codexBin ?? 'codex')
if (registered.ok) {
mcpConfig = opts.codexConfig ?? join(homedir(), '.codex', 'config.toml')
approveCodexMcpTools(mcpConfig, CALC_MCP_TOOLS)
mcpNote = `registered with \`codex mcp add\`, tools ${CALC_MCP_TOOLS.join('/')} always allowed (codex does not read a project-level .codex/config.toml)`
} else {
mcpNote = `codex mcp add failed: ${registered.detail}`
}
} else if (engine === 'claude') {
mcpConfig = join(cwd, '.mcp.json')
writeFileSync(mcpConfig, JSON.stringify({ mcpServers: { [MCP_SERVER_NAME]: { command: 'node', args: [mcpServer, mcpLog] } } }, null, 2) + '\n')
preApproveClaudeMcp(cwd, [MCP_SERVER_NAME])
allowClaudeProjectTools(cwd)
mcpNote = 'project .mcp.json; the steps\' tools always allowed in .claude/settings.json'
}
return { mcpServer, mcpConfig, mcpNote }
}
/**
* Claude Code loads a project's `.mcp.json` servers only once the person approved them; the answer
* is kept in `~/.claude.json` as `projects[<cwd>].enabledMcpjsonServers`. For a workspace this run
* made, that answer is the run's to give — same posture and same file handling as the daemon's
* `preTrustClaudeProject` (lib/claudeTrust.ts): only add, never touch a shape we did not expect.
*
* Measured against claude 2.1.x, this is NOT what carries the MCP steps, and the file says so rather
* than leaving the next reader to re-measure: claude rewrites `~/.claude.json` at startup and the
* key is gone afterwards (`claude mcp list` still says "Pending approval"). What actually reaches
* the server is the `full` permission mode the agent is created in. Kept because it costs one write
* and is the right answer if a future claude honours it; never relied on.
*/
export function preApproveClaudeMcp(cwd: string, names: string[], home = homedir()): 'approved' | 'already' | 'skipped' {
const file = join(home, '.claude.json')
if (!existsSync(file)) return 'skipped'
let config: unknown
try {
config = JSON.parse(readFileSync(file, 'utf8'))
} catch {
return 'skipped'
}
const isObj = (v: unknown): v is Record<string, unknown> => typeof v === 'object' && v !== null && !Array.isArray(v)
if (!isObj(config)) return 'skipped'
if (config.projects !== undefined && !isObj(config.projects)) return 'skipped'
const projects = (config.projects ?? {}) as Record<string, unknown>
const existing = projects[cwd]
if (existing !== undefined && !isObj(existing)) return 'skipped'
const current = Array.isArray(existing?.enabledMcpjsonServers) ? (existing!.enabledMcpjsonServers as unknown[]) : []
if (names.every((n) => current.includes(n))) return 'already'
projects[cwd] = { allowedTools: [], ...(existing ?? {}), enabledMcpjsonServers: [...current, ...names.filter((n) => !current.includes(n))] }
config.projects = projects
// Atomic, through a symlink (dotfile managers link ~/.claude.json), like the daemon does.
const target = realpathSync(file)
const tmp = `${target}.e2e-${process.pid}.tmp`
writeFileSync(tmp, JSON.stringify(config, null, 2), { mode: 0o600 })
renameSync(tmp, target)
return 'approved'
}
/**
* Accept Claude Code's Bypass Permissions warning ahead of time.
*
* `--dangerously-skip-permissions` does not go straight to the prompt the first time: an
* interactive claude draws a full-screen "WARNING: Claude Code running in Bypass Permissions mode"
* with **"No, exit" selected**, and on Enter it exits 1 — the pane becomes a shell and every check
* of the run is stuck with no engine behind it. (A headless `claude -p` never shows it, which is
* why the flag looks harmless until it runs in a pane.) The answer is remembered in `~/.claude.json`
* as `bypassPermissionsModeAccepted`, a key read out of claude 2.1.281's own binary rather than
* guessed — so the run gives it the same way a person would, once, before the agent starts.
*
* It is NOT sufficient on its own, and the next reader should not have to find that out in a failed
* run: on the pass that proved this, the flag was written (`accepted`) and the warning still came up
* — `paneProbe`'s `claude-bypass-accept` answered it, and the leg passed. So the dialog handler is
* what carries the run and this write is the belt: cheap, correct, and it may be what keeps the
* screen away on a machine that reads the flag earlier.
*/
export function preAcceptClaudeBypassMode(home = homedir()): 'accepted' | 'already' | 'skipped' {
const file = join(home, '.claude.json')
if (!existsSync(file)) return 'skipped'
let config: unknown
try {
config = JSON.parse(readFileSync(file, 'utf8'))
} catch {
return 'skipped'
}
if (typeof config !== 'object' || config === null || Array.isArray(config)) return 'skipped'
const conf = config as Record<string, unknown>
if (conf.bypassPermissionsModeAccepted === true) return 'already'
conf.bypassPermissionsModeAccepted = true
const target = realpathSync(file)
const tmp = `${target}.e2e-${process.pid}.tmp`
writeFileSync(tmp, JSON.stringify(conf, null, 2), { mode: 0o600 })
renameSync(tmp, target)
return 'accepted'
}
const WORDS = ['kiwi', 'tulip', 'cobalt', 'maple', 'otter', 'saffron', 'glacier', 'lantern', 'pepper', 'quartz', 'harbor', 'violet', 'ember', 'falcon', 'juniper', 'meadow']
/** `kiwi-4821-tulip`: two words and four digits, ~2.5 million combinations, new every run. */
export function freshToken(): string {
return `${WORDS[randomInt(WORDS.length)]}-${randomInt(1000, 10000)}-${WORDS[randomInt(WORDS.length)]}`
}
/** A workspace file's content, or null when it does not exist — what the probe checks write/edit/read against. */
export function readWorkspaceFile(cwd: string, relPath: string): string | null {
try {
return readFileSync(join(cwd, relPath), 'utf8')
} catch {
return null
}
}
/** The log's lines right now (empty when it does not exist yet). */
export function readLog(cwd: string, kind: LogKind): string[] {
try {
return readFileSync(logPath(cwd, kind), 'utf8').split('\n').filter(Boolean)
} catch {
return []
}
}