The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
/**
* Run the grid-switch journey for REAL: create a fresh agent through the daemon (like "New agent"
* in the app), run it subscription -> grid -> back home with the scenario on every leg, delete
* it, and leave a review bundle the `e2e-watchdog` agent (autonomous-grid-cli, e2e/) reads.
*
* agent_create — a new codex/claude in its own tmux pane, in a scratch folder; nobody's pane is borrowed
* subscription — checks on it as it is (its own login): the baseline
* grid — agent_retarget onto a grid model (pane respawned, session resumed), checks again
* agent_delete — gone at the end, unless E2E_KEEP_AGENT=1 (the reviewer, autonomous-grid-cli's
* e2e-watchdog, reads the bundle afterwards; keep the agent when it should type into the pane)
*
* Usage (zero-config; everything is read from this machine's harness install):
* harness e2e grid-switch (or: npx tsx src/e2e/runGridSwitchTrace.ts)
*
* Env (all optional):
* HARNESS_ENGINE=codex|claude engine under test (default codex)
* HARNESS_GRID_MODEL / HARNESS_GRID_NAME pin the grid model (default: smallest listed — pin the fastest
* one from `grid stats <grid> --verbose` when it matters)
* HARNESS_AGENT_ID + HARNESS_PANE_ID reuse an existing agent instead of creating one (not deleted)
* HARNESS_PORT / HARNESS_MACHINE_ID override what env.ts / computer-id say
* E2E_GRID_HOME + E2E_GRID_NAME (+ E2E_GRID_CONTROL_PLANE_URL) move onto a grid from ANOTHER grid home —
* a local relay checkout under test — instead of the daemon's own
* E2E_KEEP_AGENT=1 leave the created agent running for inspection
*
* The daemon is the one `harness` started (`ws://127.0.0.1:${PORT}/api/local-ws`) — the exact
* transport and commands the desktop app uses.
*/
import { mkdirSync, realpathSync, writeFileSync, cpSync, existsSync, readFileSync } from 'node:fs'
import { join } from 'node:path'
import { execFile } from 'node:child_process'
import { promisify } from 'node:util'
import { GridSwitchDriver, pickGridModel, type GridSwitchTraceStep } from './gridSwitchDriver.js'
import { probeLeg, realTmux, waitForPaneSettle } from './paneProbe.js'
import { firstStuck, plannedLegs, quotaHit, LEGS, type LegOutcome, type LogKind } from './smokeChecks.js'
const LAST_LEG = LEGS[LEGS.length - 1]
import { prepareWorkspace, readLog, readWorkspaceFile, removeCodexMcp } from './workspace.js'
import { sessionName, workspaceDirName } from './sessionName.js'
import { TESTCASE } from './matrix.js'
import { outDir, runDir } from './artifacts.js'
import { env } from '../config/env.js'
import { daemonUrl, localHeaders, machineId, waitForPane, gridOverrideFromEnv } from './harnessLocal.js'
import { parseVersionLine } from './currentVersion.js'
import { folderTrust } from '../engines/launchPrep.js'
import { toolResult, useByRef } from './sessionTools.js'
const execFileP = promisify(execFile)
function mkdirAll(dir: string): string {
mkdirSync(dir, { recursive: true })
return dir
}
export interface GridSwitchRunResult {
session: string
reportPath: string
runDir: string
stuck: ReturnType<typeof firstStuck>
error: string | null
}
/**
* The live journey, as a function: `harness e2e grid-switch` and the cron runner call this; the
* env described at the top of this file is the only configuration.
*/
export async function runGridSwitch(): Promise<GridSwitchRunResult> {
const engine = process.env.HARNESS_ENGINE ?? 'codex'
const url = daemonUrl()
const machine = await machineId()
const version = await (async () => {
try {
const { stdout } = await execFileP(engine, ['--version'], { timeout: 10_000 })
return parseVersionLine(stdout) ?? 'UNKNOWN'
} catch {
return 'UNKNOWN'
}
})()
console.error(`[grid-e2e] dialing ${url} · machine ${machine} · ${engine} ${version}`)
const legs: LegOutcome[] = []
const planned = plannedLegs()
let pane: string | null = process.env.HARNESS_PANE_ID ?? null
let workspace: string | null = null
// A subscription model answers in seconds; a local model on a grid takes about 52 s per turn (codex
// 0.166 on Qwen, measured 2026-10-30), and a switched session spends turns on tools the grid model
// does not have (`unsupported call: exec`) before it reaches one that works. The old flat 120 s cut
// the read of that run off 10 s into its third turn and marked a working model stuck. A step passes
// on its result with whichever tool gets there (smokeChecks.ts), so the grid leg gets room for
// eleven such turns. It is still a cap, not "no limit": run.sh has no timeout of its own, and a model
// that has really stopped would hold up every later run.
const CHECK_TIMEOUT_MS: Record<LegOutcome['leg'], number> = { subscription: 120_000, grid: 600_000 }
async function checks(leg: LegOutcome['leg']): Promise<void> {
const readLogs = workspace ? (kind: LogKind) => readLog(workspace!, kind) : undefined
const readFile = workspace ? (rel: string) => readWorkspaceFile(workspace!, rel) : undefined
const usedTool = workspace ? (ref: string, tool: string) => toolResult(engine, workspace!, ref, tool) : undefined
const outcome = pane
? await probeLeg(realTmux, pane, leg, { engine, checkTimeoutMs: CHECK_TIMEOUT_MS[leg], ...(readLogs ? { readLog: readLogs } : {}), ...(readFile ? { readFile } : {}), ...(usedTool ? { toolResult: usedTool } : {}) })
: planned.find((l) => l.leg === leg)!
legs.push(outcome)
console.error(`[grid-e2e] ${leg}: ${outcome.checks.map((c) => `${c.id}=${c.status}`).join(' ')}${outcome.dialogs ? ` (answered: ${outcome.dialogs.join(', ')})` : ''}`)
// The marker shows before the tool's turn is fully over (hooks still reporting); a retarget on a
// turn in flight is refused AGENT_BUSY. Let the pane go quiet first, the way a person would —
// but only when a retarget follows: after the last leg the agent is deleted, not moved.
if (pane && leg !== LAST_LEG) {
await waitForPaneSettle(realTmux, pane, { settleMs: 3_000, settleTimeoutMs: 60_000 })
await new Promise((r) => setTimeout(r, 3_000))
}
}
const driver = new GridSwitchDriver(url, machine, localHeaders())
let trace: GridSwitchTraceStep[] = []
let gridModel: string | null = null
let agentId: string | null = process.env.HARNESS_AGENT_ID ?? null
// codex's MCP server is registered globally (`codex mcp add`, see workspace.ts). Tracked so the run
// removes exactly what it added, and removes it even when the run stops early.
let registeredCodexMcp = false
// How the MCP server got registered for this engine, and in which permission mode the agent runs —
// both in the trace, because an MCP step that never fired is read very differently once you know
// the server was never registered or the engine was launched in a mode that refuses it.
let mcpSetup: { config: string | null; note: string } | null = null
// The app's own default (`kDefaultPermissionMode` in desktop/lib/core/permission_modes.dart), and
// on purpose not overridable: the run creates agents exactly as a person does. It ran in `full`
// until 2026-09-25, and that alone made codex 0.157.0 fail a switch the app does fine — `full`
// launches codex with no `-c`, which starts codex's background daemon, and the daemon kept the
// conversation after the switch ("This conversation is open in another app"). `auto`'s
// `--approve-for-me` is three `-c` overrides to codex, and codex never starts the daemon for those.
const permissionMode = 'auto'
const created = !agentId
let error: string | null = null
// Named before anything runs so the scratch folder and the trace share it; the grid model is
// stamped on once known.
const startedAt = new Date()
try {
await driver.machineSelect()
// Which grid and model — settled BEFORE the agent exists, and asked of the daemon only when the
// run did not pin them. `grid_models_list` starts one Python `grid` per grid the account can see,
// all at once (~125 MB each); on grid-dev that was ten of them, 1.2 GB, landing while claude
// started, and the 1.5 GB container OOM-killed claude. Pinned (HARNESS_GRID_NAME +
// HARNESS_GRID_MODEL, or a relay under test), the list is never needed; unpinned, it is asked
// while nothing else is running.
// A relay under test (E2E_GRID_HOME + E2E_GRID_NAME) replaces the daemon's own grid; the model
// then has to be pinned (HARNESS_GRID_MODEL), since that grid's list is not in `grid_models_list`.
const local = await gridOverrideFromEnv()
const pinned = !!process.env.HARNESS_GRID_MODEL && (!!process.env.HARNESS_GRID_NAME || !!local)
const models = pinned ? null : await driver.gridModels()
const picked = local || !models ? null : pickGridModel(models)
gridModel = process.env.HARNESS_GRID_MODEL ?? picked?.model ?? null
const gridName = local ? local.name : (process.env.HARNESS_GRID_NAME ?? (process.env.HARNESS_GRID_MODEL ? undefined : picked?.gridName))
console.error(`[grid-e2e] grid model: ${gridModel ?? '(none)'}${gridName ? ` on ${gridName}` : ''}${local ? ` (relay under test: ${local.override.baseUrl})` : ''}${pinned ? ' (pinned — no model list asked)' : ''}`)
if (created) {
// realpath: $TMPDIR is a symlink on macOS (`/var/…` → `/private/var/…`) and the engine keys its
// trust on the resolved path — a trust written for the symlink is never matched.
const cwd = realpathSync(mkdirAll(join(outDir(), 'agents', workspaceDirName(sessionName({ engine, version, testcase: TESTCASE, at: startedAt })))))
// The person's project: the notes the file steps work on, an MCP server registered for this engine, and its log.
const laid = prepareWorkspace(cwd, engine)
workspace = cwd
console.error(`[grid-e2e] workspace ${cwd} · mcp ${laid.mcpConfig ?? '(none)'} — ${laid.mcpNote}`)
// codex's registration is global (`codex mcp add`), so it is this run's to take back out.
if (engine === 'codex' && laid.mcpConfig) registeredCodexMcp = true
mcpSetup = { config: laid.mcpConfig, note: laid.mcpNote }
// The scratch folder is this run's own, so answer the engine's "trust this folder?" the way the
// daemon does for a workspace it made — otherwise the first check would be typed into that dialog.
try {
if (engine === 'claude' || engine === 'codex') folderTrust(engine)?.record(cwd)
} catch (err) {
console.error(`[grid-e2e] pre-trust ${cwd}: ${(err as Error).message}`)
}
// REGISTRATION_FAILED right after the daemon starts is not the test's to report: the saved
// registry still holds an agent from before the restart (a container, a reboot), tmux numbers
// the new pane %0 again, and the stale row owns that key until discovery confirms it gone —
// measured on grid-dev: refused at 07:41:11, the stale row forgotten at 07:41:20. Wait it out.
let made = await driver.createAgent(engine, cwd, `e2e ${engine} ${version}`, permissionMode)
for (let retry = 1; !made.ok && made.error === 'REGISTRATION_FAILED' && retry <= 3; retry++) {
console.error(`[grid-e2e] agent_create REGISTRATION_FAILED — a stale agent from before the daemon restart still holds the pane; retry ${retry}/3 in 15s`)
await new Promise((r) => setTimeout(r, 15_000))
made = await driver.createAgent(engine, cwd, `e2e ${engine} ${version}`, permissionMode)
}
if (!made.ok) throw new Error(`${made.error}: ${made.detail ?? ''}`)
agentId = made.agentId
pane = made.pane ?? (await waitForPane(agentId))
console.error(`[grid-e2e] created agent ${agentId}${pane ? ` in pane ${pane}` : ' (pane not registered yet: checks not-run)'}`)
// A fresh codex/claude has a trust / first-run screen before its prompt; let it draw.
if (pane) await waitForPaneSettle(realTmux, pane, { settleMs: 3_000, settleTimeoutMs: 90_000 })
}
// Out of usage ends the journey where it happened: an account that cannot answer on its own login
// tells us nothing about a switch, and switching it anyway only spends the grid's time too.
await checks('subscription')
if (!quotaHit(legs)) {
if (!gridModel) throw new Error('GRID_UNAVAILABLE: grid_models_list returned no model')
const onto = await driver.retargetToGrid(agentId!, gridModel, gridName, local?.override)
if (!onto.ok) throw new Error(`${onto.error}: ${onto.detail ?? ''}`)
await checks('grid')
}
} catch (err) {
error = (err as Error).message
console.error(`[grid-e2e] stopped: ${error}`)
} finally {
trace = [...driver.trace]
}
// How each step got done, and by which model: read from the engine's own session file now that the
// run is over. Reported, never judged — a step's pass/fail is its result on disk (smokeChecks.ts).
if (workspace) {
const refs = legs.flatMap((l) => l.checks.map((c) => c.ref).filter((r): r is string => !!r))
const used = useByRef(engine, workspace, refs)
for (const l of legs) for (const c of l.checks) {
const u = c.ref ? used[c.ref] : undefined
if (u) { c.tools = u.tools; c.models = u.models }
}
}
const session = sessionName({ engine, version, testcase: TESTCASE, gridModel: gridModel ?? undefined, at: startedAt })
const report = {
generatedAt: new Date().toISOString(),
mode: 'live',
engines: [
{
engine,
session,
version,
testcase: TESTCASE,
gridModel,
agentId,
pane,
/** The folder the agent worked in: notes/, out/, the MCP server and config, and .e2e/calc-mcp.log with every MCP call. */
workspace,
/** The mode the engine was launched in — always the app's default, `auto`. */
permissionMode,
/** Where this engine's MCP server was registered, and how — null config means the MCP steps could not pass. */
mcp: mcpSetup,
/** true: this run made the agent. It stays alive through the review (so the reviewer can type into the pane), then is deleted unless E2E_KEEP_AGENT=1. */
createdByRun: created,
keptAlive: !created || process.env.E2E_KEEP_AGENT === '1',
notify: error ? [error] : [],
/** Set when the tool said its account is out of usage: { leg, check, resets }. The pipeline
* reads this before anything else — no review, one plain message, and a retry later. */
quota: quotaHit(legs),
// Every leg, run or not, so a table built from this always has its three rows.
legs: [...legs, ...planned.filter((p) => !legs.some((l) => l.leg === p.leg))],
trace,
},
],
}
// The durable review bundle: everything a person (or the watchdog, later) needs to look back at this run.
const bundle = runDir(session)
const reportPath = join(bundle, 'trace.json')
;(report.engines[0] as Record<string, unknown>).runDir = bundle
writeFileSync(reportPath, JSON.stringify(report, null, 2))
if (workspace) {
for (const rel of ['notes', 'out', 'tools', '.e2e', '.codex', '.mcp.json']) {
const src = join(workspace, rel)
if (existsSync(src)) cpSync(src, join(bundle, 'workspace', rel), { recursive: true })
}
}
for (const step of trace) {
console.log(`\n▶ ${step.step}`)
console.log(` request: ${JSON.stringify(step.request)}`)
console.log(` reply: ${JSON.stringify(step.reply)}`)
}
const stuck = firstStuck(legs)
console.log(`\n=== ${session} ===`)
console.log(stuck ? `STUCK AT ${stuck.leg} / ${stuck.check} (${stuck.status})` : error ? `STOPPED: ${error}` : 'clean')
console.log(`Wrote trace → ${reportPath}`)
// Everything to look back at: the daemon's own lines for this agent, the watchdog's raw events, and
// a manifest that says how to reopen each piece.
try {
const daemonLog = join(env.ADAPTER_DATA_DIR, 'harness.log')
if (existsSync(daemonLog) && agentId) {
const short = agentId.slice(0, 8)
const lines = readFileSync(daemonLog, 'utf8').split('\n').filter((l) => l.includes(agentId!) || l.includes(short))
// The [turn]/[hooks]/[recap] lines are keyed by the ENGINE session id; find it from the attach line.
const sid = /session=([0-9a-f]{8})/.exec(lines.join('\n'))?.[1]
const all = sid ? readFileSync(daemonLog, 'utf8').split('\n').filter((l) => l.includes(agentId!) || l.includes(short) || l.includes(sid)) : lines
writeFileSync(join(bundle, 'daemon.log'), all.join('\n') + '\n')
}
} catch (err) {
console.error(`[grid-e2e] daemon log slice: ${(err as Error).message}`)
}
writeFileSync(join(bundle, 'manifest.json'), JSON.stringify({
session,
engine,
version,
agentId,
pane,
workspace,
gridModel,
reportPath,
// Filled by the reviewer's runner (autonomous-grid-cli e2e/run.sh): { model, sessionID, reopen }.
watchdog: null,
files: { trace: 'trace.json', daemonLog: 'daemon.log', watchdogEvents: 'watchdog.jsonl', watchdogReview: 'watchdog.md', workspace: 'workspace/', evidence: 'evidence/' },
generatedAt: new Date().toISOString(),
}, null, 2))
console.log(`Review bundle → ${bundle}`)
// The run is over; the agent this run made can go. Kept only on request — for a person, or for the
// reviewer to type into the pane (E2E_KEEP_AGENT=1; delete it in the app afterwards).
if (created && agentId && process.env.E2E_KEEP_AGENT !== '1') {
try {
await driver.deleteAgent(agentId)
console.log(`Deleted agent ${agentId}${pane ? ` (pane ${pane})` : ''}`)
} catch (err) {
console.log(`Could not delete agent ${agentId}: ${(err as Error).message} — delete it in the app`)
}
} else if (created && agentId) {
console.log(`Kept agent ${agentId}${pane ? ` in pane ${pane}` : ''} (E2E_KEEP_AGENT=1)`)
}
// Take the global codex entry back out. Not when the agent is being kept: someone who asked to keep
// the pane wants to type into it, and an agent whose MCP server has been unregistered is not the
// agent the run left behind.
if (registeredCodexMcp && process.env.E2E_KEEP_AGENT !== '1') {
console.log(removeCodexMcp() ? 'Unregistered the codex MCP server' : 'Could not unregister the codex MCP server — remove it with `codex mcp remove e2e_calc`')
}
driver.close()
return { session, reportPath, runDir: bundle, stuck, error }
}
// `npx tsx src/e2e/runGridSwitchTrace.ts` still works; `harness e2e grid-switch` is the same call.
if (process.argv[1] && /runGridSwitchTrace\.(ts|js)$/.test(process.argv[1])) {
runGridSwitch().then((r) => process.exit(r.error || r.stuck ? 1 : 0))
}