The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
/**
* Engines updated in place, for Claude Code and Codex. Both update themselves while agents run, Claude
* Code nearly every day: the file on disk is replaced under a running process, a release can drop a
* flag the daemon passes, the engine can be uninstalled and installed again, and its first start after
* an update can be slow. A running agent must go on working through all of it; a restart or a new agent
* must run what is on disk now, or fail quickly with a reason the window gets; and none of it may need
* the daemon itself restarted.
*/
import { mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { pathToFileURL } from 'node:url'
import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
import { LocalClient, type Frame } from './harness/client.js'
import { CLI_ROOT, IsolatedDaemon, until } from './harness/daemon.js'
type Engine = 'claude' | 'codex'
type Row = Record<string, any>
const engines: Engine[] = ['claude', 'codex']
/** A release of the fake engine, as e2e/harness/fakeEngine.mjs reads it from its config. */
interface Release { version?: string; without?: string[]; startDelayMs?: number; updateAvailable?: string }
/** OLD is what the fake engines report unless told otherwise (fakeEngine.mjs); NEW is the next release. */
const OLD: Record<Engine, string> = { claude: '2.1.270', codex: '0.160.0' }
const NEW: Record<Engine, string> = { claude: '2.1.271', codex: '0.161.0' }
const versionLine = (engine: Engine, version: string) => engine === 'claude' ? `${version} (Claude Code)` : `codex-cli ${version}`
const sleep = (ms: number) => new Promise((done) => setTimeout(done, ms))
/**
* Installs a release over the engine the way an updater does: the new file is written beside the old one
* and renamed over it, so a process already running keeps the file it started from and the next launch
* gets the new one. The config is the one IsolatedDaemon gave the engine, plus the release.
*/
function install(d: IsolatedDaemon, engine: Engine, release: Release = {}): void {
const config = { ...d.engineConfig, ...release }
const module = pathToFileURL(join(CLI_ROOT, 'e2e', 'harness', 'fakeEngine.mjs')).href
const wrapper = join(d.root, 'bin', engine)
writeFileSync(`${wrapper}.new`, `#!${process.execPath}\nimport(${JSON.stringify(module)}).then((m) => m.run(${JSON.stringify(engine)}, ${JSON.stringify(config)}))\n`, { mode: 0o755 })
renameSync(`${wrapper}.new`, wrapper)
}
const uninstall = (d: IsolatedDaemon, engine: Engine) => rmSync(join(d.root, 'bin', engine), { force: true })
const rows = async (client: LocalClient): Promise<Row[]> =>
(await client.request<{ agents: Row[] }>('agents_list', { includeStopped: true }, 30_000)).agents
const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)
const bound = (client: LocalClient, agentId: string, sessionId?: string) =>
until(`${agentId.slice(0, 8)} to bind its conversation`, async () => {
const agent = await row(client, agentId)
return agent?.sessionId && agent.status === 'active' && (!sessionId || agent.sessionId === sessionId) ? agent : null
}, 60_000, 500)
async function create(d: IsolatedDaemon, client: LocalClient, engine: Engine, folder: string): Promise<Row> {
const cwd = join(d.projectsDir, folder)
mkdirSync(cwd, { recursive: true })
const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
expect(created.error, `${folder}: ${JSON.stringify(created)}`).toBeUndefined()
return bound(client, created.agent.id)
}
const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId
async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {
const started = client.next(isTurn('turn_started', agentId), 45_000, `turn_started (${content})`)
const ended = client.next(isTurn('turn_ended', agentId), 45_000, `turn_ended (${content})`)
client.send('message', { agentId, content })
expect((await started).payload?.userMessage).toBe(content)
await ended
}
const thread = async (client: LocalClient, sessionId: string) => {
const page = await client.request<{ events?: Array<{ type: string; payload: Record<string, any> }>; error?: string }>('session_get', { sessionId, limit: 200 }, 30_000)
expect(page.error, JSON.stringify(page)).toBeUndefined()
const events = page.events ?? []
return {
users: events.filter((event) => event.type === 'user_message').map((event) => String(event.payload.content)),
answers: events.filter((event) => event.type === 'text_delta').map((event) => String(event.payload.content)),
}
}
/** The version of the engine the agent runs now, asked through the daemon and answered in its transcript. */
async function runningVersion(client: LocalClient, agentId: string): Promise<string> {
await turn(client, agentId, '!version')
const { sessionId } = (await row(client, agentId))!
return until('the version answer in the thread', async () => {
const last = (await thread(client, sessionId)).answers.at(-1)
return last && /Claude Code|codex-cli/.test(last) ? last : null
}, 15_000, 250)
}
/** What tmux shows in a pane now, or '' once the pane is gone. */
const screen = (d: IsolatedDaemon, pane: unknown): Promise<string> =>
typeof pane === 'string' && pane ? d.capture(pane).catch(() => '') : Promise.resolve('')
const errorsFor = (client: LocalClient, agentId: string, from: number) =>
client.frames.slice(from).filter((frame) => frame.type === 'error' && frame.agentId === agentId).map((frame) => String(frame.payload?.message))
describe('engines updated in place', () => {
let daemon: IsolatedDaemon | undefined
afterEach(async () => { await daemon?.close(); daemon = undefined })
const fresh = async () => {
const d = await IsolatedDaemon.create()
daemon = d
onTestFailed(() => {
console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`)
// The runner swallows console output: UPDATES_LOG=<file> keeps the daemon's whole log for reading.
if (process.env.UPDATES_LOG) writeFileSync(process.env.UPDATES_LOG, d.log())
})
await d.start()
return d
}
it.each(engines)('%s: replaced by a newer version under a running agent: the agent goes on, and a restart and a new agent run the new one', async (engine) => {
const d = await fresh()
const client = await LocalClient.connect(d)
const agent = await create(d, client, engine, `updated-${engine}`)
expect(await runningVersion(client, agent.id)).toBe(versionLine(engine, OLD[engine]))
install(d, engine, { version: NEW[engine] })
// Through a reap and a reconcile pass (every 5s here) with the file the process started from
// replaced on disk: the daemon must go on seeing the running engine as the agent's.
await sleep(7_000)
expect((await row(client, agent.id))?.status).toBe('active')
expect(await runningVersion(client, agent.id)).toBe(versionLine(engine, OLD[engine]))
await turn(client, agent.id, 'still working after the update')
const restarted = await client.request('agent_restart', { agentId: agent.id }, 90_000)
expect(restarted.error, JSON.stringify(restarted)).toBeUndefined()
expect(restarted.resumed).toBe(true)
await bound(client, agent.id, agent.sessionId)
expect(await runningVersion(client, agent.id)).toBe(versionLine(engine, NEW[engine]))
const next = await create(d, client, engine, `after-the-update-${engine}`)
expect(await runningVersion(client, next.id)).toBe(versionLine(engine, NEW[engine]))
expect(d.coresStarted()).toBe(1)
client.close()
})
it('claude: an update drops --fork-session after forks worked: the fork fails at once, its refusal reaches the window, and the source goes on', async () => {
const d = await fresh()
const client = await LocalClient.connect(d)
const source = await create(d, client, 'claude', 'fork-after-update')
await turn(client, source.id, 'before the update')
const first = await client.request('agent_fork', { agentId: source.id, name: 'before the update' }, 90_000)
expect(first.error, JSON.stringify(first)).toBeUndefined()
await bound(client, first.agent.id)
install(d, 'claude', { version: NEW.claude, without: ['--fork-session'] })
// The daemon does not read `--fork-session` from the help before it forks (only the permission flag
// is checked), so it launches the fork with it and the new release refuses it.
const startedAt = Date.now()
const forked = await client.request('agent_fork', { agentId: source.id, name: 'after the update' }, 90_000)
expect(Date.now() - startedAt).toBeLessThan(15_000)
if (!forked.error) {
const pane = forked.agent.tmuxPane
// What the daemon does today: the engine's process is there for an instant, the launch reads as
// ready, then the reconciler finds no engine and keeps the conversation-less row as stopped,
// giving the shell left in the pane a row of its own. Or, when the watcher misses that instant,
// the row itself becomes that terminal. Either way the refusal is on a terminal the window has.
const settled = await until('the fork to settle', async () => {
const now = await row(client, forked.agent.id)
return now && now.launch?.state !== 'starting' && !(now.engine === 'claude' && now.status === 'active') ? now : null
}, 30_000, 250)
expect(settled.sessionId || null).toBeNull()
const shell = (await rows(client)).find((agent) => agent.engine === 'terminal' && agent.status === 'active' && agent.tmuxPane === pane)
expect(shell, `a terminal row for the fork's pane ${pane}: ${JSON.stringify(await rows(client))}`).toBeTruthy()
expect(await screen(d, pane)).toContain("error: unknown option '--fork-session'")
} else {
expect(typeof forked.detail).toBe('string')
}
// The source is untouched: it goes on in its own conversation.
await turn(client, source.id, 'the source after the failed fork')
expect((await row(client, source.id))?.sessionId).toBe(source.sessionId)
expect(d.coresStarted()).toBe(1)
client.close()
})
it('codex: an update drops resume: a restart comes back on a new conversation and says so, the one it left is kept, and a resume says why it cannot', async () => {
const d = await fresh()
const client = await LocalClient.connect(d)
const agent = await create(d, client, 'codex', 'resume-after-update')
await turn(client, agent.id, 'before the update')
install(d, 'codex', { version: NEW.codex, without: ['resume'] })
// Restart tries `codex resume <id>`, sees no engine come up, and falls back to a fresh launch in the
// same pane: a working agent on a new conversation, with `resumed: false` to say so.
let startedAt = Date.now()
const restarted = await client.request('agent_restart', { agentId: agent.id }, 90_000)
expect(Date.now() - startedAt).toBeLessThan(45_000)
expect(restarted.error, JSON.stringify(restarted)).toBeUndefined()
expect(restarted.resumed).toBe(false)
const renewed = await until('the agent on its new conversation', async () => {
const now = await row(client, agent.id)
return now?.status === 'active' && now.sessionId && now.sessionId !== agent.sessionId ? now : null
}, 30_000, 250)
expect(await runningVersion(client, agent.id)).toBe(versionLine('codex', NEW.codex))
// The conversation the restart had to leave is kept as a stopped one of its own: listed, and its
// history readable.
const kept = await until('the conversation it left, kept as stopped', async () =>
(await rows(client)).find((one) => one.status === 'stopped' && one.sessionId === agent.sessionId), 15_000, 250)
expect(kept.id).not.toBe(agent.id)
expect((await thread(client, agent.sessionId)).users).toEqual(['before the update'])
expect((await client.request('agent_delete', { agentId: agent.id }, 60_000)).error).toBeUndefined()
await until('the agent to stop', async () => (await row(client, agent.id))?.status === 'stopped' || null, 45_000, 500)
startedAt = Date.now()
const resumed = await client.request('agent_resume', { agentId: agent.id }, 90_000)
expect(Date.now() - startedAt).toBeLessThan(30_000)
expect(resumed.error, JSON.stringify(resumed)).toBe('RESUME_FAILED')
expect(String(resumed.detail)).toContain('exited before confirming the saved conversation')
expect((await row(client, agent.id))?.sessionId).toBe(renewed.sessionId)
// A release that has resume again opens the kept conversation where it was.
install(d, 'codex', { version: '0.162.0' })
const reopened = await client.request('agent_resume', { agentId: kept.id }, 90_000)
expect(reopened.error, JSON.stringify(reopened)).toBeUndefined()
await bound(client, kept.id, agent.sessionId)
await turn(client, kept.id, 'back in the kept conversation')
expect((await thread(client, agent.sessionId)).users).toEqual(['before the update', 'back in the kept conversation'])
expect(d.coresStarted()).toBe(1)
client.close()
})
it.each([['claude', '--permission-mode'], ['codex', '--approve-for-me']] as const)('%s: an update drops %s after the daemon saw it supported: a new agent in Auto is refused with the reason, with no daemon restart', async (engine, flag) => {
const d = await fresh()
const client = await LocalClient.connect(d)
// Creating an agent in Auto reads the flag from the engine's help, and the daemon keeps the answer.
const first = await create(d, client, engine, `auto-before-${engine}`)
await turn(client, first.id, 'before the update')
install(d, engine, { version: NEW[engine], without: [flag] })
const cwd = join(d.projectsDir, `auto-after-${engine}`)
mkdirSync(cwd, { recursive: true })
const startedAt = Date.now()
const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
expect(Date.now() - startedAt).toBeLessThan(15_000)
// The answer it kept was about the file the update replaced, so the engine is asked again, and the
// launch is refused with the reason before any pane opens, as a daemon that never saw the flag
// refuses it. Before #773 it went on trusting the old answer until it was restarted: the engine
// refused the flag in a pane, and the window was shown a harness that started and stopped.
expect(created.error, JSON.stringify(created)).toBe('CODEX_CLI_TOO_OLD')
expect(String(created.detail)).toContain(flag)
expect((await rows(client)).filter((agent) => agent.status === 'active').map((agent) => agent.id)).toEqual([first.id])
// The running agent is untouched.
await turn(client, first.id, 'the first agent after the update')
expect(d.coresStarted()).toBe(1)
client.close()
})
it.each(engines)('%s: uninstalled while an agent runs: it goes on; a new request waits while restart preserves the running agent; reinstalled, both work with no daemon restart', async (engine) => {
const d = await fresh()
const client = await LocalClient.connect(d)
const agent = await create(d, client, engine, `uninstalled-${engine}`)
await turn(client, agent.id, 'before the uninstall')
uninstall(d, engine)
// Through a reap and a reconcile pass with the engine's file gone from disk.
await sleep(7_000)
expect((await row(client, agent.id))?.status).toBe('active')
await turn(client, agent.id, 'after the uninstall')
const cwd = join(d.projectsDir, `missing-${engine}`)
mkdirSync(cwd, { recursive: true })
let startedAt = Date.now()
const payload = { creationId: `uninstalled-${engine}-request`, engine, cwd, bypassPermission: true }
const created = await client.request('agent_create', payload, 90_000)
expect(Date.now() - startedAt).toBeLessThan(10_000)
expect(created).toMatchObject({ state: 'pending', held: { service: 'native' } })
expect(created.held.detail).toContain('executable')
expect((await rows(client)).map(row => row.id)).toEqual([agent.id])
// Restart holds before touching the live process. Repairing the executable
// then retrying must keep this agent's original conversation.
startedAt = Date.now()
const restarted = await client.request('agent_restart', { agentId: agent.id }, 90_000)
expect(Date.now() - startedAt).toBeLessThan(10_000)
expect(restarted.error, JSON.stringify(restarted)).toBe('NATIVE_LAUNCH_HELD')
expect(String(restarted.detail)).toContain('executable')
expect(await row(client, agent.id)).toMatchObject({ status: 'active', sessionId: agent.sessionId })
await turn(client, agent.id, 'after the refused restart')
install(d, engine, { version: NEW[engine] })
const recovered = await until('the original request after reinstall', async () => {
const reply = await client.request('agent_create_status', { creationId: payload.creationId })
return reply.state === 'created' ? reply : null
}, 60_000, 250)
const next = await bound(client, recovered.agent.id)
expect(await client.request('agent_create', payload)).toMatchObject({ state: 'created', agent: { id: next.id } })
expect(await runningVersion(client, next.id)).toBe(versionLine(engine, NEW[engine]))
// The first agent restarts onto the engine installed again, in its own conversation.
const back = await client.request('agent_restart', { agentId: agent.id }, 90_000)
expect(back.error, JSON.stringify(back)).toBeUndefined()
expect(back.resumed).toBe(true)
await bound(client, agent.id, agent.sessionId)
expect(await runningVersion(client, agent.id)).toBe(versionLine(engine, NEW[engine]))
expect((await thread(client, agent.sessionId)).users).toContain('after the refused restart')
expect(d.coresStarted()).toBe(1)
client.close()
})
it.each(engines)('%s: slow to start after an update: a new agent binds, and a message sent while it starts is taken once', async (engine) => {
const d = await fresh()
const client = await LocalClient.connect(d)
install(d, engine, { version: NEW[engine], startDelayMs: 6_000 })
const cwd = join(d.projectsDir, `slow-${engine}`)
mkdirSync(cwd, { recursive: true })
const from = client.frames.length
const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
expect(created.error, JSON.stringify(created)).toBeUndefined()
const id: string = created.agent.id
// Sent once the engine's process is there and before it has drawn or announced anything.
const starting = await until('the engine process, not yet its conversation', async () => {
const now = await row(client, id)
return now?.launch?.state === 'ready' ? now : null
}, 30_000, 100)
expect(starting.sessionId || null).toBeNull()
const content = 'sent while it was starting'
client.send('message', { agentId: id, content })
const agent = await bound(client, id)
await client.waitFor((frame) => isTurn('turn_ended', id)(frame) && client.frames.indexOf(frame) >= from, 45_000, 'the message to be answered')
// Long enough for every check and retry the daemon makes on a message it could not see taken.
await sleep(12_000)
const starts = client.frames.slice(from).filter(isTurn('turn_started', id)).map((frame) => frame.payload?.userMessage)
expect(starts).toEqual([content])
expect((await thread(client, agent.sessionId)).users).toEqual([content])
expect(errorsFor(client, id, from)).toEqual([])
expect(await runningVersion(client, id)).toBe(versionLine(engine, NEW[engine]))
client.close()
})
it('codex: a message sent while Codex asks whether to update does not update it: refused with the reason, it goes through once the question is answered', async () => {
// Codex asks at startup when a newer release is out, before its conversation starts; the prompt drops
// a paste and takes the Enter behind it as `Update now`, the highlighted row.
const d = await fresh()
const client = await LocalClient.connect(d)
install(d, 'codex', { version: OLD.codex, updateAvailable: NEW.codex })
const cwd = join(d.projectsDir, 'update-prompt')
mkdirSync(cwd, { recursive: true })
const created = await client.request('agent_create', { engine: 'codex', cwd, bypassPermission: true }, 90_000)
expect(created.error, JSON.stringify(created)).toBeUndefined()
const id: string = created.agent.id
const pane = String((await until('the engine process', async () => {
const now = await row(client, id)
return now?.tmuxPane && now.launch?.state === 'ready' ? now : null
}, 30_000, 100)).tmuxPane)
await until('the update prompt to be drawn', async () => (await d.capture(pane)).includes('Update available') || null, 30_000, 250)
const from = client.frames.length
const refused = client.next((frame) => frame.type === 'error' && frame.agentId === id, 15_000, 'the refusal')
client.send('message', { agentId: id, content: 'what changed?' })
expect(String((await refused).payload?.message)).toBe('Codex is asking whether to update. Answer it in its terminal, then send the message again.')
expect(await d.capture(pane)).not.toContain('Update ran successfully')
// Skipped in its terminal, Codex starts, and the same message goes through.
await d.tmux.run('send-keys', '-t', pane, 'Escape')
await bound(client, id)
await turn(client, id, 'what changed?')
expect(errorsFor(client, id, from)).toHaveLength(1)
client.close()
})
})