The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
End to end, round 15: chaos, upgrading from v0.3.57, and a Codex fork that never bound (#754)
Deeb72ebba0b5d2
diff --git a/cli/e2e/chaos.e2e.ts b/cli/e2e/chaos.e2e.tsnew file mode 100644index 000000000..c85e989cd--- /dev/null+++ b/cli/e2e/chaos.e2e.ts@@ -0,0 +1,198 @@+/**+ * Chaos, for Claude Code and Codex: a seeded random run of everything a desk does at once — messages,+ * slow turns with messages behind them, agent restarts, stops and resumes, new agents, forks,+ * compactions, windows coming and going, and the daemon itself restarting — and then the daemon's+ * picture of the desk is checked against tmux and the process table. Whatever the order, at the end no+ * two agents share a pane, every active agent has its pane, one engine in it and a conversation, no+ * engine runs in a pane nobody owns, every active agent answers, and the core never crashed.+ *+ * CHAOS_SEED picks the run (a failure prints its seed), CHAOS_OPS its length.+ */+import { execFileSync } from 'node:child_process'+import { mkdirSync, writeFileSync } from 'node:fs'+import { join } from 'node:path'+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'+import { LocalClient, type Frame } from './harness/client.js'+import { IsolatedDaemon, until } from './harness/daemon.js'++type Engine = 'claude' | 'codex'+const SEED = Number(process.env.CHAOS_SEED ?? 20261004)+const OPS = Number(process.env.CHAOS_OPS ?? 70)++/** mulberry32: small, seeded, and the same sequence on every machine. */+function random(seed: number): () => number {+ let a = seed >>> 0+ return () => {+ a = (a + 0x6d2b79f5) >>> 0+ let t = a+ t = Math.imul(t ^ (t >>> 15), t | 1)+ t ^= t + Math.imul(t ^ (t >>> 7), t | 61)+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296+ }+}++const rows = async (client: LocalClient) =>+ (await client.request<{ agents: Array<Record<string, any>> }>('agents_list', { includeStopped: true }, 30_000)).agents+const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)+const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId++/** Engine processes per tmux pane, from tmux's own pane list and the process table. */+function enginesByPane(daemon: IsolatedDaemon, panes: Array<{ id: string; pid: number }>): Map<string, number> {+ const table = execFileSync('ps', ['-A', '-o', 'pid=,ppid=,command=']).toString().trim().split('\n')+ .map((line) => line.trim().match(/^(\d+)\s+(\d+)\s+(.*)$/)).filter(Boolean) as RegExpMatchArray[]+ const children = new Map<number, number[]>()+ for (const [, pid, ppid] of table) children.set(Number(ppid), [...(children.get(Number(ppid)) ?? []), Number(pid)])+ const result = new Map<string, number>()+ for (const pane of panes) {+ const tree = new Set<number>([pane.pid])+ for (const pid of tree) for (const child of children.get(pid) ?? []) tree.add(child)+ // The fake engines take their engine's name as their process title.+ result.set(pane.id, table.filter(([, pid, , command]) => tree.has(Number(pid)) && /^(claude|codex)$/.test(command.trim())).length)+ }+ void daemon+ return result+}++describe('chaos', () => {+ let daemon: IsolatedDaemon | undefined+ afterEach(async () => { await daemon?.close(); daemon = undefined })++ it(`seed ${SEED}, ${OPS} operations: the desk ends coherent`, async () => {+ const d = await IsolatedDaemon.create()+ daemon = d+ const rand = random(SEED)+ const pick = <T>(items: readonly T[]): T => items[Math.floor(rand() * items.length)]+ const log: string[] = []+ onTestFailed(() => {+ console.log(`---- chaos seed ${SEED}\n${log.join('\n')}`)+ console.log(`---- daemon log\n${d.log().split('\n').slice(-200).join('\n')}`)+ // The whole of it, for a failure that began long before the end.+ if (process.env.CHAOS_LOG) writeFileSync(process.env.CHAOS_LOG, `${log.join('\n')}\n---- daemon log\n${d.log()}`)+ })+ await d.start()+ let client = await LocalClient.connect(d)+ let daemonRestarts = 0+ const unexpected: string[] = []+ // Errors a racing request may honestly get: the agent was busy, gone, or changed under it.+ const EXPECTED = new Set(['AGENT_BUSY', 'STOP_UNCONFIRMED', 'AGENT_NOT_FOUND', 'NOT_FOUND', 'AGENT_CHANGED',+ 'RESUME_UNCONFIRMED', 'RESTART_FAILED', 'UNSUPPORTED', 'NOT_STOPPED', 'ALREADY_RUNNING', 'NOT_RUNNING'])+ const expectAnswer = (op: string, answer: Record<string, any>) => {+ if (answer.error && !EXPECTED.has(String(answer.error))) unexpected.push(`${op}: ${JSON.stringify(answer).slice(0, 300)}`)+ }+ let folders = 0+ const create = async (engine: Engine) => {+ const cwd = join(d.projectsDir, `chaos-${folders++}`)+ mkdirSync(cwd, { recursive: true })+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)+ expectAnswer(`create ${engine}`, created)+ return created.agent?.id as string | undefined+ }+ const known = new Set<string>()+ for (const engine of ['claude', 'codex', 'claude', 'codex'] as const) {+ const id = await create(engine)+ if (id) known.add(id)+ }+ for (const id of known) await until(`${id.slice(0, 8)} to bind`, async () => (await row(client, id))?.sessionId || null, 60_000, 500)++ const live = async () => (await rows(client)).filter((agent) => known.has(agent.id) && agent.status === 'active' && agent.sessionId)+ const stopped = async () => (await rows(client)).filter((agent) => known.has(agent.id) && agent.status === 'stopped')+ const message = async (agentId: string, content: string, ms = 45_000) => {+ const ended = client.next(isTurn('turn_ended', agentId), ms, `turn_ended (${content})`).catch(() => null)+ client.send('message', { agentId, content })+ return ended+ }++ for (let op = 0; op < OPS; op++) {+ const roll = rand()+ const agents = await live()+ const target = agents.length ? pick(agents) : undefined+ try {+ if (roll < 0.30 && target) {+ log.push(`${op} message ${target.id.slice(0, 8)}`)+ await message(target.id, `chaos message ${op}`)+ } else if (roll < 0.40 && target) {+ log.push(`${op} slow turn with a message behind it ${target.id.slice(0, 8)}`)+ client.send('message', { agentId: target.id, content: '!slow 1500' })+ await message(target.id, `behind the slow turn ${op}`, 60_000)+ } else if (roll < 0.48 && target) {+ log.push(`${op} restart ${target.id.slice(0, 8)}`)+ expectAnswer('restart', await client.request('agent_restart', { agentId: target.id }, 90_000))+ } else if (roll < 0.55 && target) {+ log.push(`${op} stop ${target.id.slice(0, 8)}`)+ expectAnswer('stop', await client.request('agent_delete', { agentId: target.id }, 90_000))+ } else if (roll < 0.63) {+ const paused = await stopped()+ if (paused.length) {+ const agent = pick(paused)+ log.push(`${op} resume ${agent.id.slice(0, 8)}`)+ expectAnswer('resume', await client.request('agent_resume', { agentId: agent.id }, 90_000))+ }+ } else if (roll < 0.69 && known.size < 9) {+ const engine = pick(['claude', 'codex'] as const)+ log.push(`${op} create ${engine}`)+ const id = await create(engine)+ if (id) known.add(id)+ } else if (roll < 0.74 && target) {+ log.push(`${op} fork ${target.id.slice(0, 8)}`)+ const forked = await client.request('agent_fork', { agentId: target.id, name: `fork ${op}` }, 90_000)+ expectAnswer('fork', forked)+ const forkId = forked.agent?.id ?? forked.agentId+ if (forkId) known.add(forkId)+ } else if (roll < 0.80 && target) {+ log.push(`${op} compact ${target.id.slice(0, 8)}`)+ await d.tmux.run('send-keys', '-t', target.tmuxPane, '!compact', 'Enter').catch(() => {})+ } else if (roll < 0.88) {+ log.push(`${op} reconnect the window`)+ client.close()+ client = await LocalClient.connect(d)+ } else if (roll < 0.92) {+ log.push(`${op} restart the daemon`)+ client.close()+ await d.restart()+ daemonRestarts+++ client = await LocalClient.connect(d)+ } else if (target) {+ log.push(`${op} two messages at once ${target.id.slice(0, 8)}`)+ client.send('message', { agentId: target.id, content: `first of two ${op}` })+ await message(target.id, `second of two ${op}`, 60_000)+ }+ } catch (error) {+ unexpected.push(`${op}: threw ${error instanceof Error ? error.message : String(error)}`)+ }+ }++ // Settle: restarts, resumes and binds finish; every scan lands.+ log.push('settle')+ await new Promise((resolve) => setTimeout(resolve, 8_000))+ const desk = (await rows(client)).filter((agent) => known.has(agent.id))+ const active = desk.filter((agent) => agent.status === 'active')+ const paneList = (await d.tmux.run('list-panes', '-a', '-F', '#{pane_id} #{pane_pid}').catch(() => ''))+ .split('\n').filter(Boolean).map((line) => { const [id, pid] = line.split(' '); return { id, pid: Number(pid) } })+ const engines = enginesByPane(d, paneList)+ const report = `rows: ${desk.map((agent) => `${agent.id.slice(0, 8)}=${agent.status}@${agent.tmuxPane ?? '-'}${agent.sessionId ? '' : '(unbound)'}`).join(' ')}\npanes: ${[...engines].map(([pane, n]) => `${pane}:${n}`).join(' ')}`+ log.push(report)++ expect(unexpected, `unexpected answers\n${report}`).toEqual([])+ // No two active agents share a pane, and every active agent has its pane, one engine and a conversation.+ const panesOfActive = active.map((agent) => agent.tmuxPane)+ expect(new Set(panesOfActive).size, `shared panes\n${report}`).toBe(panesOfActive.length)+ for (const agent of active) {+ expect(engines.has(agent.tmuxPane), `${agent.id.slice(0, 8)} active without its pane\n${report}`).toBe(true)+ expect(engines.get(agent.tmuxPane), `${agent.id.slice(0, 8)} should have one engine\n${report}`).toBe(1)+ }+ // No engine runs in a pane no active agent owns: that would be a ghost, or a leak.+ for (const [pane, count] of engines) {+ if (count > 0) expect(panesOfActive, `an engine in ${pane}, owned by no active agent\n${report}`).toContain(pane)+ }+ // Every active agent answers, and has its conversation by then. One can be without it until now: a+ // fork whose start-up announcement fell into a daemon restart has a conversation nobody named yet+ // (only its hook knows the id a fork chose), and its first prompt's hook names it.+ for (const agent of active) {+ const ended = await message(agent.id, 'the last word', 60_000)+ expect(ended, `${agent.id.slice(0, 8)} did not answer at the end\n${report}`).toBeTruthy()+ expect((await row(client, agent.id))?.sessionId, `${agent.id.slice(0, 8)} has no conversation after a message\n${report}`).toBeTruthy()+ }+ expect(d.coresStarted(), `the core restarted on its own\n${report}`).toBe(daemonRestarts + 1)+ client.close()+ }, 900_000)+})diff --git a/cli/e2e/harness/fakeEngine.mjs b/cli/e2e/harness/fakeEngine.mjsindex 9c1cf8e93..a24d63371 100644--- a/cli/e2e/harness/fakeEngine.mjs+++ b/cli/e2e/harness/fakeEngine.mjs@@ -19,8 +19,9 @@// `/compact` does (and Claude Code then announces the same session again), `!compactmid` compacts in the// middle of a turn as an automatic compaction does, `!exit` ends the process. Everything else is// echoed as the answer.+import { execFileSync } from 'node:child_process'import { randomUUID } from 'node:crypto'-import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'+import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, writeFileSync } from 'node:fs'import { dirname, join } from 'node:path'export async function run(engine, config) {@@ -64,6 +65,21 @@ export async function run(engine, config) {: join(config.codexHome, 'sessions', '2026', '10', '03', `rollout-2026-10-03T00-00-00-${sessionId}.jsonl`)mkdirSync(dirname(transcript), { recursive: true })if (!existsSync(transcript)) writeFileSync(transcript, '')+ // What each CLI leaves for whoever needs to know which conversation this process is writing, as the+ // daemon does when the conversation's start-up hook was lost. Codex holds its rollout open for the+ // session; Claude Code keeps a record per process, `~/.claude/sessions/<pid>.json`.+ let held = null+ const procStart = (() => { try { return execFileSync('ps', ['-o', 'lstart=', '-p', String(process.pid)]).toString().trim() } catch { return '' } })()+ const announceProcess = () => {+ if (engine === 'codex') {+ if (held !== null) closeSync(held)+ held = openSync(transcript, 'r')+ } else if (procStart) {+ const sessions = join(dirname(config.claudeProjectsDir), 'sessions')+ mkdirSync(sessions, { recursive: true })+ writeFileSync(join(sessions, `${process.pid}.json`), JSON.stringify({ pid: process.pid, sessionId, cwd: process.cwd(), procStart }))+ }+ }const now = () => new Date().toISOString()const write = (record) => appendFileSync(transcript, JSON.stringify(record) + '\n')@@ -93,6 +109,7 @@ export async function run(engine, config) {if (!response.ok) process.stdout.write(`\r\n[fake ${engine}] hook ${path} failed: ${response.status}\r\n`)}+ announceProcess()const draw = (line = '') => process.stdout.write(`\r\x1b[2K› ${line}`)process.stdout.write(`\x1b[?2004h${engine === 'claude' ? '✻ Welcome to Claude Code (fake)' : '>_ OpenAI Codex (fake)'}\r\n`)process.stdout.write(` session ${sessionId}${resumed ? ' (resumed)' : ''}\r\n\r\n`)@@ -208,12 +225,16 @@ export async function run(engine, config) {parent = nullturn = 0if (engine === 'codex') codex('session_meta', { id: sessionId, cli_version: '0.159.0', cwd, source: 'cli' })+ announceProcess()process.stdout.write('\r\n(new conversation)\r\n')draw()await hook('session-start', { hookEvent: 'SessionStart', source: 'clear' })return}if (open) finish('(interrupted by a new prompt)')+ // Claude Code runs its UserPromptSubmit hook on every prompt, through the same door as SessionStart:+ // the catch hook, which re-registers a session whose start-up announcement the daemon missed.+ if (engine === 'claude') await hook('session-start', { hookEvent: 'UserPromptSubmit', prompt })turn++open = `turn-${turn}`process.stdout.write(`\r\n`)diff --git a/cli/e2e/migration.e2e.ts b/cli/e2e/migration.e2e.tsnew file mode 100644index 000000000..563de4193--- /dev/null+++ b/cli/e2e/migration.e2e.ts@@ -0,0 +1,167 @@+/**+ * Upgrading a machine from a released build, for Claude Code and Codex — what every installed machine+ * does when a release ships. A daemon of the released build (`MIGRATION_FROM`, its bundled `cli.js`)+ * runs with agents at work, finds this checkout's build in its update manifest, and hands over to it+ * the way that release hands over: it spawns the new `cli.js __run` and exits. The new core comes up+ * on its own, with no master, keeps every agent and its conversation, and the next start brings up+ * harnessd's master over the same agents.+ *+ * Skipped unless MIGRATION_FROM names a bundle, so CI does not build old releases. Before a release:+ * build the last released tag's bundle (`node build-bundle.mjs` in a checkout of it) and run+ * `MIGRATION_FROM=<that>/dist/cli.js npm run test:e2e -- migration`.+ */+import { execFileSync } from 'node:child_process'+import { createHash } from 'node:crypto'+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'+import { createServer, type Server } from 'node:http'+import { tmpdir } from 'node:os'+import { dirname, join } from 'node:path'+import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'+import { LocalClient, type Frame } from './harness/client.js'+import { CLI_ROOT, IsolatedDaemon, until } from './harness/daemon.js'++const FROM = process.env.MIGRATION_FROM+const NEXT = '0.99.0'+const sha = (bytes: Buffer): string => createHash('sha256').update(bytes).digest('hex')++describe.skipIf(!FROM)('upgrading a machine from a released build', () => {+ let scratch = ''+ let server: Server | undefined+ let daemon: IsolatedDaemon | undefined+ let offered: 'from' | 'next' = 'from'+ let fromVersion = ''+ const bundles = new Map<'from' | 'next', { cli: Buffer; notify: Buffer; version: string }>()+ const cliDir = () => join(scratch, 'cli')+ const health = async (): Promise<Record<string, any> | null> =>+ fetch(`http://127.0.0.1:${daemon!.port}/api/health`).then((response) => response.json()).catch(() => null)+ const rows = async (client: LocalClient) =>+ (await client.request<{ agents: Array<Record<string, any>> }>('agents_list', { includeStopped: true }, 30_000)).agents+ const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId+ const turn = async (client: LocalClient, agentId: string, content: string) => {+ const ended = client.next(isTurn('turn_ended', agentId), 60_000, `turn_ended (${content})`)+ client.send('message', { agentId, content })+ await ended+ }+ /** The pids answering on the daemon's port, from the process table: whatever is running it now. */+ const listeners = (): number[] => {+ try {+ return execFileSync('lsof', ['-n', '-P', '-t', `-iTCP:${daemon!.port}`, '-sTCP:LISTEN']).toString().trim().split('\n').filter(Boolean).map(Number)+ } catch { return [] }+ }++ beforeAll(async () => {+ scratch = mkdtempSync(join(tmpdir(), 'harnessd-migration-'))+ const out = join(scratch, 'build')+ execFileSync(process.execPath, ['build-bundle.mjs'], {+ cwd: CLI_ROOT, env: { ...process.env, ADAPTER_VERSION: NEXT, BUNDLE_OUT_DIR: out }, stdio: 'pipe',+ })+ bundles.set('next', { cli: readFileSync(join(out, 'cli.js')), notify: readFileSync(join(out, 'notify.mjs')), version: NEXT })+ fromVersion = execFileSync(process.execPath, [FROM!, 'version'], { encoding: 'utf8' }).trim()+ bundles.set('from', { cli: readFileSync(FROM!), notify: readFileSync(join(dirname(FROM!), 'notify.mjs')), version: fromVersion })++ server = createServer((request, response) => {+ const url = request.url ?? ''+ const origin = `http://127.0.0.1:${(server!.address() as { port: number }).port}`+ const bundle = bundles.get(offered)!+ if (url === '/metadata.json') {+ response.end(JSON.stringify({ cli: {+ version: bundle.version,+ cli: { url: `${origin}/cli-${offered}.js`, sha256: sha(bundle.cli), size: bundle.cli.length },+ notify: { url: `${origin}/notify-${offered}.mjs`, sha256: sha(bundle.notify), size: bundle.notify.length },+ } }))+ return+ }+ const asked = /^\/(cli|notify)-(from|next)\.(js|mjs)$/.exec(url)+ if (asked) { response.end(bundles.get(asked[2] as 'from' | 'next')![asked[1] as 'cli' | 'notify']); return }+ response.statusCode = 404+ response.end()+ })+ await new Promise<void>((done) => server!.listen(0, '127.0.0.1', done))+ const local = `http://127.0.0.1:${(server.address() as { port: number }).port}`++ mkdirSync(cliDir())+ copyFileSync(FROM!, join(cliDir(), 'cli.js'))+ copyFileSync(join(dirname(FROM!), 'notify.mjs'), join(cliDir(), 'notify.mjs'))+ writeFileSync(join(cliDir(), 'package.json'), '{"type":"module"}\n')+ daemon = await IsolatedDaemon.create({+ scriptPath: join(cliDir(), 'cli.js'),+ // As the released build runs: `harness start` spawned `__run` itself; it had no master.+ noMaster: true,+ env: {+ ADAPTER_CLI_DIR: cliDir(),+ ADAPTER_UPDATE_DISABLE: 'false',+ ADAPTER_UPDATE_URL: `${local}/metadata.json`,+ ADAPTER_UPDATE_CHECK_MS: '1000',+ ADAPTER_UPDATE_SLOT_SEC: '-1',+ HARNESS_TUI_MANIFEST_URL: `${local}/tui/metadata.json`,+ ADAPTER_RUNTIME_METADATA_URL: `${local}/runtime/metadata.json`,+ ADAPTER_GRID_RUNTIME_METADATA_URL: `${local}/grid/metadata.json`,+ },+ })+ // A released build may say nothing when it is ready, and has no request gate: its port is the sign.+ await daemon.start({ ready: 'port' })+ expect((await health())?.version).toBe(fromVersion)+ }, 300_000)++ afterAll(async () => {+ // Whatever runs the port now — the detached successor included — goes with the test.+ for (const pid of listeners()) { try { process.kill(pid, 'SIGTERM') } catch { /* gone */ } }+ await daemon?.close()+ await new Promise<void>((done) => server ? server.close(() => done()) : done())+ if (scratch) rmSync(scratch, { recursive: true, force: true })+ })++ it('agents at work on the released build come through its update, then through the master\'s first start', async () => {+ const d = daemon!+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-120).join('\n')}`) })+ let client = await LocalClient.connect(d)+ const agents: Array<Record<string, any>> = []+ for (const engine of ['claude', 'codex'] as const) {+ const cwd = join(d.projectsDir, `migrating-${engine}`)+ mkdirSync(cwd, { recursive: true })+ // Asked until the released build has finished wiring its handlers (it answers before it has).+ const created = await until(`${fromVersion} to create a ${engine} agent`, async () => {+ const answer = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)+ return answer.error ? null : answer+ }, 60_000, 1_000)+ agents.push(await until(`${engine} to bind on ${fromVersion}`, async () => {+ const agent = (await rows(client)).find((one) => one.id === created.agent.id)+ return agent?.sessionId && agent.status === 'active' ? agent : null+ }, 60_000, 500))+ }+ for (const agent of agents) await turn(client, agent.id, `on ${fromVersion}`)++ // The release ships: the released build finds it, stages it and hands over.+ offered = 'next'+ await until(`the machine to run ${NEXT}`, async () => (await health())?.version === NEXT || null, 120_000, 500)+ client.close()+ client = await LocalClient.connect(d)+ for (const agent of agents) {+ await until(`${agent.id.slice(0, 8)} back on ${NEXT}`, async () => {+ const now = (await rows(client)).find((one) => one.id === agent.id)+ return now?.status === 'active' && now.sessionId === agent.sessionId ? now : null+ }, 60_000, 500)+ await turn(client, agent.id, `on ${NEXT}, handed over`)+ }+ client.close()++ // The next `harness start` (a reboot, the desktop app) starts the new build as harnessd: master and core.+ // The released build's own process may still be about, waiting on its successor; it goes first.+ await d.stop()+ for (const pid of listeners()) process.kill(pid, 'SIGTERM')+ await until('the handed-over core to stop', () => listeners().length === 0 || null, 30_000, 250)+ ;(d.options as { noMaster?: boolean }).noMaster = false+ await d.start()+ expect((await health())?.version).toBe(NEXT)+ expect(d.coresStarted()).toBe(1)+ client = await LocalClient.connect(d)+ for (const agent of agents) {+ await until(`${agent.id.slice(0, 8)} back under the master`, async () => {+ const now = (await rows(client)).find((one) => one.id === agent.id)+ return now?.status === 'active' && now.sessionId === agent.sessionId ? now : null+ }, 60_000, 500)+ await turn(client, agent.id, `on ${NEXT}, under the master`)+ }+ client.close()+ }, 600_000)+})diff --git a/cli/e2e/races.e2e.ts b/cli/e2e/races.e2e.tsindex 7b7028db7..14a2369b6 100644--- a/cli/e2e/races.e2e.ts+++ b/cli/e2e/races.e2e.ts@@ -147,7 +147,12 @@ describe('lifecycle requests that race', () => {const restarted = await client.request('agent_restart', { agentId: agent.id }, 90_000)expect(restarted.error, JSON.stringify(restarted)).toBeUndefined()await active(client, agent.id, agent.sessionId)- await turn(client, agent.id, 'the next message')+ // The long message may still be on its way: it waits out the restart's hold on the terminal, and the+ // new engine answers it. The next one's own turn is the one to wait for, not just any turn's end.+ const started = client.next((frame) => isTurn('turn_started', agent.id)(frame) && frame.payload?.userMessage === 'the next message', 60_000, 'the next message\'s turn_started')+ client.send('message', { agentId: agent.id, content: 'the next message' })+ const opened = await started+ await client.waitFor(isTurn('turn_ended', agent.id), 45_000, 'the next message\'s turn_ended', client.frames.indexOf(opened) + 1)const page = await client.request<{ events: Array<{ type: string; payload: Record<string, any> }> }>('session_get', { sessionId: agent.sessionId, limit: 200 }, 30_000)const users = page.events.filter((event) => event.type === 'user_message').map((event) => String(event.payload.content))expect(users.filter((content) => content === long).length).toBeLessThanOrEqual(1)diff --git a/cli/src/lib/sessionRepair.spec.ts b/cli/src/lib/sessionRepair.spec.tsindex 019a39b38..7cf04b4d7 100644--- a/cli/src/lib/sessionRepair.spec.ts+++ b/cli/src/lib/sessionRepair.spec.ts@@ -1,6 +1,6 @@import { afterEach, describe, expect, it, vi } from 'vitest'-import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'fs'-import { execFileSync } from 'child_process'+import { mkdirSync, mkdtempSync, realpathSync, rmSync, utimesSync, writeFileSync } from 'fs'+import { execFileSync, spawn } from 'child_process'import { tmpdir } from 'os'import { join } from 'path'import { claudeContinuation } from './sessionRepair.js'@@ -498,6 +498,48 @@ describe('session repair — Codex profiles', () => {})})+describe('session repair — a Codex process names its own rollout', () => {+ it('names a fork among its siblings by the rollout its process holds open, where a scan must refuse', async () => {+ const profile = tempRoot()+ const fork = 'a1b2c3d4-1111-4a4a-8a8a-000000000003'+ const sibling = 'a1b2c3d4-1111-4a4a-8a8a-000000000004'+ const forkFile = writeCodexRollout(profile, fork, CWD, STARTED_AT + 1_000)+ const siblingFile = writeCodexRollout(profile, sibling, CWD, STARTED_AT + 2_000)+ const elsewhere = writeCodexRollout(tempRoot(), 'a1b2c3d4-1111-4a4a-8a8a-000000000005', CWD, STARTED_AT)+ vi.resetModules()+ const { codexProcessSession, findLiveSession } = await import('./sessionRepair.js')+ const sessions = join(profile, 'sessions')+ // Two born in the folder since the process started: guessing would wire a tile to a sibling.+ await expect(findLiveSession('codex', CWD, STARTED_AT, { codexHome: profile, bornOnly: true })).resolves.toBeNull()+ // The process holds one of them open, beside files that are not rollouts or not this profile's.+ const held = async () => [forkFile, '/dev/null', join(profile, 'notes.jsonl'), elsewhere]+ await expect(codexProcessSession(42, sessions, CWD, held)).resolves.toEqual({ sessionId: fork, transcriptPath: realpathSync(forkFile) })+ // Two held, or one for another folder: nothing, rather than a guess.+ await expect(codexProcessSession(42, sessions, CWD, async () => [forkFile, siblingFile])).resolves.toBeNull()+ await expect(codexProcessSession(42, sessions, '/Users/demo/elsewhere', held)).resolves.toBeNull()+ await expect(codexProcessSession(42, sessions, CWD, async () => [])).resolves.toBeNull()+ })++ it('reads a live process\'s open files, and binds the repair to the rollout it holds', async () => {+ const profile = tempRoot()+ const fork = 'a1b2c3d4-1111-4a4a-8a8a-000000000006'+ const forkFile = writeCodexRollout(profile, fork, CWD, STARTED_AT + 1_000)+ writeCodexRollout(profile, 'a1b2c3d4-1111-4a4a-8a8a-000000000007', CWD, STARTED_AT + 2_000)+ // A stand-in for Codex: a process that keeps its rollout open.+ const holder = spawn(process.execPath, ['-e', 'require("fs").openSync(process.argv[1], "r"); setInterval(() => {}, 60_000)', forkFile], { stdio: 'ignore' })+ try {+ vi.resetModules()+ const { findLiveSession, openFiles } = await import('./sessionRepair.js')+ await vi.waitFor(async () => expect(await openFiles(holder.pid!)).toContain(realpathSync(forkFile)), { timeout: 10_000, interval: 100 })+ await expect(findLiveSession('codex', CWD, STARTED_AT, { codexHome: profile, bornOnly: true, pid: holder.pid! }))+ .resolves.toEqual({ sessionId: fork, transcriptPath: realpathSync(forkFile) })+ expect(await openFiles(-1)).toEqual([])+ } finally {+ holder.kill()+ }+ })+})+describe('claudeContinuation', () => {it('follows a continued-in marker to the new session file', async () => {const dir = tempRoot()diff --git a/cli/src/lib/sessionRepair.ts b/cli/src/lib/sessionRepair.tsindex 938ef61e6..a66c72f27 100644--- a/cli/src/lib/sessionRepair.ts+++ b/cli/src/lib/sessionRepair.ts@@ -14,7 +14,8 @@* unbound until its next turn, which is recoverable — mis-binding is not.*/-import { open, readdir, readFile, realpath, stat } from 'fs/promises'+import { execFile } from 'child_process'+import { open, readdir, readFile, readlink, realpath, stat } from 'fs/promises'import { basename, dirname, join, relative, sep } from 'path'import { env } from '../config/env.js'import { museEvent, museWorkspaceRoot } from '../engines/muse/normalizer.js'@@ -291,15 +292,19 @@ export async function findLiveSession(const exact = opts?.pid ? await claudeProcessSession(opts.pid, cwd, startedAtMs) : nullreturn exact ?? fileEngineSession(env.CLAUDE_PROJECTS_DIR, cwd, startedAtMs, readClaudeTranscriptMeta, opts)}- case 'codex':+ case 'codex': {+ const sessions = join(opts?.codexHome || env.CODEX_HOME, 'sessions')+ // Exact first, as for Claude: the rollout the process holds open (see `codexProcessSession`).+ const exact = opts?.pid ? await codexProcessSession(opts.pid, sessions, cwd) : null// Codex writes no `cwd` on line one; its rollout meta carries it — and says whether the rollout// belongs to a subagent, which must never become an agent of its own.// Its session id lives INSIDE the file: the name is `rollout-<timestamp>-<id>.jsonl`, so deriving// the id from the filename produced the literal string "rollout-…" (seen on a live pane).- return fileEngineSession(join(opts?.codexHome || env.CODEX_HOME, 'sessions'), cwd, startedAtMs, async (path) => {+ return exact ?? fileEngineSession(sessions, cwd, startedAtMs, async (path) => {const meta = readCodexRolloutMeta(path)return meta && !meta.isSubagent ? { cwd: meta.cwd, sessionId: meta.id || undefined } : null}, opts)+ }case 'pi':return fileEngineSession(join(env.PI_HOME, 'agent', 'sessions'), cwd, startedAtMs, readTranscriptMeta, opts)case 'commandcode':@@ -598,6 +603,48 @@ export async function findResumedTranscript(}/** Claude's native process record is removed at exit; capture it before Stop signals the engine. */+/** The files a process holds open: `/proc` on Linux, `lsof` elsewhere. Empty when neither can say. */+export async function openFiles(pid: number): Promise<string[]> {+ if (!Number.isSafeInteger(pid) || pid <= 0) return []+ if (process.platform === 'linux') {+ const fds = await readdir(`/proc/${pid}/fd`).catch(() => [] as string[])+ const paths = await Promise.all(fds.map((fd) => readlink(`/proc/${pid}/fd/${fd}`).catch(() => null)))+ return paths.filter((path): path is string => !!path && path.startsWith('/'))+ }+ const stdout = await new Promise<string>((resolve) => {+ // `-Fn`: one `n<path>` line per open file. A process that is gone answers nothing.+ execFile('lsof', ['-n', '-P', '-p', String(pid), '-Fn'], { timeout: 3_000 }, (_error, out) => resolve(out ?? ''))+ })+ return stdout.split('\n').filter((line) => line.startsWith('n/')).map((line) => line.slice(1))+}++/**+ * The conversation a Codex process is writing: the rollout it holds open.+ *+ * The one way to name a fork's conversation when its start-up hook was lost (a daemon restart in its+ * first second). `codex fork <id>` names only its source in argv, and a fork shares its source's+ * folder with every sibling started near it, so a scan of that folder finds them all and must refuse+ * to guess — the fork stayed without a conversation for good (e2e/chaos.e2e.ts).+ */+export async function codexProcessSession(+ pid: number,+ sessionsRoot: string,+ cwd: string,+ files: (pid: number) => Promise<string[]> = openFiles,+): Promise<RepairedSession | null> {+ const root = await realpath(sessionsRoot).catch(() => sessionsRoot)+ const found = new Map<string, RepairedSession>()+ for (const path of await files(pid)) {+ if (!/rollout-[^/]*\.jsonl$/.test(path)) continue+ const real = await realpath(path).catch(() => path)+ if (!real.startsWith(`${root}${sep}`)) continue+ const meta = readCodexRolloutMeta(real)+ if (!meta || meta.isSubagent || !meta.id || !meta.cwd || !await sameDir(meta.cwd, cwd)) continue+ found.set(real, { sessionId: meta.id, transcriptPath: real })+ }+ return found.size === 1 ? [...found.values()][0] : null+}+export async function claudeProcessSession(pid: number, cwd: string, startedAtMs: number): Promise<RepairedSession | null> {if (!Number.isSafeInteger(pid) || pid <= 0 || !Number.isFinite(startedAtMs)) return nulltry {diff --git a/docs/design/2026-10-03-harnessd.md b/docs/design/2026-10-03-harnessd.mdindex f195cd123..d631285b8 100644--- a/docs/design/2026-10-03-harnessd.md+++ b/docs/design/2026-10-03-harnessd.md@@ -606,6 +606,21 @@ Found while mapping, and fixed on their own rather than inside a move:reading is cut off alone within the 40 s heartbeat, with memory bounded meanwhile.`e2e/forks.e2e.ts`: a fork asked for mid-turn is refused with the reason, then made after it;a fork of a fork; a source stopped under its fork; both back after a restart.+ - Part 21, found by the fifteenth round (`e2e/chaos.e2e.ts`, a seeded random run of messages,+ slow turns, restarts, stops, resumes, creates, forks, compactions, reconnects and daemon+ restarts, then the daemon's picture checked against tmux and the process table). A Codex fork+ whose start-up hook fell into a daemon restart never bound its conversation. `codex fork <id>`+ names only its source in argv, and the repair's scan of the folder found the fork's siblings+ born beside it and refused to guess. The repair now asks the process first, by the rollout it+ holds open, as it already asked Claude Code by its per-process record. The fake engines now leave+ those same traces, and Claude's sends its `UserPromptSubmit` catch hook on every prompt as Claude+ Code does.+ - Upgrading from a released build (`e2e/migration.e2e.ts`, run with `MIGRATION_FROM` set to a+ released bundle, and skipped otherwise). A daemon of the released build, with a Claude Code and a+ Codex agent at work, takes this checkout's build from its update manifest and hands over the way+ that release does, by spawning the new `__run` and exiting. The new core comes up on its own,+ without a master, with both agents and their conversations, and they take turns. The next start+ brings up the master over the same agents. Passed from v0.3.57, three runs of three.## Testing