The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
feat: introduce agent restoration logic after system reboot by persisting bypass permissions and clearing dormant process identities
0xken152c3d2a6ff17f
diff --git a/cli/README.md b/cli/README.mdindex 90162d28f..d8bfd4bb8 100644--- a/cli/README.md+++ b/cli/README.md@@ -139,6 +139,11 @@ claude under a terminal backend ──writes──▶ ~/.claude/projects/**.jsonnamed `harness-<engine>-<timestamp>`, which is what `agent_create` names them. A tmux session youopen by hand and run an engine in, or one an agent spawns itself with a nested `tmux new-session`,never becomes an agent — create it through the app (or `agent_create`) instead.+- **Agents survive a reboot.** The registry outlives the tmux server. On start, every registered agent+ whose pane is gone (a reboot, a `tmux kill-server`) gets a new pane running the same engine in the+ same folder, resuming its engine session when it has one, under the same agent id — so the app's+ tiles reattach by themselves. Agents that were pointed at a grid are not restored (the launch+ credential is never persisted), and an engine with no resume flag (devin) comes back fresh.- **Hooks bind mutable engine sessions** to the process agent using authenticated tmux and/or Herdrruntime hints plus verified caller ancestry. Hook socket paths are lookup hints only. They do notrequire `MACHINE_ID`. `SessionStart` and catch hooks attach transcript/store metadata; `SessionEnd` onlydiff --git a/cli/src/cli.ts b/cli/src/cli.tsindex 023ab191a..434f60aa7 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -65,6 +65,7 @@ import { restartAgent, type RestartAgentDeps } from './lib/restartAgent.js'import { claudeContinuation, findLiveSession } from './lib/sessionRepair.js'import { TmuxBackend } from './lib/tmuxBackend.js'import { createAndRegisterPane } from './lib/createAgentPane.js'+import { restoreAgents } from './lib/restoreAgents.js'import { buildHarnessSessionLabel } from './lib/harnessSessionLabel.js'import { basename } from 'node:path'import {@@ -2393,6 +2394,10 @@ async function runForeground(session: AuthSession): Promise<void> {const wasLaunching = current.launch?.state !== undefined && current.launch.state !== 'ready'registry.updateRuntimes(current.agentId, observed.runtimes, observed.primaryRuntimeKey)registry.updateProcessIdentity(current.agentId, observed.processIdentity, observed.gateway, observed.grid)+ // The live argv is the truth about the bypass flag, and this is the one place every running+ // agent passes through — so a row written before the flag was persisted at all (or by a build+ // that did not yet) learns it here, before any pane recreation ever needs it.+ registry.setBypassPermission(current.agentId, bypassPermissionActive(current.engine, observed.args))if (wasLaunching) registry.setLaunch(current.agentId, { state: 'ready' })await bindObservedAgent(observed)if (wasDormant || wasLaunching) {@@ -3196,6 +3201,64 @@ async function runForeground(session: AuthSession): Promise<void> {}watcher.start()await cursorDiscovery.start()+ // Panes that died while the daemon was down (a reboot takes the whole tmux server with it) are+ // rebuilt BEFORE the first reconcile pass: it would otherwise count them absent, and a second+ // pass five seconds later would drop the agents for good. Pane creation is awaited so the first+ // announce already shows every restored agent with a terminal; binding their engine processes+ // continues in the background, the same way `agent_create` does it.+ if (tmuxBackend) {+ const backend = tmuxBackend+ const summary = await restoreAgents({+ registry,+ // "Alive" means the pane still runs THIS row's engine — not merely that tmux knows the id.+ // A new tmux server hands out `%N` from zero again, so a stale id can name someone's shell;+ // and a pane that outlived the daemon in a session discovery no longer lists still has its+ // engine, which a second pane resuming the same session would collide with.+ liveProcess: (entry, runtime) => resolvePaneEngineProcess(runtime.paneId, entry.engine),+ buildLaunch: (entry, opts) => {+ // Mirrors `agent_create`: a Codex profile other than the default needs its hooks and its+ // CODEX_HOME; the install check runs inside the pane's own shell.+ if (entry.codexHome && !env.DISABLE_HOOK_INSTALL) installCodexHooks(hookPort, entry.codexHome)+ const argv = buildEngineLaunchArgv(entry.engine, {+ ...opts,+ bypassPermission: entry.bypassPermission === true,+ installIfMissing: enginePathOverride(entry.engine) ? undefined : engineInstallRecipe(entry.engine),+ ...(entry.cwd ? { cwd: entry.cwd } : {}),+ })+ return { argv, ...(entry.codexHome ? { env: { CODEX_HOME: entry.codexHome } } : {}) }+ },+ createPane: async (entry, launch) => {+ const created = await backend.create({+ cwd: homedir(),+ label: buildHarnessSessionLabel(entry.engine),+ command: launch.argv,+ ...(launch.env ? { env: launch.env } : {}),+ })+ return created.state === 'succeeded'+ ? { ok: true, runtime: created.runtime }+ : { ok: false, reason: created.reason }+ },+ respawn: async (runtime, launch) => {+ const result = await backend.respawn(runtime, {+ command: launch.argv,+ cwd: homedir(),+ ...(launch.env ? { env: launch.env } : {}),+ })+ return result.state === 'succeeded' ? { ok: true } : { ok: false, reason: result.reason }+ },+ probeProcess: (runtime, engine) => resolvePaneEngineProcess(runtime.paneId, engine),+ paneState: (runtime) => tmuxPaneState(runtime.paneId),+ clearRemainOnExit: (runtime) => clearPaneRemainOnExit(runtime.paneId),+ holdRoute: (key, ms) => agentReconciler.holdRoute(key, ms),+ releaseRoute: (key) => agentReconciler.releaseRoute(key),+ triggerHint: (runtime, engine) => agentReconciler.triggerHint(runtime, engine),+ log: (message) => console.log(message),+ })+ if (summary.restored.length || summary.failed.length || registry.rebootedSinceLastRun) {+ console.log(`[restore] restored ${summary.restored.length} · skipped ${summary.skipped.length} · failed ${summary.failed.length}`+ + (registry.rebootedSinceLastRun ? ' · after reboot' : ''))+ }+ }await agentReconciler.start(env.TERMINAL_RECONCILE_INTERVAL_MS ?? env.TMUX_REAP_INTERVAL_MS)for (const task of await loadCursorPendingTasks(env.ADAPTER_DATA_DIR)) {onCursorTaskStart(task.sessionId, task.toolUseId, task.input)@@ -3435,6 +3498,7 @@ async function runForeground(session: AuthSession): Promise<void> {env: gridLaunch?.env ?? (codexHome ? { CODEX_HOME: codexHome } : undefined),grid: grid ? { baseUrl: grid.baseUrl, model: grid.model ?? null } : null,codexHome,+ bypassPermission,})if (!result.ok) return { ok: false, error: result.error, detail: result.detail }const { spawned, pending } = resultdiff --git a/cli/src/lib/createAgentPane.ts b/cli/src/lib/createAgentPane.tsindex 8436cc061..631cb5c4d 100644--- a/cli/src/lib/createAgentPane.ts+++ b/cli/src/lib/createAgentPane.ts@@ -29,9 +29,11 @@ export interface CreateAgentPaneDeps {cwd?: string | nullgrid?: { baseUrl: string; model: string | null } | nullcodexHome?: string | null+ bypassPermission?: boolean}) => RegisteredSession | null }engine: AgentEnginecwd?: string | null+ bypassPermission?: boolean/** Base tmux session name (`-s`). Retries append `-r<attempt>` — see module doc. */sessionLabel: stringargv: string[]@@ -71,6 +73,7 @@ export async function createAndRegisterPane(deps: CreateAgentPaneDeps): Promise<cwd: deps.cwd,grid: deps.grid,codexHome: deps.codexHome,+ bypassPermission: deps.bypassPermission,})if (pending) return { ok: true, spawned, pending }console.warn(`[agent] create ${deps.engine} registration failed · pane ${spawned.runtime.paneId} · `diff --git a/cli/src/lib/registry.spec.ts b/cli/src/lib/registry.spec.tsindex 596a8a124..921dcd8a2 100644--- a/cli/src/lib/registry.spec.ts+++ b/cli/src/lib/registry.spec.ts@@ -1131,3 +1131,178 @@ describe('agent identity: the process owns the agent, the session is bound to itexpect(saved[0]).not.toHaveProperty('launcherId')})})++describe('registry across a reboot and pane loss', () => {+ beforeEach(() => {+ vi.useFakeTimers()+ dataDir = mkdtempSync(join(tmpdir(), 'adapter-registry-reboot-'))+ })++ afterEach(() => {+ vi.useRealTimers()+ rmSync(dataDir, { recursive: true, force: true })+ delete process.env.ADAPTER_DATA_DIR+ delete process.env.CLAUDE_PROJECTS_DIR+ delete process.env.CODEX_HOME+ delete process.env.CURSOR_HOME+ })++ /** A row as `agent_create` + a SessionStart hook leave it: bound, with a live-looking process. */+ function persistedRow(transcriptPath: string, overrides: Record<string, unknown> = {}) {+ return {+ schemaVersion: 2,+ active: true,+ launch: { state: 'ready' },+ agentId: 'agent-a',+ sessionId: 'session-a',+ boundAt: 1,+ engine: 'claude',+ gateway: null,+ grid: null,+ codexHome: null,+ bypassPermission: true,+ transcriptPath,+ projectDir: 'demo',+ cwd: '/tmp/demo',+ runtimes: [{ backend: 'tmux', paneId: '%3' }],+ primaryRuntimeKey: 'tmux\u0000%3',+ tmuxPane: '%3',+ source: null,+ title: null,+ model: null,+ cliVersion: null,+ processIdentity: processIdentity(4242),+ registeredAt: 1,+ updatedAt: 1,+ lastHookAt: 1,+ lastTranscriptAt: 1,+ ...overrides,+ }+ }++ it('keeps every agent after a reboot, clearing only what a reboot actually kills', async () => {+ const transcriptPath = join(dataDir, 'session-a.jsonl')+ writeFileSync(transcriptPath, '{}\n')+ chmodSync(dataDir, 0o755)+ writeLegacyStateFile(join(dataDir, 'registry.json'), JSON.stringify([persistedRow(transcriptPath, { codexHome: '/tmp/codex-work' })]))+ // A boot marker from the distant past: whichever way the platform identifies a boot, this is not it.+ writeLegacyStateFile(join(dataDir, 'registry-boot'), 'time:1')++ const { registry } = await loadRegistryModule()+ registry.load()++ expect(registry.rebootedSinceLastRun).toBe(true)+ expect(registry.byAgent('agent-a')).toMatchObject({+ agentId: 'agent-a',+ sessionId: 'session-a',+ engine: 'claude',+ cwd: '/tmp/demo',+ codexHome: '/tmp/codex-work',+ bypassPermission: true,+ runtimes: [{ backend: 'tmux', paneId: '%3' }],+ active: false,+ processIdentity: null,+ })+ // The cleared snapshot is on disk immediately: the boot marker was already refreshed, so a crash+ // before this save would otherwise leave the dead pid behind with no second reboot to notice it.+ const saved = JSON.parse(readFileSync(join(dataDir, 'registry.json'), 'utf-8')) as Array<Record<string, unknown>>+ expect(saved[0]).toMatchObject({ agentId: 'agent-a', processIdentity: null, codexHome: '/tmp/codex-work', bypassPermission: true })++ // The next start on the SAME boot is not a reboot, and finds the row as the previous save left it.+ const again = await loadRegistryModule()+ again.registry.load()+ expect(again.registry.rebootedSinceLastRun).toBe(false)+ expect(again.registry.byAgent('agent-a')).toMatchObject({ processIdentity: null, bypassPermission: true })+ })++ it('leaves the process identity alone when the machine did not reboot', async () => {+ const transcriptPath = join(dataDir, 'session-a.jsonl')+ writeFileSync(transcriptPath, '{}\n')+ chmodSync(dataDir, 0o755)+ writeLegacyStateFile(join(dataDir, 'registry.json'), JSON.stringify([persistedRow(transcriptPath)]))++ const { registry } = await loadRegistryModule()+ registry.load()++ expect(registry.rebootedSinceLastRun).toBe(false)+ expect(registry.byAgent('agent-a')?.processIdentity).toEqual(processIdentity(4242))+ expect(registry.byProcess('claude', processIdentity(4242))?.agentId).toBe('agent-a')+ })++ it('carries bypassPermission from agent_create through the first hook bind', async () => {+ const transcriptPath = join(dataDir, 'session-b.jsonl')+ writeFileSync(transcriptPath, '{}\n')+ const { registry } = await loadRegistryModule()+ registry.load()++ const pending = registry.openPendingAgent({+ engine: 'claude',+ runtimes: [{ backend: 'tmux', paneId: '%9' }],+ cwd: '/tmp/demo',+ bypassPermission: true,+ })+ expect(pending?.bypassPermission).toBe(true)++ const bound = registry.register({ sessionId: 'session-b', transcriptPath, tmuxPane: '%9', cwd: '/tmp/demo' })+ expect(bound?.entry.agentId).toBe(pending!.agentId)+ expect(bound?.entry.bypassPermission).toBe(true)++ expect(registry.setBypassPermission(pending!.agentId, false)).toBe(true)+ expect(registry.byAgent(pending!.agentId)).not.toHaveProperty('bypassPermission')+ const saved = JSON.parse(readFileSync(join(dataDir, 'registry.json'), 'utf-8')) as Array<Record<string, unknown>>+ expect(saved[0]).not.toHaveProperty('bypassPermission')+ })++ it('clearProcessIdentity forgets the pid everywhere it is indexed', async () => {+ const { registry } = await loadRegistryModule()+ registry.load()+ const opened = registry.openProcessAgent({ engine: 'claude', tmuxPane: '%5', processIdentity: processIdentity(77) })+ const agentId = opened!.entry.agentId+ expect(registry.byProcess('claude', processIdentity(77))?.agentId).toBe(agentId)++ expect(registry.clearProcessIdentity(agentId)).toBe(true)+ expect(registry.byAgent(agentId)?.processIdentity).toBeNull()+ expect(registry.byProcess('claude', processIdentity(77))).toBeUndefined()+ expect(registry.clearProcessIdentity(agentId)).toBe(true) // idempotent+ expect(registry.clearProcessIdentity('nobody')).toBe(false)+ })++ it('survives two restored panes swapping ids when done inside one transaction', async () => {+ const { registry } = await loadRegistryModule()+ registry.load()+ const a = registry.openProcessAgent({ engine: 'claude', tmuxPane: '%1', processIdentity: processIdentity(11) })!.entry.agentId+ const b = registry.openProcessAgent({ engine: 'claude', tmuxPane: '%2', processIdentity: processIdentity(12) })!.entry.agentId++ // A new tmux server hands out %1/%2 again, in the other order. Saved between the two updates, A+ // would claim %2 while B still lists it, and the save's route dedupe evicts one of them.+ await registry.transaction(() => {+ registry.updateRuntimes(a, [{ backend: 'tmux', paneId: '%2' }], 'tmux\u0000%2')+ registry.updateRuntimes(b, [{ backend: 'tmux', paneId: '%1' }], 'tmux\u0000%1')+ })++ expect(registry.byAgent(a)?.tmuxPane).toBe('%2')+ expect(registry.byAgent(b)?.tmuxPane).toBe('%1')+ const saved = JSON.parse(readFileSync(join(dataDir, 'registry.json'), 'utf-8')) as Array<Record<string, unknown>>+ expect(saved.map((row) => row.agentId).sort()).toEqual([a, b].sort())+ })++ it('adopts a new process into an agent whose identity was cleared, keeping its id and session', async () => {+ const transcriptPath = join(dataDir, 'session-c.jsonl')+ writeFileSync(transcriptPath, '{}\n')+ const { registry } = await loadRegistryModule()+ registry.load()+ const bound = registerProcess(registry, { sessionId: 'session-c', transcriptPath, tmuxPane: '%4', cwd: '/tmp/demo' })+ const agentId = bound!.entry.agentId++ // What restore does: forget the dead pid, point the row at the recreated pane, launch pending.+ registry.clearProcessIdentity(agentId)+ registry.updateRuntimes(agentId, [{ backend: 'tmux', paneId: '%0' }], 'tmux\u0000%0')+ registry.setLaunch(agentId, { state: 'starting' })++ const adopted = registry.openProcessAgent({ engine: 'claude', tmuxPane: '%0', processIdentity: processIdentity(500) })+ expect(adopted).toMatchObject({ isNew: false, evicted: null })+ expect(adopted?.entry.agentId).toBe(agentId)+ expect(adopted?.entry.sessionId).toBe('session-c')+ expect(registry.list()).toHaveLength(1)+ })+})diff --git a/cli/src/lib/registry.ts b/cli/src/lib/registry.tsindex 109ded03c..3d98b1241 100644--- a/cli/src/lib/registry.ts+++ b/cli/src/lib/registry.ts@@ -106,6 +106,13 @@ export interface RegisteredSession {* to a different grid.*/codexHome?: string | null+ /**+ * Whether the engine was launched with its permission prompts bypassed (`--dangerously-skip-permissions`+ * and friends, `BYPASS_PERMISSION_FLAGS`). Recorded at launch because it is otherwise only readable+ * off a LIVE process's argv — and a pane that has to be recreated after a reboot has no live process+ * to read it from. Like `codexHome`, chosen at launch and carried forward, never re-derived.+ */+ bypassPermission?: boolean/** Legacy launcher-owned snapshots may still contain this field. New records never write it. */launcherId?: stringtranscriptPath: string | null@@ -555,6 +562,12 @@ class Registry {private writeBlocked = false/** Last committed row bytes, used for a three-way merge with daemon-down hook writes. */private persistedBaseline = new Map<string, string>()+ private rebooted = false++ /** True when the last `load()` found the machine had rebooted since the previous daemon run. */+ get rebootedSinceLastRun(): boolean {+ return this.rebooted+ }private index(entry: RegisteredSession): void {this.agents.set(entry.agentId, entry)@@ -590,8 +603,10 @@ class Registry {entry.lastTranscriptAt = Date.now()}- /** Load persisted process agents. Invalid session bindings are released without dropping their agent;- * a reboot still drops all cached pane identities because no process can survive it. */+ /** Load persisted process agents. Invalid session bindings are released without dropping their agent.+ * A reboot keeps every row but clears its process identity and marks it dormant: no process survives+ * a reboot, but the agent — its id, cwd, session and pane placement — is what `restoreAgents` rebuilds+ * a pane for. */load(): void {this.agents.clear()this.sessionIndex.clear()@@ -599,6 +614,7 @@ class Registry {this.processIndex.clear()this.terminalAvailableAgents.clear()this.writeBlocked = false+ this.rebooted = falsethis.persistedBaseline.clear()try {secureStateDirectory(env.ADAPTER_DATA_DIR)@@ -645,12 +661,14 @@ class Registry {if (arr.some((row) => row.schemaVersion !== 2)) {atomicWriteJson(PRE_V2_BACKUP_FILE, arr, true)}+ let changed = falseif (rebooted) {- console.log(`[registry] machine rebooted since last run — dropping ${Array.isArray(arr) ? arr.length : 0} cached tmux session(s) (stale panes)`)- this.save() // overwrite the stale on-disk snapshot so a same-boot restart can't reload it- return+ this.rebooted = true+ console.log(`[registry] machine rebooted since last run — ${arr.length} agent(s) kept with their process identity cleared (stale panes, restored on start)`)+ // The boot marker was already refreshed above, so a crash before the save below would leave+ // pre-reboot pids on disk with no second chance to notice — force the cleared snapshot out.+ changed = true}- let changed = falsefor (const raw of Array.isArray(arr) ? arr : []) {const engine = normalizedAgentEngine(raw?.engine)const runtimes = normalizedRuntimes(raw?.runtimes, raw?.tmuxPane)@@ -705,7 +723,7 @@ class Registry {if (!agentId) { changed = true; continue }if (agentId !== raw.agentId) changed = trueconst now = Date.now()- const active = raw.active !== false+ const active = !rebooted && raw.active !== falseconst launch = normalizedLaunch(raw.launch)const s: RegisteredSession = {schemaVersion: 2,@@ -714,6 +732,10 @@ class Registry {agentId,boundAt: bound ? (typeof raw.boundAt === 'number' ? raw.boundAt : (raw.registeredAt ?? now)) : null,engine,+ gateway: raw.gateway === 'ori' ? 'ori' : null,+ grid: normalizedGridAssignment(raw.grid),+ codexHome: typeof rawCodexHome === 'string' && rawCodexHome ? rawCodexHome : null,+ ...(raw.bypassPermission === true ? { bypassPermission: true } : {}),transcriptPath,title: titleDisplayName(typeof raw.title === 'string' ? raw.title : null),sessionId: bound ? rawSessionId : '',@@ -729,7 +751,7 @@ class Registry {source: bound && typeof raw.source === 'string' ? raw.source : null,cliVersion: typeof raw.cliVersion === 'string' ? raw.cliVersion : null,model: modelString(raw.model),- processIdentity: validProcessIdentity(raw.processIdentity) ? raw.processIdentity : null,+ processIdentity: !rebooted && validProcessIdentity(raw.processIdentity) ? raw.processIdentity : null,registeredAt: typeof raw.registeredAt === 'number' ? raw.registeredAt : now,updatedAt: typeof raw.updatedAt === 'number' ? raw.updatedAt : now,lastHookAt: typeof raw.lastHookAt === 'number' ? raw.lastHookAt : (raw.updatedAt ?? now),@@ -919,6 +941,7 @@ class Registry {cwd?: string | nullgrid?: GridAssignment | nullcodexHome?: string | null+ bypassPermission?: boolean}): RegisteredSession | null {if (this.writeBlocked) return nullconst runtimes = normalizedRuntimes(input.runtimes)@@ -937,6 +960,7 @@ class Registry {gateway: null,grid: input.grid ?? null,codexHome: input.codexHome ?? null,+ ...(input.bypassPermission ? { bypassPermission: true } : {}),transcriptPath: null,projectDir: basename(input.cwd ?? '') || agentId,cwd: input.cwd ?? null,@@ -1091,6 +1115,7 @@ class Registry {// re-reads off the live process on every discovery) — a hook-triggered bind must carry it// forward or the very first SessionStart hook would silently wipe the agent's chosen profile.codexHome: existing?.codexHome ?? null,+ ...(existing?.bypassPermission ? { bypassPermission: true } : {}),processIdentity: validProcessIdentity(input.processIdentity) ? input.processIdentity : existing?.processIdentity ?? null,registeredAt: existing?.registeredAt ?? now,updatedAt: now,@@ -1241,6 +1266,30 @@ class Registry {return true}+ /** Forget the process an agent was last seen as — its pane is gone, so the pid is gone with it. */+ clearProcessIdentity(agentId: string): boolean {+ const session = this.agents.get(agentId)+ if (!session) return false+ if (!session.processIdentity) return true+ this.drop(session)+ session.processIdentity = null+ session.updatedAt = Date.now()+ this.index(session)+ this.save()+ return true+ }++ setBypassPermission(agentId: string, bypassPermission: boolean): boolean {+ const session = this.agents.get(agentId)+ if (!session) return false+ if ((session.bypassPermission === true) === bypassPermission) return true+ if (bypassPermission) session.bypassPermission = true+ else delete session.bypassPermission+ session.updatedAt = Date.now()+ this.save()+ return true+ }+touchTranscript(sessionId: string, at = Date.now()): boolean {const session = this.bySession(sessionId)if (!session) return false@@ -1501,3 +1550,11 @@ function validProcessIdentity(value: unknown): value is ProcessIdentity {const p = value as Partial<ProcessIdentity> | null | undefinedreturn !!p && Number.isSafeInteger(p.pid) && (p.pid ?? 0) > 0 && typeof p.executable === 'string' && typeof p.startMarker === 'string'}++/** A persisted grid assignment. Lenient on `model` on purpose: a row that only knows WHERE it pointed+ * must still read as a grid agent, or a restore would relaunch it on the engine's own login. */+function normalizedGridAssignment(value: unknown): GridAssignment | null {+ const g = value as Partial<GridAssignment> | null | undefined+ if (!g || typeof g.baseUrl !== 'string' || !g.baseUrl) return null+ return { baseUrl: g.baseUrl, model: typeof g.model === 'string' ? g.model : null }+}diff --git a/cli/src/lib/restoreAgents.spec.ts b/cli/src/lib/restoreAgents.spec.tsnew file mode 100644index 000000000..07efc1386--- /dev/null+++ b/cli/src/lib/restoreAgents.spec.ts@@ -0,0 +1,310 @@+import { describe, expect, it, vi } from 'vitest'+import type { AgentLaunch, ProcessIdentity, RegisteredSession } from './registry.js'+import type { TerminalRuntimeRef, TmuxRuntimeRef } from './terminalTypes.js'+import { restoreAgents, type RestoreAgentsDeps, type RestoreLaunch } from './restoreAgents.js'++const identity = (pid: number): ProcessIdentity => ({ pid, executable: 'claude', startMarker: `start ${pid}` })++function row(overrides: Partial<RegisteredSession> = {}): RegisteredSession {+ return {+ schemaVersion: 2,+ active: false,+ launch: { state: 'ready' },+ agentId: 'agent-a',+ sessionId: 'session-a',+ boundAt: 1,+ engine: 'claude',+ transcriptPath: null,+ projectDir: 'demo',+ cwd: '/tmp/demo',+ runtimes: [{ backend: 'tmux', paneId: '%3' }],+ primaryRuntimeKey: 'tmux\u0000%3',+ tmuxPane: '%3',+ source: null,+ title: null,+ model: null,+ cliVersion: null,+ processIdentity: null,+ registeredAt: 1,+ updatedAt: 1,+ lastHookAt: 1,+ lastTranscriptAt: 1,+ ...overrides,+ }+}++interface Harness {+ deps: RestoreAgentsDeps+ calls: string[]+ rows: Map<string, RegisteredSession>+ launches: Array<{ agentId: string; resumeSessionId?: string }>+ /** Per-pane scripted answers for probeProcess; shifted on each call. */+ probes: Map<string, Array<ProcessIdentity | null>>+ states: Map<string, Array<{ dead: boolean } | null>>+ nextPane: string[]+ paneCreates: number+ respawns: number+ transactionDepth: number+ released: string[]+}++function harness(rows: RegisteredSession[], opts: { livePanes?: string[]; failCreate?: boolean; budgetMs?: number; settleMs?: number } = {}): Harness {+ const h: Harness = {+ calls: [],+ rows: new Map(rows.map((r) => [r.agentId, r])),+ launches: [],+ probes: new Map(),+ states: new Map(),+ nextPane: ['%0', '%1', '%2'],+ paneCreates: 0,+ respawns: 0,+ transactionDepth: 0,+ released: [],+ deps: undefined as unknown as RestoreAgentsDeps,+ }+ const live = new Set(opts.livePanes ?? [])+ const note = (name: string) => { h.calls.push(`${name}${h.transactionDepth ? '@tx' : ''}`) }+ h.deps = {+ registry: {+ list: () => [...h.rows.values()],+ byAgent: (id) => h.rows.get(id),+ transaction: async (apply) => {+ h.transactionDepth+++ try { return await apply() } finally { h.transactionDepth-- }+ },+ clearProcessIdentity: (id) => { note(`clearIdentity:${id}`); const r = h.rows.get(id); if (r) r.processIdentity = null; return !!r },+ updateRuntimes: (id, runtimes: readonly TerminalRuntimeRef[], key) => {+ note(`updateRuntimes:${id}:${(runtimes[0] as TmuxRuntimeRef).paneId}`)+ const r = h.rows.get(id); if (!r) return false+ r.runtimes = [...runtimes]; r.primaryRuntimeKey = key ?? ''; r.active = true+ return true+ },+ setLaunch: (id, launch: AgentLaunch) => {+ note(`setLaunch:${id}:${launch.state}${launch.state === 'failed' ? `:${launch.error}` : ''}`)+ const r = h.rows.get(id); if (!r) return null+ r.launch = launch; return r+ },+ updateProcessIdentity: (id, pi) => { note(`updateIdentity:${id}:${pi.pid}`); const r = h.rows.get(id); if (r) r.processIdentity = pi; return !!r },+ unbindSession: (sid) => { note(`unbind:${sid}`); return true },+ inheritName: (from, to) => { note(`inheritName:${from}->${to}`) },+ },+ liveProcess: async (_entry, runtime) => live.has(runtime.paneId) ? identity(1000 + Number(runtime.paneId.slice(1))) : null,+ buildLaunch: (entry, o) => {+ h.launches.push({ agentId: entry.agentId, ...(o.resumeSessionId ? { resumeSessionId: o.resumeSessionId } : {}) })+ const launch: RestoreLaunch = { argv: [entry.engine, ...(o.resumeSessionId ? ['--resume', o.resumeSessionId] : [])] }+ return launch+ },+ createPane: async (entry) => {+ note(`createPane:${entry.agentId}`)+ h.paneCreates+++ if (opts.failCreate) return { ok: false, reason: 'tmux said no' }+ return { ok: true, runtime: { backend: 'tmux', paneId: h.nextPane.shift() ?? '%99' } }+ },+ respawn: async (runtime, launch) => { note(`respawn:${runtime.paneId}:${launch.argv.join(' ')}`); h.respawns++; return { ok: true } },+ probeProcess: async (runtime) => {+ const queue = h.probes.get(runtime.paneId) ?? []+ return queue.length ? queue.shift()! : null+ },+ paneState: async (runtime) => {+ const queue = h.states.get(runtime.paneId) ?? []+ return queue.length ? queue.shift()! : { dead: false }+ },+ clearRemainOnExit: async (runtime) => { note(`clearRemainOnExit:${runtime.paneId}`) },+ holdRoute: (key) => { note(`hold:${key}`) },+ releaseRoute: (key) => { note(`release:${key}`); h.released.push(key) },+ triggerHint: async (runtime) => { note(`triggerHint:${runtime.paneId}`) },+ log: () => {},+ budgetMs: opts.budgetMs ?? 60_000,+ settleMs: opts.settleMs ?? 0,+ sleep: async () => {},+ }+ return h+}++/** A watch always ends in `finally { releaseRoute }`; the success path releases once more before the+ * hint, so "settled" means at least `watches` releases and the LAST one came from a finally. */+const settled = (h: Harness, watches: number, successes = 0) =>+ vi.waitFor(() => { expect(h.released.length).toBe(watches + successes) })++describe('restoreAgents — which agents get a pane back', () => {+ it('recreates a missing pane with a resume launch and hands the row over to discovery', async () => {+ const h = harness([row()])+ h.probes.set('%0', [null, identity(500)])++ const summary = await restoreAgents(h.deps)++ expect(summary).toEqual({ restored: ['agent-a'], skipped: [], failed: [] })+ expect(h.launches).toEqual([{ agentId: 'agent-a', resumeSessionId: 'session-a' }])+ // Every registry write of phase 1 happens inside the one transaction, in this order.+ expect(h.calls.slice(0, 5)).toEqual([+ 'clearIdentity:agent-a@tx',+ 'createPane:agent-a@tx',+ 'hold:tmux\u0000%0@tx',+ 'updateRuntimes:agent-a:%0@tx',+ 'setLaunch:agent-a:starting@tx',+ ])+ await settled(h, 1, 1)+ // The route is released BEFORE the hint so the pass it triggers sees the pane again.+ expect(h.calls.slice(5)).toEqual([+ 'updateIdentity:agent-a:500',+ 'clearRemainOnExit:%0',+ 'release:tmux\u0000%0',+ 'triggerHint:%0',+ 'release:tmux\u0000%0', // the finally, idempotent by contract+ ])+ expect(h.rows.get('agent-a')?.launch).toEqual({ state: 'starting' })+ })++ it('launches fresh when the row has no session to resume', async () => {+ const h = harness([row({ sessionId: '', boundAt: null })])+ h.probes.set('%0', [identity(1)])+ await restoreAgents(h.deps)+ expect(h.launches).toEqual([{ agentId: 'agent-a' }])+ await settled(h, 1, 1)+ })++ it('leaves a pane that still runs its engine alone, re-identifying a row that lost its pid', async () => {+ const h = harness([row()], { livePanes: ['%3'] })+ const summary = await restoreAgents(h.deps)+ expect(summary).toEqual({ restored: [], skipped: [], failed: [] })+ expect(h.paneCreates).toBe(0)+ // No identity on the row (a misread reboot, a tmux server that outlived the daemon): the live+ // pid is written back so discovery adopts by process rather than by route.+ expect(h.calls).toEqual(['updateIdentity:agent-a:1003'])+ })++ it('does not touch a live row that still knows its process', async () => {+ const h = harness([row({ processIdentity: identity(1003) })], { livePanes: ['%3'] })+ await restoreAgents(h.deps)+ expect(h.calls).toEqual([])+ })++ it('skips herdr-only rows, failed launches and grid agents, saying why', async () => {+ const h = harness([+ row({ agentId: 'herdr', runtimes: [{ backend: 'herdr', sessionName: 's', paneId: 'p' } as unknown as TerminalRuntimeRef] }),+ row({ agentId: 'broken', launch: { state: 'failed', error: 'START_TIMEOUT' } }),+ row({ agentId: 'gridded', grid: { baseUrl: 'https://grid.example/relay', model: null } }),+ ])+ const summary = await restoreAgents(h.deps)+ expect(summary.restored).toEqual([])+ expect(summary.skipped.map((s) => s.agentId)).toEqual(['herdr', 'broken', 'gridded'])+ expect(summary.skipped[2].reason).toMatch(/grid/)+ expect(h.paneCreates).toBe(0)+ })++ it('records a pane that could not be opened and leaves the row untouched', async () => {+ const h = harness([row()], { failCreate: true })+ const summary = await restoreAgents(h.deps)+ expect(summary.failed).toEqual([{ agentId: 'agent-a', reason: 'tmux said no' }])+ expect(h.calls).not.toContainEqual(expect.stringMatching(/^updateRuntimes/))+ expect(h.calls).not.toContainEqual(expect.stringMatching(/^hold/))+ expect(h.rows.get('agent-a')?.tmuxPane).toBe('%3')+ })++ it('restores several agents inside one transaction so reused pane ids cannot evict each other', async () => {+ const h = harness([row({ agentId: 'a', runtimes: [{ backend: 'tmux', paneId: '%1' }] }), row({ agentId: 'b', runtimes: [{ backend: 'tmux', paneId: '%0' }] })])+ h.probes.set('%0', [identity(1)])+ h.probes.set('%1', [identity(2)])+ const summary = await restoreAgents(h.deps)+ expect(summary.restored).toEqual(['a', 'b'])+ expect(h.calls.filter((c) => c.startsWith('updateRuntimes'))).toEqual(['updateRuntimes:a:%0@tx', 'updateRuntimes:b:%1@tx'])+ await settled(h, 2, 2)+ })+})++describe('restoreAgents — waiting for the engine', () => {+ it('falls back to a fresh launch once when the resumed engine dies, and unbinds the stale session', async () => {+ const h = harness([row()])+ h.probes.set('%0', [null, null, identity(9)])+ h.states.set('%0', [{ dead: true }, { dead: false }])++ await restoreAgents(h.deps)+ await settled(h, 1, 1)++ expect(h.respawns).toBe(1)+ expect(h.launches).toEqual([+ { agentId: 'agent-a', resumeSessionId: 'session-a' },+ { agentId: 'agent-a' },+ ])+ const tail = h.calls.slice(5)+ expect(tail.slice(0, 3)).toEqual(['inheritName:session-a->agent-a', 'unbind:session-a', 'respawn:%0:claude'])+ expect(tail).toContain('updateIdentity:agent-a:9')+ expect(h.rows.get('agent-a')?.launch).toEqual({ state: 'starting' })+ })++ it('treats an engine that appears and then exits inside the settling window as a rejected resume', async () => {+ const h = harness([row()], { settleMs: 50 })+ // First probe: the resumed claude is up. Settling poll: the pane is dead. After the fresh+ // respawn: up again, and this time it stays.+ h.probes.set('%0', [identity(7), identity(8)])+ h.states.set('%0', [{ dead: true }])++ await restoreAgents(h.deps)+ await settled(h, 1, 1)++ expect(h.respawns).toBe(1)+ expect(h.launches.map((l) => l.resumeSessionId ?? 'fresh')).toEqual(['session-a', 'fresh'])+ const tail = h.calls.slice(5)+ expect(tail.slice(0, 4)).toEqual(['updateIdentity:agent-a:7', 'inheritName:session-a->agent-a', 'unbind:session-a', 'respawn:%0:claude'])+ expect(tail).toContain('updateIdentity:agent-a:8')+ expect(tail).toContain('clearRemainOnExit:%0')+ // remain-on-exit was NOT cleared before the first engine died — that is what kept the pane.+ expect(tail.indexOf('clearRemainOnExit:%0')).toBeGreaterThan(tail.indexOf('respawn:%0:claude'))+ expect(h.rows.get('agent-a')?.launch).toEqual({ state: 'starting' })+ })++ it('fails a fresh engine that exits inside the settling window', async () => {+ const h = harness([row({ sessionId: '', boundAt: null })], { settleMs: 50 })+ h.probes.set('%0', [identity(7)])+ h.states.set('%0', [{ dead: true }])+ await restoreAgents(h.deps)+ await settled(h, 1)+ expect(h.respawns).toBe(0)+ expect(h.rows.get('agent-a')?.launch).toMatchObject({ state: 'failed', error: 'ENGINE_DID_NOT_START' })+ })++ it('gives up after the fresh launch dies too', async () => {+ const h = harness([row()])+ h.states.set('%0', [{ dead: true }, { dead: true }])+ await restoreAgents(h.deps)+ await settled(h, 1)+ expect(h.respawns).toBe(1)+ expect(h.rows.get('agent-a')?.launch).toMatchObject({ state: 'failed', error: 'ENGINE_DID_NOT_START' })+ })++ it('does not retry fresh when there was no resume to blame', async () => {+ const h = harness([row({ sessionId: '', boundAt: null })])+ h.states.set('%0', [{ dead: true }])+ await restoreAgents(h.deps)+ await settled(h, 1)+ expect(h.respawns).toBe(0)+ expect(h.rows.get('agent-a')?.launch).toMatchObject({ state: 'failed', error: 'ENGINE_DID_NOT_START' })+ })++ it('fails the launch when the pane vanishes entirely', async () => {+ const h = harness([row()])+ h.states.set('%0', [null])+ await restoreAgents(h.deps)+ await settled(h, 1)+ expect(h.rows.get('agent-a')?.launch).toMatchObject({ state: 'failed', error: 'ENGINE_DID_NOT_START' })+ })++ it('times out with the launch marked, never leaving the route held', async () => {+ const h = harness([row()], { budgetMs: 0 })+ await restoreAgents(h.deps)+ await settled(h, 1)+ expect(h.rows.get('agent-a')?.launch).toMatchObject({ state: 'failed', error: 'START_TIMEOUT' })+ expect(h.released).toEqual(['tmux\u0000%0'])+ })++ it('stops quietly when the agent was deleted while its engine was starting', async () => {+ const h = harness([row()])+ h.probes.set('%0', [null])+ const deps = { ...h.deps }+ let polls = 0+ deps.probeProcess = async () => { if (++polls === 2) h.rows.delete('agent-a'); return null }+ await restoreAgents(deps)+ await settled(h, 1)+ expect(h.calls.filter((c) => c.startsWith('setLaunch:agent-a:failed'))).toEqual([])+ })+})diff --git a/cli/src/lib/restoreAgents.ts b/cli/src/lib/restoreAgents.tsnew file mode 100644index 000000000..a64f10aa6--- /dev/null+++ b/cli/src/lib/restoreAgents.ts@@ -0,0 +1,247 @@+/**+ * Bring registered agents back after their tmux panes died with the daemon down — a reboot, a+ * `tmux kill-server`, a pane closed by hand.+ *+ * The registry outlives the tmux server: it still knows each agent's id, engine, cwd, session and+ * launch shape. This recreates a pane for every agent whose pane is gone, launches the same engine+ * there — resuming its engine session when it has one — and keeps the SAME agentId, so a desktop+ * tile that remembered the agent attaches to it again without anyone recreating anything.+ *+ * Dependency-injected like restartAgent.ts: cli.ts owns the tmux, registry and reconciler wiring,+ * and this owns the ordering, which is the part worth testing without a daemon:+ *+ * 1. Every missing pane is recreated inside ONE registry transaction. A new tmux server hands out+ * pane ids from `%0` again, so a restored pane's id can equal another stale row's dead pane —+ * and `registry.save()` evicts whichever row loses that collision. Deferring the save until+ * every row holds its new pane is what keeps the second row alive.+ * 2. The new route is HELD in the reconciler until the engine process is bound here. Otherwise the+ * reconciler's discovery half sees an unclaimed engine process and mints a fresh agentId for it.+ * 3. The row keeps `launch: starting` and only gets its process identity from here. The reconcile+ * pass after the release then matches the process, sees a launch in progress, and does the+ * full ready → attach → announce sequence it already does for `agent_create`.+ */++import type { AgentEngine } from '../engines/types.js'+import type { AgentLaunch, ProcessIdentity, RegisteredSession } from './registry.js'+import type { TerminalRuntimeRef, TmuxRuntimeRef } from './terminalTypes.js'+import { terminalRouteKey } from './terminalRuntime.js'++export interface RestoreLaunch {+ argv: string[]+ env?: Record<string, string>+}++export interface RestoreAgentsDeps {+ registry: {+ list(): RegisteredSession[]+ byAgent(agentId: string): RegisteredSession | undefined+ transaction<T>(apply: () => T | Promise<T>): Promise<T>+ clearProcessIdentity(agentId: string): boolean+ updateRuntimes(agentId: string, runtimes: readonly TerminalRuntimeRef[], primaryRuntimeKey?: string): boolean+ setLaunch(agentId: string, launch: AgentLaunch): RegisteredSession | null+ updateProcessIdentity(agentId: string, processIdentity: ProcessIdentity): boolean+ unbindSession(sessionId: string): boolean+ inheritName(fromSessionId: string, toSessionId: string): void+ }+ /** The engine process still running in this row's pane, or null when tmux does not know the pane+ * at all — including when no tmux server is running — or the pane has become something else. */+ liveProcess: (entry: RegisteredSession, runtime: TmuxRuntimeRef) => Promise<ProcessIdentity | null>+ buildLaunch: (entry: RegisteredSession, opts: { resumeSessionId?: string }) => RestoreLaunch+ createPane: (entry: RegisteredSession, launch: RestoreLaunch) => Promise<+ | { ok: true; runtime: TmuxRuntimeRef }+ | { ok: false; reason: string }+ >+ /** `tmux respawn-pane` over the restored pane — the resume → fresh fallback. */+ respawn: (runtime: TmuxRuntimeRef, launch: RestoreLaunch) => Promise<{ ok: boolean; reason?: string }>+ /** One probe of the pane for a recognizable engine process. */+ probeProcess: (runtime: TmuxRuntimeRef, engine: AgentEngine) => Promise<ProcessIdentity | null>+ /** Null when tmux no longer knows the pane. */+ paneState: (runtime: TmuxRuntimeRef) => Promise<{ dead: boolean } | null>+ clearRemainOnExit: (runtime: TmuxRuntimeRef) => Promise<void>+ holdRoute: (routeKey: string, autoReleaseMs: number) => void+ releaseRoute: (routeKey: string) => void+ triggerHint: (runtime: TmuxRuntimeRef, engine: AgentEngine) => Promise<void>+ log: (message: string) => void+ /** How long a restored pane may take to show an engine process. Default matches `agent_create`. */+ budgetMs?: number+ /** How long the engine must stay up after appearing before the pane is handed over. */+ settleMs?: number+ sleep?: (ms: number) => Promise<void>+}++export interface RestoreSummary {+ /** Agents whose pane was recreated; their engine process is bound in the background. */+ restored: string[]+ skipped: Array<{ agentId: string; reason: string }>+ failed: Array<{ agentId: string; reason: string }>+}++const DEFAULT_BUDGET_MS = 10 * 60_000+const DEFAULT_SETTLE_MS = 10_000+const SETTLE_POLL_MS = 500+const HOLD_SLACK_MS = 30_000++function tmuxRuntime(entry: RegisteredSession): TmuxRuntimeRef | null {+ const runtime = entry.runtimes.find((candidate): candidate is TmuxRuntimeRef => candidate.backend === 'tmux')+ return runtime ?? null+}++function defaultSleep(ms: number): Promise<void> {+ return new Promise((resolve) => {+ const timer = setTimeout(resolve, ms)+ timer.unref?.()+ })+}++/** Watch a pane whose engine just appeared: 'settled' once it has stayed up for `settleMs`, 'gone' the+ * moment tmux reports it dead or forgotten. */+async function waitForSettle(deps: RestoreAgentsDeps, runtime: TmuxRuntimeRef, settleMs: number): Promise<'settled' | 'gone'> {+ const sleep = deps.sleep ?? defaultSleep+ const until = Date.now() + settleMs+ while (Date.now() < until) {+ await sleep(Math.min(SETTLE_POLL_MS, until - Date.now()))+ const state = await deps.paneState(runtime)+ if (!state || state.dead) return 'gone'+ }+ return 'settled'+}++/**+ * Recreate every missing pane, then return. Binding each pane's engine process continues in the+ * background; the summary says which agents that is happening for.+ */+export async function restoreAgents(deps: RestoreAgentsDeps): Promise<RestoreSummary> {+ const summary: RestoreSummary = { restored: [], skipped: [], failed: [] }+ const missing: Array<{ entry: RegisteredSession; runtime: TmuxRuntimeRef }> = []++ for (const entry of deps.registry.list()) {+ const runtime = tmuxRuntime(entry)+ if (!runtime) { summary.skipped.push({ agentId: entry.agentId, reason: 'no tmux pane' }); continue }+ if (entry.launch?.state === 'failed') { summary.skipped.push({ agentId: entry.agentId, reason: 'last launch failed' }); continue }+ const live = await deps.liveProcess(entry, runtime)+ if (live) {+ // Still running. A row that lost its identity without losing its pane (a reboot the boot+ // clock misread; a tmux server that outlived the daemon) is re-identified right here, so the+ // reconciler adopts it by process instead of treating it as an unbound route.+ if (!entry.processIdentity) deps.registry.updateProcessIdentity(entry.agentId, live)+ continue+ }+ // The registry keeps only where a grid agent pointed, never the credential the launch needs —+ // and launching it on the engine's own login instead would spend the wrong account while+ // looking identical. Refusing is the rule gridLaunch.ts already sets.+ if (entry.grid) { summary.skipped.push({ agentId: entry.agentId, reason: 'grid agent; credential not persisted' }); continue }+ missing.push({ entry, runtime })+ }+ if (!missing.length) return summary++ const budgetMs = deps.budgetMs ?? DEFAULT_BUDGET_MS+ const watches: Array<() => Promise<void>> = []+ await deps.registry.transaction(async () => {+ for (const { entry, runtime: dead } of missing) {+ // No process survives its pane; clear it now or discovery would refuse to adopt the new one+ // into this agent (route adoption requires either no identity or a matching pid).+ deps.registry.clearProcessIdentity(entry.agentId)+ const resumeSessionId = entry.sessionId || undefined+ const launch = deps.buildLaunch(entry, resumeSessionId ? { resumeSessionId } : {})+ const created = await deps.createPane(entry, launch)+ if (!created.ok) {+ summary.failed.push({ agentId: entry.agentId, reason: created.reason })+ deps.log(`[restore] ${entry.engine} · agent ${entry.agentId} · could not open a pane · ${created.reason}`)+ continue+ }+ const key = terminalRouteKey(created.runtime)+ deps.holdRoute(key, budgetMs + HOLD_SLACK_MS)+ deps.registry.updateRuntimes(entry.agentId, [created.runtime], key)+ deps.registry.setLaunch(entry.agentId, { state: 'starting' })+ summary.restored.push(entry.agentId)+ deps.log(`[restore] ${entry.engine} · agent ${entry.agentId} · pane ${dead.paneId} → ${created.runtime.paneId}`+ + (resumeSessionId ? ` · resuming ${resumeSessionId.slice(0, 8)}` : ' · fresh session'))+ watches.push(() => watchRestoredPane(deps, entry, created.runtime, !!resumeSessionId, budgetMs))+ }+ })+ for (const watch of watches) void watch()+ return summary+}++async function watchRestoredPane(+ deps: RestoreAgentsDeps,+ entry: RegisteredSession,+ runtime: TmuxRuntimeRef,+ resuming: boolean,+ budgetMs: number,+): Promise<void> {+ const { agentId, engine } = entry+ const key = terminalRouteKey(runtime)+ const sleep = deps.sleep ?? defaultSleep+ const fail = (error: string, detail: string): void => {+ deps.registry.setLaunch(agentId, { state: 'failed', error, detail })+ deps.log(`[restore] ${engine} · agent ${agentId} · failed · ${detail}`)+ }+ const settleMs = deps.settleMs ?? DEFAULT_SETTLE_MS+ let mayRetryFresh = resuming+ /** The pane's engine is gone. True when a fresh relaunch is now under way, false when this is the end. */+ const relaunchFresh = async (): Promise<boolean> => {+ if (!mayRetryFresh) {+ fail('ENGINE_DID_NOT_START', `${engine} exited before its engine process became ready. See the terminal output for details.`)+ return false+ }+ // A resume id the engine no longer honours is not worth a dead agent: the row's name comes+ // along to the agent itself, the stale binding goes, and the engine gets one fresh start.+ mayRetryFresh = false+ deps.log(`[restore] ${engine} · agent ${agentId} · did not come back up resuming its session — retrying fresh`)+ deps.registry.inheritName(entry.sessionId, agentId)+ deps.registry.unbindSession(entry.sessionId)+ const spawned = await deps.respawn(runtime, deps.buildLaunch(entry, {}))+ if (!spawned.ok) {+ fail('ENGINE_DID_NOT_START', `${engine} could not be relaunched fresh: ${spawned.reason ?? 'unknown reason'}`)+ return false+ }+ // The fresh start gets the full budget again; the hold's auto-release must outlast it.+ deps.holdRoute(key, budgetMs + HOLD_SLACK_MS)+ return true+ }+ try {+ const startedAt = Date.now()+ let delayMs = 50+ while (Date.now() - startedAt < budgetMs) {+ if (!deps.registry.byAgent(agentId)) return+ const identity = await deps.probeProcess(runtime, engine)+ if (identity) {+ deps.registry.updateProcessIdentity(agentId, identity)+ // A resume the engine rejects does not fail to start — it starts, prints why, and exits a+ // few seconds later. Keep the pane (and the fallback) for a settling window before handing+ // it over; a pane let go at first sight of the process vanished with the engine and took+ // the agent with it (measured: claude resuming a conversation another client still held).+ const settled = await waitForSettle(deps, runtime, settleMs)+ if (settled === 'gone') {+ if (!await relaunchFresh()) return+ delayMs = 50+ continue+ }+ await deps.clearRemainOnExit(runtime)+ // Release BEFORE the hint: the pass it triggers is the one that must see this route again.+ deps.releaseRoute(key)+ await deps.triggerHint(runtime, engine)+ deps.log(`[restore] ${engine} · agent ${agentId} · engine up · ${Date.now() - startedAt}ms`)+ return+ }+ const state = await deps.paneState(runtime)+ if (!state) {+ fail('ENGINE_DID_NOT_START', `${engine}'s restored pane disappeared before its engine process became ready.`)+ return+ }+ if (state.dead) {+ if (!await relaunchFresh()) return+ delayMs = 50+ continue+ }+ await sleep(delayMs)+ delayMs = Math.min(delayMs * 2, 750)+ }+ fail('START_TIMEOUT', `${engine} did not expose an engine process within ${Math.round(budgetMs / 60_000)} minutes. The terminal remains available.`)+ } catch (error) {+ deps.log(`[restore] ${engine} · agent ${agentId} · watch failed · ${error instanceof Error ? error.message : error}`)+ } finally {+ deps.releaseRoute(key)+ }+}diff --git a/cli/src/lib/terminalBinary.spec.ts b/cli/src/lib/terminalBinary.spec.tsindex c229deee2..8ba8f0b7a 100644--- a/cli/src/lib/terminalBinary.spec.ts+++ b/cli/src/lib/terminalBinary.spec.ts@@ -265,13 +265,12 @@ describe('authenticated loopback terminal framing v1', () => {compressed: false,})expect(encoded).not.toBeNull()- expect(decodeTerminalLocal(encoded!)).toEqual({- kind: TerminalBinaryKind.paste,- streamId,- seq: 0,- bytes,- compressed: false,- })+ // Metadata via deep-equal, payload via native memcmp: vitest's toEqual walks a typed array one+ // element at a time, and on a multi-MiB payload that alone took ~1-3s on a fast laptop and blew+ // through the 5s test timeout on a CI runner (v0.2.16_cli release run).+ const decoded = decodeTerminalLocal(encoded!)!+ expect(decoded).toMatchObject({ kind: TerminalBinaryKind.paste, streamId, seq: 0, compressed: false })+ expect(Buffer.compare(decoded.bytes, bytes)).toBe(0)expect(encodeTerminalLocal({kind: TerminalBinaryKind.output,@@ -295,13 +294,9 @@ describe('authenticated loopback terminal framing v1', () => {compressed: false,})expect(encoded).not.toBeNull()- expect(decodeTerminalLocal(encoded!)).toEqual({- kind: TerminalBinaryKind.imagePaste,- streamId,- seq: 0,- bytes,- compressed: false,- })+ const decoded = decodeTerminalLocal(encoded!)!+ expect(decoded).toMatchObject({ kind: TerminalBinaryKind.imagePaste, streamId, seq: 0, compressed: false })+ expect(Buffer.compare(decoded.bytes, bytes)).toBe(0) // memcmp, not element-wise — see the paste test aboveconst tooBig = new Uint8Array(TERMINAL_LOCAL_IMAGE_PASTE_MAX_PAYLOAD_BYTES + 1)expect(encodeTerminalLocal({@@ -326,10 +321,9 @@ describe('authenticated loopback terminal framing v1', () => {compressed: false,})expect(encoded).not.toBeNull()- const decoded = decodeTerminalLocal(encoded!)- expect(decoded?.kind).toBe(TerminalBinaryKind.pasteFile)- expect(decoded?.seq).toBe(2)- expect(decoded?.bytes).toEqual(content)+ const decoded = decodeTerminalLocal(encoded!)!+ expect(decoded).toMatchObject({ kind: TerminalBinaryKind.pasteFile, seq: 2 })+ expect(Buffer.compare(decoded.bytes, content)).toBe(0) // memcmp, not element-wise — see the paste test aboveconst tooBig = new Uint8Array(TERMINAL_LOCAL_PASTE_FILE_MAX_PAYLOAD_BYTES + 1)expect(encodeTerminalLocal({