Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: introduce agent restoration logic after system reboot by persisting bypass permissions and clearing dormant process identities

0xken152c3d2a6ff17f
diff --git a/cli/README.md b/cli/README.md
index 90162d28f..d8bfd4bb8 100644
--- a/cli/README.md
+++ b/cli/README.md
@@ -139,6 +139,11 @@ claude under a terminal backend ──writes──▶ ~/.claude/projects/**.json
named `harness-<engine>-<timestamp>`, which is what `agent_create` names them. A tmux session you
open by hand and run an engine in, or one an agent spawns itself with a nested `tmux new-session`,
never becomes an agent — create it through the app (or `agent_create`) instead.
+- **Agents survive a reboot.** The registry outlives the tmux server. On start, every registered agent
+ whose pane is gone (a reboot, a `tmux kill-server`) gets a new pane running the same engine in the
+ same folder, resuming its engine session when it has one, under the same agent id — so the app's
+ tiles reattach by themselves. Agents that were pointed at a grid are not restored (the launch
+ credential is never persisted), and an engine with no resume flag (devin) comes back fresh.
- **Hooks bind mutable engine sessions** to the process agent using authenticated tmux and/or Herdr
runtime hints plus verified caller ancestry. Hook socket paths are lookup hints only. They do not
require `MACHINE_ID`. `SessionStart` and catch hooks attach transcript/store metadata; `SessionEnd` only
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index 023ab191a..434f60aa7 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -65,6 +65,7 @@ import { restartAgent, type RestartAgentDeps } from './lib/restartAgent.js'
import { claudeContinuation, findLiveSession } from './lib/sessionRepair.js'
import { TmuxBackend } from './lib/tmuxBackend.js'
import { createAndRegisterPane } from './lib/createAgentPane.js'
+import { restoreAgents } from './lib/restoreAgents.js'
import { buildHarnessSessionLabel } from './lib/harnessSessionLabel.js'
import { basename } from 'node:path'
import {
@@ -2393,6 +2394,10 @@ async function runForeground(session: AuthSession): Promise<void> {
const wasLaunching = current.launch?.state !== undefined && current.launch.state !== 'ready'
registry.updateRuntimes(current.agentId, observed.runtimes, observed.primaryRuntimeKey)
registry.updateProcessIdentity(current.agentId, observed.processIdentity, observed.gateway, observed.grid)
+ // The live argv is the truth about the bypass flag, and this is the one place every running
+ // agent passes through — so a row written before the flag was persisted at all (or by a build
+ // that did not yet) learns it here, before any pane recreation ever needs it.
+ registry.setBypassPermission(current.agentId, bypassPermissionActive(current.engine, observed.args))
if (wasLaunching) registry.setLaunch(current.agentId, { state: 'ready' })
await bindObservedAgent(observed)
if (wasDormant || wasLaunching) {
@@ -3196,6 +3201,64 @@ async function runForeground(session: AuthSession): Promise<void> {
}
watcher.start()
await cursorDiscovery.start()
+ // Panes that died while the daemon was down (a reboot takes the whole tmux server with it) are
+ // rebuilt BEFORE the first reconcile pass: it would otherwise count them absent, and a second
+ // pass five seconds later would drop the agents for good. Pane creation is awaited so the first
+ // announce already shows every restored agent with a terminal; binding their engine processes
+ // continues in the background, the same way `agent_create` does it.
+ if (tmuxBackend) {
+ const backend = tmuxBackend
+ const summary = await restoreAgents({
+ registry,
+ // "Alive" means the pane still runs THIS row's engine — not merely that tmux knows the id.
+ // A new tmux server hands out `%N` from zero again, so a stale id can name someone's shell;
+ // and a pane that outlived the daemon in a session discovery no longer lists still has its
+ // engine, which a second pane resuming the same session would collide with.
+ liveProcess: (entry, runtime) => resolvePaneEngineProcess(runtime.paneId, entry.engine),
+ buildLaunch: (entry, opts) => {
+ // Mirrors `agent_create`: a Codex profile other than the default needs its hooks and its
+ // CODEX_HOME; the install check runs inside the pane's own shell.
+ if (entry.codexHome && !env.DISABLE_HOOK_INSTALL) installCodexHooks(hookPort, entry.codexHome)
+ const argv = buildEngineLaunchArgv(entry.engine, {
+ ...opts,
+ bypassPermission: entry.bypassPermission === true,
+ installIfMissing: enginePathOverride(entry.engine) ? undefined : engineInstallRecipe(entry.engine),
+ ...(entry.cwd ? { cwd: entry.cwd } : {}),
+ })
+ return { argv, ...(entry.codexHome ? { env: { CODEX_HOME: entry.codexHome } } : {}) }
+ },
+ createPane: async (entry, launch) => {
+ const created = await backend.create({
+ cwd: homedir(),
+ label: buildHarnessSessionLabel(entry.engine),
+ command: launch.argv,
+ ...(launch.env ? { env: launch.env } : {}),
+ })
+ return created.state === 'succeeded'
+ ? { ok: true, runtime: created.runtime }
+ : { ok: false, reason: created.reason }
+ },
+ respawn: async (runtime, launch) => {
+ const result = await backend.respawn(runtime, {
+ command: launch.argv,
+ cwd: homedir(),
+ ...(launch.env ? { env: launch.env } : {}),
+ })
+ return result.state === 'succeeded' ? { ok: true } : { ok: false, reason: result.reason }
+ },
+ probeProcess: (runtime, engine) => resolvePaneEngineProcess(runtime.paneId, engine),
+ paneState: (runtime) => tmuxPaneState(runtime.paneId),
+ clearRemainOnExit: (runtime) => clearPaneRemainOnExit(runtime.paneId),
+ holdRoute: (key, ms) => agentReconciler.holdRoute(key, ms),
+ releaseRoute: (key) => agentReconciler.releaseRoute(key),
+ triggerHint: (runtime, engine) => agentReconciler.triggerHint(runtime, engine),
+ log: (message) => console.log(message),
+ })
+ if (summary.restored.length || summary.failed.length || registry.rebootedSinceLastRun) {
+ console.log(`[restore] restored ${summary.restored.length} · skipped ${summary.skipped.length} · failed ${summary.failed.length}`
+ + (registry.rebootedSinceLastRun ? ' · after reboot' : ''))
+ }
+ }
await agentReconciler.start(env.TERMINAL_RECONCILE_INTERVAL_MS ?? env.TMUX_REAP_INTERVAL_MS)
for (const task of await loadCursorPendingTasks(env.ADAPTER_DATA_DIR)) {
onCursorTaskStart(task.sessionId, task.toolUseId, task.input)
@@ -3435,6 +3498,7 @@ async function runForeground(session: AuthSession): Promise<void> {
env: gridLaunch?.env ?? (codexHome ? { CODEX_HOME: codexHome } : undefined),
grid: grid ? { baseUrl: grid.baseUrl, model: grid.model ?? null } : null,
codexHome,
+ bypassPermission,
})
if (!result.ok) return { ok: false, error: result.error, detail: result.detail }
const { spawned, pending } = result
diff --git a/cli/src/lib/createAgentPane.ts b/cli/src/lib/createAgentPane.ts
index 8436cc061..631cb5c4d 100644
--- a/cli/src/lib/createAgentPane.ts
+++ b/cli/src/lib/createAgentPane.ts
@@ -29,9 +29,11 @@ export interface CreateAgentPaneDeps {
cwd?: string | null
grid?: { baseUrl: string; model: string | null } | null
codexHome?: string | null
+ bypassPermission?: boolean
}) => RegisteredSession | null }
engine: AgentEngine
cwd?: string | null
+ bypassPermission?: boolean
/** Base tmux session name (`-s`). Retries append `-r<attempt>` — see module doc. */
sessionLabel: string
argv: string[]
@@ -71,6 +73,7 @@ export async function createAndRegisterPane(deps: CreateAgentPaneDeps): Promise<
cwd: deps.cwd,
grid: deps.grid,
codexHome: deps.codexHome,
+ bypassPermission: deps.bypassPermission,
})
if (pending) return { ok: true, spawned, pending }
console.warn(`[agent] create ${deps.engine} registration failed · pane ${spawned.runtime.paneId} · `
diff --git a/cli/src/lib/registry.spec.ts b/cli/src/lib/registry.spec.ts
index 596a8a124..921dcd8a2 100644
--- a/cli/src/lib/registry.spec.ts
+++ b/cli/src/lib/registry.spec.ts
@@ -1131,3 +1131,178 @@ describe('agent identity: the process owns the agent, the session is bound to it
expect(saved[0]).not.toHaveProperty('launcherId')
})
})
+
+describe('registry across a reboot and pane loss', () => {
+ beforeEach(() => {
+ vi.useFakeTimers()
+ dataDir = mkdtempSync(join(tmpdir(), 'adapter-registry-reboot-'))
+ })
+
+ afterEach(() => {
+ vi.useRealTimers()
+ rmSync(dataDir, { recursive: true, force: true })
+ delete process.env.ADAPTER_DATA_DIR
+ delete process.env.CLAUDE_PROJECTS_DIR
+ delete process.env.CODEX_HOME
+ delete process.env.CURSOR_HOME
+ })
+
+ /** A row as `agent_create` + a SessionStart hook leave it: bound, with a live-looking process. */
+ function persistedRow(transcriptPath: string, overrides: Record<string, unknown> = {}) {
+ return {
+ schemaVersion: 2,
+ active: true,
+ launch: { state: 'ready' },
+ agentId: 'agent-a',
+ sessionId: 'session-a',
+ boundAt: 1,
+ engine: 'claude',
+ gateway: null,
+ grid: null,
+ codexHome: null,
+ bypassPermission: true,
+ transcriptPath,
+ projectDir: 'demo',
+ cwd: '/tmp/demo',
+ runtimes: [{ backend: 'tmux', paneId: '%3' }],
+ primaryRuntimeKey: 'tmux\u0000%3',
+ tmuxPane: '%3',
+ source: null,
+ title: null,
+ model: null,
+ cliVersion: null,
+ processIdentity: processIdentity(4242),
+ registeredAt: 1,
+ updatedAt: 1,
+ lastHookAt: 1,
+ lastTranscriptAt: 1,
+ ...overrides,
+ }
+ }
+
+ it('keeps every agent after a reboot, clearing only what a reboot actually kills', async () => {
+ const transcriptPath = join(dataDir, 'session-a.jsonl')
+ writeFileSync(transcriptPath, '{}\n')
+ chmodSync(dataDir, 0o755)
+ writeLegacyStateFile(join(dataDir, 'registry.json'), JSON.stringify([persistedRow(transcriptPath, { codexHome: '/tmp/codex-work' })]))
+ // A boot marker from the distant past: whichever way the platform identifies a boot, this is not it.
+ writeLegacyStateFile(join(dataDir, 'registry-boot'), 'time:1')
+
+ const { registry } = await loadRegistryModule()
+ registry.load()
+
+ expect(registry.rebootedSinceLastRun).toBe(true)
+ expect(registry.byAgent('agent-a')).toMatchObject({
+ agentId: 'agent-a',
+ sessionId: 'session-a',
+ engine: 'claude',
+ cwd: '/tmp/demo',
+ codexHome: '/tmp/codex-work',
+ bypassPermission: true,
+ runtimes: [{ backend: 'tmux', paneId: '%3' }],
+ active: false,
+ processIdentity: null,
+ })
+ // The cleared snapshot is on disk immediately: the boot marker was already refreshed, so a crash
+ // before this save would otherwise leave the dead pid behind with no second reboot to notice it.
+ const saved = JSON.parse(readFileSync(join(dataDir, 'registry.json'), 'utf-8')) as Array<Record<string, unknown>>
+ expect(saved[0]).toMatchObject({ agentId: 'agent-a', processIdentity: null, codexHome: '/tmp/codex-work', bypassPermission: true })
+
+ // The next start on the SAME boot is not a reboot, and finds the row as the previous save left it.
+ const again = await loadRegistryModule()
+ again.registry.load()
+ expect(again.registry.rebootedSinceLastRun).toBe(false)
+ expect(again.registry.byAgent('agent-a')).toMatchObject({ processIdentity: null, bypassPermission: true })
+ })
+
+ it('leaves the process identity alone when the machine did not reboot', async () => {
+ const transcriptPath = join(dataDir, 'session-a.jsonl')
+ writeFileSync(transcriptPath, '{}\n')
+ chmodSync(dataDir, 0o755)
+ writeLegacyStateFile(join(dataDir, 'registry.json'), JSON.stringify([persistedRow(transcriptPath)]))
+
+ const { registry } = await loadRegistryModule()
+ registry.load()
+
+ expect(registry.rebootedSinceLastRun).toBe(false)
+ expect(registry.byAgent('agent-a')?.processIdentity).toEqual(processIdentity(4242))
+ expect(registry.byProcess('claude', processIdentity(4242))?.agentId).toBe('agent-a')
+ })
+
+ it('carries bypassPermission from agent_create through the first hook bind', async () => {
+ const transcriptPath = join(dataDir, 'session-b.jsonl')
+ writeFileSync(transcriptPath, '{}\n')
+ const { registry } = await loadRegistryModule()
+ registry.load()
+
+ const pending = registry.openPendingAgent({
+ engine: 'claude',
+ runtimes: [{ backend: 'tmux', paneId: '%9' }],
+ cwd: '/tmp/demo',
+ bypassPermission: true,
+ })
+ expect(pending?.bypassPermission).toBe(true)
+
+ const bound = registry.register({ sessionId: 'session-b', transcriptPath, tmuxPane: '%9', cwd: '/tmp/demo' })
+ expect(bound?.entry.agentId).toBe(pending!.agentId)
+ expect(bound?.entry.bypassPermission).toBe(true)
+
+ expect(registry.setBypassPermission(pending!.agentId, false)).toBe(true)
+ expect(registry.byAgent(pending!.agentId)).not.toHaveProperty('bypassPermission')
+ const saved = JSON.parse(readFileSync(join(dataDir, 'registry.json'), 'utf-8')) as Array<Record<string, unknown>>
+ expect(saved[0]).not.toHaveProperty('bypassPermission')
+ })
+
+ it('clearProcessIdentity forgets the pid everywhere it is indexed', async () => {
+ const { registry } = await loadRegistryModule()
+ registry.load()
+ const opened = registry.openProcessAgent({ engine: 'claude', tmuxPane: '%5', processIdentity: processIdentity(77) })
+ const agentId = opened!.entry.agentId
+ expect(registry.byProcess('claude', processIdentity(77))?.agentId).toBe(agentId)
+
+ expect(registry.clearProcessIdentity(agentId)).toBe(true)
+ expect(registry.byAgent(agentId)?.processIdentity).toBeNull()
+ expect(registry.byProcess('claude', processIdentity(77))).toBeUndefined()
+ expect(registry.clearProcessIdentity(agentId)).toBe(true) // idempotent
+ expect(registry.clearProcessIdentity('nobody')).toBe(false)
+ })
+
+ it('survives two restored panes swapping ids when done inside one transaction', async () => {
+ const { registry } = await loadRegistryModule()
+ registry.load()
+ const a = registry.openProcessAgent({ engine: 'claude', tmuxPane: '%1', processIdentity: processIdentity(11) })!.entry.agentId
+ const b = registry.openProcessAgent({ engine: 'claude', tmuxPane: '%2', processIdentity: processIdentity(12) })!.entry.agentId
+
+ // A new tmux server hands out %1/%2 again, in the other order. Saved between the two updates, A
+ // would claim %2 while B still lists it, and the save's route dedupe evicts one of them.
+ await registry.transaction(() => {
+ registry.updateRuntimes(a, [{ backend: 'tmux', paneId: '%2' }], 'tmux\u0000%2')
+ registry.updateRuntimes(b, [{ backend: 'tmux', paneId: '%1' }], 'tmux\u0000%1')
+ })
+
+ expect(registry.byAgent(a)?.tmuxPane).toBe('%2')
+ expect(registry.byAgent(b)?.tmuxPane).toBe('%1')
+ const saved = JSON.parse(readFileSync(join(dataDir, 'registry.json'), 'utf-8')) as Array<Record<string, unknown>>
+ expect(saved.map((row) => row.agentId).sort()).toEqual([a, b].sort())
+ })
+
+ it('adopts a new process into an agent whose identity was cleared, keeping its id and session', async () => {
+ const transcriptPath = join(dataDir, 'session-c.jsonl')
+ writeFileSync(transcriptPath, '{}\n')
+ const { registry } = await loadRegistryModule()
+ registry.load()
+ const bound = registerProcess(registry, { sessionId: 'session-c', transcriptPath, tmuxPane: '%4', cwd: '/tmp/demo' })
+ const agentId = bound!.entry.agentId
+
+ // What restore does: forget the dead pid, point the row at the recreated pane, launch pending.
+ registry.clearProcessIdentity(agentId)
+ registry.updateRuntimes(agentId, [{ backend: 'tmux', paneId: '%0' }], 'tmux\u0000%0')
+ registry.setLaunch(agentId, { state: 'starting' })
+
+ const adopted = registry.openProcessAgent({ engine: 'claude', tmuxPane: '%0', processIdentity: processIdentity(500) })
+ expect(adopted).toMatchObject({ isNew: false, evicted: null })
+ expect(adopted?.entry.agentId).toBe(agentId)
+ expect(adopted?.entry.sessionId).toBe('session-c')
+ expect(registry.list()).toHaveLength(1)
+ })
+})
diff --git a/cli/src/lib/registry.ts b/cli/src/lib/registry.ts
index 109ded03c..3d98b1241 100644
--- a/cli/src/lib/registry.ts
+++ b/cli/src/lib/registry.ts
@@ -106,6 +106,13 @@ export interface RegisteredSession {
* to a different grid.
*/
codexHome?: string | null
+ /**
+ * Whether the engine was launched with its permission prompts bypassed (`--dangerously-skip-permissions`
+ * and friends, `BYPASS_PERMISSION_FLAGS`). Recorded at launch because it is otherwise only readable
+ * off a LIVE process's argv — and a pane that has to be recreated after a reboot has no live process
+ * to read it from. Like `codexHome`, chosen at launch and carried forward, never re-derived.
+ */
+ bypassPermission?: boolean
/** Legacy launcher-owned snapshots may still contain this field. New records never write it. */
launcherId?: string
transcriptPath: string | null
@@ -555,6 +562,12 @@ class Registry {
private writeBlocked = false
/** Last committed row bytes, used for a three-way merge with daemon-down hook writes. */
private persistedBaseline = new Map<string, string>()
+ private rebooted = false
+
+ /** True when the last `load()` found the machine had rebooted since the previous daemon run. */
+ get rebootedSinceLastRun(): boolean {
+ return this.rebooted
+ }
private index(entry: RegisteredSession): void {
this.agents.set(entry.agentId, entry)
@@ -590,8 +603,10 @@ class Registry {
entry.lastTranscriptAt = Date.now()
}
- /** Load persisted process agents. Invalid session bindings are released without dropping their agent;
- * a reboot still drops all cached pane identities because no process can survive it. */
+ /** Load persisted process agents. Invalid session bindings are released without dropping their agent.
+ * A reboot keeps every row but clears its process identity and marks it dormant: no process survives
+ * a reboot, but the agent — its id, cwd, session and pane placement — is what `restoreAgents` rebuilds
+ * a pane for. */
load(): void {
this.agents.clear()
this.sessionIndex.clear()
@@ -599,6 +614,7 @@ class Registry {
this.processIndex.clear()
this.terminalAvailableAgents.clear()
this.writeBlocked = false
+ this.rebooted = false
this.persistedBaseline.clear()
try {
secureStateDirectory(env.ADAPTER_DATA_DIR)
@@ -645,12 +661,14 @@ class Registry {
if (arr.some((row) => row.schemaVersion !== 2)) {
atomicWriteJson(PRE_V2_BACKUP_FILE, arr, true)
}
+ let changed = false
if (rebooted) {
- console.log(`[registry] machine rebooted since last run — dropping ${Array.isArray(arr) ? arr.length : 0} cached tmux session(s) (stale panes)`)
- this.save() // overwrite the stale on-disk snapshot so a same-boot restart can't reload it
- return
+ this.rebooted = true
+ console.log(`[registry] machine rebooted since last run — ${arr.length} agent(s) kept with their process identity cleared (stale panes, restored on start)`)
+ // The boot marker was already refreshed above, so a crash before the save below would leave
+ // pre-reboot pids on disk with no second chance to notice — force the cleared snapshot out.
+ changed = true
}
- let changed = false
for (const raw of Array.isArray(arr) ? arr : []) {
const engine = normalizedAgentEngine(raw?.engine)
const runtimes = normalizedRuntimes(raw?.runtimes, raw?.tmuxPane)
@@ -705,7 +723,7 @@ class Registry {
if (!agentId) { changed = true; continue }
if (agentId !== raw.agentId) changed = true
const now = Date.now()
- const active = raw.active !== false
+ const active = !rebooted && raw.active !== false
const launch = normalizedLaunch(raw.launch)
const s: RegisteredSession = {
schemaVersion: 2,
@@ -714,6 +732,10 @@ class Registry {
agentId,
boundAt: bound ? (typeof raw.boundAt === 'number' ? raw.boundAt : (raw.registeredAt ?? now)) : null,
engine,
+ gateway: raw.gateway === 'ori' ? 'ori' : null,
+ grid: normalizedGridAssignment(raw.grid),
+ codexHome: typeof rawCodexHome === 'string' && rawCodexHome ? rawCodexHome : null,
+ ...(raw.bypassPermission === true ? { bypassPermission: true } : {}),
transcriptPath,
title: titleDisplayName(typeof raw.title === 'string' ? raw.title : null),
sessionId: bound ? rawSessionId : '',
@@ -729,7 +751,7 @@ class Registry {
source: bound && typeof raw.source === 'string' ? raw.source : null,
cliVersion: typeof raw.cliVersion === 'string' ? raw.cliVersion : null,
model: modelString(raw.model),
- processIdentity: validProcessIdentity(raw.processIdentity) ? raw.processIdentity : null,
+ processIdentity: !rebooted && validProcessIdentity(raw.processIdentity) ? raw.processIdentity : null,
registeredAt: typeof raw.registeredAt === 'number' ? raw.registeredAt : now,
updatedAt: typeof raw.updatedAt === 'number' ? raw.updatedAt : now,
lastHookAt: typeof raw.lastHookAt === 'number' ? raw.lastHookAt : (raw.updatedAt ?? now),
@@ -919,6 +941,7 @@ class Registry {
cwd?: string | null
grid?: GridAssignment | null
codexHome?: string | null
+ bypassPermission?: boolean
}): RegisteredSession | null {
if (this.writeBlocked) return null
const runtimes = normalizedRuntimes(input.runtimes)
@@ -937,6 +960,7 @@ class Registry {
gateway: null,
grid: input.grid ?? null,
codexHome: input.codexHome ?? null,
+ ...(input.bypassPermission ? { bypassPermission: true } : {}),
transcriptPath: null,
projectDir: basename(input.cwd ?? '') || agentId,
cwd: input.cwd ?? null,
@@ -1091,6 +1115,7 @@ class Registry {
// re-reads off the live process on every discovery) — a hook-triggered bind must carry it
// forward or the very first SessionStart hook would silently wipe the agent's chosen profile.
codexHome: existing?.codexHome ?? null,
+ ...(existing?.bypassPermission ? { bypassPermission: true } : {}),
processIdentity: validProcessIdentity(input.processIdentity) ? input.processIdentity : existing?.processIdentity ?? null,
registeredAt: existing?.registeredAt ?? now,
updatedAt: now,
@@ -1241,6 +1266,30 @@ class Registry {
return true
}
+ /** Forget the process an agent was last seen as — its pane is gone, so the pid is gone with it. */
+ clearProcessIdentity(agentId: string): boolean {
+ const session = this.agents.get(agentId)
+ if (!session) return false
+ if (!session.processIdentity) return true
+ this.drop(session)
+ session.processIdentity = null
+ session.updatedAt = Date.now()
+ this.index(session)
+ this.save()
+ return true
+ }
+
+ setBypassPermission(agentId: string, bypassPermission: boolean): boolean {
+ const session = this.agents.get(agentId)
+ if (!session) return false
+ if ((session.bypassPermission === true) === bypassPermission) return true
+ if (bypassPermission) session.bypassPermission = true
+ else delete session.bypassPermission
+ session.updatedAt = Date.now()
+ this.save()
+ return true
+ }
+
touchTranscript(sessionId: string, at = Date.now()): boolean {
const session = this.bySession(sessionId)
if (!session) return false
@@ -1501,3 +1550,11 @@ function validProcessIdentity(value: unknown): value is ProcessIdentity {
const p = value as Partial<ProcessIdentity> | null | undefined
return !!p && Number.isSafeInteger(p.pid) && (p.pid ?? 0) > 0 && typeof p.executable === 'string' && typeof p.startMarker === 'string'
}
+
+/** A persisted grid assignment. Lenient on `model` on purpose: a row that only knows WHERE it pointed
+ * must still read as a grid agent, or a restore would relaunch it on the engine's own login. */
+function normalizedGridAssignment(value: unknown): GridAssignment | null {
+ const g = value as Partial<GridAssignment> | null | undefined
+ if (!g || typeof g.baseUrl !== 'string' || !g.baseUrl) return null
+ return { baseUrl: g.baseUrl, model: typeof g.model === 'string' ? g.model : null }
+}
diff --git a/cli/src/lib/restoreAgents.spec.ts b/cli/src/lib/restoreAgents.spec.ts
new file mode 100644
index 000000000..07efc1386
--- /dev/null
+++ b/cli/src/lib/restoreAgents.spec.ts
@@ -0,0 +1,310 @@
+import { describe, expect, it, vi } from 'vitest'
+import type { AgentLaunch, ProcessIdentity, RegisteredSession } from './registry.js'
+import type { TerminalRuntimeRef, TmuxRuntimeRef } from './terminalTypes.js'
+import { restoreAgents, type RestoreAgentsDeps, type RestoreLaunch } from './restoreAgents.js'
+
+const identity = (pid: number): ProcessIdentity => ({ pid, executable: 'claude', startMarker: `start ${pid}` })
+
+function row(overrides: Partial<RegisteredSession> = {}): RegisteredSession {
+ return {
+ schemaVersion: 2,
+ active: false,
+ launch: { state: 'ready' },
+ agentId: 'agent-a',
+ sessionId: 'session-a',
+ boundAt: 1,
+ engine: 'claude',
+ transcriptPath: null,
+ projectDir: 'demo',
+ cwd: '/tmp/demo',
+ runtimes: [{ backend: 'tmux', paneId: '%3' }],
+ primaryRuntimeKey: 'tmux\u0000%3',
+ tmuxPane: '%3',
+ source: null,
+ title: null,
+ model: null,
+ cliVersion: null,
+ processIdentity: null,
+ registeredAt: 1,
+ updatedAt: 1,
+ lastHookAt: 1,
+ lastTranscriptAt: 1,
+ ...overrides,
+ }
+}
+
+interface Harness {
+ deps: RestoreAgentsDeps
+ calls: string[]
+ rows: Map<string, RegisteredSession>
+ launches: Array<{ agentId: string; resumeSessionId?: string }>
+ /** Per-pane scripted answers for probeProcess; shifted on each call. */
+ probes: Map<string, Array<ProcessIdentity | null>>
+ states: Map<string, Array<{ dead: boolean } | null>>
+ nextPane: string[]
+ paneCreates: number
+ respawns: number
+ transactionDepth: number
+ released: string[]
+}
+
+function harness(rows: RegisteredSession[], opts: { livePanes?: string[]; failCreate?: boolean; budgetMs?: number; settleMs?: number } = {}): Harness {
+ const h: Harness = {
+ calls: [],
+ rows: new Map(rows.map((r) => [r.agentId, r])),
+ launches: [],
+ probes: new Map(),
+ states: new Map(),
+ nextPane: ['%0', '%1', '%2'],
+ paneCreates: 0,
+ respawns: 0,
+ transactionDepth: 0,
+ released: [],
+ deps: undefined as unknown as RestoreAgentsDeps,
+ }
+ const live = new Set(opts.livePanes ?? [])
+ const note = (name: string) => { h.calls.push(`${name}${h.transactionDepth ? '@tx' : ''}`) }
+ h.deps = {
+ registry: {
+ list: () => [...h.rows.values()],
+ byAgent: (id) => h.rows.get(id),
+ transaction: async (apply) => {
+ h.transactionDepth++
+ try { return await apply() } finally { h.transactionDepth-- }
+ },
+ clearProcessIdentity: (id) => { note(`clearIdentity:${id}`); const r = h.rows.get(id); if (r) r.processIdentity = null; return !!r },
+ updateRuntimes: (id, runtimes: readonly TerminalRuntimeRef[], key) => {
+ note(`updateRuntimes:${id}:${(runtimes[0] as TmuxRuntimeRef).paneId}`)
+ const r = h.rows.get(id); if (!r) return false
+ r.runtimes = [...runtimes]; r.primaryRuntimeKey = key ?? ''; r.active = true
+ return true
+ },
+ setLaunch: (id, launch: AgentLaunch) => {
+ note(`setLaunch:${id}:${launch.state}${launch.state === 'failed' ? `:${launch.error}` : ''}`)
+ const r = h.rows.get(id); if (!r) return null
+ r.launch = launch; return r
+ },
+ updateProcessIdentity: (id, pi) => { note(`updateIdentity:${id}:${pi.pid}`); const r = h.rows.get(id); if (r) r.processIdentity = pi; return !!r },
+ unbindSession: (sid) => { note(`unbind:${sid}`); return true },
+ inheritName: (from, to) => { note(`inheritName:${from}->${to}`) },
+ },
+ liveProcess: async (_entry, runtime) => live.has(runtime.paneId) ? identity(1000 + Number(runtime.paneId.slice(1))) : null,
+ buildLaunch: (entry, o) => {
+ h.launches.push({ agentId: entry.agentId, ...(o.resumeSessionId ? { resumeSessionId: o.resumeSessionId } : {}) })
+ const launch: RestoreLaunch = { argv: [entry.engine, ...(o.resumeSessionId ? ['--resume', o.resumeSessionId] : [])] }
+ return launch
+ },
+ createPane: async (entry) => {
+ note(`createPane:${entry.agentId}`)
+ h.paneCreates++
+ if (opts.failCreate) return { ok: false, reason: 'tmux said no' }
+ return { ok: true, runtime: { backend: 'tmux', paneId: h.nextPane.shift() ?? '%99' } }
+ },
+ respawn: async (runtime, launch) => { note(`respawn:${runtime.paneId}:${launch.argv.join(' ')}`); h.respawns++; return { ok: true } },
+ probeProcess: async (runtime) => {
+ const queue = h.probes.get(runtime.paneId) ?? []
+ return queue.length ? queue.shift()! : null
+ },
+ paneState: async (runtime) => {
+ const queue = h.states.get(runtime.paneId) ?? []
+ return queue.length ? queue.shift()! : { dead: false }
+ },
+ clearRemainOnExit: async (runtime) => { note(`clearRemainOnExit:${runtime.paneId}`) },
+ holdRoute: (key) => { note(`hold:${key}`) },
+ releaseRoute: (key) => { note(`release:${key}`); h.released.push(key) },
+ triggerHint: async (runtime) => { note(`triggerHint:${runtime.paneId}`) },
+ log: () => {},
+ budgetMs: opts.budgetMs ?? 60_000,
+ settleMs: opts.settleMs ?? 0,
+ sleep: async () => {},
+ }
+ return h
+}
+
+/** A watch always ends in `finally { releaseRoute }`; the success path releases once more before the
+ * hint, so "settled" means at least `watches` releases and the LAST one came from a finally. */
+const settled = (h: Harness, watches: number, successes = 0) =>
+ vi.waitFor(() => { expect(h.released.length).toBe(watches + successes) })
+
+describe('restoreAgents — which agents get a pane back', () => {
+ it('recreates a missing pane with a resume launch and hands the row over to discovery', async () => {
+ const h = harness([row()])
+ h.probes.set('%0', [null, identity(500)])
+
+ const summary = await restoreAgents(h.deps)
+
+ expect(summary).toEqual({ restored: ['agent-a'], skipped: [], failed: [] })
+ expect(h.launches).toEqual([{ agentId: 'agent-a', resumeSessionId: 'session-a' }])
+ // Every registry write of phase 1 happens inside the one transaction, in this order.
+ expect(h.calls.slice(0, 5)).toEqual([
+ 'clearIdentity:agent-a@tx',
+ 'createPane:agent-a@tx',
+ 'hold:tmux\u0000%0@tx',
+ 'updateRuntimes:agent-a:%0@tx',
+ 'setLaunch:agent-a:starting@tx',
+ ])
+ await settled(h, 1, 1)
+ // The route is released BEFORE the hint so the pass it triggers sees the pane again.
+ expect(h.calls.slice(5)).toEqual([
+ 'updateIdentity:agent-a:500',
+ 'clearRemainOnExit:%0',
+ 'release:tmux\u0000%0',
+ 'triggerHint:%0',
+ 'release:tmux\u0000%0', // the finally, idempotent by contract
+ ])
+ expect(h.rows.get('agent-a')?.launch).toEqual({ state: 'starting' })
+ })
+
+ it('launches fresh when the row has no session to resume', async () => {
+ const h = harness([row({ sessionId: '', boundAt: null })])
+ h.probes.set('%0', [identity(1)])
+ await restoreAgents(h.deps)
+ expect(h.launches).toEqual([{ agentId: 'agent-a' }])
+ await settled(h, 1, 1)
+ })
+
+ it('leaves a pane that still runs its engine alone, re-identifying a row that lost its pid', async () => {
+ const h = harness([row()], { livePanes: ['%3'] })
+ const summary = await restoreAgents(h.deps)
+ expect(summary).toEqual({ restored: [], skipped: [], failed: [] })
+ expect(h.paneCreates).toBe(0)
+ // No identity on the row (a misread reboot, a tmux server that outlived the daemon): the live
+ // pid is written back so discovery adopts by process rather than by route.
+ expect(h.calls).toEqual(['updateIdentity:agent-a:1003'])
+ })
+
+ it('does not touch a live row that still knows its process', async () => {
+ const h = harness([row({ processIdentity: identity(1003) })], { livePanes: ['%3'] })
+ await restoreAgents(h.deps)
+ expect(h.calls).toEqual([])
+ })
+
+ it('skips herdr-only rows, failed launches and grid agents, saying why', async () => {
+ const h = harness([
+ row({ agentId: 'herdr', runtimes: [{ backend: 'herdr', sessionName: 's', paneId: 'p' } as unknown as TerminalRuntimeRef] }),
+ row({ agentId: 'broken', launch: { state: 'failed', error: 'START_TIMEOUT' } }),
+ row({ agentId: 'gridded', grid: { baseUrl: 'https://grid.example/relay', model: null } }),
+ ])
+ const summary = await restoreAgents(h.deps)
+ expect(summary.restored).toEqual([])
+ expect(summary.skipped.map((s) => s.agentId)).toEqual(['herdr', 'broken', 'gridded'])
+ expect(summary.skipped[2].reason).toMatch(/grid/)
+ expect(h.paneCreates).toBe(0)
+ })
+
+ it('records a pane that could not be opened and leaves the row untouched', async () => {
+ const h = harness([row()], { failCreate: true })
+ const summary = await restoreAgents(h.deps)
+ expect(summary.failed).toEqual([{ agentId: 'agent-a', reason: 'tmux said no' }])
+ expect(h.calls).not.toContainEqual(expect.stringMatching(/^updateRuntimes/))
+ expect(h.calls).not.toContainEqual(expect.stringMatching(/^hold/))
+ expect(h.rows.get('agent-a')?.tmuxPane).toBe('%3')
+ })
+
+ it('restores several agents inside one transaction so reused pane ids cannot evict each other', async () => {
+ const h = harness([row({ agentId: 'a', runtimes: [{ backend: 'tmux', paneId: '%1' }] }), row({ agentId: 'b', runtimes: [{ backend: 'tmux', paneId: '%0' }] })])
+ h.probes.set('%0', [identity(1)])
+ h.probes.set('%1', [identity(2)])
+ const summary = await restoreAgents(h.deps)
+ expect(summary.restored).toEqual(['a', 'b'])
+ expect(h.calls.filter((c) => c.startsWith('updateRuntimes'))).toEqual(['updateRuntimes:a:%0@tx', 'updateRuntimes:b:%1@tx'])
+ await settled(h, 2, 2)
+ })
+})
+
+describe('restoreAgents — waiting for the engine', () => {
+ it('falls back to a fresh launch once when the resumed engine dies, and unbinds the stale session', async () => {
+ const h = harness([row()])
+ h.probes.set('%0', [null, null, identity(9)])
+ h.states.set('%0', [{ dead: true }, { dead: false }])
+
+ await restoreAgents(h.deps)
+ await settled(h, 1, 1)
+
+ expect(h.respawns).toBe(1)
+ expect(h.launches).toEqual([
+ { agentId: 'agent-a', resumeSessionId: 'session-a' },
+ { agentId: 'agent-a' },
+ ])
+ const tail = h.calls.slice(5)
+ expect(tail.slice(0, 3)).toEqual(['inheritName:session-a->agent-a', 'unbind:session-a', 'respawn:%0:claude'])
+ expect(tail).toContain('updateIdentity:agent-a:9')
+ expect(h.rows.get('agent-a')?.launch).toEqual({ state: 'starting' })
+ })
+
+ it('treats an engine that appears and then exits inside the settling window as a rejected resume', async () => {
+ const h = harness([row()], { settleMs: 50 })
+ // First probe: the resumed claude is up. Settling poll: the pane is dead. After the fresh
+ // respawn: up again, and this time it stays.
+ h.probes.set('%0', [identity(7), identity(8)])
+ h.states.set('%0', [{ dead: true }])
+
+ await restoreAgents(h.deps)
+ await settled(h, 1, 1)
+
+ expect(h.respawns).toBe(1)
+ expect(h.launches.map((l) => l.resumeSessionId ?? 'fresh')).toEqual(['session-a', 'fresh'])
+ const tail = h.calls.slice(5)
+ expect(tail.slice(0, 4)).toEqual(['updateIdentity:agent-a:7', 'inheritName:session-a->agent-a', 'unbind:session-a', 'respawn:%0:claude'])
+ expect(tail).toContain('updateIdentity:agent-a:8')
+ expect(tail).toContain('clearRemainOnExit:%0')
+ // remain-on-exit was NOT cleared before the first engine died — that is what kept the pane.
+ expect(tail.indexOf('clearRemainOnExit:%0')).toBeGreaterThan(tail.indexOf('respawn:%0:claude'))
+ expect(h.rows.get('agent-a')?.launch).toEqual({ state: 'starting' })
+ })
+
+ it('fails a fresh engine that exits inside the settling window', async () => {
+ const h = harness([row({ sessionId: '', boundAt: null })], { settleMs: 50 })
+ h.probes.set('%0', [identity(7)])
+ h.states.set('%0', [{ dead: true }])
+ await restoreAgents(h.deps)
+ await settled(h, 1)
+ expect(h.respawns).toBe(0)
+ expect(h.rows.get('agent-a')?.launch).toMatchObject({ state: 'failed', error: 'ENGINE_DID_NOT_START' })
+ })
+
+ it('gives up after the fresh launch dies too', async () => {
+ const h = harness([row()])
+ h.states.set('%0', [{ dead: true }, { dead: true }])
+ await restoreAgents(h.deps)
+ await settled(h, 1)
+ expect(h.respawns).toBe(1)
+ expect(h.rows.get('agent-a')?.launch).toMatchObject({ state: 'failed', error: 'ENGINE_DID_NOT_START' })
+ })
+
+ it('does not retry fresh when there was no resume to blame', async () => {
+ const h = harness([row({ sessionId: '', boundAt: null })])
+ h.states.set('%0', [{ dead: true }])
+ await restoreAgents(h.deps)
+ await settled(h, 1)
+ expect(h.respawns).toBe(0)
+ expect(h.rows.get('agent-a')?.launch).toMatchObject({ state: 'failed', error: 'ENGINE_DID_NOT_START' })
+ })
+
+ it('fails the launch when the pane vanishes entirely', async () => {
+ const h = harness([row()])
+ h.states.set('%0', [null])
+ await restoreAgents(h.deps)
+ await settled(h, 1)
+ expect(h.rows.get('agent-a')?.launch).toMatchObject({ state: 'failed', error: 'ENGINE_DID_NOT_START' })
+ })
+
+ it('times out with the launch marked, never leaving the route held', async () => {
+ const h = harness([row()], { budgetMs: 0 })
+ await restoreAgents(h.deps)
+ await settled(h, 1)
+ expect(h.rows.get('agent-a')?.launch).toMatchObject({ state: 'failed', error: 'START_TIMEOUT' })
+ expect(h.released).toEqual(['tmux\u0000%0'])
+ })
+
+ it('stops quietly when the agent was deleted while its engine was starting', async () => {
+ const h = harness([row()])
+ h.probes.set('%0', [null])
+ const deps = { ...h.deps }
+ let polls = 0
+ deps.probeProcess = async () => { if (++polls === 2) h.rows.delete('agent-a'); return null }
+ await restoreAgents(deps)
+ await settled(h, 1)
+ expect(h.calls.filter((c) => c.startsWith('setLaunch:agent-a:failed'))).toEqual([])
+ })
+})
diff --git a/cli/src/lib/restoreAgents.ts b/cli/src/lib/restoreAgents.ts
new file mode 100644
index 000000000..a64f10aa6
--- /dev/null
+++ b/cli/src/lib/restoreAgents.ts
@@ -0,0 +1,247 @@
+/**
+ * Bring registered agents back after their tmux panes died with the daemon down — a reboot, a
+ * `tmux kill-server`, a pane closed by hand.
+ *
+ * The registry outlives the tmux server: it still knows each agent's id, engine, cwd, session and
+ * launch shape. This recreates a pane for every agent whose pane is gone, launches the same engine
+ * there — resuming its engine session when it has one — and keeps the SAME agentId, so a desktop
+ * tile that remembered the agent attaches to it again without anyone recreating anything.
+ *
+ * Dependency-injected like restartAgent.ts: cli.ts owns the tmux, registry and reconciler wiring,
+ * and this owns the ordering, which is the part worth testing without a daemon:
+ *
+ * 1. Every missing pane is recreated inside ONE registry transaction. A new tmux server hands out
+ * pane ids from `%0` again, so a restored pane's id can equal another stale row's dead pane —
+ * and `registry.save()` evicts whichever row loses that collision. Deferring the save until
+ * every row holds its new pane is what keeps the second row alive.
+ * 2. The new route is HELD in the reconciler until the engine process is bound here. Otherwise the
+ * reconciler's discovery half sees an unclaimed engine process and mints a fresh agentId for it.
+ * 3. The row keeps `launch: starting` and only gets its process identity from here. The reconcile
+ * pass after the release then matches the process, sees a launch in progress, and does the
+ * full ready → attach → announce sequence it already does for `agent_create`.
+ */
+
+import type { AgentEngine } from '../engines/types.js'
+import type { AgentLaunch, ProcessIdentity, RegisteredSession } from './registry.js'
+import type { TerminalRuntimeRef, TmuxRuntimeRef } from './terminalTypes.js'
+import { terminalRouteKey } from './terminalRuntime.js'
+
+export interface RestoreLaunch {
+ argv: string[]
+ env?: Record<string, string>
+}
+
+export interface RestoreAgentsDeps {
+ registry: {
+ list(): RegisteredSession[]
+ byAgent(agentId: string): RegisteredSession | undefined
+ transaction<T>(apply: () => T | Promise<T>): Promise<T>
+ clearProcessIdentity(agentId: string): boolean
+ updateRuntimes(agentId: string, runtimes: readonly TerminalRuntimeRef[], primaryRuntimeKey?: string): boolean
+ setLaunch(agentId: string, launch: AgentLaunch): RegisteredSession | null
+ updateProcessIdentity(agentId: string, processIdentity: ProcessIdentity): boolean
+ unbindSession(sessionId: string): boolean
+ inheritName(fromSessionId: string, toSessionId: string): void
+ }
+ /** The engine process still running in this row's pane, or null when tmux does not know the pane
+ * at all — including when no tmux server is running — or the pane has become something else. */
+ liveProcess: (entry: RegisteredSession, runtime: TmuxRuntimeRef) => Promise<ProcessIdentity | null>
+ buildLaunch: (entry: RegisteredSession, opts: { resumeSessionId?: string }) => RestoreLaunch
+ createPane: (entry: RegisteredSession, launch: RestoreLaunch) => Promise<
+ | { ok: true; runtime: TmuxRuntimeRef }
+ | { ok: false; reason: string }
+ >
+ /** `tmux respawn-pane` over the restored pane — the resume → fresh fallback. */
+ respawn: (runtime: TmuxRuntimeRef, launch: RestoreLaunch) => Promise<{ ok: boolean; reason?: string }>
+ /** One probe of the pane for a recognizable engine process. */
+ probeProcess: (runtime: TmuxRuntimeRef, engine: AgentEngine) => Promise<ProcessIdentity | null>
+ /** Null when tmux no longer knows the pane. */
+ paneState: (runtime: TmuxRuntimeRef) => Promise<{ dead: boolean } | null>
+ clearRemainOnExit: (runtime: TmuxRuntimeRef) => Promise<void>
+ holdRoute: (routeKey: string, autoReleaseMs: number) => void
+ releaseRoute: (routeKey: string) => void
+ triggerHint: (runtime: TmuxRuntimeRef, engine: AgentEngine) => Promise<void>
+ log: (message: string) => void
+ /** How long a restored pane may take to show an engine process. Default matches `agent_create`. */
+ budgetMs?: number
+ /** How long the engine must stay up after appearing before the pane is handed over. */
+ settleMs?: number
+ sleep?: (ms: number) => Promise<void>
+}
+
+export interface RestoreSummary {
+ /** Agents whose pane was recreated; their engine process is bound in the background. */
+ restored: string[]
+ skipped: Array<{ agentId: string; reason: string }>
+ failed: Array<{ agentId: string; reason: string }>
+}
+
+const DEFAULT_BUDGET_MS = 10 * 60_000
+const DEFAULT_SETTLE_MS = 10_000
+const SETTLE_POLL_MS = 500
+const HOLD_SLACK_MS = 30_000
+
+function tmuxRuntime(entry: RegisteredSession): TmuxRuntimeRef | null {
+ const runtime = entry.runtimes.find((candidate): candidate is TmuxRuntimeRef => candidate.backend === 'tmux')
+ return runtime ?? null
+}
+
+function defaultSleep(ms: number): Promise<void> {
+ return new Promise((resolve) => {
+ const timer = setTimeout(resolve, ms)
+ timer.unref?.()
+ })
+}
+
+/** Watch a pane whose engine just appeared: 'settled' once it has stayed up for `settleMs`, 'gone' the
+ * moment tmux reports it dead or forgotten. */
+async function waitForSettle(deps: RestoreAgentsDeps, runtime: TmuxRuntimeRef, settleMs: number): Promise<'settled' | 'gone'> {
+ const sleep = deps.sleep ?? defaultSleep
+ const until = Date.now() + settleMs
+ while (Date.now() < until) {
+ await sleep(Math.min(SETTLE_POLL_MS, until - Date.now()))
+ const state = await deps.paneState(runtime)
+ if (!state || state.dead) return 'gone'
+ }
+ return 'settled'
+}
+
+/**
+ * Recreate every missing pane, then return. Binding each pane's engine process continues in the
+ * background; the summary says which agents that is happening for.
+ */
+export async function restoreAgents(deps: RestoreAgentsDeps): Promise<RestoreSummary> {
+ const summary: RestoreSummary = { restored: [], skipped: [], failed: [] }
+ const missing: Array<{ entry: RegisteredSession; runtime: TmuxRuntimeRef }> = []
+
+ for (const entry of deps.registry.list()) {
+ const runtime = tmuxRuntime(entry)
+ if (!runtime) { summary.skipped.push({ agentId: entry.agentId, reason: 'no tmux pane' }); continue }
+ if (entry.launch?.state === 'failed') { summary.skipped.push({ agentId: entry.agentId, reason: 'last launch failed' }); continue }
+ const live = await deps.liveProcess(entry, runtime)
+ if (live) {
+ // Still running. A row that lost its identity without losing its pane (a reboot the boot
+ // clock misread; a tmux server that outlived the daemon) is re-identified right here, so the
+ // reconciler adopts it by process instead of treating it as an unbound route.
+ if (!entry.processIdentity) deps.registry.updateProcessIdentity(entry.agentId, live)
+ continue
+ }
+ // The registry keeps only where a grid agent pointed, never the credential the launch needs —
+ // and launching it on the engine's own login instead would spend the wrong account while
+ // looking identical. Refusing is the rule gridLaunch.ts already sets.
+ if (entry.grid) { summary.skipped.push({ agentId: entry.agentId, reason: 'grid agent; credential not persisted' }); continue }
+ missing.push({ entry, runtime })
+ }
+ if (!missing.length) return summary
+
+ const budgetMs = deps.budgetMs ?? DEFAULT_BUDGET_MS
+ const watches: Array<() => Promise<void>> = []
+ await deps.registry.transaction(async () => {
+ for (const { entry, runtime: dead } of missing) {
+ // No process survives its pane; clear it now or discovery would refuse to adopt the new one
+ // into this agent (route adoption requires either no identity or a matching pid).
+ deps.registry.clearProcessIdentity(entry.agentId)
+ const resumeSessionId = entry.sessionId || undefined
+ const launch = deps.buildLaunch(entry, resumeSessionId ? { resumeSessionId } : {})
+ const created = await deps.createPane(entry, launch)
+ if (!created.ok) {
+ summary.failed.push({ agentId: entry.agentId, reason: created.reason })
+ deps.log(`[restore] ${entry.engine} · agent ${entry.agentId} · could not open a pane · ${created.reason}`)
+ continue
+ }
+ const key = terminalRouteKey(created.runtime)
+ deps.holdRoute(key, budgetMs + HOLD_SLACK_MS)
+ deps.registry.updateRuntimes(entry.agentId, [created.runtime], key)
+ deps.registry.setLaunch(entry.agentId, { state: 'starting' })
+ summary.restored.push(entry.agentId)
+ deps.log(`[restore] ${entry.engine} · agent ${entry.agentId} · pane ${dead.paneId} → ${created.runtime.paneId}`
+ + (resumeSessionId ? ` · resuming ${resumeSessionId.slice(0, 8)}` : ' · fresh session'))
+ watches.push(() => watchRestoredPane(deps, entry, created.runtime, !!resumeSessionId, budgetMs))
+ }
+ })
+ for (const watch of watches) void watch()
+ return summary
+}
+
+async function watchRestoredPane(
+ deps: RestoreAgentsDeps,
+ entry: RegisteredSession,
+ runtime: TmuxRuntimeRef,
+ resuming: boolean,
+ budgetMs: number,
+): Promise<void> {
+ const { agentId, engine } = entry
+ const key = terminalRouteKey(runtime)
+ const sleep = deps.sleep ?? defaultSleep
+ const fail = (error: string, detail: string): void => {
+ deps.registry.setLaunch(agentId, { state: 'failed', error, detail })
+ deps.log(`[restore] ${engine} · agent ${agentId} · failed · ${detail}`)
+ }
+ const settleMs = deps.settleMs ?? DEFAULT_SETTLE_MS
+ let mayRetryFresh = resuming
+ /** The pane's engine is gone. True when a fresh relaunch is now under way, false when this is the end. */
+ const relaunchFresh = async (): Promise<boolean> => {
+ if (!mayRetryFresh) {
+ fail('ENGINE_DID_NOT_START', `${engine} exited before its engine process became ready. See the terminal output for details.`)
+ return false
+ }
+ // A resume id the engine no longer honours is not worth a dead agent: the row's name comes
+ // along to the agent itself, the stale binding goes, and the engine gets one fresh start.
+ mayRetryFresh = false
+ deps.log(`[restore] ${engine} · agent ${agentId} · did not come back up resuming its session — retrying fresh`)
+ deps.registry.inheritName(entry.sessionId, agentId)
+ deps.registry.unbindSession(entry.sessionId)
+ const spawned = await deps.respawn(runtime, deps.buildLaunch(entry, {}))
+ if (!spawned.ok) {
+ fail('ENGINE_DID_NOT_START', `${engine} could not be relaunched fresh: ${spawned.reason ?? 'unknown reason'}`)
+ return false
+ }
+ // The fresh start gets the full budget again; the hold's auto-release must outlast it.
+ deps.holdRoute(key, budgetMs + HOLD_SLACK_MS)
+ return true
+ }
+ try {
+ const startedAt = Date.now()
+ let delayMs = 50
+ while (Date.now() - startedAt < budgetMs) {
+ if (!deps.registry.byAgent(agentId)) return
+ const identity = await deps.probeProcess(runtime, engine)
+ if (identity) {
+ deps.registry.updateProcessIdentity(agentId, identity)
+ // A resume the engine rejects does not fail to start — it starts, prints why, and exits a
+ // few seconds later. Keep the pane (and the fallback) for a settling window before handing
+ // it over; a pane let go at first sight of the process vanished with the engine and took
+ // the agent with it (measured: claude resuming a conversation another client still held).
+ const settled = await waitForSettle(deps, runtime, settleMs)
+ if (settled === 'gone') {
+ if (!await relaunchFresh()) return
+ delayMs = 50
+ continue
+ }
+ await deps.clearRemainOnExit(runtime)
+ // Release BEFORE the hint: the pass it triggers is the one that must see this route again.
+ deps.releaseRoute(key)
+ await deps.triggerHint(runtime, engine)
+ deps.log(`[restore] ${engine} · agent ${agentId} · engine up · ${Date.now() - startedAt}ms`)
+ return
+ }
+ const state = await deps.paneState(runtime)
+ if (!state) {
+ fail('ENGINE_DID_NOT_START', `${engine}'s restored pane disappeared before its engine process became ready.`)
+ return
+ }
+ if (state.dead) {
+ if (!await relaunchFresh()) return
+ delayMs = 50
+ continue
+ }
+ await sleep(delayMs)
+ delayMs = Math.min(delayMs * 2, 750)
+ }
+ fail('START_TIMEOUT', `${engine} did not expose an engine process within ${Math.round(budgetMs / 60_000)} minutes. The terminal remains available.`)
+ } catch (error) {
+ deps.log(`[restore] ${engine} · agent ${agentId} · watch failed · ${error instanceof Error ? error.message : error}`)
+ } finally {
+ deps.releaseRoute(key)
+ }
+}
diff --git a/cli/src/lib/terminalBinary.spec.ts b/cli/src/lib/terminalBinary.spec.ts
index c229deee2..8ba8f0b7a 100644
--- a/cli/src/lib/terminalBinary.spec.ts
+++ b/cli/src/lib/terminalBinary.spec.ts
@@ -265,13 +265,12 @@ describe('authenticated loopback terminal framing v1', () => {
compressed: false,
})
expect(encoded).not.toBeNull()
- expect(decodeTerminalLocal(encoded!)).toEqual({
- kind: TerminalBinaryKind.paste,
- streamId,
- seq: 0,
- bytes,
- compressed: false,
- })
+ // Metadata via deep-equal, payload via native memcmp: vitest's toEqual walks a typed array one
+ // element at a time, and on a multi-MiB payload that alone took ~1-3s on a fast laptop and blew
+ // through the 5s test timeout on a CI runner (v0.2.16_cli release run).
+ const decoded = decodeTerminalLocal(encoded!)!
+ expect(decoded).toMatchObject({ kind: TerminalBinaryKind.paste, streamId, seq: 0, compressed: false })
+ expect(Buffer.compare(decoded.bytes, bytes)).toBe(0)
expect(encodeTerminalLocal({
kind: TerminalBinaryKind.output,
@@ -295,13 +294,9 @@ describe('authenticated loopback terminal framing v1', () => {
compressed: false,
})
expect(encoded).not.toBeNull()
- expect(decodeTerminalLocal(encoded!)).toEqual({
- kind: TerminalBinaryKind.imagePaste,
- streamId,
- seq: 0,
- bytes,
- compressed: false,
- })
+ const decoded = decodeTerminalLocal(encoded!)!
+ expect(decoded).toMatchObject({ kind: TerminalBinaryKind.imagePaste, streamId, seq: 0, compressed: false })
+ expect(Buffer.compare(decoded.bytes, bytes)).toBe(0) // memcmp, not element-wise — see the paste test above
const tooBig = new Uint8Array(TERMINAL_LOCAL_IMAGE_PASTE_MAX_PAYLOAD_BYTES + 1)
expect(encodeTerminalLocal({
@@ -326,10 +321,9 @@ describe('authenticated loopback terminal framing v1', () => {
compressed: false,
})
expect(encoded).not.toBeNull()
- const decoded = decodeTerminalLocal(encoded!)
- expect(decoded?.kind).toBe(TerminalBinaryKind.pasteFile)
- expect(decoded?.seq).toBe(2)
- expect(decoded?.bytes).toEqual(content)
+ const decoded = decodeTerminalLocal(encoded!)!
+ expect(decoded).toMatchObject({ kind: TerminalBinaryKind.pasteFile, seq: 2 })
+ expect(Buffer.compare(decoded.bytes, content)).toBe(0) // memcmp, not element-wise — see the paste test above
const tooBig = new Uint8Array(TERMINAL_LOCAL_PASTE_FILE_MAX_PAYLOAD_BYTES + 1)
expect(encodeTerminalLocal({