Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

harnessd: engines updated in place, and a hook before its process is recorded waits for it (round 24) (#797)

Deeae01e80fa38c
diff --git a/cli/e2e/harness/fakeEngine.mjs b/cli/e2e/harness/fakeEngine.mjs
index 6028bd4cd..bd56bebe9 100644
--- a/cli/e2e/harness/fakeEngine.mjs
+++ b/cli/e2e/harness/fakeEngine.mjs
@@ -17,8 +17,13 @@
// numbered lines, the way a build log or a long diff does, `!clear` starts a new conversation in the
// same pane as Claude Code's `/clear` and Codex's `/new` do, `!compact` compacts the conversation as
// `/compact` does (and Claude Code then announces the same session again), `!compactmid` compacts in the
-// middle of a turn as an automatic compaction does, `!exit` ends the process. Everything else is
-// echoed as the answer.
+// middle of a turn as an automatic compaction does, `!version` answers with the version this
+// process is, `!exit` ends the process. Everything else is echoed as the answer.
+//
+// Which release is installed is the config's business, so a test can update an engine in place by
+// rewriting its wrapper: `version` is what it reports and writes, `without` lists the flags and
+// subcommands that release no longer has, and `startDelayMs` is how long it takes, once started, before
+// it draws or announces anything (an engine's first run after an update).
import { execFileSync } from 'node:child_process'
import { randomUUID } from 'node:crypto'
import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, writeFileSync } from 'node:fs'
@@ -26,32 +31,48 @@ import { dirname, join } from 'node:path'
export async function run(engine, config) {
const args = process.argv.slice(2)
+ const version = config.version ?? (engine === 'claude' ? '2.1.270' : '0.159.0')
+ const versionLine = engine === 'claude' ? `${version} (Claude Code)` : `codex-cli ${version}`
+ const without = new Set(config.without ?? [])
if (args.includes('--version') || args.includes('-V')) {
- process.stdout.write(engine === 'claude' ? '2.1.270 (Claude Code)\n' : 'codex-cli 0.159.0\n')
+ process.stdout.write(`${versionLine}\n`)
return
}
if (args.includes('--help') || args.includes('-h')) {
- // The flags the daemon looks for in the real CLIs' help before it launches them.
- process.stdout.write(engine === 'codex'
+ // The flags the daemon looks for in the real CLIs' help before it launches them, less the ones this
+ // release has dropped.
+ process.stdout.write((engine === 'codex'
? [
'Usage: codex [OPTIONS] [PROMPT]', '',
' -c, --config <key=value>', ' -m, --model <MODEL>', ' -s, --sandbox <SANDBOX_MODE>',
' -a, --ask-for-approval <APPROVAL_POLICY>', ' --approve-for-me',
' --dangerously-bypass-approvals-and-sandbox', ' --no-daemon',
'Commands:', ' resume Resume a previous interactive session', '',
- ].join('\n')
+ ]
: [
'Usage: claude [options] [command] [prompt]', '', 'Arguments:', ' prompt Your prompt', '',
' --model <model>', ' --permission-mode <mode> (choices: "acceptEdits", "auto", "bypassPermissions", "default", "plan")',
' --dangerously-skip-permissions', ' -r, --resume [value]', ' --fork-session', ' --session-id <uuid>', '',
- ].join('\n'))
+ ]).filter((line) => !line.split(/[^A-Za-z0-9-]+/).some((word) => without.has(word))).join('\n'))
return
}
+ // A flag or subcommand this release does not have is refused before anything is drawn, as the real
+ // CLIs' parsers refuse one: Commander (Claude Code) exits 1, clap (Codex) exits 2.
+ const refused = args.find((arg) => without.has(arg))
+ if (refused) {
+ process.stderr.write(engine === 'claude'
+ ? `error: unknown option '${refused}'\n`
+ : `error: unexpected argument '${refused}' found\n\nUsage: codex [OPTIONS] [PROMPT]\n\nFor more information, try '--help'.\n`)
+ process.exit(engine === 'claude' ? 1 : 2)
+ }
process.title = engine
// Raw at once, as the real CLIs are (Ink and ratatui take the terminal before they draw anything): a
// terminal still in line mode keeps at most 1 KiB of a line, so a paste that arrived before the
// engine was ready lost the rest of itself (measured: 1,018 of 2,406 characters).
process.stdin.setRawMode?.(true)
+ // Slow to start, as an engine is on its first run after an update: the process is there and holds
+ // the terminal, and nothing is drawn, announced or read until it is ready.
+ if (config.startDelayMs) await new Promise((resolve) => setTimeout(resolve, config.startDelayMs))
const resumeAt = engine === 'claude' ? args.indexOf('--resume') : args.indexOf('resume')
const resumed = resumeAt >= 0 && args[resumeAt + 1] && !args[resumeAt + 1].startsWith('-') ? args[resumeAt + 1] : null
@@ -101,12 +122,12 @@ export async function run(engine, config) {
let parent = null
const claude = (record) => {
const uuid = randomUUID()
- write({ parentUuid: parent, isSidechain: false, userType: 'external', cwd, sessionId, version: '2.1.270', timestamp: now(), uuid, ...record })
+ write({ parentUuid: parent, isSidechain: false, userType: 'external', cwd, sessionId, version, timestamp: now(), uuid, ...record })
parent = uuid
}
const codex = (type, payload) => write({ timestamp: now(), type, payload })
if (engine === 'codex' && readFileSync(transcript, 'utf8') === '') {
- codex('session_meta', { id: sessionId, cli_version: '0.159.0', cwd, source: 'cli' })
+ codex('session_meta', { id: sessionId, cli_version: version, cwd, source: 'cli' })
}
// The port the daemon actually bound, which it saves beside its data (lib/daemonEndpoint.ts) and
@@ -249,7 +270,7 @@ export async function run(engine, config) {
writeFileSync(transcript, '')
parent = null
turn = 0
- if (engine === 'codex') codex('session_meta', { id: sessionId, cli_version: '0.159.0', cwd, source: 'cli' })
+ if (engine === 'codex') codex('session_meta', { id: sessionId, cli_version: version, cwd, source: 'cli' })
announceProcess()
process.stdout.write('\r\n(new conversation)\r\n')
draw()
@@ -358,6 +379,7 @@ export async function run(engine, config) {
finish(`you chose ${choice}`)
return
}
+ if (directive?.[1] === 'version') { finish(versionLine); return }
if (directive?.[1] === 'slow') await new Promise((resolve) => setTimeout(resolve, Number(directive[2]) || 1000))
finish(`answer ${turn}: ${prompt}`)
}
diff --git a/cli/e2e/rotation.e2e.ts b/cli/e2e/rotation.e2e.ts
index 2f1071fb9..cb4d1b83d 100644
--- a/cli/e2e/rotation.e2e.ts
+++ b/cli/e2e/rotation.e2e.ts
@@ -2,7 +2,9 @@
* A new conversation in the same pane, for Claude Code and Codex — Claude Code's `/clear`, Codex's
* `/new`: the engine ends one conversation and starts another without leaving the pane. The agent is
* the same agent, under the same id, bound to the new conversation; its turns are seen; it comes back
- * bound to the new one after a restart, and resumes the new one after a stop.
+ * bound to the new one after a restart, and resumes the new one after a stop. The person chose to leave
+ * the old conversation, so it is not kept as a stopped harness, as one the daemon had to leave is
+ * (updates.e2e.ts): people who clear often would pile up stopped rows.
*/
import { mkdirSync } from 'node:fs'
import { join } from 'node:path'
@@ -70,8 +72,9 @@ describe('a new conversation in the same pane', () => {
const started = await turn(client, agent.id, 'in the second conversation')
expect(started.payload?.userMessage).toBe('in the second conversation')
expect(await users(client, rotated.sessionId)).toEqual(['in the second conversation'])
- // One agent for the pane, never a second minted beside it.
+ // One agent for the pane, never a second minted beside it, and no stopped row for the one left.
expect((await rows(client)).filter((one) => one.tmuxPane === agent.tmuxPane && one.status === 'active')).toHaveLength(1)
+ expect((await rows(client)).filter((one) => one.status === 'stopped')).toEqual([])
client.close()
})
@@ -140,6 +143,7 @@ describe('a new conversation in the same pane', () => {
expect(new Set([agent.sessionId, second.sessionId, third.sessionId]).size).toBe(3)
await turn(client, agent.id, 'in the third conversation')
expect(await users(client, third.sessionId)).toEqual(['in the third conversation'])
+ expect((await rows(client)).filter((one) => one.status === 'stopped')).toEqual([])
client.close()
})
})
diff --git a/cli/e2e/updates.e2e.ts b/cli/e2e/updates.e2e.ts
new file mode 100644
index 000000000..fa2a6b7a4
--- /dev/null
+++ b/cli/e2e/updates.e2e.ts
@@ -0,0 +1,316 @@
+/**
+ * Engines updated in place, for Claude Code and Codex. Both update themselves while agents run, Claude
+ * Code nearly every day: the file on disk is replaced under a running process, a release can drop a
+ * flag the daemon passes, the engine can be uninstalled and installed again, and its first start after
+ * an update can be slow. A running agent must go on working through all of it; a restart or a new agent
+ * must run what is on disk now, or fail quickly with a reason the window gets; and none of it may need
+ * the daemon itself restarted.
+ */
+import { mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
+import { LocalClient, type Frame } from './harness/client.js'
+import { CLI_ROOT, IsolatedDaemon, until } from './harness/daemon.js'
+
+type Engine = 'claude' | 'codex'
+type Row = Record<string, any>
+const engines: Engine[] = ['claude', 'codex']
+
+/** A release of the fake engine, as e2e/harness/fakeEngine.mjs reads it from its config. */
+interface Release { version?: string; without?: string[]; startDelayMs?: number }
+const OLD: Record<Engine, string> = { claude: '2.1.270', codex: '0.159.0' }
+const NEW: Record<Engine, string> = { claude: '2.1.271', codex: '0.160.0' }
+const versionLine = (engine: Engine, version: string) => engine === 'claude' ? `${version} (Claude Code)` : `codex-cli ${version}`
+const sleep = (ms: number) => new Promise((done) => setTimeout(done, ms))
+
+/**
+ * Installs a release over the engine the way an updater does: the new file is written beside the old one
+ * and renamed over it, so a process already running keeps the file it started from and the next launch
+ * gets the new one. The config is the one IsolatedDaemon gave the engine, plus the release.
+ */
+function install(d: IsolatedDaemon, engine: Engine, release: Release = {}): void {
+ const config = { port: d.port, dataDir: d.dataDir, claudeProjectsDir: d.env.CLAUDE_PROJECTS_DIR, codexHome: d.env.CODEX_HOME, ...release }
+ const module = pathToFileURL(join(CLI_ROOT, 'e2e', 'harness', 'fakeEngine.mjs')).href
+ const wrapper = join(d.root, 'bin', engine)
+ writeFileSync(`${wrapper}.new`, `#!${process.execPath}\nimport(${JSON.stringify(module)}).then((m) => m.run(${JSON.stringify(engine)}, ${JSON.stringify(config)}))\n`, { mode: 0o755 })
+ renameSync(`${wrapper}.new`, wrapper)
+}
+const uninstall = (d: IsolatedDaemon, engine: Engine) => rmSync(join(d.root, 'bin', engine), { force: true })
+
+const rows = async (client: LocalClient): Promise<Row[]> =>
+ (await client.request<{ agents: Row[] }>('agents_list', { includeStopped: true }, 30_000)).agents
+const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)
+const bound = (client: LocalClient, agentId: string, sessionId?: string) =>
+ until(`${agentId.slice(0, 8)} to bind its conversation`, async () => {
+ const agent = await row(client, agentId)
+ return agent?.sessionId && agent.status === 'active' && (!sessionId || agent.sessionId === sessionId) ? agent : null
+ }, 60_000, 500)
+async function create(d: IsolatedDaemon, client: LocalClient, engine: Engine, folder: string): Promise<Row> {
+ const cwd = join(d.projectsDir, folder)
+ mkdirSync(cwd, { recursive: true })
+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
+ expect(created.error, `${folder}: ${JSON.stringify(created)}`).toBeUndefined()
+ return bound(client, created.agent.id)
+}
+const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId
+async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {
+ const started = client.next(isTurn('turn_started', agentId), 45_000, `turn_started (${content})`)
+ const ended = client.next(isTurn('turn_ended', agentId), 45_000, `turn_ended (${content})`)
+ client.send('message', { agentId, content })
+ expect((await started).payload?.userMessage).toBe(content)
+ await ended
+}
+const thread = async (client: LocalClient, sessionId: string) => {
+ const page = await client.request<{ events?: Array<{ type: string; payload: Record<string, any> }>; error?: string }>('session_get', { sessionId, limit: 200 }, 30_000)
+ expect(page.error, JSON.stringify(page)).toBeUndefined()
+ const events = page.events ?? []
+ return {
+ users: events.filter((event) => event.type === 'user_message').map((event) => String(event.payload.content)),
+ answers: events.filter((event) => event.type === 'text_delta').map((event) => String(event.payload.content)),
+ }
+}
+/** The version of the engine the agent runs now, asked through the daemon and answered in its transcript. */
+async function runningVersion(client: LocalClient, agentId: string): Promise<string> {
+ await turn(client, agentId, '!version')
+ const { sessionId } = (await row(client, agentId))!
+ return until('the version answer in the thread', async () => {
+ const last = (await thread(client, sessionId)).answers.at(-1)
+ return last && /Claude Code|codex-cli/.test(last) ? last : null
+ }, 15_000, 250)
+}
+
+/** What tmux shows in a pane now, or '' once the pane is gone. */
+const screen = (d: IsolatedDaemon, pane: unknown): Promise<string> =>
+ typeof pane === 'string' && pane ? d.capture(pane).catch(() => '') : Promise.resolve('')
+const errorsFor = (client: LocalClient, agentId: string, from: number) =>
+ client.frames.slice(from).filter((frame) => frame.type === 'error' && frame.agentId === agentId).map((frame) => String(frame.payload?.message))
+
+describe('engines updated in place', () => {
+ let daemon: IsolatedDaemon | undefined
+ afterEach(async () => { await daemon?.close(); daemon = undefined })
+ const fresh = async () => {
+ const d = await IsolatedDaemon.create()
+ daemon = d
+ onTestFailed(() => {
+ console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`)
+ // The runner swallows console output: UPDATES_LOG=<file> keeps the daemon's whole log for reading.
+ if (process.env.UPDATES_LOG) writeFileSync(process.env.UPDATES_LOG, d.log())
+ })
+ await d.start()
+ return d
+ }
+
+ it.each(engines)('%s: replaced by a newer version under a running agent: the agent goes on, and a restart and a new agent run the new one', async (engine) => {
+ const d = await fresh()
+ const client = await LocalClient.connect(d)
+ const agent = await create(d, client, engine, `updated-${engine}`)
+ expect(await runningVersion(client, agent.id)).toBe(versionLine(engine, OLD[engine]))
+
+ install(d, engine, { version: NEW[engine] })
+ // Through a reap and a reconcile pass (every 5s here) with the file the process started from
+ // replaced on disk: the daemon must go on seeing the running engine as the agent's.
+ await sleep(7_000)
+ expect((await row(client, agent.id))?.status).toBe('active')
+ expect(await runningVersion(client, agent.id)).toBe(versionLine(engine, OLD[engine]))
+ await turn(client, agent.id, 'still working after the update')
+
+ const restarted = await client.request('agent_restart', { agentId: agent.id }, 90_000)
+ expect(restarted.error, JSON.stringify(restarted)).toBeUndefined()
+ expect(restarted.resumed).toBe(true)
+ await bound(client, agent.id, agent.sessionId)
+ expect(await runningVersion(client, agent.id)).toBe(versionLine(engine, NEW[engine]))
+
+ const next = await create(d, client, engine, `after-the-update-${engine}`)
+ expect(await runningVersion(client, next.id)).toBe(versionLine(engine, NEW[engine]))
+ expect(d.coresStarted()).toBe(1)
+ client.close()
+ })
+
+ it('claude: an update drops --fork-session after forks worked: the fork fails at once, its refusal reaches the window, and the source goes on', async () => {
+ const d = await fresh()
+ const client = await LocalClient.connect(d)
+ const source = await create(d, client, 'claude', 'fork-after-update')
+ await turn(client, source.id, 'before the update')
+ const first = await client.request('agent_fork', { agentId: source.id, name: 'before the update' }, 90_000)
+ expect(first.error, JSON.stringify(first)).toBeUndefined()
+ await bound(client, first.agent.id)
+
+ install(d, 'claude', { version: NEW.claude, without: ['--fork-session'] })
+ // The daemon does not read `--fork-session` from the help before it forks (only the permission flag
+ // is checked), so it launches the fork with it and the new release refuses it.
+ const startedAt = Date.now()
+ const forked = await client.request('agent_fork', { agentId: source.id, name: 'after the update' }, 90_000)
+ expect(Date.now() - startedAt).toBeLessThan(15_000)
+ if (!forked.error) {
+ const pane = forked.agent.tmuxPane
+ // What the daemon does today: the engine's process is there for an instant, the launch reads as
+ // ready, then the reconciler finds no engine and keeps the conversation-less row as stopped,
+ // giving the shell left in the pane a row of its own. Or, when the watcher misses that instant,
+ // the row itself becomes that terminal. Either way the refusal is on a terminal the window has.
+ const settled = await until('the fork to settle', async () => {
+ const now = await row(client, forked.agent.id)
+ return now && now.launch?.state !== 'starting' && !(now.engine === 'claude' && now.status === 'active') ? now : null
+ }, 30_000, 250)
+ expect(settled.sessionId || null).toBeNull()
+ const shell = (await rows(client)).find((agent) => agent.engine === 'terminal' && agent.status === 'active' && agent.tmuxPane === pane)
+ expect(shell, `a terminal row for the fork's pane ${pane}: ${JSON.stringify(await rows(client))}`).toBeTruthy()
+ expect(await screen(d, pane)).toContain("error: unknown option '--fork-session'")
+ } else {
+ expect(typeof forked.detail).toBe('string')
+ }
+ // The source is untouched: it goes on in its own conversation.
+ await turn(client, source.id, 'the source after the failed fork')
+ expect((await row(client, source.id))?.sessionId).toBe(source.sessionId)
+ expect(d.coresStarted()).toBe(1)
+ client.close()
+ })
+
+ it('codex: an update drops resume: a restart comes back on a new conversation and says so, the one it left is kept, and a resume says why it cannot', async () => {
+ const d = await fresh()
+ const client = await LocalClient.connect(d)
+ const agent = await create(d, client, 'codex', 'resume-after-update')
+ await turn(client, agent.id, 'before the update')
+ install(d, 'codex', { version: NEW.codex, without: ['resume'] })
+
+ // Restart tries `codex resume <id>`, sees no engine come up, and falls back to a fresh launch in the
+ // same pane: a working agent on a new conversation, with `resumed: false` to say so.
+ let startedAt = Date.now()
+ const restarted = await client.request('agent_restart', { agentId: agent.id }, 90_000)
+ expect(Date.now() - startedAt).toBeLessThan(45_000)
+ expect(restarted.error, JSON.stringify(restarted)).toBeUndefined()
+ expect(restarted.resumed).toBe(false)
+ const renewed = await until('the agent on its new conversation', async () => {
+ const now = await row(client, agent.id)
+ return now?.status === 'active' && now.sessionId && now.sessionId !== agent.sessionId ? now : null
+ }, 30_000, 250)
+ expect(await runningVersion(client, agent.id)).toBe(versionLine('codex', NEW.codex))
+ // The conversation the restart had to leave is kept as a stopped one of its own: listed, and its
+ // history readable.
+ const kept = await until('the conversation it left, kept as stopped', async () =>
+ (await rows(client)).find((one) => one.status === 'stopped' && one.sessionId === agent.sessionId), 15_000, 250)
+ expect(kept.id).not.toBe(agent.id)
+ expect((await thread(client, agent.sessionId)).users).toEqual(['before the update'])
+
+ expect((await client.request('agent_delete', { agentId: agent.id }, 60_000)).error).toBeUndefined()
+ await until('the agent to stop', async () => (await row(client, agent.id))?.status === 'stopped' || null, 45_000, 500)
+ startedAt = Date.now()
+ const resumed = await client.request('agent_resume', { agentId: agent.id }, 90_000)
+ expect(Date.now() - startedAt).toBeLessThan(30_000)
+ expect(resumed.error, JSON.stringify(resumed)).toBe('RESUME_FAILED')
+ expect(String(resumed.detail)).toContain('exited before confirming the saved conversation')
+ expect((await row(client, agent.id))?.sessionId).toBe(renewed.sessionId)
+
+ // A release that has resume again opens the kept conversation where it was.
+ install(d, 'codex', { version: '0.161.0' })
+ const reopened = await client.request('agent_resume', { agentId: kept.id }, 90_000)
+ expect(reopened.error, JSON.stringify(reopened)).toBeUndefined()
+ await bound(client, kept.id, agent.sessionId)
+ await turn(client, kept.id, 'back in the kept conversation')
+ expect((await thread(client, agent.sessionId)).users).toEqual(['before the update', 'back in the kept conversation'])
+ expect(d.coresStarted()).toBe(1)
+ client.close()
+ })
+
+ it.each([['claude', '--permission-mode'], ['codex', '--approve-for-me']] as const)('%s: an update drops %s after the daemon saw it supported: a new agent in Auto is refused with the reason, with no daemon restart', async (engine, flag) => {
+ const d = await fresh()
+ const client = await LocalClient.connect(d)
+ // Creating an agent in Auto reads the flag from the engine's help, and the daemon keeps the answer.
+ const first = await create(d, client, engine, `auto-before-${engine}`)
+ await turn(client, first.id, 'before the update')
+ install(d, engine, { version: NEW[engine], without: [flag] })
+
+ const cwd = join(d.projectsDir, `auto-after-${engine}`)
+ mkdirSync(cwd, { recursive: true })
+ const startedAt = Date.now()
+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
+ expect(Date.now() - startedAt).toBeLessThan(15_000)
+ // The answer it kept was about the file the update replaced, so the engine is asked again, and the
+ // launch is refused with the reason before any pane opens, as a daemon that never saw the flag
+ // refuses it. Before #773 it went on trusting the old answer until it was restarted: the engine
+ // refused the flag in a pane, and the window was shown a harness that started and stopped.
+ expect(created.error, JSON.stringify(created)).toBe('CODEX_CLI_TOO_OLD')
+ expect(String(created.detail)).toContain(flag)
+ expect((await rows(client)).filter((agent) => agent.status === 'active').map((agent) => agent.id)).toEqual([first.id])
+ // The running agent is untouched.
+ await turn(client, first.id, 'the first agent after the update')
+ expect(d.coresStarted()).toBe(1)
+ client.close()
+ })
+
+ it.each(engines)('%s: uninstalled while an agent runs: it goes on; a new agent and a restart fail at once with the reason; reinstalled, both work with no daemon restart', async (engine) => {
+ const d = await fresh()
+ const client = await LocalClient.connect(d)
+ const agent = await create(d, client, engine, `uninstalled-${engine}`)
+ await turn(client, agent.id, 'before the uninstall')
+ uninstall(d, engine)
+ // Through a reap and a reconcile pass with the engine's file gone from disk.
+ await sleep(7_000)
+ expect((await row(client, agent.id))?.status).toBe('active')
+ await turn(client, agent.id, 'after the uninstall')
+
+ const cwd = join(d.projectsDir, `missing-${engine}`)
+ mkdirSync(cwd, { recursive: true })
+ let startedAt = Date.now()
+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
+ const failed = await until('the launch to fail with its reason', async () => {
+ const now = created.agent?.id ? await row(client, created.agent.id) : undefined
+ return now?.launch?.state === 'failed' ? now : null
+ }, 30_000, 250)
+ expect(Date.now() - startedAt).toBeLessThan(10_000)
+ expect(failed.launch.error).toBe('ENGINE_NOT_INSTALLED')
+ expect(failed.launch.detail).toContain(`${engine} is not installed`)
+
+ // A restart cannot bring the agent back on an engine that is not there, so it is refused before the
+ // running agent is touched, with the reason, the way the agent's folder is checked first.
+ startedAt = Date.now()
+ const restarted = await client.request('agent_restart', { agentId: agent.id }, 90_000)
+ expect(Date.now() - startedAt).toBeLessThan(10_000)
+ expect(restarted.error, JSON.stringify(restarted)).toBe('ENGINE_NOT_INSTALLED')
+ expect(String(restarted.detail)).toContain(`${engine} is not installed`)
+ await turn(client, agent.id, 'after the refused restart')
+
+ install(d, engine, { version: NEW[engine] })
+ const next = await create(d, client, engine, `reinstalled-${engine}`)
+ expect(await runningVersion(client, next.id)).toBe(versionLine(engine, NEW[engine]))
+ // The first agent restarts onto the engine installed again, in its own conversation.
+ const back = await client.request('agent_restart', { agentId: agent.id }, 90_000)
+ expect(back.error, JSON.stringify(back)).toBeUndefined()
+ expect(back.resumed).toBe(true)
+ await bound(client, agent.id, agent.sessionId)
+ expect(await runningVersion(client, agent.id)).toBe(versionLine(engine, NEW[engine]))
+ expect((await thread(client, agent.sessionId)).users).toContain('after the refused restart')
+ expect(d.coresStarted()).toBe(1)
+ client.close()
+ })
+
+ it.each(engines)('%s: slow to start after an update: a new agent binds, and a message sent while it starts is taken once', async (engine) => {
+ const d = await fresh()
+ const client = await LocalClient.connect(d)
+ install(d, engine, { version: NEW[engine], startDelayMs: 6_000 })
+ const cwd = join(d.projectsDir, `slow-${engine}`)
+ mkdirSync(cwd, { recursive: true })
+ const from = client.frames.length
+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
+ expect(created.error, JSON.stringify(created)).toBeUndefined()
+ const id: string = created.agent.id
+ // Sent once the engine's process is there and before it has drawn or announced anything.
+ const starting = await until('the engine process, not yet its conversation', async () => {
+ const now = await row(client, id)
+ return now?.launch?.state === 'ready' ? now : null
+ }, 30_000, 100)
+ expect(starting.sessionId || null).toBeNull()
+ const content = 'sent while it was starting'
+ client.send('message', { agentId: id, content })
+ const agent = await bound(client, id)
+ await client.waitFor((frame) => isTurn('turn_ended', id)(frame) && client.frames.indexOf(frame) >= from, 45_000, 'the message to be answered')
+ // Long enough for every check and retry the daemon makes on a message it could not see taken.
+ await sleep(12_000)
+ const starts = client.frames.slice(from).filter(isTurn('turn_started', id)).map((frame) => frame.payload?.userMessage)
+ expect(starts).toEqual([content])
+ expect((await thread(client, agent.sessionId)).users).toEqual([content])
+ expect(errorsFor(client, id, from)).toEqual([])
+ expect(await runningVersion(client, id)).toBe(versionLine(engine, NEW[engine]))
+ client.close()
+ })
+})
diff --git a/cli/src/architecture.spec.ts b/cli/src/architecture.spec.ts
index 057c23a17..2250ee2b2 100644
--- a/cli/src/architecture.spec.ts
+++ b/cli/src/architecture.spec.ts
@@ -61,7 +61,7 @@ function runForegroundLines(): number {
* when you move code out; raising one needs a reason a reviewer agrees with, and the usual one is wrong:
* the code belongs in a module or a service.
*/
-const RUN_FOREGROUND_BUDGET = 2_690
+const RUN_FOREGROUND_BUDGET = 2_695
const BACKEND_SOCKET_BUDGET = 2_975
/** Exceptions, each with its reason. Keep this short. */
diff --git a/cli/src/backendSocket.history.spec.ts b/cli/src/backendSocket.history.spec.ts
index d61e4d7eb..94f4e8aa5 100644
--- a/cli/src/backendSocket.history.spec.ts
+++ b/cli/src/backendSocket.history.spec.ts
@@ -5,6 +5,7 @@ import { BackendSocket } from './backendSocket.js'
import { codexMessagesToEvents, windowCodexLines } from './engines/codex/normalizer.js'
import { messagesToEvents, subagentStatsFromRawLines, windowRawLines } from './lib/normalize.js'
import { registry } from './lib/registry.js'
+import { stoppedAgents } from './lib/stoppedAgents.js'
import { tailFile } from './lib/transcriptTail.js'
import { bindHistory } from './testing/socketHistory.js'
import { cl, claude, claudeScenario, codexScenario } from './testing/transcriptScenarios.js'
@@ -68,6 +69,20 @@ describe.each([
expect(registry.resolve(sessionId)?.transcriptPath).toBe(file)
})
+ it('is served from a conversation kept as a stopped one, once nothing live holds it', async () => {
+ const whole = (await ask('session_get', { sessionId })).events
+ // What a stop, or a relaunch that had to leave the conversation for a new one, keeps of it.
+ stoppedAgents.save(registry.resolve(sessionId)!)
+ registry.removeAgent(agentId)
+ try {
+ expect(registry.resolve(sessionId)).toBeUndefined()
+ expect((await ask('session_get', { sessionId })).events).toEqual(whole)
+ } finally {
+ stoppedAgents.remove(agentId)
+ }
+ expect((await ask('session_get', { sessionId })).error).toBe('NOT_FOUND')
+ })
+
it('pages back through the thread exactly as the whole-file read did, cursor for cursor', async () => {
const lines = await tailFile(file, Infinity)
for (const limit of [1, 4, 500]) {
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index d4f17164f..723c328c3 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -92,6 +92,7 @@ import { TmuxBackend } from './lib/tmuxBackend.js'
import { DEFAULT_HOST_THEME, loadHostTheme, saveHostTheme, type HostTheme } from './lib/hostTheme.js'
import { restoreAgents } from './lib/restoreAgents.js'
import { createRetainExitedSession } from './lib/retainExitedSession.js'
+import { createKeepAbandonedConversation } from './lib/keepAbandonedConversation.js'
import { OpenTabProtection } from './lib/openTabProtection.js'
import { sessionCheckpoints } from './lib/sessionCheckpoint.js'
import { repairClaudeCwd } from './lib/cwdRepair.js'
@@ -2096,6 +2097,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
// A conversation's history, a page at a time, for a window that asks (core/transcripts/history.ts).
backend.historyProvider = createHistory({
resolve: (id) => registry.resolve(id),
+ stopped: () => stoppedAgents.list(),
pages: backend.transcriptPages,
dbs: { opencode: OPENCODE_DB, kilo: KILO_DB, devin: DEVIN_DB },
hermesDb: (s) => hermesDbForSession(s),
@@ -2367,6 +2369,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
syncRecapPool,
warn: (message, error) => console.warn(message, error),
})
+ const keepAbandonedConversation = createKeepAbandonedConversation({ stoppedAgents, publishStoppedAgent: (saved) => backend.publishStoppedAgent(saved) })
@@ -3379,6 +3382,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
let paneInventory: ReturnType<typeof listTmuxPanes> | null = null
const summary = await restoreAgents({
retainStopped: retainExitedSession,
+ keepAbandoned: keepAbandonedConversation,
registry,
// "Alive" means the pane still runs THIS row's engine — not merely that tmux knows the id.
// A new tmux server hands out `%N` from zero again, so a stale id can name someone's shell;
@@ -3652,6 +3656,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
byAgent: (agentId) => registry.byAgent(agentId),
tmuxBackend,
prepareSessionResume,
+ keepAbandonedConversation,
})
const restartJobs = paneSwap.restartJobs
const sameRestartTarget = paneSwap.sameRestartTarget
@@ -3698,6 +3703,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
prepareSessionResume,
refreshGridWebSearch,
attachDsh,
+ attachSession: (session) => attachSession(session),
relaunchMarks,
})
const stopJobs = lifecycle.stopJobs
diff --git a/cli/src/core/agents/lifecycle.spec.ts b/cli/src/core/agents/lifecycle.spec.ts
index dce8309e9..f6aa2f0ab 100644
--- a/cli/src/core/agents/lifecycle.spec.ts
+++ b/cli/src/core/agents/lifecycle.spec.ts
@@ -46,6 +46,7 @@ function setup(over: Partial<LifecycleDeps> = {}) {
prepareSessionResume: vi.fn(),
refreshGridWebSearch: vi.fn(),
attachDsh: vi.fn(),
+ attachSession: vi.fn(async () => true),
...over,
}
const lifecycle = createAgentLifecycle(deps)
@@ -69,6 +70,7 @@ describe('stopping, purging and resuming an agent', () => {
stopJobs: lifecycle.stopJobs, pinnedControls: deps.pinnedControls, retainExitedSession: deps.retainExitedSession,
announceSession: deps.announceSession, relaunchOverrides: deps.relaunchOverrides, prepareSessionResume: deps.prepareSessionResume,
refreshGridWebSearch: deps.refreshGridWebSearch, clearDeleted: deps.clearDeleted, attachDsh: deps.attachDsh,
+ attachSession: deps.attachSession,
})
})
diff --git a/cli/src/core/agents/lifecycle.ts b/cli/src/core/agents/lifecycle.ts
index 9d85815ca..17d3890b7 100644
--- a/cli/src/core/agents/lifecycle.ts
+++ b/cli/src/core/agents/lifecycle.ts
@@ -37,13 +37,14 @@ export interface LifecycleDeps {
prepareSessionResume: ResumeAgentServiceDeps['prepareSessionResume']
refreshGridWebSearch: ResumeAgentServiceDeps['refreshGridWebSearch']
attachDsh: ResumeAgentServiceDeps['attachDsh']
+ attachSession: ResumeAgentServiceDeps['attachSession']
relaunchMarks?: ResumeAgentServiceDeps['relaunchMarks']
}
export function createAgentLifecycle({
registry, stoppedAgents, restartJobs, tmuxBackend, agentReconciler, forgetSession, markDeleted, clearDeleted,
sessionCheckpoints, mirror, sessionSearch, send, pinnedControls, retainExitedSession, announceSession,
- relaunchOverrides, prepareSessionResume, refreshGridWebSearch, attachDsh, relaunchMarks,
+ relaunchOverrides, prepareSessionResume, refreshGridWebSearch, attachDsh, attachSession, relaunchMarks,
}: LifecycleDeps) {
/**
* Stop Harness (`agent_delete`) archives its conversation and launch settings, removes the live
@@ -69,7 +70,7 @@ export function createAgentLifecycle({
const resume = createResumeAgentService({
registry, stoppedAgents, tmuxBackend, restartJobs, stopJobs, pinnedControls,
retainExitedSession, announceSession, relaunchOverrides, prepareSessionResume,
- refreshGridWebSearch, clearDeleted, attachDsh, relaunchMarks,
+ refreshGridWebSearch, clearDeleted, attachDsh, attachSession, relaunchMarks,
})
const resumeAgent: ResumeAgent = (id, permissionMode) => purgeAgentService.busy(id) || purgeAgentService.blocksFolder(stoppedAgents.get(id)?.cwd)
? Promise.resolve({ ok: false, error: 'AGENT_BUSY' }) : resume(id, permissionMode)
diff --git a/cli/src/core/agents/restart.spec.ts b/cli/src/core/agents/restart.spec.ts
index a89464d91..85bd6604f 100644
--- a/cli/src/core/agents/restart.spec.ts
+++ b/cli/src/core/agents/restart.spec.ts
@@ -1,6 +1,8 @@
import { homedir } from 'node:os'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
-import { buildEngineLaunchArgv } from '../../lib/engineLaunch.js'
+import { engineBin, enginePathOverride } from '../../lib/engineBin.js'
+import { engineInstallRecipe } from '../../lib/engineInstall.js'
+import { buildEngineLaunchArgv, commandAvailableInInteractiveShell } from '../../lib/engineLaunch.js'
import { probeGatewayRuntime } from '../../lib/gatewayRuntime.js'
import { probeGridAssignment } from '../../lib/gridAssignment.js'
import type { RegisteredSession } from '../../lib/registry.js'
@@ -10,7 +12,12 @@ import { clearPaneRemainOnExit } from '../../lib/tmux.js'
import { workspaceMissing } from '../../lib/workspaceCheck.js'
import { createAgentRestarter, type RestartDeps } from './restart.js'
-vi.mock('../../lib/engineLaunch.js', async (real) => ({ ...await real<object>(), buildEngineLaunchArgv: vi.fn(() => ['zsh', '-l']) }))
+vi.mock('../../lib/engineBin.js', async (real) => ({ ...await real<object>(), enginePathOverride: vi.fn(() => undefined) }))
+vi.mock('../../lib/engineLaunch.js', async (real) => ({
+ ...await real<object>(),
+ buildEngineLaunchArgv: vi.fn(() => ['zsh', '-l']),
+ commandAvailableInInteractiveShell: vi.fn(async () => true),
+}))
vi.mock('../../lib/gatewayRuntime.js', async (real) => ({ ...await real<object>(), probeGatewayRuntime: vi.fn(async () => ({ kind: 'none' })) }))
vi.mock('../../lib/gridAssignment.js', async (real) => ({ ...await real<object>(), probeGridAssignment: vi.fn(async () => undefined) }))
vi.mock('../../lib/restartAgent.js', async (real) => ({
@@ -98,6 +105,43 @@ describe('restarting an agent', () => {
expect(restartAgent).not.toHaveBeenCalled()
})
+ it('for an engine uninstalled since the agent started, before anything is touched, and the agent goes on', async () => {
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+ vi.mocked(commandAvailableInInteractiveShell).mockResolvedValueOnce(false)
+ const run = setup()
+ expect(await run.restart('a1')).toEqual({
+ ok: false, error: 'ENGINE_NOT_INSTALLED', detail: 'claude is not installed. Install it, then restart again. Nothing was stopped.',
+ })
+ // Asked as create asks: the launch's own command, and the install recipe unless a path was set.
+ expect(commandAvailableInInteractiveShell).toHaveBeenCalledWith(engineBin('claude'), undefined, engineInstallRecipe('claude'))
+ expect(console.warn).toHaveBeenCalledWith('[restart] a1 refused · claude is not installed')
+ expect(run.deps.relaunchOverrides).not.toHaveBeenCalled()
+ expect(run.deps.agentReconciler.holdRoute).not.toHaveBeenCalled()
+ expect(restartAgent).not.toHaveBeenCalled()
+
+ vi.mocked(enginePathOverride).mockReturnValueOnce('/opt/claude')
+ vi.mocked(commandAvailableInInteractiveShell).mockResolvedValueOnce(false)
+ expect(await setup().restart('a1')).toMatchObject({ ok: false, error: 'ENGINE_NOT_INSTALLED' })
+ expect(vi.mocked(commandAvailableInInteractiveShell).mock.calls[1][2]).toBeUndefined()
+ })
+
+ it('but not on a no from a shell that gave up: a slow rc file is not a missing engine', async () => {
+ // The check starts at 0, and the shell's no comes at 5s, when the probe gives up waiting for it.
+ const clock = vi.spyOn(Date, 'now').mockReturnValue(0)
+ vi.mocked(commandAvailableInInteractiveShell).mockImplementationOnce(async () => { clock.mockReturnValue(5_000); return false })
+ const run = setup()
+ expect(await run.restart('a1')).toEqual({ ok: true, session: agent(), resumed: true })
+ expect(restartAgent).toHaveBeenCalled()
+ })
+
+ it('and stops, touching nothing, when the operation is overtaken while the engine is looked for', async () => {
+ const run = setup()
+ vi.mocked(commandAvailableInInteractiveShell).mockImplementationOnce(async () => { run.state.live = false; return false })
+ expect(await run.restart('a1')).toEqual(changed)
+ expect(run.deps.relaunchOverrides).not.toHaveBeenCalled()
+ expect(restartAgent).not.toHaveBeenCalled()
+ })
+
it('for an engine with no process to replace, or a launch it cannot honour', async () => {
expect(await setup(agent({ processIdentity: undefined } as Partial<RegisteredSession>)).restart('a1')).toEqual({ ok: false, error: 'NO_ACTIVE_PROCESS' })
const refused = setup(agent(), { relaunchOverrides: vi.fn(async () => ({ ok: false, error: 'API_UNAVAILABLE', detail: 'removed' })) as never })
diff --git a/cli/src/core/agents/restart.ts b/cli/src/core/agents/restart.ts
index fe8548040..ae0657644 100644
--- a/cli/src/core/agents/restart.ts
+++ b/cli/src/core/agents/restart.ts
@@ -5,8 +5,10 @@
*/
import { homedir } from 'node:os'
import type { BackendSocket } from '../../backendSocket.js'
-import { isTerminalEngine } from '../../engines/types.js'
-import { buildEngineLaunchArgv } from '../../lib/engineLaunch.js'
+import { isTerminalEngine, type AgentEngine } from '../../engines/types.js'
+import { engineBin, enginePathOverride } from '../../lib/engineBin.js'
+import { engineInstallRecipe } from '../../lib/engineInstall.js'
+import { buildEngineLaunchArgv, commandAvailableInInteractiveShell } from '../../lib/engineLaunch.js'
import { probeGatewayRuntime } from '../../lib/gatewayRuntime.js'
import { probeGridAssignment } from '../../lib/gridAssignment.js'
import { sid } from '../../lib/log.js'
@@ -23,6 +25,24 @@ import type { PaneSwap } from './swap.js'
type LaunchHelpers = ReturnType<typeof createLaunchHelpers>
+/**
+ * Longer than any shell that answered: `commandAvailableInInteractiveShell` gives up after 5s and then
+ * says no. A no that took that long is a slow rc file, not a missing engine, and must not refuse a
+ * restart that would work — the same rule `commandSupportsFlagInInteractiveShell` keeps for `unknown`.
+ */
+const ENGINE_CHECK_GAVE_UP_MS = 4_500
+
+/**
+ * Whether the engine's command is on this machine, asked as create asks of a pane that died: the same
+ * interactive shell, the same command, the same install recipe. Null when the shell gave up first.
+ */
+async function engineInstalled(engine: AgentEngine): Promise<boolean | null> {
+ const startedAt = Date.now()
+ const installed = await commandAvailableInInteractiveShell(engineBin(engine), undefined,
+ enginePathOverride(engine) ? undefined : engineInstallRecipe(engine))
+ return installed || Date.now() - startedAt < ENGINE_CHECK_GAVE_UP_MS ? installed : null
+}
+
type RestartAgentHandler = NonNullable<BackendSocket['onRestartAgent']>
export interface RestartDeps {
@@ -119,6 +139,17 @@ export function createAgentRestarter({
}
if (!session.processIdentity) return { ok: false, error: 'NO_ACTIVE_PROCESS' }
+ // The relaunch runs the engine's command again, so an engine uninstalled since the agent started
+ // cannot come back. Asked anyway, the swap killed the working agent and gave up twenty seconds
+ // later with "did not come back up" (updates.e2e.ts, round 24). Asked here, before anything is
+ // touched, the agent goes on running and the person is told why.
+ const installed = await engineInstalled(engine)
+ if (!current()) return changed
+ if (installed === false) {
+ console.warn(`[restart] ${sid(session.agentId)} refused · ${engine} is not installed`)
+ return { ok: false, error: 'ENGINE_NOT_INSTALLED', detail: `${engine} is not installed. Install it, then restart again. Nothing was stopped.` }
+ }
+
// The replacement is launched WITH what the original was: its grid's env and argv (a bare
// respawn would inherit the tmux session's variables but never the codex `-c …` / pi `--model`
// half, and an agent moved here by a retarget has nothing in the session env at all), or its
diff --git a/cli/src/core/agents/swap.spec.ts b/cli/src/core/agents/swap.spec.ts
index e8adf607e..3d10a71db 100644
--- a/cli/src/core/agents/swap.spec.ts
+++ b/cli/src/core/agents/swap.spec.ts
@@ -29,6 +29,7 @@ function setup(row: RegisteredSession | null = session()) {
byAgent: vi.fn(() => row ?? undefined),
tmuxBackend: tmuxBackend as unknown as PaneSwapDeps['tmuxBackend'],
prepareSessionResume: vi.fn(),
+ keepAbandonedConversation: vi.fn(),
}
return { deps, tmuxBackend, swap: createPaneSwap(deps) }
}
@@ -54,6 +55,8 @@ describe('the pane-process swap', () => {
const swapDeps = swap.paneSwapDeps(session(), runtime, { env: { GRID_KEY: 'k' } })
swapDeps.prepareResume?.()
expect(deps.prepareSessionResume).toHaveBeenCalledWith(session())
+ swapDeps.keepAbandoned?.()
+ expect(deps.keepAbandonedConversation).toHaveBeenCalledWith(session())
expect(await swapDeps.holdOpen()).toEqual({ ok: true })
tmuxBackend.holdOpen.mockResolvedValueOnce({ state: 'failed', reason: 'pane gone' } as never).mockResolvedValueOnce({ state: 'unknown' } as never)
expect(await swapDeps.holdOpen()).toEqual({ ok: false, reason: 'pane gone' })
diff --git a/cli/src/core/agents/swap.ts b/cli/src/core/agents/swap.ts
index 9135a5d3b..f8541524b 100644
--- a/cli/src/core/agents/swap.ts
+++ b/cli/src/core/agents/swap.ts
@@ -20,9 +20,11 @@ export interface PaneSwapDeps {
/** The tmux backend; a swap only ever runs where there is one. */
tmuxBackend: TmuxBackend | null
prepareSessionResume: (session: RegisteredSession) => void
+ /** Keeps a conversation a swap had to leave for a new one as a stopped harness (lib/keepAbandonedConversation.ts). */
+ keepAbandonedConversation: (left: RegisteredSession) => void
}
-export function createPaneSwap({ byAgent, tmuxBackend, prepareSessionResume }: PaneSwapDeps) {
+export function createPaneSwap({ byAgent, tmuxBackend, prepareSessionResume, keepAbandonedConversation }: PaneSwapDeps) {
/**
* The dependencies a pane-process swap needs, for both callers that do one.
*
@@ -48,6 +50,8 @@ export function createPaneSwap({ byAgent, tmuxBackend, prepareSessionResume }: P
permissionMode: string | null = session.permissionMode ?? null,
): RestartAgentDeps => ({
prepareResume: () => prepareSessionResume(session),
+ // The row as the swap found it: the conversation a fallback to a fresh start leaves behind.
+ keepAbandoned: () => keepAbandonedConversation(session),
holdOpen: async () => {
const result = await tmuxBackend!.holdOpen(runtime)
return result.state === 'succeeded'
diff --git a/cli/src/core/engines/hooks.spec.ts b/cli/src/core/engines/hooks.spec.ts
index 07c343c00..14a12d8da 100644
--- a/cli/src/core/engines/hooks.spec.ts
+++ b/cli/src/core/engines/hooks.spec.ts
@@ -5,7 +5,7 @@ import { processRows } from '../../lib/terminalAgentDiscovery.js'
import { dirname } from 'node:path'
import { env } from '../../config/env.js'
import { adoptEngineHomes, movedEngineHomes } from '../../lib/engineHomes.js'
-import { createEngineHooks, installEngineHooks, type EngineHookDeps } from './hooks.js'
+import { createEngineHooks, installEngineHooks, PROCESS_RECORD_WAIT_MS, type EngineHookDeps } from './hooks.js'
vi.mock('../../lib/engineHomes.js', () => ({
adoptEngineHomes: vi.fn(() => ({ claude: null, codex: null })),
@@ -85,11 +85,19 @@ describe('which agent a hook belongs to', () => {
+ ' · callerPid=300 · caller is not a descendant of that engine process')
})
- it('none for an agent with no process to descend from, and a process table that loops', async () => {
- vi.mocked(processRows).mockResolvedValueOnce(tree([300, 400], [400, 300]) as never)
- const { hooks } = setup({ '%1': agent('no-pid'), '%2': agent('a2', 100) })
- expect(await hooks.resolveHookAgent({ engine: 'claude', runtimeHints: [hint('%1'), hint('%2')], callerPid: 300 })).toBeNull()
- expect(console.log).toHaveBeenCalledWith(expect.stringContaining('agentsOnRuntime=2 · callerPid=300 · caller is not a descendant'))
+ it('none for an agent with no process to descend from once the wait for one is over, and a process table that loops', async () => {
+ vi.useFakeTimers()
+ try {
+ vi.mocked(processRows).mockResolvedValue(tree([300, 400], [400, 300], [100, 1]) as never)
+ const { hooks } = setup({ '%1': agent('no-pid'), '%2': agent('a2', 100) })
+ const answer = hooks.resolveHookAgent({ engine: 'claude', runtimeHints: [hint('%1'), hint('%2')], callerPid: 300 })
+ await vi.advanceTimersByTimeAsync(PROCESS_RECORD_WAIT_MS)
+ expect(await answer).toBeNull()
+ expect(console.log).toHaveBeenCalledWith(expect.stringContaining('agentsOnRuntime=2 · callerPid=300 · caller is not a descendant'))
+ } finally {
+ vi.useRealTimers()
+ vi.mocked(processRows).mockReset()
+ }
})
it('none when two agents could own the hook, saying it is ambiguous', async () => {
@@ -114,6 +122,82 @@ describe('which agent a hook belongs to', () => {
expect(console.log).toHaveBeenLastCalledWith('[hooks] unmatched claude hook · hints=none · resolvedRuntimes=0 · agentsOnRuntime=0 · callerPid=300')
})
+ describe('a hook that comes in before its agent has recorded the process it came from', () => {
+ afterEach(() => { vi.mocked(processRows).mockReset() })
+ // The new engine is 200 and the hook's process, 300, is its child. Pid 100 was the engine a restart
+ // killed; nothing in the table is it any more.
+ const newEngine = () => tree([300, 200], [200, 1])
+ /** Asks, and says when the hook has started waiting: after it read the process table once. */
+ const ask = (hooks: ReturnType<typeof setup>['hooks'], panes = ['%1']) => {
+ let answer: RegisteredSession | null | undefined
+ const asked = hooks.resolveHookAgent({ engine: 'claude', runtimeHints: panes.map(hint), callerPid: 300 }).then((agent) => { answer = agent })
+ const waiting = () => vi.waitFor(() => expect(processRows).toHaveBeenCalledTimes(1))
+ return { asked, waiting, answer: () => answer }
+ }
+
+ it('in a restart, waits while the recorded process is one that is gone, and belongs to the agent once the new one is recorded', async () => {
+ const onPane: Record<string, RegisteredSession> = { '%1': agent('a1', 100) }
+ vi.mocked(processRows).mockResolvedValue(newEngine() as never)
+ const hook = ask(setup(onPane).hooks, ['%1', '%9'])
+ await hook.waiting()
+ await new Promise((resolve) => setTimeout(resolve, 250))
+ expect(hook.answer()).toBeUndefined()
+ // What restart.ts does once the new process is found and probed.
+ onPane['%1'] = agent('a1', 200)
+ await hook.asked
+ expect(hook.answer()).toEqual(agent('a1', 200))
+ expect(console.log).not.toHaveBeenCalled()
+ })
+
+ it('in a resume, waits while the row has no process yet, and belongs to it once the resume records one', async () => {
+ const onPane: Record<string, RegisteredSession> = { '%1': agent('a1') }
+ vi.mocked(processRows).mockResolvedValue(newEngine() as never)
+ const hook = ask(setup(onPane).hooks)
+ await hook.waiting()
+ expect(hook.answer()).toBeUndefined()
+ // What resumeAgentService.ts does once waitForResumedAgent finds the engine in the pane.
+ onPane['%1'] = agent('a1', 200)
+ await hook.asked
+ expect(hook.answer()).toEqual(agent('a1', 200))
+ })
+
+ it('none when what is recorded is a process it does not come from, or the table cannot be read again', async () => {
+ const onPane: Record<string, RegisteredSession> = { '%1': agent('a1') }
+ vi.mocked(processRows).mockResolvedValue(newEngine() as never)
+ const other = ask(setup(onPane).hooks)
+ await other.waiting()
+ onPane['%1'] = agent('a1', 999)
+ await other.asked
+ expect(other.answer()).toBeNull()
+ expect(console.log).toHaveBeenLastCalledWith('[hooks] unmatched claude hook · hints=tmux:%1 · resolvedRuntimes=1 · agentsOnRuntime=1'
+ + ' · callerPid=300 · caller is not a descendant of that engine process')
+
+ vi.mocked(processRows).mockReset()
+ const pane: Record<string, RegisteredSession> = { '%1': agent('a1') }
+ vi.mocked(processRows).mockResolvedValueOnce(newEngine() as never).mockResolvedValueOnce(null)
+ const unreadable = ask(setup(pane).hooks)
+ await unreadable.waiting()
+ pane['%1'] = agent('a1', 200)
+ await unreadable.asked
+ expect(unreadable.answer()).toBeNull()
+ })
+
+ it('waits no longer than its bound for a record that does not come', async () => {
+ vi.useFakeTimers()
+ try {
+ vi.mocked(processRows).mockResolvedValue(newEngine() as never)
+ const hook = ask(setup({ '%1': agent('a1', 100) }).hooks)
+ await vi.advanceTimersByTimeAsync(PROCESS_RECORD_WAIT_MS - 1)
+ expect(hook.answer()).toBeUndefined()
+ await vi.advanceTimersByTimeAsync(1)
+ await hook.asked
+ expect(hook.answer()).toBeNull()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+ })
+
it('a SessionEnd only asks for a reconcile: discovery decides whether the agent still exists', () => {
const { deps, hooks } = setup()
hooks.onSessionEnd('s1', 'exit')
diff --git a/cli/src/core/engines/hooks.ts b/cli/src/core/engines/hooks.ts
index f89147216..962d84dba 100644
--- a/cli/src/core/engines/hooks.ts
+++ b/cli/src/core/engines/hooks.ts
@@ -28,21 +28,39 @@ export interface EngineHookDeps {
registry: Pick<typeof registry, 'byRuntimeEngine'>
}
-export function createEngineHooks({ tmuxBackend, agentReconciler, registry }: EngineHookDeps) {
- const resolveHookAgent: ResolveHookAgent = async ({ engine, runtimeHints, callerPid }) => {
- if (!callerPid) return null
- const resolved: TerminalRuntimeRef[] = []
- for (const hint of runtimeHints ?? []) {
- if (tmuxBackend) resolved.push({ backend: 'tmux', paneId: hint.paneId })
- }
- for (const runtime of resolved) await agentReconciler.triggerHint(runtime, engine)
+/**
+ * The longest a hook waits for the agent on its pane to record the process it came from. Every relaunch
+ * starts its engine before it records it: a resume looks for the process every 250ms, a create or a
+ * fork backs off to 750ms, and a restart finds it the same way and then probes it, twice over when it
+ * falls back to a new conversation (two 8s discovery budgets, swap.ts). Bounded all the same: the engine
+ * kills its own hook command at 5s, and an answer later than that only keeps the daemon's record.
+ */
+export const PROCESS_RECORD_WAIT_MS = 20_000
+/** How often a waiting hook reads the rows on its panes again: memory, not a process table. */
+const PROCESS_RECORD_POLL_MS = 100
+
+type HookQuery = Parameters<ResolveHookAgent>[0]
+
+/** Which agent holds a pane, and the process it has recorded: what a waiting hook watches change. */
+const recordKey = (session: RegisteredSession | undefined): string =>
+ session ? `${session.agentId}\u0000${session.processIdentity?.pid ?? ''}\u0000${session.processIdentity?.startMarker ?? ''}` : ''
+export function createEngineHooks({ tmuxBackend, agentReconciler, registry }: EngineHookDeps) {
+ /**
+ * The agents on the hinted panes: those whose recorded process the caller descends from, the rest,
+ * and of the rest those with no live process recorded at all (none yet, or one that has exited).
+ */
+ const matchCaller = async (resolved: TerminalRuntimeRef[], engine: HookQuery['engine'], callerPid: number) => {
const rows = await processRows()
if (!rows) return null
+ const byPid = new Map(rows.map((row) => [row.pid, row.parentPid]))
+ const recordedAlive = (session: RegisteredSession): boolean => {
+ const recorded = session.processIdentity
+ return !!recorded && rows.some((row) => row.pid === recorded.pid && row.startMarker === recorded.startMarker)
+ }
const callerBelongsTo = (session: RegisteredSession): boolean => {
const expectedPid = session.processIdentity?.pid
if (!expectedPid) return false
- const byPid = new Map(rows.map((row) => [row.pid, row.parentPid]))
let pid = callerPid
const visited = new Set<number>()
while (pid > 0 && !visited.has(pid)) {
@@ -66,13 +84,54 @@ export function createEngineHooks({ tmuxBackend, agentReconciler, registry }: En
* Codex (or other engine's) old transcript belongs to the new process.
*/
const onHintedRuntime = new Map<string, RegisteredSession>()
+ const unrecorded: RegisteredSession[] = []
for (const runtime of resolved) {
const candidate = registry.byRuntimeEngine(runtime, engine)
if (!candidate) continue
if (callerBelongsTo(candidate)) candidates.set(candidate.agentId, candidate)
- else onHintedRuntime.set(candidate.agentId, candidate)
+ else {
+ onHintedRuntime.set(candidate.agentId, candidate)
+ if (!recordedAlive(candidate)) unrecorded.push(candidate)
+ }
+ }
+ return { candidates, onHintedRuntime, unrecorded }
+ }
+
+ /** Resolves when the row on any of these panes records another process (or another agent takes the
+ * pane), or when the wait is over. */
+ const recordChanged = async (resolved: TerminalRuntimeRef[], engine: HookQuery['engine']): Promise<void> => {
+ const before = resolved.map((runtime) => recordKey(registry.byRuntimeEngine(runtime, engine)))
+ const deadline = Date.now() + PROCESS_RECORD_WAIT_MS
+ while (Date.now() < deadline) {
+ await new Promise((resolve) => setTimeout(resolve, PROCESS_RECORD_POLL_MS))
+ if (resolved.some((runtime, i) => recordKey(registry.byRuntimeEngine(runtime, engine)) !== before[i])) return
+ }
+ }
+
+ const resolveHookAgent: ResolveHookAgent = async ({ engine, runtimeHints, callerPid }) => {
+ if (!callerPid) return null
+ const resolved: TerminalRuntimeRef[] = []
+ for (const hint of runtimeHints ?? []) {
+ if (tmuxBackend) resolved.push({ backend: 'tmux', paneId: hint.paneId })
+ }
+ for (const runtime of resolved) await agentReconciler.triggerHint(runtime, engine)
+
+ let found = await matchCaller(resolved, engine, callerPid)
+ if (!found) return null
+ let choice = chooseHookAgent([...found.candidates.values()], [...found.onHintedRuntime.values()], engine)
+ // Every relaunch starts its engine before it records the new process — create, fork, resume, a
+ // restart and the new conversation it falls back to, a retarget — so a SessionStart in between came
+ // from a pid no agent had yet, and was dropped. A resumed Codex agent then never showed its turns
+ // (round 23), and a restart's new conversation was never bound (round 24). While the agent on the
+ // hook's pane has no live process recorded, the hook now waits for it to record one, and is matched
+ // again against that: one rule for every relaunch, with nothing for any of them to remember to do.
+ if (!choice.agent && choice.reason === 'none' && found.unrecorded.length) {
+ await recordChanged(resolved, engine)
+ found = await matchCaller(resolved, engine, callerPid)
+ if (!found) return null
+ choice = chooseHookAgent([...found.candidates.values()], [...found.onHintedRuntime.values()], engine)
}
- const choice = chooseHookAgent([...candidates.values()], [...onHintedRuntime.values()], engine)
+ const { candidates } = found
if (choice.agent) {
if (choice.reason === 'runtime') {
console.log(`[hooks] ${engine} hook accepted on runtime evidence alone`
diff --git a/cli/src/core/transcripts/history.spec.ts b/cli/src/core/transcripts/history.spec.ts
index 2d51f47af..3230b33fd 100644
--- a/cli/src/core/transcripts/history.spec.ts
+++ b/cli/src/core/transcripts/history.spec.ts
@@ -106,13 +106,14 @@ function transcript(lines: string[], name = 'session.jsonl'): string {
return file
}
-function setup(s?: RegisteredSession) {
+function setup(s?: RegisteredSession, kept: RegisteredSession[] = []) {
const pages = {
claude: vi.fn(async (_path: string, _opts: { limit?: number; before?: string }) => page([], false, null)),
codex: vi.fn(async (_path: string, _opts: { limit?: number; before?: string }) => page([], false, null)),
}
const deps = {
resolve: vi.fn((id: string) => (s && (id === s.sessionId || id === s.agentId) ? s : undefined)),
+ stopped: vi.fn(() => kept),
pages,
dbs: { opencode: '/stores/opencode.db', kilo: '/stores/kilo.db', devin: '/stores/sessions.db' },
hermesDb: vi.fn(async () => '/profiles/work/state.db'),
@@ -144,6 +145,19 @@ describe('what cannot be served', () => {
expect(pages.claude).not.toHaveBeenCalled()
})
+ it('a conversation kept as a stopped harness, once nothing live holds it, but never by its harness\'s id', async () => {
+ const live = session('claude')
+ const kept = { ...session('claude'), agentId: 'kept-agent', sessionId: 'kept-session' } as RegisteredSession
+ const { sessionGet, deps } = setup(live, [kept])
+ expect(await sessionGet({ sessionId: 'kept-session', limit: 50 })).toMatchObject({ id: 'kept-session', engine: 'claude', events: [] })
+ expect(deps.resolve).toHaveBeenCalledWith('kept-session')
+ // What is live is answered from the registry, without reading the saved ones.
+ deps.stopped.mockClear()
+ expect(await sessionGet({ sessionId: live.sessionId, limit: 50 })).toMatchObject({ id: live.sessionId })
+ expect(deps.stopped).not.toHaveBeenCalled()
+ expect(await sessionGet({ sessionId: 'kept-agent', limit: 50 })).toStrictEqual({ error: 'NOT_FOUND' })
+ })
+
it('found by agent id too, and still answers with the id it was asked for', async () => {
const s = session('claude')
const { sessionGet } = setup(s)
diff --git a/cli/src/core/transcripts/history.ts b/cli/src/core/transcripts/history.ts
index 7ff17c7ed..c5d9d1e21 100644
--- a/cli/src/core/transcripts/history.ts
+++ b/cli/src/core/transcripts/history.ts
@@ -44,6 +44,9 @@ import { streamRecords, tailFileCapped, WHOLE_READ_CAP_BYTES } from '../../lib/t
export interface HistoryDeps {
/** The registry's lookup, by agent or session id. */
resolve: (id: string) => RegisteredSession | undefined
+ /** The conversations kept as stopped harnesses (lib/stoppedAgents.ts): a stop's, an exited engine's,
+ * and one a restart, a move or a restore had to leave for a new one. */
+ stopped: () => readonly RegisteredSession[]
/** Claude Code's and Codex's pages. The socket's pager, so the line index it keeps for a transcript is
* shared with the line counts `sessions_list` answers, not built twice. */
pages: Pick<TranscriptPager, 'claude' | 'codex'>
@@ -157,7 +160,7 @@ function databasePage<M>(
}
}
-export function createHistory({ resolve, pages, dbs, hermesDb }: HistoryDeps) {
+export function createHistory({ resolve, stopped, pages, dbs, hermesDb }: HistoryDeps) {
// Devin, OpenCode and Kilo keep one store on this machine. Hermes keeps one per HOME, so its path is
// the session's own (`hermesDb`) rather than this machine's default.
const databases = new Map<string, DatabasePage>([
@@ -171,10 +174,12 @@ export function createHistory({ resolve, pages, dbs, hermesDb }: HistoryDeps) {
const sessionGet = async (payload: Record<string, unknown>): Promise<Record<string, unknown>> => {
const sessionId = payload.sessionId as string | undefined
if (!sessionId) return { error: 'MISSING_SESSION_ID' }
- // Only serve transcripts for a REGISTERED tmux session, read from its own trusted
- // transcriptPath — never resolve an arbitrary request-supplied id to a file (that let a
- // caller read any *.jsonl on the computer, incl. unshared claude history / traversal).
- const s = resolve(sessionId)
+ // Only serve transcripts for a session this daemon REGISTERED — live, or kept as a stopped one —
+ // read from its own trusted transcriptPath: never resolve an arbitrary request-supplied id to a
+ // file (that let a caller read any *.jsonl on the computer, incl. unshared claude history /
+ // traversal). A kept conversation used to answer NOT_FOUND, so a conversation the daemon had to
+ // leave for a new one could no longer be read at all (round 24).
+ const s = resolve(sessionId) ?? stopped().find((saved) => saved.sessionId === sessionId)
if (!s) return { error: 'NOT_FOUND' }
// Optional pagination: `limit` = window size; `before` = cursor (the oldest record the
// client already holds). Absent → full transcript (legacy). Clamp limit defensively.
diff --git a/cli/src/lib/keepAbandonedConversation.spec.ts b/cli/src/lib/keepAbandonedConversation.spec.ts
new file mode 100644
index 000000000..d9f975670
--- /dev/null
+++ b/cli/src/lib/keepAbandonedConversation.spec.ts
@@ -0,0 +1,71 @@
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { createKeepAbandonedConversation } from './keepAbandonedConversation.js'
+import type { RegisteredSession } from './registry.js'
+import { StoppedAgentStore } from './stoppedAgents.js'
+
+const pane = { backend: 'tmux' as const, paneId: '%4' }
+/** The agent's row as it was in the conversation it left: its conversation, pane and live process. */
+const left = (over: Partial<RegisteredSession> = {}) => ({
+ schemaVersion: 2, agentId: 'agent-1', sessionId: 'session-0', engine: 'codex', codexHome: null, cwd: '/work',
+ transcriptPath: '/work/rollout.jsonl', projectDir: 'work', runtimes: [pane], primaryRuntimeKey: '', tmuxPane: '%4',
+ processIdentity: { pid: 7, startMarker: 'Mon Oct 5 03:00:00 2026', executable: 'codex' }, active: true,
+ registeredAt: 1, touchedAt: 1, lastHookAt: 1, lastTranscriptAt: 1, boundAt: 1, ...over,
+}) as unknown as RegisteredSession
+
+describe('keeping a conversation the daemon left for a new one', () => {
+ let dir: string
+ let store: StoppedAgentStore
+ let publish: ReturnType<typeof vi.fn<(saved: RegisteredSession) => Promise<void>>>
+ beforeEach(() => {
+ dir = mkdtempSync(join(tmpdir(), 'keep-abandoned-'))
+ store = new StoppedAgentStore(join(dir, 'saved'))
+ publish = vi.fn<(saved: RegisteredSession) => Promise<void>>(async () => {})
+ vi.spyOn(console, 'log').mockImplementation(() => {})
+ })
+ afterEach(() => { vi.restoreAllMocks(); rmSync(dir, { recursive: true, force: true }) })
+
+ it('keeps it as a stopped harness of its own, without the live process, and tells the windows', () => {
+ // The agent's own record, which follows it into the new conversation: it is not the one kept.
+ store.save(left())
+ createKeepAbandonedConversation({ stoppedAgents: store, publishStoppedAgent: publish })(left())
+ const kept = store.list().find((saved) => saved.agentId !== 'agent-1')!
+ expect(kept).toMatchObject({ sessionId: 'session-0', engine: 'codex', transcriptPath: '/work/rollout.jsonl', processIdentity: null, active: false })
+ expect(publish).toHaveBeenCalledWith(expect.objectContaining({ agentId: kept.agentId, sessionId: 'session-0' }))
+ expect(console.log).toHaveBeenCalledWith(`[agent] agent-1 kept the conversation it left, session-, as stopped harness ${kept.agentId.slice(0, 8)}`)
+ })
+
+ it('keeps it once, keeps the same id under another Codex profile, and keeps nothing without a conversation', () => {
+ const keep = createKeepAbandonedConversation({ stoppedAgents: store, publishStoppedAgent: publish })
+ keep(left())
+ keep(left())
+ expect(store.list()).toHaveLength(1)
+ keep(left({ codexHome: '/profile' }))
+ expect(store.list()).toHaveLength(2)
+ keep(left({ sessionId: '' }))
+ expect(store.list()).toHaveLength(2)
+ expect(publish).toHaveBeenCalledTimes(2)
+ })
+
+ it('says why it could not, and is never in the way; a window that cannot be told is not an error', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const full = { list: () => [], get: () => null, save: vi.fn<(row: RegisteredSession) => void>(() => { throw new Error('ENOSPC') }) }
+ const keep = createKeepAbandonedConversation({ stoppedAgents: full, publishStoppedAgent: publish })
+ expect(() => keep(left())).not.toThrow()
+ expect(warn).toHaveBeenCalledWith('[agent] agent-1 could not keep the conversation it left, session- · ENOSPC')
+ full.save.mockImplementationOnce(() => { throw 'locked' })
+ keep(left())
+ expect(warn).toHaveBeenCalledWith('[agent] agent-1 could not keep the conversation it left, session- · locked')
+ // Saved, but it did not read back: nothing to tell the windows.
+ full.save.mockImplementationOnce(() => {})
+ keep(left())
+ expect(publish).not.toHaveBeenCalled()
+
+ publish.mockRejectedValueOnce(new Error('no window'))
+ createKeepAbandonedConversation({ stoppedAgents: store, publishStoppedAgent: publish })(left())
+ await new Promise((resolve) => setTimeout(resolve, 0))
+ expect(publish).toHaveBeenCalledTimes(1)
+ })
+})
diff --git a/cli/src/lib/keepAbandonedConversation.ts b/cli/src/lib/keepAbandonedConversation.ts
new file mode 100644
index 000000000..d3ed83170
--- /dev/null
+++ b/cli/src/lib/keepAbandonedConversation.ts
@@ -0,0 +1,49 @@
+/**
+ * A conversation the daemon itself left for a new one, kept as a stopped harness of its own.
+ *
+ * Three relaunches fall back to a new conversation when the engine cannot reopen the one the agent was
+ * in, so that the agent goes on working: a restart and a retarget (restartAgent.ts) and the restore after
+ * a reboot (restoreAgents.ts). The agent then follows the new conversation, and the bind that points it
+ * there also rewrites the agent's own stopped record, so the one it left was no longer listed,
+ * `session_get` could not find it, and nothing could resume it (round 24). It is kept under an identity
+ * of its own, without the process, which stays the live agent's.
+ *
+ * Only what the daemon abandoned. A /clear or /new is the person's own choice and keeps nothing: people
+ * who clear often would pile up stopped rows.
+ */
+import { randomUUID } from 'node:crypto'
+import { sid } from './log.js'
+import type { RegisteredSession } from './registry.js'
+import type { StoppedAgentStore } from './stoppedAgents.js'
+
+export interface KeepAbandonedConversationDeps {
+ stoppedAgents: Pick<StoppedAgentStore, 'save' | 'get' | 'list'>
+ /** Tells the windows of a stopped harness (BackendSocket.publishStoppedAgent). */
+ publishStoppedAgent: (saved: RegisteredSession) => Promise<void>
+}
+
+/** One conversation: the same id under another Codex profile is another one. */
+const conversation = (row: RegisteredSession) => `${row.engine}\u0000${row.codexHome ?? ''}\u0000${row.sessionId}`
+
+export function createKeepAbandonedConversation({ stoppedAgents, publishStoppedAgent }: KeepAbandonedConversationDeps) {
+ /**
+ * Keeps `left`, the agent's row as it was in the conversation it left. Saved once: a conversation
+ * already kept is not kept again, though the agent's own record does not count, since it follows the
+ * agent into its new conversation (bind.ts). Never in the way of the relaunch: a failure is only said.
+ */
+ return (left: RegisteredSession): void => {
+ if (!left.sessionId) return
+ try {
+ if (stoppedAgents.list().some((saved) => saved.agentId !== left.agentId && conversation(saved) === conversation(left))) return
+ const kept: RegisteredSession = { ...left, agentId: randomUUID(), processIdentity: null }
+ stoppedAgents.save(kept)
+ const saved = stoppedAgents.get(kept.agentId)
+ if (saved) void publishStoppedAgent(saved).catch(() => {})
+ console.log(`[agent] ${sid(left.agentId)} kept the conversation it left, ${sid(left.sessionId)}, as stopped harness ${sid(kept.agentId)}`)
+ } catch (error) {
+ console.warn(`[agent] ${sid(left.agentId)} could not keep the conversation it left, ${sid(left.sessionId)} · ${error instanceof Error ? error.message : error}`)
+ }
+ }
+}
+
+export type KeepAbandonedConversation = ReturnType<typeof createKeepAbandonedConversation>
diff --git a/cli/src/lib/restartAgent.spec.ts b/cli/src/lib/restartAgent.spec.ts
index 63ef5c2cd..32cd8536a 100644
--- a/cli/src/lib/restartAgent.spec.ts
+++ b/cli/src/lib/restartAgent.spec.ts
@@ -31,6 +31,7 @@ function deps(over: Overrides = {}): RestartAgentDeps & { calls: string[] } {
waitForProcess: async () => { calls.push('waitForProcess'); return waitForProcessImpl() },
buildArgv: (opts) => { calls.push('buildArgv'); return buildArgvImpl(opts) },
log: () => {},
+ keepAbandoned: () => { calls.push('keepAbandoned') },
}
}
@@ -103,6 +104,8 @@ describe('restartAgent', () => {
const d = deps({ buildArgv: (opts) => { argvCalls.push(opts); return ['claude'] } })
await restartAgent({ engine: 'claude', sessionId: 'sess-42' }, true, d)
expect(argvCalls).toEqual([{ bypassPermission: true, resumeSessionId: 'sess-42' }])
+ // It reopened the conversation: nothing was left behind.
+ expect(d.calls).not.toContain('keepAbandoned')
})
it('launches fresh (no resumeSessionId) when the session has none', async () => {
@@ -126,6 +129,8 @@ describe('restartAgent', () => {
{ bypassPermission: false },
])
expect(outcome).toEqual({ ok: true, processIdentity: IDENTITY, resumed: false })
+ // The conversation it could not reopen is kept, once the fresh start is up.
+ expect(d.calls.at(-1)).toBe('keepAbandoned')
// respawn + waitForProcess ran twice (once per attempt); holdOpen/terminate ran once.
expect(d.calls.filter((c) => c === 'respawn')).toHaveLength(2)
expect(d.calls.filter((c) => c === 'holdOpen')).toHaveLength(1)
@@ -136,6 +141,8 @@ describe('restartAgent', () => {
const d = deps({ waitForProcess: async () => null })
const outcome = await restartAgent({ engine: 'codex', sessionId: 'sess-1' }, false, d)
expect(outcome).toEqual({ ok: false, detail: 'codex did not come back up after restart' })
+ // Nothing runs in a new conversation, so the row still holds the one it was in: nothing to keep.
+ expect(d.calls).not.toContain('keepAbandoned')
})
it('never retries a resume fallback for an engine with no session to resume in the first place', async () => {
diff --git a/cli/src/lib/restartAgent.ts b/cli/src/lib/restartAgent.ts
index aedbf391a..beccb15eb 100644
--- a/cli/src/lib/restartAgent.ts
+++ b/cli/src/lib/restartAgent.ts
@@ -90,6 +90,8 @@ export interface RestartAgentDeps {
prepareResume?: () => void | Promise<void>
buildArgv: (opts: { bypassPermission: boolean; resumeSessionId?: string }) => string[]
log: (message: string) => void
+ /** Keeps the conversation the fresh fallback left as a stopped harness (keepAbandonedConversation.ts). */
+ keepAbandoned?: () => void
}
/** Outcomes that mean the old process is confirmed gone — safe to respawn over the pane. `not-ours` and
@@ -156,5 +158,8 @@ export async function restartAgent(
if (!identity) {
return { ok: false, detail: `${session.engine} did not come back up after restart` }
}
+ // The agent works on in a new conversation; the one it was in, which the engine could not reopen, is
+ // kept as a stopped harness to read and to resume once it can (round 24). A /clear keeps nothing.
+ if (resumeSessionId && !resumed) deps.keepAbandoned?.()
return { ok: true, processIdentity: identity, resumed }
}
diff --git a/cli/src/lib/restoreAgents.spec.ts b/cli/src/lib/restoreAgents.spec.ts
index 11717fc67..84c66c87d 100644
--- a/cli/src/lib/restoreAgents.spec.ts
+++ b/cli/src/lib/restoreAgents.spec.ts
@@ -326,6 +326,19 @@ describe('restoreAgents — waiting for the engine', () => {
expect(h.rows.get('agent-a')?.launch).toEqual({ state: 'starting' })
})
+ it('keeps the conversation the fresh launch leaves, before it is unbound', async () => {
+ const h = harness([row()])
+ const kept: Array<{ agentId: string; sessionId: string; unbound: boolean }> = []
+ h.deps.keepAbandoned = (left) => { kept.push({ agentId: left.agentId, sessionId: left.sessionId, unbound: h.calls.includes('unbind:session-a') }) }
+ h.probes.set('%0', [null, null, identity(9)])
+ h.states.set('%0', [{ dead: true }, { dead: false }])
+
+ await restoreAgents(h.deps)
+ await settled(h, 1, 1)
+
+ expect(kept).toEqual([{ agentId: 'agent-a', sessionId: 'session-a', unbound: false }])
+ })
+
it('treats an engine that appears and then exits inside the settling window as a rejected resume', async () => {
const h = harness([row()], { settleMs: 50 })
// First probe: the resumed claude is up. Settling poll: the pane is dead. After the fresh
diff --git a/cli/src/lib/restoreAgents.ts b/cli/src/lib/restoreAgents.ts
index e68ee97bb..d76c15cc0 100644
--- a/cli/src/lib/restoreAgents.ts
+++ b/cli/src/lib/restoreAgents.ts
@@ -44,6 +44,8 @@ export const GRID_CREDENTIAL_REQUIRED = 'GRID_CREDENTIAL_REQUIRED'
export interface RestoreAgentsDeps {
retainStopped?: (entry: RegisteredSession, paneAlive: boolean) => void
+ /** Keeps a conversation the restore had to leave for a new one as a stopped harness (keepAbandonedConversation.ts). */
+ keepAbandoned?: (left: RegisteredSession) => void
registry: {
list(): RegisteredSession[]
@@ -303,6 +305,9 @@ async function watchRestoredPane(
// along to the agent itself, the stale binding goes, and the engine gets one fresh start.
mayRetryFresh = false
deps.log(`[restore] ${engine} · agent ${agentId} · did not come back up resuming its session — retrying fresh`)
+ // Unbound below and replaced by the fresh start's own, the conversation the agent was in is kept as
+ // a stopped harness first, to read and to resume once the engine can again.
+ deps.keepAbandoned?.({ ...entry })
deps.registry.inheritName(entry.sessionId, agentId)
deps.registry.unbindSession(entry.sessionId)
const launch = await deps.buildLaunch(entry, {})
diff --git a/cli/src/lib/resumeAgentService.spec.ts b/cli/src/lib/resumeAgentService.spec.ts
index 35b4431c8..5b446da53 100644
--- a/cli/src/lib/resumeAgentService.spec.ts
+++ b/cli/src/lib/resumeAgentService.spec.ts
@@ -79,6 +79,7 @@ beforeEach(() => {
retainExitedSession: vi.fn((row, alive) => { store.save(row); if (alive) registry.releaseEngine(row.agentId, true); else registry.removeAgent(row.agentId) }),
announceSession: vi.fn(), relaunchOverrides: vi.fn(async () => ({ ok: true as const, overrides: { env: {}, extraArgs: [], clearEnv: [] } })),
prepareSessionResume: vi.fn(), refreshGridWebSearch: vi.fn(), clearDeleted: vi.fn(), attachDsh: vi.fn(),
+ attachSession: vi.fn(async () => true),
}
})
afterEach(() => { vi.useRealTimers(); for (const row of registry.list()) registry.removeAgent(row.agentId); rmSync(dir, { recursive: true, force: true }) })
@@ -195,6 +196,42 @@ describe('production resume handler', () => {
expect(registry.byAgent(saved.agentId)?.processIdentity).toEqual(identity)
})
+ it('attaches a resume its own process confirmed when no hook registered it, and says so if that fails', async () => {
+ // Round 23: the SessionStart was refused, and discovery, finding the row ready already, did not
+ // attach it either, so the resumed agent never showed a turn.
+ vi.mocked(resolvePaneEngineProcess).mockResolvedValue(identity)
+ expect(await start()).toMatchObject({ ok: true, session: { launch: { state: 'ready' } } })
+ expect(deps.attachSession).toHaveBeenCalledTimes(1)
+ expect(deps.attachSession).toHaveBeenCalledWith(expect.objectContaining({ agentId: saved.agentId, sessionId: saved.sessionId, processIdentity: identity }))
+
+ for (const row of registry.list()) registry.removeAgent(row.agentId)
+ deps.stoppedAgents.finishResume(saved.agentId)
+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.mocked(deps.attachSession).mockRejectedValueOnce(new Error('transcript unreadable'))
+ expect(await start()).toMatchObject({ ok: true })
+ await vi.waitFor(() => expect(error).toHaveBeenCalledWith(`[resume] ${saved.agentId.slice(0, 8)} attach failed:`, 'transcript unreadable'))
+ vi.mocked(deps.attachSession).mockRejectedValueOnce('gone')
+ for (const row of registry.list()) registry.removeAgent(row.agentId)
+ deps.stoppedAgents.finishResume(saved.agentId)
+ expect(await start()).toMatchObject({ ok: true })
+ await vi.waitFor(() => expect(error).toHaveBeenCalledWith(`[resume] ${saved.agentId.slice(0, 8)} attach failed:`, 'gone'))
+ error.mockRestore()
+ })
+
+ it('attaches nothing for a resume that started a new conversation: the engine has yet to name it', async () => {
+ // A stopped harness that never had a conversation resumes into a new one.
+ rewrite({ sessionId: '', transcriptPath: '' })
+ vi.mocked(resolvePaneEngineProcess).mockResolvedValue(identity)
+ expect(await start()).toMatchObject({ ok: true, resumed: false, session: { launch: { state: 'ready' } } })
+ expect(deps.attachSession).not.toHaveBeenCalled()
+ })
+
+ it('leaves the attach to the hook that registered the resume first', async () => {
+ // The fixture's probe imitates a hook that already landed (`lastHookAt`): its registration attached it.
+ expect(await start()).toMatchObject({ ok: true, session: { launch: { state: 'ready' } } })
+ expect(deps.attachSession).not.toHaveBeenCalled()
+ })
+
it('opens a retained terminal as a new shell without a vendor resume argument', async () => {
rewrite({ engine: 'terminal', sessionId: '', transcriptPath: '' })
expect(await start()).toMatchObject({ ok: true, resumed: true, session: { engine: 'terminal', launch: { state: 'ready' } } })
diff --git a/cli/src/lib/resumeAgentService.ts b/cli/src/lib/resumeAgentService.ts
index 5529547d1..d42d52b35 100644
--- a/cli/src/lib/resumeAgentService.ts
+++ b/cli/src/lib/resumeAgentService.ts
@@ -32,6 +32,8 @@ export interface ResumeAgentServiceDeps {
refreshGridWebSearch(agentId: string, overrides: LaunchOverrides): void
clearDeleted(agentId: string): void
attachDsh(session: RegisteredSession): void
+ /** Follows a session: its history read, then its tail (core/transcripts/attach.ts). */
+ attachSession(session: RegisteredSession): Promise<boolean>
/** Where the relaunched engine's own writing begins, for the attach that follows (core/transcripts/relaunch.ts). */
relaunchMarks?: Pick<RelaunchMarks, 'note'>
}
@@ -39,7 +41,7 @@ export interface ResumeAgentServiceDeps {
export function createResumeAgentService(deps: ResumeAgentServiceDeps) {
const { registry, stoppedAgents, tmuxBackend, restartJobs, stopJobs, pinnedControls,
retainExitedSession, announceSession, relaunchOverrides, prepareSessionResume,
- refreshGridWebSearch, clearDeleted, attachDsh, relaunchMarks } = deps
+ refreshGridWebSearch, clearDeleted, attachDsh, attachSession, relaunchMarks } = deps
const resumeConversations = new Set<string>()
/** Same short form every other `[…]` line in the daemon logs uses. */
const sid = (id: string) => id.slice(0, 8)
@@ -103,9 +105,22 @@ export function createResumeAgentService(deps: ResumeAgentServiceDeps) {
const ready = confirmed?.launch?.state === 'ready'
? confirmed
: registry.setLaunch(saved.agentId, { state: 'ready' }) ?? result.session
+ const hooked = (confirmed?.lastHookAt ?? 0) > 0
console.log(`[resume] ${sid(saved.agentId)} confirmed · engine=${saved.engine}`
- + ` · hook=${(confirmed?.lastHookAt ?? 0) > 0 ? 'yes' : 'no'} · ${result.resumed ? 'same conversation' : 'fresh'}`)
+ + ` · hook=${hooked ? 'yes' : 'no'} · ${result.resumed ? 'same conversation' : 'fresh'}`)
announceSession(ready)
+ // Confirmed by its own process, the conversation is followed whether or not a hook ever comes.
+ // Only the hook's registration attached it: discovery attaches a row it finds still launching,
+ // and this one is ready already. With its SessionStart refused (round 23: the hook beat the
+ // process being recorded) or never sent, nothing attached it, and the resumed agent never showed
+ // a turn. A hook that registers later attaches it again, as one does a created agent. Only the
+ // conversation it reopened: a resume that started a new one has nothing to follow until the engine
+ // names it, and the row's old conversation is not being written any more.
+ if (!hooked && result.resumed) {
+ void attachSession(ready).catch((error) => {
+ console.error(`[resume] ${sid(saved.agentId)} attach failed:`, error instanceof Error ? error.message : error)
+ })
+ }
return { ...result, session: ready }
} else if (result.error !== 'AGENT_CHANGED') {
const row = registry.byAgent(saved.agentId)!
diff --git a/cli/src/testing/socketHistory.ts b/cli/src/testing/socketHistory.ts
index 3e716b7a4..9219f8616 100644
--- a/cli/src/testing/socketHistory.ts
+++ b/cli/src/testing/socketHistory.ts
@@ -1,8 +1,8 @@
/**
* `session_get` answered the way the daemon answers it, for a socket a spec builds on its own. The request
* left the socket's switch for the core (core/transcripts/history.ts), and cli.ts binds it, so a bare
- * socket answers it UNSUPPORTED. This binds it the same way: over the registry, the socket's own pager and
- * the engines' stores.
+ * socket answers it UNSUPPORTED. This binds it the same way: over the registry and the saved harnesses, the
+ * socket's own pager and the engines' stores.
*/
import { join } from 'node:path'
import type { BackendSocket } from '../backendSocket.js'
@@ -10,10 +10,12 @@ import { env } from '../config/env.js'
import { createHistory } from '../core/transcripts/history.js'
import { hermesDbForSession } from '../lib/hermesHome.js'
import { registry } from '../lib/registry.js'
+import { stoppedAgents } from '../lib/stoppedAgents.js'
export function bindHistory(socket: BackendSocket): void {
socket.historyProvider = createHistory({
resolve: (id) => registry.resolve(id),
+ stopped: () => stoppedAgents.list(),
pages: socket.transcriptPages,
dbs: { opencode: join(env.OPENCODE_DATA_DIR, 'opencode.db'), kilo: join(env.KILO_DATA_DIR, 'kilo.db'), devin: join(env.DEVIN_HOME, 'sessions.db') },
hermesDb: (s) => hermesDbForSession(s),