Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Preserve crash-resume boundaries across failed transcript attaches (#1072)

Dee9ad4bb087019
diff --git a/cli/e2e/machine.e2e.ts b/cli/e2e/machine.e2e.ts
index 5e27aaeae..2ad25a837 100644
--- a/cli/e2e/machine.e2e.ts
+++ b/cli/e2e/machine.e2e.ts
@@ -6,7 +6,7 @@
* must bring each back with its conversation when it is opened again. After a restart it brings them
* back on its own.
*/
-import { mkdirSync, writeFileSync } from 'node:fs'
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
import { LocalClient, type Frame } from './harness/client.js'
@@ -103,6 +103,33 @@ describe('what the machine does to the daemon', () => {
client.close()
})
+ it.each(['claude', 'codex'] as const)('%s: the engine dies mid-turn while its shell survives: resume never reopens the interrupted turn', async engine => {
+ const d = await fresh()
+ const client = await LocalClient.connect(d)
+ const agent = await create(d, client, engine, `engine-crash-${engine}`)
+ client.send('message', { agentId: agent.id, content: '!slow 8000' })
+ await client.next(isTurn('turn_started', agent.id), 30_000, 'turn_started')
+ const working = await until('the isolated engine to read as working', async () => {
+ const now = await row(client, agent.id)
+ // Process identity is private registry state, not part of the clients' agent list.
+ const registered = JSON.parse(readFileSync(join(d.dataDir, 'registry.json'), 'utf8'))
+ .find((entry: Row) => entry.agentId === agent.id)
+ return now?.activity?.state === 'working' && registered?.processIdentity?.pid
+ ? registered : null
+ }, 15_000, 250)
+ // This PID came from the disposable daemon's private pane, never the owner's daemon or client.
+ expect(working.cwd).toBe(join(d.projectsDir, `engine-crash-${engine}`))
+ const killedAt = client.frames.length
+ process.kill(working.processIdentity.pid, 'SIGKILL')
+ await until('the engine to be retained as stopped', async () => down(await row(client, agent.id)) || null, 45_000, 500)
+ expect(await d.tmux.run('display-message', '-p', '-t', working.tmuxPane, '#{pane_dead}')).toBe('0')
+ await reopen(client, agent, 'back after the engine died mid-turn')
+ expect(client.frames.slice(killedAt).filter(isTurn('turn_started', agent.id)).map(frame => frame.payload?.userMessage))
+ .toEqual(['back after the engine died mid-turn'])
+ expect(d.coresStarted()).toBe(1)
+ client.close()
+ })
+
it('the machine restarts while the daemon is down: every agent comes back on its own, with its conversation', async () => {
const d = await fresh()
let client = await LocalClient.connect(d)
diff --git a/cli/src/core/agents/forget.spec.ts b/cli/src/core/agents/forget.spec.ts
index 9ae4318f4..b49908ae1 100644
--- a/cli/src/core/agents/forget.spec.ts
+++ b/cli/src/core/agents/forget.spec.ts
@@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'
import { removePendingCursorTasks } from '../engines/cursorTasks.js'
import type { RegisteredSession } from '../../lib/registry.js'
import { createForgetSession, type ForgetDeps } from './forget.js'
+import { createRelaunchMarks } from '../transcripts/relaunch.js'
// Cursor's queued Tasks are cleared through the core's own door to them (core/engines/cursorTasks.ts).
vi.mock('../engines/cursorTasks.js', () => ({ removePendingCursorTasks: vi.fn(async () => {}) }))
@@ -61,6 +62,15 @@ function expectLetGo(deps: ForgetDeps, sessionId: string, agentId: string) {
describe('forgetting a session', () => {
afterEach(() => { vi.restoreAllMocks(); vi.mocked(removePendingCursorTasks).mockClear() })
+ it('drops a crash-resume boundary when the conversation is stopped or unbound', () => {
+ const { deps } = setup()
+ const marks = createRelaunchMarks()
+ marks.note('s1', 20, true)
+ marks.read('s1')
+ createForgetSession({ ...deps, relaunchMarks: marks })('a1', { keepAgent: true })
+ expect(marks.size).toBe(0)
+ })
+
it('releases a session and keeps its agent: everything per-session goes, the agent and its tiles stay', () => {
const log = vi.spyOn(console, 'log').mockImplementation(() => {})
const { deps, forgetSession } = setup()
diff --git a/cli/src/core/agents/forget.ts b/cli/src/core/agents/forget.ts
index eed225824..d3ea4d650 100644
--- a/cli/src/core/agents/forget.ts
+++ b/cli/src/core/agents/forget.ts
@@ -18,6 +18,7 @@ import type { StoppedAgentStore } from '../../lib/stoppedAgents.js'
import type { SwarmPromptScopes } from '../../teams/promptScope.js'
import type { Watcher } from '../../watcher/watcher.js'
import type { SessionNormalizers } from '../transcripts/normalizers.js'
+import type { RelaunchMarks } from '../transcripts/relaunch.js'
type Frame = { type: string; payload: Record<string, unknown> }
@@ -30,6 +31,7 @@ export interface ForgetDeps {
/** Attach's two per-session marks (core/transcripts/attach.ts). */
neverFoldedHistory: Set<string>
replayedFirstTurn: Set<string>
+ relaunchMarks?: Pick<RelaunchMarks, 'forget'>
clearAgyIdleWatch: (sessionId: string) => void
cursorDiscovery: Pick<CursorTranscriptDiscovery, 'remove'>
cursorSubagents: Pick<CursorSubagentManager, 'forget'>
@@ -47,7 +49,7 @@ export interface ForgetDeps {
}
export function createForgetSession({
- registry, stoppedAgents, syncRecapPool, normalizers, turnStartedAt, neverFoldedHistory, replayedFirstTurn,
+ registry, stoppedAgents, syncRecapPool, normalizers, turnStartedAt, neverFoldedHistory, replayedFirstTurn, relaunchMarks,
clearAgyIdleWatch, cursorDiscovery, cursorSubagents, runtimeProfiles, watcher, stopHeartbeat, teams, input,
deviceInput, detachDsh, mirror, clients, dataDir,
}: ForgetDeps) {
@@ -81,6 +83,7 @@ export function createForgetSession({
// that runs for days, and a session forgotten then re-registered under the same id would inherit a
// stale "already replayed" and lose a first turn it was entitled to.
replayedFirstTurn.delete(sessionId)
+ relaunchMarks?.forget(sessionId)
clearAgyIdleWatch(sessionId)
cursorDiscovery.remove(sessionId)
cursorSubagents.forget(sessionId)
diff --git a/cli/src/core/main.ts b/cli/src/core/main.ts
index d446a81bb..1eb66b5d9 100644
--- a/cli/src/core/main.ts
+++ b/cli/src/core/main.ts
@@ -1450,6 +1450,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
// Release a session's binding, or remove a process-owned agent everywhere (core/agents/forget.ts).
const forgetSession = createForgetSession({
+ relaunchMarks,
registry,
stoppedAgents,
syncRecapPool,
@@ -1473,6 +1474,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
})
const retainExitedSession = createRetainExitedSession({
+ forgetRelaunch: relaunchMarks.forget,
stoppedAgents,
registry,
send: frame => backend.send(frame),
diff --git a/cli/src/core/transcripts/attach.spec.ts b/cli/src/core/transcripts/attach.spec.ts
index 245e97eb6..7c2a2ec28 100644
--- a/cli/src/core/transcripts/attach.spec.ts
+++ b/cli/src/core/transcripts/attach.spec.ts
@@ -8,7 +8,7 @@ import type { TailHold } from '../../watcher/watcher.js'
import { createAttach, type AttachDeps } from './attach.js'
import { loadEngine } from '../../engines/inProcess.js'
import { createSessionNormalizers } from './normalizers.js'
-import type { RelaunchMark } from './relaunch.js'
+import { createRelaunchMarks, type RelaunchMark } from './relaunch.js'
import type { PreparedLive } from '../engines/liveSessions.js'
import type { LiveFrame } from '../../engines/worker/liveProtocol.js'
@@ -76,7 +76,7 @@ vi.mock('../../engines/inProcess.js', async (real) => {
})
const IDLE_AGY = 'done\n\n ? for shortcuts'
-const started = (userMessage = 'go') => ({ type: 'turn_started', payload: { userMessage } })
+const started = (userMessage = 'go') => ({ type: 'turn_started' as const, payload: { userMessage } })
const CLAUDE_PROMPT = { type: 'user', message: { role: 'user', content: 'hello' }, uuid: 'u1' }
function setup(over: Partial<AttachDeps> = {}) {
@@ -113,7 +113,7 @@ function setup(over: Partial<AttachDeps> = {}) {
const made = (kind: string) => fakes.made.filter((entry) => entry.kind === kind).map((entry) => entry.instance)
-function remoteSetup(open = false) {
+function remoteSetup(open = false, over: Partial<AttachDeps> = {}) {
const turn = { identity: 'worker:turn', turnOpen: open, continued: false }
const handle = { engine: 'claude', turnOpen: open, snapshot: () => ({ ...turn }), closeTurn: vi.fn() }
const candidate = { state: { handle }, page: { cursor: { offset: 20 }, lastStarted: open ? started() : null },
@@ -127,7 +127,7 @@ function remoteSetup(open = false) {
prepare: vi.fn(async (_session, _options, observe, observePage) => { await observePage?.(frames); frames.forEach(observe); return candidate }),
install: vi.fn(() => true), discard: vi.fn(), retry: vi.fn(),
}
- const p = setup({ remoteLive: remote, liveFor: vi.fn(() => { throw new Error('isolated parser must stay in its worker') }) })
+ const p = setup({ remoteLive: remote, liveFor: vi.fn(() => { throw new Error('isolated parser must stay in its worker') }), ...over })
return { ...p, remote, candidate, handle }
}
@@ -207,6 +207,77 @@ describe('attaching a session', () => {
expect(p.normalizers.liveParsers.get(s.sessionId)).toBe(p.handle)
})
+ it.each(['begin', 'prepare', 'config', 'commit', 'install'])('keeps the crash-resume boundary after a failed %s until an attach commits', async phase => {
+ let now = 0
+ vi.spyOn(Date, 'now').mockImplementation(() => now)
+ const marks = createRelaunchMarks()
+ const p = remoteSetup(true, { relaunchMarks: marks }), s = session('claude', '/fixture/transcript')
+ marks.note(s.sessionId, 20, true)
+ const unavailable = () => { throw new Error('ENGINE_STALE_REPLY') }
+ if (phase === 'begin') vi.mocked(p.deps.runtimeProfiles.beginHydrate).mockImplementationOnce(unavailable)
+ if (phase === 'prepare') vi.mocked(p.remote.prepare).mockImplementationOnce(unavailable)
+ if (phase === 'config') vi.mocked(p.deps.runtimeProfiles.beginHydrate).mockReturnValueOnce({ ...p.profile, config: async () => unavailable() })
+ if (phase === 'commit') vi.mocked(p.deps.runtimeProfiles.beginHydrate).mockReturnValueOnce({ ...p.profile, commitWith: () => false })
+ if (phase === 'install') vi.mocked(p.remote.install).mockReturnValueOnce(false)
+ await p.attach.attachSession(s).catch(() => {})
+ expect(p.deps.emit).not.toHaveBeenCalled()
+ expect(p.remote.retry).toHaveBeenCalledWith(s)
+ expect(marks.size).toBe(1)
+ // Worker availability cannot age a confirmed resume back into an interrupted live turn.
+ now = 60 * 60_000
+ await p.attach.attachSession(s, true)
+ expect(p.remote.prepare).toHaveBeenLastCalledWith(s, { live: false, end: 20 }, expect.any(Function), undefined)
+ expect(p.handle.closeTurn).toHaveBeenCalledExactlyOnceWith('abandoned')
+ expect(p.deps.emit).not.toHaveBeenCalled()
+ expect(marks.size).toBe(0)
+ })
+
+ it('closes the interrupted history before the resumed engine can deliver a new live turn', async () => {
+ const marks = createRelaunchMarks()
+ const p = remoteSetup(true, { relaunchMarks: marks }), s = session('claude', '/fixture/transcript')
+ marks.note(s.sessionId, 20, true)
+ vi.mocked(p.deps.watcher.addSession).mockImplementationOnce(async () => {
+ expect(p.handle.closeTurn).toHaveBeenCalledExactlyOnceWith('abandoned')
+ p.deps.emit(s.sessionId, [started('after the crash')])
+ p.handle.closeTurn.mockClear()
+ })
+ await p.attach.attachSession(s)
+ expect(p.deps.emit).toHaveBeenCalledExactlyOnceWith(s.sessionId, [started('after the crash')])
+ expect(p.handle.closeTurn).not.toHaveBeenCalled()
+ })
+
+ it.each([false, true])('uses a held tail from the old engine only when the engine survived: restarted %s', async restarted => {
+ const marks = createRelaunchMarks()
+ const p = remoteSetup(true, { relaunchMarks: marks }), s = session('claude', '/fixture/transcript')
+ const old = { ...p.handle }
+ p.normalizers.liveParsers.set(s.sessionId, old)
+ const hold: TailHold = { offset: 12, expired: false, release: vi.fn() }
+ vi.mocked(p.deps.watcher.hold).mockResolvedValue(hold)
+ vi.mocked(p.deps.watcher.tails).mockReturnValue(true)
+ marks.note(s.sessionId, 20, restarted)
+ await p.attach.attachSession(s)
+ expect(p.remote.prepare).toHaveBeenCalledWith(s, { live: false, end: restarted ? 20 : 12 }, expect.any(Function), undefined)
+ expect(p.handle.closeTurn).toHaveBeenCalledTimes(restarted ? 1 : 0)
+ expect(p.deps.emit).not.toHaveBeenCalled()
+ expect(hold.release).toHaveBeenCalled()
+ expect(marks.size).toBe(0)
+ })
+
+ it.each(['prepare', 'config'])('retries a resumed attach whose tail hold expires during %s', async stage => {
+ const marks = createRelaunchMarks()
+ const p = remoteSetup(true, { relaunchMarks: marks }), s = session('claude', '/fixture/transcript')
+ let expired = stage === 'prepare'
+ const hold: TailHold = { offset: 12, get expired() { return expired }, release: vi.fn() }
+ vi.mocked(p.deps.watcher.hold).mockResolvedValue(hold)
+ if (stage === 'config') vi.mocked(p.deps.runtimeProfiles.ingestConfig).mockImplementationOnce(async () => { expired = true; return false })
+ marks.note(s.sessionId, 20, true)
+ await p.attach.attachSession(s)
+ expect(p.remote.retry).toHaveBeenCalledWith(s)
+ expect(p.remote.install).not.toHaveBeenCalled()
+ expect(p.deps.emit).not.toHaveBeenCalled()
+ expect(marks.read(s.sessionId)).toEqual({ offset: 20, engineStarted: true })
+ })
+
it.each([false, true])('retains a binding and retries a failed worker prepare, with an existing tail: %s', async held => {
const p = remoteSetup(), s = session('claude', '/private/transcript')
const old = { ...p.handle }
@@ -401,21 +472,21 @@ describe('attaching a session', () => {
it('folds a resumed conversation only up to where its relaunched engine began, and tails the rest live', async () => {
vi.spyOn(console, 'log').mockImplementation(() => {})
- const take = vi.fn((_sessionId: string): RelaunchMark | undefined => undefined)
- const run = setup({ relaunchMarks: { take } })
+ const read = vi.fn((_sessionId: string): RelaunchMark | undefined => undefined)
+ const run = setup({ relaunchMarks: { read, complete: vi.fn() } })
const history = JSON.stringify({ ...CLAUDE_PROMPT, uuid: 'before' })
// The relaunched engine answered a message before this attach: a whole turn after the mark.
const s = session('claude', transcript([history, { ...CLAUDE_PROMPT, uuid: 'after' }, { type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text: 'hi' }], stop_reason: 'end_turn' }, uuid: 'a1' }]))
const mark = Buffer.byteLength(history) + 1
- take.mockReturnValueOnce({ offset: mark, engineStarted: true })
+ read.mockReturnValueOnce({ offset: mark, engineStarted: true })
expect(await run.attach.attachSession(s)).toBe(true)
- expect(take).toHaveBeenCalledWith(s.sessionId)
+ expect(read).toHaveBeenCalledWith(s.sessionId)
// The tail starts where the engine's own writing began, so its turn reaches every window live.
expect(vi.mocked(run.deps.watcher.addSession).mock.calls[0][1]).toEqual({ fromOffset: mark })
// A mark is for the next attach only: taken even by an attach that only makes sure the tail runs.
- take.mockReturnValueOnce({ offset: mark, engineStarted: true })
+ read.mockReturnValueOnce({ offset: mark, engineStarted: true })
expect(await run.attach.attachSession(s)).toBe(true)
- expect(take).toHaveBeenCalledTimes(2)
+ expect(read).toHaveBeenCalledTimes(2)
})
it('takes over a tail it holds for a reset, resuming it where the read stopped', async () => {
@@ -481,8 +552,8 @@ describe('attaching a session', () => {
// what the first-turn rule asks, but its engine was relaunched on it, so it is history.
const lines = [CLAUDE_PROMPT, { type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text: 'hi' }], stop_reason: 'end_turn' }, uuid: 'a1' }]
const s = session('claude', transcript(lines))
- const take = vi.fn((_sessionId: string): RelaunchMark | undefined => ({ offset: statSync(s.transcriptPath!).size, engineStarted: true }))
- const run = setup({ relaunchMarks: { take } })
+ const read = vi.fn((_sessionId: string): RelaunchMark | undefined => ({ offset: statSync(s.transcriptPath!).size, engineStarted: true }))
+ const run = setup({ relaunchMarks: { read, complete: vi.fn() } })
await run.attach.attachSession(s, false, false, true)
expect(run.deps.emit).not.toHaveBeenCalled()
expect(log.mock.calls.map(([line]) => String(line)).some((line) => line.includes('replayed the first turn'))).toBe(false)
@@ -498,11 +569,11 @@ describe('attaching a session', () => {
const s = session('claude', transcript([CLAUDE_PROMPT]))
const size = statSync(s.transcriptPath!).size
// A daemon restart marks every conversation, and an engine that kept running is still in that turn.
- const survived = setup({ relaunchMarks: { take: () => ({ offset: size, engineStarted: false }) } })
+ const survived = setup({ relaunchMarks: { complete: vi.fn(), read: () => ({ offset: size, engineStarted: false }) } })
await survived.attach.attachSession(s)
expect(vi.mocked(survived.deps.emit).mock.calls.some((call) => call[2]?.resumed)).toBe(true)
// A resume, or a restore that rebuilt the pane, started a new engine: that turn died with the old one.
- const restarted = setup({ relaunchMarks: { take: () => ({ offset: size, engineStarted: true }) } })
+ const restarted = setup({ relaunchMarks: { complete: vi.fn(), read: () => ({ offset: size, engineStarted: true }) } })
await restarted.attach.attachSession(s)
expect(restarted.deps.emit).not.toHaveBeenCalled()
expect(log.mock.calls.some(([line]) => String(line).includes('left the turn open at attach as history'))).toBe(true)
@@ -511,7 +582,7 @@ describe('attaching a session', () => {
expect(survived.normalizers.liveParsers.get(s.sessionId)?.turnOpen).toBe(true)
// Codex's own normalizer, likewise.
const codex = session('codex', transcript([{ open: true }]))
- const resumed = setup({ relaunchMarks: { take: () => ({ offset: statSync(codex.transcriptPath!).size, engineStarted: true }) } })
+ const resumed = setup({ relaunchMarks: { complete: vi.fn(), read: () => ({ offset: statSync(codex.transcriptPath!).size, engineStarted: true }) } })
await resumed.attach.attachSession(codex)
expect(resumed.normalizers.liveParsers.get(codex.sessionId)?.turnOpen).toBe(false)
})
diff --git a/cli/src/core/transcripts/attach.ts b/cli/src/core/transcripts/attach.ts
index bc1b7593e..d8c2ea24d 100644
--- a/cli/src/core/transcripts/attach.ts
+++ b/cli/src/core/transcripts/attach.ts
@@ -56,7 +56,7 @@ export interface AttachDeps {
/** How many sessions attach at once. */
concurrency: number
/** Where an engine's writing became live again after a relaunch (core/transcripts/relaunch.ts): the fold stops there. */
- relaunchMarks?: Pick<RelaunchMarks, 'take'>
+ relaunchMarks?: Pick<RelaunchMarks, 'read' | 'complete'>
/** The most of a transcript an engine without its own reader from the end folds, from its end. */
wholeReadCapBytes?: number
/** The session attached with its last turn already over (core/turns/recaps.ts `settled`). */
@@ -133,11 +133,12 @@ export function createAttach({
}
const engine = <Name extends keyof InProcessModules>(name: Name): InProcessModules[Name] | null =>
session.engine === name ? other as InProcessModules[Name] | null : null
- // Taken whichever way this attach goes: a mark is for the next attach of the conversation only.
- const relaunch = relaunchMarks?.take(session.sessionId)
+ // Reading does not consume the boundary. A stale profile or unavailable worker can abandon this
+ // attach before installing anything; the next attach must still know the old engine's turn died.
+ const relaunch = relaunchMarks?.read(session.sessionId)
const relaunchedAt = relaunch?.offset
const remote = remoteLive?.handles(session.engine) ? remoteLive : undefined
- if (!reset && normalizers.hasState(session.sessionId) && (!remote || remote.current(session))) {
+ if (!relaunch && !reset && normalizers.hasState(session.sessionId) && (!remote || remote.current(session))) {
if (session.transcriptPath) {
const unseen = neverFoldedHistory.delete(session.sessionId)
await watcher.addSession(
@@ -207,15 +208,19 @@ export function createAttach({
console.warn(`[agent] ${sid(session.agentId)} kept its live normalizer · the re-read ${why}`)
handover.hold?.release()
handover.next = null
+ if (relaunch) remote?.retry(session)
return true
}
+ // A surviving shell can leave its old tail installed. Its cursor is the old engine's delivery
+ // position; a newly started engine's history ends at the launch boundary instead.
+ const historyEnd = () => relaunch?.engineStarted ? relaunch.offset : handover.hold?.offset ?? relaunchedAt
if (remote) {
if (session.transcriptPath) handover.hold = await watcher.hold(session.sessionId, session.transcriptPath)
const live = replayLive && handover.hold === null
- const profile = runtimeProfiles.beginHydrate(session)
- profileHydration = profile
try {
- prepared = await remote.prepare(session, { live, end: handover.hold?.offset ?? relaunchedAt }, frame => {
+ const profile = runtimeProfiles.beginHydrate(session)
+ profileHydration = profile
+ prepared = await remote.prepare(session, { live, end: historyEnd() }, frame => {
if (frame.profile && !profile.ingestFrames) sideRead('runtime profile', session.sessionId, () => profile.ingest(frame.raw))
if (frame.observe && observe) observe(frame.raw)
}, profile.ingestFrames)
@@ -226,7 +231,7 @@ export function createAttach({
return handover.hold ? keepLiveNormalizer('could not reach its engine worker') : true
}
if (handover.hold?.expired) { remote.discard(prepared); return keepLiveNormalizer('outlasted its hold on the tail') }
- if (!profile.commitWith) profile.commit()
+ if (!profileHydration.commitWith) profileHydration.commit()
fromEnd = { next: prepared.page.cursor.offset, records: prepared.records, content: prepared.content }
handover.next = fromEnd.next
historyTurnOpen = !live && prepared.state.handle.turnOpen
@@ -246,7 +251,7 @@ export function createAttach({
profile: (line) => profile.ingest(line),
fold: (line) => stream.push(line),
observe,
- }, { fromStart: live, end: handover.hold?.offset ?? relaunchedAt })
+ }, { fromStart: live, end: historyEnd() })
if (handover.hold && (read.failed || handover.hold.expired)) {
return keepLiveNormalizer(read.failed ? 'could not read the transcript' : 'outlasted its hold on the tail')
}
@@ -410,6 +415,18 @@ export function createAttach({
historyTurnOpen = fold((line) => normalizer.ingest(line), () => normalizer.turnOpen)
commandcodeNormalizers.set(session.sessionId, normalizer)
}
+ // Close the old engine's turn before starting the tail: addSession may synchronously deliver a
+ // new turn written after resume. Closing afterward would abandon that new turn instead.
+ const abandonedHistory = historyTurnOpen && relaunch?.engineStarted
+ if (abandonedHistory) {
+ console.log(`[agent] ${sid(session.agentId)} left the turn open at attach as history · it began before its engine was started again`)
+ const folds: Array<Map<string, { closeTurn(): unknown }>> = [cursorNormalizers, museNormalizers, ampNormalizers,
+ grokNormalizers, agyNormalizers, copilotNormalizers, piNormalizers, commandcodeNormalizers]
+ for (const fold of folds) fold.get(session.sessionId)?.closeTurn()
+ liveParsers.get(session.sessionId)?.closeTurn('abandoned')
+ }
+ // From this point the installed parser owns the boundary, including the closed-turn checkpoint.
+ if (relaunch) relaunchMarks!.complete(session.sessionId, relaunch)
if (session.transcriptPath) {
neverFoldedHistory.delete(session.sessionId)
// Deliberately NOT `fromStart`, even when the caller asked for it: this branch has just folded the
@@ -451,16 +468,7 @@ export function createAttach({
// Nor for a turn left open before a new engine was started on the conversation (a resume, or a restore
// that rebuilt the pane): that turn died with the engine before, and announcing it showed the
// interrupted message starting anew (core/transcripts/relaunch.ts).
- if (historyTurnOpen && !handover.hold && relaunch?.engineStarted) {
- console.log(`[agent] ${sid(session.agentId)} left the turn open at attach as history · it began before its engine was started again`)
- // Closed in the normalizer too, and as silently. Left open, the next message's start ended it first:
- // a turn_ended for a turn no client saw start (a "done" and its notification), a team's or the
- // orchestrator's delivery read as ended before it started, and the agent working until then.
- const folds: Array<Map<string, { closeTurn(): unknown }>> = [cursorNormalizers, museNormalizers, ampNormalizers,
- grokNormalizers, agyNormalizers, copilotNormalizers, piNormalizers, commandcodeNormalizers]
- for (const fold of folds) fold.get(session.sessionId)?.closeTurn()
- liveParsers.get(session.sessionId)?.closeTurn('abandoned')
- } else if (historyTurnOpen && !handover.hold) {
+ if (historyTurnOpen && !handover.hold && !abandonedHistory) {
const opened = historyEvents.findLast((event) => event.type === 'turn_started')
if (opened) {
console.log(`[agent] ${sid(session.agentId)} resumed the turn already open at attach`)
diff --git a/cli/src/core/transcripts/relaunch.spec.ts b/cli/src/core/transcripts/relaunch.spec.ts
index 6bc038ab1..80ae44bea 100644
--- a/cli/src/core/transcripts/relaunch.spec.ts
+++ b/cli/src/core/transcripts/relaunch.spec.ts
@@ -1,28 +1,32 @@
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
-import { describe, expect, it } from 'vitest'
+import { afterEach, describe, expect, it, vi } from 'vitest'
import { createRelaunchMarks, transcriptSize } from './relaunch.js'
describe('relaunch marks', () => {
- it('gives the attach the byte the relaunched engine began at, once', () => {
+ afterEach(() => vi.restoreAllMocks())
+ it('keeps the byte the relaunched engine began at until its attach commits', () => {
const marks = createRelaunchMarks()
marks.note('conversation', 1457)
expect(marks.size).toBe(1)
- expect(marks.take('conversation')).toEqual({ offset: 1457, engineStarted: false })
- expect(marks.take('conversation')).toBeUndefined()
+ expect(marks.read('conversation')).toEqual({ offset: 1457, engineStarted: false })
+ const mark = marks.read('conversation')!
+ expect(mark).toEqual({ offset: 1457, engineStarted: false })
+ marks.complete('conversation', mark)
+ expect(marks.read('conversation')).toBeUndefined()
expect(marks.size).toBe(0)
})
it('has nothing for a conversation that was not relaunched', () => {
- expect(createRelaunchMarks().take('never')).toBeUndefined()
+ expect(createRelaunchMarks().read('never')).toBeUndefined()
})
it('keeps the latest relaunch of a conversation', () => {
const marks = createRelaunchMarks()
marks.note('conversation', 10)
marks.note('conversation', 20)
- expect(marks.take('conversation')).toEqual({ offset: 20, engineStarted: false })
+ expect(marks.read('conversation')).toEqual({ offset: 20, engineStarted: false })
})
it('says whether a new engine was started on the conversation: by a resume, or by a restore that rebuilt its pane', () => {
@@ -33,9 +37,22 @@ describe('relaunch marks', () => {
marks.note('survived', 8)
// Nothing to mark for a conversation the daemon's start did not note.
marks.engineStarted('never')
- expect(marks.take('resumed')).toEqual({ offset: 3, engineStarted: true })
- expect(marks.take('restored')).toEqual({ offset: 5, engineStarted: true })
- expect(marks.take('survived')).toEqual({ offset: 8, engineStarted: false })
+ expect(marks.read('resumed')).toEqual({ offset: 3, engineStarted: true })
+ expect(marks.read('restored')).toEqual({ offset: 5, engineStarted: true })
+ expect(marks.read('survived')).toEqual({ offset: 8, engineStarted: false })
+ expect(marks.size).toBe(3)
+ })
+
+ it('never lets an older attach consume a newer resume, and forgets stopped conversations', () => {
+ const marks = createRelaunchMarks()
+ marks.note('conversation', 10, true)
+ const before = marks.read('conversation')!
+ marks.note('conversation', 20, true)
+ marks.complete('conversation', before)
+ expect(marks.read('conversation')).toEqual({ offset: 20, engineStarted: true })
+ marks.forget('conversation')
+ marks.complete('conversation', before)
+ expect(marks.read('conversation')).toBeUndefined()
expect(marks.size).toBe(0)
})
@@ -50,15 +67,16 @@ describe('relaunch marks', () => {
}
})
- it('drops a mark older than a resume can wait for', () => {
+ it('keeps a boundary when its first attach waits beyond the resume deadline', () => {
let clock = 0
- const marks = createRelaunchMarks({ now: () => clock, maxAgeMs: 1_000 })
- marks.note('stale', 5)
- marks.note('fresh', 7)
- clock = 1_000
- expect(marks.take('fresh')).toEqual({ offset: 7, engineStarted: false })
- clock = 1_001
- expect(marks.take('stale')).toBeUndefined()
- expect(marks.size).toBe(0)
+ vi.spyOn(Date, 'now').mockImplementation(() => clock)
+ const marks = createRelaunchMarks()
+ marks.note('queued', 5, true)
+ marks.note('restoring', 7)
+ clock = 60 * 60_000
+ marks.engineStarted('restoring')
+ expect(marks.read('queued')).toEqual({ offset: 5, engineStarted: true })
+ expect(marks.read('restoring')).toEqual({ offset: 7, engineStarted: true })
+ expect(marks.size).toBe(2)
})
})
diff --git a/cli/src/core/transcripts/relaunch.ts b/cli/src/core/transcripts/relaunch.ts
index 270d8843e..8136a8524 100644
--- a/cli/src/core/transcripts/relaunch.ts
+++ b/cli/src/core/transcripts/relaunch.ts
@@ -13,22 +13,17 @@
* So the transcript's size is noted where the engine's own writing begins: by a resume just before it
* launches the engine, and by the daemon at start for every conversation it restores. The attach that
* follows folds the conversation only up to that byte and tails from there, so what was written after
- * it is live. A mark is used once, by the next attach of that conversation; one older than a resume can
- * wait for (`RESUME_READINESS_BUDGET_MS`) belongs to an operation that is over and is not used.
+ * it is live. A mark is committed only after an attach installs its state. Failed worker reads must
+ * retain it: otherwise a retry reopens an interrupted turn as live. Time passing is not evidence that
+ * the old turn resumed: a queued attach keeps its boundary until it commits or the session is released.
*/
import { statSync } from 'node:fs'
-import { RESUME_READINESS_BUDGET_MS } from '../../lib/resumeStoppedAgent.js'
/** A transcript's size now, or null when there is no file to tail. One stat: nothing for a launch to wait on. */
export function transcriptSize(path: string): number | null {
try { return statSync(path).size } catch { return null }
}
-export interface RelaunchMarkOptions {
- now?: () => number
- maxAgeMs?: number
-}
-
/**
* Where a relaunched conversation's live writing begins, and whether a new engine was started on it. A
* daemon's start marks every conversation it restores, including those whose engine kept running in
@@ -42,24 +37,29 @@ export interface RelaunchMark {
engineStarted: boolean
}
-export function createRelaunchMarks({ now = Date.now, maxAgeMs = RESUME_READINESS_BUDGET_MS }: RelaunchMarkOptions = {}) {
- const marks = new Map<string, RelaunchMark & { at: number }>()
+export function createRelaunchMarks() {
+ const marks = new Map<string, RelaunchMark>()
return {
/** Before the engine is launched: its own writing starts at `offset`. */
note(sessionId: string, offset: number, engineStarted = false): void {
- marks.set(sessionId, { offset, engineStarted, at: now() })
+ marks.set(sessionId, { offset, engineStarted })
},
/** A restore rebuilt this conversation's pane, so its engine is a new one. */
engineStarted(sessionId: string): void {
const mark = marks.get(sessionId)
if (mark) mark.engineStarted = true
},
- /** At the attach: where the fold stops, and whether the engine is new, once. */
- take(sessionId: string): RelaunchMark | undefined {
- const mark = marks.get(sessionId)
- if (!mark) return undefined
+ /** At the attach: retain the boundary through retries, however long its worker is unavailable. */
+ read(sessionId: string): RelaunchMark | undefined {
+ return marks.get(sessionId)
+ },
+ /** An installed attach acknowledges only the boundary it read, never a newer resume's. */
+ complete(sessionId: string, mark: RelaunchMark): void {
+ if (marks.get(sessionId) === mark) marks.delete(sessionId)
+ },
+ /** A stopped or unbound conversation has no pending attach to preserve. */
+ forget(sessionId: string): void {
marks.delete(sessionId)
- return now() - mark.at <= maxAgeMs ? { offset: mark.offset, engineStarted: mark.engineStarted } : undefined
},
/** How many marks are waiting for an attach — for tests and diagnostics. */
get size(): number {
diff --git a/cli/src/lib/retainExitedSession.spec.ts b/cli/src/lib/retainExitedSession.spec.ts
index c229fb67c..c6a0c8ac9 100644
--- a/cli/src/lib/retainExitedSession.spec.ts
+++ b/cli/src/lib/retainExitedSession.spec.ts
@@ -76,6 +76,13 @@ describe('retainExitedSession', () => {
}
})
+ it.each([false, true])('releases a pending resume boundary when the engine exits, pane alive: %s', paneAlive => {
+ const forgetRelaunch = vi.fn()
+ const h = harness({ forgetRelaunch })
+ h.retain(row(), paneAlive)
+ expect(forgetRelaunch).toHaveBeenCalledExactlyOnceWith('session-a')
+ })
+
it('an archive that cannot be announced is warned about, never thrown at the caller', async () => {
const warn = vi.fn()
const h = harness({ publishStoppedAgent: () => Promise.reject(new Error('offline')), warn })
diff --git a/cli/src/lib/retainExitedSession.ts b/cli/src/lib/retainExitedSession.ts
index 27dad2df1..fc7f915ae 100644
--- a/cli/src/lib/retainExitedSession.ts
+++ b/cli/src/lib/retainExitedSession.ts
@@ -29,6 +29,7 @@ export interface RetainExitedSessionDeps {
announceSession(session: RegisteredSession): void
invalidateTerminalControl(agentId: string): void
forgetInput(agentId: string): void
+ forgetRelaunch?(sessionId: string): void
detachDsh(agentId: string): void
syncRecapPool(): void
warn(message: string, error: unknown): void
@@ -41,6 +42,7 @@ export function createRetainExitedSession(deps: RetainExitedSessionDeps) {
const saved = deps.stoppedAgents.get(entry.agentId)!
deps.invalidateTerminalControl(entry.agentId)
deps.forgetInput(entry.agentId)
+ deps.forgetRelaunch?.(entry.sessionId)
deps.detachDsh(entry.agentId)
// CLOSE THE VIEWS OF THE AGENT THAT JUST ENDED, the way a stop does.
//