Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

A turn left open before a new engine started is never announced as starting again (#825)

Dee52bf7f9a0436
diff --git a/cli/e2e/machine.e2e.ts b/cli/e2e/machine.e2e.ts
index deab0d19b..5e27aaeae 100644
--- a/cli/e2e/machine.e2e.ts
+++ b/cli/e2e/machine.e2e.ts
@@ -91,9 +91,14 @@ describe('what the machine does to the daemon', () => {
await client.next(isTurn('turn_started', agent.id), 30_000, 'turn_started')
await until('the agent to read as working', async () => (await row(client, agent.id))?.activity?.state === 'working' || null, 15_000, 250)
+ const killedAt = client.frames.length
await d.tmux.run('kill-server')
await until('the agent to stop reading as working', async () => down(await row(client, agent.id)) || null, 45_000, 500)
await reopen(client, agent, 'back after the server died mid-turn')
+ // The interrupted turn died with its engine: it is never announced as starting again. Checked on every
+ // frame since the kill, not only the next one, which caught it only when it came late (Codex).
+ const restarted = client.frames.slice(killedAt).filter(isTurn('turn_started', agent.id)).map((frame) => frame.payload?.userMessage)
+ expect(restarted).toEqual(['back after the server died mid-turn'])
expect(d.coresStarted()).toBe(1)
client.close()
})
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index 47a947ec8..b459fa4e6 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -3363,6 +3363,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
retainStopped: retainExitedSession,
keepAbandoned: keepAbandonedConversation,
registry,
+ engineStarted: (sessionId) => relaunchMarks.engineStarted(sessionId),
// "Alive" means the pane still runs THIS row's engine — not merely that tmux knows the id.
// A new tmux server hands out `%N` from zero again, so a stale id can name someone's shell;
// and a pane that outlived the daemon in a session discovery no longer lists still has its
diff --git a/cli/src/core/transcripts/attach.spec.ts b/cli/src/core/transcripts/attach.spec.ts
index a2c0d825d..28a77cdac 100644
--- a/cli/src/core/transcripts/attach.spec.ts
+++ b/cli/src/core/transcripts/attach.spec.ts
@@ -6,6 +6,7 @@ import type { RegisteredSession } from '../../lib/registry.js'
import type { TailHold } from '../../watcher/watcher.js'
import { createAttach, type AttachDeps } from './attach.js'
import { createSessionNormalizers } from './normalizers.js'
+import type { RelaunchMark } from './relaunch.js'
/**
* The engines' own normalizers and readers are tested with each engine. Here each is a fake that
@@ -197,19 +198,19 @@ describe('attaching a session', () => {
it('folds a resumed conversation only up to where its relaunched engine began, and tails the rest live', async () => {
vi.spyOn(console, 'log').mockImplementation(() => {})
- const take = vi.fn((_sessionId: string): number | undefined => undefined)
+ const take = vi.fn((_sessionId: string): RelaunchMark | undefined => undefined)
const run = setup({ relaunchMarks: { take } })
const history = JSON.stringify({ ...CLAUDE_PROMPT, uuid: 'before' })
// The relaunched engine answered a message before this attach: a whole turn after the mark.
const s = session('claude', transcript([history, { ...CLAUDE_PROMPT, uuid: 'after' }, { type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text: 'hi' }], stop_reason: 'end_turn' }, uuid: 'a1' }]))
const mark = Buffer.byteLength(history) + 1
- take.mockReturnValueOnce(mark)
+ take.mockReturnValueOnce({ offset: mark, engineStarted: true })
expect(await run.attach.attachSession(s)).toBe(true)
expect(take).toHaveBeenCalledWith(s.sessionId)
// The tail starts where the engine's own writing began, so its turn reaches every window live.
expect(vi.mocked(run.deps.watcher.addSession).mock.calls[0][1]).toEqual({ fromOffset: mark })
// A mark is for the next attach only: taken even by an attach that only makes sure the tail runs.
- take.mockReturnValueOnce(mark)
+ take.mockReturnValueOnce({ offset: mark, engineStarted: true })
expect(await run.attach.attachSession(s)).toBe(true)
expect(take).toHaveBeenCalledTimes(2)
})
@@ -277,7 +278,7 @@ describe('attaching a session', () => {
// what the first-turn rule asks, but its engine was relaunched on it, so it is history.
const lines = [CLAUDE_PROMPT, { type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text: 'hi' }], stop_reason: 'end_turn' }, uuid: 'a1' }]
const s = session('claude', transcript(lines))
- const take = vi.fn((_sessionId: string): number | undefined => statSync(s.transcriptPath!).size)
+ const take = vi.fn((_sessionId: string): RelaunchMark | undefined => ({ offset: statSync(s.transcriptPath!).size, engineStarted: true }))
const run = setup({ relaunchMarks: { take } })
await run.attach.attachSession(s, false, false, true)
expect(run.deps.emit).not.toHaveBeenCalled()
@@ -288,6 +289,22 @@ describe('attaching a session', () => {
expect(run.deps.emit).not.toHaveBeenCalled()
})
+ it('announces a turn open at attach while its engine may still be running, never one a new engine left behind', async () => {
+ const log = vi.spyOn(console, 'log').mockImplementation(() => {})
+ // The engine was killed mid-turn (the tmux server died): the prompt is in, its answer never came.
+ const s = session('claude', transcript([CLAUDE_PROMPT]))
+ const size = statSync(s.transcriptPath!).size
+ // A daemon restart marks every conversation, and an engine that kept running is still in that turn.
+ const survived = setup({ relaunchMarks: { take: () => ({ offset: size, engineStarted: false }) } })
+ await survived.attach.attachSession(s)
+ expect(vi.mocked(survived.deps.emit).mock.calls.some((call) => call[2]?.resumed)).toBe(true)
+ // A resume, or a restore that rebuilt the pane, started a new engine: that turn died with the old one.
+ const restarted = setup({ relaunchMarks: { take: () => ({ offset: size, engineStarted: true }) } })
+ await restarted.attach.attachSession(s)
+ expect(restarted.deps.emit).not.toHaveBeenCalled()
+ expect(log.mock.calls.some(([line]) => String(line).includes('left the turn open at attach as history'))).toBe(true)
+ })
+
it('replays a Claude Code first turn live when born after its agent, unless a held tail already delivered it', async () => {
vi.spyOn(console, 'log').mockImplementation(() => {})
const run = setup()
diff --git a/cli/src/core/transcripts/attach.ts b/cli/src/core/transcripts/attach.ts
index 089a19c41..4f7d151a0 100644
--- a/cli/src/core/transcripts/attach.ts
+++ b/cli/src/core/transcripts/attach.ts
@@ -124,7 +124,8 @@ export function createAttach({
// e2e/clockjump.e2e.ts). A terminal that really is gone is retired by the reconciler's confirmed scans.
if (await terminalGone(session)) return false
// Taken whichever way this attach goes: a mark is for the next attach of the conversation only.
- const relaunchedAt = relaunchMarks?.take(session.sessionId)
+ const relaunch = relaunchMarks?.take(session.sessionId)
+ const relaunchedAt = relaunch?.offset
if (!reset && normalizers.hasState(session.sessionId)) {
if (session.transcriptPath) {
const unseen = neverFoldedHistory.delete(session.sessionId)
@@ -409,7 +410,12 @@ export function createAttach({
// start included. Claude Code announces its session again when it compacts, often in the middle of
// a long turn, and replaying the start showed every window that turn starting twice
// (e2e/compaction.e2e.ts).
- if (historyTurnOpen && !handover.hold) {
+ // Nor for a turn left open before a new engine was started on the conversation (a resume, or a restore
+ // that rebuilt the pane): that turn died with the engine before, and announcing it showed the
+ // interrupted message starting anew (core/transcripts/relaunch.ts).
+ if (historyTurnOpen && !handover.hold && relaunch?.engineStarted) {
+ console.log(`[agent] ${sid(session.agentId)} left the turn open at attach as history · it began before its engine was started again`)
+ } else if (historyTurnOpen && !handover.hold) {
const opened = historyEvents.findLast((event) => event.type === 'turn_started')
if (opened) {
console.log(`[agent] ${sid(session.agentId)} resumed the turn already open at attach`)
diff --git a/cli/src/core/transcripts/relaunch.spec.ts b/cli/src/core/transcripts/relaunch.spec.ts
index 69cc5216b..6bc038ab1 100644
--- a/cli/src/core/transcripts/relaunch.spec.ts
+++ b/cli/src/core/transcripts/relaunch.spec.ts
@@ -9,7 +9,7 @@ describe('relaunch marks', () => {
const marks = createRelaunchMarks()
marks.note('conversation', 1457)
expect(marks.size).toBe(1)
- expect(marks.take('conversation')).toBe(1457)
+ expect(marks.take('conversation')).toEqual({ offset: 1457, engineStarted: false })
expect(marks.take('conversation')).toBeUndefined()
expect(marks.size).toBe(0)
})
@@ -22,7 +22,21 @@ describe('relaunch marks', () => {
const marks = createRelaunchMarks()
marks.note('conversation', 10)
marks.note('conversation', 20)
- expect(marks.take('conversation')).toBe(20)
+ expect(marks.take('conversation')).toEqual({ offset: 20, engineStarted: false })
+ })
+
+ it('says whether a new engine was started on the conversation: by a resume, or by a restore that rebuilt its pane', () => {
+ const marks = createRelaunchMarks()
+ marks.note('resumed', 3, true)
+ marks.note('restored', 5)
+ marks.engineStarted('restored')
+ marks.note('survived', 8)
+ // Nothing to mark for a conversation the daemon's start did not note.
+ marks.engineStarted('never')
+ expect(marks.take('resumed')).toEqual({ offset: 3, engineStarted: true })
+ expect(marks.take('restored')).toEqual({ offset: 5, engineStarted: true })
+ expect(marks.take('survived')).toEqual({ offset: 8, engineStarted: false })
+ expect(marks.size).toBe(0)
})
it('reads a transcript\'s size, or nothing when there is no file', () => {
@@ -42,7 +56,7 @@ describe('relaunch marks', () => {
marks.note('stale', 5)
marks.note('fresh', 7)
clock = 1_000
- expect(marks.take('fresh')).toBe(7)
+ expect(marks.take('fresh')).toEqual({ offset: 7, engineStarted: false })
clock = 1_001
expect(marks.take('stale')).toBeUndefined()
expect(marks.size).toBe(0)
diff --git a/cli/src/core/transcripts/relaunch.ts b/cli/src/core/transcripts/relaunch.ts
index 20b3e33c8..270d8843e 100644
--- a/cli/src/core/transcripts/relaunch.ts
+++ b/cli/src/core/transcripts/relaunch.ts
@@ -29,19 +29,37 @@ export interface RelaunchMarkOptions {
maxAgeMs?: number
}
+/**
+ * Where a relaunched conversation's live writing begins, and whether a new engine was started on it. A
+ * daemon's start marks every conversation it restores, including those whose engine kept running in
+ * tmux; only a resume, or a restore that rebuilt the pane, starts a new engine. A turn left open before
+ * a new engine started died with the engine before it: announced again at the attach, it showed the
+ * interrupted message starting anew in every window (e2e/machine.e2e.ts, Codex after the tmux server
+ * died mid-turn).
+ */
+export interface RelaunchMark {
+ offset: number
+ engineStarted: boolean
+}
+
export function createRelaunchMarks({ now = Date.now, maxAgeMs = RESUME_READINESS_BUDGET_MS }: RelaunchMarkOptions = {}) {
- const marks = new Map<string, { offset: number; at: number }>()
+ const marks = new Map<string, RelaunchMark & { at: number }>()
return {
/** Before the engine is launched: its own writing starts at `offset`. */
- note(sessionId: string, offset: number): void {
- marks.set(sessionId, { offset, at: now() })
+ note(sessionId: string, offset: number, engineStarted = false): void {
+ marks.set(sessionId, { offset, engineStarted, at: now() })
+ },
+ /** A restore rebuilt this conversation's pane, so its engine is a new one. */
+ engineStarted(sessionId: string): void {
+ const mark = marks.get(sessionId)
+ if (mark) mark.engineStarted = true
},
- /** At the attach: where the fold stops, once. */
- take(sessionId: string): number | undefined {
+ /** At the attach: where the fold stops, and whether the engine is new, once. */
+ take(sessionId: string): RelaunchMark | undefined {
const mark = marks.get(sessionId)
if (!mark) return undefined
marks.delete(sessionId)
- return now() - mark.at <= maxAgeMs ? mark.offset : undefined
+ return now() - mark.at <= maxAgeMs ? { offset: mark.offset, engineStarted: mark.engineStarted } : undefined
},
/** How many marks are waiting for an attach — for tests and diagnostics. */
get size(): number {
diff --git a/cli/src/lib/restoreAgents.spec.ts b/cli/src/lib/restoreAgents.spec.ts
index 71dedcb6e..e25c45c0d 100644
--- a/cli/src/lib/restoreAgents.spec.ts
+++ b/cli/src/lib/restoreAgents.spec.ts
@@ -242,8 +242,25 @@ describe('restoreAgents — which agents get a pane back', () => {
it('launches fresh when the row has no session to resume', async () => {
const h = harness([row({ sessionId: '', boundAt: null })])
h.probes.set('%0', [identity(1)])
- await restoreAgents(h.deps)
+ const engineStarted = vi.fn()
+ await restoreAgents({ ...h.deps, engineStarted })
expect(h.launches).toEqual([{ agentId: 'agent-a' }])
+ // No conversation to carry over, so none whose open turn a new engine ends.
+ expect(engineStarted).not.toHaveBeenCalled()
+ await settled(h, 1, 1)
+ })
+
+ it('says a rebuilt pane runs a new engine on its conversation, and only once the pane is up', async () => {
+ const h = harness([row(), row({ agentId: 'agent-b', sessionId: 'session-b', tmuxPane: '%1' })])
+ h.probes.set('%0', [null, identity(500)])
+ h.probes.set('%1', [null, identity(501)])
+ const engineStarted = vi.fn()
+ let panes = 0
+ const createPane = h.deps.createPane
+ // The second pane cannot be opened: its conversation keeps whatever engine it had.
+ const summary = await restoreAgents({ ...h.deps, engineStarted, createPane: async (entry, launch) => (++panes === 2 ? { ok: false, reason: 'no room' } : createPane(entry, launch)) })
+ expect(summary.restored).toEqual(['agent-a'])
+ expect(engineStarted.mock.calls).toEqual([['session-a']])
await settled(h, 1, 1)
})
diff --git a/cli/src/lib/restoreAgents.ts b/cli/src/lib/restoreAgents.ts
index a7fec7ba6..71525572b 100644
--- a/cli/src/lib/restoreAgents.ts
+++ b/cli/src/lib/restoreAgents.ts
@@ -81,6 +81,8 @@ export interface RestoreAgentsDeps {
>
/** `tmux respawn-pane` over the restored pane — the resume → fresh fallback. */
respawn: (runtime: TmuxRuntimeRef, launch: RestoreLaunch) => Promise<{ ok: boolean; reason?: string }>
+ /** A pane was rebuilt on this conversation: its engine is a new one (core/transcripts/relaunch.ts). */
+ engineStarted?: (sessionId: string) => void
/** One probe of the pane for a recognizable engine process. */
probeProcess: (runtime: TmuxRuntimeRef, engine: AgentEngine) => Promise<ProcessIdentity | null>
/** `'gone'` when tmux no longer knows the pane, `'unknown'` when it could not be asked (`tmuxPaneState`).
@@ -317,6 +319,8 @@ export async function restoreAgents(deps: RestoreAgentsDeps): Promise<RestoreSum
deps.log(`[restore] ${entry.engine} · agent ${entry.agentId} · could not open a pane · ${created.reason}`)
continue
}
+ // A new engine on the conversation: a turn it left open is over (core/transcripts/relaunch.ts).
+ if (resumeSessionId) deps.engineStarted?.(resumeSessionId)
const key = terminalRouteKey(created.runtime)
// A terminal is up the moment its pane is — no engine to wait for, no route to hold.
if (isTerminalEngine(entry.engine)) {
diff --git a/cli/src/lib/resumeAgentService.spec.ts b/cli/src/lib/resumeAgentService.spec.ts
index e434622a7..6683f4401 100644
--- a/cli/src/lib/resumeAgentService.spec.ts
+++ b/cli/src/lib/resumeAgentService.spec.ts
@@ -106,14 +106,15 @@ describe('production resume handler', () => {
it('notes where the relaunched engine\'s writing begins, after the history is prepared and before the launch', async () => {
writeFileSync(saved.transcriptPath!, '{"type":"session_meta"}\n')
- const note = vi.fn((_sessionId: string, _offset: number) => {
+ const note = vi.fn((_sessionId: string, _offset: number, _engineStarted?: boolean) => {
// The pane is not allocated yet: the engine has written nothing of its own.
expect(create).not.toHaveBeenCalled()
})
vi.mocked(deps.prepareSessionResume).mockImplementation(() => { writeFileSync(saved.transcriptPath!, '{"type":"session_meta"}\n{"prepared":true}\n') })
deps = { ...deps, relaunchMarks: { note } }
expect(await start()).toMatchObject({ ok: true, resumed: true })
- expect(note).toHaveBeenCalledWith(saved.sessionId, Buffer.byteLength('{"type":"session_meta"}\n{"prepared":true}\n'))
+ // A resume always starts a new engine: a turn left open before it is over.
+ expect(note).toHaveBeenCalledWith(saved.sessionId, Buffer.byteLength('{"type":"session_meta"}\n{"prepared":true}\n'), true)
})
it('notes nothing for a conversation with no file to tail, or a file that is gone', async () => {
diff --git a/cli/src/lib/resumeAgentService.ts b/cli/src/lib/resumeAgentService.ts
index 3a3c3a9c1..d9112d177 100644
--- a/cli/src/lib/resumeAgentService.ts
+++ b/cli/src/lib/resumeAgentService.ts
@@ -223,7 +223,8 @@ export function createResumeAgentService(deps: ResumeAgentServiceDeps) {
// conversation only up to this byte. Taken after the history is prepared, which can rewrite it.
if (resumeSessionId && saved.sessionId && saved.transcriptPath) {
const offset = transcriptSize(saved.transcriptPath)
- if (offset !== null) relaunchMarks?.note(saved.sessionId, offset)
+ // A resume always starts a new engine: a turn left open before it is over.
+ if (offset !== null) relaunchMarks?.note(saved.sessionId, offset, true)
}
const { extraArgs, clearEnv } = built.overrides
const launchEnv = permissionMode === undefined ? built.overrides.env