The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
A turn left open before a new engine started is never announced as starting again (#825)
Dee52bf7f9a0436
diff --git a/cli/e2e/machine.e2e.ts b/cli/e2e/machine.e2e.tsindex deab0d19b..5e27aaeae 100644--- a/cli/e2e/machine.e2e.ts+++ b/cli/e2e/machine.e2e.ts@@ -91,9 +91,14 @@ describe('what the machine does to the daemon', () => {await client.next(isTurn('turn_started', agent.id), 30_000, 'turn_started')await until('the agent to read as working', async () => (await row(client, agent.id))?.activity?.state === 'working' || null, 15_000, 250)+ const killedAt = client.frames.lengthawait d.tmux.run('kill-server')await until('the agent to stop reading as working', async () => down(await row(client, agent.id)) || null, 45_000, 500)await reopen(client, agent, 'back after the server died mid-turn')+ // The interrupted turn died with its engine: it is never announced as starting again. Checked on every+ // frame since the kill, not only the next one, which caught it only when it came late (Codex).+ const restarted = client.frames.slice(killedAt).filter(isTurn('turn_started', agent.id)).map((frame) => frame.payload?.userMessage)+ expect(restarted).toEqual(['back after the server died mid-turn'])expect(d.coresStarted()).toBe(1)client.close()})diff --git a/cli/src/cli.ts b/cli/src/cli.tsindex 47a947ec8..b459fa4e6 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -3363,6 +3363,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {retainStopped: retainExitedSession,keepAbandoned: keepAbandonedConversation,registry,+ engineStarted: (sessionId) => relaunchMarks.engineStarted(sessionId),// "Alive" means the pane still runs THIS row's engine — not merely that tmux knows the id.// A new tmux server hands out `%N` from zero again, so a stale id can name someone's shell;// and a pane that outlived the daemon in a session discovery no longer lists still has itsdiff --git a/cli/src/core/transcripts/attach.spec.ts b/cli/src/core/transcripts/attach.spec.tsindex a2c0d825d..28a77cdac 100644--- a/cli/src/core/transcripts/attach.spec.ts+++ b/cli/src/core/transcripts/attach.spec.ts@@ -6,6 +6,7 @@ import type { RegisteredSession } from '../../lib/registry.js'import type { TailHold } from '../../watcher/watcher.js'import { createAttach, type AttachDeps } from './attach.js'import { createSessionNormalizers } from './normalizers.js'+import type { RelaunchMark } from './relaunch.js'/*** The engines' own normalizers and readers are tested with each engine. Here each is a fake that@@ -197,19 +198,19 @@ describe('attaching a session', () => {it('folds a resumed conversation only up to where its relaunched engine began, and tails the rest live', async () => {vi.spyOn(console, 'log').mockImplementation(() => {})- const take = vi.fn((_sessionId: string): number | undefined => undefined)+ const take = vi.fn((_sessionId: string): RelaunchMark | undefined => undefined)const run = setup({ relaunchMarks: { take } })const history = JSON.stringify({ ...CLAUDE_PROMPT, uuid: 'before' })// The relaunched engine answered a message before this attach: a whole turn after the mark.const s = session('claude', transcript([history, { ...CLAUDE_PROMPT, uuid: 'after' }, { type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text: 'hi' }], stop_reason: 'end_turn' }, uuid: 'a1' }]))const mark = Buffer.byteLength(history) + 1- take.mockReturnValueOnce(mark)+ take.mockReturnValueOnce({ offset: mark, engineStarted: true })expect(await run.attach.attachSession(s)).toBe(true)expect(take).toHaveBeenCalledWith(s.sessionId)// The tail starts where the engine's own writing began, so its turn reaches every window live.expect(vi.mocked(run.deps.watcher.addSession).mock.calls[0][1]).toEqual({ fromOffset: mark })// A mark is for the next attach only: taken even by an attach that only makes sure the tail runs.- take.mockReturnValueOnce(mark)+ take.mockReturnValueOnce({ offset: mark, engineStarted: true })expect(await run.attach.attachSession(s)).toBe(true)expect(take).toHaveBeenCalledTimes(2)})@@ -277,7 +278,7 @@ describe('attaching a session', () => {// what the first-turn rule asks, but its engine was relaunched on it, so it is history.const lines = [CLAUDE_PROMPT, { type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text: 'hi' }], stop_reason: 'end_turn' }, uuid: 'a1' }]const s = session('claude', transcript(lines))- const take = vi.fn((_sessionId: string): number | undefined => statSync(s.transcriptPath!).size)+ const take = vi.fn((_sessionId: string): RelaunchMark | undefined => ({ offset: statSync(s.transcriptPath!).size, engineStarted: true }))const run = setup({ relaunchMarks: { take } })await run.attach.attachSession(s, false, false, true)expect(run.deps.emit).not.toHaveBeenCalled()@@ -288,6 +289,22 @@ describe('attaching a session', () => {expect(run.deps.emit).not.toHaveBeenCalled()})+ it('announces a turn open at attach while its engine may still be running, never one a new engine left behind', async () => {+ const log = vi.spyOn(console, 'log').mockImplementation(() => {})+ // The engine was killed mid-turn (the tmux server died): the prompt is in, its answer never came.+ const s = session('claude', transcript([CLAUDE_PROMPT]))+ const size = statSync(s.transcriptPath!).size+ // A daemon restart marks every conversation, and an engine that kept running is still in that turn.+ const survived = setup({ relaunchMarks: { take: () => ({ offset: size, engineStarted: false }) } })+ await survived.attach.attachSession(s)+ expect(vi.mocked(survived.deps.emit).mock.calls.some((call) => call[2]?.resumed)).toBe(true)+ // A resume, or a restore that rebuilt the pane, started a new engine: that turn died with the old one.+ const restarted = setup({ relaunchMarks: { take: () => ({ offset: size, engineStarted: true }) } })+ await restarted.attach.attachSession(s)+ expect(restarted.deps.emit).not.toHaveBeenCalled()+ expect(log.mock.calls.some(([line]) => String(line).includes('left the turn open at attach as history'))).toBe(true)+ })+it('replays a Claude Code first turn live when born after its agent, unless a held tail already delivered it', async () => {vi.spyOn(console, 'log').mockImplementation(() => {})const run = setup()diff --git a/cli/src/core/transcripts/attach.ts b/cli/src/core/transcripts/attach.tsindex 089a19c41..4f7d151a0 100644--- a/cli/src/core/transcripts/attach.ts+++ b/cli/src/core/transcripts/attach.ts@@ -124,7 +124,8 @@ export function createAttach({// e2e/clockjump.e2e.ts). A terminal that really is gone is retired by the reconciler's confirmed scans.if (await terminalGone(session)) return false// Taken whichever way this attach goes: a mark is for the next attach of the conversation only.- const relaunchedAt = relaunchMarks?.take(session.sessionId)+ const relaunch = relaunchMarks?.take(session.sessionId)+ const relaunchedAt = relaunch?.offsetif (!reset && normalizers.hasState(session.sessionId)) {if (session.transcriptPath) {const unseen = neverFoldedHistory.delete(session.sessionId)@@ -409,7 +410,12 @@ export function createAttach({// start included. Claude Code announces its session again when it compacts, often in the middle of// a long turn, and replaying the start showed every window that turn starting twice// (e2e/compaction.e2e.ts).- if (historyTurnOpen && !handover.hold) {+ // Nor for a turn left open before a new engine was started on the conversation (a resume, or a restore+ // that rebuilt the pane): that turn died with the engine before, and announcing it showed the+ // interrupted message starting anew (core/transcripts/relaunch.ts).+ if (historyTurnOpen && !handover.hold && relaunch?.engineStarted) {+ console.log(`[agent] ${sid(session.agentId)} left the turn open at attach as history · it began before its engine was started again`)+ } else if (historyTurnOpen && !handover.hold) {const opened = historyEvents.findLast((event) => event.type === 'turn_started')if (opened) {console.log(`[agent] ${sid(session.agentId)} resumed the turn already open at attach`)diff --git a/cli/src/core/transcripts/relaunch.spec.ts b/cli/src/core/transcripts/relaunch.spec.tsindex 69cc5216b..6bc038ab1 100644--- a/cli/src/core/transcripts/relaunch.spec.ts+++ b/cli/src/core/transcripts/relaunch.spec.ts@@ -9,7 +9,7 @@ describe('relaunch marks', () => {const marks = createRelaunchMarks()marks.note('conversation', 1457)expect(marks.size).toBe(1)- expect(marks.take('conversation')).toBe(1457)+ expect(marks.take('conversation')).toEqual({ offset: 1457, engineStarted: false })expect(marks.take('conversation')).toBeUndefined()expect(marks.size).toBe(0)})@@ -22,7 +22,21 @@ describe('relaunch marks', () => {const marks = createRelaunchMarks()marks.note('conversation', 10)marks.note('conversation', 20)- expect(marks.take('conversation')).toBe(20)+ expect(marks.take('conversation')).toEqual({ offset: 20, engineStarted: false })+ })++ it('says whether a new engine was started on the conversation: by a resume, or by a restore that rebuilt its pane', () => {+ const marks = createRelaunchMarks()+ marks.note('resumed', 3, true)+ marks.note('restored', 5)+ marks.engineStarted('restored')+ marks.note('survived', 8)+ // Nothing to mark for a conversation the daemon's start did not note.+ marks.engineStarted('never')+ expect(marks.take('resumed')).toEqual({ offset: 3, engineStarted: true })+ expect(marks.take('restored')).toEqual({ offset: 5, engineStarted: true })+ expect(marks.take('survived')).toEqual({ offset: 8, engineStarted: false })+ expect(marks.size).toBe(0)})it('reads a transcript\'s size, or nothing when there is no file', () => {@@ -42,7 +56,7 @@ describe('relaunch marks', () => {marks.note('stale', 5)marks.note('fresh', 7)clock = 1_000- expect(marks.take('fresh')).toBe(7)+ expect(marks.take('fresh')).toEqual({ offset: 7, engineStarted: false })clock = 1_001expect(marks.take('stale')).toBeUndefined()expect(marks.size).toBe(0)diff --git a/cli/src/core/transcripts/relaunch.ts b/cli/src/core/transcripts/relaunch.tsindex 20b3e33c8..270d8843e 100644--- a/cli/src/core/transcripts/relaunch.ts+++ b/cli/src/core/transcripts/relaunch.ts@@ -29,19 +29,37 @@ export interface RelaunchMarkOptions {maxAgeMs?: number}+/**+ * Where a relaunched conversation's live writing begins, and whether a new engine was started on it. A+ * daemon's start marks every conversation it restores, including those whose engine kept running in+ * tmux; only a resume, or a restore that rebuilt the pane, starts a new engine. A turn left open before+ * a new engine started died with the engine before it: announced again at the attach, it showed the+ * interrupted message starting anew in every window (e2e/machine.e2e.ts, Codex after the tmux server+ * died mid-turn).+ */+export interface RelaunchMark {+ offset: number+ engineStarted: boolean+}+export function createRelaunchMarks({ now = Date.now, maxAgeMs = RESUME_READINESS_BUDGET_MS }: RelaunchMarkOptions = {}) {- const marks = new Map<string, { offset: number; at: number }>()+ const marks = new Map<string, RelaunchMark & { at: number }>()return {/** Before the engine is launched: its own writing starts at `offset`. */- note(sessionId: string, offset: number): void {- marks.set(sessionId, { offset, at: now() })+ note(sessionId: string, offset: number, engineStarted = false): void {+ marks.set(sessionId, { offset, engineStarted, at: now() })+ },+ /** A restore rebuilt this conversation's pane, so its engine is a new one. */+ engineStarted(sessionId: string): void {+ const mark = marks.get(sessionId)+ if (mark) mark.engineStarted = true},- /** At the attach: where the fold stops, once. */- take(sessionId: string): number | undefined {+ /** At the attach: where the fold stops, and whether the engine is new, once. */+ take(sessionId: string): RelaunchMark | undefined {const mark = marks.get(sessionId)if (!mark) return undefinedmarks.delete(sessionId)- return now() - mark.at <= maxAgeMs ? mark.offset : undefined+ return now() - mark.at <= maxAgeMs ? { offset: mark.offset, engineStarted: mark.engineStarted } : undefined},/** How many marks are waiting for an attach — for tests and diagnostics. */get size(): number {diff --git a/cli/src/lib/restoreAgents.spec.ts b/cli/src/lib/restoreAgents.spec.tsindex 71dedcb6e..e25c45c0d 100644--- a/cli/src/lib/restoreAgents.spec.ts+++ b/cli/src/lib/restoreAgents.spec.ts@@ -242,8 +242,25 @@ describe('restoreAgents — which agents get a pane back', () => {it('launches fresh when the row has no session to resume', async () => {const h = harness([row({ sessionId: '', boundAt: null })])h.probes.set('%0', [identity(1)])- await restoreAgents(h.deps)+ const engineStarted = vi.fn()+ await restoreAgents({ ...h.deps, engineStarted })expect(h.launches).toEqual([{ agentId: 'agent-a' }])+ // No conversation to carry over, so none whose open turn a new engine ends.+ expect(engineStarted).not.toHaveBeenCalled()+ await settled(h, 1, 1)+ })++ it('says a rebuilt pane runs a new engine on its conversation, and only once the pane is up', async () => {+ const h = harness([row(), row({ agentId: 'agent-b', sessionId: 'session-b', tmuxPane: '%1' })])+ h.probes.set('%0', [null, identity(500)])+ h.probes.set('%1', [null, identity(501)])+ const engineStarted = vi.fn()+ let panes = 0+ const createPane = h.deps.createPane+ // The second pane cannot be opened: its conversation keeps whatever engine it had.+ const summary = await restoreAgents({ ...h.deps, engineStarted, createPane: async (entry, launch) => (++panes === 2 ? { ok: false, reason: 'no room' } : createPane(entry, launch)) })+ expect(summary.restored).toEqual(['agent-a'])+ expect(engineStarted.mock.calls).toEqual([['session-a']])await settled(h, 1, 1)})diff --git a/cli/src/lib/restoreAgents.ts b/cli/src/lib/restoreAgents.tsindex a7fec7ba6..71525572b 100644--- a/cli/src/lib/restoreAgents.ts+++ b/cli/src/lib/restoreAgents.ts@@ -81,6 +81,8 @@ export interface RestoreAgentsDeps {>/** `tmux respawn-pane` over the restored pane — the resume → fresh fallback. */respawn: (runtime: TmuxRuntimeRef, launch: RestoreLaunch) => Promise<{ ok: boolean; reason?: string }>+ /** A pane was rebuilt on this conversation: its engine is a new one (core/transcripts/relaunch.ts). */+ engineStarted?: (sessionId: string) => void/** One probe of the pane for a recognizable engine process. */probeProcess: (runtime: TmuxRuntimeRef, engine: AgentEngine) => Promise<ProcessIdentity | null>/** `'gone'` when tmux no longer knows the pane, `'unknown'` when it could not be asked (`tmuxPaneState`).@@ -317,6 +319,8 @@ export async function restoreAgents(deps: RestoreAgentsDeps): Promise<RestoreSumdeps.log(`[restore] ${entry.engine} · agent ${entry.agentId} · could not open a pane · ${created.reason}`)continue}+ // A new engine on the conversation: a turn it left open is over (core/transcripts/relaunch.ts).+ if (resumeSessionId) deps.engineStarted?.(resumeSessionId)const key = terminalRouteKey(created.runtime)// A terminal is up the moment its pane is — no engine to wait for, no route to hold.if (isTerminalEngine(entry.engine)) {diff --git a/cli/src/lib/resumeAgentService.spec.ts b/cli/src/lib/resumeAgentService.spec.tsindex e434622a7..6683f4401 100644--- a/cli/src/lib/resumeAgentService.spec.ts+++ b/cli/src/lib/resumeAgentService.spec.ts@@ -106,14 +106,15 @@ describe('production resume handler', () => {it('notes where the relaunched engine\'s writing begins, after the history is prepared and before the launch', async () => {writeFileSync(saved.transcriptPath!, '{"type":"session_meta"}\n')- const note = vi.fn((_sessionId: string, _offset: number) => {+ const note = vi.fn((_sessionId: string, _offset: number, _engineStarted?: boolean) => {// The pane is not allocated yet: the engine has written nothing of its own.expect(create).not.toHaveBeenCalled()})vi.mocked(deps.prepareSessionResume).mockImplementation(() => { writeFileSync(saved.transcriptPath!, '{"type":"session_meta"}\n{"prepared":true}\n') })deps = { ...deps, relaunchMarks: { note } }expect(await start()).toMatchObject({ ok: true, resumed: true })- expect(note).toHaveBeenCalledWith(saved.sessionId, Buffer.byteLength('{"type":"session_meta"}\n{"prepared":true}\n'))+ // A resume always starts a new engine: a turn left open before it is over.+ expect(note).toHaveBeenCalledWith(saved.sessionId, Buffer.byteLength('{"type":"session_meta"}\n{"prepared":true}\n'), true)})it('notes nothing for a conversation with no file to tail, or a file that is gone', async () => {diff --git a/cli/src/lib/resumeAgentService.ts b/cli/src/lib/resumeAgentService.tsindex 3a3c3a9c1..d9112d177 100644--- a/cli/src/lib/resumeAgentService.ts+++ b/cli/src/lib/resumeAgentService.ts@@ -223,7 +223,8 @@ export function createResumeAgentService(deps: ResumeAgentServiceDeps) {// conversation only up to this byte. Taken after the history is prepared, which can rewrite it.if (resumeSessionId && saved.sessionId && saved.transcriptPath) {const offset = transcriptSize(saved.transcriptPath)- if (offset !== null) relaunchMarks?.note(saved.sessionId, offset)+ // A resume always starts a new engine: a turn left open before it is over.+ if (offset !== null) relaunchMarks?.note(saved.sessionId, offset, true)}const { extraArgs, clearEnv } = built.overridesconst launchEnv = permissionMode === undefined ? built.overrides.env