Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Preserve restored session identities and stream large Codex histories (#502)

Dee402e61d3a14b
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index acd71abba..5cbcf808a 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -3458,6 +3458,9 @@ async function runForeground(session: AuthSession | null): Promise<void> {
}
const agentReconciler = new TerminalAgentReconciler({
+ // The hook server starts before restore. Its early SessionStart hints must not run a full
+ // discovery scan over rows whose panes have not been recreated yet (and archive those rows).
+ deferUntilStart: true,
current: () => registry.list(),
backends: terminalBackends,
backendOrder: terminalConfig.backends,
diff --git a/cli/src/engines/codex/portableHistory.spec.ts b/cli/src/engines/codex/portableHistory.spec.ts
index 77a88af7c..2c9831e72 100644
--- a/cli/src/engines/codex/portableHistory.spec.ts
+++ b/cli/src/engines/codex/portableHistory.spec.ts
@@ -1,9 +1,15 @@
-import { afterEach, beforeEach, describe, expect, it } from 'vitest'
-import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { constants as bufferConstants } from 'node:buffer'
+import { createHash } from 'node:crypto'
+import { closeSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, rmSync, statSync, symlinkSync, writeFileSync, writeSync } from 'node:fs'
+import * as fs from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { portableCodexHistory, prepareCodexResume } from './portableHistory.js'
+// Keep the real filesystem, with configurable exports for simulating a concurrent writer.
+vi.mock('node:fs', async importOriginal => ({ ...await importOriginal<typeof import('node:fs')>() }))
+
const SESSION = '01a0a3cd-a374-71f2-a11a-1fc1cc41b36d'
const BAD_ID = 'msg_aY9suLof0GA06VzeKPXLM7rjUhoRRF9d'
const summary = [{ type: 'summary_text', text: 'Prior engine summary.' }]
@@ -68,7 +74,7 @@ describe('preparing a stopped Codex session for resume', () => {
mkdirSync(dir, { recursive: true })
file = join(dir, `rollout-2026-09-15T13-42-38-${SESSION}.jsonl`)
})
- afterEach(() => { rmSync(profile, { recursive: true, force: true }) })
+ afterEach(() => { vi.restoreAllMocks(); rmSync(profile, { recursive: true, force: true }) })
const source = () => ({ engine: 'codex', sessionId: SESSION, transcriptPath: file, codexHome: profile })
it('backs up the exact original, repairs the same session, and is a no-op on the next resume', () => {
@@ -89,6 +95,71 @@ describe('preparing a stopped Codex session for resume', () => {
expect(readdirSync(join(profile, 'sessions', '2026', '09', '15'))).toHaveLength(2)
})
+ it('resumes and repairs a rollout larger than the engine single-string limit without losing history', () => {
+ const chunk = Buffer.from(record('event_msg', { type: 'agent_message', message: 'x'.repeat(512 * 1024) }) + '\n')
+ const fd = openSync(file, 'wx')
+ const originalHash = createHash('sha256')
+ const write = (bytes: string | Buffer) => { writeSync(fd, Buffer.from(bytes)); originalHash.update(bytes) }
+ try {
+ write(meta + '\r\n')
+ for (let bytes = 0; bytes <= bufferConstants.MAX_STRING_LENGTH; bytes += chunk.length) write(chunk)
+ } finally { closeSync(fd) }
+ const before = statSync(file)
+ expect(before.size).toBeGreaterThan(bufferConstants.MAX_STRING_LENGTH)
+ // The common case must not rewrite or create a backup just to reopen an existing session.
+ expect(prepareCodexResume(source())).toEqual({ repairedItems: 0 })
+ expect(statSync(file).ino).toBe(before.ino)
+ expect(readdirSync(join(profile, 'sessions', '2026', '09', '15'))).toHaveLength(1)
+
+ const tail = record('response_item', badReasoning())
+ writeFileSync(file, tail, { flag: 'a' })
+ originalHash.update(tail)
+ const result = prepareCodexResume(source())
+ expect(result.repairedItems).toBe(1)
+ expect(result.repairedBytes).toBe(statSync(file).size)
+ const backupHash = createHash('sha256')
+ const backup = openSync(result.backupPath!, 'r')
+ try {
+ const buffer = Buffer.alloc(64 * 1024)
+ let read: number
+ while ((read = readSync(backup, buffer)) > 0) backupHash.update(buffer.subarray(0, read))
+ } finally { closeSync(backup) }
+ expect(backupHash.digest('hex')).toBe(originalHash.digest('hex'))
+ const repaired = openSync(file, 'r')
+ try {
+ const expected = Buffer.from(record('response_item', { type: 'reasoning', summary, content: [] }))
+ const actual = Buffer.alloc(expected.length)
+ readSync(repaired, actual, 0, actual.length, statSync(file).size - actual.length)
+ expect(actual).toEqual(expected)
+ } finally { closeSync(repaired) }
+ }, 30_000)
+
+ it('preserves UTF-8, CRLF, blank lines, and a final record without a newline across chunk boundaries', () => {
+ const original = '\r\n' + history(badReasoning(), { ...message,
+ content: [{ type: 'output_text', text: '𐐀é'.repeat(40_000) }],
+ }).replaceAll('\n', '\r\n').trimEnd()
+ writeFileSync(file, original)
+ const result = prepareCodexResume(source())
+ expect(result.repairedItems).toBe(1)
+ expect(readFileSync(result.backupPath!, 'utf8')).toBe(original)
+ expect(readFileSync(file, 'utf8')).toBe(portableCodexHistory(original, SESSION).history)
+ })
+
+ it('keeps a concurrent writer’s update instead of replacing it with prepared history', () => {
+ writeFileSync(file, history(badReasoning()))
+ const originalRead = fs.readSync
+ let changed = false
+ const appended = record('event_msg', { type: 'agent_message', message: 'new work' }) + '\n'
+ vi.spyOn(fs, 'readSync').mockImplementation((...args: Parameters<typeof fs.readSync>) => {
+ const read = originalRead(...args)
+ if (!changed) { changed = true; writeFileSync(file, appended, { flag: 'a' }) }
+ return read
+ })
+ expect(() => prepareCodexResume(source())).toThrow('changed during resume preparation')
+ expect(readFileSync(file, 'utf8')).toBe(history(badReasoning()) + appended)
+ expect(readdirSync(join(profile, 'sessions', '2026', '09', '15'))).toHaveLength(1)
+ })
+
it('resolves a rollout in the selected profile when the registry has no transcript path', () => {
writeFileSync(file, history(badReasoning()))
expect(prepareCodexResume({ ...source(), transcriptPath: null }).repairedItems).toBe(1)
diff --git a/cli/src/engines/codex/portableHistory.ts b/cli/src/engines/codex/portableHistory.ts
index 085d9ed01..372a93b18 100644
--- a/cli/src/engines/codex/portableHistory.ts
+++ b/cli/src/engines/codex/portableHistory.ts
@@ -1,5 +1,5 @@
-import { randomUUID } from 'node:crypto'
-import { lstatSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs'
+import { createHash, randomUUID } from 'node:crypto'
+import { closeSync, constants, fsyncSync, fstatSync, lstatSync, openSync, readSync, realpathSync, renameSync, rmSync, writeSync, type Stats } from 'node:fs'
import { homedir } from 'node:os'
import { isAbsolute, join, relative } from 'node:path'
import { resolveCodexRollout } from './rollout.js'
@@ -30,17 +30,18 @@ function repairReasoning(item: unknown): boolean {
return changed
}
-/** Only replayed API items are repaired. Other records (including readable event history) and
- * unchanged lines retain their exact bytes. Invalid JSON aborts the whole repair. */
-export function portableCodexHistory(history: string, sessionId: string): { history: string; repairedItems: number } {
+/** Shared by the string helper and the file reader, which retains only one record at a time. */
+function historyRepair(sessionId: string) {
let sawMeta = false
let repairedItems = 0
- const lines = history.split('\n').map((line, index) => {
+ let lineNumber = 0
+ const line = (line: string): string => {
+ lineNumber++
if (!line.trim()) return line
let record: unknown
try { record = JSON.parse(line) } catch {
// JSON.parse's own error can quote conversation text; keep it out of daemon logs.
- throw new Error(`invalid JSON in Codex rollout at line ${index + 1}`)
+ throw new Error(`invalid JSON in Codex rollout at line ${lineNumber}`)
}
if (!object(record)) throw new Error('invalid Codex rollout record')
if (!sawMeta) {
@@ -57,9 +58,78 @@ export function portableCodexHistory(history: string, sessionId: string): { hist
if (repairReasoning(item)) { repairedItems++; changed = true }
}
return changed ? JSON.stringify(record) + (line.endsWith('\r') ? '\r' : '') : line
- })
- if (!sawMeta) throw new Error('Codex rollout has no session metadata')
- return { history: lines.join('\n'), repairedItems }
+ }
+ return { line, finish: () => {
+ if (!sawMeta) throw new Error('Codex rollout has no session metadata')
+ return repairedItems
+ } }
+}
+
+/** Only replayed API items are repaired. Other records (including readable event history) and
+ * unchanged lines retain their exact bytes. Invalid JSON aborts the whole repair. */
+export function portableCodexHistory(history: string, sessionId: string): { history: string; repairedItems: number } {
+ const repair = historyRepair(sessionId)
+ const lines = history.split('\n').map(repair.line)
+ return { history: lines.join('\n'), repairedItems: repair.finish() }
+}
+
+const READ_BYTES = 64 * 1024
+const changedDuringPreparation = () => new Error('Codex rollout changed during resume preparation; retry after its writer stops')
+
+function sameFile(a: Stats, b: Stats): boolean {
+ return b.isFile() && a.ino === b.ino && a.dev === b.dev && a.size === b.size
+ && a.mtimeMs === b.mtimeMs && a.ctimeMs === b.ctimeMs
+}
+
+/** A rollout can exceed V8's single-string limit. Decode complete JSONL records, preserving raw
+ * bytes for unchanged records and hashing the whole file without holding it in memory. Reads are
+ * bounded to the captured length so an unexpected writer cannot make preparation run forever. */
+function scanHistory(fd: number, size: number, sessionId: string, emit?: (original: Buffer, repaired: Buffer) => void) {
+ const repair = historyRepair(sessionId)
+ const hash = createHash('sha256')
+ const chunk = Buffer.allocUnsafe(READ_BYTES)
+ let fragments: Buffer[] = []
+ let position = 0
+ let repairedBytes = 0
+ const line = (bytes: Buffer) => {
+ const newline = bytes.at(-1) === 10
+ const original = bytes.subarray(0, bytes.length - (newline ? 1 : 0)).toString('utf8')
+ const repaired = repair.line(original)
+ const output = repaired === original ? bytes : Buffer.from(repaired + (newline ? '\n' : ''))
+ repairedBytes += output.length
+ emit?.(bytes, output)
+ }
+ while (position < size) {
+ const count = readSync(fd, chunk, 0, Math.min(chunk.length, size - position), position)
+ if (!count) throw changedDuringPreparation()
+ position += count
+ hash.update(chunk.subarray(0, count))
+ let start = 0
+ let end: number
+ while ((end = chunk.indexOf(10, start)) >= 0 && end < count) {
+ const part = chunk.subarray(start, end + 1)
+ if (fragments.length) {
+ fragments.push(part)
+ line(Buffer.concat(fragments))
+ fragments = []
+ } else line(part)
+ start = end + 1
+ }
+ // Copy the trailing fragment before the next read reuses the chunk buffer. In particular,
+ // a multibyte UTF-8 codepoint split across reads is decoded only after its line is complete.
+ if (start < count) fragments.push(Buffer.from(chunk.subarray(start, count)))
+ }
+ if (fragments.length) line(Buffer.concat(fragments))
+ return { repairedItems: repair.finish(), repairedBytes, digest: hash.digest('hex') }
+}
+
+function writeAll(fd: number, bytes: Buffer): void {
+ let written = 0
+ while (written < bytes.length) {
+ const count = writeSync(fd, bytes, written, bytes.length - written)
+ if (!count) throw new Error('Could not write the prepared Codex history')
+ written += count
+ }
}
export interface CodexResumeSource {
@@ -96,25 +166,44 @@ export function prepareCodexResume(source: CodexResumeSource): { repairedItems:
|| (typeof process.getuid === 'function' && before.uid !== process.getuid())) {
throw new Error('Codex rollout is outside the session profile or is not an owned regular file')
}
- const original = readFileSync(file, 'utf8')
- const repaired = portableCodexHistory(original, source.sessionId)
- if (!repaired.repairedItems) return { repairedItems: 0 }
-
- const suffix = randomUUID()
- const backupPath = `${file}.reasoning-backup-${suffix}`
- const temporary = `${file}.reasoning-tmp-${suffix}`
- // Writing the bytes we inspected avoids copying a concurrent writer's partial record.
- writeFileSync(backupPath, original, { flag: 'wx', mode: 0o600, flush: true })
+ const input = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW)
try {
- writeFileSync(temporary, repaired.history, { flag: 'wx', mode: 0o600, flush: true })
- const current = lstatSync(file)
- if (current.ino !== before.ino || current.dev !== before.dev || current.size !== before.size
- || current.mtimeMs !== before.mtimeMs || readFileSync(file, 'utf8') !== original) {
- throw new Error('Codex rollout changed during resume preparation; retry after its writer stops')
+ const unchanged = () => {
+ if (!sameFile(before, fstatSync(input)) || !sameFile(before, lstatSync(file!))) throw changedDuringPreparation()
}
- renameSync(temporary, file)
- } finally {
- rmSync(temporary, { force: true })
- }
- return { repairedItems: repaired.repairedItems, repairedBytes: Buffer.byteLength(repaired.history, 'utf8'), backupPath }
+ unchanged()
+ const inspected = scanHistory(input, before.size, source.sessionId)
+ unchanged()
+ if (!inspected.repairedItems) return { repairedItems: 0 }
+
+ const suffix = randomUUID()
+ const backupPath = `${file}.reasoning-backup-${suffix}`
+ const temporary = `${file}.reasoning-tmp-${suffix}`
+ let backup: number | undefined
+ let output: number | undefined
+ let verifiedBackup = false
+ try {
+ backup = openSync(backupPath, 'wx', 0o600)
+ output = openSync(temporary, 'wx', 0o600)
+ // Re-read into private files only after every record has validated. The digest proves the
+ // backup and repaired output came from the exact bytes inspected, even across chunk reads.
+ const copied = scanHistory(input, before.size, source.sessionId, (original, repaired) => {
+ writeAll(backup!, original)
+ writeAll(output!, repaired)
+ })
+ if (copied.digest !== inspected.digest) throw changedDuringPreparation()
+ fsyncSync(backup)
+ fsyncSync(output)
+ unchanged()
+ verifiedBackup = true
+ renameSync(temporary, file)
+ return { repairedItems: copied.repairedItems, repairedBytes: copied.repairedBytes, backupPath }
+ } finally {
+ if (output !== undefined) { closeSync(output); rmSync(temporary, { force: true }) }
+ if (backup !== undefined) {
+ closeSync(backup)
+ if (!verifiedBackup) rmSync(backupPath, { force: true })
+ }
+ }
+ } finally { closeSync(input) }
}
diff --git a/cli/src/lib/terminalAgentReconciler.spec.ts b/cli/src/lib/terminalAgentReconciler.spec.ts
index de65d5ef1..ffbf2c4d4 100644
--- a/cli/src/lib/terminalAgentReconciler.spec.ts
+++ b/cli/src/lib/terminalAgentReconciler.spec.ts
@@ -31,6 +31,42 @@ function probe(targets: TerminalAgentProbe['targets'], agents: DiscoveredTermina
}
describe('composite terminal reconciliation', () => {
+ it('defers early startup hooks until restored panes have their original owners', async () => {
+ const current = { ...session([tmux]), sessionId: 'saved-conversation', processIdentity: null }
+ const onRemoved = vi.fn()
+ const onDiscovered = vi.fn()
+ const onObserved = vi.fn()
+ const probed = vi.fn(async (_hints: ReadonlyMap<string, unknown>) =>
+ probe([{ instanceId: 'tmux:default', result: { state: 'available' as const, roots: [] } }]))
+ const reconciler = new TerminalAgentReconciler({
+ deferUntilStart: true,
+ current: () => [current], backends: [], backendOrder: ['tmux'],
+ onDiscovered, onObserved, onDormant: vi.fn(), onRemoved, probe: probed,
+ })
+ const restored: TerminalRuntimeRef = { backend: 'tmux', paneId: '%24' }
+ // Hooks from an earlier restored pane used to run two full negative inventories here and
+ // remove this row before its new pane was allocated. Never block a hook waiting for boot.
+ await reconciler.triggerHint(restored, 'claude')
+ await reconciler.trigger()
+ expect(probed).not.toHaveBeenCalled()
+ expect(onRemoved).not.toHaveBeenCalled()
+
+ current.runtimes = [restored]
+ current.primaryRuntimeKey = terminalRouteKey(restored)
+ current.launch = { state: 'starting' }
+ const live = observed([restored])
+ probed.mockImplementation(async () => probe([
+ { instanceId: 'tmux:default', result: { state: 'available', roots: [{ runtime: restored, rootPid: 1, cwd: '/work' }] } },
+ ], [live]))
+ try {
+ await reconciler.start(60_000)
+ expect(probed).toHaveBeenCalledWith(new Map([[terminalRouteKey(restored), 'claude']]))
+ expect(onObserved).toHaveBeenCalledWith(live, current)
+ expect(onDiscovered).not.toHaveBeenCalled()
+ expect(onRemoved).not.toHaveBeenCalled()
+ } finally { reconciler.stop() }
+ })
+
it('advertises a retained pane even when no engine process is observed', async () => {
const current = { ...session([tmux]), active: false }
const onTerminalAvailability = vi.fn()
diff --git a/cli/src/lib/terminalAgentReconciler.ts b/cli/src/lib/terminalAgentReconciler.ts
index 527809e9d..986ea1e1c 100644
--- a/cli/src/lib/terminalAgentReconciler.ts
+++ b/cli/src/lib/terminalAgentReconciler.ts
@@ -25,6 +25,9 @@ export interface TerminalAgentReconcilerDeps {
transaction?: <T>(apply: () => T | Promise<T>) => Promise<T>
probe?: (hints: ReadonlyMap<string, AgentEngine>) => Promise<TerminalAgentProbe>
daemonPid?: number
+ /** Hooks may arrive while reboot restoration is still allocating panes. Keep their hints,
+ * but do not scan or retire any saved owners until start() opens discovery. */
+ deferUntilStart?: boolean
}
function currentProcessKey(session: RegisteredSession): string | null {
@@ -108,8 +111,11 @@ export class TerminalAgentReconciler {
private pending = false
private inFlight: Promise<void> | null = null
private timer: NodeJS.Timeout | null = null
+ private waitingForStart: boolean
- constructor(private readonly deps: TerminalAgentReconcilerDeps) {}
+ constructor(private readonly deps: TerminalAgentReconcilerDeps) {
+ this.waitingForStart = deps.deferUntilStart === true
+ }
/**
* Arm the interval FIRST, then run the opening pass.
@@ -120,6 +126,7 @@ export class TerminalAgentReconciler {
* pass is reported and dropped; the interval retries it a few seconds later.
*/
async start(intervalMs: number): Promise<void> {
+ this.waitingForStart = false
this.timer = setInterval(() => { void this.trigger() }, intervalMs)
this.timer.unref?.()
await this.trigger().catch((error) => {
@@ -202,6 +209,9 @@ export class TerminalAgentReconciler {
trigger(): Promise<void> {
this.pending = true
+ // Return promptly to startup hooks: waiting for start() here can hold up the very engines
+ // restore is trying to launch. The opening pass consumes every retained hint after restore.
+ if (this.waitingForStart) return Promise.resolve()
if (!this.inFlight) this.inFlight = this.drain().finally(() => { this.inFlight = null })
return this.inFlight
}