Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Bound history pages and the live tail by what they serve (#713)

Dee79f40741a687
diff --git a/cli/e2e/bounded.e2e.ts b/cli/e2e/bounded.e2e.ts
new file mode 100644
index 000000000..a6def0647
--- /dev/null
+++ b/cli/e2e/bounded.e2e.ts
@@ -0,0 +1,91 @@
+/**
+ * Memory follows activity, never history. On 2026-10-03 a Codex rollout of 803 MB crash-looped the
+ * daemon at its 4 GB heap: attaching read the whole transcript. These run the daemon under a heap far
+ * smaller than the transcripts it serves.
+ */
+import { mkdirSync, statSync, truncateSync } from 'node:fs'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
+import { LocalClient, type Frame } from './harness/client.js'
+import { IsolatedDaemon, until } from './harness/daemon.js'
+
+const HEAP_MIB = 256
+const TRANSCRIPT_MIB = 300
+
+describe('bounded memory', () => {
+ let daemon: IsolatedDaemon | undefined
+ afterEach(async () => { await daemon?.close(); daemon = undefined })
+
+ it.each(['claude', 'codex'] as const)(`%s: a ${TRANSCRIPT_MIB} MiB transcript re-attaches under a ${HEAP_MIB} MiB heap`, async (engine) => {
+ daemon = await IsolatedDaemon.create({ heapMiB: HEAP_MIB })
+ const d = daemon
+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-60).join('\n')}`) })
+ await d.start()
+ let client = await LocalClient.connect(d)
+ const cwd = join(d.projectsDir, engine)
+ mkdirSync(cwd, { recursive: true })
+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 60_000)
+ const agentId: string = created.agent.id
+ const row = async (c: LocalClient) => ((await c.request('agents_list', { includeStopped: true })).agents as Array<Record<string, any>>)
+ .find((agent) => agent.id === agentId)
+ await until('the conversation to bind', async () => (await row(client))?.sessionId, 45_000, 250)
+
+ const isEnd = (frame: Frame) => frame.type === 'turn_ended' && frame.agentId === agentId
+ const grown = client.next(isEnd, 170_000, 'the growing turn to end')
+ client.send('message', { agentId, content: `!grow ${TRANSCRIPT_MIB}` })
+ await grown
+ expect(d.child, 'the daemon survived the live tail').not.toBeNull()
+
+ await d.restart()
+ client = await LocalClient.connect(d)
+ const back = await until('the agent to be back', async () => {
+ const agent = await row(client)
+ return agent?.status && agent.status !== 'stopped' ? agent : null
+ }, 60_000, 250)
+ expect(back.status).not.toBe('stopped')
+ const next = client.next(isEnd, 30_000, 'a turn after the restart')
+ client.send('message', { agentId, content: 'still here?' })
+ await next
+ expect(d.child, 'the daemon survived the attach').not.toBeNull()
+ expect(await d.rssMiB()).toBeLessThan(HEAP_MIB + 200)
+
+ // A phone or the web opening the thread: its length, then page after page back into the turn that
+ // wrote the 300 MiB — each page read on its own, never the whole file (lib/transcriptPages.ts).
+ const sessionId: string = (await row(client))!.sessionId
+ const listed = await client.request('sessions_list', { agentId })
+ expect(listed.sessions[0].messageCount).toBeGreaterThan(TRANSCRIPT_MIB)
+ let before: string | undefined
+ let pages = 0
+ for (; pages < 8; pages++) {
+ const page = await client.request('session_get', { sessionId, limit: 50, before }, 60_000)
+ expect(page.staleCursor, `page ${pages}`).toBeUndefined()
+ // Compaction records render nothing, so a page inside the long turn can hold no events: what
+ // matters is that every page moves back.
+ expect(Array.isArray(page.events), `page ${pages}`).toBe(true)
+ expect(page.oldestCursor, `page ${pages}`).not.toBe(before)
+ if (!page.hasMore || !page.oldestCursor) break
+ before = page.oldestCursor
+ }
+ expect(pages, 'pages read back into the long turn').toBe(8)
+ // An app from before paging asks for the whole thread: it gets the newest page's worth, and a cursor.
+ const whole = await client.request('session_get', { sessionId }, 60_000)
+ expect(whole).toMatchObject({ hasMore: true })
+ expect(d.child, 'the daemon survived the paging').not.toBeNull()
+ expect(await d.rssMiB()).toBeLessThan(HEAP_MIB + 200)
+
+ // The transcript rewritten in place, two thirds of it left: more history than the heap holds, so it
+ // cannot be replayed in one batch. The tail moves to the new end and the session is attached again,
+ // from that end.
+ const transcript = engine === 'claude'
+ ? join(d.env.CLAUDE_PROJECTS_DIR!, cwd.replace(/[^a-zA-Z0-9]/g, '-'), `${sessionId}.jsonl`)
+ : join(d.env.CODEX_HOME!, 'sessions', '2026', '10', '03', `rollout-2026-10-03T00-00-00-${sessionId}.jsonl`)
+ truncateSync(transcript, Math.floor(statSync(transcript).size * 2 / 3))
+ const after = client.next(isEnd, 60_000, 'a turn after the rewrite')
+ client.send('message', { agentId, content: 'and after a rewrite?' })
+ await after
+ await until('the rewrite to be attached again', () => /transcript rewritten in place — attaching it again/.test(d.log()), 30_000)
+ expect(d.child, 'the daemon survived the rewrite').not.toBeNull()
+ expect(await d.rssMiB()).toBeLessThan(HEAP_MIB + 200)
+ })
+})
+
diff --git a/cli/e2e/harness/client.ts b/cli/e2e/harness/client.ts
new file mode 100644
index 000000000..ced0e48f3
--- /dev/null
+++ b/cli/e2e/harness/client.ts
@@ -0,0 +1,78 @@
+/**
+ * A client that speaks to the daemon exactly as the desktop app does: the local WebSocket over the
+ * daemon's Unix socket (or its loopback port), `machine_select`, request frames answered by
+ * `<type>_result`, and the event stream in between.
+ */
+import { randomUUID } from 'node:crypto'
+import WebSocket from 'ws'
+import type { IsolatedDaemon } from './daemon.js'
+
+export type Frame = { type: string; payload?: Record<string, any>; [key: string]: unknown }
+
+export class LocalClient {
+ readonly frames: Frame[] = []
+ private waiters: Array<{ test: (frame: Frame) => boolean; done: (frame: Frame) => void }> = []
+ closed = false
+
+ private constructor(private readonly ws: WebSocket) {
+ ws.on('message', (raw, binary) => {
+ if (binary) return
+ let frame: Frame
+ try { frame = JSON.parse(raw.toString()) as Frame } catch { return }
+ this.frames.push(frame)
+ for (const waiter of [...this.waiters]) {
+ if (waiter.test(frame)) { this.waiters.splice(this.waiters.indexOf(waiter), 1); waiter.done(frame) }
+ }
+ })
+ ws.on('close', () => { this.closed = true })
+ }
+
+ static async connect(daemon: IsolatedDaemon, options: { tcp?: boolean; tool?: boolean } = {}): Promise<LocalClient> {
+ const url = options.tcp
+ ? `ws://127.0.0.1:${daemon.port}/api/local-ws`
+ : `ws+unix://${daemon.socketPath}:/api/local-ws`
+ const ws = new WebSocket(url)
+ await new Promise<void>((resolve, reject) => {
+ ws.once('open', () => resolve())
+ ws.once('error', reject)
+ })
+ const client = new LocalClient(ws)
+ client.send('machine_select', { machineId: daemon.computerId, localProtocolVersion: 1, ...(options.tool ? { tool: true } : {}) })
+ await client.waitFor((frame) => frame.type === 'connected', 10_000, 'connected')
+ return client
+ }
+
+ send(type: string, payload: Record<string, unknown>): void {
+ this.ws.send(JSON.stringify({ type, payload }))
+ }
+
+ /** Resolves with the first frame, past or future, that passes `test`. */
+ waitFor(test: (frame: Frame) => boolean, ms = 20_000, what = 'a frame', since = 0): Promise<Frame> {
+ const seen = this.frames.slice(since).find(test)
+ if (seen) return Promise.resolve(seen)
+ return new Promise((resolve, reject) => {
+ const timer = setTimeout(() => {
+ this.waiters = this.waiters.filter((waiter) => waiter.done !== done)
+ reject(new Error(`timed out after ${ms}ms waiting for ${what}; last frames: ${JSON.stringify(this.frames.slice(-8).map((f) => f.type))}`))
+ }, ms)
+ const done = (frame: Frame) => { clearTimeout(timer); resolve(frame) }
+ this.waiters.push({ test, done })
+ })
+ }
+
+ /** Only frames that arrive after this call. */
+ next(test: (frame: Frame) => boolean, ms = 20_000, what = 'a frame'): Promise<Frame> {
+ return this.waitFor(test, ms, what, this.frames.length)
+ }
+
+ async request<T = Record<string, any>>(type: string, payload: Record<string, unknown> = {}, ms = 30_000): Promise<T> {
+ const requestId = randomUUID()
+ const answer = this.next((frame) => frame.type === `${type}_result` && frame.payload?.requestId === requestId, ms, `${type}_result`)
+ this.send(type, { requestId, ...payload })
+ return (await answer).payload as T
+ }
+
+ close(): void {
+ try { this.ws.close() } catch { /* already closed */ }
+ }
+}
diff --git a/cli/e2e/harness/daemon.ts b/cli/e2e/harness/daemon.ts
new file mode 100644
index 000000000..129d2cd0e
--- /dev/null
+++ b/cli/e2e/harness/daemon.ts
@@ -0,0 +1,202 @@
+/**
+ * A daemon under test: the real daemon from this checkout, booted under a throwaway home with its own
+ * data directory, port, private tmux server and fake Claude Code and Codex engines, signed out and with
+ * every optional feature that reaches beyond the machine turned off. Nothing it does can touch the
+ * person's own daemon, tmux server, transcripts or credentials.
+ */
+import { execFile, spawn, type ChildProcess } from 'node:child_process'
+import { existsSync, readFileSync } from 'node:fs'
+import { mkdir, writeFile } from 'node:fs/promises'
+import { createServer } from 'node:net'
+import { dirname, join, resolve } from 'node:path'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+import { promisify } from 'node:util'
+import { isolatedTmux, type IsolatedTmux } from '../../src/testing/isolatedTmux.js'
+
+const exec = promisify(execFile)
+const here = dirname(fileURLToPath(import.meta.url))
+export const CLI_ROOT = resolve(here, '../..')
+
+export interface DaemonOptions {
+ /** Extra environment for the daemon. */
+ env?: Record<string, string>
+ /** V8 heap ceiling for the daemon process, MiB. Defaults to Node's own. */
+ heapMiB?: number
+ /** Models the fake engines report. */
+ claudeModel?: string
+ codexModel?: string
+}
+
+async function freePort(): Promise<number> {
+ const server = createServer()
+ await new Promise<void>((done) => server.listen(0, '127.0.0.1', done))
+ const address = server.address()
+ await new Promise<void>((done) => server.close(() => done()))
+ if (!address || typeof address === 'string') throw new Error('no free port')
+ return address.port
+}
+
+export const until = async <T>(what: string, probe: () => Promise<T | null | undefined | false> | T | null | undefined | false, ms = 30_000, every = 100): Promise<T> => {
+ const deadline = Date.now() + ms
+ let last: unknown
+ while (Date.now() < deadline) {
+ try {
+ const value = await probe()
+ if (value) return value
+ } catch (error) { last = error }
+ await new Promise((done) => setTimeout(done, every))
+ }
+ throw new Error(`timed out after ${ms}ms waiting for ${what}${last ? ` (last error: ${String(last)})` : ''}`)
+}
+
+export class IsolatedDaemon {
+ child: ChildProcess | null = null
+ private output = ''
+
+ private constructor(
+ readonly root: string,
+ readonly tmux: IsolatedTmux,
+ readonly port: number,
+ readonly env: NodeJS.ProcessEnv,
+ readonly options: DaemonOptions,
+ ) {}
+
+ static async create(options: DaemonOptions = {}): Promise<IsolatedDaemon> {
+ const tmux = await isolatedTmux()
+ const root = tmux.root
+ const port = await freePort()
+ const dirs = {
+ home: join(root, 'home'), data: join(root, 'data'), runtime: join(root, 'runtime'), auth: join(root, 'auth'),
+ bin: join(root, 'bin'), dsh: join(root, 'dsh'), claudeProjects: join(root, 'claude', 'projects'), codexHome: join(root, 'codex'),
+ projects: join(root, 'projects'),
+ }
+ for (const dir of Object.values(dirs)) await mkdir(dir, { recursive: true })
+ const config = {
+ port, dataDir: dirs.data, claudeProjectsDir: dirs.claudeProjects, codexHome: dirs.codexHome,
+ claudeModel: options.claudeModel, codexModel: options.codexModel,
+ }
+ const engine = pathToFileURL(join(here, 'fakeEngine.mjs')).href
+ for (const name of ['claude', 'codex']) {
+ await writeFile(join(dirs.bin, name),
+ `#!${process.execPath}\nimport(${JSON.stringify(engine)}).then((m) => m.run(${JSON.stringify(name)}, ${JSON.stringify(config)}))\n`,
+ { mode: 0o755 })
+ }
+ const env: NodeJS.ProcessEnv = {
+ ...tmux.env,
+ NODE_ENV: 'test',
+ HOME: dirs.home,
+ // A login shell in a pane must not load anyone's zsh configuration.
+ ZDOTDIR: dirs.home,
+ PORT: String(port),
+ ADAPTER_DATA_DIR: dirs.data,
+ ADAPTER_RUNTIME_DIR: dirs.runtime,
+ ADAPTER_COMPUTER_ID: 'e2e-computer-0000-0000-000000000001',
+ ADAPTER_COMPUTER_ID_FILE: join(root, 'computer-id'),
+ HARNESS_AUTH_DIR: dirs.auth,
+ DSH_DIR: dirs.dsh,
+ CLAUDE_PATH: join(dirs.bin, 'claude'),
+ CODEX_PATH: join(dirs.bin, 'codex'),
+ CLAUDE_PROJECTS_DIR: dirs.claudeProjects,
+ CODEX_HOME: dirs.codexHome,
+ PATH: `${dirs.bin}:${process.env.PATH}`,
+ // Signed out, and nothing reaches beyond the machine.
+ BACKEND_WS_URL: 'ws://127.0.0.1:9',
+ WEB_URL: 'http://127.0.0.1:9',
+ DISABLE_HOOK_INSTALL: 'true',
+ ADAPTER_UPDATE_DISABLE: 'true',
+ ANALYTICS_ENABLED: 'false',
+ RECAP_FORCE: 'false',
+ RECAP_WITHOUT_DEVICE: 'false',
+ CABLE_DISABLE: 'true',
+ CABLE_FW_DISABLE: 'true',
+ TERMINAL_BACKENDS: 'tmux',
+ TMUX_REAP_INTERVAL_MS: '5000',
+ TERMINAL_RECONCILE_INTERVAL_MS: '5000',
+ ...options.env,
+ }
+ return new IsolatedDaemon(root, tmux, port, env, options)
+ }
+
+ get dataDir(): string { return this.env.ADAPTER_DATA_DIR! }
+ get computerId(): string { return this.env.ADAPTER_COMPUTER_ID! }
+ get socketPath(): string { return join(this.dataDir, `daemon-${this.port}.sock`) }
+ get projectsDir(): string { return join(this.root, 'projects') }
+ get pid(): number | null { return this.child?.pid ?? null }
+ log(): string { return this.output }
+
+ /** `ready: 'port'` returns as soon as the port answers, the way a client sees a restarting daemon. */
+ async start(options: { ready?: 'port' | 'wired' } = {}): Promise<void> {
+ if (this.child) throw new Error('already running')
+ const heap = this.options.heapMiB ? [`--max-old-space-size=${this.options.heapMiB}`] : []
+ const child = spawn(process.execPath, [...heap, '--import', 'tsx', 'src/cli.ts', '__run'], {
+ cwd: CLI_ROOT, env: this.env, stdio: ['ignore', 'pipe', 'pipe'],
+ })
+ this.child = child
+ const take = (chunk: Buffer) => { this.output += chunk.toString('utf8') }
+ child.stdout!.on('data', take)
+ child.stderr!.on('data', take)
+ const exited = new Promise<never>((_, reject) => child.once('exit', (code, signal) => {
+ if (this.child === child) this.child = null
+ reject(new Error(`daemon exited during boot (${signal ?? code}):\n${this.output.slice(-4000)}`))
+ }))
+ exited.catch(() => {}) // observed below while booting; afterwards an exit is the test's business
+ await Promise.race([exited, (async () => {
+ await until('the daemon to answer on its port', async () => {
+ const response = await fetch(`http://127.0.0.1:${this.port}/api/health`).catch(() => null)
+ return response?.ok ? true : null
+ }, 60_000, 200)
+ await until('the daemon socket', () => existsSync(this.socketPath), 10_000)
+ // The port answers ~1,100 lines of startup before every handler is wired (see the startup-race
+ // test); this line is printed once they are.
+ if (options.ready !== 'port') {
+ await until('the daemon to finish starting', () => /\[cli\] (dialing|not signed in)/.test(this.output), 60_000, 100)
+ }
+ })()])
+ }
+
+ /** SIGTERM, then SIGKILL after `ms`. Resolves when the process is gone. */
+ async stop(ms = 15_000): Promise<void> {
+ const child = this.child
+ if (!child) return
+ const exited = new Promise<void>((done) => child.once('exit', () => done()))
+ child.kill('SIGTERM')
+ const timer = setTimeout(() => child.kill('SIGKILL'), ms)
+ await exited
+ clearTimeout(timer)
+ }
+
+ /** The way a crash ends it: no shutdown path runs. */
+ async kill(): Promise<void> {
+ const child = this.child
+ if (!child) return
+ const exited = new Promise<void>((done) => child.once('exit', () => done()))
+ child.kill('SIGKILL')
+ await exited
+ }
+
+ async restart(): Promise<void> {
+ await this.stop()
+ await this.start()
+ }
+
+ /** Resident memory of the daemon process, MiB. */
+ async rssMiB(): Promise<number> {
+ if (!this.child?.pid) return 0
+ const { stdout } = await exec('ps', ['-o', 'rss=', '-p', String(this.child.pid)])
+ return Number(stdout.trim()) / 1024
+ }
+
+ /** What the daemon's pane shows, plain text. */
+ async capture(pane: string): Promise<string> {
+ return this.tmux.run('capture-pane', '-p', '-t', pane)
+ }
+
+ hookCredential(): string {
+ return readFileSync(join(this.dataDir, 'hook-credential'), 'utf8').trim()
+ }
+
+ async close(): Promise<void> {
+ await this.stop().catch(() => {})
+ await this.tmux.close()
+ }
+}
diff --git a/cli/e2e/harness/fakeEngine.mjs b/cli/e2e/harness/fakeEngine.mjs
new file mode 100644
index 000000000..1b9d24ad0
--- /dev/null
+++ b/cli/e2e/harness/fakeEngine.mjs
@@ -0,0 +1,178 @@
+// A deterministic stand-in for Claude Code and Codex, run inside a real tmux pane by the daemon
+// under test. It behaves the way the daemon depends on the real CLIs behaving, and nothing more:
+//
+// - answers `--version` / `--help` probes;
+// - announces its session through the same authenticated hook the real `notify.mjs` uses, from
+// inside the pane (so the daemon's pane/process binding is exercised for real);
+// - draws a composer, turns on bracketed paste, and treats Enter as submit;
+// - writes its conversation to a transcript in the engine's real record shapes, so the daemon's
+// normalizers, turn lifecycle, chips and attach read are exercised for real.
+//
+// A prompt can carry directives that script the turn: `!slow <ms>` holds the answer back,
+// `!tool <command>` runs a tool call first, `!grow <MiB>` appends that much compaction history
+// before answering (the transcripts that crashed the daemon on 2026-10-03), `!hold` leaves the turn
+// open until the next prompt, `!exit` ends the process. Everything else is echoed as the answer.
+import { randomUUID } from 'node:crypto'
+import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+
+export async function run(engine, config) {
+ const args = process.argv.slice(2)
+ if (args.includes('--version') || args.includes('-V')) {
+ process.stdout.write(engine === 'claude' ? '2.1.270 (Claude Code)\n' : 'codex-cli 0.159.0\n')
+ return
+ }
+ if (args.includes('--help') || args.includes('-h')) {
+ // The flags the daemon looks for in the real CLIs' help before it launches them.
+ process.stdout.write(engine === 'codex'
+ ? [
+ 'Usage: codex [OPTIONS] [PROMPT]', '',
+ ' -c, --config <key=value>', ' -m, --model <MODEL>', ' -s, --sandbox <SANDBOX_MODE>',
+ ' -a, --ask-for-approval <APPROVAL_POLICY>', ' --approve-for-me',
+ ' --dangerously-bypass-approvals-and-sandbox', ' --no-daemon',
+ 'Commands:', ' resume Resume a previous interactive session', '',
+ ].join('\n')
+ : [
+ 'Usage: claude [options] [command] [prompt]', '', 'Arguments:', ' prompt Your prompt', '',
+ ' --model <model>', ' --permission-mode <mode> (choices: "acceptEdits", "auto", "bypassPermissions", "default", "plan")',
+ ' --dangerously-skip-permissions', ' -r, --resume [value]', ' --fork-session', ' --session-id <uuid>', '',
+ ].join('\n'))
+ return
+ }
+ process.title = engine
+
+ const resumeAt = engine === 'claude' ? args.indexOf('--resume') : args.indexOf('resume')
+ const resumed = resumeAt >= 0 && args[resumeAt + 1] && !args[resumeAt + 1].startsWith('-') ? args[resumeAt + 1] : null
+ const fork = args.includes('--fork-session')
+ const sessionId = resumed && !fork ? resumed : randomUUID()
+ const cwd = process.cwd()
+ const transcript = engine === 'claude'
+ ? join(config.claudeProjectsDir, cwd.replace(/[^a-zA-Z0-9]/g, '-'), `${sessionId}.jsonl`)
+ : resumed && !fork && config.rolloutFor?.[resumed]
+ ? config.rolloutFor[resumed]
+ : join(config.codexHome, 'sessions', '2026', '10', '03', `rollout-2026-10-03T00-00-00-${sessionId}.jsonl`)
+ mkdirSync(dirname(transcript), { recursive: true })
+ if (!existsSync(transcript)) writeFileSync(transcript, '')
+
+ const now = () => new Date().toISOString()
+ const write = (record) => appendFileSync(transcript, JSON.stringify(record) + '\n')
+ let parent = null
+ const claude = (record) => {
+ const uuid = randomUUID()
+ write({ parentUuid: parent, isSidechain: false, userType: 'external', cwd, sessionId, version: '2.1.270', timestamp: now(), uuid, ...record })
+ parent = uuid
+ }
+ const codex = (type, payload) => write({ timestamp: now(), type, payload })
+ if (engine === 'codex' && readFileSync(transcript, 'utf8') === '') {
+ codex('session_meta', { id: sessionId, cli_version: '0.159.0', cwd, source: 'cli' })
+ }
+
+ const hook = async (path, body) => {
+ const token = readFileSync(join(config.dataDir, 'hook-credential'), 'utf8').trim()
+ const pane = process.env.TMUX_PANE
+ const response = await fetch(`http://127.0.0.1:${config.port}/api/hook/${path}`, {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', 'x-harness-hook-token': token },
+ body: JSON.stringify({
+ engine, sessionId, transcriptPath: transcript, cwd,
+ tmuxPane: pane, runtimeHints: pane ? [{ backend: 'tmux', paneId: pane }] : [], callerPid: process.pid,
+ ...body,
+ }),
+ }).catch((error) => ({ ok: false, status: 0, text: async () => String(error) }))
+ if (!response.ok) process.stdout.write(`\r\n[fake ${engine}] hook ${path} failed: ${response.status}\r\n`)
+ }
+
+ const draw = (line = '') => process.stdout.write(`\r\x1b[2K› ${line}`)
+ process.stdout.write(`\x1b[?2004h${engine === 'claude' ? '✻ Welcome to Claude Code (fake)' : '>_ OpenAI Codex (fake)'}\r\n`)
+ process.stdout.write(` session ${sessionId}${resumed ? ' (resumed)' : ''}\r\n\r\n`)
+ draw()
+ await hook('session-start', { hookEvent: resumed ? 'SessionStart' : 'SessionStart', source: resumed ? 'resume' : 'startup' })
+
+ let turn = 0
+ let open = null
+ const finish = (text) => {
+ if (engine === 'claude') {
+ claude({ type: 'assistant', message: { id: `msg_${turn}`, role: 'assistant', model: config.claudeModel ?? 'claude-opus-5-5', content: [{ type: 'text', text }], stop_reason: 'end_turn' } })
+ } else {
+ codex('event_msg', { type: 'item_completed', item: { type: 'AgentMessage', content: [{ type: 'Text', text }], phase: 'final_answer' } })
+ codex('event_msg', { type: 'task_complete', turn_id: open, last_agent_message: text })
+ }
+ process.stdout.write(`\r\n${text}\r\n\r\n`)
+ open = null
+ draw()
+ if (engine === 'claude') void hook('turn-stop', { status: 'ok' })
+ }
+
+ const handle = async (raw) => {
+ const prompt = raw.trim()
+ if (!prompt) { draw(); return }
+ if (prompt === '!exit') { process.stdout.write('\x1b[?2004l\r\n'); process.exit(0) }
+ if (open) finish('(interrupted by a new prompt)')
+ turn++
+ open = `turn-${turn}`
+ process.stdout.write(`\r\n`)
+ if (engine === 'claude') {
+ claude({ type: 'user', message: { role: 'user', content: prompt } })
+ claude({ type: 'assistant', message: { id: `msg_${turn}_t`, role: 'assistant', model: config.claudeModel ?? 'claude-opus-5-5', content: [{ type: 'thinking', thinking: `considering: ${prompt}` }], stop_reason: null } })
+ } else {
+ codex('event_msg', { type: 'task_started', turn_id: open })
+ codex('turn_context', { turn_id: open, model: config.codexModel ?? 'gpt-6', reasoning_effort: 'high', collaboration_mode: { mode: 'default' } })
+ codex('event_msg', { type: 'item_completed', turn_id: open, item: { type: 'UserMessage', id: open, content: [{ type: 'text', text: prompt, text_elements: [] }] } })
+ codex('response_item', { type: 'reasoning', summary: [{ type: 'summary_text', text: `considering: ${prompt}` }] })
+ }
+ const directive = /^!(\w+)\s*(.*)$/.exec(prompt)
+ if (directive?.[1] === 'grow' || directive?.[1] === 'burst') {
+ // `grow` writes the way Codex compacts — a snapshot at a time, with the engine still working in
+ // between — so a live tail sees a few MiB per read. `burst` writes it all at once.
+ const mib = Number(directive[2]) || 1
+ const chunk = 'x'.repeat(1024 * 1024 - 256)
+ for (let i = 0; i < mib; i++) {
+ if (engine === 'claude') claude({ type: 'user', isCompactSummary: true, message: { role: 'user', content: chunk } })
+ else codex('compacted', { message: chunk, replacement_history: [] })
+ if (directive[1] === 'grow' && i % 4 === 3) await new Promise((resolve) => setTimeout(resolve, 60))
+ }
+ }
+ if (directive?.[1] === 'tool') {
+ const id = `call_${turn}`
+ if (engine === 'claude') {
+ claude({ type: 'assistant', message: { id: `msg_${turn}_u`, role: 'assistant', model: config.claudeModel ?? 'claude-opus-5-5', content: [{ type: 'tool_use', id, name: 'Bash', input: { command: directive[2] } }], stop_reason: 'tool_use' } })
+ claude({ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: id, content: `ran ${directive[2]}` }] } })
+ } else {
+ codex('response_item', { type: 'function_call', call_id: id, name: 'exec_command', arguments: JSON.stringify({ cmd: directive[2] }) })
+ codex('response_item', { type: 'function_call_output', call_id: id, output: `ran ${directive[2]}` })
+ }
+ }
+ if (directive?.[1] === 'hold') return
+ if (directive?.[1] === 'slow') await new Promise((resolve) => setTimeout(resolve, Number(directive[2]) || 1000))
+ finish(`answer ${turn}: ${prompt}`)
+ }
+
+ // Raw input: bracketed paste brackets the text, Enter (\r) submits, Ctrl-C interrupts.
+ process.stdin.setRawMode?.(true)
+ process.stdin.setEncoding('utf8')
+ let buffer = ''
+ let queue = Promise.resolve()
+ process.stdin.on('data', (chunk) => {
+ for (const part of chunk.replace(/\x1b\[20[01]~/g, '').split(/(\r|\n|\x03)/)) {
+ if (part === '\x03') {
+ if (open) {
+ if (engine === 'claude') claude({ type: 'user', message: { role: 'user', content: [{ type: 'text', text: '[Request interrupted by user]' }] } })
+ else codex('event_msg', { type: 'turn_aborted', turn_id: open })
+ open = null
+ process.stdout.write('\r\n(interrupted)\r\n')
+ draw()
+ }
+ buffer = ''
+ } else if (part === '\r' || part === '\n') {
+ const line = buffer
+ buffer = ''
+ queue = queue.then(() => handle(line))
+ } else if (part) {
+ buffer += part
+ draw(buffer)
+ }
+ }
+ })
+ process.on('SIGTERM', () => process.exit(0))
+ setInterval(() => {}, 60_000)
+}
diff --git a/cli/e2e/lifecycle.e2e.ts b/cli/e2e/lifecycle.e2e.ts
new file mode 100644
index 000000000..18f5c997c
--- /dev/null
+++ b/cli/e2e/lifecycle.e2e.ts
@@ -0,0 +1,76 @@
+/**
+ * The daemon's everyday contract, end to end, for Claude Code and Codex: a client connects the way the
+ * desktop app does, starts an agent, the agent binds its conversation, a message becomes a turn that
+ * starts and ends, the daemon restarts and picks the agent back up, and the agent stops.
+ */
+import { mkdirSync } from 'node:fs'
+import { join } from 'node:path'
+import { afterAll, afterEach, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
+import { LocalClient, type Frame } from './harness/client.js'
+import { IsolatedDaemon, until } from './harness/daemon.js'
+
+const engines = ['claude', 'codex'] as const
+
+describe('the daemon, end to end', () => {
+ let daemon: IsolatedDaemon
+ beforeAll(async () => { daemon = await IsolatedDaemon.create(); await daemon.start() })
+ afterAll(async () => { await daemon?.close() })
+ afterEach(() => {})
+
+
+ const row = async (client: LocalClient, agentId: string) =>
+ ((await client.request<{ agents: Array<Record<string, any>> }>('agents_list', { includeStopped: true })).agents)
+ .find((agent) => agent.id === agentId)
+
+ it('answers the desktop handshake on its socket and its port, with no agents', async () => {
+ for (const tcp of [false, true]) {
+ const client = await LocalClient.connect(daemon, { tcp })
+ expect((await client.request('agents_list', {})).agents).toEqual([])
+ client.close()
+ }
+ })
+
+ it.each(engines)('%s: start, bind, a message becomes a turn, restart, stop', async (engine) => {
+ onTestFailed(() => { console.log(`---- daemon log (${engine})\n${daemon.log().split('\n').slice(-80).join('\n')}`) })
+ const client = await LocalClient.connect(daemon)
+ const cwd = join(daemon.projectsDir, engine)
+ mkdirSync(cwd, { recursive: true })
+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 60_000)
+ expect(created.error, JSON.stringify(created)).toBeUndefined()
+ const agentId: string = created.agent.id
+
+ const bound = await until(`${engine} to bind its conversation`, async () => {
+ const agent = await row(client, agentId)
+ return agent?.sessionId ? agent : null
+ }, 45_000, 500)
+ expect(bound.status).not.toBe('stopped')
+
+ const isTurn = (type: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId
+ const started = client.next(isTurn('turn_started'), 30_000, 'turn_started')
+ const ended = client.next(isTurn('turn_ended'), 30_000, 'turn_ended')
+ client.send('message', { agentId, content: 'hello there' })
+ expect((await started).payload?.userMessage).toBe('hello there')
+ await ended
+
+ // A restart re-attaches the session from its transcript: no turn is replayed as open.
+ await daemon.restart()
+ const again = await LocalClient.connect(daemon)
+ const seen: string[] = []
+ const back = await until(`${engine} to be back after a restart`, async () => {
+ const agent = await row(again, agentId)
+ seen.push(`${agent?.status}:${agent?.sessionId?.slice(0, 8)}`)
+ return agent?.sessionId === bound.sessionId && agent.status !== 'stopped' ? agent : null
+ }, 45_000, 250)
+ console.log(`${engine} rows seen after the restart: ${seen.join(' ')}`)
+ expect(back.status).not.toBe('stopped')
+ const second = again.next(isTurn('turn_ended'), 30_000, 'second turn_ended')
+ again.send('message', { agentId, content: 'and again' })
+ await second
+
+ const stopped = await again.request('agent_delete', { agentId }, 60_000)
+ expect(stopped.error, JSON.stringify(stopped)).toBeUndefined()
+ await until(`${engine} to stop`, async () => (await row(again, agentId))?.status === 'stopped' || null, 45_000, 500)
+ again.close()
+ client.close()
+ })
+})
diff --git a/cli/e2e/startup.e2e.ts b/cli/e2e/startup.e2e.ts
new file mode 100644
index 000000000..78385ffbc
--- /dev/null
+++ b/cli/e2e/startup.e2e.ts
@@ -0,0 +1,71 @@
+/**
+ * What a client sees while the daemon starts. The desktop app connects the moment the daemon's port
+ * answers, so everything the daemon says from then on is something a person sees.
+ *
+ * Known defects of the current daemon are pinned with `it.fails`: each passes today only because it
+ * fails, and harnessd must turn it green (then it becomes an ordinary `it`).
+ */
+import { mkdirSync } from 'node:fs'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import { LocalClient } from './harness/client.js'
+import { IsolatedDaemon, until } from './harness/daemon.js'
+
+describe('starting up', () => {
+ let daemon: IsolatedDaemon | undefined
+ afterEach(async () => { await daemon?.close(); daemon = undefined })
+
+ // The port answers, and the pid file is written, long before every request handler is wired:
+ // cli.ts writes PID_FILE at ~4325, assigns backend.onCreateAgent at ~5435 and onForkAgent later
+ // still, with the restore and the first full reconcile awaited in between. A request in that window
+ // is refused with UNSUPPORTED_ON_REMOTE, which no client expects from its own machine.
+ it.fails('answers every request with its real handler from the instant its port answers', async () => {
+ daemon = await IsolatedDaemon.create()
+ await daemon.start()
+ const setup = await LocalClient.connect(daemon)
+ for (const name of ['a', 'b', 'c']) {
+ const cwd = join(daemon.projectsDir, name)
+ mkdirSync(cwd, { recursive: true })
+ await setup.request('agent_create', { engine: 'claude', cwd, bypassPermission: true }, 60_000)
+ }
+ setup.close()
+ await daemon.stop()
+ await daemon.start({ ready: 'port' })
+ const client = await LocalClient.connect(daemon)
+ const errors: string[] = []
+ const deadline = Date.now() + 3_000
+ while (Date.now() < deadline) {
+ errors.push((await client.request('agent_fork', { agentId: 'no-such-agent' })).error)
+ }
+ expect(errors).not.toContain('UNSUPPORTED_ON_REMOTE')
+ })
+
+ // After a restart the registry is served before the first reconcile confirms the panes: a running
+ // agent lists as `stopped` (when it had a conversation) or not at all, for ~100-200 ms after the
+ // daemon says it is ready. The app shows it paused or gone meanwhile.
+ it.fails('never shows a running agent as stopped or missing across a restart', async () => {
+ daemon = await IsolatedDaemon.create()
+ await daemon.start()
+ let client = await LocalClient.connect(daemon)
+ const cwd = join(daemon.projectsDir, 'restart')
+ mkdirSync(cwd, { recursive: true })
+ const agentId: string = (await client.request('agent_create', { engine: 'claude', cwd, bypassPermission: true }, 60_000)).agent.id
+ const row = async () => ((await client.request('agents_list', { includeStopped: true })).agents as Array<Record<string, any>>)
+ .find((agent) => agent.id === agentId)
+ await until('the conversation to bind', async () => (await row())?.sessionId, 45_000, 200)
+ const ended = client.next((frame) => frame.type === 'turn_ended' && frame.agentId === agentId)
+ client.send('message', { agentId, content: 'hello' })
+ await ended
+
+ await daemon.stop()
+ await daemon.start({ ready: 'port' })
+ client = await LocalClient.connect(daemon)
+ const seen: string[] = []
+ const deadline = Date.now() + 5_000
+ while (Date.now() < deadline) {
+ seen.push((await row())?.status ?? 'absent')
+ await new Promise((done) => setTimeout(done, 50))
+ }
+ expect(seen.filter((status) => status === 'stopped' || status === 'absent')).toEqual([])
+ })
+})
diff --git a/cli/package.json b/cli/package.json
index 052545b18..474a6fc60 100644
--- a/cli/package.json
+++ b/cli/package.json
@@ -14,6 +14,7 @@
"start": "node dist/cli.js",
"test": "vitest run",
"test:local-models": "vitest run src/lib/localModels.spec.ts src/lib/localModels.integration.spec.ts src/dsh/builtins.spec.ts src/lib/gridFleetRpc.spec.ts --coverage --coverage.include=src/lib/localModels.ts --coverage.include=src/dsh/builtins.ts --coverage.thresholds.lines=100 --coverage.thresholds.statements=100 --coverage.thresholds.functions=100 --coverage.thresholds.branches=100 --coverage.reporter=text --coverage.reporter=json-summary --coverage.reportsDirectory=coverage/local-models",
+ "test:e2e": "vitest run --config vitest.e2e.config.ts",
"test:orchestrator": "vitest run src/orchestrator --maxWorkers=1 --coverage --coverage.include='src/orchestrator/**' --coverage.thresholds.lines=100 --coverage.thresholds.statements=100 --coverage.thresholds.functions=100 --coverage.thresholds.branches=100 --coverage.reporter=text --coverage.reporter=json --coverage.reportsDirectory=coverage/orchestrator",
"test:dsh-updates": "vitest run --config vitest.dsh-updates.config.ts",
"test:sharing": "vitest run src/sharing --coverage --coverage.include='src/sharing/**' --coverage.thresholds.lines=100 --coverage.thresholds.statements=100 --coverage.thresholds.branches=100 --coverage.thresholds.functions=100",
diff --git a/cli/src/backendSocket.history.spec.ts b/cli/src/backendSocket.history.spec.ts
new file mode 100644
index 000000000..3ee8c3268
--- /dev/null
+++ b/cli/src/backendSocket.history.spec.ts
@@ -0,0 +1,153 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { BackendSocket } from './backendSocket.js'
+import { codexMessagesToEvents, windowCodexLines } from './engines/codex/normalizer.js'
+import { messagesToEvents, subagentStatsFromRawLines, windowRawLines } from './lib/normalize.js'
+import { registry } from './lib/registry.js'
+import { tailFile } from './lib/transcriptTail.js'
+import { cl, claude, claudeScenario, codexScenario } from './testing/transcriptScenarios.js'
+
+// Transcripts are only served from the engines' own folders. Those default to this computer's real
+// ~/.claude and ~/.codex, so they are pointed at throwaway ones before the config is read.
+const roots = vi.hoisted(() => {
+ const base = `${process.env.TMPDIR || '/tmp'}/history-roots-${process.pid}-${Date.now()}`
+ process.env.CLAUDE_PROJECTS_DIR = `${base}/claude-projects`
+ process.env.CODEX_HOME = `${base}/codex`
+ return { base, claude: process.env.CLAUDE_PROJECTS_DIR, codex: `${process.env.CODEX_HOME}/sessions` }
+})
+afterAll(() => rmSync(roots.base, { recursive: true, force: true }))
+
+/**
+ * `session_get` and `sessions_list` for Claude Code and Codex, through the socket: what a client
+ * scrolling a thread gets is what the whole-file read gave it, page for page (lib/transcriptPages.ts).
+ */
+describe.each([
+ ['claude', claudeScenario, windowRawLines, (lines: string[]) => messagesToEvents(lines)],
+ ['codex', codexScenario, windowCodexLines, (lines: string[]) => codexMessagesToEvents(lines)],
+] as const)('%s history through the socket', (engine, scenario, window, toEvents) => {
+ let dir: string
+ let socket: BackendSocket
+ let frames: Array<{ type: string; payload: Record<string, any> }>
+ let agentId: string
+ let sessionId: string
+ let file: string
+ let pane = 7400
+
+ beforeEach(() => {
+ const root = engine === 'claude' ? roots.claude : roots.codex
+ mkdirSync(root, { recursive: true })
+ dir = mkdtempSync(join(root, 'history-'))
+ file = join(dir, 'transcript.jsonl')
+ writeFileSync(file, scenario().join('\n') + '\n')
+ socket = new BackendSocket('fixture')
+ frames = []
+ socket.registerLocalClient('local:history', { sendFrame: (frame) => { frames.push(frame as never); return true }, sendBinary: () => true })
+ const paneId = `%${++pane}`
+ agentId = registry.openPendingAgent({ engine, runtimes: [{ backend: 'tmux', paneId }], cwd: dir })!.agentId
+ sessionId = `${engine}-history-${pane}`
+ registry.register({ engine, sessionId, transcriptPath: file, tmuxPane: paneId, cwd: dir })
+ })
+
+ afterEach(async () => {
+ registry.removeAgent(agentId)
+ await socket.stop()
+ rmSync(dir, { recursive: true, force: true })
+ })
+
+ let requests = 0
+ async function ask(type: string, payload: Record<string, unknown>): Promise<Record<string, any>> {
+ const requestId = `r${++requests}`
+ await (socket as any).dispatchDown({ type, payload: { requestId, ...payload } }, 'local:history', 'local')
+ return frames.find((frame) => frame.type === `${type}_result` && frame.payload.requestId === requestId)!.payload
+ }
+
+ it('is bound to the transcript under test', () => {
+ expect(registry.resolve(sessionId)?.transcriptPath).toBe(file)
+ })
+
+ it('pages back through the thread exactly as the whole-file read did, cursor for cursor', async () => {
+ const lines = await tailFile(file, Infinity)
+ for (const limit of [1, 4, 500]) {
+ let before: string | undefined
+ for (let page = 0; page < lines.length + 1; page++) {
+ const reply = await ask('session_get', { sessionId, limit, before })
+ const old = window(lines, { limit, before })
+ const events = toEvents(old.window)
+ if (before && events[events.length - 1]?.type === 'done') events.pop()
+ expect(reply.events, `limit ${limit}, page ${page}`).toEqual(JSON.parse(JSON.stringify(events)))
+ expect({ hasMore: reply.hasMore, oldestCursor: reply.oldestCursor }).toEqual({ hasMore: old.hasMore, oldestCursor: old.oldestCursor })
+ if (!old.hasMore || !old.oldestCursor) break
+ before = old.oldestCursor
+ }
+ }
+ })
+
+ it('answers a request with no limit with the whole thread, in the reply it always had', async () => {
+ const reply = await ask('session_get', { sessionId, before: 'ignored without a limit' })
+ expect(reply.events).toEqual(JSON.parse(JSON.stringify(toEvents(await tailFile(file, Infinity)))))
+ expect(reply).not.toHaveProperty('hasMore')
+ expect(reply).not.toHaveProperty('oldestCursor')
+ })
+
+ it('says a cursor it cannot find is stale, so the client reloads', async () => {
+ const reply = await ask('session_get', { sessionId, limit: 5, before: 'not-a-cursor-in-this-file' })
+ expect(reply).toMatchObject({ events: [], hasMore: false, oldestCursor: null, staleCursor: true })
+ })
+
+ it('totals a sub-agent from its own transcript, read a line at a time', async () => {
+ if (engine !== 'claude') return
+ const launch = cl({ type: 'assistant', message: { id: 'mt', role: 'assistant', model: 'claude-opus-5-5', content: [{ type: 'tool_use', id: 'toolu_task', name: 'Task', input: { description: 'look around' } }], stop_reason: 'tool_use' } })
+ const result = cl({
+ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'toolu_task', content: 'launched' }] },
+ toolUseResult: { isAsync: true, status: 'async_launched', agentId: 'agent7' },
+ })
+ writeFileSync(file, [claude.user('send a helper'), launch, result].join('\n') + '\n')
+ const sub = [
+ JSON.stringify({ type: 'user', timestamp: '2026-10-04T10:00:00.000Z', message: { role: 'user', content: 'go' } }),
+ JSON.stringify({ type: 'assistant', timestamp: '2026-10-04T10:00:05.000Z', message: { content: [{ type: 'tool_use', id: 'a' }, { type: 'tool_use', id: 'b' }], usage: { input_tokens: 10, output_tokens: 5 } } }),
+ '',
+ JSON.stringify({ type: 'assistant', timestamp: '2026-10-04T10:01:00.000Z', message: { content: [{ type: 'tool_use', id: 'c' }], usage: { cache_read_input_tokens: 7 } } }),
+ ]
+ mkdirSync(join(dir, 'transcript', 'subagents'), { recursive: true })
+ writeFileSync(join(dir, 'transcript', 'subagents', 'agent-agent7.jsonl'), sub.join('\r\n'))
+ const reply = await ask('session_get', { sessionId, limit: 50 })
+ const end = reply.events.find((event: { type: string; payload: { subagent?: unknown } }) => event.type === 'tool_end' && event.payload.subagent)
+ expect(end.payload.subagent).toMatchObject({ agentId: 'agent7', ...subagentStatsFromRawLines(sub) })
+ expect(end.payload.subagent).toMatchObject({ totalToolUseCount: 3, totalDurationMs: 60_000, totalTokens: 22 })
+ // Totals the launch already reported stand; only what it left out is filled in.
+ const reported = cl({
+ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'toolu_task', content: 'launched' }] },
+ toolUseResult: { agentId: 'agent7', totalDurationMs: 5, totalTokens: 9 },
+ })
+ writeFileSync(file, [claude.user('send a helper'), launch, reported].join('\n') + '\n')
+ const again = await ask('session_get', { sessionId, limit: 50 })
+ const kept = again.events.find((event: { type: string; payload: { subagent?: unknown } }) => event.type === 'tool_end' && event.payload.subagent)
+ expect(kept.payload.subagent).toMatchObject({ agentId: 'agent7', totalToolUseCount: 3, totalDurationMs: 5, totalTokens: 9 })
+ })
+
+ it('answers for a transcript that is gone as for an empty one', async () => {
+ rmSync(file)
+ const reply = await ask('session_get', { sessionId, limit: 5 })
+ // What the whole-file read made of the empty history it read there: no lines, then the end marker.
+ expect(reply.events).toEqual(JSON.parse(JSON.stringify(toEvents([]))))
+ expect(reply).toMatchObject({ hasMore: false, oldestCursor: window([], { limit: 5 }).oldestCursor })
+ expect(Date.parse(reply.timestamp)).toBeGreaterThan(Date.now() - 60_000)
+ })
+
+ it('counts the thread without reading it whole', async () => {
+ const reply = await ask('sessions_list', { agentId })
+ expect(reply.sessions).toHaveLength(1)
+ expect(reply.sessions[0]).toMatchObject({ id: sessionId, messageCount: (await tailFile(file, Infinity)).length })
+ })
+
+ it('counts nothing for an agent whose engine keeps no transcript file', async () => {
+ const paneId = `%${++pane}`
+ const store = registry.openPendingAgent({ engine: 'opencode', runtimes: [{ backend: 'tmux', paneId }], cwd: dir })!.agentId
+ try {
+ registry.register({ engine: 'opencode', sessionId: `ses_history${pane}`, tmuxPane: paneId, cwd: dir })
+ const reply = await ask('sessions_list', { agentId: store })
+ expect(reply.sessions).toEqual([expect.objectContaining({ id: `ses_history${pane}`, messageCount: 0 })])
+ } finally { registry.removeAgent(store) }
+ })
+})
diff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.ts
index e87b91e82..631d66d84 100644
--- a/cli/src/backendSocket.ts
+++ b/cli/src/backendSocket.ts
@@ -24,7 +24,7 @@ import type { AutonomousDeviceService, AutonomousDeviceFrame } from './lib/auton
import { WebSocket } from 'ws'
import { BACKEND_IDLE_DEADLINE_MS, watchSocketLiveness, type LivenessWatch } from './lib/wsLiveness.js'
-import { stat, readFile, readdir } from 'fs/promises'
+import { stat, readdir } from 'fs/promises'
import { existsSync } from 'node:fs'
import { dirname, isAbsolute, join } from 'path'
import { hostname, homedir } from 'os'
@@ -91,13 +91,15 @@ import { terminalHandoffRequest } from './lib/terminalHandoff.js'
import { routeVoiceTask } from './lib/voiceRouter.js'
import { tailFile } from './lib/sessions.js'
import { stoppedAgents } from './lib/stoppedAgents.js'
-import { messagesToEvents, windowRawLines, subagentStatsFromRawLines, type SessionEvent } from './lib/normalize.js'
+import { messagesToEvents, windowRawLines, SubagentStats, type SessionEvent } from './lib/normalize.js'
+import { TranscriptPager } from './lib/transcriptPages.js'
+import { streamRecords } from './lib/transcriptTail.js'
import { MediaPreviewError, readMediaPreviewChunk } from './lib/mediaPreview.js'
import { ViewerForwarder } from './lib/viewerForwarder.js'
import { InteractiveViewers } from './lib/interactiveViewer.js'
import { OwnerCommands, OWNER_COMMAND_TYPES } from './lib/ownerCommands.js'
import { VIEWER_DOWN_TYPES } from './lib/viewerWire.js'
-import { codexMessagesToEvents, windowCodexLines } from './engines/codex/normalizer.js'
+import { codexMessagesToEvents } from './engines/codex/normalizer.js'
import { codexSubagentResolverFor } from './engines/codex/subagent.js'
import { parseHostTheme, type HostTheme } from './lib/hostTheme.js'
import { cursorMessagesToEvents, windowCursorLines } from './engines/cursor/normalizer.js'
@@ -150,6 +152,8 @@ import {
// OpenCode has no per-session transcript file — its history is read from this SQLite store.
const OPENCODE_DB = join(env.OPENCODE_DATA_DIR, 'opencode.db')
+/** History pages and line counts for the threads clients read (lib/transcriptPages.ts). */
+const transcriptPages = new TranscriptPager()
// Kilo keeps history the same way opencode does, in its own store.
const KILO_DB = join(env.KILO_DATA_DIR, 'kilo.db')
const DEVIN_DB = join(env.DEVIN_HOME, 'sessions.db')
@@ -398,8 +402,11 @@ async function enrichSubagentStats(events: SessionEvent[], transcriptPath: strin
const sub = e.payload.subagent
if (!sub?.agentId || typeof sub.totalToolUseCount === 'number') continue
try {
- const txt = await readFile(join(subagentsDir, `agent-${sub.agentId}.jsonl`), 'utf8')
- const stats = subagentStatsFromRawLines(txt.split('\n'))
+ // A line at a time: a long sub-agent's transcript is never held whole to count its calls.
+ const file = join(subagentsDir, `agent-${sub.agentId}.jsonl`)
+ const totals = new SubagentStats()
+ await streamRecords(file, 0, (await stat(file)).size, (line) => { totals.push(line) }, () => true)
+ const stats = totals.result()
sub.totalToolUseCount = stats.totalToolUseCount
if (sub.totalDurationMs === undefined) sub.totalDurationMs = stats.totalDurationMs
if (sub.totalTokens === undefined) sub.totalTokens = stats.totalTokens
@@ -2157,13 +2164,14 @@ export class BackendSocket {
// plainly beats inventing one: the web then shows the tab with an empty thread until the bind
// lands, instead of pinning `currentSessionId` to an id no event will ever carry.
if (!s || !s.sessionId) { reply(type, requestId, { sessions: [] }); return }
- const lines = s.transcriptPath ? await tailFile(s.transcriptPath, Infinity) : []
+ // Counted from an index kept as the file grows (lib/transcriptPages.ts), not by reading it whole.
+ const messageCount = s.transcriptPath ? await transcriptPages.lineCount(s.transcriptPath) : 0
reply(type, requestId, {
sessions: [{
id: s.sessionId,
title: projectDisplayName(s),
timestamp: new Date(s.registeredAt).toISOString(),
- messageCount: lines.length,
+ messageCount,
lastActivity: new Date(await lastActivityAt(s)).toISOString(),
participants: [],
}],
@@ -2284,14 +2292,42 @@ export class BackendSocket {
const rawLimit = payload.limit
const limit = typeof rawLimit === 'number' && rawLimit > 0 ? Math.min(Math.floor(rawLimit), 500) : undefined
const before = typeof payload.before === 'string' ? payload.before : undefined
+ if (s.engine === 'claude' || s.engine === 'codex') {
+ // Only the page asked for is read (lib/transcriptPages.ts): the whole history of a long session
+ // is more memory than the daemon has. Without a limit, the newest lines that fit a page, and
+ // the cursor to the rest when they do not all fit — a cursor the full-transcript reply never had.
+ const opts = limit ? { limit, before } : {}
+ const page = s.engine === 'codex'
+ ? await transcriptPages.codex(s.transcriptPath, opts)
+ : await transcriptPages.claude(s.transcriptPath, opts)
+ const st = await stat(s.transcriptPath).catch(() => null)
+ const timestamp = new Date(st?.mtimeMs ?? Date.now()).toISOString()
+ if (page.staleCursor) {
+ reply(type, requestId, { id: sessionId, title: projectDisplayName(s), events: [], timestamp, engine: s.engine, hasMore: false, oldestCursor: null, staleCursor: true })
+ return
+ }
+ const events = s.engine === 'codex'
+ ? codexMessagesToEvents(page.lines, codexSubagentResolverFor(s.codexHome))
+ : messagesToEvents(page.lines)
+ await enrichSubagentStats(events, s.transcriptPath)
+ // Older pages must not inject a spurious end-of-transcript marker mid-scroll.
+ if (limit && before && events[events.length - 1]?.type === 'done') events.pop()
+ reply(type, requestId, {
+ id: sessionId,
+ title: projectDisplayName(s),
+ events,
+ timestamp,
+ engine: s.engine,
+ ...(limit || page.hasMore ? { hasMore: page.hasMore, oldestCursor: page.oldestCursor } : {}),
+ })
+ return
+ }
const lines = await tailFile(s.transcriptPath, Infinity)
const st = await stat(s.transcriptPath).catch(() => null)
const timestamp = new Date(st?.mtimeMs ?? Date.now()).toISOString()
if (!limit) {
- const fullEvents = s.engine === 'codex'
- ? codexMessagesToEvents(lines, codexSubagentResolverFor(s.codexHome))
- : s.engine === 'cursor'
+ const fullEvents = s.engine === 'cursor'
? cursorMessagesToEvents(lines, sessionId, await loadCursorReplayTaskLinks(cursorConfigDir(), sessionId, cursorDataDir()))
: s.engine === 'muse'
? museMessagesToEvents(lines)
@@ -2349,9 +2385,7 @@ export class BackendSocket {
})
return
}
- const w = s.engine === 'codex'
- ? windowCodexLines(lines, { limit, before })
- : s.engine === 'cursor'
+ const w = s.engine === 'cursor'
? windowCursorLines(lines, { limit, before })
: s.engine === 'pi'
? windowPiLines(lines, { limit, before })
@@ -2362,9 +2396,7 @@ export class BackendSocket {
reply(type, requestId, { id: sessionId, title: projectDisplayName(s), events: [], timestamp, engine: s.engine, hasMore: false, oldestCursor: null, staleCursor: true })
return
}
- const events = s.engine === 'codex'
- ? codexMessagesToEvents(w.window, codexSubagentResolverFor(s.codexHome))
- : s.engine === 'cursor'
+ const events = s.engine === 'cursor'
? cursorMessagesToEvents(
w.window,
sessionId,
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index cba739d21..329bf2804 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -170,7 +170,7 @@ import {
type TerminalRuntimeRef,
type TmuxRuntimeRef,
} from './lib/terminalTypes.js'
-import { Watcher, type HistoryEvent, type LineEvent, type TailHold } from './watcher/watcher.js'
+import { Watcher, type HistoryEvent, type LineEvent, type RewrittenEvent, type TailHold } from './watcher/watcher.js'
import { chooseHookAgent, startHookServer } from './hookServer.js'
import { isLocalSocketName, localSocketPath, type LocalSocketServer } from './lib/localSocket.js'
import { legacyDaemonStatus, localDaemonStatus, saveDaemonPort } from './lib/daemonEndpoint.js'
@@ -4939,6 +4939,14 @@ async function runForeground(session: AuthSession | null): Promise<void> {
// A transcript catch-up is history, including an unfinished last turn.
// Its content still streams, but only fresh events or live inspection can
// establish Working; replay cannot create a new completion notification.
+ // A transcript rewritten in place with too much history to replay: its tail already starts at the new
+ // end, and the session is attached again, from that end, so its turn state is rebuilt from the file.
+ watcher.on('rewritten', (event: RewrittenEvent) => {
+ const session = registry.bySession(event.sessionId)
+ if (!session || session.transcriptPath !== event.transcriptPath) return
+ console.log(`[watcher] ${sid(event.sessionId)} transcript rewritten in place — attaching it again from its end`)
+ void attachSession(session, true).catch((err) => console.error(`[cli] re-attach after rewrite failed (session ${sid(event.sessionId)}):`, err instanceof Error ? err.message : err))
+ })
watcher.on('history', (batch: HistoryEvent) => {
try {
type Events = ReturnType<CursorNormalizer['ingest']>
diff --git a/cli/src/engines/codex/normalizer.ts b/cli/src/engines/codex/normalizer.ts
index b659023c5..b37268835 100644
--- a/cli/src/engines/codex/normalizer.ts
+++ b/cli/src/engines/codex/normalizer.ts
@@ -667,6 +667,19 @@ export function lastCodexTurnText(rawLines: string[]): LastTurnText | null {
return text ? { userMessage, assistantText: text } : null
}
+/**
+ * Whether a Codex history page may start at this line: a user's message, or the injected context a goal
+ * turn starts on — so a turn is never split. Shared by `windowCodexLines` and the bounded pager
+ * (lib/transcriptPages.ts), so the two cannot drift apart.
+ */
+export function codexPageStart(line: string): boolean {
+ const raw = parse(line)
+ const item = raw ? payload(raw) : null
+ if (raw?.type === 'event_msg' && item && USER_TURN_TYPES.has(string(item.type))) return true
+ // A goal turn starts on the injected goal context, not on a `user_message`.
+ return raw?.type === 'response_item' && !!item && string(item.type) === 'message' && !!goalObjective(item)
+}
+
export function windowCodexLines(
rawLines: string[],
opts: { limit: number; before?: string },
@@ -682,14 +695,7 @@ export function windowCodexLines(
}
let start = Math.max(0, endIndex - opts.limit)
- while (start > 0) {
- const raw = parse(rawLines[start])
- const item = raw ? payload(raw) : null
- if (raw?.type === 'event_msg' && item && USER_TURN_TYPES.has(string(item.type))) break
- // A goal turn starts on the injected goal context, not on a `user_message`.
- if (raw?.type === 'response_item' && item && string(item.type) === 'message' && goalObjective(item)) break
- start--
- }
+ while (start > 0 && !codexPageStart(rawLines[start])) start--
return {
window: rawLines.slice(start, endIndex),
hasMore: start > 0,
diff --git a/cli/src/lib/attachTranscript.spec.ts b/cli/src/lib/attachTranscript.spec.ts
index 6f543ac74..59bc849fd 100644
--- a/cli/src/lib/attachTranscript.spec.ts
+++ b/cli/src/lib/attachTranscript.spec.ts
@@ -4,6 +4,7 @@ import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { CodexNormalizer, codexGoalOf, startsCodexTurn } from '../engines/codex/normalizer.js'
import { canonical, fromTheEnd, goalForgotten, newFold, session, wholeHistory } from '../testing/transcriptOracle.js'
+import { claude, claudeScenario, codex, codexScenario, cx } from '../testing/transcriptScenarios.js'
import {
attachTranscript,
claudeAttachRules,
@@ -19,117 +20,6 @@ import { RuntimeProfileManager } from './runtimeProfile.js'
import { tailFile, tailFileUntil } from './transcriptTail.js'
import { Watcher, type LineEvent } from '../watcher/watcher.js'
-// ── Codex records, in the shapes a 0.159 rollout writes ──────────────────────────────────────────────
-const cx = (type: string, payload: Record<string, unknown>): string =>
- JSON.stringify({ timestamp: '2026-10-03T23:21:44.458Z', type, payload })
-const codex = {
- meta: (version = '0.159.0') => cx('session_meta', { id: 'thread-1', cli_version: version, cwd: '/tmp/work' }),
- context: (model: string, effort: string | null, mode: string | null) => cx('turn_context', {
- turn_id: 't', model, ...(effort ? { reasoning_effort: effort } : {}), ...(mode ? { collaboration_mode: { mode } } : {}),
- }),
- settings: (model: string, effort: string, mode: string) => cx('event_msg', {
- type: 'thread_settings_applied', thread_settings: { model, reasoning_effort: effort, collaboration_mode: { mode } },
- }),
- started: (turn: string) => cx('event_msg', { type: 'task_started', turn_id: turn }),
- user: (text: string) => cx('event_msg', { type: 'user_message', message: text }),
- userItem: (text: string) => cx('event_msg', {
- type: 'item_completed', turn_id: 't', item: { type: 'UserMessage', id: 'u', content: [{ type: 'text', text, text_elements: [] }] },
- }),
- goal: (objective: string) => cx('response_item', {
- type: 'message', role: 'user',
- content: [{ type: 'input_text', text: `<codex_internal_context source="goal">\n<objective>${objective}</objective>\n</codex_internal_context>` }],
- }),
- developer: (text: string) => cx('response_item', { type: 'message', role: 'developer', content: [{ type: 'input_text', text }] }),
- agent: (text: string, phase = 'final_answer') => cx('event_msg', {
- type: 'item_completed', item: { type: 'AgentMessage', content: [{ type: 'Text', text }], phase },
- }),
- reasoning: (text: string) => cx('response_item', { type: 'reasoning', summary: [{ type: 'summary_text', text }] }),
- call: (id: string, cmd: string) => cx('response_item', { type: 'function_call', call_id: id, name: 'exec_command', arguments: JSON.stringify({ cmd }) }),
- output: (id: string, output: string) => cx('response_item', { type: 'function_call_output', call_id: id, output }),
- complete: (turn: string) => cx('event_msg', { type: 'task_complete', turn_id: turn, last_agent_message: 'done' }),
- aborted: (turn: string) => cx('event_msg', { type: 'turn_aborted', turn_id: turn }),
- compacted: (bytes: number) => cx('compacted', { message: '📘é'.repeat(bytes / 6), replacement_history: [] }),
- contextCompacted: () => cx('event_msg', { type: 'context_compacted' }),
- tokens: () => cx('event_msg', { type: 'token_count', info: { total_token_usage: { input_tokens: 1 } } }),
-}
-
-/** A rollout with every shape attach has to get right, in the order Codex writes them. */
-const codexScenario = (): string[] => [
- codex.meta(),
- codex.developer('<permissions instructions>'),
- codex.settings('gpt-5.5', 'high', 'default'),
- codex.started('t1'), codex.context('gpt-5.5', 'high', 'default'), codex.user('first question'),
- codex.reasoning('Planning'), codex.call('c1', 'ls'), codex.output('c1', 'a\nb'), codex.agent('one answer'), codex.complete('t1'),
- codex.started('t2'), codex.context('gpt-6', 'xhigh', 'plan'), codex.userItem('second, in the item shape'),
- codex.reasoning('Thinking 漢字 📘'), codex.agent('looking', 'commentary'),
- codex.compacted(150_000), codex.contextCompacted(), codex.tokens(),
- codex.call('c2', 'npm test'), codex.output('c2', 'ok'), codex.agent('second answer'), codex.complete('t2'),
- codex.settings('gpt-6', 'low', 'default'),
- codex.started('t3'), codex.context('gpt-6', null, 'default'), codex.user('third: effort left out'),
- codex.call('c3', 'sleep 1'),
- codex.user(''), // an empty message opens nothing
- codex.output('c3', 'slept'), codex.complete('t3'),
- codex.started('t4'), codex.context('gpt-6', 'medium', 'default'), codex.user('interrupted one'), codex.aborted('t4'),
- codex.started('t5'), codex.context('gpt-6', 'medium', 'default'), codex.goal('ship the release'), codex.reasoning('goal work'), codex.complete('t5'),
- codex.started('t6'), codex.context('gpt-6', 'medium', 'default'), codex.goal('ship the release'), codex.agent('continuing'), codex.complete('t6'),
- codex.started('t7'), codex.context('gpt-6', 'medium', 'default'), codex.user('a question mid-goal'), codex.complete('t7'),
- codex.started('t8'), codex.context('gpt-6', 'medium', 'default'), codex.goal('ship the release'), codex.reasoning('still the goal'),
- codex.call('c8', 'make'), codex.complete('t8'),
- codex.started('t9'), codex.context('gpt-6', 'max', 'plan'), codex.goal('a different goal'), codex.reasoning('new goal'),
- codex.complete('t9'),
- // A message sent mid-task, with a call still in flight: its output arrives after the new message.
- codex.started('t10'), codex.context('gpt-6', 'max', 'default'), codex.user('start the build'),
- codex.call('c10', 'make all'), codex.user('also run the tests while that builds'),
- codex.output('c10', 'built'), codex.reasoning('now the tests'), codex.call('c11', 'npm test'),
-]
-
-// ── Claude Code records ───────────────────────────────────────────────────────────────────────────────
-let uuid = 0
-const cl = (record: Record<string, unknown>): string => JSON.stringify({
- parentUuid: null, isSidechain: false, userType: 'external', cwd: '/tmp', sessionId: 's', version: '2.1.212',
- timestamp: '2026-10-03T23:21:44.458Z', uuid: `u${++uuid}`, ...record,
-})
-const claude = {
- user: (content: unknown, extra: Record<string, unknown> = {}) => cl({ type: 'user', message: { role: 'user', content }, ...extra }),
- assistant: (content: unknown[], stop: string | null, model = 'claude-opus-5-5') =>
- cl({ type: 'assistant', message: { id: `m${uuid}`, role: 'assistant', model, content, stop_reason: stop } }),
- text: (text: string) => ({ type: 'text', text }),
- thinking: (thinking: string) => ({ type: 'thinking', thinking }),
- tool: (id: string, command: string) => ({ type: 'tool_use', id, name: 'Bash', input: { command } }),
- result: (id: string, content: string) => cl({ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: id, content }] } }),
- compact: () => cl({ type: 'system', subtype: 'compact_boundary', content: 'Conversation compacted', compactMetadata: { trigger: 'auto' } }),
- summary: () => cl({ type: 'user', isCompactSummary: true, message: { role: 'user', content: 'Summary of the earlier conversation' } }),
-}
-
-const claudeScenario = (): string[] => [
- JSON.stringify({ type: 'summary', summary: 'Earlier work', leafUuid: 'x' }),
- claude.user('first prompt'),
- claude.assistant([claude.thinking('let me think'), claude.text('First answer.')], 'end_turn'),
- claude.user('second prompt with a tool'),
- claude.assistant([claude.text('Checking.'), claude.tool('toolu_1', 'ls')], 'tool_use'),
- claude.result('toolu_1', 'files'),
- claude.assistant([claude.thinking('found them 漢字'), claude.text('Done: '.repeat(20_000))], 'end_turn', 'claude-fable-5-1'),
- claude.user('<local-command-stdout>Set model to Opus 5.5</local-command-stdout>'),
- claude.user('<bash-input>git status</bash-input><bash-stdout>clean</bash-stdout>'),
- claude.user('third prompt, interrupted'),
- claude.assistant([claude.tool('toolu_2', 'sleep 100')], 'tool_use'),
- claude.user([{ type: 'text', text: '[Request interrupted by user for tool use]' }]),
- claude.user('<task-notification><task-id>a</task-id><tool-use-id>toolu_9</tool-use-id><status>completed</status><summary>bg done</summary></task-notification>'),
- claude.compact(), claude.summary(),
- claude.user('loop iteration', { isMeta: true, promptSource: 'system' }),
- claude.assistant([claude.text('Loop ran.')], 'end_turn', '<synthetic>'),
- claude.user('a bookkeeping line', { isMeta: true }),
- claude.user('fourth prompt'),
- claude.assistant([claude.text('Working on it'), claude.tool('toolu_3', 'make')], 'tool_use'),
- claude.result('toolu_3', 'built'),
- claude.assistant([claude.thinking('almost'), claude.text('Built it.')], 'end_turn'),
- claude.user('fifth prompt, left running'),
- claude.assistant([claude.tool('toolu_4', 'npm test')], 'tool_use'),
- // A prompt that lands while a call is in flight: the result arrives after it, naming its call.
- claude.user('sixth prompt, typed during the tests'),
- claude.result('toolu_4', 'tests passed'),
- claude.assistant([claude.thinking('reading results'), claude.text('All green.')], 'end_turn'),
-]
describe('attachTranscript', () => {
let dir: string, file: string
diff --git a/cli/src/lib/normalize.ts b/cli/src/lib/normalize.ts
index 3664c6ca2..75c6b1ba9 100644
--- a/cli/src/lib/normalize.ts
+++ b/cli/src/lib/normalize.ts
@@ -491,38 +491,50 @@ export function lastTurnTextFromRawLines(rawLines: string[]): LastTurnText | nul
/** Aggregates computed from a sub-agent's OWN transcript. Async/background agents never get totals
* in the launcher's toolUseResult (it only records `{isAsync, status:'async_launched', agentId}` at
* launch) — the real numbers live in `<session>/subagents/agent-<id>.jsonl`. totalTokens mirrors the
- * CLI's definition: input + output + cache_read + cache_creation summed over assistant turns. */
-export function subagentStatsFromRawLines(rawLines: string[]): { totalToolUseCount: number; totalDurationMs?: number; totalTokens?: number } {
- let toolCount = 0
- let tokens = 0
- let first: number | undefined
- let last: number | undefined
- for (const line of rawLines) {
- if (!line.trim()) continue
+ * CLI's definition: input + output + cache_read + cache_creation summed over assistant turns. Taken a
+ * line at a time, so a long transcript is never held whole. */
+export class SubagentStats {
+ private toolCount = 0
+ private tokens = 0
+ private first: number | undefined
+ private last: number | undefined
+
+ push(line: string): void {
+ if (!line.trim()) return
let raw: Record<string, unknown>
- try { raw = JSON.parse(line) as Record<string, unknown> } catch { continue }
- if (!raw || typeof raw !== 'object') continue
+ try { raw = JSON.parse(line) as Record<string, unknown> } catch { return }
+ if (!raw || typeof raw !== 'object') return
const ts = typeof raw.timestamp === 'string' ? Date.parse(raw.timestamp) : NaN
- if (!Number.isNaN(ts)) { if (first === undefined) first = ts; last = ts }
- if (raw.type !== 'assistant') continue
+ if (!Number.isNaN(ts)) { if (this.first === undefined) this.first = ts; this.last = ts }
+ if (raw.type !== 'assistant') return
const msg = raw.message as { content?: unknown; usage?: Record<string, unknown> } | undefined
- if (!msg) continue
+ if (!msg) return
if (Array.isArray(msg.content)) {
- for (const b of msg.content) if ((b as { type?: string }).type === 'tool_use') toolCount++
+ for (const b of msg.content) if ((b as { type?: string }).type === 'tool_use') this.toolCount++
}
const u = msg.usage
if (u) {
- tokens += (Number(u.input_tokens) || 0) + (Number(u.output_tokens) || 0)
+ this.tokens += (Number(u.input_tokens) || 0) + (Number(u.output_tokens) || 0)
+ (Number(u.cache_read_input_tokens) || 0) + (Number(u.cache_creation_input_tokens) || 0)
}
}
- return {
- totalToolUseCount: toolCount,
- totalDurationMs: first !== undefined && last !== undefined && last > first ? last - first : undefined,
- totalTokens: tokens || undefined,
+
+ result(): { totalToolUseCount: number; totalDurationMs?: number; totalTokens?: number } {
+ const { first, last } = this
+ return {
+ totalToolUseCount: this.toolCount,
+ totalDurationMs: first !== undefined && last !== undefined && last > first ? last - first : undefined,
+ totalTokens: this.tokens || undefined,
+ }
}
}
+export function subagentStatsFromRawLines(rawLines: string[]): { totalToolUseCount: number; totalDurationMs?: number; totalTokens?: number } {
+ const stats = new SubagentStats()
+ for (const line of rawLines) stats.push(line)
+ return stats.result()
+}
+
/** Emit tool_end events for a user message's tool_result blocks. */
function toolResultEvents(msg: NormalizedMessage, toolIdToName: Map<string, string>): SessionEvent[] {
const events: SessionEvent[] = []
@@ -713,19 +725,28 @@ export function messagesToEvents(rawLines: string[]): SessionEvent[] {
* would drop tool names. A user-prompt line never sits between a tool_use and its tool_result, so
* snapping there keeps every turn whole.
*/
+/**
+ * What a Claude history page knows of one line: the cursor that names it, and whether a page may start
+ * there — a real user prompt, so a turn is never split. Shared by `windowRawLines` and the bounded pager
+ * (lib/transcriptPages.ts), so the two cannot drift apart.
+ */
+export function claudePageLine(line: string): { cursor: string | null; startsPage: boolean } {
+ if (!line.trim()) return { cursor: null, startsPage: false }
+ let raw: Record<string, unknown>
+ try { raw = JSON.parse(line) as Record<string, unknown> } catch { return { cursor: null, startsPage: false } }
+ const cursor = (raw.uuid ?? raw.id ?? raw.message_id ?? null) as string | null
+ const msg = transformLine(raw)
+ return { cursor, startsPage: msg ? realUserText(msg) !== null : false }
+}
+
export function windowRawLines(
rawLines: string[],
opts: { limit: number; before?: string },
): { window: string[]; hasMore: boolean; oldestCursor: string | null; staleCursor?: boolean } {
// Per-line uuid + whether the line is a real user-prompt turn start (parse each line once).
const meta = rawLines.map((line) => {
- if (!line.trim()) return { uuid: null as string | null, turnStart: false }
- let raw: Record<string, unknown>
- try { raw = JSON.parse(line) as Record<string, unknown> } catch { return { uuid: null as string | null, turnStart: false } }
- const uuid = (raw.uuid ?? raw.id ?? raw.message_id ?? null) as string | null
- const msg = transformLine(raw)
- const turnStart = msg ? realUserText(msg) !== null : false
- return { uuid, turnStart }
+ const { cursor, startsPage } = claudePageLine(line)
+ return { uuid: cursor, turnStart: startsPage }
})
let endIndex = rawLines.length
diff --git a/cli/src/lib/transcriptPages.spec.ts b/cli/src/lib/transcriptPages.spec.ts
new file mode 100644
index 000000000..28b9f001a
--- /dev/null
+++ b/cli/src/lib/transcriptPages.spec.ts
@@ -0,0 +1,414 @@
+import { appendFileSync, chmodSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs'
+import { open } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { codexPageStart, windowCodexLines } from '../engines/codex/normalizer.js'
+import { claude, claudeScenario, codex, codexScenario } from '../testing/transcriptScenarios.js'
+import { claudePageLine, windowRawLines } from './normalize.js'
+import { tailFile } from './transcriptTail.js'
+import { LineIndex, pageBefore, TranscriptPager, type HistoryPage, type OpenFile } from './transcriptPages.js'
+
+let dir: string
+let files = 0
+const fresh = (content: string | Buffer): string => {
+ const file = join(dir, `t${++files}.jsonl`)
+ writeFileSync(file, content)
+ return file
+}
+beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'transcript-pages-')) })
+afterEach(() => rmSync(dir, { recursive: true, force: true }))
+
+type Engine = 'claude' | 'codex'
+const window = (engine: Engine, lines: string[], opts: { limit: number; before?: string }) =>
+ engine === 'claude' ? windowRawLines(lines, opts) : windowCodexLines(lines, opts)
+const asOld = (page: HistoryPage) => ({ window: page.lines, hasMore: page.hasMore, oldestCursor: page.oldestCursor, staleCursor: page.staleCursor })
+
+/** Page back through `file` from its end, both ways, until the start; every page and cursor must match. */
+async function pageThrough(engine: Engine, file: string, limit: number): Promise<number> {
+ const lines = await tailFile(file, Infinity)
+ const pager = new TranscriptPager()
+ let before: string | undefined
+ for (let pages = 1; ; pages++) {
+ const old = window(engine, lines, { limit, before })
+ const now = await pager[engine](file, { limit, before })
+ expect(asOld(now), `${engine} page ${pages}, limit ${limit}, before ${before}`).toEqual(old)
+ expect(now.clipped).toBe(false)
+ if (!old.hasMore || old.staleCursor || old.oldestCursor === null) return pages
+ before = old.oldestCursor
+ }
+}
+
+describe('history pages equal the windows cut from the whole file', () => {
+ describe.each([['claude', claudeScenario], ['codex', codexScenario]] as const)('%s', (engine, scenario) => {
+ const records = scenario()
+ const cuts = records.map((_, index) => index + 1)
+
+ it.each(cuts)('after record %i, with every ending, a half-written record or none, and every page size', async (cut) => {
+ for (const ending of ['\n', '\r\n']) {
+ const body = records.slice(0, cut).join(ending) + ending
+ const variants = [body]
+ if (cut < records.length) variants.push(body + records[cut].slice(0, Math.floor(records[cut].length / 2)))
+ for (const content of variants) {
+ const file = fresh(content)
+ expect(await new TranscriptPager().lineCount(file)).toBe((await tailFile(file, Infinity)).length)
+ for (const limit of [1, 3, 7, 500]) await pageThrough(engine, file, limit)
+ }
+ }
+ })
+
+ it('reads lines ended by CR alone, and passes over blank ones', async () => {
+ const file = fresh(['', ...records.slice(0, 20), ' ', '\t'].join('\r') + '\r\r\n\n')
+ for (const limit of [1, 4, 500]) await pageThrough(engine, file, limit)
+ })
+
+ it('says a cursor it cannot find is stale, as the old window did', async () => {
+ const file = fresh(records.join('\n') + '\n')
+ const lines = await tailFile(file, Infinity)
+ const pager = new TranscriptPager()
+ for (const before of ['nothing-like-it', 'codex:x', `codex:${lines.length + 1}`, 'codex:99999999999999999999']) {
+ expect(asOld(await pager[engine](file, { limit: 5, before }))).toEqual(window(engine, lines, { limit: 5, before }))
+ }
+ })
+
+ it('gives every line, oldest first, when asked for no page at all', async () => {
+ const file = fresh(records.join('\n') + '\n')
+ const page = await new TranscriptPager()[engine](file, {})
+ expect(page.lines).toEqual(await tailFile(file, Infinity))
+ expect(page.hasMore).toBe(false)
+ })
+ })
+
+ it('reads a transcript that is missing, or cannot be read, as an empty one', async () => {
+ const pager = new TranscriptPager()
+ const missing = join(dir, 'gone.jsonl')
+ for (const before of [undefined, 'u1']) {
+ expect(asOld(await pager.claude(missing, { limit: 5, before }))).toEqual(windowRawLines([], { limit: 5, before }))
+ }
+ for (const before of [undefined, 'codex:0', 'codex:1']) {
+ expect(asOld(await pager.codex(missing, { limit: 5, before }))).toEqual(windowCodexLines([], { limit: 5, before }))
+ }
+ expect(await pager.lineCount(missing)).toBe(0)
+ const locked = fresh(claudeScenario().join('\n') + '\n')
+ chmodSync(locked, 0o000)
+ try {
+ expect(asOld(await pager.claude(locked, { limit: 5 }))).toEqual(windowRawLines([], { limit: 5 }))
+ expect((await pager.claude(locked, { limit: 5, before: 'u1' })).staleCursor).toBe(true)
+ expect(await pageBefore(locked, 100, 5, { startsPage: () => true })).toEqual({ lines: [], count: 0, start: 0, hasMore: false, clipped: false })
+ } finally { chmodSync(locked, 0o600) }
+ })
+
+ it('names the line a cursor stood on when no page is left before it', async () => {
+ const records = claudeScenario()
+ const file = fresh(records.join('\n') + '\n')
+ const first = claudePageLine(records[1]).cursor!
+ const lines = await tailFile(file, Infinity)
+ const pager = new TranscriptPager()
+ // Paging back from the second line leaves only the first, which has no id of its own.
+ expect(asOld(await pager.claude(file, { limit: 5, before: first }))).toEqual(windowRawLines(lines, { limit: 5, before: first }))
+ const top = claudePageLine(records[0]).cursor
+ expect(top).toBeNull()
+ const withId = fresh(records.slice(1).join('\n') + '\n')
+ const page = await pager.claude(withId, { limit: 5, before: first })
+ expect(asOld(page)).toEqual(windowRawLines(await tailFile(withId, Infinity), { limit: 5, before: first }))
+ expect(page.oldestCursor).toBe(first)
+ })
+})
+
+describe('a Claude cursor', () => {
+ it('names the newest line with its id, where the old window took the oldest', async () => {
+ const twice = claude.user('a prompt written twice')
+ const records = [claude.user('one'), twice, claude.user('two'), twice, claude.user('three')]
+ const file = fresh(records.join('\n') + '\n')
+ const before = claudePageLine(twice).cursor!
+ const page = await new TranscriptPager().claude(file, { limit: 1, before })
+ expect(page.lines).toEqual([records[2]])
+ expect(windowRawLines(await tailFile(file, Infinity), { limit: 1, before }).window).toEqual([records[0]])
+ })
+
+ it('is found again from where it was served, and by walking the file when that no longer names it', async () => {
+ const records = claudeScenario()
+ const file = fresh(records.join('\n') + '\n')
+ const pager = new TranscriptPager()
+ const first = await pager.claude(file, { limit: 3 })
+ const again = await pager.claude(file, { limit: 3, before: first.oldestCursor! })
+ expect(again).toEqual(await new TranscriptPager().claude(file, { limit: 3, before: first.oldestCursor! }))
+ // Rewritten so the byte remembered for that cursor now starts another line.
+ writeFileSync(file, [claude.user('a new first line, longer than before'), ...records].join('\n') + '\n')
+ expect(await pager.claude(file, { limit: 3, before: first.oldestCursor! }))
+ .toEqual(await new TranscriptPager().claude(file, { limit: 3, before: first.oldestCursor! }))
+ // And when the file shrank past it.
+ writeFileSync(file, records.slice(0, 2).join('\n') + '\n')
+ expect((await pager.claude(file, { limit: 3, before: first.oldestCursor! })).staleCursor).toBe(true)
+ })
+
+ it('is matched as a whole id, and an id with characters JSON may escape is read the slow way', async () => {
+ const odd = 'id with "quotes" and /slashes/'
+ const records = [claude.user('one'), JSON.stringify({ type: 'user', id: odd, message: { role: 'user', content: 'two' } }), claude.user('three')]
+ const file = fresh(records.join('\n') + '\n')
+ const lines = await tailFile(file, Infinity)
+ const pager = new TranscriptPager()
+ expect(asOld(await pager.claude(file, { limit: 1, before: odd }))).toEqual(windowRawLines(lines, { limit: 1, before: odd }))
+ const prefix = claudePageLine(records[2]).cursor!.slice(0, -1)
+ expect((await pager.claude(file, { limit: 1, before: prefix })).staleCursor).toBe(true)
+ })
+
+ it('remembers a bounded number of cursors and files', async () => {
+ const pager = new TranscriptPager({ capacity: 2 })
+ const records = Array.from({ length: 80 }, (_, i) => claude.user(`prompt ${i}`))
+ const file = fresh(records.join('\n') + '\n')
+ let before: string | undefined
+ for (let i = 0; i < 70; i++) before = (await pager.claude(file, { limit: 1, before })).oldestCursor!
+ for (let i = 0; i < 3; i++) await pager.claude(fresh(records.join('\n') + '\n'), { limit: 1 })
+ expect(await pager.claude(file, { limit: 1, before })).toEqual(await new TranscriptPager().claude(file, { limit: 1, before }))
+ })
+})
+
+describe('a page held to its size', () => {
+ const rules = { startsPage: (line: string) => line.startsWith('{"start"') }
+ const start = (n: number, pad = 0) => JSON.stringify({ start: n, pad: 'x'.repeat(pad) })
+ const step = (n: number, pad = 0) => JSON.stringify({ step: n, pad: 'x'.repeat(pad) })
+
+ it('ends at the oldest turn that fits, short of its limit, so no turn is split', async () => {
+ const records = [start(1), step(1), start(2, 100), step(2, 100), start(3), step(3)]
+ const file = fresh(records.join('\n') + '\n')
+ const end = Buffer.byteLength(records.join('\n') + '\n')
+ // 39 bytes reach back to turn 3, 278 to turn 2: a page of 200 holds turn 3 alone.
+ expect(await pageBefore(file, end, 6, rules, 200)).toMatchObject({ lines: records.slice(4), count: 2, hasMore: true, clipped: false })
+ expect(await pageBefore(file, end, 6, rules, 300)).toMatchObject({ lines: records.slice(2), count: 4, hasMore: true, clipped: false })
+ })
+
+ it('cuts a turn that cannot fit inside it, and says so', async () => {
+ const records = [start(1), step(1, 100), step(2, 100), step(3, 100)]
+ const file = fresh(records.join('\n') + '\n')
+ const page = (await pageBefore(file, Buffer.byteLength(records.join('\n') + '\n'), 2, rules, 250))!
+ expect(page).toMatchObject({ lines: records.slice(2), hasMore: true, clipped: true })
+ })
+
+ it('counts a line too long to hold where it stands, and leaves it out', async () => {
+ const records = [start(1), step(1), step(2, 400), step(3), start(2), step(4)]
+ const file = fresh(records.join('\n') + '\n')
+ const end = Buffer.byteLength(records.join('\n') + '\n')
+ expect(await pageBefore(file, end, 2, rules, 300)).toMatchObject({ lines: records.slice(4), count: 2, hasMore: true })
+ // Turn 1 holds the long line: it counts, and the page holds the rest of the turn around it.
+ expect(await pageBefore(file, end, 3, rules, 300))
+ .toMatchObject({ lines: [records[0], records[1], records[3], records[4], records[5]], count: 6, hasMore: false })
+ // Asked for the page right after one ended on a long line: the line still counts toward it.
+ const fromLong = (await pageBefore(file, Buffer.byteLength(records.slice(0, 3).join('\n') + '\n'), 1, rules, 300))!
+ expect(fromLong).toMatchObject({ lines: [records[0], records[1]], count: 3, hasMore: false })
+ const afterReach = (await pageBefore(file, Buffer.byteLength(records.slice(0, 4).join('\n') + '\n'), 1, rules, 300))!
+ expect(afterReach).toMatchObject({ lines: [records[0], records[1], records[3]], count: 4 })
+ // A long line just before a page that is already complete is the next page's.
+ const complete = (await pageBefore(file, Buffer.byteLength(records.slice(0, 4).join('\n') + '\n'), 1, { startsPage: () => true }, 300))!
+ expect(complete).toMatchObject({ lines: [records[3]], count: 1, hasMore: true })
+ })
+
+ it('pages a Codex rollout through lines too long to hold, every line in exactly one page', async () => {
+ const records = codexScenario()
+ const file = fresh(records.join('\n') + '\n')
+ const pager = new TranscriptPager({ maxBytes: 20_000 })
+ const seen: string[] = []
+ let before: string | undefined
+ let endIndex = records.length
+ for (;;) {
+ const page = await pager.codex(file, { limit: 4, before })
+ seen.unshift(...page.lines)
+ const start = Number(page.oldestCursor!.slice('codex:'.length))
+ expect(start).toBeLessThan(endIndex)
+ endIndex = start
+ if (!page.hasMore) break
+ before = page.oldestCursor!
+ }
+ expect(endIndex).toBe(0)
+ expect(seen).toEqual(records.filter((record) => Buffer.byteLength(record) <= 20_000))
+ })
+
+ it('is no page at all when the file shrinks under the walk', async () => {
+ const records = codexScenario()
+ const file = fresh(records.join('\n') + '\n')
+ const end = Buffer.byteLength(records.join('\n') + '\n')
+ let cut = false
+ const shrinking = { startsPage: (line: string) => { if (!cut) { cut = true; truncateSync(file, 10) } return codexPageStart(line) } }
+ expect(await pageBefore(file, end, 500, shrinking)).toBeNull()
+ })
+})
+
+describe('LineIndex', () => {
+ const tricky = [
+ '{"a":1}', '', ' ', '\t', ' ', ' 
', '', '\u0001', 'é', '{"b":2}',
+ Buffer.from([0xff, 0xfe]).toString('latin1'),
+ ]
+
+ it('counts what the whole-file reader counts, however lines end and whatever whitespace they hold', async () => {
+ for (const ending of ['\n', '\r\n', '\r']) {
+ for (let cut = 0; cut <= tricky.length; cut++) {
+ for (const last of ['', ending]) {
+ const content = tricky.slice(0, cut).join(ending) + (cut ? last : '')
+ const file = fresh(content)
+ const index = new LineIndex(file)
+ await index.update()
+ expect(index.count(), JSON.stringify({ ending, cut, last })).toBe((await tailFile(file, Infinity)).length)
+ }
+ }
+ }
+ })
+
+ it('reads invalid UTF-8 as a line, as the whole-file reader does', async () => {
+ const file = fresh(Buffer.concat([Buffer.from('{"a":1}\n'), Buffer.from([0xc3]), Buffer.from('\n'), Buffer.from([0xe2, 0x80]), Buffer.from('\n')]))
+ const index = new LineIndex(file)
+ await index.update()
+ expect(index.count()).toBe((await tailFile(file, Infinity)).length)
+ })
+
+ it('keeps up as the file grows a byte at a time, and finds where any line ends', async () => {
+ const content = Buffer.from([...codexScenario().slice(0, 12), '', ' ', claude.user('漢字 📘')].join('\r\n') + '\n')
+ const file = fresh('')
+ const index = new LineIndex(file, { lines: 2, bytes: 4096 })
+ // Byte by byte through the first records, a character split across looks included; then in strides.
+ for (let at = 0; at < content.length;) {
+ const next = at < 600 ? at + 1 : Math.min(content.length, at + 997)
+ appendFileSync(file, content.subarray(at, next))
+ at = next
+ await index.update()
+ expect(index.count(), `after ${at} bytes`).toBe((await tailFile(file, Infinity)).length)
+ }
+ const lines = await tailFile(file, Infinity)
+ expect(index.count()).toBe(lines.length)
+ expect(index.length()).toBe(content.length)
+ for (let n = 0; n <= lines.length + 1; n++) {
+ const end = await index.endOf(n)
+ const prefix = fresh(content.subarray(0, end))
+ expect((await tailFile(prefix, Infinity)).length, `line ${n}`).toBe(Math.min(n, lines.length))
+ }
+ })
+
+ it('marks lines by bytes too, so a long stretch of long lines is never read whole to find one', async () => {
+ const content = Buffer.from(Array.from({ length: 40 }, (_, i) => codex.reasoning(`${i} ${'x'.repeat(3000)}`)).join('\n') + '\n')
+ const file = fresh(content)
+ const index = new LineIndex(file, { lines: 1000, bytes: 10_000 })
+ await index.update()
+ for (const n of [0, 1, 3, 4, 17, 39, 40]) {
+ expect((await tailFile(fresh(content.subarray(0, await index.endOf(n))), Infinity)).length, `line ${n}`).toBe(n)
+ }
+ })
+
+ it('indexes a file again from the start when it shrank, its first bytes changed, or an ending moved', async () => {
+ const records = codexScenario()
+ const file = fresh(records.join('\n') + '\n')
+ const index = new LineIndex(file)
+ await index.update()
+ expect(index.count()).toBe(records.length)
+ writeFileSync(file, records.slice(0, 5).join('\n') + '\n')
+ await index.update()
+ expect(index.count()).toBe(5)
+ // Longer than before, with an ending where one was: only its first bytes tell it is not the file indexed.
+ writeFileSync(file, [codex.meta('99.159.0'), ...records.slice(1, 5)].join('\n') + '\n')
+ await index.update()
+ expect(index.count()).toBe(5)
+ const longer = [codex.reasoning('x'.repeat(30)), ...records.slice(1, 5)].join('\n') + '\n'
+ expect(Buffer.byteLength(longer)).toBeGreaterThan(Buffer.byteLength([codex.meta('99.159.0'), ...records.slice(1, 5)].join('\n') + '\n'))
+ writeFileSync(file, longer)
+ await index.update()
+ expect(index.count()).toBe(5)
+ writeFileSync(file, records.slice(0, 5).join(' ') + '\n' + records.slice(5, 9).join('\n') + '\n')
+ await index.update()
+ expect(index.count()).toBe((await tailFile(file, Infinity)).length)
+ })
+
+ it('reads a file that is gone as empty, and one that appears later from its start', async () => {
+ const file = join(dir, 'later.jsonl')
+ const index = new LineIndex(file)
+ await index.update()
+ expect([index.count(), index.length()]).toEqual([0, 0])
+ writeFileSync(file, '{"a":1}\n{"b":2}')
+ await index.update()
+ expect(index.count()).toBe(2)
+ expect(await index.endOf(1)).toBe(8)
+ rmSync(file)
+ await index.update()
+ expect(index.count()).toBe(0)
+ })
+
+ it('looks and finds one at a time, in the order asked', async () => {
+ const records = codexScenario()
+ const file = fresh(records.slice(0, 10).join('\n') + '\n')
+ const index = new LineIndex(file, { lines: 3, bytes: 1 << 30 })
+ await index.update()
+ appendFileSync(file, records.slice(10).join('\n') + '\n')
+ const [, end] = await Promise.all([index.update(), index.endOf(records.length - 1)])
+ expect(end).toBe(Buffer.byteLength(records.slice(0, -1).join('\n') + '\n'))
+ // A look that fails leaves the queue running.
+ rmSync(file)
+ await expect(index.update()).resolves.toBeUndefined()
+ await expect(index.endOf(0)).resolves.toBe(0)
+ })
+
+ it('stops where a file cut short under it ends, whatever its length said', async () => {
+ for (const content of ['{"a":1}\n{"b"', '{"a":1}\n']) {
+ const file = fresh(content)
+ const shorted: OpenFile = async (path) => {
+ const handle = await open(path, 'r')
+ return Object.assign(Object.create(handle), {
+ stat: async () => ({ ...(await handle.stat()), size: Buffer.byteLength(content) + 50 }),
+ read: handle.read.bind(handle),
+ close: handle.close.bind(handle),
+ })
+ }
+ const index = new LineIndex(file, undefined, shorted)
+ await index.update()
+ expect(index.count()).toBe((await tailFile(file, Infinity)).length)
+ }
+ })
+
+ it('reports a file it cannot read, and keeps answering after', async () => {
+ const index = new LineIndex(dir)
+ await expect(index.update()).rejects.toThrow()
+ const locked = fresh('{"a":1}\n')
+ chmodSync(locked, 0o000)
+ try { await expect(new LineIndex(locked).update()).rejects.toThrow() } finally { chmodSync(locked, 0o600) }
+ const records = codexScenario()
+ const file = fresh(records.join('\n') + '\n')
+ const gone = new LineIndex(file, { lines: 4, bytes: 1 << 30 })
+ await gone.update()
+ rmSync(file)
+ await expect(gone.endOf(6)).rejects.toThrow()
+ await expect(gone.update()).resolves.toBeUndefined()
+ expect(gone.count()).toBe(0)
+ const pager = new TranscriptPager()
+ expect(await pager.lineCount(dir)).toBe(0)
+ expect(asOld(await pager.codex(dir, { limit: 2 }))).toEqual(windowCodexLines([], { limit: 2 }))
+ expect(asOld(await pager.codex(dir, { limit: 2, before: 'codex:3' }))).toEqual(windowCodexLines([], { limit: 2, before: 'codex:3' }))
+ expect(asOld(await pager.codex(dir, {}))).toEqual(windowCodexLines([], { limit: Infinity }))
+ })
+})
+
+describe('the pager', () => {
+ it('says a page is stale when the file shrank under its walk', async () => {
+ const pager = new TranscriptPager({ walk: async () => false })
+ expect((await pager.claude(fresh(claudeScenario().join('\n') + '\n'), { limit: 2 })).staleCursor).toBe(true)
+ expect((await pager.codex(fresh(codexScenario().join('\n') + '\n'), { limit: 2 })).staleCursor).toBe(true)
+ })
+
+ it('keeps a bounded number of indexes', async () => {
+ const pager = new TranscriptPager({ capacity: 2 })
+ const records = codexScenario()
+ const files = [1, 2, 3].map(() => fresh(records.join('\n') + '\n'))
+ for (const file of [...files, files[0]]) expect(await pager.lineCount(file)).toBe(records.length)
+ })
+
+ it('finds a remembered cursor on a last line not yet ended', async () => {
+ const records = [claude.user('one'), claude.assistant([claude.text('a')], 'end_turn'), claude.user('two, still being written')]
+ const file = fresh(records.join('\n'))
+ const pager = new TranscriptPager()
+ const first = await pager.claude(file, { limit: 1 })
+ expect(first.lines).toEqual([records[2]])
+ appendFileSync(file, '\n' + claude.user('three'))
+ const lines = await tailFile(file, Infinity)
+ expect(asOld(await pager.claude(file, { limit: 1, before: first.oldestCursor! }))).toEqual(windowRawLines(lines, { limit: 1, before: first.oldestCursor! }))
+ const unended = fresh(records.join('\n'))
+ const page = await pager.claude(unended, { limit: 1 })
+ expect(asOld(await pager.claude(unended, { limit: 1, before: page.oldestCursor! })))
+ .toEqual(windowRawLines(await tailFile(unended, Infinity), { limit: 1, before: page.oldestCursor! }))
+ })
+})
+
diff --git a/cli/src/lib/transcriptPages.ts b/cli/src/lib/transcriptPages.ts
new file mode 100644
index 000000000..16757a6dd
--- /dev/null
+++ b/cli/src/lib/transcriptPages.ts
@@ -0,0 +1,443 @@
+/**
+ * A transcript's history a page at a time — what a client scrolling a thread asks for (`session_get`)
+ * and how long the thread is (`sessions_list`) — without reading the whole file. Attach's rule
+ * (lib/attachTranscript.ts) holds here too: the work a request costs grows with the page, never with the
+ * history. Both requests used to read every line into memory first, so a phone opening the thread of a
+ * long Codex session asked the daemon for as much memory as the crash of 2026-10-03 took.
+ *
+ * Lines are `tailFile(path, Infinity)`'s, the reader every page came from: LF, CR and CRLF end a line,
+ * a line that is only whitespace is no line, and a last line with no ending yet is one. Pages are the
+ * windows `windowRawLines` and `windowCodexLines` cut from those lines, with the same cursors, except:
+ * - a page holds at most `PAGE_MAX_BYTES` (see `pageBefore`), and a line longer than that is counted
+ * but left out — the old window held whatever its turn did, a whole file at worst;
+ * - a Claude cursor names the newest line with that id, where the old window took the oldest. Ids are
+ * unique in practice, and finding the oldest would mean reading back to the start every time.
+ */
+import { open, stat } from 'node:fs/promises'
+import { StringDecoder } from 'node:string_decoder'
+import { codexPageStart, windowCodexLines } from '../engines/codex/normalizer.js'
+import { claudePageLine } from './normalize.js'
+import { scanRecordsBackward, streamRecords } from './transcriptTail.js'
+
+/** The most a page holds. A page used to be however long its turn was, and a turn can be gigabytes. */
+export const PAGE_MAX_BYTES = 32 * 1024 * 1024
+
+/** A line index notes where a line ends at least every this many lines and this many bytes, so finding
+ * any line reads no more than that. */
+export const INDEX_STRIDE_LINES = 4096
+export const INDEX_STRIDE_BYTES = 4 * 1024 * 1024
+
+const CHUNK_BYTES = 1024 * 1024
+const HEAD_BYTES = 64
+const NOT_WHITESPACE = /\S/
+const EMPTY = Buffer.alloc(0)
+
+type Handle = Awaited<ReturnType<typeof open>>
+/** How a file is opened for reading — swapped in tests for one that is cut short as it is read. */
+export type OpenFile = (filePath: string) => Promise<Handle>
+const openForReading: OpenFile = (filePath) => open(filePath, 'r')
+/** How a page is walked — swapped in tests for a walk the file shrinks under. */
+export type Walk = typeof scanRecordsBackward
+
+/** `isBlank`'s rule (lib/transcriptTail.ts) — `line.trim()` is empty — fed a line a chunk at a time. */
+class BlankTest {
+ private found = false
+ private decoder: StringDecoder | null = null
+
+ get seen(): boolean { return this.found }
+
+ feed(bytes: Buffer): void {
+ if (this.found || !bytes.length) return
+ for (const byte of bytes) if (byte > 0x20 && byte < 0x7f) { this.found = true; this.decoder = null; return }
+ this.decoder ??= new StringDecoder('utf8')
+ if (NOT_WHITESPACE.test(this.decoder.write(bytes))) { this.found = true; this.decoder = null }
+ }
+
+ /** Whether the line held anything but whitespace. Resets for the next line. */
+ end(): boolean {
+ const line = this.found || (this.decoder !== null && NOT_WHITESPACE.test(this.decoder.end()))
+ this.found = false
+ this.decoder = null
+ return line
+ }
+}
+
+async function readAt(handle: Handle, position: number, length: number): Promise<Buffer> {
+ const buffer = Buffer.alloc(Math.max(0, length))
+ let read = 0
+ while (read < buffer.length) {
+ const { bytesRead } = await handle.read(buffer, read, buffer.length - read, position + read)
+ if (!bytesRead) break
+ read += bytesRead
+ }
+ return buffer.subarray(0, read)
+}
+
+/**
+ * Walk `[from, to)` — `from` just past a line ending, or 0 — calling `onLine` with the byte just past
+ * each line's ending and `onEnding` past every ending, a blank line's included. Returning true from
+ * `onLine` stops the walk. What follows the last ending is left unread.
+ */
+async function walkLines(
+ handle: Handle,
+ from: number,
+ to: number,
+ onLine: (lineEnd: number) => boolean,
+ onEnding: (end: number) => void = () => {},
+): Promise<void> {
+ const test = new BlankTest()
+ for (let position = from; position < to;) {
+ const chunk = await readAt(handle, position, Math.min(CHUNK_BYTES, to - position))
+ if (!chunk.length) return
+ let lineStart = 0
+ for (let i = 0; i < chunk.length; i++) {
+ if (chunk[i] !== 10 && chunk[i] !== 13) continue
+ test.feed(chunk.subarray(lineStart, i))
+ lineStart = i + 1
+ const end = position + i + 1
+ if (test.end() && onLine(end)) return
+ onEnding(end)
+ }
+ if (lineStart < chunk.length) test.feed(chunk.subarray(lineStart))
+ position += chunk.length
+ }
+}
+
+/**
+ * How many lines a transcript holds and where any of them ends, kept up to date as the file grows:
+ * each look reads only what was appended since the last. A file rewritten in place — it shrank, its
+ * first bytes changed, or a line ending is no longer where one was — is indexed again from the start.
+ * Looks and lookups queue, so neither sees the other half done.
+ */
+export class LineIndex {
+ /** Indexed up to here: just past a line ending, or 0. */
+ private bytes = 0
+ /** Lines that end before `bytes`. */
+ private lines = 0
+ /** Bytes with a known number of lines before them, ascending: `marks[k]` lines before `at[k]`. */
+ private marks: number[] = [0]
+ private at: number[] = [0]
+ private head: Buffer = EMPTY
+ /** The bytes after the last line ending are a line still being written. */
+ private trailing = false
+ private size = 0
+ private queue: Promise<unknown> = Promise.resolve()
+
+ constructor(
+ readonly filePath: string,
+ private readonly stride: { lines: number; bytes: number } = { lines: INDEX_STRIDE_LINES, bytes: INDEX_STRIDE_BYTES },
+ private readonly openFile: OpenFile = openForReading,
+ ) {}
+
+ /** Bring the index up to the file as it is now. */
+ update(): Promise<void> {
+ return this.serial(() => this.look())
+ }
+
+ /** The file's lines, one still being written included: `tailFile(path, Infinity).length`. */
+ count(): number {
+ return this.lines + (this.trailing ? 1 : 0)
+ }
+
+ /** The file's length at the last look. */
+ length(): number {
+ return this.size
+ }
+
+ /** A byte with exactly `n` lines before it — where a page that ends before line `n` ends. */
+ endOf(n: number): Promise<number> {
+ return this.serial(() => this.find(n))
+ }
+
+ private serial<T>(task: () => Promise<T>): Promise<T> {
+ const run = this.queue.then(task)
+ this.queue = run.catch(() => {})
+ return run
+ }
+
+ private reset(): void {
+ this.bytes = 0
+ this.lines = 0
+ this.marks = [0]
+ this.at = [0]
+ this.head = EMPTY
+ this.trailing = false
+ this.size = 0
+ }
+
+ /** A file that is not there is an empty one; any other failure to read it is the caller's to hear. */
+ private async look(): Promise<void> {
+ let handle: Handle
+ try { handle = await this.openFile(this.filePath) } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error
+ this.reset()
+ return
+ }
+ try {
+ const { size } = await handle.stat()
+ if (size < this.bytes || !await this.unchanged(handle)) this.reset()
+ if (this.head.length < HEAD_BYTES && size > this.head.length) this.head = await readAt(handle, 0, Math.min(size, HEAD_BYTES))
+ await walkLines(handle, this.bytes, size, (lineEnd) => {
+ this.lines++
+ const last = this.marks.length - 1
+ if (this.lines - this.marks[last] >= this.stride.lines || lineEnd - this.at[last] >= this.stride.bytes) {
+ this.marks.push(this.lines)
+ this.at.push(lineEnd)
+ }
+ return false
+ }, (end) => { this.bytes = end })
+ this.trailing = await this.trailingLine(handle, size)
+ this.size = size
+ } finally { await handle.close() }
+ }
+
+ private async unchanged(handle: Handle): Promise<boolean> {
+ if (this.head.length && !(await readAt(handle, 0, this.head.length)).equals(this.head)) return false
+ if (!this.bytes) return true
+ const [ending] = await readAt(handle, this.bytes - 1, 1)
+ return ending === 10 || ending === 13
+ }
+
+ /** Whether the bytes after the last line ending hold a line: read only until they show they do. */
+ private async trailingLine(handle: Handle, size: number): Promise<boolean> {
+ const test = new BlankTest()
+ for (let position = this.bytes; position < size && !test.seen;) {
+ const chunk = await readAt(handle, position, Math.min(CHUNK_BYTES, size - position))
+ if (!chunk.length) break
+ test.feed(chunk)
+ position += chunk.length
+ }
+ return test.end()
+ }
+
+ private async find(n: number): Promise<number> {
+ if (n >= this.count()) return this.size
+ if (n <= 0) return 0
+ // The last mark at or before line n.
+ let low = 0
+ for (let high = this.marks.length - 1; low < high;) {
+ const middle = (low + high + 1) >> 1
+ if (this.marks[middle] <= n) low = middle
+ else high = middle - 1
+ }
+ let found = this.at[low]
+ let seen = this.marks[low]
+ if (seen === n) return found
+ const handle = await this.openFile(this.filePath)
+ try {
+ await walkLines(handle, found, this.bytes, (lineEnd) => {
+ found = lineEnd
+ return ++seen === n
+ })
+ } finally { await handle.close() }
+ return found
+ }
+}
+
+/** What the pager asks of an engine's lines. */
+export interface PageRules {
+ /** A page may start at this line — a turn's first, so a turn is never split. */
+ startsPage(line: string): boolean
+}
+
+export interface Page {
+ /** Oldest first. */
+ lines: string[]
+ /** Lines in the page, those too long to hold included. */
+ count: number
+ /** Where the page's first line starts. */
+ start: number
+ /** Lines remain before the page. */
+ hasMore: boolean
+ /** The page starts inside a turn: that one turn held more than a page may. */
+ clipped: boolean
+}
+
+/**
+ * The page that ends just before byte `end`: the `limit` lines before it, and as many more as reach
+ * back to a line a page may start at — the window `windowRawLines` and `windowCodexLines` cut. Null when
+ * the file shrank under the walk.
+ *
+ * Bounded at `maxBytes`, which the old window was not: a page that would hold more ends at the oldest
+ * start that fits, short of `limit`, so a turn is still never split. Only a turn too long to fit at all
+ * is cut at a line inside it (`clipped`), its older part left for the next page. A single line over
+ * `maxBytes` is never held: it is counted where it stands and left out. A file that cannot be read is an
+ * empty one, as it was to the whole-file reader.
+ */
+export async function pageBefore(
+ filePath: string,
+ end: number,
+ limit: number,
+ rules: PageRules,
+ maxBytes = PAGE_MAX_BYTES,
+ walk: Walk = scanRecordsBackward,
+): Promise<Page | null> {
+ const lines: string[] = []
+ let count = 0
+ let bytes = 0
+ let start = end
+ // The page up to its oldest start that fits: where a page that runs out of room ends.
+ let fits: { lines: number; count: number; start: number } | null = null
+ let reached = false
+ let hasMore = false
+ let clipped = false
+ const stop = (): true => { hasMore = true; return true }
+ const walking = end <= 0 ? Promise.resolve(true) : walk(filePath, end, (record, offset) => {
+ if (reached) return stop()
+ if (lines.length && bytes + record.length > maxBytes) {
+ if (fits) {
+ lines.length = fits.lines
+ count = fits.count
+ start = fits.start
+ } else clipped = true
+ return stop()
+ }
+ const line = record.toString('utf8')
+ lines.push(line)
+ count++
+ bytes += record.length
+ start = offset
+ if (rules.startsPage(line)) {
+ fits = { lines: lines.length, count, start }
+ if (count >= limit) reached = true
+ }
+ }, maxBytes, (offset) => {
+ if (reached) return stop()
+ count++
+ start = offset
+ })
+ const whole = await walking.catch(() => 'unreadable' as const)
+ if (whole === 'unreadable') return { lines: [], count: 0, start: 0, hasMore: false, clipped: false }
+ if (!whole) return null
+ lines.reverse()
+ return { lines, count, start, hasMore, clipped }
+}
+
+/** One page of a thread, and the cursor that asks for the page before it. */
+export interface HistoryPage {
+ lines: string[]
+ hasMore: boolean
+ oldestCursor: string | null
+ /** The cursor named nothing in the file: the client reloads the thread. */
+ staleCursor?: true
+ clipped: boolean
+}
+
+const STALE: HistoryPage = { lines: [], hasMore: false, oldestCursor: null, staleCursor: true, clipped: false }
+/** An id that is written into its record exactly as it reads, so its bytes can be searched for. */
+const PLAIN_ID = /^[A-Za-z0-9_.:-]+$/
+
+/**
+ * Pages for the threads clients read. Holds an index per transcript (for Codex's line cursors and line
+ * counts) and the byte each recently served Claude cursor names, so scrolling back one page after
+ * another does not walk the file again from its end.
+ */
+export class TranscriptPager {
+ private readonly indexes = new Map<string, LineIndex>()
+ private readonly cursors = new Map<string, Map<string, number>>()
+
+ constructor(
+ private readonly options: {
+ capacity?: number
+ maxBytes?: number
+ stride?: { lines: number; bytes: number }
+ walk?: Walk
+ } = {},
+ ) {}
+
+ private get capacity(): number { return this.options.capacity ?? 256 }
+ private get maxBytes(): number { return this.options.maxBytes ?? PAGE_MAX_BYTES }
+
+ /** `tailFile(path, Infinity).length`, read once and then only as the file grows. A file that cannot
+ * be read has none, as it had to the whole-file reader. */
+ async lineCount(filePath: string): Promise<number> {
+ try { return (await this.index(filePath)).count() } catch { return 0 }
+ }
+
+ /** `windowRawLines` over the whole file, without it. No `limit` asks for every line, as far as fits. */
+ async claude(filePath: string, opts: { limit?: number; before?: string }): Promise<HistoryPage> {
+ let end = await length(filePath)
+ if (opts.before !== undefined) {
+ const at = await this.findClaude(filePath, end, opts.before)
+ if (at === null) return STALE
+ end = at
+ }
+ const page = await pageBefore(filePath, end, opts.limit ?? Infinity, { startsPage: (line) => claudePageLine(line).startsPage }, this.maxBytes, this.options.walk)
+ if (!page) return STALE
+ // An empty window before a cursor still names that cursor's line, as the old one did.
+ const oldestCursor = page.lines.length ? claudePageLine(page.lines[0]).cursor : opts.before ?? null
+ if (page.lines.length && oldestCursor !== null) this.remember(filePath, oldestCursor, page.start)
+ return { lines: page.lines, hasMore: page.hasMore, oldestCursor, clipped: page.clipped }
+ }
+
+ /** `windowCodexLines` over the whole file, without it. No `limit` asks for every line, as far as fits.
+ * A file that cannot be read is an empty one, as it was to the whole-file reader. */
+ async codex(filePath: string, opts: { limit?: number; before?: string }): Promise<HistoryPage> {
+ try { return await this.codexPage(filePath, opts) } catch {
+ const old = windowCodexLines([], { limit: opts.limit ?? Infinity, before: opts.before })
+ return old.staleCursor ? STALE : { lines: [], hasMore: false, oldestCursor: old.oldestCursor, clipped: false }
+ }
+ }
+
+ private async codexPage(filePath: string, opts: { limit?: number; before?: string }): Promise<HistoryPage> {
+ const index = await this.index(filePath)
+ const count = index.count()
+ let endIndex = count
+ if (opts.before !== undefined) {
+ const match = /^codex:(\d+)$/.exec(opts.before)
+ if (!match) return STALE
+ endIndex = Number(match[1])
+ if (!Number.isSafeInteger(endIndex) || endIndex > count) return STALE
+ }
+ const page = await pageBefore(filePath, await index.endOf(endIndex), opts.limit ?? Infinity, { startsPage: codexPageStart }, this.maxBytes, this.options.walk)
+ if (!page) return STALE
+ return { lines: page.lines, hasMore: page.hasMore, oldestCursor: `codex:${endIndex - page.count}`, clipped: page.clipped }
+ }
+
+ private async index(filePath: string): Promise<LineIndex> {
+ let index = this.indexes.get(filePath)
+ if (index) this.indexes.delete(filePath)
+ index ??= new LineIndex(filePath, this.options.stride)
+ this.indexes.set(filePath, index)
+ while (this.indexes.size > this.capacity) this.indexes.delete(this.indexes.keys().next().value!)
+ await index.update()
+ return index
+ }
+
+ private remember(filePath: string, cursor: string, offset: number): void {
+ let known = this.cursors.get(filePath)
+ if (known) this.cursors.delete(filePath)
+ else known = new Map()
+ this.cursors.set(filePath, known)
+ known.delete(cursor)
+ known.set(cursor, offset)
+ while (known.size > 64) known.delete(known.keys().next().value!)
+ while (this.cursors.size > this.capacity) this.cursors.delete(this.cursors.keys().next().value!)
+ }
+
+ /** Where the newest line with this cursor starts, or null. A remembered byte is checked first. */
+ private async findClaude(filePath: string, end: number, cursor: string): Promise<number | null> {
+ const names = (line: string): boolean => claudePageLine(line).cursor === cursor
+ const known = this.cursors.get(filePath)?.get(cursor)
+ if (known !== undefined && known < end) {
+ let line: string | null = null
+ const read = await streamRecords(filePath, known, end, (record) => { line = record; return true }, () => true, this.maxBytes)
+ if (read && line !== null && names(line)) return known
+ this.cursors.get(filePath)?.delete(cursor)
+ }
+ if (end <= 0) return null
+ const needle = PLAIN_ID.test(cursor) ? Buffer.from(`"${cursor}"`) : null
+ let found: number | null = null
+ const whole = await scanRecordsBackward(filePath, end, (record, offset) => {
+ if (needle && record.indexOf(needle) < 0) return false
+ if (!names(record.toString('utf8'))) return false
+ found = offset
+ return true
+ }, this.maxBytes).catch(() => false)
+ return whole ? found : null
+ }
+}
+
+async function length(filePath: string): Promise<number> {
+ try { return (await stat(filePath)).size } catch { return 0 }
+}
diff --git a/cli/src/lib/transcriptTail.ts b/cli/src/lib/transcriptTail.ts
index 4831b2ee0..f4117d33a 100644
--- a/cli/src/lib/transcriptTail.ts
+++ b/cli/src/lib/transcriptTail.ts
@@ -31,14 +31,16 @@ async function readFully(handle: FileHandle, chunk: Buffer, position: number): P
*
* `visit` gets each non-blank record's bytes — valid only during the call — and the offset it starts
* at; returning true stops the walk. LF, CR and CRLF each end a record, readline's rule. A record over
- * `maxRecordBytes` is passed over unvisited. Resolves false when the file shrank under the walk: what
- * was visited may then mix two versions of it.
+ * `maxRecordBytes` is passed over unvisited, and only told to `passed` (its offset), if given; returning
+ * true from that stops the walk too. Resolves false when the file shrank under the walk: what was
+ * visited may then mix two versions of it.
*/
export async function scanRecordsBackward(
filePath: string,
end: number,
visit: (record: Buffer, offset: number) => boolean | void,
maxRecordBytes = MAX_RECORD_BYTES,
+ passed?: (offset: number) => boolean | void,
): Promise<boolean> {
const handle = await open(filePath, 'r')
try {
@@ -53,7 +55,8 @@ export async function scanRecordsBackward(
fragments = []
fragmentBytes = 0
oversized = false
- if (skip || isBlank(bytes)) return false
+ if (skip) return passed?.(offset) === true
+ if (isBlank(bytes)) return false
return visit(bytes, offset) === true
}
while (position > 0) {
diff --git a/cli/src/testing/transcriptScenarios.ts b/cli/src/testing/transcriptScenarios.ts
new file mode 100644
index 000000000..536f92207
--- /dev/null
+++ b/cli/src/testing/transcriptScenarios.ts
@@ -0,0 +1,116 @@
+/**
+ * Realistic Claude Code and Codex transcripts, record by record, in the shapes the engines write — the
+ * fixtures attach (lib/attachTranscript.spec.ts) and paging (lib/transcriptPages.spec.ts) are checked
+ * against at every cut point.
+ */
+// ── Codex records, in the shapes a 0.159 rollout writes ──────────────────────────────────────────────
+export const cx = (type: string, payload: Record<string, unknown>): string =>
+ JSON.stringify({ timestamp: '2026-10-03T23:21:44.458Z', type, payload })
+export const codex = {
+ meta: (version = '0.159.0') => cx('session_meta', { id: 'thread-1', cli_version: version, cwd: '/tmp/work' }),
+ context: (model: string, effort: string | null, mode: string | null) => cx('turn_context', {
+ turn_id: 't', model, ...(effort ? { reasoning_effort: effort } : {}), ...(mode ? { collaboration_mode: { mode } } : {}),
+ }),
+ settings: (model: string, effort: string, mode: string) => cx('event_msg', {
+ type: 'thread_settings_applied', thread_settings: { model, reasoning_effort: effort, collaboration_mode: { mode } },
+ }),
+ started: (turn: string) => cx('event_msg', { type: 'task_started', turn_id: turn }),
+ user: (text: string) => cx('event_msg', { type: 'user_message', message: text }),
+ userItem: (text: string) => cx('event_msg', {
+ type: 'item_completed', turn_id: 't', item: { type: 'UserMessage', id: 'u', content: [{ type: 'text', text, text_elements: [] }] },
+ }),
+ goal: (objective: string) => cx('response_item', {
+ type: 'message', role: 'user',
+ content: [{ type: 'input_text', text: `<codex_internal_context source="goal">\n<objective>${objective}</objective>\n</codex_internal_context>` }],
+ }),
+ developer: (text: string) => cx('response_item', { type: 'message', role: 'developer', content: [{ type: 'input_text', text }] }),
+ agent: (text: string, phase = 'final_answer') => cx('event_msg', {
+ type: 'item_completed', item: { type: 'AgentMessage', content: [{ type: 'Text', text }], phase },
+ }),
+ reasoning: (text: string) => cx('response_item', { type: 'reasoning', summary: [{ type: 'summary_text', text }] }),
+ call: (id: string, cmd: string) => cx('response_item', { type: 'function_call', call_id: id, name: 'exec_command', arguments: JSON.stringify({ cmd }) }),
+ output: (id: string, output: string) => cx('response_item', { type: 'function_call_output', call_id: id, output }),
+ complete: (turn: string) => cx('event_msg', { type: 'task_complete', turn_id: turn, last_agent_message: 'done' }),
+ aborted: (turn: string) => cx('event_msg', { type: 'turn_aborted', turn_id: turn }),
+ compacted: (bytes: number) => cx('compacted', { message: '📘é'.repeat(bytes / 6), replacement_history: [] }),
+ contextCompacted: () => cx('event_msg', { type: 'context_compacted' }),
+ tokens: () => cx('event_msg', { type: 'token_count', info: { total_token_usage: { input_tokens: 1 } } }),
+}
+
+/** A rollout with every shape attach has to get right, in the order Codex writes them. */
+export const codexScenario = (): string[] => [
+ codex.meta(),
+ codex.developer('<permissions instructions>'),
+ codex.settings('gpt-5.5', 'high', 'default'),
+ codex.started('t1'), codex.context('gpt-5.5', 'high', 'default'), codex.user('first question'),
+ codex.reasoning('Planning'), codex.call('c1', 'ls'), codex.output('c1', 'a\nb'), codex.agent('one answer'), codex.complete('t1'),
+ codex.started('t2'), codex.context('gpt-6', 'xhigh', 'plan'), codex.userItem('second, in the item shape'),
+ codex.reasoning('Thinking 漢字 📘'), codex.agent('looking', 'commentary'),
+ codex.compacted(150_000), codex.contextCompacted(), codex.tokens(),
+ codex.call('c2', 'npm test'), codex.output('c2', 'ok'), codex.agent('second answer'), codex.complete('t2'),
+ codex.settings('gpt-6', 'low', 'default'),
+ codex.started('t3'), codex.context('gpt-6', null, 'default'), codex.user('third: effort left out'),
+ codex.call('c3', 'sleep 1'),
+ codex.user(''), // an empty message opens nothing
+ codex.output('c3', 'slept'), codex.complete('t3'),
+ codex.started('t4'), codex.context('gpt-6', 'medium', 'default'), codex.user('interrupted one'), codex.aborted('t4'),
+ codex.started('t5'), codex.context('gpt-6', 'medium', 'default'), codex.goal('ship the release'), codex.reasoning('goal work'), codex.complete('t5'),
+ codex.started('t6'), codex.context('gpt-6', 'medium', 'default'), codex.goal('ship the release'), codex.agent('continuing'), codex.complete('t6'),
+ codex.started('t7'), codex.context('gpt-6', 'medium', 'default'), codex.user('a question mid-goal'), codex.complete('t7'),
+ codex.started('t8'), codex.context('gpt-6', 'medium', 'default'), codex.goal('ship the release'), codex.reasoning('still the goal'),
+ codex.call('c8', 'make'), codex.complete('t8'),
+ codex.started('t9'), codex.context('gpt-6', 'max', 'plan'), codex.goal('a different goal'), codex.reasoning('new goal'),
+ codex.complete('t9'),
+ // A message sent mid-task, with a call still in flight: its output arrives after the new message.
+ codex.started('t10'), codex.context('gpt-6', 'max', 'default'), codex.user('start the build'),
+ codex.call('c10', 'make all'), codex.user('also run the tests while that builds'),
+ codex.output('c10', 'built'), codex.reasoning('now the tests'), codex.call('c11', 'npm test'),
+]
+
+// ── Claude Code records ───────────────────────────────────────────────────────────────────────────────
+let uuid = 0
+export const cl = (record: Record<string, unknown>): string => JSON.stringify({
+ parentUuid: null, isSidechain: false, userType: 'external', cwd: '/tmp', sessionId: 's', version: '2.1.212',
+ timestamp: '2026-10-03T23:21:44.458Z', uuid: `u${++uuid}`, ...record,
+})
+export const claude = {
+ user: (content: unknown, extra: Record<string, unknown> = {}) => cl({ type: 'user', message: { role: 'user', content }, ...extra }),
+ assistant: (content: unknown[], stop: string | null, model = 'claude-opus-5-5') =>
+ cl({ type: 'assistant', message: { id: `m${uuid}`, role: 'assistant', model, content, stop_reason: stop } }),
+ text: (text: string) => ({ type: 'text', text }),
+ thinking: (thinking: string) => ({ type: 'thinking', thinking }),
+ tool: (id: string, command: string) => ({ type: 'tool_use', id, name: 'Bash', input: { command } }),
+ result: (id: string, content: string) => cl({ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: id, content }] } }),
+ compact: () => cl({ type: 'system', subtype: 'compact_boundary', content: 'Conversation compacted', compactMetadata: { trigger: 'auto' } }),
+ summary: () => cl({ type: 'user', isCompactSummary: true, message: { role: 'user', content: 'Summary of the earlier conversation' } }),
+}
+
+export const claudeScenario = (): string[] => [
+ JSON.stringify({ type: 'summary', summary: 'Earlier work', leafUuid: 'x' }),
+ claude.user('first prompt'),
+ claude.assistant([claude.thinking('let me think'), claude.text('First answer.')], 'end_turn'),
+ claude.user('second prompt with a tool'),
+ claude.assistant([claude.text('Checking.'), claude.tool('toolu_1', 'ls')], 'tool_use'),
+ claude.result('toolu_1', 'files'),
+ claude.assistant([claude.thinking('found them 漢字'), claude.text('Done: '.repeat(20_000))], 'end_turn', 'claude-fable-5-1'),
+ claude.user('<local-command-stdout>Set model to Opus 5.5</local-command-stdout>'),
+ claude.user('<bash-input>git status</bash-input><bash-stdout>clean</bash-stdout>'),
+ claude.user('third prompt, interrupted'),
+ claude.assistant([claude.tool('toolu_2', 'sleep 100')], 'tool_use'),
+ claude.user([{ type: 'text', text: '[Request interrupted by user for tool use]' }]),
+ claude.user('<task-notification><task-id>a</task-id><tool-use-id>toolu_9</tool-use-id><status>completed</status><summary>bg done</summary></task-notification>'),
+ claude.compact(), claude.summary(),
+ claude.user('loop iteration', { isMeta: true, promptSource: 'system' }),
+ claude.assistant([claude.text('Loop ran.')], 'end_turn', '<synthetic>'),
+ claude.user('a bookkeeping line', { isMeta: true }),
+ claude.user('fourth prompt'),
+ claude.assistant([claude.text('Working on it'), claude.tool('toolu_3', 'make')], 'tool_use'),
+ claude.result('toolu_3', 'built'),
+ claude.assistant([claude.thinking('almost'), claude.text('Built it.')], 'end_turn'),
+ claude.user('fifth prompt, left running'),
+ claude.assistant([claude.tool('toolu_4', 'npm test')], 'tool_use'),
+ // A prompt that lands while a call is in flight: the result arrives after it, naming its call.
+ claude.user('sixth prompt, typed during the tests'),
+ claude.result('toolu_4', 'tests passed'),
+ claude.assistant([claude.thinking('reading results'), claude.text('All green.')], 'end_turn'),
+]
diff --git a/cli/src/watcher/watcher.spec.ts b/cli/src/watcher/watcher.spec.ts
index 4d6b9d126..d8e217662 100644
--- a/cli/src/watcher/watcher.spec.ts
+++ b/cli/src/watcher/watcher.spec.ts
@@ -1,9 +1,9 @@
import { chmodSync } from 'fs'
-import { appendFile, mkdtemp, rm, writeFile } from 'fs/promises'
+import { appendFile, mkdtemp, open, rm, writeFile } from 'fs/promises'
import { tmpdir } from 'os'
import { join } from 'path'
import { afterEach, describe, expect, it, vi } from 'vitest'
-import { Watcher, type HistoryEvent, type LineEvent } from './watcher.js'
+import { Watcher, type HistoryEvent, type LineEvent, type RewrittenEvent } from './watcher.js'
const cleanup: string[] = []
@@ -639,3 +639,88 @@ describe('Watcher, at its edges', () => {
await watcher.stop()
})
})
+
+describe('Watcher, reading a backlog', () => {
+ const backlog = async (options: ConstructorParameters<typeof Watcher>[0], content = '') => {
+ const dir = await mkdtemp(join(tmpdir(), 'machine-watcher-backlog-'))
+ cleanup.push(dir)
+ const transcriptPath = join(dir, 'session.jsonl')
+ await writeFile(transcriptPath, content)
+ const watcher = new Watcher(options)
+ const lines: string[] = []
+ const history: string[] = []
+ const rewritten: RewrittenEvent[] = []
+ watcher.on('line', (event: LineEvent) => lines.push(event.text))
+ watcher.on('history', (event: HistoryEvent) => history.push(...event.lines.map((line) => line.text)))
+ watcher.on('rewritten', (event: RewrittenEvent) => rewritten.push(event))
+ await watcher.addSession({ sessionId: 's1', engine: 'codex', transcriptPath })
+ return { transcriptPath, watcher, lines, history, rewritten }
+ }
+ const records = [
+ '{"n":1}', '{"n":2,"text":"漢字 📘 across a chunk"}', `{"n":3,"long":"${'x'.repeat(200)}"}`, '{"n":4}', '', ' ', '{"n":5}',
+ ]
+
+ it('reads what fell behind a chunk at a time: lines and characters split between chunks, a line longer than many', async () => {
+ for (const chunk of [1, 3, 16, 64, 1 << 20]) {
+ const { transcriptPath, watcher, lines } = await backlog({ readChunkBytes: chunk })
+ await appendFile(transcriptPath, records.join('\r\n') + '\r\n{"n":6, still')
+ await watcher.pollSession('s1')
+ expect(lines, `chunks of ${chunk}`).toEqual(records.filter((record) => record.trim()))
+ await appendFile(transcriptPath, ' being written}\n')
+ await watcher.pollSession('s1')
+ expect(lines.at(-1)).toBe('{"n":6, still being written}')
+ await watcher.stop()
+ }
+ })
+
+ it('stops a long read at a hold that lands in it, and hands over every line exactly once', async () => {
+ const all = Array.from({ length: 400 }, (_, i) => `{"n":${i},"pad":"${'y'.repeat(i % 37)}"}`)
+ const { transcriptPath, watcher, lines } = await backlog({ readChunkBytes: 32 })
+ await appendFile(transcriptPath, all.join('\n') + '\n')
+ const reading = watcher.pollAll()
+ const hold = (await watcher.hold('s1', transcriptPath))!
+ await reading
+ const size = Buffer.byteLength(all.join('\n') + '\n')
+ expect(hold.offset).toBeLessThan(size)
+ // Delivery stopped exactly at a line: what came out is everything before that byte.
+ expect(Buffer.byteLength(lines.map((line) => line + '\n').join(''))).toBe(hold.offset)
+ hold.release(hold.offset)
+ await watcher.pollSession('s1')
+ expect(lines).toEqual(all)
+ await watcher.stop()
+ })
+
+ it('attaches a file that shrank to more history than one batch should hold, instead of replaying it', async () => {
+ const { transcriptPath, watcher, lines, history, rewritten } = await backlog({ historyMaxBytes: 40 }, records.join('\n') + '\n')
+ await writeFile(transcriptPath, records.slice(0, 4).join('\n') + '\n')
+ await watcher.pollSession('s1')
+ expect(rewritten).toEqual([{ sessionId: 's1', engine: 'codex', transcriptPath }])
+ expect({ lines, history }).toEqual({ lines: [], history: [] })
+ // The tail goes on from the new end.
+ await appendFile(transcriptPath, '{"n":7}\n')
+ await watcher.pollSession('s1')
+ expect(lines).toEqual(['{"n":7}'])
+ // A shrink small enough is still read as history.
+ await writeFile(transcriptPath, '{"n":8}\n')
+ await watcher.pollSession('s1')
+ expect(history).toEqual(['{"n":8}'])
+ await watcher.stop()
+ })
+
+ it('stops where a file cut short under the read ends', async () => {
+ const shorted = async (path: string) => {
+ const handle = await open(path, 'r')
+ let reads = 0
+ return Object.assign(Object.create(handle), {
+ read: (...args: Parameters<typeof handle.read>) => (++reads > 1 ? Promise.resolve({ bytesRead: 0, buffer: args[0] }) : handle.read(...args)),
+ close: () => handle.close(),
+ })
+ }
+ const { transcriptPath, watcher, lines } = await backlog({ readChunkBytes: 8, openFile: shorted })
+ await appendFile(transcriptPath, '{"n":1}\n{"n":2}\n')
+ await watcher.pollSession('s1')
+ expect(lines).toEqual(['{"n":1}'])
+ await watcher.stop()
+ })
+})
+
diff --git a/cli/src/watcher/watcher.ts b/cli/src/watcher/watcher.ts
index de32bd697..338c9d4c7 100644
--- a/cli/src/watcher/watcher.ts
+++ b/cli/src/watcher/watcher.ts
@@ -48,19 +48,34 @@ export interface TailHold {
release(offset?: number | null): void
}
+/** A transcript rewritten in place with more history than one batch should hold — a long one whose
+ * owner rewrote it. The tail now starts at its new end; the session's state is rebuilt by attaching it
+ * again, from its end. */
+export interface RewrittenEvent {
+ sessionId: string
+ engine: AgentEngine
+ transcriptPath: string
+}
+
+/** How much the tail reads at once: a tail that fell behind reads its backlog a chunk at a time, never
+ * in one buffer as large as everything appended since. */
+export const READ_CHUNK_BYTES = 8 * 1024 * 1024
+/** The most history replayed as one batch. A file that shrank to more than this is attached again. */
+export const HISTORY_MAX_BYTES = 32 * 1024 * 1024
+
/** The longest a session's live delivery may stay held. An attach still running by then lets it go
* (`TailHold.expired`) and must keep the normalizer delivery went back to. */
export const HOLD_TIMEOUT_MS = 30_000
-const EMPTY = Buffer.alloc(0)
-
interface FileState extends WatchedSession {
offset: number
/** Bytes below this were on disk before the current read cursor was placed — history, not live. */
historicalUntil: number
/** The end of the file not yet ending in a newline: raw bytes, decoded only once the line is whole,
- * so a character split between two reads is not mangled and positions stay byte-exact. */
- partial: Buffer
+ * so a character split between two reads is not mangled and positions stay byte-exact. Kept as the
+ * pieces read, joined once when the line ends: a long line is not copied again at every read. */
+ partial: Buffer[]
+ partialBytes: number
seq: number
debounce: NodeJS.Timeout | null
reading: boolean
@@ -82,6 +97,18 @@ export class Watcher extends EventEmitter {
private files = new Map<string, FileState>()
private bySession = new Map<string, string>()
+ private readonly readChunkBytes: number
+ private readonly historyMaxBytes: number
+ private readonly openFile: (filePath: string) => ReturnType<typeof open>
+
+ /** `openFile` is how a transcript is opened for reading: swapped in tests for one cut short as it is read. */
+ constructor(options: { readChunkBytes?: number; historyMaxBytes?: number; openFile?: (filePath: string) => ReturnType<typeof open> } = {}) {
+ super()
+ this.readChunkBytes = options.readChunkBytes ?? READ_CHUNK_BYTES
+ this.historyMaxBytes = options.historyMaxBytes ?? HISTORY_MAX_BYTES
+ this.openFile = options.openFile ?? ((filePath) => open(filePath, 'r'))
+ }
+
start(): void {
if (this.watcher) return
this.watcher = chokidar.watch([], { ignoreInitial: false, alwaysStat: true })
@@ -129,7 +156,8 @@ export class Watcher extends EventEmitter {
offset,
// A `fromStart` tail of a file that already has content reads that content as history.
historicalUntil: opts.fromStart ? size : 0,
- partial: EMPTY,
+ partial: [],
+ partialBytes: 0,
seq: 0,
debounce: null,
reading: false,
@@ -173,7 +201,8 @@ export class Watcher extends EventEmitter {
state.holds.delete(hold)
if (to !== null && state.moves === moves) {
state.offset = to
- state.partial = EMPTY
+ state.partial = []
+ state.partialBytes = 0
}
if (state.holds.size) return
state.settleUnheld?.()
@@ -195,7 +224,7 @@ export class Watcher extends EventEmitter {
}
state.holds.add(hold)
while (state.reading) await new Promise((resolve) => setTimeout(resolve, 5))
- offset = state.offset - state.partial.length
+ offset = state.offset - state.partialBytes
return hold
}
@@ -218,7 +247,8 @@ export class Watcher extends EventEmitter {
if (!state) return
if (state.debounce) { clearTimeout(state.debounce); state.debounce = null }
state.offset = offset
- state.partial = EMPTY
+ state.partial = []
+ state.partialBytes = 0
// An attach holding the tail read the file as it was; where it stopped is no position in this one.
state.moves++
}
@@ -315,43 +345,61 @@ export class Watcher extends EventEmitter {
let size: number
try { size = (await stat(filePath)).size } catch { return }
if (size < state.offset) {
+ state.partial = []
+ state.partialBytes = 0
+ if (size > this.historyMaxBytes) {
+ // Too much to replay as one batch of history: the tail starts at the new end, and the session is
+ // attached again, which reads what it needs from that end (lib/attachTranscript.ts).
+ state.offset = size
+ state.historicalUntil = 0
+ this.emit('rewritten', { sessionId: state.sessionId, engine: state.engine, transcriptPath: filePath } satisfies RewrittenEvent)
+ return
+ }
// The file shrank: rewritten in place (or recreated). Whatever it holds now is read from byte 0,
// and none of it is new to the world — it is history until the write that grows it past this.
state.offset = 0
- state.partial = EMPTY
state.historicalUntil = size
}
if (size <= state.offset) return
- const start = state.offset
- let fresh: Buffer
+ const history: LineEvent[] = []
try {
- const fh = await open(filePath, 'r')
+ const fh = await this.openFile(filePath)
try {
- const buf = Buffer.alloc(size - start)
- const { bytesRead } = await fh.read(buf, 0, buf.length, start)
- fresh = buf.subarray(0, bytesRead)
- state.offset = start + bytesRead
+ while (state.offset < size) {
+ const start = state.offset
+ const chunk = Buffer.alloc(Math.min(this.readChunkBytes, size - start))
+ const { bytesRead } = await fh.read(chunk, 0, chunk.length, start)
+ if (!bytesRead) break
+ state.offset = start + bytesRead
+ this.takeChunk(filePath, state, chunk.subarray(0, bytesRead), start, history)
+ // A hold taken meanwhile stops the read here; `release` reads what is left.
+ if (state.holds.size) break
+ }
} finally {
await fh.close()
}
} catch (err) {
console.error(`[watcher] read failed (${basename(filePath)}):`, err)
- return
}
+ if (history.length) this.flushHistory(state, history)
+ }
- // Lines are cut on the newline BYTE and decoded only once whole. Each line's byte position tells
- // the historical prefix of this read from the live rest: a file appended to between the cursor
- // being placed and this read has both. The carried partial line began that many bytes before `start`.
- const base = start - state.partial.length
- const combined = state.partial.length ? Buffer.concat([state.partial, fresh]) : fresh
+ /**
+ * Lines are cut on the newline BYTE and decoded only once whole. Each line's byte position tells the
+ * historical prefix of a read from its live rest: a file appended to between the cursor being placed
+ * and this read has both. A line carried over from the last chunk began `partialBytes` before `start`.
+ */
+ private takeChunk(filePath: string, state: FileState, chunk: Buffer, start: number, history: LineEvent[]): void {
let lineStart = 0
- const history: LineEvent[] = []
- for (let newline = combined.indexOf(10); newline >= 0; newline = combined.indexOf(10, lineStart)) {
- const linePos = base + lineStart
- const raw = combined.subarray(lineStart, newline).toString('utf8')
+ for (let newline = chunk.indexOf(10); newline >= 0; newline = chunk.indexOf(10, lineStart)) {
+ const linePos = lineStart === 0 ? start - state.partialBytes : start + lineStart
+ const piece = chunk.subarray(lineStart, newline)
+ const bytes = state.partial.length ? Buffer.concat([...state.partial, piece]) : piece
+ state.partial = []
+ state.partialBytes = 0
lineStart = newline + 1
- const text = raw.replace(/\r$/, '')
+ const text = bytes.toString('utf8').replace(/\r$/, '')
if (!text.trim()) continue
const evt: LineEvent = {
sessionId: state.sessionId,
@@ -365,9 +413,12 @@ export class Watcher extends EventEmitter {
if (history.length) this.flushHistory(state, history)
this.emit('line', evt)
}
- if (history.length) this.flushHistory(state, history)
- // A copy: the read buffer can be large, and only its unfinished last line is kept.
- state.partial = lineStart < combined.length ? Buffer.from(combined.subarray(lineStart)) : EMPTY
+ if (lineStart < chunk.length) {
+ // A copy of a chunk's tail, so the rest of the chunk is not kept for it; a chunk that is all one
+ // line is kept as read.
+ state.partial.push(lineStart ? Buffer.from(chunk.subarray(lineStart)) : chunk)
+ state.partialBytes += chunk.length - lineStart
+ }
}
private flushHistory(state: FileState, lines: LineEvent[]): void {
@@ -381,7 +432,8 @@ export class Watcher extends EventEmitter {
while (common < state.cursorLines.length && common < next.length && state.cursorLines[common] === next[common]) common++
state.cursorLines = next
state.offset = 0
- state.partial = EMPTY
+ state.partial = []
+ state.partialBytes = 0
state.historicalUntil = 0
// `completeLines` has already dropped blank lines.
for (const text of next.slice(common)) {
diff --git a/cli/vitest.e2e.config.ts b/cli/vitest.e2e.config.ts
new file mode 100644
index 000000000..f0ae27724
--- /dev/null
+++ b/cli/vitest.e2e.config.ts
@@ -0,0 +1,18 @@
+import { defineConfig } from 'vitest/config'
+
+/**
+ * End-to-end: each file boots real daemons (see e2e/harness/daemon.ts) under throwaway homes with
+ * private tmux servers and fake engines. Files run one at a time so their timing is the daemon's,
+ * not the host's contention.
+ */
+export default defineConfig({
+ test: {
+ environment: 'node',
+ setupFiles: ['./vitest.setup.ts'],
+ include: ['e2e/**/*.e2e.ts'],
+ testTimeout: 180_000,
+ hookTimeout: 180_000,
+ fileParallelism: false,
+ pool: 'forks',
+ },
+})