The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Bound history pages and the live tail by what they serve (#713)
Dee79f40741a687
diff --git a/cli/e2e/bounded.e2e.ts b/cli/e2e/bounded.e2e.tsnew file mode 100644index 000000000..a6def0647--- /dev/null+++ b/cli/e2e/bounded.e2e.ts@@ -0,0 +1,91 @@+/**+ * Memory follows activity, never history. On 2026-10-03 a Codex rollout of 803 MB crash-looped the+ * daemon at its 4 GB heap: attaching read the whole transcript. These run the daemon under a heap far+ * smaller than the transcripts it serves.+ */+import { mkdirSync, statSync, truncateSync } from 'node:fs'+import { join } from 'node:path'+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'+import { LocalClient, type Frame } from './harness/client.js'+import { IsolatedDaemon, until } from './harness/daemon.js'++const HEAP_MIB = 256+const TRANSCRIPT_MIB = 300++describe('bounded memory', () => {+ let daemon: IsolatedDaemon | undefined+ afterEach(async () => { await daemon?.close(); daemon = undefined })++ it.each(['claude', 'codex'] as const)(`%s: a ${TRANSCRIPT_MIB} MiB transcript re-attaches under a ${HEAP_MIB} MiB heap`, async (engine) => {+ daemon = await IsolatedDaemon.create({ heapMiB: HEAP_MIB })+ const d = daemon+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-60).join('\n')}`) })+ await d.start()+ let client = await LocalClient.connect(d)+ const cwd = join(d.projectsDir, engine)+ mkdirSync(cwd, { recursive: true })+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 60_000)+ const agentId: string = created.agent.id+ const row = async (c: LocalClient) => ((await c.request('agents_list', { includeStopped: true })).agents as Array<Record<string, any>>)+ .find((agent) => agent.id === agentId)+ await until('the conversation to bind', async () => (await row(client))?.sessionId, 45_000, 250)++ const isEnd = (frame: Frame) => frame.type === 'turn_ended' && frame.agentId === agentId+ const grown = client.next(isEnd, 170_000, 'the growing turn to end')+ client.send('message', { agentId, content: `!grow ${TRANSCRIPT_MIB}` })+ await grown+ expect(d.child, 'the daemon survived the live tail').not.toBeNull()++ await d.restart()+ client = await LocalClient.connect(d)+ const back = await until('the agent to be back', async () => {+ const agent = await row(client)+ return agent?.status && agent.status !== 'stopped' ? agent : null+ }, 60_000, 250)+ expect(back.status).not.toBe('stopped')+ const next = client.next(isEnd, 30_000, 'a turn after the restart')+ client.send('message', { agentId, content: 'still here?' })+ await next+ expect(d.child, 'the daemon survived the attach').not.toBeNull()+ expect(await d.rssMiB()).toBeLessThan(HEAP_MIB + 200)++ // A phone or the web opening the thread: its length, then page after page back into the turn that+ // wrote the 300 MiB — each page read on its own, never the whole file (lib/transcriptPages.ts).+ const sessionId: string = (await row(client))!.sessionId+ const listed = await client.request('sessions_list', { agentId })+ expect(listed.sessions[0].messageCount).toBeGreaterThan(TRANSCRIPT_MIB)+ let before: string | undefined+ let pages = 0+ for (; pages < 8; pages++) {+ const page = await client.request('session_get', { sessionId, limit: 50, before }, 60_000)+ expect(page.staleCursor, `page ${pages}`).toBeUndefined()+ // Compaction records render nothing, so a page inside the long turn can hold no events: what+ // matters is that every page moves back.+ expect(Array.isArray(page.events), `page ${pages}`).toBe(true)+ expect(page.oldestCursor, `page ${pages}`).not.toBe(before)+ if (!page.hasMore || !page.oldestCursor) break+ before = page.oldestCursor+ }+ expect(pages, 'pages read back into the long turn').toBe(8)+ // An app from before paging asks for the whole thread: it gets the newest page's worth, and a cursor.+ const whole = await client.request('session_get', { sessionId }, 60_000)+ expect(whole).toMatchObject({ hasMore: true })+ expect(d.child, 'the daemon survived the paging').not.toBeNull()+ expect(await d.rssMiB()).toBeLessThan(HEAP_MIB + 200)++ // The transcript rewritten in place, two thirds of it left: more history than the heap holds, so it+ // cannot be replayed in one batch. The tail moves to the new end and the session is attached again,+ // from that end.+ const transcript = engine === 'claude'+ ? join(d.env.CLAUDE_PROJECTS_DIR!, cwd.replace(/[^a-zA-Z0-9]/g, '-'), `${sessionId}.jsonl`)+ : join(d.env.CODEX_HOME!, 'sessions', '2026', '10', '03', `rollout-2026-10-03T00-00-00-${sessionId}.jsonl`)+ truncateSync(transcript, Math.floor(statSync(transcript).size * 2 / 3))+ const after = client.next(isEnd, 60_000, 'a turn after the rewrite')+ client.send('message', { agentId, content: 'and after a rewrite?' })+ await after+ await until('the rewrite to be attached again', () => /transcript rewritten in place — attaching it again/.test(d.log()), 30_000)+ expect(d.child, 'the daemon survived the rewrite').not.toBeNull()+ expect(await d.rssMiB()).toBeLessThan(HEAP_MIB + 200)+ })+})+diff --git a/cli/e2e/harness/client.ts b/cli/e2e/harness/client.tsnew file mode 100644index 000000000..ced0e48f3--- /dev/null+++ b/cli/e2e/harness/client.ts@@ -0,0 +1,78 @@+/**+ * A client that speaks to the daemon exactly as the desktop app does: the local WebSocket over the+ * daemon's Unix socket (or its loopback port), `machine_select`, request frames answered by+ * `<type>_result`, and the event stream in between.+ */+import { randomUUID } from 'node:crypto'+import WebSocket from 'ws'+import type { IsolatedDaemon } from './daemon.js'++export type Frame = { type: string; payload?: Record<string, any>; [key: string]: unknown }++export class LocalClient {+ readonly frames: Frame[] = []+ private waiters: Array<{ test: (frame: Frame) => boolean; done: (frame: Frame) => void }> = []+ closed = false++ private constructor(private readonly ws: WebSocket) {+ ws.on('message', (raw, binary) => {+ if (binary) return+ let frame: Frame+ try { frame = JSON.parse(raw.toString()) as Frame } catch { return }+ this.frames.push(frame)+ for (const waiter of [...this.waiters]) {+ if (waiter.test(frame)) { this.waiters.splice(this.waiters.indexOf(waiter), 1); waiter.done(frame) }+ }+ })+ ws.on('close', () => { this.closed = true })+ }++ static async connect(daemon: IsolatedDaemon, options: { tcp?: boolean; tool?: boolean } = {}): Promise<LocalClient> {+ const url = options.tcp+ ? `ws://127.0.0.1:${daemon.port}/api/local-ws`+ : `ws+unix://${daemon.socketPath}:/api/local-ws`+ const ws = new WebSocket(url)+ await new Promise<void>((resolve, reject) => {+ ws.once('open', () => resolve())+ ws.once('error', reject)+ })+ const client = new LocalClient(ws)+ client.send('machine_select', { machineId: daemon.computerId, localProtocolVersion: 1, ...(options.tool ? { tool: true } : {}) })+ await client.waitFor((frame) => frame.type === 'connected', 10_000, 'connected')+ return client+ }++ send(type: string, payload: Record<string, unknown>): void {+ this.ws.send(JSON.stringify({ type, payload }))+ }++ /** Resolves with the first frame, past or future, that passes `test`. */+ waitFor(test: (frame: Frame) => boolean, ms = 20_000, what = 'a frame', since = 0): Promise<Frame> {+ const seen = this.frames.slice(since).find(test)+ if (seen) return Promise.resolve(seen)+ return new Promise((resolve, reject) => {+ const timer = setTimeout(() => {+ this.waiters = this.waiters.filter((waiter) => waiter.done !== done)+ reject(new Error(`timed out after ${ms}ms waiting for ${what}; last frames: ${JSON.stringify(this.frames.slice(-8).map((f) => f.type))}`))+ }, ms)+ const done = (frame: Frame) => { clearTimeout(timer); resolve(frame) }+ this.waiters.push({ test, done })+ })+ }++ /** Only frames that arrive after this call. */+ next(test: (frame: Frame) => boolean, ms = 20_000, what = 'a frame'): Promise<Frame> {+ return this.waitFor(test, ms, what, this.frames.length)+ }++ async request<T = Record<string, any>>(type: string, payload: Record<string, unknown> = {}, ms = 30_000): Promise<T> {+ const requestId = randomUUID()+ const answer = this.next((frame) => frame.type === `${type}_result` && frame.payload?.requestId === requestId, ms, `${type}_result`)+ this.send(type, { requestId, ...payload })+ return (await answer).payload as T+ }++ close(): void {+ try { this.ws.close() } catch { /* already closed */ }+ }+}diff --git a/cli/e2e/harness/daemon.ts b/cli/e2e/harness/daemon.tsnew file mode 100644index 000000000..129d2cd0e--- /dev/null+++ b/cli/e2e/harness/daemon.ts@@ -0,0 +1,202 @@+/**+ * A daemon under test: the real daemon from this checkout, booted under a throwaway home with its own+ * data directory, port, private tmux server and fake Claude Code and Codex engines, signed out and with+ * every optional feature that reaches beyond the machine turned off. Nothing it does can touch the+ * person's own daemon, tmux server, transcripts or credentials.+ */+import { execFile, spawn, type ChildProcess } from 'node:child_process'+import { existsSync, readFileSync } from 'node:fs'+import { mkdir, writeFile } from 'node:fs/promises'+import { createServer } from 'node:net'+import { dirname, join, resolve } from 'node:path'+import { fileURLToPath, pathToFileURL } from 'node:url'+import { promisify } from 'node:util'+import { isolatedTmux, type IsolatedTmux } from '../../src/testing/isolatedTmux.js'++const exec = promisify(execFile)+const here = dirname(fileURLToPath(import.meta.url))+export const CLI_ROOT = resolve(here, '../..')++export interface DaemonOptions {+ /** Extra environment for the daemon. */+ env?: Record<string, string>+ /** V8 heap ceiling for the daemon process, MiB. Defaults to Node's own. */+ heapMiB?: number+ /** Models the fake engines report. */+ claudeModel?: string+ codexModel?: string+}++async function freePort(): Promise<number> {+ const server = createServer()+ await new Promise<void>((done) => server.listen(0, '127.0.0.1', done))+ const address = server.address()+ await new Promise<void>((done) => server.close(() => done()))+ if (!address || typeof address === 'string') throw new Error('no free port')+ return address.port+}++export const until = async <T>(what: string, probe: () => Promise<T | null | undefined | false> | T | null | undefined | false, ms = 30_000, every = 100): Promise<T> => {+ const deadline = Date.now() + ms+ let last: unknown+ while (Date.now() < deadline) {+ try {+ const value = await probe()+ if (value) return value+ } catch (error) { last = error }+ await new Promise((done) => setTimeout(done, every))+ }+ throw new Error(`timed out after ${ms}ms waiting for ${what}${last ? ` (last error: ${String(last)})` : ''}`)+}++export class IsolatedDaemon {+ child: ChildProcess | null = null+ private output = ''++ private constructor(+ readonly root: string,+ readonly tmux: IsolatedTmux,+ readonly port: number,+ readonly env: NodeJS.ProcessEnv,+ readonly options: DaemonOptions,+ ) {}++ static async create(options: DaemonOptions = {}): Promise<IsolatedDaemon> {+ const tmux = await isolatedTmux()+ const root = tmux.root+ const port = await freePort()+ const dirs = {+ home: join(root, 'home'), data: join(root, 'data'), runtime: join(root, 'runtime'), auth: join(root, 'auth'),+ bin: join(root, 'bin'), dsh: join(root, 'dsh'), claudeProjects: join(root, 'claude', 'projects'), codexHome: join(root, 'codex'),+ projects: join(root, 'projects'),+ }+ for (const dir of Object.values(dirs)) await mkdir(dir, { recursive: true })+ const config = {+ port, dataDir: dirs.data, claudeProjectsDir: dirs.claudeProjects, codexHome: dirs.codexHome,+ claudeModel: options.claudeModel, codexModel: options.codexModel,+ }+ const engine = pathToFileURL(join(here, 'fakeEngine.mjs')).href+ for (const name of ['claude', 'codex']) {+ await writeFile(join(dirs.bin, name),+ `#!${process.execPath}\nimport(${JSON.stringify(engine)}).then((m) => m.run(${JSON.stringify(name)}, ${JSON.stringify(config)}))\n`,+ { mode: 0o755 })+ }+ const env: NodeJS.ProcessEnv = {+ ...tmux.env,+ NODE_ENV: 'test',+ HOME: dirs.home,+ // A login shell in a pane must not load anyone's zsh configuration.+ ZDOTDIR: dirs.home,+ PORT: String(port),+ ADAPTER_DATA_DIR: dirs.data,+ ADAPTER_RUNTIME_DIR: dirs.runtime,+ ADAPTER_COMPUTER_ID: 'e2e-computer-0000-0000-000000000001',+ ADAPTER_COMPUTER_ID_FILE: join(root, 'computer-id'),+ HARNESS_AUTH_DIR: dirs.auth,+ DSH_DIR: dirs.dsh,+ CLAUDE_PATH: join(dirs.bin, 'claude'),+ CODEX_PATH: join(dirs.bin, 'codex'),+ CLAUDE_PROJECTS_DIR: dirs.claudeProjects,+ CODEX_HOME: dirs.codexHome,+ PATH: `${dirs.bin}:${process.env.PATH}`,+ // Signed out, and nothing reaches beyond the machine.+ BACKEND_WS_URL: 'ws://127.0.0.1:9',+ WEB_URL: 'http://127.0.0.1:9',+ DISABLE_HOOK_INSTALL: 'true',+ ADAPTER_UPDATE_DISABLE: 'true',+ ANALYTICS_ENABLED: 'false',+ RECAP_FORCE: 'false',+ RECAP_WITHOUT_DEVICE: 'false',+ CABLE_DISABLE: 'true',+ CABLE_FW_DISABLE: 'true',+ TERMINAL_BACKENDS: 'tmux',+ TMUX_REAP_INTERVAL_MS: '5000',+ TERMINAL_RECONCILE_INTERVAL_MS: '5000',+ ...options.env,+ }+ return new IsolatedDaemon(root, tmux, port, env, options)+ }++ get dataDir(): string { return this.env.ADAPTER_DATA_DIR! }+ get computerId(): string { return this.env.ADAPTER_COMPUTER_ID! }+ get socketPath(): string { return join(this.dataDir, `daemon-${this.port}.sock`) }+ get projectsDir(): string { return join(this.root, 'projects') }+ get pid(): number | null { return this.child?.pid ?? null }+ log(): string { return this.output }++ /** `ready: 'port'` returns as soon as the port answers, the way a client sees a restarting daemon. */+ async start(options: { ready?: 'port' | 'wired' } = {}): Promise<void> {+ if (this.child) throw new Error('already running')+ const heap = this.options.heapMiB ? [`--max-old-space-size=${this.options.heapMiB}`] : []+ const child = spawn(process.execPath, [...heap, '--import', 'tsx', 'src/cli.ts', '__run'], {+ cwd: CLI_ROOT, env: this.env, stdio: ['ignore', 'pipe', 'pipe'],+ })+ this.child = child+ const take = (chunk: Buffer) => { this.output += chunk.toString('utf8') }+ child.stdout!.on('data', take)+ child.stderr!.on('data', take)+ const exited = new Promise<never>((_, reject) => child.once('exit', (code, signal) => {+ if (this.child === child) this.child = null+ reject(new Error(`daemon exited during boot (${signal ?? code}):\n${this.output.slice(-4000)}`))+ }))+ exited.catch(() => {}) // observed below while booting; afterwards an exit is the test's business+ await Promise.race([exited, (async () => {+ await until('the daemon to answer on its port', async () => {+ const response = await fetch(`http://127.0.0.1:${this.port}/api/health`).catch(() => null)+ return response?.ok ? true : null+ }, 60_000, 200)+ await until('the daemon socket', () => existsSync(this.socketPath), 10_000)+ // The port answers ~1,100 lines of startup before every handler is wired (see the startup-race+ // test); this line is printed once they are.+ if (options.ready !== 'port') {+ await until('the daemon to finish starting', () => /\[cli\] (dialing|not signed in)/.test(this.output), 60_000, 100)+ }+ })()])+ }++ /** SIGTERM, then SIGKILL after `ms`. Resolves when the process is gone. */+ async stop(ms = 15_000): Promise<void> {+ const child = this.child+ if (!child) return+ const exited = new Promise<void>((done) => child.once('exit', () => done()))+ child.kill('SIGTERM')+ const timer = setTimeout(() => child.kill('SIGKILL'), ms)+ await exited+ clearTimeout(timer)+ }++ /** The way a crash ends it: no shutdown path runs. */+ async kill(): Promise<void> {+ const child = this.child+ if (!child) return+ const exited = new Promise<void>((done) => child.once('exit', () => done()))+ child.kill('SIGKILL')+ await exited+ }++ async restart(): Promise<void> {+ await this.stop()+ await this.start()+ }++ /** Resident memory of the daemon process, MiB. */+ async rssMiB(): Promise<number> {+ if (!this.child?.pid) return 0+ const { stdout } = await exec('ps', ['-o', 'rss=', '-p', String(this.child.pid)])+ return Number(stdout.trim()) / 1024+ }++ /** What the daemon's pane shows, plain text. */+ async capture(pane: string): Promise<string> {+ return this.tmux.run('capture-pane', '-p', '-t', pane)+ }++ hookCredential(): string {+ return readFileSync(join(this.dataDir, 'hook-credential'), 'utf8').trim()+ }++ async close(): Promise<void> {+ await this.stop().catch(() => {})+ await this.tmux.close()+ }+}diff --git a/cli/e2e/harness/fakeEngine.mjs b/cli/e2e/harness/fakeEngine.mjsnew file mode 100644index 000000000..1b9d24ad0--- /dev/null+++ b/cli/e2e/harness/fakeEngine.mjs@@ -0,0 +1,178 @@+// A deterministic stand-in for Claude Code and Codex, run inside a real tmux pane by the daemon+// under test. It behaves the way the daemon depends on the real CLIs behaving, and nothing more:+//+// - answers `--version` / `--help` probes;+// - announces its session through the same authenticated hook the real `notify.mjs` uses, from+// inside the pane (so the daemon's pane/process binding is exercised for real);+// - draws a composer, turns on bracketed paste, and treats Enter as submit;+// - writes its conversation to a transcript in the engine's real record shapes, so the daemon's+// normalizers, turn lifecycle, chips and attach read are exercised for real.+//+// A prompt can carry directives that script the turn: `!slow <ms>` holds the answer back,+// `!tool <command>` runs a tool call first, `!grow <MiB>` appends that much compaction history+// before answering (the transcripts that crashed the daemon on 2026-10-03), `!hold` leaves the turn+// open until the next prompt, `!exit` ends the process. Everything else is echoed as the answer.+import { randomUUID } from 'node:crypto'+import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'+import { dirname, join } from 'node:path'++export async function run(engine, config) {+ const args = process.argv.slice(2)+ if (args.includes('--version') || args.includes('-V')) {+ process.stdout.write(engine === 'claude' ? '2.1.270 (Claude Code)\n' : 'codex-cli 0.159.0\n')+ return+ }+ if (args.includes('--help') || args.includes('-h')) {+ // The flags the daemon looks for in the real CLIs' help before it launches them.+ process.stdout.write(engine === 'codex'+ ? [+ 'Usage: codex [OPTIONS] [PROMPT]', '',+ ' -c, --config <key=value>', ' -m, --model <MODEL>', ' -s, --sandbox <SANDBOX_MODE>',+ ' -a, --ask-for-approval <APPROVAL_POLICY>', ' --approve-for-me',+ ' --dangerously-bypass-approvals-and-sandbox', ' --no-daemon',+ 'Commands:', ' resume Resume a previous interactive session', '',+ ].join('\n')+ : [+ 'Usage: claude [options] [command] [prompt]', '', 'Arguments:', ' prompt Your prompt', '',+ ' --model <model>', ' --permission-mode <mode> (choices: "acceptEdits", "auto", "bypassPermissions", "default", "plan")',+ ' --dangerously-skip-permissions', ' -r, --resume [value]', ' --fork-session', ' --session-id <uuid>', '',+ ].join('\n'))+ return+ }+ process.title = engine++ const resumeAt = engine === 'claude' ? args.indexOf('--resume') : args.indexOf('resume')+ const resumed = resumeAt >= 0 && args[resumeAt + 1] && !args[resumeAt + 1].startsWith('-') ? args[resumeAt + 1] : null+ const fork = args.includes('--fork-session')+ const sessionId = resumed && !fork ? resumed : randomUUID()+ const cwd = process.cwd()+ const transcript = engine === 'claude'+ ? join(config.claudeProjectsDir, cwd.replace(/[^a-zA-Z0-9]/g, '-'), `${sessionId}.jsonl`)+ : resumed && !fork && config.rolloutFor?.[resumed]+ ? config.rolloutFor[resumed]+ : join(config.codexHome, 'sessions', '2026', '10', '03', `rollout-2026-10-03T00-00-00-${sessionId}.jsonl`)+ mkdirSync(dirname(transcript), { recursive: true })+ if (!existsSync(transcript)) writeFileSync(transcript, '')++ const now = () => new Date().toISOString()+ const write = (record) => appendFileSync(transcript, JSON.stringify(record) + '\n')+ let parent = null+ const claude = (record) => {+ const uuid = randomUUID()+ write({ parentUuid: parent, isSidechain: false, userType: 'external', cwd, sessionId, version: '2.1.270', timestamp: now(), uuid, ...record })+ parent = uuid+ }+ const codex = (type, payload) => write({ timestamp: now(), type, payload })+ if (engine === 'codex' && readFileSync(transcript, 'utf8') === '') {+ codex('session_meta', { id: sessionId, cli_version: '0.159.0', cwd, source: 'cli' })+ }++ const hook = async (path, body) => {+ const token = readFileSync(join(config.dataDir, 'hook-credential'), 'utf8').trim()+ const pane = process.env.TMUX_PANE+ const response = await fetch(`http://127.0.0.1:${config.port}/api/hook/${path}`, {+ method: 'POST',+ headers: { 'content-type': 'application/json', 'x-harness-hook-token': token },+ body: JSON.stringify({+ engine, sessionId, transcriptPath: transcript, cwd,+ tmuxPane: pane, runtimeHints: pane ? [{ backend: 'tmux', paneId: pane }] : [], callerPid: process.pid,+ ...body,+ }),+ }).catch((error) => ({ ok: false, status: 0, text: async () => String(error) }))+ if (!response.ok) process.stdout.write(`\r\n[fake ${engine}] hook ${path} failed: ${response.status}\r\n`)+ }++ const draw = (line = '') => process.stdout.write(`\r\x1b[2K› ${line}`)+ process.stdout.write(`\x1b[?2004h${engine === 'claude' ? '✻ Welcome to Claude Code (fake)' : '>_ OpenAI Codex (fake)'}\r\n`)+ process.stdout.write(` session ${sessionId}${resumed ? ' (resumed)' : ''}\r\n\r\n`)+ draw()+ await hook('session-start', { hookEvent: resumed ? 'SessionStart' : 'SessionStart', source: resumed ? 'resume' : 'startup' })++ let turn = 0+ let open = null+ const finish = (text) => {+ if (engine === 'claude') {+ claude({ type: 'assistant', message: { id: `msg_${turn}`, role: 'assistant', model: config.claudeModel ?? 'claude-opus-5-5', content: [{ type: 'text', text }], stop_reason: 'end_turn' } })+ } else {+ codex('event_msg', { type: 'item_completed', item: { type: 'AgentMessage', content: [{ type: 'Text', text }], phase: 'final_answer' } })+ codex('event_msg', { type: 'task_complete', turn_id: open, last_agent_message: text })+ }+ process.stdout.write(`\r\n${text}\r\n\r\n`)+ open = null+ draw()+ if (engine === 'claude') void hook('turn-stop', { status: 'ok' })+ }++ const handle = async (raw) => {+ const prompt = raw.trim()+ if (!prompt) { draw(); return }+ if (prompt === '!exit') { process.stdout.write('\x1b[?2004l\r\n'); process.exit(0) }+ if (open) finish('(interrupted by a new prompt)')+ turn+++ open = `turn-${turn}`+ process.stdout.write(`\r\n`)+ if (engine === 'claude') {+ claude({ type: 'user', message: { role: 'user', content: prompt } })+ claude({ type: 'assistant', message: { id: `msg_${turn}_t`, role: 'assistant', model: config.claudeModel ?? 'claude-opus-5-5', content: [{ type: 'thinking', thinking: `considering: ${prompt}` }], stop_reason: null } })+ } else {+ codex('event_msg', { type: 'task_started', turn_id: open })+ codex('turn_context', { turn_id: open, model: config.codexModel ?? 'gpt-6', reasoning_effort: 'high', collaboration_mode: { mode: 'default' } })+ codex('event_msg', { type: 'item_completed', turn_id: open, item: { type: 'UserMessage', id: open, content: [{ type: 'text', text: prompt, text_elements: [] }] } })+ codex('response_item', { type: 'reasoning', summary: [{ type: 'summary_text', text: `considering: ${prompt}` }] })+ }+ const directive = /^!(\w+)\s*(.*)$/.exec(prompt)+ if (directive?.[1] === 'grow' || directive?.[1] === 'burst') {+ // `grow` writes the way Codex compacts — a snapshot at a time, with the engine still working in+ // between — so a live tail sees a few MiB per read. `burst` writes it all at once.+ const mib = Number(directive[2]) || 1+ const chunk = 'x'.repeat(1024 * 1024 - 256)+ for (let i = 0; i < mib; i++) {+ if (engine === 'claude') claude({ type: 'user', isCompactSummary: true, message: { role: 'user', content: chunk } })+ else codex('compacted', { message: chunk, replacement_history: [] })+ if (directive[1] === 'grow' && i % 4 === 3) await new Promise((resolve) => setTimeout(resolve, 60))+ }+ }+ if (directive?.[1] === 'tool') {+ const id = `call_${turn}`+ if (engine === 'claude') {+ claude({ type: 'assistant', message: { id: `msg_${turn}_u`, role: 'assistant', model: config.claudeModel ?? 'claude-opus-5-5', content: [{ type: 'tool_use', id, name: 'Bash', input: { command: directive[2] } }], stop_reason: 'tool_use' } })+ claude({ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: id, content: `ran ${directive[2]}` }] } })+ } else {+ codex('response_item', { type: 'function_call', call_id: id, name: 'exec_command', arguments: JSON.stringify({ cmd: directive[2] }) })+ codex('response_item', { type: 'function_call_output', call_id: id, output: `ran ${directive[2]}` })+ }+ }+ if (directive?.[1] === 'hold') return+ if (directive?.[1] === 'slow') await new Promise((resolve) => setTimeout(resolve, Number(directive[2]) || 1000))+ finish(`answer ${turn}: ${prompt}`)+ }++ // Raw input: bracketed paste brackets the text, Enter (\r) submits, Ctrl-C interrupts.+ process.stdin.setRawMode?.(true)+ process.stdin.setEncoding('utf8')+ let buffer = ''+ let queue = Promise.resolve()+ process.stdin.on('data', (chunk) => {+ for (const part of chunk.replace(/\x1b\[20[01]~/g, '').split(/(\r|\n|\x03)/)) {+ if (part === '\x03') {+ if (open) {+ if (engine === 'claude') claude({ type: 'user', message: { role: 'user', content: [{ type: 'text', text: '[Request interrupted by user]' }] } })+ else codex('event_msg', { type: 'turn_aborted', turn_id: open })+ open = null+ process.stdout.write('\r\n(interrupted)\r\n')+ draw()+ }+ buffer = ''+ } else if (part === '\r' || part === '\n') {+ const line = buffer+ buffer = ''+ queue = queue.then(() => handle(line))+ } else if (part) {+ buffer += part+ draw(buffer)+ }+ }+ })+ process.on('SIGTERM', () => process.exit(0))+ setInterval(() => {}, 60_000)+}diff --git a/cli/e2e/lifecycle.e2e.ts b/cli/e2e/lifecycle.e2e.tsnew file mode 100644index 000000000..18f5c997c--- /dev/null+++ b/cli/e2e/lifecycle.e2e.ts@@ -0,0 +1,76 @@+/**+ * The daemon's everyday contract, end to end, for Claude Code and Codex: a client connects the way the+ * desktop app does, starts an agent, the agent binds its conversation, a message becomes a turn that+ * starts and ends, the daemon restarts and picks the agent back up, and the agent stops.+ */+import { mkdirSync } from 'node:fs'+import { join } from 'node:path'+import { afterAll, afterEach, beforeAll, describe, expect, it, onTestFailed } from 'vitest'+import { LocalClient, type Frame } from './harness/client.js'+import { IsolatedDaemon, until } from './harness/daemon.js'++const engines = ['claude', 'codex'] as const++describe('the daemon, end to end', () => {+ let daemon: IsolatedDaemon+ beforeAll(async () => { daemon = await IsolatedDaemon.create(); await daemon.start() })+ afterAll(async () => { await daemon?.close() })+ afterEach(() => {})+++ const row = async (client: LocalClient, agentId: string) =>+ ((await client.request<{ agents: Array<Record<string, any>> }>('agents_list', { includeStopped: true })).agents)+ .find((agent) => agent.id === agentId)++ it('answers the desktop handshake on its socket and its port, with no agents', async () => {+ for (const tcp of [false, true]) {+ const client = await LocalClient.connect(daemon, { tcp })+ expect((await client.request('agents_list', {})).agents).toEqual([])+ client.close()+ }+ })++ it.each(engines)('%s: start, bind, a message becomes a turn, restart, stop', async (engine) => {+ onTestFailed(() => { console.log(`---- daemon log (${engine})\n${daemon.log().split('\n').slice(-80).join('\n')}`) })+ const client = await LocalClient.connect(daemon)+ const cwd = join(daemon.projectsDir, engine)+ mkdirSync(cwd, { recursive: true })+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 60_000)+ expect(created.error, JSON.stringify(created)).toBeUndefined()+ const agentId: string = created.agent.id++ const bound = await until(`${engine} to bind its conversation`, async () => {+ const agent = await row(client, agentId)+ return agent?.sessionId ? agent : null+ }, 45_000, 500)+ expect(bound.status).not.toBe('stopped')++ const isTurn = (type: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId+ const started = client.next(isTurn('turn_started'), 30_000, 'turn_started')+ const ended = client.next(isTurn('turn_ended'), 30_000, 'turn_ended')+ client.send('message', { agentId, content: 'hello there' })+ expect((await started).payload?.userMessage).toBe('hello there')+ await ended++ // A restart re-attaches the session from its transcript: no turn is replayed as open.+ await daemon.restart()+ const again = await LocalClient.connect(daemon)+ const seen: string[] = []+ const back = await until(`${engine} to be back after a restart`, async () => {+ const agent = await row(again, agentId)+ seen.push(`${agent?.status}:${agent?.sessionId?.slice(0, 8)}`)+ return agent?.sessionId === bound.sessionId && agent.status !== 'stopped' ? agent : null+ }, 45_000, 250)+ console.log(`${engine} rows seen after the restart: ${seen.join(' ')}`)+ expect(back.status).not.toBe('stopped')+ const second = again.next(isTurn('turn_ended'), 30_000, 'second turn_ended')+ again.send('message', { agentId, content: 'and again' })+ await second++ const stopped = await again.request('agent_delete', { agentId }, 60_000)+ expect(stopped.error, JSON.stringify(stopped)).toBeUndefined()+ await until(`${engine} to stop`, async () => (await row(again, agentId))?.status === 'stopped' || null, 45_000, 500)+ again.close()+ client.close()+ })+})diff --git a/cli/e2e/startup.e2e.ts b/cli/e2e/startup.e2e.tsnew file mode 100644index 000000000..78385ffbc--- /dev/null+++ b/cli/e2e/startup.e2e.ts@@ -0,0 +1,71 @@+/**+ * What a client sees while the daemon starts. The desktop app connects the moment the daemon's port+ * answers, so everything the daemon says from then on is something a person sees.+ *+ * Known defects of the current daemon are pinned with `it.fails`: each passes today only because it+ * fails, and harnessd must turn it green (then it becomes an ordinary `it`).+ */+import { mkdirSync } from 'node:fs'+import { join } from 'node:path'+import { afterEach, describe, expect, it } from 'vitest'+import { LocalClient } from './harness/client.js'+import { IsolatedDaemon, until } from './harness/daemon.js'++describe('starting up', () => {+ let daemon: IsolatedDaemon | undefined+ afterEach(async () => { await daemon?.close(); daemon = undefined })++ // The port answers, and the pid file is written, long before every request handler is wired:+ // cli.ts writes PID_FILE at ~4325, assigns backend.onCreateAgent at ~5435 and onForkAgent later+ // still, with the restore and the first full reconcile awaited in between. A request in that window+ // is refused with UNSUPPORTED_ON_REMOTE, which no client expects from its own machine.+ it.fails('answers every request with its real handler from the instant its port answers', async () => {+ daemon = await IsolatedDaemon.create()+ await daemon.start()+ const setup = await LocalClient.connect(daemon)+ for (const name of ['a', 'b', 'c']) {+ const cwd = join(daemon.projectsDir, name)+ mkdirSync(cwd, { recursive: true })+ await setup.request('agent_create', { engine: 'claude', cwd, bypassPermission: true }, 60_000)+ }+ setup.close()+ await daemon.stop()+ await daemon.start({ ready: 'port' })+ const client = await LocalClient.connect(daemon)+ const errors: string[] = []+ const deadline = Date.now() + 3_000+ while (Date.now() < deadline) {+ errors.push((await client.request('agent_fork', { agentId: 'no-such-agent' })).error)+ }+ expect(errors).not.toContain('UNSUPPORTED_ON_REMOTE')+ })++ // After a restart the registry is served before the first reconcile confirms the panes: a running+ // agent lists as `stopped` (when it had a conversation) or not at all, for ~100-200 ms after the+ // daemon says it is ready. The app shows it paused or gone meanwhile.+ it.fails('never shows a running agent as stopped or missing across a restart', async () => {+ daemon = await IsolatedDaemon.create()+ await daemon.start()+ let client = await LocalClient.connect(daemon)+ const cwd = join(daemon.projectsDir, 'restart')+ mkdirSync(cwd, { recursive: true })+ const agentId: string = (await client.request('agent_create', { engine: 'claude', cwd, bypassPermission: true }, 60_000)).agent.id+ const row = async () => ((await client.request('agents_list', { includeStopped: true })).agents as Array<Record<string, any>>)+ .find((agent) => agent.id === agentId)+ await until('the conversation to bind', async () => (await row())?.sessionId, 45_000, 200)+ const ended = client.next((frame) => frame.type === 'turn_ended' && frame.agentId === agentId)+ client.send('message', { agentId, content: 'hello' })+ await ended++ await daemon.stop()+ await daemon.start({ ready: 'port' })+ client = await LocalClient.connect(daemon)+ const seen: string[] = []+ const deadline = Date.now() + 5_000+ while (Date.now() < deadline) {+ seen.push((await row())?.status ?? 'absent')+ await new Promise((done) => setTimeout(done, 50))+ }+ expect(seen.filter((status) => status === 'stopped' || status === 'absent')).toEqual([])+ })+})diff --git a/cli/package.json b/cli/package.jsonindex 052545b18..474a6fc60 100644--- a/cli/package.json+++ b/cli/package.json@@ -14,6 +14,7 @@"start": "node dist/cli.js","test": "vitest run","test:local-models": "vitest run src/lib/localModels.spec.ts src/lib/localModels.integration.spec.ts src/dsh/builtins.spec.ts src/lib/gridFleetRpc.spec.ts --coverage --coverage.include=src/lib/localModels.ts --coverage.include=src/dsh/builtins.ts --coverage.thresholds.lines=100 --coverage.thresholds.statements=100 --coverage.thresholds.functions=100 --coverage.thresholds.branches=100 --coverage.reporter=text --coverage.reporter=json-summary --coverage.reportsDirectory=coverage/local-models",+ "test:e2e": "vitest run --config vitest.e2e.config.ts","test:orchestrator": "vitest run src/orchestrator --maxWorkers=1 --coverage --coverage.include='src/orchestrator/**' --coverage.thresholds.lines=100 --coverage.thresholds.statements=100 --coverage.thresholds.functions=100 --coverage.thresholds.branches=100 --coverage.reporter=text --coverage.reporter=json --coverage.reportsDirectory=coverage/orchestrator","test:dsh-updates": "vitest run --config vitest.dsh-updates.config.ts","test:sharing": "vitest run src/sharing --coverage --coverage.include='src/sharing/**' --coverage.thresholds.lines=100 --coverage.thresholds.statements=100 --coverage.thresholds.branches=100 --coverage.thresholds.functions=100",diff --git a/cli/src/backendSocket.history.spec.ts b/cli/src/backendSocket.history.spec.tsnew file mode 100644index 000000000..3ee8c3268--- /dev/null+++ b/cli/src/backendSocket.history.spec.ts@@ -0,0 +1,153 @@+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'+import { join } from 'node:path'+import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { BackendSocket } from './backendSocket.js'+import { codexMessagesToEvents, windowCodexLines } from './engines/codex/normalizer.js'+import { messagesToEvents, subagentStatsFromRawLines, windowRawLines } from './lib/normalize.js'+import { registry } from './lib/registry.js'+import { tailFile } from './lib/transcriptTail.js'+import { cl, claude, claudeScenario, codexScenario } from './testing/transcriptScenarios.js'++// Transcripts are only served from the engines' own folders. Those default to this computer's real+// ~/.claude and ~/.codex, so they are pointed at throwaway ones before the config is read.+const roots = vi.hoisted(() => {+ const base = `${process.env.TMPDIR || '/tmp'}/history-roots-${process.pid}-${Date.now()}`+ process.env.CLAUDE_PROJECTS_DIR = `${base}/claude-projects`+ process.env.CODEX_HOME = `${base}/codex`+ return { base, claude: process.env.CLAUDE_PROJECTS_DIR, codex: `${process.env.CODEX_HOME}/sessions` }+})+afterAll(() => rmSync(roots.base, { recursive: true, force: true }))++/**+ * `session_get` and `sessions_list` for Claude Code and Codex, through the socket: what a client+ * scrolling a thread gets is what the whole-file read gave it, page for page (lib/transcriptPages.ts).+ */+describe.each([+ ['claude', claudeScenario, windowRawLines, (lines: string[]) => messagesToEvents(lines)],+ ['codex', codexScenario, windowCodexLines, (lines: string[]) => codexMessagesToEvents(lines)],+] as const)('%s history through the socket', (engine, scenario, window, toEvents) => {+ let dir: string+ let socket: BackendSocket+ let frames: Array<{ type: string; payload: Record<string, any> }>+ let agentId: string+ let sessionId: string+ let file: string+ let pane = 7400++ beforeEach(() => {+ const root = engine === 'claude' ? roots.claude : roots.codex+ mkdirSync(root, { recursive: true })+ dir = mkdtempSync(join(root, 'history-'))+ file = join(dir, 'transcript.jsonl')+ writeFileSync(file, scenario().join('\n') + '\n')+ socket = new BackendSocket('fixture')+ frames = []+ socket.registerLocalClient('local:history', { sendFrame: (frame) => { frames.push(frame as never); return true }, sendBinary: () => true })+ const paneId = `%${++pane}`+ agentId = registry.openPendingAgent({ engine, runtimes: [{ backend: 'tmux', paneId }], cwd: dir })!.agentId+ sessionId = `${engine}-history-${pane}`+ registry.register({ engine, sessionId, transcriptPath: file, tmuxPane: paneId, cwd: dir })+ })++ afterEach(async () => {+ registry.removeAgent(agentId)+ await socket.stop()+ rmSync(dir, { recursive: true, force: true })+ })++ let requests = 0+ async function ask(type: string, payload: Record<string, unknown>): Promise<Record<string, any>> {+ const requestId = `r${++requests}`+ await (socket as any).dispatchDown({ type, payload: { requestId, ...payload } }, 'local:history', 'local')+ return frames.find((frame) => frame.type === `${type}_result` && frame.payload.requestId === requestId)!.payload+ }++ it('is bound to the transcript under test', () => {+ expect(registry.resolve(sessionId)?.transcriptPath).toBe(file)+ })++ it('pages back through the thread exactly as the whole-file read did, cursor for cursor', async () => {+ const lines = await tailFile(file, Infinity)+ for (const limit of [1, 4, 500]) {+ let before: string | undefined+ for (let page = 0; page < lines.length + 1; page++) {+ const reply = await ask('session_get', { sessionId, limit, before })+ const old = window(lines, { limit, before })+ const events = toEvents(old.window)+ if (before && events[events.length - 1]?.type === 'done') events.pop()+ expect(reply.events, `limit ${limit}, page ${page}`).toEqual(JSON.parse(JSON.stringify(events)))+ expect({ hasMore: reply.hasMore, oldestCursor: reply.oldestCursor }).toEqual({ hasMore: old.hasMore, oldestCursor: old.oldestCursor })+ if (!old.hasMore || !old.oldestCursor) break+ before = old.oldestCursor+ }+ }+ })++ it('answers a request with no limit with the whole thread, in the reply it always had', async () => {+ const reply = await ask('session_get', { sessionId, before: 'ignored without a limit' })+ expect(reply.events).toEqual(JSON.parse(JSON.stringify(toEvents(await tailFile(file, Infinity)))))+ expect(reply).not.toHaveProperty('hasMore')+ expect(reply).not.toHaveProperty('oldestCursor')+ })++ it('says a cursor it cannot find is stale, so the client reloads', async () => {+ const reply = await ask('session_get', { sessionId, limit: 5, before: 'not-a-cursor-in-this-file' })+ expect(reply).toMatchObject({ events: [], hasMore: false, oldestCursor: null, staleCursor: true })+ })++ it('totals a sub-agent from its own transcript, read a line at a time', async () => {+ if (engine !== 'claude') return+ const launch = cl({ type: 'assistant', message: { id: 'mt', role: 'assistant', model: 'claude-opus-5-5', content: [{ type: 'tool_use', id: 'toolu_task', name: 'Task', input: { description: 'look around' } }], stop_reason: 'tool_use' } })+ const result = cl({+ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'toolu_task', content: 'launched' }] },+ toolUseResult: { isAsync: true, status: 'async_launched', agentId: 'agent7' },+ })+ writeFileSync(file, [claude.user('send a helper'), launch, result].join('\n') + '\n')+ const sub = [+ JSON.stringify({ type: 'user', timestamp: '2026-10-04T10:00:00.000Z', message: { role: 'user', content: 'go' } }),+ JSON.stringify({ type: 'assistant', timestamp: '2026-10-04T10:00:05.000Z', message: { content: [{ type: 'tool_use', id: 'a' }, { type: 'tool_use', id: 'b' }], usage: { input_tokens: 10, output_tokens: 5 } } }),+ '',+ JSON.stringify({ type: 'assistant', timestamp: '2026-10-04T10:01:00.000Z', message: { content: [{ type: 'tool_use', id: 'c' }], usage: { cache_read_input_tokens: 7 } } }),+ ]+ mkdirSync(join(dir, 'transcript', 'subagents'), { recursive: true })+ writeFileSync(join(dir, 'transcript', 'subagents', 'agent-agent7.jsonl'), sub.join('\r\n'))+ const reply = await ask('session_get', { sessionId, limit: 50 })+ const end = reply.events.find((event: { type: string; payload: { subagent?: unknown } }) => event.type === 'tool_end' && event.payload.subagent)+ expect(end.payload.subagent).toMatchObject({ agentId: 'agent7', ...subagentStatsFromRawLines(sub) })+ expect(end.payload.subagent).toMatchObject({ totalToolUseCount: 3, totalDurationMs: 60_000, totalTokens: 22 })+ // Totals the launch already reported stand; only what it left out is filled in.+ const reported = cl({+ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'toolu_task', content: 'launched' }] },+ toolUseResult: { agentId: 'agent7', totalDurationMs: 5, totalTokens: 9 },+ })+ writeFileSync(file, [claude.user('send a helper'), launch, reported].join('\n') + '\n')+ const again = await ask('session_get', { sessionId, limit: 50 })+ const kept = again.events.find((event: { type: string; payload: { subagent?: unknown } }) => event.type === 'tool_end' && event.payload.subagent)+ expect(kept.payload.subagent).toMatchObject({ agentId: 'agent7', totalToolUseCount: 3, totalDurationMs: 5, totalTokens: 9 })+ })++ it('answers for a transcript that is gone as for an empty one', async () => {+ rmSync(file)+ const reply = await ask('session_get', { sessionId, limit: 5 })+ // What the whole-file read made of the empty history it read there: no lines, then the end marker.+ expect(reply.events).toEqual(JSON.parse(JSON.stringify(toEvents([]))))+ expect(reply).toMatchObject({ hasMore: false, oldestCursor: window([], { limit: 5 }).oldestCursor })+ expect(Date.parse(reply.timestamp)).toBeGreaterThan(Date.now() - 60_000)+ })++ it('counts the thread without reading it whole', async () => {+ const reply = await ask('sessions_list', { agentId })+ expect(reply.sessions).toHaveLength(1)+ expect(reply.sessions[0]).toMatchObject({ id: sessionId, messageCount: (await tailFile(file, Infinity)).length })+ })++ it('counts nothing for an agent whose engine keeps no transcript file', async () => {+ const paneId = `%${++pane}`+ const store = registry.openPendingAgent({ engine: 'opencode', runtimes: [{ backend: 'tmux', paneId }], cwd: dir })!.agentId+ try {+ registry.register({ engine: 'opencode', sessionId: `ses_history${pane}`, tmuxPane: paneId, cwd: dir })+ const reply = await ask('sessions_list', { agentId: store })+ expect(reply.sessions).toEqual([expect.objectContaining({ id: `ses_history${pane}`, messageCount: 0 })])+ } finally { registry.removeAgent(store) }+ })+})diff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.tsindex e87b91e82..631d66d84 100644--- a/cli/src/backendSocket.ts+++ b/cli/src/backendSocket.ts@@ -24,7 +24,7 @@ import type { AutonomousDeviceService, AutonomousDeviceFrame } from './lib/autonimport { WebSocket } from 'ws'import { BACKEND_IDLE_DEADLINE_MS, watchSocketLiveness, type LivenessWatch } from './lib/wsLiveness.js'-import { stat, readFile, readdir } from 'fs/promises'+import { stat, readdir } from 'fs/promises'import { existsSync } from 'node:fs'import { dirname, isAbsolute, join } from 'path'import { hostname, homedir } from 'os'@@ -91,13 +91,15 @@ import { terminalHandoffRequest } from './lib/terminalHandoff.js'import { routeVoiceTask } from './lib/voiceRouter.js'import { tailFile } from './lib/sessions.js'import { stoppedAgents } from './lib/stoppedAgents.js'-import { messagesToEvents, windowRawLines, subagentStatsFromRawLines, type SessionEvent } from './lib/normalize.js'+import { messagesToEvents, windowRawLines, SubagentStats, type SessionEvent } from './lib/normalize.js'+import { TranscriptPager } from './lib/transcriptPages.js'+import { streamRecords } from './lib/transcriptTail.js'import { MediaPreviewError, readMediaPreviewChunk } from './lib/mediaPreview.js'import { ViewerForwarder } from './lib/viewerForwarder.js'import { InteractiveViewers } from './lib/interactiveViewer.js'import { OwnerCommands, OWNER_COMMAND_TYPES } from './lib/ownerCommands.js'import { VIEWER_DOWN_TYPES } from './lib/viewerWire.js'-import { codexMessagesToEvents, windowCodexLines } from './engines/codex/normalizer.js'+import { codexMessagesToEvents } from './engines/codex/normalizer.js'import { codexSubagentResolverFor } from './engines/codex/subagent.js'import { parseHostTheme, type HostTheme } from './lib/hostTheme.js'import { cursorMessagesToEvents, windowCursorLines } from './engines/cursor/normalizer.js'@@ -150,6 +152,8 @@ import {// OpenCode has no per-session transcript file — its history is read from this SQLite store.const OPENCODE_DB = join(env.OPENCODE_DATA_DIR, 'opencode.db')+/** History pages and line counts for the threads clients read (lib/transcriptPages.ts). */+const transcriptPages = new TranscriptPager()// Kilo keeps history the same way opencode does, in its own store.const KILO_DB = join(env.KILO_DATA_DIR, 'kilo.db')const DEVIN_DB = join(env.DEVIN_HOME, 'sessions.db')@@ -398,8 +402,11 @@ async function enrichSubagentStats(events: SessionEvent[], transcriptPath: strinconst sub = e.payload.subagentif (!sub?.agentId || typeof sub.totalToolUseCount === 'number') continuetry {- const txt = await readFile(join(subagentsDir, `agent-${sub.agentId}.jsonl`), 'utf8')- const stats = subagentStatsFromRawLines(txt.split('\n'))+ // A line at a time: a long sub-agent's transcript is never held whole to count its calls.+ const file = join(subagentsDir, `agent-${sub.agentId}.jsonl`)+ const totals = new SubagentStats()+ await streamRecords(file, 0, (await stat(file)).size, (line) => { totals.push(line) }, () => true)+ const stats = totals.result()sub.totalToolUseCount = stats.totalToolUseCountif (sub.totalDurationMs === undefined) sub.totalDurationMs = stats.totalDurationMsif (sub.totalTokens === undefined) sub.totalTokens = stats.totalTokens@@ -2157,13 +2164,14 @@ export class BackendSocket {// plainly beats inventing one: the web then shows the tab with an empty thread until the bind// lands, instead of pinning `currentSessionId` to an id no event will ever carry.if (!s || !s.sessionId) { reply(type, requestId, { sessions: [] }); return }- const lines = s.transcriptPath ? await tailFile(s.transcriptPath, Infinity) : []+ // Counted from an index kept as the file grows (lib/transcriptPages.ts), not by reading it whole.+ const messageCount = s.transcriptPath ? await transcriptPages.lineCount(s.transcriptPath) : 0reply(type, requestId, {sessions: [{id: s.sessionId,title: projectDisplayName(s),timestamp: new Date(s.registeredAt).toISOString(),- messageCount: lines.length,+ messageCount,lastActivity: new Date(await lastActivityAt(s)).toISOString(),participants: [],}],@@ -2284,14 +2292,42 @@ export class BackendSocket {const rawLimit = payload.limitconst limit = typeof rawLimit === 'number' && rawLimit > 0 ? Math.min(Math.floor(rawLimit), 500) : undefinedconst before = typeof payload.before === 'string' ? payload.before : undefined+ if (s.engine === 'claude' || s.engine === 'codex') {+ // Only the page asked for is read (lib/transcriptPages.ts): the whole history of a long session+ // is more memory than the daemon has. Without a limit, the newest lines that fit a page, and+ // the cursor to the rest when they do not all fit — a cursor the full-transcript reply never had.+ const opts = limit ? { limit, before } : {}+ const page = s.engine === 'codex'+ ? await transcriptPages.codex(s.transcriptPath, opts)+ : await transcriptPages.claude(s.transcriptPath, opts)+ const st = await stat(s.transcriptPath).catch(() => null)+ const timestamp = new Date(st?.mtimeMs ?? Date.now()).toISOString()+ if (page.staleCursor) {+ reply(type, requestId, { id: sessionId, title: projectDisplayName(s), events: [], timestamp, engine: s.engine, hasMore: false, oldestCursor: null, staleCursor: true })+ return+ }+ const events = s.engine === 'codex'+ ? codexMessagesToEvents(page.lines, codexSubagentResolverFor(s.codexHome))+ : messagesToEvents(page.lines)+ await enrichSubagentStats(events, s.transcriptPath)+ // Older pages must not inject a spurious end-of-transcript marker mid-scroll.+ if (limit && before && events[events.length - 1]?.type === 'done') events.pop()+ reply(type, requestId, {+ id: sessionId,+ title: projectDisplayName(s),+ events,+ timestamp,+ engine: s.engine,+ ...(limit || page.hasMore ? { hasMore: page.hasMore, oldestCursor: page.oldestCursor } : {}),+ })+ return+ }const lines = await tailFile(s.transcriptPath, Infinity)const st = await stat(s.transcriptPath).catch(() => null)const timestamp = new Date(st?.mtimeMs ?? Date.now()).toISOString()if (!limit) {- const fullEvents = s.engine === 'codex'- ? codexMessagesToEvents(lines, codexSubagentResolverFor(s.codexHome))- : s.engine === 'cursor'+ const fullEvents = s.engine === 'cursor'? cursorMessagesToEvents(lines, sessionId, await loadCursorReplayTaskLinks(cursorConfigDir(), sessionId, cursorDataDir())): s.engine === 'muse'? museMessagesToEvents(lines)@@ -2349,9 +2385,7 @@ export class BackendSocket {})return}- const w = s.engine === 'codex'- ? windowCodexLines(lines, { limit, before })- : s.engine === 'cursor'+ const w = s.engine === 'cursor'? windowCursorLines(lines, { limit, before }): s.engine === 'pi'? windowPiLines(lines, { limit, before })@@ -2362,9 +2396,7 @@ export class BackendSocket {reply(type, requestId, { id: sessionId, title: projectDisplayName(s), events: [], timestamp, engine: s.engine, hasMore: false, oldestCursor: null, staleCursor: true })return}- const events = s.engine === 'codex'- ? codexMessagesToEvents(w.window, codexSubagentResolverFor(s.codexHome))- : s.engine === 'cursor'+ const events = s.engine === 'cursor'? cursorMessagesToEvents(w.window,sessionId,diff --git a/cli/src/cli.ts b/cli/src/cli.tsindex cba739d21..329bf2804 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -170,7 +170,7 @@ import {type TerminalRuntimeRef,type TmuxRuntimeRef,} from './lib/terminalTypes.js'-import { Watcher, type HistoryEvent, type LineEvent, type TailHold } from './watcher/watcher.js'+import { Watcher, type HistoryEvent, type LineEvent, type RewrittenEvent, type TailHold } from './watcher/watcher.js'import { chooseHookAgent, startHookServer } from './hookServer.js'import { isLocalSocketName, localSocketPath, type LocalSocketServer } from './lib/localSocket.js'import { legacyDaemonStatus, localDaemonStatus, saveDaemonPort } from './lib/daemonEndpoint.js'@@ -4939,6 +4939,14 @@ async function runForeground(session: AuthSession | null): Promise<void> {// A transcript catch-up is history, including an unfinished last turn.// Its content still streams, but only fresh events or live inspection can// establish Working; replay cannot create a new completion notification.+ // A transcript rewritten in place with too much history to replay: its tail already starts at the new+ // end, and the session is attached again, from that end, so its turn state is rebuilt from the file.+ watcher.on('rewritten', (event: RewrittenEvent) => {+ const session = registry.bySession(event.sessionId)+ if (!session || session.transcriptPath !== event.transcriptPath) return+ console.log(`[watcher] ${sid(event.sessionId)} transcript rewritten in place — attaching it again from its end`)+ void attachSession(session, true).catch((err) => console.error(`[cli] re-attach after rewrite failed (session ${sid(event.sessionId)}):`, err instanceof Error ? err.message : err))+ })watcher.on('history', (batch: HistoryEvent) => {try {type Events = ReturnType<CursorNormalizer['ingest']>diff --git a/cli/src/engines/codex/normalizer.ts b/cli/src/engines/codex/normalizer.tsindex b659023c5..b37268835 100644--- a/cli/src/engines/codex/normalizer.ts+++ b/cli/src/engines/codex/normalizer.ts@@ -667,6 +667,19 @@ export function lastCodexTurnText(rawLines: string[]): LastTurnText | null {return text ? { userMessage, assistantText: text } : null}+/**+ * Whether a Codex history page may start at this line: a user's message, or the injected context a goal+ * turn starts on — so a turn is never split. Shared by `windowCodexLines` and the bounded pager+ * (lib/transcriptPages.ts), so the two cannot drift apart.+ */+export function codexPageStart(line: string): boolean {+ const raw = parse(line)+ const item = raw ? payload(raw) : null+ if (raw?.type === 'event_msg' && item && USER_TURN_TYPES.has(string(item.type))) return true+ // A goal turn starts on the injected goal context, not on a `user_message`.+ return raw?.type === 'response_item' && !!item && string(item.type) === 'message' && !!goalObjective(item)+}+export function windowCodexLines(rawLines: string[],opts: { limit: number; before?: string },@@ -682,14 +695,7 @@ export function windowCodexLines(}let start = Math.max(0, endIndex - opts.limit)- while (start > 0) {- const raw = parse(rawLines[start])- const item = raw ? payload(raw) : null- if (raw?.type === 'event_msg' && item && USER_TURN_TYPES.has(string(item.type))) break- // A goal turn starts on the injected goal context, not on a `user_message`.- if (raw?.type === 'response_item' && item && string(item.type) === 'message' && goalObjective(item)) break- start--- }+ while (start > 0 && !codexPageStart(rawLines[start])) start--return {window: rawLines.slice(start, endIndex),hasMore: start > 0,diff --git a/cli/src/lib/attachTranscript.spec.ts b/cli/src/lib/attachTranscript.spec.tsindex 6f543ac74..59bc849fd 100644--- a/cli/src/lib/attachTranscript.spec.ts+++ b/cli/src/lib/attachTranscript.spec.ts@@ -4,6 +4,7 @@ import { join } from 'node:path'import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import { CodexNormalizer, codexGoalOf, startsCodexTurn } from '../engines/codex/normalizer.js'import { canonical, fromTheEnd, goalForgotten, newFold, session, wholeHistory } from '../testing/transcriptOracle.js'+import { claude, claudeScenario, codex, codexScenario, cx } from '../testing/transcriptScenarios.js'import {attachTranscript,claudeAttachRules,@@ -19,117 +20,6 @@ import { RuntimeProfileManager } from './runtimeProfile.js'import { tailFile, tailFileUntil } from './transcriptTail.js'import { Watcher, type LineEvent } from '../watcher/watcher.js'-// ── Codex records, in the shapes a 0.159 rollout writes ──────────────────────────────────────────────-const cx = (type: string, payload: Record<string, unknown>): string =>- JSON.stringify({ timestamp: '2026-10-03T23:21:44.458Z', type, payload })-const codex = {- meta: (version = '0.159.0') => cx('session_meta', { id: 'thread-1', cli_version: version, cwd: '/tmp/work' }),- context: (model: string, effort: string | null, mode: string | null) => cx('turn_context', {- turn_id: 't', model, ...(effort ? { reasoning_effort: effort } : {}), ...(mode ? { collaboration_mode: { mode } } : {}),- }),- settings: (model: string, effort: string, mode: string) => cx('event_msg', {- type: 'thread_settings_applied', thread_settings: { model, reasoning_effort: effort, collaboration_mode: { mode } },- }),- started: (turn: string) => cx('event_msg', { type: 'task_started', turn_id: turn }),- user: (text: string) => cx('event_msg', { type: 'user_message', message: text }),- userItem: (text: string) => cx('event_msg', {- type: 'item_completed', turn_id: 't', item: { type: 'UserMessage', id: 'u', content: [{ type: 'text', text, text_elements: [] }] },- }),- goal: (objective: string) => cx('response_item', {- type: 'message', role: 'user',- content: [{ type: 'input_text', text: `<codex_internal_context source="goal">\n<objective>${objective}</objective>\n</codex_internal_context>` }],- }),- developer: (text: string) => cx('response_item', { type: 'message', role: 'developer', content: [{ type: 'input_text', text }] }),- agent: (text: string, phase = 'final_answer') => cx('event_msg', {- type: 'item_completed', item: { type: 'AgentMessage', content: [{ type: 'Text', text }], phase },- }),- reasoning: (text: string) => cx('response_item', { type: 'reasoning', summary: [{ type: 'summary_text', text }] }),- call: (id: string, cmd: string) => cx('response_item', { type: 'function_call', call_id: id, name: 'exec_command', arguments: JSON.stringify({ cmd }) }),- output: (id: string, output: string) => cx('response_item', { type: 'function_call_output', call_id: id, output }),- complete: (turn: string) => cx('event_msg', { type: 'task_complete', turn_id: turn, last_agent_message: 'done' }),- aborted: (turn: string) => cx('event_msg', { type: 'turn_aborted', turn_id: turn }),- compacted: (bytes: number) => cx('compacted', { message: '📘é'.repeat(bytes / 6), replacement_history: [] }),- contextCompacted: () => cx('event_msg', { type: 'context_compacted' }),- tokens: () => cx('event_msg', { type: 'token_count', info: { total_token_usage: { input_tokens: 1 } } }),-}--/** A rollout with every shape attach has to get right, in the order Codex writes them. */-const codexScenario = (): string[] => [- codex.meta(),- codex.developer('<permissions instructions>'),- codex.settings('gpt-5.5', 'high', 'default'),- codex.started('t1'), codex.context('gpt-5.5', 'high', 'default'), codex.user('first question'),- codex.reasoning('Planning'), codex.call('c1', 'ls'), codex.output('c1', 'a\nb'), codex.agent('one answer'), codex.complete('t1'),- codex.started('t2'), codex.context('gpt-6', 'xhigh', 'plan'), codex.userItem('second, in the item shape'),- codex.reasoning('Thinking 漢字 📘'), codex.agent('looking', 'commentary'),- codex.compacted(150_000), codex.contextCompacted(), codex.tokens(),- codex.call('c2', 'npm test'), codex.output('c2', 'ok'), codex.agent('second answer'), codex.complete('t2'),- codex.settings('gpt-6', 'low', 'default'),- codex.started('t3'), codex.context('gpt-6', null, 'default'), codex.user('third: effort left out'),- codex.call('c3', 'sleep 1'),- codex.user(''), // an empty message opens nothing- codex.output('c3', 'slept'), codex.complete('t3'),- codex.started('t4'), codex.context('gpt-6', 'medium', 'default'), codex.user('interrupted one'), codex.aborted('t4'),- codex.started('t5'), codex.context('gpt-6', 'medium', 'default'), codex.goal('ship the release'), codex.reasoning('goal work'), codex.complete('t5'),- codex.started('t6'), codex.context('gpt-6', 'medium', 'default'), codex.goal('ship the release'), codex.agent('continuing'), codex.complete('t6'),- codex.started('t7'), codex.context('gpt-6', 'medium', 'default'), codex.user('a question mid-goal'), codex.complete('t7'),- codex.started('t8'), codex.context('gpt-6', 'medium', 'default'), codex.goal('ship the release'), codex.reasoning('still the goal'),- codex.call('c8', 'make'), codex.complete('t8'),- codex.started('t9'), codex.context('gpt-6', 'max', 'plan'), codex.goal('a different goal'), codex.reasoning('new goal'),- codex.complete('t9'),- // A message sent mid-task, with a call still in flight: its output arrives after the new message.- codex.started('t10'), codex.context('gpt-6', 'max', 'default'), codex.user('start the build'),- codex.call('c10', 'make all'), codex.user('also run the tests while that builds'),- codex.output('c10', 'built'), codex.reasoning('now the tests'), codex.call('c11', 'npm test'),-]--// ── Claude Code records ───────────────────────────────────────────────────────────────────────────────-let uuid = 0-const cl = (record: Record<string, unknown>): string => JSON.stringify({- parentUuid: null, isSidechain: false, userType: 'external', cwd: '/tmp', sessionId: 's', version: '2.1.212',- timestamp: '2026-10-03T23:21:44.458Z', uuid: `u${++uuid}`, ...record,-})-const claude = {- user: (content: unknown, extra: Record<string, unknown> = {}) => cl({ type: 'user', message: { role: 'user', content }, ...extra }),- assistant: (content: unknown[], stop: string | null, model = 'claude-opus-5-5') =>- cl({ type: 'assistant', message: { id: `m${uuid}`, role: 'assistant', model, content, stop_reason: stop } }),- text: (text: string) => ({ type: 'text', text }),- thinking: (thinking: string) => ({ type: 'thinking', thinking }),- tool: (id: string, command: string) => ({ type: 'tool_use', id, name: 'Bash', input: { command } }),- result: (id: string, content: string) => cl({ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: id, content }] } }),- compact: () => cl({ type: 'system', subtype: 'compact_boundary', content: 'Conversation compacted', compactMetadata: { trigger: 'auto' } }),- summary: () => cl({ type: 'user', isCompactSummary: true, message: { role: 'user', content: 'Summary of the earlier conversation' } }),-}--const claudeScenario = (): string[] => [- JSON.stringify({ type: 'summary', summary: 'Earlier work', leafUuid: 'x' }),- claude.user('first prompt'),- claude.assistant([claude.thinking('let me think'), claude.text('First answer.')], 'end_turn'),- claude.user('second prompt with a tool'),- claude.assistant([claude.text('Checking.'), claude.tool('toolu_1', 'ls')], 'tool_use'),- claude.result('toolu_1', 'files'),- claude.assistant([claude.thinking('found them 漢字'), claude.text('Done: '.repeat(20_000))], 'end_turn', 'claude-fable-5-1'),- claude.user('<local-command-stdout>Set model to Opus 5.5</local-command-stdout>'),- claude.user('<bash-input>git status</bash-input><bash-stdout>clean</bash-stdout>'),- claude.user('third prompt, interrupted'),- claude.assistant([claude.tool('toolu_2', 'sleep 100')], 'tool_use'),- claude.user([{ type: 'text', text: '[Request interrupted by user for tool use]' }]),- claude.user('<task-notification><task-id>a</task-id><tool-use-id>toolu_9</tool-use-id><status>completed</status><summary>bg done</summary></task-notification>'),- claude.compact(), claude.summary(),- claude.user('loop iteration', { isMeta: true, promptSource: 'system' }),- claude.assistant([claude.text('Loop ran.')], 'end_turn', '<synthetic>'),- claude.user('a bookkeeping line', { isMeta: true }),- claude.user('fourth prompt'),- claude.assistant([claude.text('Working on it'), claude.tool('toolu_3', 'make')], 'tool_use'),- claude.result('toolu_3', 'built'),- claude.assistant([claude.thinking('almost'), claude.text('Built it.')], 'end_turn'),- claude.user('fifth prompt, left running'),- claude.assistant([claude.tool('toolu_4', 'npm test')], 'tool_use'),- // A prompt that lands while a call is in flight: the result arrives after it, naming its call.- claude.user('sixth prompt, typed during the tests'),- claude.result('toolu_4', 'tests passed'),- claude.assistant([claude.thinking('reading results'), claude.text('All green.')], 'end_turn'),-]describe('attachTranscript', () => {let dir: string, file: stringdiff --git a/cli/src/lib/normalize.ts b/cli/src/lib/normalize.tsindex 3664c6ca2..75c6b1ba9 100644--- a/cli/src/lib/normalize.ts+++ b/cli/src/lib/normalize.ts@@ -491,38 +491,50 @@ export function lastTurnTextFromRawLines(rawLines: string[]): LastTurnText | nul/** Aggregates computed from a sub-agent's OWN transcript. Async/background agents never get totals* in the launcher's toolUseResult (it only records `{isAsync, status:'async_launched', agentId}` at* launch) — the real numbers live in `<session>/subagents/agent-<id>.jsonl`. totalTokens mirrors the- * CLI's definition: input + output + cache_read + cache_creation summed over assistant turns. */-export function subagentStatsFromRawLines(rawLines: string[]): { totalToolUseCount: number; totalDurationMs?: number; totalTokens?: number } {- let toolCount = 0- let tokens = 0- let first: number | undefined- let last: number | undefined- for (const line of rawLines) {- if (!line.trim()) continue+ * CLI's definition: input + output + cache_read + cache_creation summed over assistant turns. Taken a+ * line at a time, so a long transcript is never held whole. */+export class SubagentStats {+ private toolCount = 0+ private tokens = 0+ private first: number | undefined+ private last: number | undefined++ push(line: string): void {+ if (!line.trim()) returnlet raw: Record<string, unknown>- try { raw = JSON.parse(line) as Record<string, unknown> } catch { continue }- if (!raw || typeof raw !== 'object') continue+ try { raw = JSON.parse(line) as Record<string, unknown> } catch { return }+ if (!raw || typeof raw !== 'object') returnconst ts = typeof raw.timestamp === 'string' ? Date.parse(raw.timestamp) : NaN- if (!Number.isNaN(ts)) { if (first === undefined) first = ts; last = ts }- if (raw.type !== 'assistant') continue+ if (!Number.isNaN(ts)) { if (this.first === undefined) this.first = ts; this.last = ts }+ if (raw.type !== 'assistant') returnconst msg = raw.message as { content?: unknown; usage?: Record<string, unknown> } | undefined- if (!msg) continue+ if (!msg) returnif (Array.isArray(msg.content)) {- for (const b of msg.content) if ((b as { type?: string }).type === 'tool_use') toolCount+++ for (const b of msg.content) if ((b as { type?: string }).type === 'tool_use') this.toolCount++}const u = msg.usageif (u) {- tokens += (Number(u.input_tokens) || 0) + (Number(u.output_tokens) || 0)+ this.tokens += (Number(u.input_tokens) || 0) + (Number(u.output_tokens) || 0)+ (Number(u.cache_read_input_tokens) || 0) + (Number(u.cache_creation_input_tokens) || 0)}}- return {- totalToolUseCount: toolCount,- totalDurationMs: first !== undefined && last !== undefined && last > first ? last - first : undefined,- totalTokens: tokens || undefined,++ result(): { totalToolUseCount: number; totalDurationMs?: number; totalTokens?: number } {+ const { first, last } = this+ return {+ totalToolUseCount: this.toolCount,+ totalDurationMs: first !== undefined && last !== undefined && last > first ? last - first : undefined,+ totalTokens: this.tokens || undefined,+ }}}+export function subagentStatsFromRawLines(rawLines: string[]): { totalToolUseCount: number; totalDurationMs?: number; totalTokens?: number } {+ const stats = new SubagentStats()+ for (const line of rawLines) stats.push(line)+ return stats.result()+}+/** Emit tool_end events for a user message's tool_result blocks. */function toolResultEvents(msg: NormalizedMessage, toolIdToName: Map<string, string>): SessionEvent[] {const events: SessionEvent[] = []@@ -713,19 +725,28 @@ export function messagesToEvents(rawLines: string[]): SessionEvent[] {* would drop tool names. A user-prompt line never sits between a tool_use and its tool_result, so* snapping there keeps every turn whole.*/+/**+ * What a Claude history page knows of one line: the cursor that names it, and whether a page may start+ * there — a real user prompt, so a turn is never split. Shared by `windowRawLines` and the bounded pager+ * (lib/transcriptPages.ts), so the two cannot drift apart.+ */+export function claudePageLine(line: string): { cursor: string | null; startsPage: boolean } {+ if (!line.trim()) return { cursor: null, startsPage: false }+ let raw: Record<string, unknown>+ try { raw = JSON.parse(line) as Record<string, unknown> } catch { return { cursor: null, startsPage: false } }+ const cursor = (raw.uuid ?? raw.id ?? raw.message_id ?? null) as string | null+ const msg = transformLine(raw)+ return { cursor, startsPage: msg ? realUserText(msg) !== null : false }+}+export function windowRawLines(rawLines: string[],opts: { limit: number; before?: string },): { window: string[]; hasMore: boolean; oldestCursor: string | null; staleCursor?: boolean } {// Per-line uuid + whether the line is a real user-prompt turn start (parse each line once).const meta = rawLines.map((line) => {- if (!line.trim()) return { uuid: null as string | null, turnStart: false }- let raw: Record<string, unknown>- try { raw = JSON.parse(line) as Record<string, unknown> } catch { return { uuid: null as string | null, turnStart: false } }- const uuid = (raw.uuid ?? raw.id ?? raw.message_id ?? null) as string | null- const msg = transformLine(raw)- const turnStart = msg ? realUserText(msg) !== null : false- return { uuid, turnStart }+ const { cursor, startsPage } = claudePageLine(line)+ return { uuid: cursor, turnStart: startsPage }})let endIndex = rawLines.lengthdiff --git a/cli/src/lib/transcriptPages.spec.ts b/cli/src/lib/transcriptPages.spec.tsnew file mode 100644index 000000000..28b9f001a--- /dev/null+++ b/cli/src/lib/transcriptPages.spec.ts@@ -0,0 +1,414 @@+import { appendFileSync, chmodSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs'+import { open } from 'node:fs/promises'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { afterEach, beforeEach, describe, expect, it } from 'vitest'+import { codexPageStart, windowCodexLines } from '../engines/codex/normalizer.js'+import { claude, claudeScenario, codex, codexScenario } from '../testing/transcriptScenarios.js'+import { claudePageLine, windowRawLines } from './normalize.js'+import { tailFile } from './transcriptTail.js'+import { LineIndex, pageBefore, TranscriptPager, type HistoryPage, type OpenFile } from './transcriptPages.js'++let dir: string+let files = 0+const fresh = (content: string | Buffer): string => {+ const file = join(dir, `t${++files}.jsonl`)+ writeFileSync(file, content)+ return file+}+beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'transcript-pages-')) })+afterEach(() => rmSync(dir, { recursive: true, force: true }))++type Engine = 'claude' | 'codex'+const window = (engine: Engine, lines: string[], opts: { limit: number; before?: string }) =>+ engine === 'claude' ? windowRawLines(lines, opts) : windowCodexLines(lines, opts)+const asOld = (page: HistoryPage) => ({ window: page.lines, hasMore: page.hasMore, oldestCursor: page.oldestCursor, staleCursor: page.staleCursor })++/** Page back through `file` from its end, both ways, until the start; every page and cursor must match. */+async function pageThrough(engine: Engine, file: string, limit: number): Promise<number> {+ const lines = await tailFile(file, Infinity)+ const pager = new TranscriptPager()+ let before: string | undefined+ for (let pages = 1; ; pages++) {+ const old = window(engine, lines, { limit, before })+ const now = await pager[engine](file, { limit, before })+ expect(asOld(now), `${engine} page ${pages}, limit ${limit}, before ${before}`).toEqual(old)+ expect(now.clipped).toBe(false)+ if (!old.hasMore || old.staleCursor || old.oldestCursor === null) return pages+ before = old.oldestCursor+ }+}++describe('history pages equal the windows cut from the whole file', () => {+ describe.each([['claude', claudeScenario], ['codex', codexScenario]] as const)('%s', (engine, scenario) => {+ const records = scenario()+ const cuts = records.map((_, index) => index + 1)++ it.each(cuts)('after record %i, with every ending, a half-written record or none, and every page size', async (cut) => {+ for (const ending of ['\n', '\r\n']) {+ const body = records.slice(0, cut).join(ending) + ending+ const variants = [body]+ if (cut < records.length) variants.push(body + records[cut].slice(0, Math.floor(records[cut].length / 2)))+ for (const content of variants) {+ const file = fresh(content)+ expect(await new TranscriptPager().lineCount(file)).toBe((await tailFile(file, Infinity)).length)+ for (const limit of [1, 3, 7, 500]) await pageThrough(engine, file, limit)+ }+ }+ })++ it('reads lines ended by CR alone, and passes over blank ones', async () => {+ const file = fresh(['', ...records.slice(0, 20), ' ', '\t'].join('\r') + '\r\r\n\n')+ for (const limit of [1, 4, 500]) await pageThrough(engine, file, limit)+ })++ it('says a cursor it cannot find is stale, as the old window did', async () => {+ const file = fresh(records.join('\n') + '\n')+ const lines = await tailFile(file, Infinity)+ const pager = new TranscriptPager()+ for (const before of ['nothing-like-it', 'codex:x', `codex:${lines.length + 1}`, 'codex:99999999999999999999']) {+ expect(asOld(await pager[engine](file, { limit: 5, before }))).toEqual(window(engine, lines, { limit: 5, before }))+ }+ })++ it('gives every line, oldest first, when asked for no page at all', async () => {+ const file = fresh(records.join('\n') + '\n')+ const page = await new TranscriptPager()[engine](file, {})+ expect(page.lines).toEqual(await tailFile(file, Infinity))+ expect(page.hasMore).toBe(false)+ })+ })++ it('reads a transcript that is missing, or cannot be read, as an empty one', async () => {+ const pager = new TranscriptPager()+ const missing = join(dir, 'gone.jsonl')+ for (const before of [undefined, 'u1']) {+ expect(asOld(await pager.claude(missing, { limit: 5, before }))).toEqual(windowRawLines([], { limit: 5, before }))+ }+ for (const before of [undefined, 'codex:0', 'codex:1']) {+ expect(asOld(await pager.codex(missing, { limit: 5, before }))).toEqual(windowCodexLines([], { limit: 5, before }))+ }+ expect(await pager.lineCount(missing)).toBe(0)+ const locked = fresh(claudeScenario().join('\n') + '\n')+ chmodSync(locked, 0o000)+ try {+ expect(asOld(await pager.claude(locked, { limit: 5 }))).toEqual(windowRawLines([], { limit: 5 }))+ expect((await pager.claude(locked, { limit: 5, before: 'u1' })).staleCursor).toBe(true)+ expect(await pageBefore(locked, 100, 5, { startsPage: () => true })).toEqual({ lines: [], count: 0, start: 0, hasMore: false, clipped: false })+ } finally { chmodSync(locked, 0o600) }+ })++ it('names the line a cursor stood on when no page is left before it', async () => {+ const records = claudeScenario()+ const file = fresh(records.join('\n') + '\n')+ const first = claudePageLine(records[1]).cursor!+ const lines = await tailFile(file, Infinity)+ const pager = new TranscriptPager()+ // Paging back from the second line leaves only the first, which has no id of its own.+ expect(asOld(await pager.claude(file, { limit: 5, before: first }))).toEqual(windowRawLines(lines, { limit: 5, before: first }))+ const top = claudePageLine(records[0]).cursor+ expect(top).toBeNull()+ const withId = fresh(records.slice(1).join('\n') + '\n')+ const page = await pager.claude(withId, { limit: 5, before: first })+ expect(asOld(page)).toEqual(windowRawLines(await tailFile(withId, Infinity), { limit: 5, before: first }))+ expect(page.oldestCursor).toBe(first)+ })+})++describe('a Claude cursor', () => {+ it('names the newest line with its id, where the old window took the oldest', async () => {+ const twice = claude.user('a prompt written twice')+ const records = [claude.user('one'), twice, claude.user('two'), twice, claude.user('three')]+ const file = fresh(records.join('\n') + '\n')+ const before = claudePageLine(twice).cursor!+ const page = await new TranscriptPager().claude(file, { limit: 1, before })+ expect(page.lines).toEqual([records[2]])+ expect(windowRawLines(await tailFile(file, Infinity), { limit: 1, before }).window).toEqual([records[0]])+ })++ it('is found again from where it was served, and by walking the file when that no longer names it', async () => {+ const records = claudeScenario()+ const file = fresh(records.join('\n') + '\n')+ const pager = new TranscriptPager()+ const first = await pager.claude(file, { limit: 3 })+ const again = await pager.claude(file, { limit: 3, before: first.oldestCursor! })+ expect(again).toEqual(await new TranscriptPager().claude(file, { limit: 3, before: first.oldestCursor! }))+ // Rewritten so the byte remembered for that cursor now starts another line.+ writeFileSync(file, [claude.user('a new first line, longer than before'), ...records].join('\n') + '\n')+ expect(await pager.claude(file, { limit: 3, before: first.oldestCursor! }))+ .toEqual(await new TranscriptPager().claude(file, { limit: 3, before: first.oldestCursor! }))+ // And when the file shrank past it.+ writeFileSync(file, records.slice(0, 2).join('\n') + '\n')+ expect((await pager.claude(file, { limit: 3, before: first.oldestCursor! })).staleCursor).toBe(true)+ })++ it('is matched as a whole id, and an id with characters JSON may escape is read the slow way', async () => {+ const odd = 'id with "quotes" and /slashes/'+ const records = [claude.user('one'), JSON.stringify({ type: 'user', id: odd, message: { role: 'user', content: 'two' } }), claude.user('three')]+ const file = fresh(records.join('\n') + '\n')+ const lines = await tailFile(file, Infinity)+ const pager = new TranscriptPager()+ expect(asOld(await pager.claude(file, { limit: 1, before: odd }))).toEqual(windowRawLines(lines, { limit: 1, before: odd }))+ const prefix = claudePageLine(records[2]).cursor!.slice(0, -1)+ expect((await pager.claude(file, { limit: 1, before: prefix })).staleCursor).toBe(true)+ })++ it('remembers a bounded number of cursors and files', async () => {+ const pager = new TranscriptPager({ capacity: 2 })+ const records = Array.from({ length: 80 }, (_, i) => claude.user(`prompt ${i}`))+ const file = fresh(records.join('\n') + '\n')+ let before: string | undefined+ for (let i = 0; i < 70; i++) before = (await pager.claude(file, { limit: 1, before })).oldestCursor!+ for (let i = 0; i < 3; i++) await pager.claude(fresh(records.join('\n') + '\n'), { limit: 1 })+ expect(await pager.claude(file, { limit: 1, before })).toEqual(await new TranscriptPager().claude(file, { limit: 1, before }))+ })+})++describe('a page held to its size', () => {+ const rules = { startsPage: (line: string) => line.startsWith('{"start"') }+ const start = (n: number, pad = 0) => JSON.stringify({ start: n, pad: 'x'.repeat(pad) })+ const step = (n: number, pad = 0) => JSON.stringify({ step: n, pad: 'x'.repeat(pad) })++ it('ends at the oldest turn that fits, short of its limit, so no turn is split', async () => {+ const records = [start(1), step(1), start(2, 100), step(2, 100), start(3), step(3)]+ const file = fresh(records.join('\n') + '\n')+ const end = Buffer.byteLength(records.join('\n') + '\n')+ // 39 bytes reach back to turn 3, 278 to turn 2: a page of 200 holds turn 3 alone.+ expect(await pageBefore(file, end, 6, rules, 200)).toMatchObject({ lines: records.slice(4), count: 2, hasMore: true, clipped: false })+ expect(await pageBefore(file, end, 6, rules, 300)).toMatchObject({ lines: records.slice(2), count: 4, hasMore: true, clipped: false })+ })++ it('cuts a turn that cannot fit inside it, and says so', async () => {+ const records = [start(1), step(1, 100), step(2, 100), step(3, 100)]+ const file = fresh(records.join('\n') + '\n')+ const page = (await pageBefore(file, Buffer.byteLength(records.join('\n') + '\n'), 2, rules, 250))!+ expect(page).toMatchObject({ lines: records.slice(2), hasMore: true, clipped: true })+ })++ it('counts a line too long to hold where it stands, and leaves it out', async () => {+ const records = [start(1), step(1), step(2, 400), step(3), start(2), step(4)]+ const file = fresh(records.join('\n') + '\n')+ const end = Buffer.byteLength(records.join('\n') + '\n')+ expect(await pageBefore(file, end, 2, rules, 300)).toMatchObject({ lines: records.slice(4), count: 2, hasMore: true })+ // Turn 1 holds the long line: it counts, and the page holds the rest of the turn around it.+ expect(await pageBefore(file, end, 3, rules, 300))+ .toMatchObject({ lines: [records[0], records[1], records[3], records[4], records[5]], count: 6, hasMore: false })+ // Asked for the page right after one ended on a long line: the line still counts toward it.+ const fromLong = (await pageBefore(file, Buffer.byteLength(records.slice(0, 3).join('\n') + '\n'), 1, rules, 300))!+ expect(fromLong).toMatchObject({ lines: [records[0], records[1]], count: 3, hasMore: false })+ const afterReach = (await pageBefore(file, Buffer.byteLength(records.slice(0, 4).join('\n') + '\n'), 1, rules, 300))!+ expect(afterReach).toMatchObject({ lines: [records[0], records[1], records[3]], count: 4 })+ // A long line just before a page that is already complete is the next page's.+ const complete = (await pageBefore(file, Buffer.byteLength(records.slice(0, 4).join('\n') + '\n'), 1, { startsPage: () => true }, 300))!+ expect(complete).toMatchObject({ lines: [records[3]], count: 1, hasMore: true })+ })++ it('pages a Codex rollout through lines too long to hold, every line in exactly one page', async () => {+ const records = codexScenario()+ const file = fresh(records.join('\n') + '\n')+ const pager = new TranscriptPager({ maxBytes: 20_000 })+ const seen: string[] = []+ let before: string | undefined+ let endIndex = records.length+ for (;;) {+ const page = await pager.codex(file, { limit: 4, before })+ seen.unshift(...page.lines)+ const start = Number(page.oldestCursor!.slice('codex:'.length))+ expect(start).toBeLessThan(endIndex)+ endIndex = start+ if (!page.hasMore) break+ before = page.oldestCursor!+ }+ expect(endIndex).toBe(0)+ expect(seen).toEqual(records.filter((record) => Buffer.byteLength(record) <= 20_000))+ })++ it('is no page at all when the file shrinks under the walk', async () => {+ const records = codexScenario()+ const file = fresh(records.join('\n') + '\n')+ const end = Buffer.byteLength(records.join('\n') + '\n')+ let cut = false+ const shrinking = { startsPage: (line: string) => { if (!cut) { cut = true; truncateSync(file, 10) } return codexPageStart(line) } }+ expect(await pageBefore(file, end, 500, shrinking)).toBeNull()+ })+})++describe('LineIndex', () => {+ const tricky = [+ '{"a":1}', '', ' ', '\t', ' ', ' ', '', '\u0001', 'é', '{"b":2}',+ Buffer.from([0xff, 0xfe]).toString('latin1'),+ ]++ it('counts what the whole-file reader counts, however lines end and whatever whitespace they hold', async () => {+ for (const ending of ['\n', '\r\n', '\r']) {+ for (let cut = 0; cut <= tricky.length; cut++) {+ for (const last of ['', ending]) {+ const content = tricky.slice(0, cut).join(ending) + (cut ? last : '')+ const file = fresh(content)+ const index = new LineIndex(file)+ await index.update()+ expect(index.count(), JSON.stringify({ ending, cut, last })).toBe((await tailFile(file, Infinity)).length)+ }+ }+ }+ })++ it('reads invalid UTF-8 as a line, as the whole-file reader does', async () => {+ const file = fresh(Buffer.concat([Buffer.from('{"a":1}\n'), Buffer.from([0xc3]), Buffer.from('\n'), Buffer.from([0xe2, 0x80]), Buffer.from('\n')]))+ const index = new LineIndex(file)+ await index.update()+ expect(index.count()).toBe((await tailFile(file, Infinity)).length)+ })++ it('keeps up as the file grows a byte at a time, and finds where any line ends', async () => {+ const content = Buffer.from([...codexScenario().slice(0, 12), '', ' ', claude.user('漢字 📘')].join('\r\n') + '\n')+ const file = fresh('')+ const index = new LineIndex(file, { lines: 2, bytes: 4096 })+ // Byte by byte through the first records, a character split across looks included; then in strides.+ for (let at = 0; at < content.length;) {+ const next = at < 600 ? at + 1 : Math.min(content.length, at + 997)+ appendFileSync(file, content.subarray(at, next))+ at = next+ await index.update()+ expect(index.count(), `after ${at} bytes`).toBe((await tailFile(file, Infinity)).length)+ }+ const lines = await tailFile(file, Infinity)+ expect(index.count()).toBe(lines.length)+ expect(index.length()).toBe(content.length)+ for (let n = 0; n <= lines.length + 1; n++) {+ const end = await index.endOf(n)+ const prefix = fresh(content.subarray(0, end))+ expect((await tailFile(prefix, Infinity)).length, `line ${n}`).toBe(Math.min(n, lines.length))+ }+ })++ it('marks lines by bytes too, so a long stretch of long lines is never read whole to find one', async () => {+ const content = Buffer.from(Array.from({ length: 40 }, (_, i) => codex.reasoning(`${i} ${'x'.repeat(3000)}`)).join('\n') + '\n')+ const file = fresh(content)+ const index = new LineIndex(file, { lines: 1000, bytes: 10_000 })+ await index.update()+ for (const n of [0, 1, 3, 4, 17, 39, 40]) {+ expect((await tailFile(fresh(content.subarray(0, await index.endOf(n))), Infinity)).length, `line ${n}`).toBe(n)+ }+ })++ it('indexes a file again from the start when it shrank, its first bytes changed, or an ending moved', async () => {+ const records = codexScenario()+ const file = fresh(records.join('\n') + '\n')+ const index = new LineIndex(file)+ await index.update()+ expect(index.count()).toBe(records.length)+ writeFileSync(file, records.slice(0, 5).join('\n') + '\n')+ await index.update()+ expect(index.count()).toBe(5)+ // Longer than before, with an ending where one was: only its first bytes tell it is not the file indexed.+ writeFileSync(file, [codex.meta('99.159.0'), ...records.slice(1, 5)].join('\n') + '\n')+ await index.update()+ expect(index.count()).toBe(5)+ const longer = [codex.reasoning('x'.repeat(30)), ...records.slice(1, 5)].join('\n') + '\n'+ expect(Buffer.byteLength(longer)).toBeGreaterThan(Buffer.byteLength([codex.meta('99.159.0'), ...records.slice(1, 5)].join('\n') + '\n'))+ writeFileSync(file, longer)+ await index.update()+ expect(index.count()).toBe(5)+ writeFileSync(file, records.slice(0, 5).join(' ') + '\n' + records.slice(5, 9).join('\n') + '\n')+ await index.update()+ expect(index.count()).toBe((await tailFile(file, Infinity)).length)+ })++ it('reads a file that is gone as empty, and one that appears later from its start', async () => {+ const file = join(dir, 'later.jsonl')+ const index = new LineIndex(file)+ await index.update()+ expect([index.count(), index.length()]).toEqual([0, 0])+ writeFileSync(file, '{"a":1}\n{"b":2}')+ await index.update()+ expect(index.count()).toBe(2)+ expect(await index.endOf(1)).toBe(8)+ rmSync(file)+ await index.update()+ expect(index.count()).toBe(0)+ })++ it('looks and finds one at a time, in the order asked', async () => {+ const records = codexScenario()+ const file = fresh(records.slice(0, 10).join('\n') + '\n')+ const index = new LineIndex(file, { lines: 3, bytes: 1 << 30 })+ await index.update()+ appendFileSync(file, records.slice(10).join('\n') + '\n')+ const [, end] = await Promise.all([index.update(), index.endOf(records.length - 1)])+ expect(end).toBe(Buffer.byteLength(records.slice(0, -1).join('\n') + '\n'))+ // A look that fails leaves the queue running.+ rmSync(file)+ await expect(index.update()).resolves.toBeUndefined()+ await expect(index.endOf(0)).resolves.toBe(0)+ })++ it('stops where a file cut short under it ends, whatever its length said', async () => {+ for (const content of ['{"a":1}\n{"b"', '{"a":1}\n']) {+ const file = fresh(content)+ const shorted: OpenFile = async (path) => {+ const handle = await open(path, 'r')+ return Object.assign(Object.create(handle), {+ stat: async () => ({ ...(await handle.stat()), size: Buffer.byteLength(content) + 50 }),+ read: handle.read.bind(handle),+ close: handle.close.bind(handle),+ })+ }+ const index = new LineIndex(file, undefined, shorted)+ await index.update()+ expect(index.count()).toBe((await tailFile(file, Infinity)).length)+ }+ })++ it('reports a file it cannot read, and keeps answering after', async () => {+ const index = new LineIndex(dir)+ await expect(index.update()).rejects.toThrow()+ const locked = fresh('{"a":1}\n')+ chmodSync(locked, 0o000)+ try { await expect(new LineIndex(locked).update()).rejects.toThrow() } finally { chmodSync(locked, 0o600) }+ const records = codexScenario()+ const file = fresh(records.join('\n') + '\n')+ const gone = new LineIndex(file, { lines: 4, bytes: 1 << 30 })+ await gone.update()+ rmSync(file)+ await expect(gone.endOf(6)).rejects.toThrow()+ await expect(gone.update()).resolves.toBeUndefined()+ expect(gone.count()).toBe(0)+ const pager = new TranscriptPager()+ expect(await pager.lineCount(dir)).toBe(0)+ expect(asOld(await pager.codex(dir, { limit: 2 }))).toEqual(windowCodexLines([], { limit: 2 }))+ expect(asOld(await pager.codex(dir, { limit: 2, before: 'codex:3' }))).toEqual(windowCodexLines([], { limit: 2, before: 'codex:3' }))+ expect(asOld(await pager.codex(dir, {}))).toEqual(windowCodexLines([], { limit: Infinity }))+ })+})++describe('the pager', () => {+ it('says a page is stale when the file shrank under its walk', async () => {+ const pager = new TranscriptPager({ walk: async () => false })+ expect((await pager.claude(fresh(claudeScenario().join('\n') + '\n'), { limit: 2 })).staleCursor).toBe(true)+ expect((await pager.codex(fresh(codexScenario().join('\n') + '\n'), { limit: 2 })).staleCursor).toBe(true)+ })++ it('keeps a bounded number of indexes', async () => {+ const pager = new TranscriptPager({ capacity: 2 })+ const records = codexScenario()+ const files = [1, 2, 3].map(() => fresh(records.join('\n') + '\n'))+ for (const file of [...files, files[0]]) expect(await pager.lineCount(file)).toBe(records.length)+ })++ it('finds a remembered cursor on a last line not yet ended', async () => {+ const records = [claude.user('one'), claude.assistant([claude.text('a')], 'end_turn'), claude.user('two, still being written')]+ const file = fresh(records.join('\n'))+ const pager = new TranscriptPager()+ const first = await pager.claude(file, { limit: 1 })+ expect(first.lines).toEqual([records[2]])+ appendFileSync(file, '\n' + claude.user('three'))+ const lines = await tailFile(file, Infinity)+ expect(asOld(await pager.claude(file, { limit: 1, before: first.oldestCursor! }))).toEqual(windowRawLines(lines, { limit: 1, before: first.oldestCursor! }))+ const unended = fresh(records.join('\n'))+ const page = await pager.claude(unended, { limit: 1 })+ expect(asOld(await pager.claude(unended, { limit: 1, before: page.oldestCursor! })))+ .toEqual(windowRawLines(await tailFile(unended, Infinity), { limit: 1, before: page.oldestCursor! }))+ })+})+diff --git a/cli/src/lib/transcriptPages.ts b/cli/src/lib/transcriptPages.tsnew file mode 100644index 000000000..16757a6dd--- /dev/null+++ b/cli/src/lib/transcriptPages.ts@@ -0,0 +1,443 @@+/**+ * A transcript's history a page at a time — what a client scrolling a thread asks for (`session_get`)+ * and how long the thread is (`sessions_list`) — without reading the whole file. Attach's rule+ * (lib/attachTranscript.ts) holds here too: the work a request costs grows with the page, never with the+ * history. Both requests used to read every line into memory first, so a phone opening the thread of a+ * long Codex session asked the daemon for as much memory as the crash of 2026-10-03 took.+ *+ * Lines are `tailFile(path, Infinity)`'s, the reader every page came from: LF, CR and CRLF end a line,+ * a line that is only whitespace is no line, and a last line with no ending yet is one. Pages are the+ * windows `windowRawLines` and `windowCodexLines` cut from those lines, with the same cursors, except:+ * - a page holds at most `PAGE_MAX_BYTES` (see `pageBefore`), and a line longer than that is counted+ * but left out — the old window held whatever its turn did, a whole file at worst;+ * - a Claude cursor names the newest line with that id, where the old window took the oldest. Ids are+ * unique in practice, and finding the oldest would mean reading back to the start every time.+ */+import { open, stat } from 'node:fs/promises'+import { StringDecoder } from 'node:string_decoder'+import { codexPageStart, windowCodexLines } from '../engines/codex/normalizer.js'+import { claudePageLine } from './normalize.js'+import { scanRecordsBackward, streamRecords } from './transcriptTail.js'++/** The most a page holds. A page used to be however long its turn was, and a turn can be gigabytes. */+export const PAGE_MAX_BYTES = 32 * 1024 * 1024++/** A line index notes where a line ends at least every this many lines and this many bytes, so finding+ * any line reads no more than that. */+export const INDEX_STRIDE_LINES = 4096+export const INDEX_STRIDE_BYTES = 4 * 1024 * 1024++const CHUNK_BYTES = 1024 * 1024+const HEAD_BYTES = 64+const NOT_WHITESPACE = /\S/+const EMPTY = Buffer.alloc(0)++type Handle = Awaited<ReturnType<typeof open>>+/** How a file is opened for reading — swapped in tests for one that is cut short as it is read. */+export type OpenFile = (filePath: string) => Promise<Handle>+const openForReading: OpenFile = (filePath) => open(filePath, 'r')+/** How a page is walked — swapped in tests for a walk the file shrinks under. */+export type Walk = typeof scanRecordsBackward++/** `isBlank`'s rule (lib/transcriptTail.ts) — `line.trim()` is empty — fed a line a chunk at a time. */+class BlankTest {+ private found = false+ private decoder: StringDecoder | null = null++ get seen(): boolean { return this.found }++ feed(bytes: Buffer): void {+ if (this.found || !bytes.length) return+ for (const byte of bytes) if (byte > 0x20 && byte < 0x7f) { this.found = true; this.decoder = null; return }+ this.decoder ??= new StringDecoder('utf8')+ if (NOT_WHITESPACE.test(this.decoder.write(bytes))) { this.found = true; this.decoder = null }+ }++ /** Whether the line held anything but whitespace. Resets for the next line. */+ end(): boolean {+ const line = this.found || (this.decoder !== null && NOT_WHITESPACE.test(this.decoder.end()))+ this.found = false+ this.decoder = null+ return line+ }+}++async function readAt(handle: Handle, position: number, length: number): Promise<Buffer> {+ const buffer = Buffer.alloc(Math.max(0, length))+ let read = 0+ while (read < buffer.length) {+ const { bytesRead } = await handle.read(buffer, read, buffer.length - read, position + read)+ if (!bytesRead) break+ read += bytesRead+ }+ return buffer.subarray(0, read)+}++/**+ * Walk `[from, to)` — `from` just past a line ending, or 0 — calling `onLine` with the byte just past+ * each line's ending and `onEnding` past every ending, a blank line's included. Returning true from+ * `onLine` stops the walk. What follows the last ending is left unread.+ */+async function walkLines(+ handle: Handle,+ from: number,+ to: number,+ onLine: (lineEnd: number) => boolean,+ onEnding: (end: number) => void = () => {},+): Promise<void> {+ const test = new BlankTest()+ for (let position = from; position < to;) {+ const chunk = await readAt(handle, position, Math.min(CHUNK_BYTES, to - position))+ if (!chunk.length) return+ let lineStart = 0+ for (let i = 0; i < chunk.length; i++) {+ if (chunk[i] !== 10 && chunk[i] !== 13) continue+ test.feed(chunk.subarray(lineStart, i))+ lineStart = i + 1+ const end = position + i + 1+ if (test.end() && onLine(end)) return+ onEnding(end)+ }+ if (lineStart < chunk.length) test.feed(chunk.subarray(lineStart))+ position += chunk.length+ }+}++/**+ * How many lines a transcript holds and where any of them ends, kept up to date as the file grows:+ * each look reads only what was appended since the last. A file rewritten in place — it shrank, its+ * first bytes changed, or a line ending is no longer where one was — is indexed again from the start.+ * Looks and lookups queue, so neither sees the other half done.+ */+export class LineIndex {+ /** Indexed up to here: just past a line ending, or 0. */+ private bytes = 0+ /** Lines that end before `bytes`. */+ private lines = 0+ /** Bytes with a known number of lines before them, ascending: `marks[k]` lines before `at[k]`. */+ private marks: number[] = [0]+ private at: number[] = [0]+ private head: Buffer = EMPTY+ /** The bytes after the last line ending are a line still being written. */+ private trailing = false+ private size = 0+ private queue: Promise<unknown> = Promise.resolve()++ constructor(+ readonly filePath: string,+ private readonly stride: { lines: number; bytes: number } = { lines: INDEX_STRIDE_LINES, bytes: INDEX_STRIDE_BYTES },+ private readonly openFile: OpenFile = openForReading,+ ) {}++ /** Bring the index up to the file as it is now. */+ update(): Promise<void> {+ return this.serial(() => this.look())+ }++ /** The file's lines, one still being written included: `tailFile(path, Infinity).length`. */+ count(): number {+ return this.lines + (this.trailing ? 1 : 0)+ }++ /** The file's length at the last look. */+ length(): number {+ return this.size+ }++ /** A byte with exactly `n` lines before it — where a page that ends before line `n` ends. */+ endOf(n: number): Promise<number> {+ return this.serial(() => this.find(n))+ }++ private serial<T>(task: () => Promise<T>): Promise<T> {+ const run = this.queue.then(task)+ this.queue = run.catch(() => {})+ return run+ }++ private reset(): void {+ this.bytes = 0+ this.lines = 0+ this.marks = [0]+ this.at = [0]+ this.head = EMPTY+ this.trailing = false+ this.size = 0+ }++ /** A file that is not there is an empty one; any other failure to read it is the caller's to hear. */+ private async look(): Promise<void> {+ let handle: Handle+ try { handle = await this.openFile(this.filePath) } catch (error) {+ if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error+ this.reset()+ return+ }+ try {+ const { size } = await handle.stat()+ if (size < this.bytes || !await this.unchanged(handle)) this.reset()+ if (this.head.length < HEAD_BYTES && size > this.head.length) this.head = await readAt(handle, 0, Math.min(size, HEAD_BYTES))+ await walkLines(handle, this.bytes, size, (lineEnd) => {+ this.lines+++ const last = this.marks.length - 1+ if (this.lines - this.marks[last] >= this.stride.lines || lineEnd - this.at[last] >= this.stride.bytes) {+ this.marks.push(this.lines)+ this.at.push(lineEnd)+ }+ return false+ }, (end) => { this.bytes = end })+ this.trailing = await this.trailingLine(handle, size)+ this.size = size+ } finally { await handle.close() }+ }++ private async unchanged(handle: Handle): Promise<boolean> {+ if (this.head.length && !(await readAt(handle, 0, this.head.length)).equals(this.head)) return false+ if (!this.bytes) return true+ const [ending] = await readAt(handle, this.bytes - 1, 1)+ return ending === 10 || ending === 13+ }++ /** Whether the bytes after the last line ending hold a line: read only until they show they do. */+ private async trailingLine(handle: Handle, size: number): Promise<boolean> {+ const test = new BlankTest()+ for (let position = this.bytes; position < size && !test.seen;) {+ const chunk = await readAt(handle, position, Math.min(CHUNK_BYTES, size - position))+ if (!chunk.length) break+ test.feed(chunk)+ position += chunk.length+ }+ return test.end()+ }++ private async find(n: number): Promise<number> {+ if (n >= this.count()) return this.size+ if (n <= 0) return 0+ // The last mark at or before line n.+ let low = 0+ for (let high = this.marks.length - 1; low < high;) {+ const middle = (low + high + 1) >> 1+ if (this.marks[middle] <= n) low = middle+ else high = middle - 1+ }+ let found = this.at[low]+ let seen = this.marks[low]+ if (seen === n) return found+ const handle = await this.openFile(this.filePath)+ try {+ await walkLines(handle, found, this.bytes, (lineEnd) => {+ found = lineEnd+ return ++seen === n+ })+ } finally { await handle.close() }+ return found+ }+}++/** What the pager asks of an engine's lines. */+export interface PageRules {+ /** A page may start at this line — a turn's first, so a turn is never split. */+ startsPage(line: string): boolean+}++export interface Page {+ /** Oldest first. */+ lines: string[]+ /** Lines in the page, those too long to hold included. */+ count: number+ /** Where the page's first line starts. */+ start: number+ /** Lines remain before the page. */+ hasMore: boolean+ /** The page starts inside a turn: that one turn held more than a page may. */+ clipped: boolean+}++/**+ * The page that ends just before byte `end`: the `limit` lines before it, and as many more as reach+ * back to a line a page may start at — the window `windowRawLines` and `windowCodexLines` cut. Null when+ * the file shrank under the walk.+ *+ * Bounded at `maxBytes`, which the old window was not: a page that would hold more ends at the oldest+ * start that fits, short of `limit`, so a turn is still never split. Only a turn too long to fit at all+ * is cut at a line inside it (`clipped`), its older part left for the next page. A single line over+ * `maxBytes` is never held: it is counted where it stands and left out. A file that cannot be read is an+ * empty one, as it was to the whole-file reader.+ */+export async function pageBefore(+ filePath: string,+ end: number,+ limit: number,+ rules: PageRules,+ maxBytes = PAGE_MAX_BYTES,+ walk: Walk = scanRecordsBackward,+): Promise<Page | null> {+ const lines: string[] = []+ let count = 0+ let bytes = 0+ let start = end+ // The page up to its oldest start that fits: where a page that runs out of room ends.+ let fits: { lines: number; count: number; start: number } | null = null+ let reached = false+ let hasMore = false+ let clipped = false+ const stop = (): true => { hasMore = true; return true }+ const walking = end <= 0 ? Promise.resolve(true) : walk(filePath, end, (record, offset) => {+ if (reached) return stop()+ if (lines.length && bytes + record.length > maxBytes) {+ if (fits) {+ lines.length = fits.lines+ count = fits.count+ start = fits.start+ } else clipped = true+ return stop()+ }+ const line = record.toString('utf8')+ lines.push(line)+ count+++ bytes += record.length+ start = offset+ if (rules.startsPage(line)) {+ fits = { lines: lines.length, count, start }+ if (count >= limit) reached = true+ }+ }, maxBytes, (offset) => {+ if (reached) return stop()+ count+++ start = offset+ })+ const whole = await walking.catch(() => 'unreadable' as const)+ if (whole === 'unreadable') return { lines: [], count: 0, start: 0, hasMore: false, clipped: false }+ if (!whole) return null+ lines.reverse()+ return { lines, count, start, hasMore, clipped }+}++/** One page of a thread, and the cursor that asks for the page before it. */+export interface HistoryPage {+ lines: string[]+ hasMore: boolean+ oldestCursor: string | null+ /** The cursor named nothing in the file: the client reloads the thread. */+ staleCursor?: true+ clipped: boolean+}++const STALE: HistoryPage = { lines: [], hasMore: false, oldestCursor: null, staleCursor: true, clipped: false }+/** An id that is written into its record exactly as it reads, so its bytes can be searched for. */+const PLAIN_ID = /^[A-Za-z0-9_.:-]+$/++/**+ * Pages for the threads clients read. Holds an index per transcript (for Codex's line cursors and line+ * counts) and the byte each recently served Claude cursor names, so scrolling back one page after+ * another does not walk the file again from its end.+ */+export class TranscriptPager {+ private readonly indexes = new Map<string, LineIndex>()+ private readonly cursors = new Map<string, Map<string, number>>()++ constructor(+ private readonly options: {+ capacity?: number+ maxBytes?: number+ stride?: { lines: number; bytes: number }+ walk?: Walk+ } = {},+ ) {}++ private get capacity(): number { return this.options.capacity ?? 256 }+ private get maxBytes(): number { return this.options.maxBytes ?? PAGE_MAX_BYTES }++ /** `tailFile(path, Infinity).length`, read once and then only as the file grows. A file that cannot+ * be read has none, as it had to the whole-file reader. */+ async lineCount(filePath: string): Promise<number> {+ try { return (await this.index(filePath)).count() } catch { return 0 }+ }++ /** `windowRawLines` over the whole file, without it. No `limit` asks for every line, as far as fits. */+ async claude(filePath: string, opts: { limit?: number; before?: string }): Promise<HistoryPage> {+ let end = await length(filePath)+ if (opts.before !== undefined) {+ const at = await this.findClaude(filePath, end, opts.before)+ if (at === null) return STALE+ end = at+ }+ const page = await pageBefore(filePath, end, opts.limit ?? Infinity, { startsPage: (line) => claudePageLine(line).startsPage }, this.maxBytes, this.options.walk)+ if (!page) return STALE+ // An empty window before a cursor still names that cursor's line, as the old one did.+ const oldestCursor = page.lines.length ? claudePageLine(page.lines[0]).cursor : opts.before ?? null+ if (page.lines.length && oldestCursor !== null) this.remember(filePath, oldestCursor, page.start)+ return { lines: page.lines, hasMore: page.hasMore, oldestCursor, clipped: page.clipped }+ }++ /** `windowCodexLines` over the whole file, without it. No `limit` asks for every line, as far as fits.+ * A file that cannot be read is an empty one, as it was to the whole-file reader. */+ async codex(filePath: string, opts: { limit?: number; before?: string }): Promise<HistoryPage> {+ try { return await this.codexPage(filePath, opts) } catch {+ const old = windowCodexLines([], { limit: opts.limit ?? Infinity, before: opts.before })+ return old.staleCursor ? STALE : { lines: [], hasMore: false, oldestCursor: old.oldestCursor, clipped: false }+ }+ }++ private async codexPage(filePath: string, opts: { limit?: number; before?: string }): Promise<HistoryPage> {+ const index = await this.index(filePath)+ const count = index.count()+ let endIndex = count+ if (opts.before !== undefined) {+ const match = /^codex:(\d+)$/.exec(opts.before)+ if (!match) return STALE+ endIndex = Number(match[1])+ if (!Number.isSafeInteger(endIndex) || endIndex > count) return STALE+ }+ const page = await pageBefore(filePath, await index.endOf(endIndex), opts.limit ?? Infinity, { startsPage: codexPageStart }, this.maxBytes, this.options.walk)+ if (!page) return STALE+ return { lines: page.lines, hasMore: page.hasMore, oldestCursor: `codex:${endIndex - page.count}`, clipped: page.clipped }+ }++ private async index(filePath: string): Promise<LineIndex> {+ let index = this.indexes.get(filePath)+ if (index) this.indexes.delete(filePath)+ index ??= new LineIndex(filePath, this.options.stride)+ this.indexes.set(filePath, index)+ while (this.indexes.size > this.capacity) this.indexes.delete(this.indexes.keys().next().value!)+ await index.update()+ return index+ }++ private remember(filePath: string, cursor: string, offset: number): void {+ let known = this.cursors.get(filePath)+ if (known) this.cursors.delete(filePath)+ else known = new Map()+ this.cursors.set(filePath, known)+ known.delete(cursor)+ known.set(cursor, offset)+ while (known.size > 64) known.delete(known.keys().next().value!)+ while (this.cursors.size > this.capacity) this.cursors.delete(this.cursors.keys().next().value!)+ }++ /** Where the newest line with this cursor starts, or null. A remembered byte is checked first. */+ private async findClaude(filePath: string, end: number, cursor: string): Promise<number | null> {+ const names = (line: string): boolean => claudePageLine(line).cursor === cursor+ const known = this.cursors.get(filePath)?.get(cursor)+ if (known !== undefined && known < end) {+ let line: string | null = null+ const read = await streamRecords(filePath, known, end, (record) => { line = record; return true }, () => true, this.maxBytes)+ if (read && line !== null && names(line)) return known+ this.cursors.get(filePath)?.delete(cursor)+ }+ if (end <= 0) return null+ const needle = PLAIN_ID.test(cursor) ? Buffer.from(`"${cursor}"`) : null+ let found: number | null = null+ const whole = await scanRecordsBackward(filePath, end, (record, offset) => {+ if (needle && record.indexOf(needle) < 0) return false+ if (!names(record.toString('utf8'))) return false+ found = offset+ return true+ }, this.maxBytes).catch(() => false)+ return whole ? found : null+ }+}++async function length(filePath: string): Promise<number> {+ try { return (await stat(filePath)).size } catch { return 0 }+}diff --git a/cli/src/lib/transcriptTail.ts b/cli/src/lib/transcriptTail.tsindex 4831b2ee0..f4117d33a 100644--- a/cli/src/lib/transcriptTail.ts+++ b/cli/src/lib/transcriptTail.ts@@ -31,14 +31,16 @@ async function readFully(handle: FileHandle, chunk: Buffer, position: number): P** `visit` gets each non-blank record's bytes — valid only during the call — and the offset it starts* at; returning true stops the walk. LF, CR and CRLF each end a record, readline's rule. A record over- * `maxRecordBytes` is passed over unvisited. Resolves false when the file shrank under the walk: what- * was visited may then mix two versions of it.+ * `maxRecordBytes` is passed over unvisited, and only told to `passed` (its offset), if given; returning+ * true from that stops the walk too. Resolves false when the file shrank under the walk: what was+ * visited may then mix two versions of it.*/export async function scanRecordsBackward(filePath: string,end: number,visit: (record: Buffer, offset: number) => boolean | void,maxRecordBytes = MAX_RECORD_BYTES,+ passed?: (offset: number) => boolean | void,): Promise<boolean> {const handle = await open(filePath, 'r')try {@@ -53,7 +55,8 @@ export async function scanRecordsBackward(fragments = []fragmentBytes = 0oversized = false- if (skip || isBlank(bytes)) return false+ if (skip) return passed?.(offset) === true+ if (isBlank(bytes)) return falsereturn visit(bytes, offset) === true}while (position > 0) {diff --git a/cli/src/testing/transcriptScenarios.ts b/cli/src/testing/transcriptScenarios.tsnew file mode 100644index 000000000..536f92207--- /dev/null+++ b/cli/src/testing/transcriptScenarios.ts@@ -0,0 +1,116 @@+/**+ * Realistic Claude Code and Codex transcripts, record by record, in the shapes the engines write — the+ * fixtures attach (lib/attachTranscript.spec.ts) and paging (lib/transcriptPages.spec.ts) are checked+ * against at every cut point.+ */+// ── Codex records, in the shapes a 0.159 rollout writes ──────────────────────────────────────────────+export const cx = (type: string, payload: Record<string, unknown>): string =>+ JSON.stringify({ timestamp: '2026-10-03T23:21:44.458Z', type, payload })+export const codex = {+ meta: (version = '0.159.0') => cx('session_meta', { id: 'thread-1', cli_version: version, cwd: '/tmp/work' }),+ context: (model: string, effort: string | null, mode: string | null) => cx('turn_context', {+ turn_id: 't', model, ...(effort ? { reasoning_effort: effort } : {}), ...(mode ? { collaboration_mode: { mode } } : {}),+ }),+ settings: (model: string, effort: string, mode: string) => cx('event_msg', {+ type: 'thread_settings_applied', thread_settings: { model, reasoning_effort: effort, collaboration_mode: { mode } },+ }),+ started: (turn: string) => cx('event_msg', { type: 'task_started', turn_id: turn }),+ user: (text: string) => cx('event_msg', { type: 'user_message', message: text }),+ userItem: (text: string) => cx('event_msg', {+ type: 'item_completed', turn_id: 't', item: { type: 'UserMessage', id: 'u', content: [{ type: 'text', text, text_elements: [] }] },+ }),+ goal: (objective: string) => cx('response_item', {+ type: 'message', role: 'user',+ content: [{ type: 'input_text', text: `<codex_internal_context source="goal">\n<objective>${objective}</objective>\n</codex_internal_context>` }],+ }),+ developer: (text: string) => cx('response_item', { type: 'message', role: 'developer', content: [{ type: 'input_text', text }] }),+ agent: (text: string, phase = 'final_answer') => cx('event_msg', {+ type: 'item_completed', item: { type: 'AgentMessage', content: [{ type: 'Text', text }], phase },+ }),+ reasoning: (text: string) => cx('response_item', { type: 'reasoning', summary: [{ type: 'summary_text', text }] }),+ call: (id: string, cmd: string) => cx('response_item', { type: 'function_call', call_id: id, name: 'exec_command', arguments: JSON.stringify({ cmd }) }),+ output: (id: string, output: string) => cx('response_item', { type: 'function_call_output', call_id: id, output }),+ complete: (turn: string) => cx('event_msg', { type: 'task_complete', turn_id: turn, last_agent_message: 'done' }),+ aborted: (turn: string) => cx('event_msg', { type: 'turn_aborted', turn_id: turn }),+ compacted: (bytes: number) => cx('compacted', { message: '📘é'.repeat(bytes / 6), replacement_history: [] }),+ contextCompacted: () => cx('event_msg', { type: 'context_compacted' }),+ tokens: () => cx('event_msg', { type: 'token_count', info: { total_token_usage: { input_tokens: 1 } } }),+}++/** A rollout with every shape attach has to get right, in the order Codex writes them. */+export const codexScenario = (): string[] => [+ codex.meta(),+ codex.developer('<permissions instructions>'),+ codex.settings('gpt-5.5', 'high', 'default'),+ codex.started('t1'), codex.context('gpt-5.5', 'high', 'default'), codex.user('first question'),+ codex.reasoning('Planning'), codex.call('c1', 'ls'), codex.output('c1', 'a\nb'), codex.agent('one answer'), codex.complete('t1'),+ codex.started('t2'), codex.context('gpt-6', 'xhigh', 'plan'), codex.userItem('second, in the item shape'),+ codex.reasoning('Thinking 漢字 📘'), codex.agent('looking', 'commentary'),+ codex.compacted(150_000), codex.contextCompacted(), codex.tokens(),+ codex.call('c2', 'npm test'), codex.output('c2', 'ok'), codex.agent('second answer'), codex.complete('t2'),+ codex.settings('gpt-6', 'low', 'default'),+ codex.started('t3'), codex.context('gpt-6', null, 'default'), codex.user('third: effort left out'),+ codex.call('c3', 'sleep 1'),+ codex.user(''), // an empty message opens nothing+ codex.output('c3', 'slept'), codex.complete('t3'),+ codex.started('t4'), codex.context('gpt-6', 'medium', 'default'), codex.user('interrupted one'), codex.aborted('t4'),+ codex.started('t5'), codex.context('gpt-6', 'medium', 'default'), codex.goal('ship the release'), codex.reasoning('goal work'), codex.complete('t5'),+ codex.started('t6'), codex.context('gpt-6', 'medium', 'default'), codex.goal('ship the release'), codex.agent('continuing'), codex.complete('t6'),+ codex.started('t7'), codex.context('gpt-6', 'medium', 'default'), codex.user('a question mid-goal'), codex.complete('t7'),+ codex.started('t8'), codex.context('gpt-6', 'medium', 'default'), codex.goal('ship the release'), codex.reasoning('still the goal'),+ codex.call('c8', 'make'), codex.complete('t8'),+ codex.started('t9'), codex.context('gpt-6', 'max', 'plan'), codex.goal('a different goal'), codex.reasoning('new goal'),+ codex.complete('t9'),+ // A message sent mid-task, with a call still in flight: its output arrives after the new message.+ codex.started('t10'), codex.context('gpt-6', 'max', 'default'), codex.user('start the build'),+ codex.call('c10', 'make all'), codex.user('also run the tests while that builds'),+ codex.output('c10', 'built'), codex.reasoning('now the tests'), codex.call('c11', 'npm test'),+]++// ── Claude Code records ───────────────────────────────────────────────────────────────────────────────+let uuid = 0+export const cl = (record: Record<string, unknown>): string => JSON.stringify({+ parentUuid: null, isSidechain: false, userType: 'external', cwd: '/tmp', sessionId: 's', version: '2.1.212',+ timestamp: '2026-10-03T23:21:44.458Z', uuid: `u${++uuid}`, ...record,+})+export const claude = {+ user: (content: unknown, extra: Record<string, unknown> = {}) => cl({ type: 'user', message: { role: 'user', content }, ...extra }),+ assistant: (content: unknown[], stop: string | null, model = 'claude-opus-5-5') =>+ cl({ type: 'assistant', message: { id: `m${uuid}`, role: 'assistant', model, content, stop_reason: stop } }),+ text: (text: string) => ({ type: 'text', text }),+ thinking: (thinking: string) => ({ type: 'thinking', thinking }),+ tool: (id: string, command: string) => ({ type: 'tool_use', id, name: 'Bash', input: { command } }),+ result: (id: string, content: string) => cl({ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: id, content }] } }),+ compact: () => cl({ type: 'system', subtype: 'compact_boundary', content: 'Conversation compacted', compactMetadata: { trigger: 'auto' } }),+ summary: () => cl({ type: 'user', isCompactSummary: true, message: { role: 'user', content: 'Summary of the earlier conversation' } }),+}++export const claudeScenario = (): string[] => [+ JSON.stringify({ type: 'summary', summary: 'Earlier work', leafUuid: 'x' }),+ claude.user('first prompt'),+ claude.assistant([claude.thinking('let me think'), claude.text('First answer.')], 'end_turn'),+ claude.user('second prompt with a tool'),+ claude.assistant([claude.text('Checking.'), claude.tool('toolu_1', 'ls')], 'tool_use'),+ claude.result('toolu_1', 'files'),+ claude.assistant([claude.thinking('found them 漢字'), claude.text('Done: '.repeat(20_000))], 'end_turn', 'claude-fable-5-1'),+ claude.user('<local-command-stdout>Set model to Opus 5.5</local-command-stdout>'),+ claude.user('<bash-input>git status</bash-input><bash-stdout>clean</bash-stdout>'),+ claude.user('third prompt, interrupted'),+ claude.assistant([claude.tool('toolu_2', 'sleep 100')], 'tool_use'),+ claude.user([{ type: 'text', text: '[Request interrupted by user for tool use]' }]),+ claude.user('<task-notification><task-id>a</task-id><tool-use-id>toolu_9</tool-use-id><status>completed</status><summary>bg done</summary></task-notification>'),+ claude.compact(), claude.summary(),+ claude.user('loop iteration', { isMeta: true, promptSource: 'system' }),+ claude.assistant([claude.text('Loop ran.')], 'end_turn', '<synthetic>'),+ claude.user('a bookkeeping line', { isMeta: true }),+ claude.user('fourth prompt'),+ claude.assistant([claude.text('Working on it'), claude.tool('toolu_3', 'make')], 'tool_use'),+ claude.result('toolu_3', 'built'),+ claude.assistant([claude.thinking('almost'), claude.text('Built it.')], 'end_turn'),+ claude.user('fifth prompt, left running'),+ claude.assistant([claude.tool('toolu_4', 'npm test')], 'tool_use'),+ // A prompt that lands while a call is in flight: the result arrives after it, naming its call.+ claude.user('sixth prompt, typed during the tests'),+ claude.result('toolu_4', 'tests passed'),+ claude.assistant([claude.thinking('reading results'), claude.text('All green.')], 'end_turn'),+]diff --git a/cli/src/watcher/watcher.spec.ts b/cli/src/watcher/watcher.spec.tsindex 4d6b9d126..d8e217662 100644--- a/cli/src/watcher/watcher.spec.ts+++ b/cli/src/watcher/watcher.spec.ts@@ -1,9 +1,9 @@import { chmodSync } from 'fs'-import { appendFile, mkdtemp, rm, writeFile } from 'fs/promises'+import { appendFile, mkdtemp, open, rm, writeFile } from 'fs/promises'import { tmpdir } from 'os'import { join } from 'path'import { afterEach, describe, expect, it, vi } from 'vitest'-import { Watcher, type HistoryEvent, type LineEvent } from './watcher.js'+import { Watcher, type HistoryEvent, type LineEvent, type RewrittenEvent } from './watcher.js'const cleanup: string[] = []@@ -639,3 +639,88 @@ describe('Watcher, at its edges', () => {await watcher.stop()})})++describe('Watcher, reading a backlog', () => {+ const backlog = async (options: ConstructorParameters<typeof Watcher>[0], content = '') => {+ const dir = await mkdtemp(join(tmpdir(), 'machine-watcher-backlog-'))+ cleanup.push(dir)+ const transcriptPath = join(dir, 'session.jsonl')+ await writeFile(transcriptPath, content)+ const watcher = new Watcher(options)+ const lines: string[] = []+ const history: string[] = []+ const rewritten: RewrittenEvent[] = []+ watcher.on('line', (event: LineEvent) => lines.push(event.text))+ watcher.on('history', (event: HistoryEvent) => history.push(...event.lines.map((line) => line.text)))+ watcher.on('rewritten', (event: RewrittenEvent) => rewritten.push(event))+ await watcher.addSession({ sessionId: 's1', engine: 'codex', transcriptPath })+ return { transcriptPath, watcher, lines, history, rewritten }+ }+ const records = [+ '{"n":1}', '{"n":2,"text":"漢字 📘 across a chunk"}', `{"n":3,"long":"${'x'.repeat(200)}"}`, '{"n":4}', '', ' ', '{"n":5}',+ ]++ it('reads what fell behind a chunk at a time: lines and characters split between chunks, a line longer than many', async () => {+ for (const chunk of [1, 3, 16, 64, 1 << 20]) {+ const { transcriptPath, watcher, lines } = await backlog({ readChunkBytes: chunk })+ await appendFile(transcriptPath, records.join('\r\n') + '\r\n{"n":6, still')+ await watcher.pollSession('s1')+ expect(lines, `chunks of ${chunk}`).toEqual(records.filter((record) => record.trim()))+ await appendFile(transcriptPath, ' being written}\n')+ await watcher.pollSession('s1')+ expect(lines.at(-1)).toBe('{"n":6, still being written}')+ await watcher.stop()+ }+ })++ it('stops a long read at a hold that lands in it, and hands over every line exactly once', async () => {+ const all = Array.from({ length: 400 }, (_, i) => `{"n":${i},"pad":"${'y'.repeat(i % 37)}"}`)+ const { transcriptPath, watcher, lines } = await backlog({ readChunkBytes: 32 })+ await appendFile(transcriptPath, all.join('\n') + '\n')+ const reading = watcher.pollAll()+ const hold = (await watcher.hold('s1', transcriptPath))!+ await reading+ const size = Buffer.byteLength(all.join('\n') + '\n')+ expect(hold.offset).toBeLessThan(size)+ // Delivery stopped exactly at a line: what came out is everything before that byte.+ expect(Buffer.byteLength(lines.map((line) => line + '\n').join(''))).toBe(hold.offset)+ hold.release(hold.offset)+ await watcher.pollSession('s1')+ expect(lines).toEqual(all)+ await watcher.stop()+ })++ it('attaches a file that shrank to more history than one batch should hold, instead of replaying it', async () => {+ const { transcriptPath, watcher, lines, history, rewritten } = await backlog({ historyMaxBytes: 40 }, records.join('\n') + '\n')+ await writeFile(transcriptPath, records.slice(0, 4).join('\n') + '\n')+ await watcher.pollSession('s1')+ expect(rewritten).toEqual([{ sessionId: 's1', engine: 'codex', transcriptPath }])+ expect({ lines, history }).toEqual({ lines: [], history: [] })+ // The tail goes on from the new end.+ await appendFile(transcriptPath, '{"n":7}\n')+ await watcher.pollSession('s1')+ expect(lines).toEqual(['{"n":7}'])+ // A shrink small enough is still read as history.+ await writeFile(transcriptPath, '{"n":8}\n')+ await watcher.pollSession('s1')+ expect(history).toEqual(['{"n":8}'])+ await watcher.stop()+ })++ it('stops where a file cut short under the read ends', async () => {+ const shorted = async (path: string) => {+ const handle = await open(path, 'r')+ let reads = 0+ return Object.assign(Object.create(handle), {+ read: (...args: Parameters<typeof handle.read>) => (++reads > 1 ? Promise.resolve({ bytesRead: 0, buffer: args[0] }) : handle.read(...args)),+ close: () => handle.close(),+ })+ }+ const { transcriptPath, watcher, lines } = await backlog({ readChunkBytes: 8, openFile: shorted })+ await appendFile(transcriptPath, '{"n":1}\n{"n":2}\n')+ await watcher.pollSession('s1')+ expect(lines).toEqual(['{"n":1}'])+ await watcher.stop()+ })+})+diff --git a/cli/src/watcher/watcher.ts b/cli/src/watcher/watcher.tsindex de32bd697..338c9d4c7 100644--- a/cli/src/watcher/watcher.ts+++ b/cli/src/watcher/watcher.ts@@ -48,19 +48,34 @@ export interface TailHold {release(offset?: number | null): void}+/** A transcript rewritten in place with more history than one batch should hold — a long one whose+ * owner rewrote it. The tail now starts at its new end; the session's state is rebuilt by attaching it+ * again, from its end. */+export interface RewrittenEvent {+ sessionId: string+ engine: AgentEngine+ transcriptPath: string+}++/** How much the tail reads at once: a tail that fell behind reads its backlog a chunk at a time, never+ * in one buffer as large as everything appended since. */+export const READ_CHUNK_BYTES = 8 * 1024 * 1024+/** The most history replayed as one batch. A file that shrank to more than this is attached again. */+export const HISTORY_MAX_BYTES = 32 * 1024 * 1024+/** The longest a session's live delivery may stay held. An attach still running by then lets it go* (`TailHold.expired`) and must keep the normalizer delivery went back to. */export const HOLD_TIMEOUT_MS = 30_000-const EMPTY = Buffer.alloc(0)-interface FileState extends WatchedSession {offset: number/** Bytes below this were on disk before the current read cursor was placed — history, not live. */historicalUntil: number/** The end of the file not yet ending in a newline: raw bytes, decoded only once the line is whole,- * so a character split between two reads is not mangled and positions stay byte-exact. */- partial: Buffer+ * so a character split between two reads is not mangled and positions stay byte-exact. Kept as the+ * pieces read, joined once when the line ends: a long line is not copied again at every read. */+ partial: Buffer[]+ partialBytes: numberseq: numberdebounce: NodeJS.Timeout | nullreading: boolean@@ -82,6 +97,18 @@ export class Watcher extends EventEmitter {private files = new Map<string, FileState>()private bySession = new Map<string, string>()+ private readonly readChunkBytes: number+ private readonly historyMaxBytes: number+ private readonly openFile: (filePath: string) => ReturnType<typeof open>++ /** `openFile` is how a transcript is opened for reading: swapped in tests for one cut short as it is read. */+ constructor(options: { readChunkBytes?: number; historyMaxBytes?: number; openFile?: (filePath: string) => ReturnType<typeof open> } = {}) {+ super()+ this.readChunkBytes = options.readChunkBytes ?? READ_CHUNK_BYTES+ this.historyMaxBytes = options.historyMaxBytes ?? HISTORY_MAX_BYTES+ this.openFile = options.openFile ?? ((filePath) => open(filePath, 'r'))+ }+start(): void {if (this.watcher) returnthis.watcher = chokidar.watch([], { ignoreInitial: false, alwaysStat: true })@@ -129,7 +156,8 @@ export class Watcher extends EventEmitter {offset,// A `fromStart` tail of a file that already has content reads that content as history.historicalUntil: opts.fromStart ? size : 0,- partial: EMPTY,+ partial: [],+ partialBytes: 0,seq: 0,debounce: null,reading: false,@@ -173,7 +201,8 @@ export class Watcher extends EventEmitter {state.holds.delete(hold)if (to !== null && state.moves === moves) {state.offset = to- state.partial = EMPTY+ state.partial = []+ state.partialBytes = 0}if (state.holds.size) returnstate.settleUnheld?.()@@ -195,7 +224,7 @@ export class Watcher extends EventEmitter {}state.holds.add(hold)while (state.reading) await new Promise((resolve) => setTimeout(resolve, 5))- offset = state.offset - state.partial.length+ offset = state.offset - state.partialBytesreturn hold}@@ -218,7 +247,8 @@ export class Watcher extends EventEmitter {if (!state) returnif (state.debounce) { clearTimeout(state.debounce); state.debounce = null }state.offset = offset- state.partial = EMPTY+ state.partial = []+ state.partialBytes = 0// An attach holding the tail read the file as it was; where it stopped is no position in this one.state.moves++}@@ -315,43 +345,61 @@ export class Watcher extends EventEmitter {let size: numbertry { size = (await stat(filePath)).size } catch { return }if (size < state.offset) {+ state.partial = []+ state.partialBytes = 0+ if (size > this.historyMaxBytes) {+ // Too much to replay as one batch of history: the tail starts at the new end, and the session is+ // attached again, which reads what it needs from that end (lib/attachTranscript.ts).+ state.offset = size+ state.historicalUntil = 0+ this.emit('rewritten', { sessionId: state.sessionId, engine: state.engine, transcriptPath: filePath } satisfies RewrittenEvent)+ return+ }// The file shrank: rewritten in place (or recreated). Whatever it holds now is read from byte 0,// and none of it is new to the world — it is history until the write that grows it past this.state.offset = 0- state.partial = EMPTYstate.historicalUntil = size}if (size <= state.offset) return- const start = state.offset- let fresh: Buffer+ const history: LineEvent[] = []try {- const fh = await open(filePath, 'r')+ const fh = await this.openFile(filePath)try {- const buf = Buffer.alloc(size - start)- const { bytesRead } = await fh.read(buf, 0, buf.length, start)- fresh = buf.subarray(0, bytesRead)- state.offset = start + bytesRead+ while (state.offset < size) {+ const start = state.offset+ const chunk = Buffer.alloc(Math.min(this.readChunkBytes, size - start))+ const { bytesRead } = await fh.read(chunk, 0, chunk.length, start)+ if (!bytesRead) break+ state.offset = start + bytesRead+ this.takeChunk(filePath, state, chunk.subarray(0, bytesRead), start, history)+ // A hold taken meanwhile stops the read here; `release` reads what is left.+ if (state.holds.size) break+ }} finally {await fh.close()}} catch (err) {console.error(`[watcher] read failed (${basename(filePath)}):`, err)- return}+ if (history.length) this.flushHistory(state, history)+ }- // Lines are cut on the newline BYTE and decoded only once whole. Each line's byte position tells- // the historical prefix of this read from the live rest: a file appended to between the cursor- // being placed and this read has both. The carried partial line began that many bytes before `start`.- const base = start - state.partial.length- const combined = state.partial.length ? Buffer.concat([state.partial, fresh]) : fresh+ /**+ * Lines are cut on the newline BYTE and decoded only once whole. Each line's byte position tells the+ * historical prefix of a read from its live rest: a file appended to between the cursor being placed+ * and this read has both. A line carried over from the last chunk began `partialBytes` before `start`.+ */+ private takeChunk(filePath: string, state: FileState, chunk: Buffer, start: number, history: LineEvent[]): void {let lineStart = 0- const history: LineEvent[] = []- for (let newline = combined.indexOf(10); newline >= 0; newline = combined.indexOf(10, lineStart)) {- const linePos = base + lineStart- const raw = combined.subarray(lineStart, newline).toString('utf8')+ for (let newline = chunk.indexOf(10); newline >= 0; newline = chunk.indexOf(10, lineStart)) {+ const linePos = lineStart === 0 ? start - state.partialBytes : start + lineStart+ const piece = chunk.subarray(lineStart, newline)+ const bytes = state.partial.length ? Buffer.concat([...state.partial, piece]) : piece+ state.partial = []+ state.partialBytes = 0lineStart = newline + 1- const text = raw.replace(/\r$/, '')+ const text = bytes.toString('utf8').replace(/\r$/, '')if (!text.trim()) continueconst evt: LineEvent = {sessionId: state.sessionId,@@ -365,9 +413,12 @@ export class Watcher extends EventEmitter {if (history.length) this.flushHistory(state, history)this.emit('line', evt)}- if (history.length) this.flushHistory(state, history)- // A copy: the read buffer can be large, and only its unfinished last line is kept.- state.partial = lineStart < combined.length ? Buffer.from(combined.subarray(lineStart)) : EMPTY+ if (lineStart < chunk.length) {+ // A copy of a chunk's tail, so the rest of the chunk is not kept for it; a chunk that is all one+ // line is kept as read.+ state.partial.push(lineStart ? Buffer.from(chunk.subarray(lineStart)) : chunk)+ state.partialBytes += chunk.length - lineStart+ }}private flushHistory(state: FileState, lines: LineEvent[]): void {@@ -381,7 +432,8 @@ export class Watcher extends EventEmitter {while (common < state.cursorLines.length && common < next.length && state.cursorLines[common] === next[common]) common++state.cursorLines = nextstate.offset = 0- state.partial = EMPTY+ state.partial = []+ state.partialBytes = 0state.historicalUntil = 0// `completeLines` has already dropped blank lines.for (const text of next.slice(common)) {diff --git a/cli/vitest.e2e.config.ts b/cli/vitest.e2e.config.tsnew file mode 100644index 000000000..f0ae27724--- /dev/null+++ b/cli/vitest.e2e.config.ts@@ -0,0 +1,18 @@+import { defineConfig } from 'vitest/config'++/**+ * End-to-end: each file boots real daemons (see e2e/harness/daemon.ts) under throwaway homes with+ * private tmux servers and fake engines. Files run one at a time so their timing is the daemon's,+ * not the host's contention.+ */+export default defineConfig({+ test: {+ environment: 'node',+ setupFiles: ['./vitest.setup.ts'],+ include: ['e2e/**/*.e2e.ts'],+ testTimeout: 180_000,+ hookTimeout: 180_000,+ fileParallelism: false,+ pool: 'forks',+ },+})