The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Round 40b: a full disk, a throwing teardown and a stalled link no longer break an update (#817)
Dee72c66453ab1c
diff --git a/cli/e2e/diskfull.e2e.ts b/cli/e2e/diskfull.e2e.tsindex bcb51f8a2..7138318f7 100644--- a/cli/e2e/diskfull.e2e.ts+++ b/cli/e2e/diskfull.e2e.ts@@ -9,7 +9,7 @@* `DISKFULL=1 npm run test:e2e -- diskfull`.*/import { execFileSync } from 'node:child_process'-import { mkdirSync, mkdtempSync, openSync, closeSync, rmSync, writeFileSync, writeSync } from 'node:fs'+import { existsSync, mkdirSync, mkdtempSync, openSync, closeSync, rmSync, writeFileSync, writeSync } from 'node:fs'import { tmpdir } from 'node:os'import { join } from 'node:path'import { afterAll, afterEach, beforeAll, describe, expect, it, onTestFailed } from 'vitest'@@ -141,4 +141,37 @@ describe.skipIf(!ON)('a full disk', () => {if (process.env.DISKFULL_LOG) writeFileSync(process.env.DISKFULL_LOG, d.log())client.close()}, 600_000)++ it('the first turn on a data folder that never ran a project ends, though the disk is full', async () => {+ // A turn's end asks the orchestrator whether its agent is a specialist, and reading makes the+ // orchestrator's folder: on a full disk the mkdir threw out of the transcript's line handler, and the+ // turn never ended for the windows (found by e2e/updateHostile.e2e.ts, round 40). The turn's end no+ // longer depends on it (core/turns/recaps.ts); the orchestrator itself is left as it is for now.+ const data = join(volume, 'd2')+ mkdirSync(data, { recursive: true })+ const d = await IsolatedDaemon.create({ dataDir: data })+ daemon = d+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`) })+ await d.start()+ const client = await LocalClient.connect(d)+ const cwd = join(d.projectsDir, 'first-turn-full')+ mkdirSync(cwd, { recursive: true })+ const created = await client.request('agent_create', { engine: 'claude', cwd, bypassPermission: true }, 90_000)+ expect(created.error, JSON.stringify(created)).toBeUndefined()+ const agent = await until('the agent to bind', async () => {+ const now = await row(client, created.agent.id)+ return now?.sessionId && now.status === 'active' ? now : null+ }, 60_000, 500)+ expect(existsSync(join(data, 'orchestrator')), 'no turn has asked the orchestrator yet').toBe(false)+ const filler = fill(volume)+ try {+ await turn(client, agent.id, 'the first turn, on a full disk')+ expect(d.log()).not.toContain('line handler error')+ } finally {+ rmSync(filler, { force: true })+ rmSync(join(volume, 'filler-tail'), { force: true })+ }+ expect(d.coresStarted()).toBe(1)+ client.close()+ }, 300_000)})diff --git a/cli/e2e/updateHostile.e2e.ts b/cli/e2e/updateHostile.e2e.tsindex ee8346b83..f2df725b0 100644--- a/cli/e2e/updateHostile.e2e.ts+++ b/cli/e2e/updateHostile.e2e.ts@@ -29,6 +29,12 @@ const FIRST = '43.0.1'const DISKFULL = process.env.DISKFULL === '1' && process.platform === 'darwin'/** Passes the updater's canary (`cli.js version`), then dies on every start as a core. */const CRASHES_AS_A_CORE = 'if(process.argv[2]==="__run")process.exit(3);'+/**+ * A core that runs as any other, until it hands over to an update: then the first interval it clears+ * throws, as a teardown step that fails does (core/updateHandoff.ts). Told by the line the handoff+ * starts with, wherever the daemon's timestamp puts it among the arguments.+ */+const TEARDOWN_THROWS = 'if(process.argv[2]==="__run"){let f=false;const l=console.log;console.log=function(...a){if(a.some((x)=>typeof x==="string"&&x.includes("[update] applying")))f=true;return l.apply(this,a)};const c=globalThis.clearInterval;globalThis.clearInterval=function(...a){if(f){f=false;throw new Error("a teardown step that throws")}return c.apply(this,a)}}'const sha = (bytes: Buffer): string => createHash('sha256').update(bytes).digest('hex')const count = (text: string, part: string): number => text.split(part).length - 1const sleep = (ms: number) => new Promise((done) => setTimeout(done, ms))@@ -70,8 +76,9 @@ class Releases {}/** What a download of cli.js meets: cut short with a length to match, a connection that dies halfway- * through the length it promised, or a wait until the test lets it through. */-type Fault = 'truncate' | 'drop' | 'hold'+ * through the length it promised, one that stays open halfway through and sends nothing more, or a+ * wait until the test lets it through. */+type Fault = 'truncate' | 'drop' | 'stall' | 'hold'interface Offer {version: string@@ -89,6 +96,8 @@ class UpdateServer {/** Every download of a cli.js asked for, by version, in order. */readonly downloads: string[] = []private held: Array<() => void> = []+ /** Answers left open halfway, ended only when the server stops. */+ private stalled: ServerResponse[] = []private server: Server | null = nullconstructor(private readonly releases: Releases, first: string) { this.offer = { version: first } }@@ -104,6 +113,7 @@ class UpdateServer {stop(): Promise<void> {this.release()+ for (const response of this.stalled) response.socket?.destroy()return new Promise((done) => this.server ? this.server.close(() => done()) : done())}@@ -142,6 +152,10 @@ class UpdateServer {} else if (fault === 'drop') {response.setHeader('content-length', bytes.length)response.write(half, () => setTimeout(() => response.socket?.destroy(), 20))+ } else if (fault === 'stall') {+ response.setHeader('content-length', bytes.length)+ response.write(half)+ this.stalled.push(response)} else if (fault === 'hold') {this.held.push(() => response.end(bytes))} else {@@ -174,7 +188,7 @@ class Installed {constructor(readonly releases: Releases, readonly server: UpdateServer, readonly cliDir: string) {}- async boot(first: string, dataDir?: string): Promise<void> {+ async boot(first: string, dataDir?: string, env: Record<string, string> = {}): Promise<void> {mkdirSync(this.cliDir, { recursive: true })writeFileSync(join(this.cliDir, 'cli.js'), this.releases.cli.get(first)!)writeFileSync(join(this.cliDir, 'notify.mjs'), this.releases.notify.get(first)!)@@ -195,6 +209,7 @@ class Installed {HARNESS_TUI_MANIFEST_URL: `${local}/tui/metadata.json`,ADAPTER_RUNTIME_METADATA_URL: `${local}/runtime/metadata.json`,ADAPTER_GRID_RUNTIME_METADATA_URL: `${local}/grid/metadata.json`,+ ...env,},})await this.daemon.start()@@ -211,10 +226,6 @@ class Installed {return row?.sessionId && row.status === 'active' ? row : null}, 90_000, 500))}- // At work before any update: a daemon that has finished a turn has made the folders a turn's end- // writes to. (On a data folder that never had them, a turn ending while the disk is full is lost:- // the orchestrator's state folder cannot be made. Reported with round 40, not part of updating.)- for (const agent of this.agents) await turn(client, agent.id, `before any update (${agent.engine})`)} finally { client.close() }}@@ -314,12 +325,15 @@ describe('an update under hostile conditions', () => {let machine: InstalledbeforeAll(async () => {- for (let patch = 1; patch <= 12; patch++) releases.add(`43.0.${patch}`, patch === 10 || patch === 11 ? CRASHES_AS_A_CORE : '')+ for (let patch = 1; patch <= 15; patch++) {+ releases.add(`43.0.${patch}`, patch === 10 || patch === 11 ? CRASHES_AS_A_CORE : patch === 14 ? TEARDOWN_THROWS : '')+ }scratch = mkdtempSync(join(tmpdir(), 'harnessd-hostile-'))server = new UpdateServer(releases, FIRST)await server.start()machine = new Installed(releases, server, join(scratch, 'cli'))- await machine.boot(FIRST)+ // A link that sends nothing for two seconds is given up on (lib/selfUpdate.ts TransferLimits).+ await machine.boot(FIRST, undefined, { ADAPTER_UPDATE_IDLE_MS: '2000' })}, 300_000)afterAll(async () => {@@ -454,6 +468,30 @@ describe('an update under hostile conditions', () => {server.offer = { version: '43.0.12' }await machine.settled('43.0.12', { rejected: ['43.0.10', '43.0.11'] })}, 300_000)++ it('a link that stalls halfway without dropping is given up on, and the build lands at the next check', async () => {+ machine.showLogOnFailure()+ const from = machine.log().length+ server.faults = ['stall']+ server.offer = { version: '43.0.13' }+ // Before, the check waited out undici's five-minute body timeout, every later check skipped meanwhile.+ await until('the stalled download to be given up', () => machine.log().slice(from).includes('cli-43.0.13.js sent nothing for 2000 ms'), 60_000)+ await machine.settled('43.0.13', { rejected: ['43.0.10', '43.0.11'] })+ expect(server.downloadsOf('43.0.13')).toBe(2)+ }, 300_000)++ it('a core whose teardown throws as it hands over still hands over, and the new build is kept', async () => {+ machine.showLogOnFailure()+ server.offer = { version: '43.0.14' }+ await machine.settled('43.0.14', { rejected: ['43.0.10', '43.0.11'] })+ // 43.0.14's own handoff is the one that throws. Before, the core stayed on it, half torn down, with+ // 43.0.15 staged and never judged.+ const from = machine.log().length+ server.offer = { version: '43.0.15' }+ await machine.settled('43.0.15', { rejected: ['43.0.10', '43.0.11'] })+ expect(machine.log().slice(from)).toMatch(/\[update\] [^\n]* did not let go \(a teardown step that throws\) — handing over all the same/)+ expect(machine.log().slice(from)).not.toContain('restart failed — staying on current build')+ }, 300_000)})describe.skipIf(!DISKFULL)('an update on a disk that fills', () => {@@ -565,21 +603,25 @@ describe.skipIf(!DISKFULL)('an update on a disk that fills', () => {await machine.settled('44.0.3', { rejected: [] })}, 400_000)- it('the disk fills just after the swap: the master outlives it, and the daemon is back on a working build, the new one kept or the old one with the new remembered', async () => {+ it('the disk fills just after the swap: the new build cannot start for want of room, which is no verdict on it, and it is kept once there is room', async () => {machine.showLogOnFailure()const from = machine.log().lengthconst master = machine.daemon.pidserver.offer = { version: '44.0.4' }await until('the core to stage 44.0.4', () => machine.log().slice(from).includes('[update] staged 44.0.4'), 90_000, 50)fill()- // A core cannot open its socket on a full disk (listen ENOSPC): one started on the new bundle comes up- // in safe mode, and the master rolls the update back, the rollback itself freeing the room the build- // before needs. One that came up before the disk filled stays up and is kept.- await until('the update to be kept or rolled back on the full disk', () => /the update stayed up — keeping it|the updated core failed[^\n]*rolled back/.test(machine.log().slice(from)), 150_000, 100)+ // A core cannot claim its socket on a full disk (listen ENOSPC), so the core on the new bundle starts in+ // safe mode. The build before would too: the master keeps the update on trial and tries again,+ // backing off. Before, it rolled back (the rollback freeing the room the build before needed) and+ // remembered a good build as bad.+ await until('the master to try the new bundle again for want of room', () => count(machine.log().slice(from), 'had no room to start') >= 2, 150_000, 100)+ expect(machine.log().slice(from)).toContain('could not start on the new bundle for want of room on the disk (listen ENOSPC')+ expect(machine.log().slice(from)).not.toContain('the updated core failed')expect(IsolatedDaemon.alive(master), 'the master outlives a full disk in the middle of an update').toBe(true)+ machine.holds('44.0.4')unfill()- if (machine.log().slice(from).includes('the update stayed up — keeping it')) await machine.settled('44.0.4', { masters: ['44.0.3', '44.0.4'], rejected: [] })- else await machine.settled('44.0.3', { masters: ['44.0.3', '44.0.4'], rejected: ['44.0.4'] })+ await machine.settled('44.0.4', { masters: ['44.0.3', '44.0.4'], rejected: [] })+ expect(machine.log().slice(from)).toContain('the update stayed up — keeping it')expect(machine.daemon.pid).toBe(master)}, 400_000)diff --git a/cli/src/backendSocket.spec.ts b/cli/src/backendSocket.spec.tsindex d9895b845..48cca89b1 100644--- a/cli/src/backendSocket.spec.ts+++ b/cli/src/backendSocket.spec.ts@@ -628,6 +628,21 @@ describe('BackendSocket outbound queue', () => {await socket.stop()})+ it('sends a frame though the orchestrator watching the frames cannot read its state', async () => {+ // A full disk: reading the orchestrator's state makes its folder, and that threw out of every send,+ // so a turn's end never reached a window (e2e/diskfull.e2e.ts).+ const socket = new BackendSocket('token')+ ;(socket as unknown as { orchestratorService: { ingest(): void; stop(): void } }).orchestratorService = {+ ingest: () => { throw new Error('ENOSPC: no space left on device, mkdir') }, stop: () => {},+ }+ socket.connect()+ const ws = wsMock.instances[0]+ ws.open()+ socket.send({ type: 'turn_ended', dbSessionId: 's1', payload: { sessionId: 's1' } })+ expect(parseSent(ws).map((sent) => (sent.frame as { type?: string }).type)).toContain('turn_ended')+ await socket.stop()+ })+it('bounds the opening handshake and reconnects when it times out', async () => {// A connect attempt whose TCP side came up but whose upgrade was never answered used to sit in// CONNECTING forever: no 'open', so no heartbeat to terminate it, and `this.ws` set, so everydiff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.tsindex 5b64a0c75..47f052eb2 100644--- a/cli/src/backendSocket.ts+++ b/cli/src/backendSocket.ts@@ -992,8 +992,9 @@ export class BackendSocket {send(frame: Frame): void {this.monitorCompletions.observe(frame)// Only an already-open orchestration service observes events; ordinary sessions- // do not create project state or incur disk work. Project payloads stay local.- this.orchestratorService?.ingest(frame)+ // do not create project state or incur disk work. Project payloads stay local. Its state it cannot+ // read (a full disk: reading makes its folder) must not cost every frame after it (e2e/diskfull.e2e.ts).+ try { this.orchestratorService?.ingest(frame) } catch { /* the frame goes out regardless */ }if (env.LOG_FRAMES) logFrame('→', 'web', frame)for (const [connId, sink] of this.localClients) {if (!sink.sendFrame(frame)) void this.unregisterLocalClient(connId)diff --git a/cli/src/cli.ts b/cli/src/cli.tsindex 47a1fb1a8..d01d358b7 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -141,6 +141,7 @@ import { createQuestions } from './core/questions.js'import { createTurnActivity } from './core/turns/activity.js'import { createLastTurnReader } from './core/transcripts/lastTurn.js'import { createRecaps } from './core/turns/recaps.js'+import { createUpdateHandoff, type HandoffChild } from './core/updateHandoff.js'import { createHeartbeats } from './core/turns/heartbeats.js'import { createEventFunnel, outsideConsumers } from './core/turns/funnel.js'import { createAgyBackstop } from './core/turns/agyBackstop.js'@@ -554,6 +555,64 @@ function bootHandoff(version: string): void {})}+/**+ * The update handoff of a core run on its own (core/updateHandoff.ts, once the teardown is done): hand+ * off to a freshly spawned daemon running the just-swapped cli.js, then SUPERVISE it and roll back to the+ * .prev bytes if it fails to come up. NOT launch() — that refuses while a daemon is alive.+ *+ * Runs under the spawn lock for its whole length (the updater's `withLock` wraps the staging and this+ * together), so no `harness start` can spawn into the seconds where the port is free and the pid file+ * names nothing. `track` names the child a signal mid-handoff must take down with us, rather than leave+ * two daemons — see shutdown(); null the moment the handoff is CONFIRMED, when that child is the daemon.+ */+async function handOffWithoutMaster(newVersion: string, track: (child: HandoffChild | null) => void): Promise<void> {+ const sinceOffset = existsSync(LOG_FILE) ? statSync(LOG_FILE).size : 0+ const child = spawnDaemonChild({ ADAPTER_UPDATED_TO: newVersion })+ track(child)+ let childExited = false+ child.on('exit', () => { childExited = true })++ // Two phases. First the child has to BIND the port — it claims the pid file itself at that+ // moment, and nothing else writes that file any more. A child that exits or stalls before then+ // is a bad build (or a port it could not take): roll back at once instead of burning the whole+ // connect window on it. Then, bound, wait for the backend: KEEP on connected/unreachable/busy+ // (the new build RAN), ROLL BACK only on `fatal`. unreachable = backend transient, not a bad build.+ const bind = await waitForBind(child.pid ?? -1, () => childExited, BIND_WAIT_MS, launchDeps)+ const ready = bind === 'bound' ? await waitForReady(sinceOffset, 30_000, launchDeps) : null+ if (bind === 'bound' && !childExited && ready?.state !== 'fatal') {+ // Confirmed: it is the daemon now. Let go of it BEFORE anything else — a SIGTERM landing between+ // here and the exit below must not take it down with us (see shutdown()).+ track(null)+ child.unref()+ confirmUpdate(env.ADAPTER_CLI_DIR) // drop the .prev backups+ console.log(`[update] now running ${newVersion} (pid ${child.pid})`)+ process.exit(0)+ }+ console.error(`[update] new build failed to start (${bind !== 'bound' ? bind : childExited ? 'exited' : ready?.state}) — rolling back`)+ try { if (child.pid) process.kill(child.pid, 'SIGKILL') } catch { /* ignore */ }+ // A killed child cannot remove its own pid file; do it for it — but only once it is actually+ // dead (SIGKILL is asynchronous, and a child mid-bind could still write the file after our+ // removal) and only if it is still ITS file.+ if (child.pid) {+ const gone = Date.now() + 2_000+ while (Date.now() < gone && isAlive(child.pid)) await new Promise((r) => setTimeout(r, 50))+ }+ removePidFileIf(child.pid)+ restoreUpdate(env.ADAPTER_CLI_DIR) // restore .prev → cli.js/notify.mjs+ const good = spawnDaemonChild({})+ track(good)+ let goodExited = false+ good.on('exit', () => { goodExited = true })+ // Hold the lock — and this process — until the rollback child has bound too. Exiting the moment it+ // is spawned would free the lock while the port is still unclaimed, which is the window this whole+ // arrangement exists to close. Nothing to do if it fails: the .prev bytes were the build that was+ // running a minute ago, and `harness start` can be tried by hand.+ const goodBind = await waitForBind(good.pid ?? -1, () => goodExited, BIND_WAIT_MS, launchDeps)+ if (goodBind !== 'bound') console.error(`[update] rollback build did not come up either (${goodBind}) — run harness start`)+ good.unref()+ process.exit(0)+}+/** This computer's identity — see lib/computerIdentity.ts. Sent on connect so the backend can enforce* one machine per computer, and used by `harness start` to reconnect to the machine already* bound to this box instead of minting a second one. */@@ -1603,16 +1662,9 @@ async function runForeground(session: AuthSession | null): Promise<void> {// there is no supervisor, and the desktop app only re-runs `harness start` on the same broken bytes.// Started here, a published fix lands on its own however badly the rest of the boot goes.//- // `onStaged` is one indirection on purpose: `restartForUpdate` does not exist yet and must not move+ // `onStaged` is one indirection on purpose: the handoff's teardown does not exist yet and must not move// (it tears down two dozen subsystems declared further down). Until it is ready, a staged update is// applied by `bootHandoff`, which hands the machine over without finishing start-up.- // Update-handoff state, declared here — ahead of the /api/status handler that reads `restarting` —- // rather than beside the updater that writes it, so the closure never reaches a `let` in its TDZ.- let restarting = false- // The child a handoff is supervising, so a signal that lands mid-handoff can take it down with us- // rather than leaving two daemons — see shutdown(). Cleared the moment the handoff is CONFIRMED:- // from then on that child is the daemon, and a signal must not take it down with the old one.- let handoffChild: ReturnType<typeof spawn> | null = null// Self-update ONLY manages the INSTALLED copy (`~/.harness/cli/cli.js`). A dev/repo run — `tsx`// (`npm run dev`) OR `node dist/cli.js` from the checkout — must NEVER self-update: it would swap@@ -1636,6 +1688,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {onWaiting: (owner) => console.log(`[update] waiting — the daemon is ${describeSpawnLockOwner(owner)}`),}),onStaged: (v) => daemonBoot.applyStagedUpdate(v),+ limits: { idleMs: env.ADAPTER_UPDATE_IDLE_MS, deadlineMs: env.ADAPTER_UPDATE_DEADLINE_MS },})const slotted = env.ADAPTER_UPDATE_SLOT_SEC >= 0 && 60_000 % env.ADAPTER_UPDATE_CHECK_MS === 0console.log(`[update] self-update on · v${VERSION} · every ${Math.round(env.ADAPTER_UPDATE_CHECK_MS / 1000)}s`@@ -1652,6 +1705,13 @@ async function runForeground(session: AuthSession | null): Promise<void> {intervalMs: env.ADAPTER_UPDATE_CHECK_MS,slotSecond: env.ADAPTER_UPDATE_SLOT_SEC,})+ // The update handoff (core/updateHandoff.ts): after the updaters, never above them (startupOrder.spec.ts),+ // and ahead of the /api/status handler that reads whether it is under way and of shutdown(), which takes+ // a successor it is judging down with us.+ const updateHandoff = createUpdateHandoff({+ version: VERSION, supervised: coreLink.supervised, exitForUpdate: () => process.exit(CORE_EXIT_UPDATE),+ handOff: handOffWithoutMaster, log: (line) => console.log(line), error: (line) => console.error(line),+ })// Another daemon serves this data folder: leave before reading or writing anything of its — its// registry, its viewers, its panes. Just after the updaters, never before them (startupOrder.spec.ts);@@ -2796,7 +2856,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {harnessd: coreLink.supervised ? { masterPid: coreLink.masterPid, ...coreLink.status() } : null,// True for the few hundred ms between an update being staged and this server closing for the// handoff. Informational: nothing should build readiness on a field the server stops serving.- restarting,+ restarting: updateHandoff.restarting(),discoveryReady,discoveryError: discoveryError ?? (tmuxUnavailable ? `tmux unavailable: ${tmuxUnavailable}` : null),// Present only when start-up failed and this daemon is holding the machine open for its@@ -3713,122 +3773,33 @@ async function runForeground(session: AuthSession | null): Promise<void> {console.log(`[cli] not signed in — serving this computer only · watching registered sessions for ${ENGINES.length} engines`)}- // ── self-update: poll GCS for a newer bundle → verify+swap → restart IMMEDIATELY (supervised rollback) ──- //- // The restart used to wait for the computer to go idle. That wait was unbounded, and "idle" is a set of- // latches — open turn, settling composer, awaited submit, control lock, recap in flight — so ONE latch- // left stuck deferred the restart forever. Seen on 2026-07-31: 0.0.26 staged, then eight minutes of- // "deferring restart — sessions still processing" with the daemon otherwise silent. A daemon that- // quietly never updates is the exact failure this updater exists to prevent, so the wait is gone- // (owner call, 2026-07-31): staged means restart now.- //- // The cost is real and accepted: a turn streaming at that moment loses the rest of its events, and its- // clients see no turn_end for it until the new daemon re-attaches the session and the next turn runs.-- // Hand off to a freshly-spawned daemon running the just-swapped cli.js, then SUPERVISE it and roll- // back to the .prev bytes if it fails to come up. NOT launch() — that refuses while a daemon is alive.- //- // Runs under the spawn lock for its whole length (the updater's `withLock` wraps the staging and- // this together), so no `harness start` can spawn into the seconds where the port is free and the- // pid file names nothing.- const restartForUpdate = async (newVersion: string): Promise<void> => {- if (restarting) return- restarting = true- console.log(`[update] applying ${VERSION} → ${newVersion} — restarting daemon`)- registry.flush()- daemonBoot.updater?.stop()- daemonBoot.tuiUpdater?.stop()- agentReconciler.stop()- clearInterval(logTrimTimer)- clearInterval(runtimeReconcileTimer)- clearInterval(paneTitleSyncTimer)- questionWatcher.stopAll()- for (const t of heartbeats.values()) clearInterval(t)- heartbeats.clear()- cursorSubagents.stop()- normalizers.stopPollers()- await cursorDiscovery.stop()- await watcher.stop()- // Fully release the FIXED hook port BEFORE the child binds (no fallback → EADDRINUSE otherwise).- ;(hookServer as unknown as { closeAllConnections?: () => void }).closeAllConnections?.()- // Release the fixed hook port before the child binds. Process-owned agents stay in the persisted- // registry and are revalidated by the new daemon's first discovery passes.- shareRelay.close()- sharedViewers.stop()- await localWsServer.close()- hookServer.close()- await localSocket?.close()- codexActivity.close()- shutdownVoiceRouter()- // The new daemon starts its own viewers for the agents it restores; ours must not hold the ports.- await ports.viewers?.stop()- autonomousDeviceDirect?.stop()- await backend.stop() // graceful WS close → releases the Redis machine-owner claim- await new Promise((r) => setTimeout(r, 1000)) // grace before the same-machine reclaim-- if (coreLink.supervised) {- // Everything above is released; the master starts the new bundle as soon as this exits and- // rolls back to the .prev bytes if it does not come up and stay up (harnessd/supervisor.ts).- console.log(`[update] handing ${newVersion} to harnessd`)- process.exit(CORE_EXIT_UPDATE)- }- const sinceOffset = existsSync(LOG_FILE) ? statSync(LOG_FILE).size : 0- const child = spawnDaemonChild({ ADAPTER_UPDATED_TO: newVersion })- handoffChild = child- let childExited = false- child.on('exit', () => { childExited = true })-- // Two phases. First the child has to BIND the port — it claims the pid file itself at that- // moment, and nothing else writes that file any more. A child that exits or stalls before then- // is a bad build (or a port it could not take): roll back at once instead of burning the whole- // connect window on it. Then, bound, wait for the backend: KEEP on connected/unreachable/busy- // (the new build RAN), ROLL BACK only on `fatal`. unreachable = backend transient, not a bad build.- const bind = await waitForBind(child.pid ?? -1, () => childExited, BIND_WAIT_MS, launchDeps)- const ready = bind === 'bound' ? await waitForReady(sinceOffset, 30_000, launchDeps) : null- if (bind === 'bound' && !childExited && ready?.state !== 'fatal') {- // Confirmed: it is the daemon now. Let go of it BEFORE anything else — a SIGTERM landing between- // here and the exit below must not take it down with us (see shutdown()).- handoffChild = null- child.unref()- confirmUpdate(env.ADAPTER_CLI_DIR) // drop the .prev backups- console.log(`[update] now running ${newVersion} (pid ${child.pid})`)- process.exit(0)- }- console.error(`[update] new build failed to start (${bind !== 'bound' ? bind : childExited ? 'exited' : ready?.state}) — rolling back`)- try { if (child.pid) process.kill(child.pid, 'SIGKILL') } catch { /* ignore */ }- // A killed child cannot remove its own pid file; do it for it — but only once it is actually- // dead (SIGKILL is asynchronous, and a child mid-bind could still write the file after our- // removal) and only if it is still ITS file.- if (child.pid) {- const gone = Date.now() + 2_000- while (Date.now() < gone && isAlive(child.pid)) await new Promise((r) => setTimeout(r, 50))- }- removePidFileIf(child.pid)- restoreUpdate(env.ADAPTER_CLI_DIR) // restore .prev → cli.js/notify.mjs- const good = spawnDaemonChild({})- handoffChild = good- let goodExited = false- good.on('exit', () => { goodExited = true })- // Hold the lock — and this process — until the rollback child has bound too. Exiting the moment it- // is spawned would free the lock while the port is still unclaimed, which is the window this whole- // arrangement exists to close. Nothing to do if it fails: the .prev bytes were the build that was- // running a minute ago, and `harness start` can be tried by hand.- const goodBind = await waitForBind(good.pid ?? -1, () => goodExited, BIND_WAIT_MS, launchDeps)- if (goodBind !== 'bound') console.error(`[update] rollback build did not come up either (${goodBind}) — run harness start`)- good.unref()- process.exit(0)- }-- // The handoff handler stops being `bootHandoff` HERE, and not a line earlier: everything- // `restartForUpdate` tears down — the hook server, the reconciler, the three interval timers, the- // watcher, the backend socket — exists by now. A straight-line assignment, never a wait: if the- // body never reaches this line the handler stays `bootHandoff`, and the fix still lands.- daemonBoot.applyStagedUpdate = (v) => restartForUpdate(v).catch((err) => {- // If the restart handoff itself throws/rejects (I/O fault during teardown), don't let it become- // an unhandledRejection — log, un-latch `restarting`, and stay on the current build.+ // ── self-update: a staged bundle restarts the daemon IMMEDIATELY (core/updateHandoff.ts, and+ // handOffWithoutMaster for a core run on its own). The handler stops being `bootHandoff` HERE, and not a+ // line earlier: everything the teardown releases exists by now. A straight-line assignment, never a+ // wait: if the body never reaches this line the handler stays `bootHandoff`, and the fix still lands.+ daemonBoot.applyStagedUpdate = (v) => updateHandoff.restartForUpdate(v, [+ ['the registry', () => registry.flush()], ['the updaters', () => { daemonBoot.updater?.stop(); daemonBoot.tuiUpdater?.stop() }],+ ['the reconciler', () => agentReconciler.stop()],+ ['the timers', () => { clearInterval(logTrimTimer); clearInterval(runtimeReconcileTimer); clearInterval(paneTitleSyncTimer) }],+ ['the question watchers', () => questionWatcher.stopAll()],+ ['the turn heartbeats', () => { for (const t of heartbeats.values()) clearInterval(t); heartbeats.clear() }],+ ['the Cursor sub-agents', () => cursorSubagents.stop()], ['the normalizers', () => normalizers.stopPollers()],+ ['Cursor discovery', () => cursorDiscovery.stop()], ['the transcript watcher', () => watcher.stop()],+ // The FIXED hook port, released before the successor binds it (no fallback → EADDRINUSE otherwise).+ // Process-owned agents stay in the persisted registry and are revalidated by its first discovery passes.+ ['the hook connections', () => (hookServer as unknown as { closeAllConnections?: () => void }).closeAllConnections?.()],+ ['the share relay', () => shareRelay.close()], ['the shared viewers', () => sharedViewers.stop()],+ ['the local websocket', () => localWsServer.close()], ['the hook server', () => hookServer.close()],+ ['the local socket', () => localSocket?.close()], ['Codex activity', () => codexActivity.close()],+ ['the voice router', () => shutdownVoiceRouter()],+ // The successor starts its own viewers for the agents it restores; ours must not hold the ports.+ ['the viewers', () => ports.viewers?.stop()], ['the device link', () => autonomousDeviceDirect?.stop()],+ // A graceful close releases the backend's one-machine claim, given a moment before the reclaim.+ ['the backend', () => backend.stop()], ['a grace', () => new Promise((r) => setTimeout(r, 1000))],+ ]).catch((err) => {+ // Only without a master: a step or the successor failed before anything was handed over.console.error('[update] restart failed — staying on current build:', err instanceof Error ? err.message : err)- restarting = false- handoffChild = null+ updateHandoff.abandon()})/** Stopping for good — removed from the account, or connected from elsewhere: tell harnessd's master,@@ -3839,8 +3810,8 @@ async function runForeground(session: AuthSession | null): Promise<void> {// Mid-handoff everything below has already been torn down once, and the daemon that matters is// the child being supervised. Take it down with us and leave — a second teardown of closed servers// is noise, and a child left running would be a daemon nothing manages.- if (restarting) {- const child = handoffChild // null once the handoff was confirmed — that daemon stays up+ if (updateHandoff.restarting()) {+ const child = updateHandoff.child() // null once the handoff was confirmed — that daemon stays upif (child?.pid) {console.log(`[cli] ${signal} during an update handoff — stopping the new daemon (pid ${child.pid}) too`)try { process.kill(child.pid, 'SIGTERM') } catch { /* ignore */ }diff --git a/cli/src/config/env.ts b/cli/src/config/env.tsindex bf4437a66..646c94745 100644--- a/cli/src/config/env.ts+++ b/cli/src/config/env.ts@@ -347,6 +347,10 @@ const envSchema = z.object({// keeping the update handoff 30s away from that is what stops the two from fighting over the spawn// lock. Set to a negative number to keep the plain interval.ADAPTER_UPDATE_SLOT_SEC: z.string().default('45').transform(Number),+ // How long an update's download may go without a byte, and take in all, before the check gives up and+ // the next one tries again (lib/selfUpdate.ts `TransferLimits`). The manifest gets at most a minute.+ ADAPTER_UPDATE_IDLE_MS: z.string().default('60000').transform(Number),+ ADAPTER_UPDATE_DEADLINE_MS: z.string().default('900000').transform(Number),// Set 'true' to disable self-update entirely.ADAPTER_UPDATE_DISABLE: z.string().default('false').transform((v) => v === 'true'),/** How long a daemon whose start-up failed stays up serving nothing but its updater, before itdiff --git a/cli/src/core/turns/recaps.spec.ts b/cli/src/core/turns/recaps.spec.tsindex 9f09bab7d..d18a851fc 100644--- a/cli/src/core/turns/recaps.spec.ts+++ b/cli/src/core/turns/recaps.spec.ts@@ -59,6 +59,12 @@ describe('recaps', () => {expect(role).toHaveBeenCalledWith('a1')})+ it('takes an agent whose role cannot be read for anyone\'s: the turn keeps its end (e2e/diskfull.e2e.ts)', () => {+ const { deps, recaps } = setup()+ vi.mocked(deps.orchestratorRoleOf).mockImplementationOnce(() => { throw new Error('ENOSPC: no space left on device, mkdir') })+ expect(recaps.isSubagentSession('s1')).toBe(false)+ })+it('gives the mirror the clients, the device gates and an excerpt for a recap', async () => {const { deps, opts } = setup()vi.mocked(deps.turnActivity.snapshot).mockReturnValueOnce({ state: 'working' } as ActivityFrame).mockReturnValueOnce({ state: 'idle' } as ActivityFrame)diff --git a/cli/src/core/turns/recaps.ts b/cli/src/core/turns/recaps.tsindex 10514dfbc..f232226a0 100644--- a/cli/src/core/turns/recaps.ts+++ b/cli/src/core/turns/recaps.ts@@ -64,7 +64,10 @@ export function createRecaps({const isSubagentSession = (sessionId: string): boolean => {const agentId = bySession(sessionId)?.agentIdif (!agentId) return false- const role = orchestratorRoleOf(agentId)+ // Asked at every turn's end, from the transcript's line handler. Reading the orchestrator makes its+ // folder, which throws on a full disk (e2e/diskfull.e2e.ts): announced as anyone's, the turn keeps its end.+ let role: ReturnType<typeof orchestratorRoleOf>+ try { role = orchestratorRoleOf(agentId) } catch { return false }return role?.role === 'worker' || (role?.role === 'director' && role.busy)}const mirror = new CommanderMirror({diff --git a/cli/src/core/updateHandoff.spec.ts b/cli/src/core/updateHandoff.spec.tsnew file mode 100644index 000000000..2319d1b19--- /dev/null+++ b/cli/src/core/updateHandoff.spec.ts@@ -0,0 +1,92 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import { TEARDOWN_DEADLINE_MS, createUpdateHandoff, type HandoffChild, type TeardownStep, type UpdateHandoffDeps } from './updateHandoff.js'++describe('the update handoff', () => {+ afterEach(() => { vi.useRealTimers() })++ const make = (over: Partial<UpdateHandoffDeps> = {}) => {+ const calls: string[] = []+ const tracked: Array<HandoffChild | null> = []+ const handoff = createUpdateHandoff({+ version: '1.0.0', supervised: true,+ exitForUpdate: () => calls.push('exit 75'),+ handOff: async (version, track) => {+ calls.push(`hand off ${version}`)+ track({ pid: 42 })+ tracked.push(handoff.child())+ track(null)+ tracked.push(handoff.child())+ },+ log: (line) => calls.push(line),+ error: (line) => calls.push(line),+ ...over,+ })+ return { handoff, calls, tracked }+ }+ const step = (calls: string[], name: string, release: () => unknown = () => {}): TeardownStep => [name, () => { calls.push(`release ${name}`); return release() }]++ it('under harnessd, releases everything in order and exits for the update, once', async () => {+ const { handoff, calls } = make()+ expect(handoff.restarting()).toBe(false)+ const teardown = [step(calls, 'the registry'), step(calls, 'the backend', () => Promise.resolve())]+ const first = handoff.restartForUpdate('2.0.0', teardown)+ expect(handoff.restarting()).toBe(true)+ await handoff.restartForUpdate('2.0.0', teardown)+ await first+ expect(calls).toEqual([+ '[update] applying 1.0.0 → 2.0.0 — restarting daemon', 'release the registry', 'release the backend',+ '[update] handing 2.0.0 to harnessd', 'exit 75',+ ])+ })++ it('under harnessd, tries every step when one throws, and exits for the update all the same (round 40)', async () => {+ const { handoff, calls } = make()+ await handoff.restartForUpdate('2.0.0', [+ step(calls, 'the timers', () => { throw new Error('a teardown step that throws') }),+ step(calls, 'the hook server', () => Promise.reject('odd')),+ step(calls, 'the backend'),+ ])+ expect(calls).toEqual([+ '[update] applying 1.0.0 → 2.0.0 — restarting daemon',+ 'release the timers', '[update] the timers did not let go (a teardown step that throws) — handing over all the same',+ 'release the hook server', '[update] the hook server did not let go (odd) — handing over all the same',+ 'release the backend', '[update] handing 2.0.0 to harnessd', 'exit 75',+ ])+ })++ it('under harnessd, hands over all the same when a step hangs past the deadline', async () => {+ vi.useFakeTimers()+ const { handoff, calls } = make({ teardownDeadlineMs: 5_000 })+ const done = handoff.restartForUpdate('2.0.0', [step(calls, 'the registry'), step(calls, 'the backend', () => new Promise(() => {}))])+ await vi.advanceTimersByTimeAsync(4_999)+ expect(calls).not.toContain('exit 75')+ await vi.advanceTimersByTimeAsync(1)+ await done+ expect(calls.slice(-3)).toEqual([+ '[update] the teardown did not finish within 5000 ms (at the backend) — handing over all the same', '[update] handing 2.0.0 to harnessd', 'exit 75',+ ])+ expect(TEARDOWN_DEADLINE_MS).toBe(15_000)+ const plain = make()+ vi.useRealTimers()+ await plain.handoff.restartForUpdate('2.0.0', [])+ expect(plain.calls.at(-1)).toBe('exit 75')+ })++ it('without a master, releases in order, hands off, and tracks the successor until it is the daemon', async () => {+ const { handoff, calls, tracked } = make({ supervised: false })+ expect(handoff.child()).toBeNull()+ await handoff.restartForUpdate('2.0.0', [step(calls, 'the registry'), step(calls, 'the backend')])+ expect(calls).toEqual(['[update] applying 1.0.0 → 2.0.0 — restarting daemon', 'release the registry', 'release the backend', 'hand off 2.0.0'])+ expect(tracked).toEqual([{ pid: 42 }, null])+ })++ it('without a master, stops at a step that fails, hands nothing over, and lets this core carry on once abandoned', async () => {+ const { handoff, calls } = make({ supervised: false })+ await expect(handoff.restartForUpdate('2.0.0', [step(calls, 'the timers', () => { throw new Error('no') }), step(calls, 'the backend')])).rejects.toThrow('no')+ expect(calls).toEqual(['[update] applying 1.0.0 → 2.0.0 — restarting daemon', 'release the timers'])+ expect(handoff.restarting()).toBe(true)+ handoff.abandon()+ expect(handoff.restarting()).toBe(false)+ expect(handoff.child()).toBeNull()+ })+})diff --git a/cli/src/core/updateHandoff.ts b/cli/src/core/updateHandoff.tsnew file mode 100644index 000000000..1a8075ac9--- /dev/null+++ b/cli/src/core/updateHandoff.ts@@ -0,0 +1,110 @@+/**+ * The core handing the machine to a newer build its updater has just staged (`selfUpdate.ts`+ * `startSelfUpdater`'s `onStaged`), once start-up has finished: release what the next core needs (the+ * fixed ports, the backend's one-machine claim, the watchers and timers), then go.+ *+ * Under harnessd the master starts the new bundle the moment this core exits 75, and judges it. A+ * teardown step that threw used to end the handoff where it stood: the core stayed on the old build with+ * its servers half closed, its updater stopped and the new bundle staged but never judged+ * (e2e/updateHostile.e2e.ts, round 40). Now every step is tried, one that fails is said and passed, and+ * the core exits 75 whatever happened; one that hangs is given up on after `TEARDOWN_DEADLINE_MS`.+ *+ * Without a master (a core run on its own) the old way stands: the first step that fails ends the+ * handoff, and `handOff` spawns the successor and judges it itself.+ *+ * Staged means restart now. The restart once waited for the computer to go idle, and "idle" is a set of+ * latches (an open turn, a settling composer, an awaited submit, the control lock, a recap in flight):+ * one stuck latch deferred it for ever (0.0.26 on 2026-07-31, eight minutes of "deferring restart"), and+ * a daemon that quietly never updates is the failure the updater exists to prevent. A turn streaming at+ * that moment goes on in its pane, and the new core picks it up at attach and reads how it ends.+ *+ * Moved out of `runForeground` (src/architecture.spec.ts).+ */++/** One thing the next core needs released, named for the log. */+export type TeardownStep = readonly [name: string, release: () => unknown]++/** The successor a handoff without a master spawned, which a signal must take down with this core. */+export interface HandoffChild { pid?: number }++/** How long the teardown may take before the core hands over all the same: it takes about a second. */+export const TEARDOWN_DEADLINE_MS = 15_000++export interface UpdateHandoffDeps {+ /** This core's version, for the log. */+ version: string+ /** A harnessd master runs this core. */+ supervised: boolean+ /** Exit for the update (`CORE_EXIT_UPDATE`), to the master that starts the new bundle. */+ exitForUpdate(): void+ /**+ * The handoff without a master: spawn the successor on the staged bundle, keep it or roll back.+ * `track` names the successor a signal must take down with this core, and null once it is the daemon.+ */+ handOff(newVersion: string, track: (child: HandoffChild | null) => void): Promise<void>+ log(line: string): void+ error(line: string): void+ teardownDeadlineMs?: number+}++export function createUpdateHandoff(deps: UpdateHandoffDeps) {+ let restarting = false+ let child: HandoffChild | null = null+ const deadlineMs = deps.teardownDeadlineMs ?? TEARDOWN_DEADLINE_MS+ const message = (error: unknown): string => error instanceof Error ? error.message : String(error)++ /** Every step, in order, each failure said and passed; given up on, all the same, after the deadline. */+ const releaseAll = async (teardown: readonly TeardownStep[]): Promise<void> => {+ let at = ''+ const steps = (async () => {+ for (const [name, release] of teardown) {+ at = name+ try { await release() } catch (error) { deps.error(`[update] ${name} did not let go (${message(error)}) — handing over all the same`) }+ }+ at = ''+ })()+ let timer: ReturnType<typeof setTimeout> | undefined+ const late = new Promise<void>((resolve) => {+ timer = setTimeout(() => {+ deps.error(`[update] the teardown did not finish within ${deadlineMs} ms (at ${at}) — handing over all the same`)+ resolve()+ }, deadlineMs)+ })+ await Promise.race([steps, late])+ clearTimeout(timer)+ }++ return {+ /** Between an update being staged and this core leaving (`/api/status`, and a signal mid-handoff). */+ restarting: (): boolean => restarting,+ /** The successor a handoff without a master is judging, until it is the daemon. */+ child: (): HandoffChild | null => child,+ /**+ * Hand over to `newVersion`, releasing `teardown` first. Once: a second call while one is under way+ * does nothing. Rejects only without a master, when a step failed or the successor could not be+ * started; then `abandon` lets this core carry on.+ */+ async restartForUpdate(newVersion: string, teardown: readonly TeardownStep[]): Promise<void> {+ if (restarting) return+ restarting = true+ deps.log(`[update] applying ${deps.version} → ${newVersion} — restarting daemon`)+ if (deps.supervised) {+ await releaseAll(teardown)+ // Everything above is released, or said not to be; the master starts the new bundle as soon as+ // this exits and rolls back to the .prev bytes if it does not come up and stay up.+ deps.log(`[update] handing ${newVersion} to harnessd`)+ deps.exitForUpdate()+ return+ }+ for (const [, release] of teardown) await release()+ await deps.handOff(newVersion, (successor) => { child = successor })+ },+ /** A handoff without a master that failed before it handed anything over: this core stays. */+ abandon(): void {+ restarting = false+ child = null+ },+ }+}++export type UpdateHandoff = ReturnType<typeof createUpdateHandoff>diff --git a/cli/src/harnessd/supervisor.spec.ts b/cli/src/harnessd/supervisor.spec.tsindex 99b9b5eae..1f1c39ae7 100644--- a/cli/src/harnessd/supervisor.spec.ts+++ b/cli/src/harnessd/supervisor.spec.ts@@ -1,7 +1,7 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import { CORE_EXIT_STOP, CORE_EXIT_UPDATE, HARNESSD_PROTOCOL, type MasterMessage } from './protocol.js'import {- DEFAULT_SUPERVISOR_OPTIONS, Supervisor, type CoreHandle, type ReexecOutcome, type ResumeState, type SupervisorDeps, type SupervisorOptions,+ DEFAULT_SUPERVISOR_OPTIONS, Supervisor, namesTheDisk, type CoreHandle, type ReexecOutcome, type ResumeState, type SupervisorDeps, type SupervisorOptions,type SupervisorStatus,} from './supervisor.js'@@ -482,6 +482,66 @@ describe('Supervisor', () => {expect(calls).toEqual(['claim', 'restore'])})+ it('takes a core on the new bundle that has no room on the disk to start for no verdict: the update stays on trial, tried again, backing off', () => {+ const supervisor = make()+ supervisor.start()+ core().up()+ core().exit(CORE_EXIT_UPDATE)+ vi.advanceTimersByTime(0)+ // Its socket's claim could not be written: the disk is full, and the build before would fail too.+ core().bind()+ core().ready('listen ENOSPC: no space left on device /data/.claim-1')+ expect(core().kills).toEqual(['SIGKILL'])+ expect(lines.at(-1)).toBe('[harnessd] core could not start on the new bundle for want of room on the disk (listen ENOSPC: no space left on device /data/.claim-1) — not a verdict on the build — killing it')+ core().exit(null, 'SIGKILL')+ expect(calls).toEqual(['claim'])+ expect(lines.at(-1)).toBe(`[harnessd] the updated core had no room to start (signal SIGKILL) — keeping the update on trial; trying again in ${options.initialBackoffMs} ms`)+ expect(supervisor.status()).toMatchObject({ state: 'restarting', lastExitReason: 'not-ready', safeMode: null })+ vi.advanceTimersByTime(options.initialBackoffMs - 1)+ expect(cores).toHaveLength(2)+ vi.advanceTimersByTime(1)+ expect(cores).toHaveLength(3)+ // Not a crash: no safe mode for the next, however many times the disk is still full.+ expect(core().env.HARNESSD_SAFE_MODE).toBeUndefined()+ core().bind()+ core().ready('EDQUOT: disk quota exceeded, write')+ core().exit(null, 'SIGKILL')+ expect(lines.at(-1)).toContain(`trying again in ${options.initialBackoffMs * 2} ms`)+ vi.advanceTimersByTime(options.initialBackoffMs * 2)+ core().bind()+ core().ready('EROFS: read-only file system, open')+ core().exit(null, 'SIGKILL')+ vi.advanceTimersByTime(options.initialBackoffMs * 4)+ expect(cores).toHaveLength(5)+ expect(core().env.HARNESSD_SAFE_MODE).toBeUndefined()+ // Room again: it comes up, and is judged on the update from ready, as any update is.+ core().up()+ vi.advanceTimersByTime(options.updateProbationMs)+ expect(calls).toEqual(['claim', 'confirm'])+ })++ it('still rolls back an update whose core fails on the disk once it was up, or for any other reason', () => {+ const supervisor = make()+ supervisor.start()+ core().up()+ core().exit(CORE_EXIT_UPDATE)+ vi.advanceTimersByTime(0)+ core().up()+ // Up and on probation, then gone: whatever it says, the build did not stay up.+ core().exit(1)+ expect(calls).toEqual(['claim', 'restore'])+ expect(supervisor.status().state).toBe('restarting')+ })++ it('knows a failure that names the disk from one that names the build', () => {+ for (const disk of ['listen ENOSPC: no space left on device /d/.claim-1', 'ENOSPC: no space left on device, write', 'EDQUOT: disk quota exceeded', 'EROFS: read-only file system, mkdir \'/d\'']) {+ expect(namesTheDisk(disk), disk).toBe(true)+ }+ for (const build of ['start-up failed', 'Cannot find module \'./x.js\'', 'SyntaxError: Unexpected token', 'listen EADDRINUSE: address already in use', 'spaces left over']) {+ expect(namesTheDisk(build), build).toBe(false)+ }+ })+it('judges the newer build a core on probation staged, instead of rolling it back as a failure', () => {let disk = 'v1'const supervisor = make({}, { bundle: () => disk })diff --git a/cli/src/harnessd/supervisor.ts b/cli/src/harnessd/supervisor.tsindex 15b85d1ca..dc920b652 100644--- a/cli/src/harnessd/supervisor.ts+++ b/cli/src/harnessd/supervisor.ts@@ -176,6 +176,15 @@ type TimerName = 'bindTimer' | 'readyTimer' | 'heartbeatTimer' | 'killTimer' | 'const describeExit = (code: number | null, signal: NodeJS.Signals | null): string =>signal ? `signal ${signal}` : `code ${code}`++/**+ * Whether why a core could not start names the disk rather than the build: no space left, a quota+ * spent, a file system mounted read-only. Read off the reason a core gives for its safe mode, which is+ * the error it failed on (`listen ENOSPC: no space left on device …` for its socket's claim).+ */+export function namesTheDisk(reason: string): boolean {+ return /\b(ENOSPC|EDQUOT|EROFS)\b|no space left on device|disk quota exceeded|read-only file system/i.test(reason)+}const MIB = 1024 * 1024export class Supervisor {@@ -195,6 +204,8 @@ export class Supervisor {private ending: 'stop' | 'restart' | null = null/** Why the master killed the running core, when it did. */private killReason: ExitReason | null = null+ /** The core on an update could not start for want of room on the disk: its end is no verdict. */+ private noRoom = false/** A core exited for an update: the next one runs the new bundle, on probation until it proves it. */private update: 'pending' | 'probation' | null = null/** Crashes inside the crash-loop window, on the monotonic clock. */@@ -363,6 +374,15 @@ export class Supervisor {case 'harnessd:ready':if (!this.bound || this.state !== 'listening') returnthis.clearTimer('readyTimer')+ if (message.safeMode !== undefined && this.update && namesTheDisk(message.safeMode)) {+ // A disk too full to start on says nothing of the build: the one before would fail the same+ // way, and a rollback that frees the room only for the updater to stage the build again is a+ // loop. Before, the good build was rolled back and remembered as bad (e2e/updateHostile.e2e.ts).+ // Started again, on the same bundle and still on trial, once there may be room.+ this.noRoom = true+ this.kill(core, 'not-ready', `could not start on the new bundle for want of room on the disk (${message.safeMode}) — not a verdict on the build`)+ return+ }if (message.safeMode !== undefined && this.update) {// The new bundle could not start: that is the update failing, however long it stays up.this.kill(core, 'crashed', `started in safe mode on the new bundle (${message.safeMode})`)@@ -437,6 +457,17 @@ export class Supervisor {// that found it while the one before was on probation (e2e/updateHostile.e2e.ts: a fix published// moments after the release it fixes).const superseded = this.update !== null && reason === 'update' && this.stagedSinceSpawn()+ const noRoom = this.noRoom+ this.noRoom = false+ if (this.update && noRoom) {+ // Still on trial: the next core on this bundle is judged on it, from ready. Backed off like a+ // crash, without counting as one: safe mode would not make room either.+ const delay = this.backoff+ this.backoff = Math.min(this.backoff * 2, this.options.maxBackoffMs)+ this.deps.log(`[harnessd] the updated core had no room to start (${exit}) — keeping the update on trial; trying again in ${delay} ms`)+ this.scheduleSpawn(delay)+ return+ }if (this.update && !superseded) {// The core on the new bundle did not come up, or did not stay up: the bundle before it did.this.update = nulldiff --git a/cli/src/lib/runtimeInstall.spec.ts b/cli/src/lib/runtimeInstall.spec.tsindex 7552fbd55..ca276a008 100644--- a/cli/src/lib/runtimeInstall.spec.ts+++ b/cli/src/lib/runtimeInstall.spec.ts@@ -137,6 +137,24 @@ describe('ensureManagedRuntime', () => {expect(readdirSync(runtimeDir).filter((n) => n.startsWith('.node-staging-'))).toEqual([])})+ it('downloads a runtime with no deadline, only a silence of five minutes: a slow link must be able to finish it', async () => {+ // The bundle's own limits (a quarter of an hour in all) would cut a runtime of tens of MB off on a+ // slow link, and every daemon runs on it (lib/selfUpdate.ts RUNTIME_DOWNLOAD_LIMITS).+ const key = currentPlatformKey()+ const { bytes, root: archiveRoot } = buildArchive('v22.23.3', key)+ stubFetch({ node: { [key]: { version: 'v22.23.3', url: 'https://example.test/runtime/node.tar.gz', sha256: createHash('sha256').update(bytes).digest('hex'), size: bytes.length, archiveRoot } } }, bytes)+ const limits: unknown[] = []+ vi.doMock('./selfUpdate.js', async (importOriginal) => {+ const actual = await importOriginal<typeof import('./selfUpdate.js')>()+ return { ...actual, downloadVerified: (ref: Parameters<typeof actual.downloadVerified>[0], given?: Parameters<typeof actual.downloadVerified>[1]) => { limits.push(given); return actual.downloadVerified(ref, given) } }+ })+ try {+ const { ensureManagedRuntime } = await load()+ expect(await ensureManagedRuntime()).toBe(join(runtimeDir, `node-v22.23.3-${key}`, 'bin', 'node'))+ expect(limits).toEqual([{ idleMs: 300_000, deadlineMs: Number.POSITIVE_INFINITY }])+ } finally { vi.doUnmock('./selfUpdate.js') }+ })+it('refuses an archive whose bytes do not match the manifest', async () => {const key = currentPlatformKey()const { bytes, root: archiveRoot } = buildArchive('v22.23.2', key)diff --git a/cli/src/lib/runtimeInstall.ts b/cli/src/lib/runtimeInstall.tsindex c5d56db82..049980560 100644--- a/cli/src/lib/runtimeInstall.ts+++ b/cli/src/lib/runtimeInstall.ts@@ -5,7 +5,7 @@ import { dirname, join, sep } from 'node:path'import { env } from '../config/env.js'import { gridChildEnv, managedGridPath, meetsVersionFloor } from './gridExec.js'import { managedNodePath } from './nodeRuntime.js'-import { downloadVerified } from './selfUpdate.js'+import { RUNTIME_DOWNLOAD_LIMITS, downloadVerified } from './selfUpdate.js'/*** Provisioning the Node the CLI runs on, from inside the CLI.@@ -123,7 +123,7 @@ async function ensureManagedArchive(spec: ManagedArchive): Promise<string | null// downloadVerified checks the sha256 but not the length, so check it here: a truncated body// that somehow collided would be caught by the hash anyway, but a mismatch here is the cheaper// and clearer failure.- const bytes = await downloadVerified(artifact)+ const bytes = await downloadVerified(artifact, RUNTIME_DOWNLOAD_LIMITS)if (artifact.size !== undefined && bytes.length !== artifact.size) return installedmkdirSync(env.ADAPTER_RUNTIME_DIR, { recursive: true, mode: 0o700 })diff --git a/cli/src/lib/selfUpdate.spec.ts b/cli/src/lib/selfUpdate.spec.tsindex f003eec8e..cafe20b7c 100644--- a/cli/src/lib/selfUpdate.spec.ts+++ b/cli/src/lib/selfUpdate.spec.ts@@ -4,8 +4,8 @@ import { join } from 'path'import { createHash } from 'crypto'import { afterEach, describe, expect, it, vi } from 'vitest'import {- REFUSED_RETRY_MAX_MS, canary, confirm, downloadVerified, fetchManifest, isLocalDevBuild, msUntilSlot, rejectedVersions, restore, runCanary, semverGt,- settleRolledBack, shouldAutoUpdate, stage, startSelfUpdater, unjudgedUpdate,+ DOWNLOAD_LIMITS, MANIFEST_LIMITS, REFUSED_RETRY_MAX_MS, RUNTIME_DOWNLOAD_LIMITS, TransferStalledError, canary, confirm, downloadVerified, fetchManifest, isLocalDevBuild,+ msUntilSlot, rejectedVersions, restore, runCanary, semverGt, settleRolledBack, shouldAutoUpdate, stage, startSelfUpdater, unjudgedUpdate,} from './selfUpdate.js'import { SpawnLockBusyError } from './daemonSpawnLock.js'@@ -213,7 +213,8 @@ describe('startSelfUpdater', () => {vi.useFakeTimers()try {vi.setSystemTime(new Date('2026-09-15T10:00:44.000Z'))- // The first manifest fetch hangs for two minutes (a stalled link); later ones answer at once.+ // The first manifest fetch outlasts the interval (a slow link, inside the manifest's own limits);+ // later ones answer at once.let release: () => void = () => {}const stalled = new Promise<Response>((resolve) => { release = () => resolve(new Response(JSON.stringify({ adapter: { version: '1.0.0' } }))) })const fetchMock = vi.fn()@@ -222,14 +223,14 @@ describe('startSelfUpdater', () => {vi.stubGlobal('fetch', fetchMock)const poller = startSelfUpdater({currentVersion: '1.0.0', url: 'https://updates.test/metadata.json', key: 'adapter', dir: tempDir(),- intervalMs: 60_000, slotSecond: 45, onStaged: () => {},+ intervalMs: 20_000, slotSecond: 45, onStaged: () => {},})- await vi.advanceTimersByTimeAsync(1_000) // :45 — the stalled check starts+ await vi.advanceTimersByTimeAsync(1_000) // :45 — the slow check startsexpect(fetchMock).toHaveBeenCalledTimes(1)- await vi.advanceTimersByTimeAsync(60_000) // next :45 — skipped, still checking+ await vi.advanceTimersByTimeAsync(20_000) // :05 — skipped, still checkingexpect(fetchMock).toHaveBeenCalledTimes(1)release()- await vi.advanceTimersByTimeAsync(60_000) // the :45 after that — the chain is still booked+ await vi.advanceTimersByTimeAsync(20_000) // :25 — the chain is still bookedexpect(fetchMock).toHaveBeenCalledTimes(2)poller.stop()} finally {@@ -799,3 +800,107 @@ describe('a rollback a full disk kept from being remembered (e2e/updateHostile.e} finally { poller.stop() }})})++describe('a link that stalls without dropping (e2e/updateHostile.e2e.ts)', () => {+ const sha = (b: Buffer): string => createHash('sha256').update(b).digest('hex')+ afterEach(() => {+ vi.unstubAllGlobals()+ vi.restoreAllMocks()+ for (const dir of dirs) rmSync(dir, { recursive: true, force: true })+ dirs = []+ })++ /** A body that sends `first`, then a byte every `everyMs` if asked, and never ends. */+ const stalling = (first: Buffer, everyMs?: number): ReadableStream<Uint8Array> => {+ let timer: NodeJS.Timeout | undefined+ return new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(new Uint8Array(first))+ if (everyMs) timer = setInterval(() => controller.enqueue(new Uint8Array([7])), everyMs)+ },+ cancel() { clearInterval(timer) },+ })+ }+ const limits = { idleMs: 60, deadlineMs: 400 }++ it('gives up on a download that goes quiet, one that trickles past its deadline, and one whose answer never starts', async () => {+ vi.stubGlobal('fetch', async (url: string) => {+ if (url.endsWith('/quiet.js')) return new Response(stalling(Buffer.from('half of it')))+ if (url.endsWith('/trickle.js')) return new Response(stalling(Buffer.from('a'), 20))+ return new Promise<Response>(() => {})+ })+ const started = Date.now()+ await expect(downloadVerified({ url: 'https://updates.test/quiet.js', sha256: 'x' }, limits))+ .rejects.toThrow(new TransferStalledError('https://updates.test/quiet.js sent nothing for 60 ms'))+ await expect(downloadVerified({ url: 'https://updates.test/trickle.js', sha256: 'x' }, limits))+ .rejects.toThrow('https://updates.test/trickle.js took longer than 400 ms in all')+ await expect(downloadVerified({ url: 'https://updates.test/silent.js', sha256: 'x' }, limits))+ .rejects.toBeInstanceOf(TransferStalledError)+ await expect(fetchManifest('https://updates.test/silent.json', 'adapter', limits)).rejects.toBeInstanceOf(TransferStalledError)+ expect(Date.now() - started).toBeLessThan(5_000)+ })++ it('takes a whole answer that keeps coming, however slowly, within its deadline, and one with no body', async () => {+ const bytes = Buffer.from('console.log("9.9.9")\n')+ vi.stubGlobal('fetch', async (url: string) => {+ if (url.endsWith('/empty.js')) return new Response(null, { status: 200 })+ let sent = 0+ return new Response(new ReadableStream<Uint8Array>({+ async pull(controller) {+ await new Promise((resolve) => setTimeout(resolve, 30))+ if (sent >= bytes.length) { controller.close(); return }+ controller.enqueue(new Uint8Array(bytes.subarray(sent, sent + 4)))+ sent += 4+ },+ }))+ })+ await expect(downloadVerified({ url: 'https://updates.test/slow.js', sha256: sha(bytes) }, { idleMs: 200, deadlineMs: 5_000 })).resolves.toEqual(bytes)+ await expect(downloadVerified({ url: 'https://updates.test/empty.js', sha256: sha(Buffer.alloc(0)) }, limits)).resolves.toEqual(Buffer.alloc(0))+ })++ it('reads a response with no stream whole, and sets no deadline for an infinite one', async () => {+ const bytes = Buffer.from('console.log("9.9.9")\n')+ vi.stubGlobal('fetch', async (url: string) => url.endsWith('/buffered.js')+ ? { ok: true, status: 200, body: null, arrayBuffer: async () => bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) } as unknown as Response+ : url.endsWith('/gone.js') ? { ok: false, status: 404, body: null } as unknown as Response+ : new Response(new ReadableStream<Uint8Array>({+ async pull(controller) { await new Promise((resolve) => setTimeout(resolve, 40)); controller.enqueue(new Uint8Array(bytes)); controller.close() },+ })))+ await expect(downloadVerified({ url: 'https://updates.test/buffered.js', sha256: sha(bytes) }, limits)).resolves.toEqual(bytes)+ await expect(downloadVerified({ url: 'https://updates.test/gone.js', sha256: 'x' }, limits)).rejects.toThrow('HTTP 404')+ // A timer given Infinity fires at once: an infinite deadline must be no deadline at all.+ await expect(downloadVerified({ url: 'https://updates.test/slow.js', sha256: sha(bytes) }, RUNTIME_DOWNLOAD_LIMITS)).resolves.toEqual(bytes)+ expect(RUNTIME_DOWNLOAD_LIMITS).toEqual({ idleMs: 300_000, deadlineMs: Number.POSITIVE_INFINITY })+ })++ it('is a failed check, tried again at the next one, with the limits it was given; the manifest gets at most its own', async () => {+ const cli = Buffer.from('console.log("9.9.9")\n')+ const notify = Buffer.from('export {}\n')+ let attempts = 0+ vi.stubGlobal('fetch', async (url: string) => {+ if (url.endsWith('/metadata.json')) {+ return new Response(JSON.stringify({ adapter: {+ version: '9.9.9',+ cli: { url: 'https://updates.test/cli.js', sha256: sha(cli) },+ notify: { url: 'https://updates.test/notify.mjs', sha256: sha(notify) },+ } }))+ }+ if (url.endsWith('cli.js') && ++attempts === 1) return new Response(stalling(cli.subarray(0, 4)))+ return new Response(url.endsWith('cli.js') ? cli : notify)+ })+ const errors: string[] = []+ vi.spyOn(console, 'log').mockImplementation(() => {})+ vi.spyOn(console, 'error').mockImplementation((...parts: unknown[]) => { errors.push(parts.join(' ')) })+ const staged: string[] = []+ const poller = startSelfUpdater({+ currentVersion: '1.0.0', url: 'https://updates.test/metadata.json', key: 'adapter', dir: tempDir(), intervalMs: 10,+ limits: { idleMs: 50, deadlineMs: 120_000 }, onStaged: (version) => { staged.push(version) },+ })+ try {+ await vi.waitFor(() => expect(staged).toEqual(['9.9.9']), { timeout: 5_000 })+ expect(attempts).toBe(2)+ expect(errors).toEqual(['[update] check failed (will retry): https://updates.test/cli.js sent nothing for 50 ms'])+ } finally { poller.stop() }+ expect(MANIFEST_LIMITS.deadlineMs).toBeLessThan(DOWNLOAD_LIMITS.deadlineMs)+ })+})diff --git a/cli/src/lib/selfUpdate.ts b/cli/src/lib/selfUpdate.tsindex c983cf4da..95ba5f614 100644--- a/cli/src/lib/selfUpdate.ts+++ b/cli/src/lib/selfUpdate.ts@@ -134,12 +134,79 @@ export function shouldAutoUpdate(candidate: string, current: string): boolean {return semverGt(candidate, current)}+/**+ * How long a transfer may go without a byte (`idleMs`) and take in all (`deadlineMs`). A link that+ * stalls without dropping (a lid closed, a network changed, a proxy that holds the connection) answered+ * nothing and closed nothing: the check waited on undici's own five-minute body timeout, and a trickle+ * held it for as long as the trickle lasted, every later check skipped meanwhile, with a fix waiting+ * (e2e/updateHostile.e2e.ts). Each is now its own failure, retried at the next check.+ */+export interface TransferLimits { idleMs: number; deadlineMs: number }+/** A bundle: a minute without a byte, a quarter of an hour in all (4.4 MB at 5 KB/s). */+export const DOWNLOAD_LIMITS: TransferLimits = { idleMs: 60_000, deadlineMs: 15 * 60_000 }+/** The manifest is a few hundred bytes. */+export const MANIFEST_LIMITS: TransferLimits = { idleMs: 30_000, deadlineMs: 60_000 }+/**+ * A managed runtime (lib/runtimeInstall.ts): tens of MB every daemon runs on, which a slow link must be+ * able to finish however long it takes, so no deadline; five minutes without a byte, undici's own body+ * timeout and what these downloads always had.+ */+export const RUNTIME_DOWNLOAD_LIMITS: TransferLimits = { idleMs: 300_000, deadlineMs: Number.POSITIVE_INFINITY }++/** The transfer went quiet for too long, or took too long in all. */+export class TransferStalledError extends Error {}++/** GET `url` whole within `limits`: the status, and the body once it has all arrived. */+async function fetchWithin(url: string, limits: TransferLimits): Promise<{ ok: boolean; status: number; body: Buffer }> {+ const controller = new AbortController()+ let stalled!: (error: TransferStalledError) => void+ const gaveUp = new Promise<never>((_, reject) => { stalled = reject })+ gaveUp.catch(() => {})+ let reader: ReadableStreamDefaultReader<Uint8Array> | null = null+ const giveUp = (why: string): void => {+ stalled(new TransferStalledError(`${url} ${why}`))+ controller.abort()+ void reader?.cancel().catch(() => {})+ }+ const timer = (ms: number, why: string): NodeJS.Timeout => {+ const handle = setTimeout(() => giveUp(why), ms)+ handle.unref?.()+ return handle+ }+ // No deadline at all for an infinite one: a timer given Infinity fires at once.+ const deadline = Number.isFinite(limits.deadlineMs) ? timer(limits.deadlineMs, `took longer than ${limits.deadlineMs} ms in all`) : undefined+ let idle = timer(limits.idleMs, `sent nothing for ${limits.idleMs} ms`)+ const heard = (): void => { clearTimeout(idle); idle = timer(limits.idleMs, `sent nothing for ${limits.idleMs} ms`) }+ try {+ const res = await Promise.race([fetch(url, { signal: controller.signal }), gaveUp])+ heard()+ // Read as it arrives, so the idle timer hears every chunk. A response with no stream (a body-less+ // status, or a Response-like object that only buffers) is read whole, within the same limits.+ if (!res.body) {+ const whole = res.ok && typeof res.arrayBuffer === 'function' ? Buffer.from(await Promise.race([res.arrayBuffer(), gaveUp])) : Buffer.alloc(0)+ return { ok: res.ok, status: res.status, body: whole }+ }+ const chunks: Buffer[] = []+ reader = res.body.getReader()+ for (;;) {+ const { done, value } = await Promise.race([reader.read(), gaveUp])+ if (done) break+ chunks.push(Buffer.from(value))+ heard()+ }+ return { ok: res.ok, status: res.status, body: Buffer.concat(chunks) }+ } finally {+ clearTimeout(deadline)+ clearTimeout(idle)+ }+}+/** Fetch + parse the manifest; return this adapter's entry, or null (unreachable / malformed / absent). */-export async function fetchManifest(url: string, key: string): Promise<UpdateEntry | null> {+export async function fetchManifest(url: string, key: string, limits: TransferLimits = MANIFEST_LIMITS): Promise<UpdateEntry | null> {// No `cache` option needed: undici doesn't HTTP-cache by default and GCS serves the manifest no-cache.- const res = await fetch(url)+ const res = await fetchWithin(url, limits)if (!res.ok) return null- const json = (await res.json()) as Record<string, unknown>+ const json = JSON.parse(res.body.toString('utf8')) as Record<string, unknown>const entry = json?.[key] as Partial<UpdateEntry> | undefinedconst okFile = (f: unknown): f is FileRef =>!!f && typeof (f as FileRef).url === 'string' && typeof (f as FileRef).sha256 === 'string'@@ -150,16 +217,16 @@ export async function fetchManifest(url: string, key: string): Promise<UpdateEnt/** The whole download arrived, and it is not what the manifest names. */export class DigestMismatchError extends Error {}-/** Download one file and verify its sha256 in memory; throws on a non-2xx or a digest mismatch. */-export async function downloadVerified(ref: FileRef): Promise<Buffer> {- const res = await fetch(ref.url)+/** Download one file within `limits` and verify its sha256 in memory; throws on a non-2xx, a stall or a+ * digest mismatch. */+export async function downloadVerified(ref: FileRef, limits: TransferLimits = DOWNLOAD_LIMITS): Promise<Buffer> {+ const res = await fetchWithin(ref.url, limits)if (!res.ok) throw new Error(`download ${ref.url} → HTTP ${res.status}`)- const buf = Buffer.from(await res.arrayBuffer())- const got = sha256(buf)+ const got = sha256(res.body)if (got.toLowerCase() !== ref.sha256.toLowerCase()) {throw new DigestMismatchError(`sha256 mismatch for ${ref.url}: expected ${ref.sha256}, got ${got}`)}- return buf+ return res.body}/**@@ -383,6 +450,8 @@ export function startSelfUpdater(opts: {/** Wrap the swap + `onStaged` in a mutual exclusion with every other process that writes the* bundle or spawns the daemon. Default: none (tests, and callers that hold their own). */withLock?: <T>(fn: () => Promise<T>) => Promise<T>+ /** How long a download may go without a byte, and take in all; the manifest gets at most its own. */+ limits?: TransferLimits}): Poller {let checking = falselet done = false@@ -399,6 +468,10 @@ export function startSelfUpdater(opts: {let toldRejected: string | null = nullconst now = opts.now ?? Date.now+ const download = opts.limits ?? DOWNLOAD_LIMITS+ const manifest: TransferLimits = {+ idleMs: Math.min(download.idleMs, MANIFEST_LIMITS.idleMs), deadlineMs: Math.min(download.deadlineMs, MANIFEST_LIMITS.deadlineMs),+ }const stop = (): void => { if (timer) { clearTimeout(timer); timer = null } }const schedule = (): void => {stop()@@ -416,7 +489,7 @@ export function startSelfUpdater(opts: {if (checking) returnchecking = truetry {- const entry = await fetchManifest(opts.url, opts.key)+ const entry = await fetchManifest(opts.url, opts.key, manifest)if (!entry || !shouldAutoUpdate(entry.version, opts.currentVersion)) returnconst unremembered = settleRolledBack(opts.dir)if (unremembered === entry.version || rejectedVersions(opts.dir).includes(entry.version)) {@@ -439,8 +512,8 @@ export function startSelfUpdater(opts: {let cliBuf: Bufferlet notifyBuf: Buffertry {- cliBuf = await downloadVerified(entry.cli)- notifyBuf = await downloadVerified(entry.notify)+ cliBuf = await downloadVerified(entry.cli, download)+ notifyBuf = await downloadVerified(entry.notify, download)} catch (error) {if (!(error instanceof DigestMismatchError)) throw errorrefuse(`does not match its manifest (${error.message})`)diff --git a/cli/src/startupOrder.spec.ts b/cli/src/startupOrder.spec.tsindex 62ad51e36..febcbd955 100644--- a/cli/src/startupOrder.spec.ts+++ b/cli/src/startupOrder.spec.ts@@ -67,6 +67,18 @@ function callArgument(source: string, call: string): { body: string; at: numberthrow new Error(`unbalanced braces after ${call}`)}+/** A call and its arguments, by parenthesis balance: what it is handed, whatever brackets they use. */+function callArguments(source: string, call: string): { text: string; at: number } {+ const at = source.indexOf(call)+ expect(at, `cli.ts still contains \`${call}\``).toBeGreaterThan(-1)+ let depth = 0+ for (let i = at + call.length - 1; i < source.length; i++) {+ if (source[i] === '(') depth+++ else if (source[i] === ')' && --depth === 0) return { text: source.slice(at, i + 1), at }+ }+ throw new Error(`unbalanced parentheses after ${call}`)+}+/** Every call whose dependencies run DURING start-up, before `runForeground` has finished its body. */const STARTUP_CALLS = ['await repairClaudeCwd({', 'await restoreAgents({', 'startSelfUpdater({', 'startTuiUpdater({']@@ -128,15 +140,19 @@ describe('cli.ts start-up order', () => {})it('only swaps in the full restart handler once everything it tears down exists', () => {- // `bootHandoff` hands the machine over without finishing start-up; `restartForUpdate` tears down- // two dozen subsystems and may only be armed after every one of them is built.+ // `bootHandoff` hands the machine over without finishing start-up; the full handoff+ // (core/updateHandoff.ts) tears down two dozen subsystems, handed to it here as its teardown, and may+ // only be armed after every one of them is built.const source = code(SOURCE)const flip = source.indexOf('daemonBoot.applyStagedUpdate = ')expect(flip, 'the handoff handler is still swapped in cli.ts').toBeGreaterThan(-1)- const { body } = callArgument(source, 'const restartForUpdate = ')+ const body = callArguments(source, 'updateHandoff.restartForUpdate(')+ expect(body.at, 'the handler is the one it swaps in').toBeGreaterThan(flip)const declaredAt = localsOf(source)- const late = referencedIn(body).filter(used => (declaredAt.get(used) ?? -1) > flip).sort()- expect(late, 'these are torn down by restartForUpdate but declared after it is armed').toEqual([])+ const late = referencedIn(body.text).filter(used => (declaredAt.get(used) ?? -1) > flip).sort()+ expect(late, 'these are torn down by the update handoff but declared after it is armed').toEqual([])+ // What it tears down is all there is to it: the handoff module reaches nothing of runForeground's.+ expect(referencedIn(body.text)).toEqual(expect.arrayContaining(['registry', 'hookServer', 'localWsServer', 'backend', 'watcher']))})it('the daemon arm survives its own start-up failure', () => {diff --git a/docs/design/2026-10-03-harnessd.md b/docs/design/2026-10-03-harnessd.mdindex 4053b7e4e..6a8fd046c 100644--- a/docs/design/2026-10-03-harnessd.md+++ b/docs/design/2026-10-03-harnessd.md@@ -694,24 +694,39 @@ Done so far:the build (the rollback frees the room the list needs, so this takes a disk filled again atonce, as by a log storm); its pending note now names the build until the list can be written.+ Left after round 40, and fixed since (round 40b), each with a test that fails without it:+ - A core that could not start for want of room on its disk (`listen ENOSPC` claiming its socket) was+ read as the new build failing: rolled back, the rollback freeing the room the build before needed,+ and a good build remembered as bad. A failure that names the disk (`ENOSPC`, `EDQUOT`, `EROFS`) is+ no verdict on the build now: the master keeps the update on trial and starts the core again,+ backing off, without counting it as a crash (`namesTheDisk`, harnessd/supervisor.ts). End to end,+ the build staged just before the disk filled is kept once there is room, nothing rejected.+ - A handoff whose teardown threw stayed on the old build, its servers half closed, the new bundle+ staged and never judged. The handoff is its own core module now (core/updateHandoff.ts, out of+ `runForeground`, 89 lines shorter): under harnessd every step is tried, one that fails is said+ and passed, one that hangs is given up on after 15 s, and the core exits 75 regardless. End to end,+ a build whose teardown throws still hands over, and the next build is kept.+ - Downloads had no deadline of their own: a link that stalled without dropping held the check on+ undici's five-minute body timeout, and a trickle for as long as it lasted. A transfer now gives up+ after a minute without a byte or a quarter of an hour in all (`ADAPTER_UPDATE_IDLE_MS`,+ `ADAPTER_UPDATE_DEADLINE_MS`; the manifest at most 30 s and a minute), and the next check tries+ again. End to end, a download that stalls halfway is given up on and the build lands.+ - A turn that ended while the disk was full, on a data folder that had never run a project, was lost:+ `OrchestratorService.roleOf`, asked at every turn's end, makes its state folder to read it, and the+ mkdir threw out of the transcript's line handler; once open, the orchestrator watches every frame+ sent, and the same mkdir threw out of every send after it. Neither depends on it now: a role that+ cannot be read is anyone's (core/turns/recaps.ts), and a frame goes out whatever the orchestrator+ makes of it (`BackendSocket.send`). End to end in `e2e/diskfull.e2e.ts`.+Left, larger:- The manifest is not signed. Its sha256s are only as good as the TLS connection to the manifesthost: one that served a matching manifest and bundle would be installed, its canary run first.Signing needs a release key and its public half pinned in the bundle.- - A core cannot start while its data folder's disk is full: binding its socket needs a directory- entry (`listen ENOSPC`), so it starts in safe mode. During an update the master reads that as the- new build failing, rolls back (the rollback frees the room the build before needs) and remembers- a good build as bad until the next release. Telling a start that failed for want of room from a- bad build would let it be tried again instead.- - A handoff whose teardown throws (`restartForUpdate` in `runForeground`) stays on the old build- with servers already closed and the new bundle staged, unjudged; under harnessd it should exit- 75 regardless. It moves with `runForeground`'s next step.- - Downloads have no deadline of their own: a stalled transfer is dropped by undici's 300 s body- timeout, and a trickle can hold the check longer.+ - A master that dies while a disk-starved update waits for room leaves its re-execution marker, and+ the next start rolls the update back as a re-execution that never came up.- Bundles in versioned directories behind one pointer (above) would make the swap a single rename.- - Unrelated to updating: a turn that ends while the disk is full, on a data folder that never had- an `orchestrator` folder, is lost (`line handler error … mkdir …/orchestrator`):- `OrchestratorService.roleOf`, asked at every turn's end, creates its state folder to read it.+ - After the release (the orchestrator is out of scope until then): `OrchestratorService` should read+ its state without making its folder, which only its first save needs.Next, in order: