The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
/**
* Round 40: the update path under hostile conditions, end to end on real release bundles (built from
* this checkout at made-up versions and served from a local manifest, as in e2e/update.e2e.ts and
* e2e/reexec.e2e.ts). It is how every fix reaches every machine, so it has to hold when the link
* drops, the release is broken, the manifest lies, two checks race the master, agents are mid-turn,
* the new core cannot start, the master dies mid-update and the disk fills.
*
* After each: the daemon runs a working build (the old one or the new one) under its master, every
* agent is back and takes a turn, nothing half-written is left in the install for the next update to
* trip on, and a build that failed on this machine is remembered, while one that only met a bad moment
* (a dropped link, a full disk) is installed once the moment passes.
*
* The full-disk half mounts a small disk image of its own (macOS `hdiutil`), as e2e/diskfull.e2e.ts
* does, so it is opt-in: `DISKFULL=1 npm run test:e2e -- updateHostile`.
*/
import { execFileSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import {
closeSync, existsSync, mkdirSync, mkdtempSync, openSync, readdirSync, readFileSync, rmSync, statSync, truncateSync, writeFileSync, writeSync,
} from 'node:fs'
import { createServer, type Server, type ServerResponse } from 'node:http'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterAll, afterEach, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
import { LocalClient, type Frame } from './harness/client.js'
import { CLI_ROOT, IsolatedDaemon, until } from './harness/daemon.js'
import { atVersion, withFault } from './harness/release.js'
const FIRST = '43.0.1'
const DISKFULL = process.env.DISKFULL === '1' && process.platform === 'darwin'
/** Passes the updater's canary (`cli.js version`), then dies on every start as a core. */
const CRASHES_AS_A_CORE = 'if(process.argv[2]==="__run")process.exit(3);'
/**
* A core that runs as any other, until it hands over to an update: then the first interval it clears
* throws, as a teardown step that fails does (core/updateHandoff.ts). Told by the line the handoff
* starts with, wherever the daemon's timestamp puts it among the arguments.
*/
const TEARDOWN_THROWS = 'if(process.argv[2]==="__run"){let f=false;const l=console.log;console.log=function(...a){if(a.some((x)=>typeof x==="string"&&x.includes("[update] applying")))f=true;return l.apply(this,a)};const c=globalThis.clearInterval;globalThis.clearInterval=function(...a){if(f){f=false;throw new Error("a teardown step that throws")}return c.apply(this,a)}}'
const sha = (bytes: Buffer): string => createHash('sha256').update(bytes).digest('hex')
const count = (text: string, part: string): number => text.split(part).length - 1
const sleep = (ms: number) => new Promise((done) => setTimeout(done, ms))
type Row = Record<string, any>
/** One bundle built from this checkout; every release below is it at another version. */
let built = ''
let builtNotify = ''
let buildDir = ''
beforeAll(() => {
buildDir = mkdtempSync(join(tmpdir(), 'harnessd-hostile-build-'))
execFileSync(process.execPath, ['build-bundle.mjs'], {
cwd: CLI_ROOT, env: { ...process.env, ADAPTER_VERSION: FIRST, BUNDLE_OUT_DIR: buildDir }, stdio: 'pipe',
})
built = readFileSync(join(buildDir, 'cli.js'), 'utf8')
builtNotify = readFileSync(join(buildDir, 'notify.mjs'), 'utf8')
// The version is baked in at build time; the others are the same bytes with it swapped.
expect(count(built, FIRST)).toBeGreaterThanOrEqual(2)
}, 180_000)
afterAll(() => { if (buildDir) rmSync(buildDir, { recursive: true, force: true }) })
/**
* Releases: the build at another version, with a fault put in at the top where a test needs one, and a
* notify.mjs of its own, so an install mixed from two releases shows.
*/
class Releases {
readonly cli = new Map<string, Buffer>()
readonly notify = new Map<string, Buffer>()
add(version: string, inject = ''): void {
// In the lean bundle cli.js carries for the master, the services and the core too (e2e/harness/release.ts).
const source = withFault(atVersion(built, FIRST, version), inject)
const file = join(buildDir, `cli-${version}.js`)
writeFileSync(file, source)
expect(execFileSync(process.execPath, [file, 'version'], { encoding: 'utf8' }).trim()).toBe(version)
this.cli.set(version, Buffer.from(source))
this.notify.set(version, Buffer.from(`${builtNotify}\n// release ${version}\n`))
}
}
/** What a download of cli.js meets: cut short with a length to match, a connection that dies halfway
* through the length it promised, one that stays open halfway through and sends nothing more, or a
* wait until the test lets it through. */
type Fault = 'truncate' | 'drop' | 'stall' | 'hold'
interface Offer {
version: string
/** Bytes served as this version's cli.js instead of its own: a release whose manifest lies about it. */
bytes?: Buffer
/** The sha256 the manifest names, when it is not that of the bytes served. */
manifestSha?: string
}
/** The release channel: a manifest naming one build, and the files it names, with the faults asked for. */
class UpdateServer {
offer: Offer
/** What the next downloads of cli.js meet, one fault each. */
faults: Fault[] = []
/** Every download of a cli.js asked for, by version, in order. */
readonly downloads: string[] = []
private held: Array<() => void> = []
/** Answers left open halfway, ended only when the server stops. */
private stalled: ServerResponse[] = []
private server: Server | null = null
constructor(private readonly releases: Releases, first: string) { this.offer = { version: first } }
get origin(): string { return `http://127.0.0.1:${(this.server!.address() as { port: number }).port}` }
get holding(): number { return this.held.length }
downloadsOf(version: string): number { return this.downloads.filter((one) => one === version).length }
async start(): Promise<void> {
this.server = createServer((request, response) => this.answer(request.url ?? '', response))
await new Promise<void>((done) => this.server!.listen(0, '127.0.0.1', done))
}
stop(): Promise<void> {
this.release()
for (const response of this.stalled) response.socket?.destroy()
return new Promise((done) => this.server ? this.server.close(() => done()) : done())
}
/** Every held download goes through. */
release(): void {
const held = this.held
this.held = []
for (const go of held) go()
}
private bytesOf(version: string): Buffer {
return this.offer.version === version && this.offer.bytes ? this.offer.bytes : this.releases.cli.get(version)!
}
private answer(url: string, response: ServerResponse): void {
if (url === '/metadata.json') {
const { version } = this.offer
const cli = this.bytesOf(version)
const notify = this.releases.notify.get(version)!
response.end(JSON.stringify({ cli: {
version,
cli: { url: `${this.origin}/cli-${version}.js`, sha256: this.offer.manifestSha ?? sha(cli), size: cli.length },
notify: { url: `${this.origin}/notify-${version}.mjs`, sha256: sha(notify), size: notify.length },
} }))
return
}
const cliVersion = /^\/cli-(.+)\.js$/.exec(url)?.[1]
if (cliVersion && this.releases.cli.has(cliVersion)) {
this.downloads.push(cliVersion)
const bytes = this.bytesOf(cliVersion)
const half = bytes.subarray(0, bytes.length >> 1)
const fault = this.faults.shift()
if (fault === 'truncate') {
response.setHeader('content-length', half.length)
response.end(half)
} else if (fault === 'drop') {
response.setHeader('content-length', bytes.length)
response.write(half, () => setTimeout(() => response.socket?.destroy(), 20))
} else if (fault === 'stall') {
response.setHeader('content-length', bytes.length)
response.write(half)
this.stalled.push(response)
} else if (fault === 'hold') {
this.held.push(() => response.end(bytes))
} else {
response.end(bytes)
}
return
}
const notifyVersion = /^\/notify-(.+)\.mjs$/.exec(url)?.[1]
if (notifyVersion && this.releases.notify.has(notifyVersion)) { response.end(this.releases.notify.get(notifyVersion)); return }
response.statusCode = 404
response.end()
}
}
const rows = async (client: LocalClient): Promise<Row[]> =>
(await client.request<{ agents: Row[] }>('agents_list', { includeStopped: true }, 30_000)).agents
const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId
async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {
const started = client.next(isTurn('turn_started', agentId), 60_000, `turn_started (${content})`)
const ended = client.next(isTurn('turn_ended', agentId), 60_000, `turn_ended (${content})`)
client.send('message', { agentId, content })
expect((await started).payload?.userMessage).toBe(content)
await ended
}
/** A machine with a release installed, a daemon running it under harnessd, and two agents at work. */
class Installed {
daemon!: IsolatedDaemon
agents: Row[] = []
constructor(readonly releases: Releases, readonly server: UpdateServer, readonly cliDir: string) {}
async boot(first: string, dataDir?: string, env: Record<string, string> = {}): Promise<void> {
mkdirSync(this.cliDir, { recursive: true })
writeFileSync(join(this.cliDir, 'cli.js'), this.releases.cli.get(first)!)
writeFileSync(join(this.cliDir, 'notify.mjs'), this.releases.notify.get(first)!)
writeFileSync(join(this.cliDir, 'package.json'), '{"type":"module"}\n')
const local = this.server.origin
this.daemon = await IsolatedDaemon.create({
scriptPath: join(this.cliDir, 'cli.js'),
...(dataDir ? { dataDir } : {}),
env: {
ADAPTER_CLI_DIR: this.cliDir,
ADAPTER_UPDATE_DISABLE: 'false',
ADAPTER_UPDATE_URL: `${local}/metadata.json`,
ADAPTER_UPDATE_CHECK_MS: '1000',
ADAPTER_UPDATE_SLOT_SEC: '-1',
HARNESSD_UPDATE_PROBATION_MS: '6000',
HARNESSD_INITIAL_BACKOFF_MS: '100',
// Every other download an installed daemon looks for stays on this machine too.
HARNESS_TUI_MANIFEST_URL: `${local}/tui/metadata.json`,
ADAPTER_RUNTIME_METADATA_URL: `${local}/runtime/metadata.json`,
ADAPTER_GRID_RUNTIME_METADATA_URL: `${local}/grid/metadata.json`,
...env,
},
})
await this.daemon.start()
expect(await this.status()).toMatchObject({ version: first, harnessd: { masterVersion: first } })
const client = await LocalClient.connect(this.daemon)
try {
for (const engine of ['claude', 'codex'] as const) {
const cwd = join(this.daemon.projectsDir, `hostile-${engine}`)
mkdirSync(cwd, { recursive: true })
const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
expect(created.error, JSON.stringify(created)).toBeUndefined()
this.agents.push(await until(`${engine} to bind its conversation`, async () => {
const row = (await rows(client)).find((one) => one.id === created.agent.id)
return row?.sessionId && row.status === 'active' ? row : null
}, 90_000, 500))
}
} finally { client.close() }
}
log(): string { return this.daemon.log() }
showLogOnFailure(): void {
onTestFailed(() => {
console.log(`---- daemon log\n${this.daemon?.log().split('\n').slice(-200).join('\n')}`)
// The runner keeps only so much: UPDATE_HOSTILE_LOG=<file> keeps the daemon's whole log for reading.
if (process.env.UPDATE_HOSTILE_LOG) writeFileSync(process.env.UPDATE_HOSTILE_LOG, this.daemon?.log() ?? '')
})
}
async status(): Promise<Row | null> {
return fetch(`http://127.0.0.1:${this.daemon.port}/api/status`, { headers: this.daemon.localHeaders() }).then((response) => response.json() as Promise<Row>).catch(() => null)
}
async version(): Promise<string | null> { return (await this.status())?.version ?? null }
/** The core runs `version`, ready, under a master that runs one of `masters`. */
on(version: string, masters: string[] = [version], ms = 150_000): Promise<Row> {
return until(`the core on ${version}, ready, under a master on ${masters.join(' or ')}`, async () => {
const now = await this.status()
return now?.version === version && now?.harnessd?.state === 'running' && masters.includes(now?.harnessd?.masterVersion) ? now : null
}, ms, 250)
}
/** Nothing is waiting to be kept or rolled back. */
judged(ms = 90_000): Promise<true> {
return until('the update to be kept or rolled back', () => !['update-pending.json', 'cli.js.prev', 'notify.mjs.prev'].some((name) => existsSync(join(this.cliDir, name))), ms, 250)
}
/** The install holds `version`'s own files, both of them. */
holds(version: string): void {
expect(sha(readFileSync(join(this.cliDir, 'cli.js'))), `cli.js is ${version}'s`).toBe(sha(this.releases.cli.get(version)!))
expect(sha(readFileSync(join(this.cliDir, 'notify.mjs'))), `notify.mjs is ${version}'s`).toBe(sha(this.releases.notify.get(version)!))
}
rejected(): string[] {
try { return JSON.parse(readFileSync(join(this.cliDir, 'update-rejected.json'), 'utf8')) as string[] } catch { return [] }
}
/** Whatever an update left in the install or the data folder that the next one could trip on. */
leftovers(): string[] {
const kept = new Set(['cli.js', 'notify.mjs', 'package.json', 'update-rejected.json'])
const install = readdirSync(this.cliDir).filter((name) => !kept.has(name)).map((name) => `install: ${name}`)
const data = readdirSync(this.daemon.dataDir).filter((name) => name.startsWith('harnessd-reexec.json') || name === 'adapter.spawn.lock')
.map((name) => `data: ${name}`)
return [...install, ...data]
}
/** Both agents are back on their own conversations, and each takes a turn. */
async agentsBack(label: string): Promise<void> {
const client = await LocalClient.connect(this.daemon)
try {
for (const agent of this.agents) {
await until(`the ${agent.engine} agent back on its conversation`, async () => {
const row = (await rows(client)).find((one) => one.id === agent.id)
return row?.status === 'active' && row.sessionId === agent.sessionId ? row : null
}, 90_000, 500)
await turn(client, agent.id, `${label} (${agent.engine})`)
}
} finally { client.close() }
}
/** Every process running a build of this install, from the process table. */
daemonProcesses(): Array<{ pid: number; ppid: number; command: string }> {
return execFileSync('ps', ['-axo', 'pid=,ppid=,command='], { encoding: 'utf8' }).split('\n')
.map((line) => /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line))
.filter((match): match is RegExpExecArray => !!match && (match[3].includes(join(this.cliDir, 'cli.js')) || (!!this.daemon && match[3].includes(join(this.daemon.dataDir, 'lean')))))
.map((match) => ({ pid: Number(match[1]), ppid: Number(match[2]), command: match[3] }))
}
/** One core, the master's child, and nothing else running this install: no orphan, no probe left. */
async nothingOrphaned(): Promise<void> {
const master = this.daemon.pid!
await until('only the master\'s own core', () => this.daemonProcesses().every((one) => one.ppid === master && one.command.endsWith(' __run')), 15_000, 200)
expect(this.daemonProcesses().filter((one) => one.command.endsWith(' __run'))).toHaveLength(1)
}
/** What every scenario ends in: one build installed and running under its master, every agent back,
* nothing in between left behind, and exactly these builds remembered as rolled back. */
async settled(version: string, expected: { masters?: string[]; rejected: string[] }): Promise<void> {
await this.on(version, expected.masters)
await this.judged()
this.holds(version)
expect(this.leftovers()).toEqual([])
expect(this.rejected()).toEqual(expected.rejected)
await this.agentsBack(`settled on ${version}`)
await this.nothingOrphaned()
}
}
describe('an update under hostile conditions', () => {
const releases = new Releases()
let server: UpdateServer
let scratch = ''
let machine: Installed
beforeAll(async () => {
for (let patch = 1; patch <= 15; patch++) {
releases.add(`43.0.${patch}`, patch === 10 || patch === 11 ? CRASHES_AS_A_CORE : patch === 14 ? TEARDOWN_THROWS : '')
}
scratch = mkdtempSync(join(tmpdir(), 'harnessd-hostile-'))
server = new UpdateServer(releases, FIRST)
await server.start()
machine = new Installed(releases, server, join(scratch, 'cli'))
// A link that sends nothing for two seconds is given up on (lib/selfUpdate.ts TransferLimits).
await machine.boot(FIRST, undefined, { ADAPTER_UPDATE_IDLE_MS: '2000' })
}, 300_000)
afterAll(async () => {
await machine?.daemon?.close()
await server?.stop()
if (scratch) rmSync(scratch, { recursive: true, force: true })
})
it('a link that drops mid-download: nothing is written, and the build lands once the link holds', async () => {
machine.showLogOnFailure()
const from = machine.log().length
server.faults = ['drop', 'drop', 'drop']
server.offer = { version: '43.0.2' }
await until('three downloads dropped halfway', () => server.downloadsOf('43.0.2') >= 3, 90_000)
expect(await machine.version()).toBe(FIRST)
machine.holds(FIRST)
await machine.settled('43.0.2', { rejected: [] })
expect(machine.log().slice(from)).toContain('[update] check failed (will retry)')
}, 300_000)
it('a download cut short is refused, and the whole build lands', async () => {
machine.showLogOnFailure()
const from = machine.log().length
server.faults = ['truncate', 'truncate']
server.offer = { version: '43.0.3' }
await until('two downloads cut short', () => server.downloadsOf('43.0.3') >= 2, 90_000)
expect(await machine.version()).toBe('43.0.2')
await machine.settled('43.0.3', { rejected: [] })
expect(machine.log().slice(from)).toContain('sha256 mismatch')
}, 300_000)
/** A build that cannot be installed must cost one download now and then, not one at every check. */
const refusedQuietly = async (version: string): Promise<void> => {
const cores = machine.daemon.coresStarted()
await until(`a download of ${version}`, () => server.downloadsOf(version) >= 1, 90_000)
await sleep(12_000)
expect(machine.daemon.coresStarted(), 'no restart for a build that cannot be installed').toBe(cores)
expect(await machine.version()).toBe('43.0.3')
machine.holds('43.0.3')
expect(machine.leftovers()).toEqual([])
expect(server.downloadsOf(version), 'a check a second, and only a handful of downloads').toBeLessThanOrEqual(5)
}
it('a release published broken (its manifest names the cut-short bytes) fails its canary, is never staged, and is not downloaded at every check', async () => {
machine.showLogOnFailure()
const whole = releases.cli.get('43.0.4')!
server.offer = { version: '43.0.4', bytes: whole.subarray(0, whole.length >> 1) }
await refusedQuietly('43.0.4')
}, 300_000)
it('a manifest whose hash lies is refused, without a download at every check', async () => {
machine.showLogOnFailure()
server.offer = { version: '43.0.5', manifestSha: sha(Buffer.from('not what is served')) }
await refusedQuietly('43.0.5')
}, 300_000)
it('a manifest whose version lies (it names the bytes of the build running) does not restart the daemon over and over', async () => {
machine.showLogOnFailure()
server.offer = { version: '43.0.6', bytes: releases.cli.get('43.0.3') }
await refusedQuietly('43.0.6')
}, 300_000)
it('a newer build published while the last is on probation: two checks race the master, and the newest is kept, nothing rejected', async () => {
machine.showLogOnFailure()
const from = machine.log().length
server.offer = { version: '43.0.7' }
await until('the core on 43.0.7', async () => (await machine.version()) === '43.0.7', 150_000, 100)
// Its probation has six seconds to run, and its updater checks every second.
expect(existsSync(join(machine.cliDir, 'update-pending.json'))).toBe(true)
server.offer = { version: '43.0.8' }
await machine.settled('43.0.8', { rejected: [] })
expect(machine.log().slice(from)).toContain('[harnessd] the updated core staged a newer build before it was kept')
expect(machine.log().slice(from)).not.toContain('the updated core failed')
}, 300_000)
it('an update while both agents are mid-turn: their turns finish, both are back, and the next turns work', async () => {
machine.showLogOnFailure()
const client = await LocalClient.connect(machine.daemon)
for (const agent of machine.agents) {
const started = client.next(isTurn('turn_started', agent.id), 60_000, `the long turn of ${agent.engine}`)
client.send('message', { agentId: agent.id, content: '!slow 15000' })
await started
}
server.offer = { version: '43.0.9' }
await machine.on('43.0.9')
client.close()
// The turns were running in their panes through the swap; the core on the new build reads how they end.
const after = await LocalClient.connect(machine.daemon)
try {
for (const agent of machine.agents) {
const ended = after.frames.find(isTurn('turn_ended', agent.id)) ?? await after.waitFor(isTurn('turn_ended', agent.id), 120_000, `the long turn of ${agent.engine} to end`)
expect(ended).toBeTruthy()
}
const now = await rows(after)
for (const agent of machine.agents) expect(now.find((one) => one.id === agent.id)?.activity?.state, `${agent.engine} is not left working`).not.toBe('working')
} finally { after.close() }
await machine.settled('43.0.9', { rejected: [] })
}, 300_000)
it('a build whose master answers its probe but whose core crashes at start is rolled back and remembered, and the master goes back with it', async () => {
machine.showLogOnFailure()
const from = machine.log().length
server.offer = { version: '43.0.10' }
await until('the rollback', () => machine.log().slice(from).includes('the updated core failed (code 3) — rolled back'), 150_000)
expect(machine.log().slice(from)).toContain(`master re-executed (pid ${machine.daemon.pid}) · now v43.0.10`)
await machine.settled('43.0.9', { rejected: ['43.0.10'] })
const cores = machine.daemon.coresStarted()
await sleep(5_000)
expect(machine.daemon.coresStarted()).toBe(cores)
expect(server.downloadsOf('43.0.10')).toBe(1)
}, 300_000)
it('the master killed mid-update, as a power cut would: the next start judges the build it finds staged, and rolls back one whose core crashes', async () => {
machine.showLogOnFailure()
const from = machine.log().length
server.offer = { version: '43.0.11' }
await until('the core to stage 43.0.11', () => machine.log().slice(from).includes('[update] staged 43.0.11'), 90_000)
const core = machine.daemon.corePid()
await machine.daemon.kill()
await until('the core it was running to be gone', () => !IsolatedDaemon.alive(core), 30_000, 100)
machine.holds('43.0.11')
// What launchd restarting the master, or the desktop app's `harness start`, does.
const restart = machine.log().length
await machine.daemon.start()
await machine.settled('43.0.9', { rejected: ['43.0.10', '43.0.11'] })
expect(machine.log().slice(restart)).toContain('[harnessd] the bundle on disk is 43.0.11, an update no master kept or rolled back — its first core is on probation')
expect(machine.log().slice(restart)).toContain('the updated core failed (code 3) — rolled back')
}, 300_000)
it('moves on when a newer build is published', async () => {
machine.showLogOnFailure()
server.offer = { version: '43.0.12' }
await machine.settled('43.0.12', { rejected: ['43.0.10', '43.0.11'] })
}, 300_000)
it('a link that stalls halfway without dropping is given up on, and the build lands at the next check', async () => {
machine.showLogOnFailure()
const from = machine.log().length
server.faults = ['stall']
server.offer = { version: '43.0.13' }
// Before, the check waited out undici's five-minute body timeout, every later check skipped meanwhile.
await until('the stalled download to be given up', () => machine.log().slice(from).includes('cli-43.0.13.js sent nothing for 2000 ms'), 60_000)
await machine.settled('43.0.13', { rejected: ['43.0.10', '43.0.11'] })
expect(server.downloadsOf('43.0.13')).toBe(2)
}, 300_000)
it('a core whose teardown throws as it hands over still hands over, and the new build is kept', async () => {
machine.showLogOnFailure()
server.offer = { version: '43.0.14' }
await machine.settled('43.0.14', { rejected: ['43.0.10', '43.0.11'] })
// 43.0.14's own handoff is the one that throws. Before, the core stayed on it, half torn down, with
// 43.0.15 staged and never judged.
const from = machine.log().length
server.offer = { version: '43.0.15' }
await machine.settled('43.0.15', { rejected: ['43.0.10', '43.0.11'] })
expect(machine.log().slice(from)).toMatch(/\[update\] [^\n]* did not let go \(a teardown step that throws\) — handing over all the same/)
expect(machine.log().slice(from)).not.toContain('restart failed — staying on current build')
}, 300_000)
})
describe.skipIf(!DISKFULL)('an update on a disk that fills', () => {
const releases = new Releases()
let server: UpdateServer
let scratch = ''
let volume = ''
let machine: Installed
const filler = () => join(volume, 'filler')
const lock = () => join(machine.daemon.dataDir, 'adapter.spawn.lock')
/** Writes until the volume refuses. */
const fill = (): void => {
const fd = openSync(filler(), 'a')
const chunk = Buffer.alloc(1024 * 1024, 7)
try { for (;;) writeSync(fd, chunk) } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOSPC') throw error } finally { closeSync(fd) }
// The last blocks a metadata write may still find: take those too.
const tail = openSync(join(volume, 'filler-tail'), 'a')
try { for (;;) writeSync(tail, Buffer.alloc(4096, 7)) } catch { /* full */ } finally { closeSync(tail) }
}
/** Gives back this much room. */
const room = (bytes: number): void => {
rmSync(join(volume, 'filler-tail'), { force: true })
truncateSync(filler(), Math.max(0, statSync(filler()).size - bytes))
}
const unfill = (): void => {
rmSync(filler(), { force: true })
rmSync(join(volume, 'filler-tail'), { force: true })
}
/** Holds the daemon spawn lock as a `harness start` would: the updater waits for it with its build verified. */
const holdLock = (): void => {
mkdirSync(lock(), { mode: 0o700 })
writeFileSync(join(lock(), 'owner.json'), JSON.stringify({ pid: process.pid, startMarker: '', token: 'e2e-update-hostile', purpose: 'start', since: Date.now() }), { mode: 0o600 })
}
const releaseLock = (): void => { rmSync(lock(), { recursive: true, force: true }) }
beforeAll(async () => {
for (let patch = 1; patch <= 5; patch++) releases.add(`44.0.${patch}`, patch === 5 ? CRASHES_AS_A_CORE : '')
// Short names: the daemon's socket lives in the data folder, and a Unix socket's path has a
// 104-byte limit on macOS that a long temporary folder alone nearly uses up.
scratch = mkdtempSync(join(tmpdir(), 'hu-'))
volume = join(scratch, 'v')
mkdirSync(volume)
const image = join(scratch, 'u.dmg')
execFileSync('hdiutil', ['create', '-size', '96m', '-fs', 'HFS+', '-volname', 'hutest', '-type', 'UDIF', image], { stdio: 'pipe' })
execFileSync('hdiutil', ['attach', image, '-mountpoint', volume, '-nobrowse', '-noverify', '-noautoopen'], { stdio: 'pipe' })
server = new UpdateServer(releases, '44.0.1')
await server.start()
machine = new Installed(releases, server, join(volume, 'c'))
const data = join(volume, 'd')
mkdirSync(data)
await machine.boot('44.0.1', data)
}, 300_000)
// A scenario that fails part way must not leave the next one a full disk or a held lock.
afterEach(() => {
try { unfill() } catch { /* not mounted */ }
if (machine?.daemon) { try { if (readFileSync(join(lock(), 'owner.json'), 'utf8').includes('e2e-update-hostile')) releaseLock() } catch { /* not held */ } }
})
afterAll(async () => {
try { unfill() } catch { /* not mounted */ }
if (process.env.UPDATE_HOSTILE_LOG && machine?.daemon) writeFileSync(process.env.UPDATE_HOSTILE_LOG, machine.daemon.log())
await machine?.daemon?.close()
await server?.stop()
if (volume) { try { execFileSync('hdiutil', ['detach', volume, '-force'], { stdio: 'pipe' }) } catch { /* already gone */ } }
if (scratch) rmSync(scratch, { recursive: true, force: true })
})
it('the disk fills while the build downloads: the canary cannot be written, nothing is left, the agents keep working, and the build lands once there is room', async () => {
machine.showLogOnFailure()
const from = machine.log().length
server.faults = ['hold']
server.offer = { version: '44.0.2' }
await until('the download to be under way', () => server.holding === 1, 90_000)
fill()
server.release()
await until('the canary to fail on the full disk', () => /\[update\][^\n]*canary/.test(machine.log().slice(from)), 60_000)
await sleep(3_000)
expect(await machine.version()).toBe('44.0.1')
machine.holds('44.0.1')
expect(machine.leftovers()).toEqual([])
await machine.agentsBack('while the disk is full')
unfill()
await machine.settled('44.0.2', { rejected: [] })
}, 400_000)
it('the disk fills between the canary and the swap: the install is left as it was, the spawn lock is not left behind, and the build lands once there is room', async () => {
machine.showLogOnFailure()
const from = machine.log().length
holdLock()
server.offer = { version: '44.0.3' }
await until('the updater to wait for the lock with the build verified', () => machine.log().slice(from).includes('[update] waiting — the daemon is'), 90_000)
fill()
releaseLock()
await until('a swap to fail on the full disk', () => count(machine.log().slice(from), '[update] check failed') >= 1, 60_000)
// Room for the lock and part of one copy of the bundle, not all of it.
room(3 * 1024 * 1024)
const second = machine.log().length
await until('another swap to fail', () => count(machine.log().slice(second), '[update] check failed') >= 1, 60_000)
await until('the updater to let go of the lock', () => !existsSync(lock()), 30_000, 50)
// Held again, so nothing is mid-write while the install is looked at.
holdLock()
try {
machine.holds('44.0.2')
expect(machine.leftovers().filter((one) => one !== 'data: adapter.spawn.lock')).toEqual([])
} finally { releaseLock() }
unfill()
await machine.settled('44.0.3', { rejected: [] })
}, 400_000)
it('the disk fills just after the swap: the new build cannot start for want of room, which is no verdict on it, and it is kept once there is room', async () => {
machine.showLogOnFailure()
const from = machine.log().length
const master = machine.daemon.pid
server.offer = { version: '44.0.4' }
await until('the core to stage 44.0.4', () => machine.log().slice(from).includes('[update] staged 44.0.4'), 90_000, 50)
fill()
// A core cannot claim its socket on a full disk (listen ENOSPC), so the core on the new bundle starts in
// safe mode. The build before would too: the master keeps the update on trial and tries again,
// backing off. Before, it rolled back (the rollback freeing the room the build before needed) and
// remembered a good build as bad.
// The volume is kept full until then. The master may be writing its lean bundle out as the volume fills: what
// it had written is given back when that write fails (leanBundle.ts leaves no half behind), and with that
// room the core starts and the update simply stays up, which is right and is not this test (a slow macOS
// runner, 2026-10-12).
await until('the master to try the new bundle again for want of room', () => { fill(); return count(machine.log().slice(from), 'had no room to start') >= 2 }, 150_000, 100)
expect(machine.log().slice(from)).toContain('could not start on the new bundle for want of room on the disk (listen ENOSPC')
expect(machine.log().slice(from)).not.toContain('the updated core failed')
expect(IsolatedDaemon.alive(master), 'the master outlives a full disk in the middle of an update').toBe(true)
machine.holds('44.0.4')
unfill()
await machine.settled('44.0.4', { masters: ['44.0.3', '44.0.4'], rejected: [] })
expect(machine.log().slice(from)).toContain('the update stayed up — keeping it')
expect(machine.daemon.pid).toBe(master)
}, 400_000)
it('a build that crashes on a full disk is rolled back and remembered, and never staged again', async () => {
machine.showLogOnFailure()
const from = machine.log().length
const before = (await machine.version())!
const rejectedBefore = machine.rejected()
server.offer = { version: '44.0.5' }
await until('the core to stage 44.0.5', () => machine.log().slice(from).includes('[update] staged 44.0.5'), 90_000, 50)
fill()
await until('the rollback', () => machine.log().slice(from).includes('the updated core failed (code 3) — rolled back'), 150_000)
await until(`the core back on ${before}`, async () => (await machine.version()) === before, 150_000, 250)
unfill()
await machine.settled(before, { masters: ['44.0.3', '44.0.4'], rejected: [...rejectedBefore, '44.0.5'] })
await sleep(5_000)
expect(server.downloadsOf('44.0.5')).toBe(1)
}, 400_000)
})