The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
harnessd: a master that keeps the daemon's core running (#715)
Dee076aff72a91c
diff --git a/cli/build-bundle.mjs b/cli/build-bundle.mjsindex 893e3b532..783fa7a57 100644--- a/cli/build-bundle.mjs+++ b/cli/build-bundle.mjs@@ -31,12 +31,15 @@ const processImages = readProcessImageBundle({required: process.env.HARNESS_REQUIRE_PROCESS_IMAGES === '1',})+// `BUNDLE_OUT_DIR`: somewhere other than dist/ (the end-to-end update test builds real releases).+const outDir = process.env.BUNDLE_OUT_DIR || 'dist'+// Start clean so no stale per-file `dist/*.js` / sourcemaps leak into the release artifact.-rmSync('dist', { recursive: true, force: true })+rmSync(outDir, { recursive: true, force: true })await esbuild.build({entryPoints: ['src/cli.ts'],- outfile: 'dist/cli.js',+ outfile: `${outDir}/cli.js`,bundle: true,platform: 'node',format: 'esm',@@ -64,6 +67,6 @@ await esbuild.build({logLevel: 'info',})-copyFileSync('hook/notify.mjs', 'dist/notify.mjs')+copyFileSync('hook/notify.mjs', `${outDir}/notify.mjs`)-console.log(`✓ Bundled dist/cli.js (v${version}) + dist/notify.mjs`)+console.log(`✓ Bundled ${outDir}/cli.js (v${version}) + ${outDir}/notify.mjs`)diff --git a/cli/e2e/harness/daemon.ts b/cli/e2e/harness/daemon.tsindex 129d2cd0e..6742a5dca 100644--- a/cli/e2e/harness/daemon.ts+++ b/cli/e2e/harness/daemon.ts@@ -25,6 +25,10 @@ export interface DaemonOptions {/** Models the fake engines report. */claudeModel?: stringcodexModel?: string+ /** Boot the core on its own (`__run`) instead of under harnessd's master (`__harnessd`). */+ noMaster?: boolean+ /** Run this bundle (an installed `cli.js`) instead of the checkout's source. */+ scriptPath?: string}async function freePort(): Promise<number> {@@ -121,14 +125,32 @@ export class IsolatedDaemon {get computerId(): string { return this.env.ADAPTER_COMPUTER_ID! }get socketPath(): string { return join(this.dataDir, `daemon-${this.port}.sock`) }get projectsDir(): string { return join(this.root, 'projects') }+ /** The process `harness start` would have started: the master, or a core run on its own. */get pid(): number | null { return this.child?.pid ?? null }++ /** The core the master is running now (the last one it said it started). */+ corePid(): number | null {+ if (this.options.noMaster) return this.pid+ const started = [...this.output.matchAll(/\[harnessd\] core started \(pid (\d+)\)/g)]+ return started.length ? Number(started[started.length - 1][1]) : null+ }++ /** How many cores the master has started since this daemon object first started one. */+ coresStarted(): number {+ return [...this.output.matchAll(/\[harnessd\] core started/g)].length+ }log(): string { return this.output }- /** `ready: 'port'` returns as soon as the port answers, the way a client sees a restarting daemon. */- async start(options: { ready?: 'port' | 'wired' } = {}): Promise<void> {+ /** `ready: 'port'` returns as soon as the port answers, the way a client sees a restarting daemon;+ * `'none'` as soon as the process is started. */+ async start(options: { ready?: 'none' | 'port' | 'wired' } = {}): Promise<void> {if (this.child) throw new Error('already running')const heap = this.options.heapMiB ? [`--max-old-space-size=${this.options.heapMiB}`] : []- const child = spawn(process.execPath, [...heap, '--import', 'tsx', 'src/cli.ts', '__run'], {+ const entry = this.options.noMaster ? '__run' : '__harnessd'+ // Readiness is judged from what this start prints, never from an earlier boot's lines.+ const from = this.output.length+ const script = this.options.scriptPath ? [this.options.scriptPath] : ['--import', 'tsx', 'src/cli.ts']+ const child = spawn(process.execPath, [...heap, ...script, entry], {cwd: CLI_ROOT, env: this.env, stdio: ['ignore', 'pipe', 'pipe'],})this.child = child@@ -140,6 +162,7 @@ export class IsolatedDaemon {reject(new Error(`daemon exited during boot (${signal ?? code}):\n${this.output.slice(-4000)}`))}))exited.catch(() => {}) // observed below while booting; afterwards an exit is the test's business+ if (options.ready === 'none') returnawait Promise.race([exited, (async () => {await until('the daemon to answer on its port', async () => {const response = await fetch(`http://127.0.0.1:${this.port}/api/health`).catch(() => null)@@ -149,7 +172,7 @@ export class IsolatedDaemon {// The port answers ~1,100 lines of startup before every handler is wired (see the startup-race// test); this line is printed once they are.if (options.ready !== 'port') {- await until('the daemon to finish starting', () => /\[cli\] (dialing|not signed in)/.test(this.output), 60_000, 100)+ await until('the daemon to finish starting', () => /\[cli\] ready/.test(this.output.slice(from)), 60_000, 100)}})()])}@@ -179,13 +202,20 @@ export class IsolatedDaemon {await this.start()}- /** Resident memory of the daemon process, MiB. */+ /** Resident memory of the core, MiB. */async rssMiB(): Promise<number> {- if (!this.child?.pid) return 0- const { stdout } = await exec('ps', ['-o', 'rss=', '-p', String(this.child.pid)])+ const pid = this.corePid()+ if (!pid) return 0+ const { stdout } = await exec('ps', ['-o', 'rss=', '-p', String(pid)]).catch(() => ({ stdout: '0' }))return Number(stdout.trim()) / 1024}+ /** Whether a process is alive (signal 0). */+ static alive(pid: number | null): boolean {+ if (!pid) return false+ try { process.kill(pid, 0); return true } catch { return false }+ }+/** What the daemon's pane shows, plain text. */async capture(pane: string): Promise<string> {return this.tmux.run('capture-pane', '-p', '-t', pane)diff --git a/cli/e2e/master.e2e.ts b/cli/e2e/master.e2e.tsnew file mode 100644index 000000000..936174dde--- /dev/null+++ b/cli/e2e/master.e2e.ts@@ -0,0 +1,174 @@+/**+ * harnessd's master over the real daemon: whatever happens to the core, the daemon comes back on its+ * own, the agents keep running in tmux, and a client picks up where it was. Whatever happens to the+ * master, nothing is left behind holding the port.+ */+import { existsSync, mkdirSync, readFileSync } from 'node:fs'+import { join } from 'node:path'+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'+import { LocalClient, type Frame } from './harness/client.js'+import { IsolatedDaemon, until } from './harness/daemon.js'++const ready = /\[cli\] ready/g+const wiredCount = (daemon: IsolatedDaemon) => [...daemon.log().matchAll(ready)].length++async function withAgent(daemon: IsolatedDaemon) {+ const client = await LocalClient.connect(daemon)+ const cwd = join(daemon.projectsDir, 'work')+ mkdirSync(cwd, { recursive: true })+ const agentId: string = (await client.request('agent_create', { engine: 'claude', cwd, bypassPermission: true }, 60_000)).agent.id+ await until('the conversation to bind', async () => rowOf(client, agentId).then((row) => row?.sessionId), 45_000, 250)+ return { client, agentId }+}++const rowOf = async (client: LocalClient, agentId: string) =>+ ((await client.request('agents_list', { includeStopped: true })).agents as Array<Record<string, any>>).find((agent) => agent.id === agentId)++/** A full turn on a fresh connection: proof the restarted daemon serves the same agent. */+async function turnWorks(daemon: IsolatedDaemon, agentId: string) {+ const client = await LocalClient.connect(daemon)+ await until('the agent to be live again', async () => {+ const row = await rowOf(client, agentId)+ return row && row.status !== 'stopped' ? row : null+ }, 60_000, 250)+ const ended = client.next((frame: Frame) => frame.type === 'turn_ended' && frame.agentId === agentId, 30_000, 'a turn after the restart')+ client.send('message', { agentId, content: 'are you still there?' })+ await ended+ client.close()+}++describe('harnessd', () => {+ let daemon: IsolatedDaemon | undefined+ afterEach(async () => { await daemon?.close(); daemon = undefined })+ const make = async (env: Record<string, string> = {}, { start = true } = {}) => {+ daemon = await IsolatedDaemon.create({ env: { HARNESSD_INITIAL_BACKOFF_MS: '100', ...env } })+ const d = daemon+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-80).join('\n')}`) })+ if (start) await d.start()+ return d+ }++ it('runs the core under a master that claims the pid file for itself', async () => {+ const d = await make()+ expect(d.corePid()).not.toBe(d.pid)+ expect(readFileSync(join(d.dataDir, 'adapter.pid'), 'utf8').trim()).toBe(String(d.pid))+ // The app judges who owns the daemon by this pid: the master's, as in the pid file, not the core's.+ const status = await (await fetch(`http://127.0.0.1:${d.port}/api/status`)).json() as Record<string, any>+ expect(status.pid).toBe(d.pid)+ expect(status.corePid).toBe(d.corePid())+ expect(status.harnessd.masterPid).toBe(d.pid)+ })++ it('restarts a core that crashes; the agent and its client carry on', async () => {+ const d = await make()+ const { client, agentId } = await withAgent(d)+ const first = d.corePid()!+ const wired = wiredCount(d)+ process.kill(first, 'SIGKILL')+ await until('a new core to finish starting', () => wiredCount(d) > wired, 60_000)+ expect(d.corePid()).not.toBe(first)+ expect(IsolatedDaemon.alive(d.pid)).toBe(true)+ await until('the old connection to close', () => client.closed, 10_000)+ await turnWorks(d, agentId)+ })++ it('kills and restarts a core that hangs', async () => {+ const d = await make({ HARNESSD_HEARTBEAT_TIMEOUT_MS: '3000' })+ const { agentId } = await withAgent(d)+ const hung = d.corePid()!+ const wired = wiredCount(d)+ process.kill(hung, 'SIGSTOP')+ await until('the master to notice', () => d.log().includes('it is hung'), 20_000)+ await until('a new core to finish starting', () => wiredCount(d) > wired, 60_000)+ expect(IsolatedDaemon.alive(hung)).toBe(false)+ await turnWorks(d, agentId)+ })++ it('takes the core down with it when the master is killed, and starts clean again', async () => {+ const d = await make()+ const { agentId } = await withAgent(d)+ const core = d.corePid()!+ await d.kill()+ await until('the core to follow its master', () => !IsolatedDaemon.alive(core), 10_000)+ await d.start()+ await turnWorks(d, agentId)+ })++ it('stops core and master within the grace harness stop gives them, removing the pid file', async () => {+ const d = await make()+ const core = d.corePid()!+ const master = d.pid!+ const started = Date.now()+ process.kill(master, 'SIGTERM')+ await until('both to exit', () => !IsolatedDaemon.alive(master) && !IsolatedDaemon.alive(core), 10_000, 25)+ expect(Date.now() - started).toBeLessThan(3_000)+ expect(existsSync(join(d.dataDir, 'adapter.pid'))).toBe(false)+ })++ it('restarts a core that outgrows its memory budget, backing off while it keeps doing so', async () => {+ daemon = await IsolatedDaemon.create({ env: { HARNESSD_INITIAL_BACKOFF_MS: '200', HARNESSD_RSS_LIMIT_MIB: '1' } })+ const d = daemon+ await d.start({ ready: 'none' })+ await until('the master to restart it for its memory', () => /over its 1 MiB budget — restarting it/.test(d.log()), 30_000)+ await until('another core', () => d.coresStarted() >= 2, 30_000)+ // 200, 400, 800, 1600 ms… — never as fast as a core can bind.+ await new Promise((resolve) => setTimeout(resolve, 3_000))+ expect(d.coresStarted()).toBeLessThanOrEqual(6)+ })++ it('restarts a core that something other than its master stops — it does not take the daemon down', async () => {+ const d = await make()+ const { agentId } = await withAgent(d)+ const first = d.corePid()!+ const wired = wiredCount(d)+ process.kill(first, 'SIGTERM')+ await until('a new core to finish starting', () => wiredCount(d) > wired, 60_000)+ expect(IsolatedDaemon.alive(d.pid)).toBe(true)+ expect(d.log()).toMatch(/core exited \(code 0, crashed\) — restarting/)+ await turnWorks(d, agentId)+ })++ it('records what the master knows where `harness status` reads it, the core alive or not', async () => {+ const d = await make()+ const statusFile = join(d.dataDir, 'harnessd-status.json')+ const read = () => JSON.parse(readFileSync(statusFile, 'utf8'))+ await until('the status file to say running', () => existsSync(statusFile) && read().state === 'running', 10_000)+ expect(read()).toMatchObject({ masterPid: d.pid, corePid: d.corePid(), restarts: 0, safeMode: null, protocol: 2 })+ const core = d.corePid()!+ process.kill(core, 'SIGKILL')+ await until('the status file to name the crash', () => read().restarts === 1 && read().lastExitReason === 'crashed', 10_000)+ await until('the status file to say running again', () => read().state === 'running', 60_000)+ })++ it('puts a core that keeps crashing in safe mode, and tries a normal one once safe mode runs out', async () => {+ const d = await make({ HARNESSD_CRASH_LOOP_WINDOW_MS: '120000', ADAPTER_SAFE_MODE_MS: '6000' })+ const statusFile = join(d.dataDir, 'harnessd-status.json')+ const read = () => JSON.parse(readFileSync(statusFile, 'utf8'))+ for (let crash = 1; crash <= 3; crash++) {+ const core = d.corePid()!+ process.kill(core, 'SIGKILL')+ await until(`core ${crash + 1} to start`, () => d.coresStarted() > crash, 30_000)+ await until(`core ${crash + 1} to bind`, () => d.corePid() !== core && read().state !== 'restarting', 60_000)+ }+ expect(d.log()).toMatch(/core crashed 3 times in 2 min — starting it in safe mode/)+ await until('the safe-mode core to say so', () => read().safeMode !== null && read().state === 'running', 30_000)+ expect(read().safeMode).toMatch(/crash again and again/)+ // Its time runs out with no fix: a normal core starts and finishes starting.+ const wired = wiredCount(d)+ await until('a normal core again', () => wiredCount(d) > wired, 60_000)+ await until('the status file to drop safe mode', () => read().safeMode === null && read().state === 'running', 30_000)+ })++ it('kills a core that binds but never finishes starting, and keeps doing so into safe mode', async () => {+ const d = await make({ HARNESSD_TEST_HOLD_READY: '1', HARNESSD_READY_TIMEOUT_MS: '3000', HARNESSD_CRASH_LOOP_WINDOW_MS: '120000' }, { start: false })+ await d.start({ ready: 'none' })+ await until('the master to give up on a start-up', () => /core bound but not ready within 3000 ms — killing it/.test(d.log()), 30_000)+ await until('safe mode', () => /starting it in safe mode/.test(d.log()), 60_000)+ const statusFile = join(d.dataDir, 'harnessd-status.json')+ await until('the safe-mode core to be up', () => {+ const status = JSON.parse(readFileSync(statusFile, 'utf8'))+ return status.state === 'running' && status.safeMode !== null+ }, 30_000)+ })+})+diff --git a/cli/e2e/startup.e2e.ts b/cli/e2e/startup.e2e.tsindex 78385ffbc..f15f1aea3 100644--- a/cli/e2e/startup.e2e.ts+++ b/cli/e2e/startup.e2e.ts@@ -2,8 +2,8 @@* What a client sees while the daemon starts. The desktop app connects the moment the daemon's port* answers, so everything the daemon says from then on is something a person sees.*- * Known defects of the current daemon are pinned with `it.fails`: each passes today only because it- * fails, and harnessd must turn it green (then it becomes an ordinary `it`).+ * Each of these was a defect of the daemon before harnessd, found by this suite and pinned with+ * `it.fails` until the request gate (BackendSocket.openRequests) fixed it.*/import { mkdirSync } from 'node:fs'import { join } from 'node:path'@@ -15,11 +15,10 @@ describe('starting up', () => {let daemon: IsolatedDaemon | undefinedafterEach(async () => { await daemon?.close(); daemon = undefined })- // The port answers, and the pid file is written, long before every request handler is wired:- // cli.ts writes PID_FILE at ~4325, assigns backend.onCreateAgent at ~5435 and onForkAgent later- // still, with the restore and the first full reconcile awaited in between. A request in that window- // is refused with UNSUPPORTED_ON_REMOTE, which no client expects from its own machine.- it.fails('answers every request with its real handler from the instant its port answers', async () => {+ // The port answers long before every request handler is wired (the restore and the first full+ // reconcile come in between). Before the gate a request in that window was refused with+ // UNSUPPORTED_ON_REMOTE, which no client expects from its own machine.+ it('answers every request with its real handler from the instant its port answers', async () => {daemon = await IsolatedDaemon.create()await daemon.start()const setup = await LocalClient.connect(daemon)@@ -40,10 +39,30 @@ describe('starting up', () => {expect(errors).not.toContain('UNSUPPORTED_ON_REMOTE')})- // After a restart the registry is served before the first reconcile confirms the panes: a running- // agent lists as `stopped` (when it had a conversation) or not at all, for ~100-200 ms after the- // daemon says it is ready. The app shows it paused or gone meanwhile.- it.fails('never shows a running agent as stopped or missing across a restart', async () => {+ // Worse than a refusal: `message` has no reply, so before the gate a message that landed before+ // `onMessage` was wired was dropped with only a log line. Found by master.e2e.ts, whose client+ // reconnects the moment a restarted daemon's port answers.+ it('delivers a message sent the instant its port answers', async () => {+ daemon = await IsolatedDaemon.create()+ await daemon.start()+ const setup = await LocalClient.connect(daemon)+ const cwd = join(daemon.projectsDir, 'message')+ mkdirSync(cwd, { recursive: true })+ const agentId: string = (await setup.request('agent_create', { engine: 'claude', cwd, bypassPermission: true }, 60_000)).agent.id+ await until('the conversation to bind', async () => ((await setup.request('agents_list', {})).agents as Array<Record<string, any>>)+ .find((agent) => agent.id === agentId)?.sessionId, 45_000, 200)+ setup.close()+ await daemon.stop()+ await daemon.start({ ready: 'port' })+ const client = await LocalClient.connect(daemon)+ const ended = client.next((frame) => frame.type === 'turn_ended' && frame.agentId === agentId, 20_000, 'the turn the message started')+ client.send('message', { agentId, content: 'sent the moment the port answered' })+ await ended+ })++ // Before the gate the registry was served before the restore confirmed the panes: a running agent+ // listed as `stopped` (when it had a conversation) or not at all, and the app showed it paused or gone.+ it('never shows a running agent as stopped or missing across a restart', async () => {daemon = await IsolatedDaemon.create()await daemon.start()let client = await LocalClient.connect(daemon)diff --git a/cli/e2e/update.e2e.ts b/cli/e2e/update.e2e.tsnew file mode 100644index 000000000..83c5552d0--- /dev/null+++ b/cli/e2e/update.e2e.ts@@ -0,0 +1,163 @@+/**+ * A self-update end to end, on real release bundles: the daemon finds a newer build in its manifest,+ * stages it, and harnessd's master restarts the core on it. A build that stays up is kept. One that+ * crashes is rolled back, remembered, and not tried again until a newer one is published — before,+ * every machine on it restarted once a minute until a fix shipped.+ *+ * The bundles are built from this checkout at made-up versions and served from a local manifest, so+ * nothing here reaches beyond the machine.+ */+import { execFileSync } from 'node:child_process'+import { createHash } from 'node:crypto'+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'+import { createServer, type Server } from 'node:http'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'+import { CLI_ROOT, IsolatedDaemon, until } from './harness/daemon.js'++const FIRST = '41.0.1'+const sha = (bytes: Buffer): string => createHash('sha256').update(bytes).digest('hex')+const count = (text: string, part: string): number => text.split(part).length - 1+const sleep = (ms: number) => new Promise((done) => setTimeout(done, ms))++describe('a self-update under harnessd', () => {+ let scratch = ''+ let server: Server | undefined+ let daemon: IsolatedDaemon | undefined+ let master: number | null = null+ const releases = new Map<string, Buffer>()+ let notify = Buffer.alloc(0)+ let offered = FIRST+ const downloads: string[] = []+ const cliDir = () => join(scratch, 'cli')+ const installed = () => readFileSync(join(cliDir(), 'cli.js'))+ const rejected = () => JSON.parse(readFileSync(join(cliDir(), 'update-rejected.json'), 'utf8')) as string[]+ const running = async (): Promise<string | null> => {+ const health = await fetch(`http://127.0.0.1:${daemon!.port}/api/health`).then((response) => response.json()).catch(() => null)+ return (health as { version?: string } | null)?.version ?? null+ }++ beforeAll(async () => {+ scratch = mkdtempSync(join(tmpdir(), 'harnessd-update-'))+ const out = join(scratch, 'build')+ execFileSync(process.execPath, ['build-bundle.mjs'], {+ cwd: CLI_ROOT, env: { ...process.env, ADAPTER_VERSION: FIRST, BUNDLE_OUT_DIR: out }, stdio: 'pipe',+ })+ const first = readFileSync(join(out, 'cli.js'), 'utf8')+ notify = readFileSync(join(out, 'notify.mjs'))+ // The version is baked in at build time; the others are the same bytes with it swapped.+ expect(count(first, FIRST)).toBeGreaterThanOrEqual(2)+ const release = (version: string, crashesOnStart = false): void => {+ let source = first.replaceAll(FIRST, version)+ // Passes the updater's canary (`cli.js version`), then dies on every start as a core.+ if (crashesOnStart) source = source.replace('\n', '\nif(process.argv[2]==="__run")process.exit(3);\n')+ const file = join(out, `cli-${version}.js`)+ writeFileSync(file, source)+ expect(execFileSync(process.execPath, [file, 'version'], { encoding: 'utf8' }).trim()).toBe(version)+ releases.set(version, Buffer.from(source))+ }+ release(FIRST)+ release('41.0.2')+ release('41.0.3', true)+ release('41.0.4')++ server = createServer((request, response) => {+ const url = request.url ?? ''+ const origin = `http://127.0.0.1:${(server!.address() as { port: number }).port}`+ if (url === '/metadata.json') {+ const cli = releases.get(offered)!+ response.end(JSON.stringify({ cli: {+ version: offered,+ cli: { url: `${origin}/cli-${offered}.js`, sha256: sha(cli), size: cli.length },+ notify: { url: `${origin}/notify.mjs`, sha256: sha(notify), size: notify.length },+ } }))+ return+ }+ const asked = /^\/cli-(.+)\.js$/.exec(url)?.[1]+ if (asked && releases.has(asked)) { downloads.push(asked); response.end(releases.get(asked)); return }+ if (url === '/notify.mjs') { response.end(notify); return }+ response.statusCode = 404+ response.end()+ })+ await new Promise<void>((done) => server!.listen(0, '127.0.0.1', done))+ const local = `http://127.0.0.1:${(server.address() as { port: number }).port}`++ mkdirSync(cliDir())+ writeFileSync(join(cliDir(), 'cli.js'), releases.get(FIRST)!)+ writeFileSync(join(cliDir(), 'notify.mjs'), notify)+ writeFileSync(join(cliDir(), 'package.json'), '{"type":"module"}\n')+ daemon = await IsolatedDaemon.create({+ scriptPath: join(cliDir(), 'cli.js'),+ env: {+ ADAPTER_CLI_DIR: cliDir(),+ ADAPTER_UPDATE_DISABLE: 'false',+ ADAPTER_UPDATE_URL: `${local}/metadata.json`,+ ADAPTER_UPDATE_CHECK_MS: '1000',+ ADAPTER_UPDATE_SLOT_SEC: '-1',+ HARNESSD_UPDATE_PROBATION_MS: '3000',+ HARNESSD_INITIAL_BACKOFF_MS: '100',+ // Every other download an installed daemon looks for stays on this machine too.+ HARNESS_TUI_MANIFEST_URL: `${local}/tui/metadata.json`,+ ADAPTER_RUNTIME_METADATA_URL: `${local}/runtime/metadata.json`,+ ADAPTER_GRID_RUNTIME_METADATA_URL: `${local}/grid/metadata.json`,+ },+ })+ await daemon.start()+ master = daemon.pid+ expect(await running()).toBe(FIRST)+ })++ afterAll(async () => {+ await daemon?.close()+ await new Promise<void>((done) => server ? server.close(() => done()) : done())+ if (scratch) rmSync(scratch, { recursive: true, force: true })+ })++ const showLog = () => onTestFailed(() => { console.log(`---- daemon log\n${daemon?.log().split('\n').slice(-120).join('\n')}`) })++ it('keeps a newer build that stays up', async () => {+ showLog()+ offered = '41.0.2'+ await until('the core to run 41.0.2', async () => (await running()) === '41.0.2', 90_000, 250)+ await until('the master to keep it', () => daemon!.log().includes('the update stayed up — keeping it'), 30_000)+ expect(daemon!.log()).toContain('core exited (code 75) for an update — restarting')+ expect(installed()).toEqual(releases.get('41.0.2'))+ expect(existsSync(join(cliDir(), 'cli.js.prev'))).toBe(false)+ expect(existsSync(join(cliDir(), 'update-pending.json'))).toBe(false)+ expect(existsSync(join(cliDir(), 'update-rejected.json'))).toBe(false)+ expect(daemon!.pid).toBe(master)+ })++ it('rolls back a build that crashes, and never stages it again', async () => {+ showLog()+ const from = daemon!.log().length+ offered = '41.0.3'+ await until('the master to roll it back', () => daemon!.log().slice(from).includes('the updated core failed (code 3) — rolled back'), 90_000)+ await until('the previous build to answer again', async () => (await running()) === '41.0.2', 60_000, 250)+ expect(installed()).toEqual(releases.get('41.0.2'))+ expect(rejected()).toEqual(['41.0.3'])+ expect(existsSync(join(cliDir(), 'update-pending.json'))).toBe(false)+ await until('the core to say it is waiting for a newer build', () => daemon!.log().slice(from).includes('41.0.3 was rolled back on this machine'), 30_000)+ // A check every second: five more of them, and not one restart.+ const cores = daemon!.coresStarted()+ await sleep(5_000)+ expect(daemon!.coresStarted()).toBe(cores)+ expect(await running()).toBe('41.0.2')+ expect(downloads.filter((version) => version === '41.0.3')).toHaveLength(1)+ expect(count(daemon!.log().slice(from), '41.0.3 was rolled back on this machine')).toBe(1)+ expect(daemon!.pid).toBe(master)+ })++ it('moves on when a newer build is published', async () => {+ showLog()+ offered = '41.0.4'+ await until('the core to run 41.0.4', async () => (await running()) === '41.0.4', 90_000, 250)+ await until('the master to keep it', () => count(daemon!.log(), 'the update stayed up — keeping it') === 2, 30_000)+ expect(installed()).toEqual(releases.get('41.0.4'))+ // Still remembered: only installing it on purpose takes a version off the list.+ expect(rejected()).toEqual(['41.0.3'])+ expect(daemon!.pid).toBe(master)+ expect(IsolatedDaemon.alive(master)).toBe(true)+ })+})diff --git a/cli/package.json b/cli/package.jsonindex 474a6fc60..eddc77c32 100644--- a/cli/package.json+++ b/cli/package.json@@ -14,10 +14,11 @@"start": "node dist/cli.js","test": "vitest run","test:local-models": "vitest run src/lib/localModels.spec.ts src/lib/localModels.integration.spec.ts src/dsh/builtins.spec.ts src/lib/gridFleetRpc.spec.ts --coverage --coverage.include=src/lib/localModels.ts --coverage.include=src/dsh/builtins.ts --coverage.thresholds.lines=100 --coverage.thresholds.statements=100 --coverage.thresholds.functions=100 --coverage.thresholds.branches=100 --coverage.reporter=text --coverage.reporter=json-summary --coverage.reportsDirectory=coverage/local-models",- "test:e2e": "vitest run --config vitest.e2e.config.ts","test:orchestrator": "vitest run src/orchestrator --maxWorkers=1 --coverage --coverage.include='src/orchestrator/**' --coverage.thresholds.lines=100 --coverage.thresholds.statements=100 --coverage.thresholds.functions=100 --coverage.thresholds.branches=100 --coverage.reporter=text --coverage.reporter=json --coverage.reportsDirectory=coverage/orchestrator","test:dsh-updates": "vitest run --config vitest.dsh-updates.config.ts","test:sharing": "vitest run src/sharing --coverage --coverage.include='src/sharing/**' --coverage.thresholds.lines=100 --coverage.thresholds.statements=100 --coverage.thresholds.branches=100 --coverage.thresholds.functions=100",+ "test:harnessd": "vitest run --config vitest.harnessd.config.ts",+ "test:e2e": "vitest run --config vitest.e2e.config.ts","test:sharing-e2e": "tsx scripts/share-harness-e2e.ts","test:remote-viewers": "vitest run --config vitest.viewer.config.ts","test:cursor-e2e": "RUN_CURSOR_E2E=1 vitest run src/engines/cursor/localE2e.spec.ts",diff --git a/cli/src/backendSocket.gate.spec.ts b/cli/src/backendSocket.gate.spec.tsnew file mode 100644index 000000000..6d0037e5d--- /dev/null+++ b/cli/src/backendSocket.gate.spec.ts@@ -0,0 +1,70 @@+import { describe, expect, it, vi } from 'vitest'+import { BackendSocket } from './backendSocket.js'++/** The request gate: a daemon that is still starting answers nothing, and then everything, in order. */+describe('BackendSocket request gate', () => {+ const settle = () => new Promise((resolve) => setTimeout(resolve, 10))+ const client = (socket: BackendSocket, connId: string) => {+ const frames: Array<Record<string, any>> = []+ socket.registerLocalClient(connId, { sendFrame: (frame) => { frames.push(frame); return true }, sendBinary: () => true })+ return frames+ }++ it('answers at once when nothing holds it — the way every socket but the daemon\'s is built', async () => {+ const socket = new BackendSocket('fixture')+ const cancelled: string[] = []+ socket.onCancel = (target) => cancelled.push(target)+ client(socket, 'local:open')+ socket.handleLocalFrame('local:open', { type: 'cancel', payload: { agentId: 'a1' } })+ await settle()+ expect(cancelled).toEqual(['a1'])+ })++ it('holds requests until it is opened, then dispatches them in the order they came', async () => {+ const socket = new BackendSocket('fixture')+ socket.holdRequests()+ socket.holdRequests()+ const cancelled: string[] = []+ client(socket, 'local:held')+ socket.handleLocalFrame('local:held', { type: 'cancel', payload: { agentId: 'a1' } })+ socket.handleLocalFrame('local:held', { type: 'cancel', payload: { agentId: 'a2' } })+ await settle()+ // Wired only now, as start-up would: the held requests meet the handler, not its absence.+ socket.onCancel = (target) => cancelled.push(target)+ expect(cancelled).toEqual([])+ socket.openRequests()+ socket.openRequests()+ await settle()+ expect(cancelled).toEqual(['a1', 'a2'])+ socket.handleLocalFrame('local:held', { type: 'cancel', payload: { agentId: 'a3' } })+ await settle()+ expect(cancelled).toEqual(['a1', 'a2', 'a3'])+ })++ it('closes a local client that floods a daemon still starting, and keeps the others', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})+ try {+ const socket = new BackendSocket('fixture')+ socket.holdRequests()+ const cancelled: string[] = []+ socket.onCancel = (target) => cancelled.push(target)+ client(socket, 'local:flood')+ client(socket, 'local:calm')+ for (let i = 0; i < 300; i++) socket.handleLocalFrame('local:flood', { type: 'cancel', payload: { agentId: `f${i}` } })+ socket.handleLocalFrame('local:calm', { type: 'cancel', payload: { agentId: 'calm' } })+ await settle()+ expect(warn).toHaveBeenCalledOnce()+ expect(warn.mock.calls[0][0]).toContain('sent 257 requests before the daemon was ready — closing it')+ expect(socket.localClientIds()).not.toContain('local:flood')+ socket.handleLocalFrame('local:flood', { type: 'cancel', payload: { agentId: 'gone' } })+ socket.openRequests()+ await settle()+ // What it queued before it was closed is still answered; nothing after.+ expect(cancelled.filter((target) => target.startsWith('f'))).toHaveLength(256)+ expect(cancelled).toContain('calm')+ expect(cancelled).not.toContain('gone')+ } finally {+ warn.mockRestore()+ }+ })+})diff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.tsindex 631d66d84..1c2d4ca29 100644--- a/cli/src/backendSocket.ts+++ b/cli/src/backendSocket.ts@@ -183,6 +183,9 @@ export const AGENT_OPENED_THROTTLE_MS = 3_000const BASE_DELAY_MS = 1_000const MAX_DELAY_MS = 30_000const QUEUE_MAX = 2_000+/** Requests one local connection may queue before the daemon is ready (see `openRequests`). The app+ * sends a handful on connect; hundreds is a client looping, not a person. */+const MAX_REQUESTS_BEFORE_READY = 256const DEVICE_AGENT_LIST_LIMIT = 100const DEVICE_AGENT_NAME_MAX_CODEPOINTS = 15const DEVICE_AGENT_NAME_MAX_BYTES = 39 // device project_t.name[40], including trailing NUL on-device.@@ -488,6 +491,17 @@ export class BackendSocket {// real and must be counted once, so it is owed to the next link — not turned into a `ping`.private appOpenOwed = falseprivate readonly downChains = new Map<string, Promise<void>>()+ /**+ * Requests wait here until the daemon is ready (`openRequests`), each in its connection's own order.+ * The port answers long before start-up has wired every handler and confirmed the agents it+ * restored; a request answered in between met a handler that was not there yet — refused with+ * UNSUPPORTED_ON_REMOTE, or for `message` silently dropped — or a registry not yet reconciled.+ */+ private requestsOpen = true+ private openRequestGate: () => void = () => {}+ private requestGate: Promise<void> = Promise.resolve()+ /** Requests waiting at the gate, per connection: a client that floods a starting daemon is closed. */+ private readonly waitingAtGate = new Map<string, number>()private readonly localClients = new Map<string, LocalClientSink>()/*** Loopback clients that are TOOLS, not windows (`machine_select { tool: true }`): `harness pair`, the@@ -1556,11 +1570,40 @@ export class BackendSocket {console.warn(`[terminal-p2p] conn=${sid(connId)} fallback=relay reason=${reason}`)}+ /** Hold requests at the gate until `openRequests`. The daemon calls this the moment it builds this+ * socket, before anything can reach it; a socket built without it answers at once. */+ holdRequests(): void {+ if (!this.requestsOpen) return+ this.requestsOpen = false+ this.requestGate = new Promise<void>((resolve) => { this.openRequestGate = resolve })+ }++ /** Let requests through: every handler is wired and the agents the daemon restored are confirmed.+ * Idempotent. Called once at the end of start-up — and by safe mode, so a daemon that could not+ * start still answers rather than leaving its clients waiting. */+ openRequests(): void {+ if (this.requestsOpen) return+ this.requestsOpen = true+ this.waitingAtGate.clear()+ this.openRequestGate()+ }+private enqueueDown(frame: Frame, connId: string, transport: DownTransport = 'relay'): void {const key = connId || '__backend__'+ if (!this.requestsOpen) {+ const waiting = (this.waitingAtGate.get(key) ?? 0) + 1+ this.waitingAtGate.set(key, waiting)+ if (waiting > MAX_REQUESTS_BEFORE_READY && transport === 'local') {+ console.warn(`[backend] local client ${key} sent ${waiting} requests before the daemon was ready — closing it`)+ this.waitingAtGate.delete(key)+ void this.unregisterLocalClient(connId)+ return+ }+ }const previous = this.downChains.get(key) ?? Promise.resolve()const next = previous.catch(() => { /* prior failure is already logged */ })+ .then(() => this.requestGate).then(() => this.dispatchDown(frame, connId, transport)).catch((err) => {console.error('[backend] down-frame dispatch failed:', err instanceof Error ? err.message : err)diff --git a/cli/src/cli.ts b/cli/src/cli.tsindex 8860078df..0dc7f12af 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -55,7 +55,7 @@ import { registry, projectDisplayName, sessionDisplayTitle, validTranscriptPath,import { engineSessionTitle } from './lib/sessionTitle.js'import { installAmpPlugin, installCodexHooks, installCommandCodeHooks, installCursorHooks, installDevinHooks, installGrokHooks, installAgyHooks, installCopilotHooks, installHermesHooks, installKiloPlugin, installOpencodePlugin, installPiExtension, installSessionHooks } from './lib/hooks.js'import { PID_FILE, daemonPort, isAlive, isDaemonRunning, readPid } from './lib/daemonState.js'-import { clearSafeModeMarker, readSafeModeMarker, runBootHandoff, safeModeDisposition, safeModeStatusBody, writeSafeModeMarker } from './lib/daemonSafeMode.js'+import { clearSafeModeMarker, readSafeModeMarker, runBootHandoff, safeModeDisposition, safeModeStatusBody, SafeModeRequest, writeSafeModeMarker } from './lib/daemonSafeMode.js'import { awakeTimeout } from './lib/sleepAware.js'import {BIND_WAIT_MS, connectFailure, defaultLaunchDeps, removePidFileIf, waitForBind, waitForReady,@@ -172,6 +172,9 @@ import {} from './lib/terminalTypes.js'import { Watcher, type HistoryEvent, type LineEvent, type RewrittenEvent, type TailHold } from './watcher/watcher.js'import { chooseHookAgent, startHookServer } from './hookServer.js'+import { connectToMaster } from './harnessd/coreLink.js'+import { describeMasterStatus, readStatusFile, runMaster } from './harnessd/master.js'+import { CORE_EXIT_STOP, CORE_EXIT_UPDATE } from './harnessd/protocol.js'import { isLocalSocketName, localSocketPath, type LocalSocketServer } from './lib/localSocket.js'import { legacyDaemonStatus, localDaemonStatus, saveDaemonPort } from './lib/daemonEndpoint.js'import { commandBarService } from './lib/commandBar.js'@@ -287,6 +290,8 @@ const STOP_HOOK_GRACE_MS = 1_500// Daemon stdout/stderr. Capped at LOG_MAX_BYTES — see prepareLogFile/trimLogFile in lib/log.ts.const LOG_FILE = join(env.ADAPTER_DATA_DIR, 'harness.log')+/** What harnessd's master last said about itself, for `harness status` when no core answers. */+const HARNESSD_STATUS_FILE = join(env.ADAPTER_DATA_DIR, 'harnessd-status.json')// Pre-rename name. Adopted (renamed, keeping the inode) the first time a daemon opens the log, so a// machine that updates mid-run keeps its history instead of stranding it in a file nobody tails.// The log has had three names; this slot holds the OLDEST. The middle one (`machine.log`) is adopted@@ -532,6 +537,10 @@ function spawnDaemonChild(extraEnv: Record<string, string>): ReturnType<typeof s* exists — hence the indirection: `applyStagedUpdate` is `bootHandoff` until the body has built* everything `restartForUpdate` tears down, and is swapped for it at that one line.*/+/** This process's channel to a harnessd master, when one started it (see harnessd/coreLink.ts).+ * Inert otherwise: a daemon run on its own claims its pid file and hands off updates itself. */+const coreLink = connectToMaster()+const daemonBoot: {updater: Poller | nulltuiUpdater: Poller | null@@ -545,7 +554,9 @@ const daemonBoot: {safeMode: string | nullhandingOff: booleanapplyStagedUpdate: (version: string) => void | Promise<void>-} = { updater: null, tuiUpdater: null, hookServer: null, localSocket: null, markNotReady: null, safeMode: null, handingOff: false, applyStagedUpdate: bootHandoff }+ /** Opens the request gate of a start-up that did not finish, so its clients are answered (safe mode). */+ openRequests: (() => void) | null+} = { updater: null, tuiUpdater: null, hookServer: null, localSocket: null, markNotReady: null, safeMode: null, handingOff: false, applyStagedUpdate: bootHandoff, openRequests: null }/*** Hand the machine to a newer build without finishing start-up.@@ -564,6 +575,14 @@ function bootHandoff(version: string): void {if (daemonBoot.handingOff) returndaemonBoot.handingOff = truedaemonBoot.tuiUpdater?.stop()+ if (coreLink.supervised) {+ // The master starts the new bundle the moment this exits, and rolls it back if it does not stay+ // up; a successor spawned from here would be a daemon outside its supervision.+ console.log(`[update] ${VERSION} → ${version} staged during start-up — handing back to harnessd`)+ try { daemonBoot.hookServer?.close() } catch { /* already gone */ }+ try { daemonBoot.localSocket?.closeSync() } catch { /* already gone */ }+ process.exit(CORE_EXIT_UPDATE)+ }runBootHandoff(VERSION, version, {// The hook port has no fallback: a successor that cannot bind it is a daemon that does not come up.closeServer: () => {@@ -1391,7 +1410,8 @@ async function downloadCanaryStage(entry: UpdateEntry, dir: string, log: (m: strconst cliBuf = await downloadVerified(entry.cli)const notifyBuf = await downloadVerified(entry.notify)if (!canary(cliBuf, dir)) { log(` ✗ the new build failed its self-check — keeping v${VERSION}`); return false }- stage(dir, cliBuf, notifyBuf)+ // Asked for by name, so a version this machine once rolled back is installed and no longer rejected.+ stage(dir, cliBuf, notifyBuf, entry.version)confirmUpdate(dir) // canary passed + bytes already verified ⇒ drop the .prev backupsreturn true}@@ -1671,6 +1691,10 @@ async function runForeground(session: AuthSession | null): Promise<void> {slotSecond: env.ADAPTER_UPDATE_SLOT_SEC,})+ // harnessd's master saw this core crash again and again: start nothing that could do it again. The+ // updaters above keep running, so a published fix still lands (`enterSafeMode`).+ if (process.env.HARNESSD_SAFE_MODE) throw new SafeModeRequest(process.env.HARNESSD_SAFE_MODE)+const savedApis = new ApiConnections(env.ADAPTER_DATA_DIR)// Before any agent is probed: one already running on a saved API's model reports that model.rememberSavedApis(savedApis)@@ -2078,6 +2102,9 @@ async function runForeground(session: AuthSession | null): Promise<void> {})}, computerId(), autonomousEnv)backendRef = backend+ // Nothing is answered until start-up is done (see the end of this function).+ backend.holdRequests()+ daemonBoot.openRequests = () => backend.openRequests()backend.viewerTargetProvider = (agentId) => dshViewers.forwardingUrl(agentId)ensureBundledCoreHarnesses()@@ -2123,6 +2150,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {log: gridLog,}, { ownGrid, signedInThisRun }),})+backend.ensureGrid = (request) => gridAccess.ensure(request)// Offline, for every list read: is there a `grid` here holding a sign-in? What decides whether the// picker offers local and shared models or a Set up row.@@ -4305,8 +4333,16 @@ async function runForeground(session: AuthSession | null): Promise<void> {deviceTransportConnected: backend.hasCommander(),deviceE2eeConnected: backend.deviceE2eeConnected(),uptimeSec: Math.round((Date.now() - startedAt) / 1000),- pid: process.pid,+ // The daemon as everything outside knows it: the pid file's pid, which `harness stop` signals+ // and the desktop app judges the owner of. Under a master that is the master's. A core's pid+ // changes with every restart, and macOS counts a core as its master's, so an app judging the+ // core would read a daemon started from tmux or ssh as "owned by node" and restart it on sight.+ pid: coreLink.masterPid ?? process.pid,+ corePid: process.pid,startedAt,+ // The master keeping this core running, when one is: how often it has restarted it, and why the+ // last one ended. Null for a core run on its own.+ harnessd: coreLink.supervised ? { masterPid: coreLink.masterPid, ...coreLink.status() } : null,// True for the few hundred ms between an update being staged and this server closing for the// handoff. Informational: nothing should build readiness on a field the server stops serving.restarting,@@ -4379,7 +4415,13 @@ async function runForeground(session: AuthSession | null): Promise<void> {// that LOST the port to a sibling still left a file naming itself, a corpse, over the winner. A// process that is running AND holds the port is the only honest author of its own pid; that claim// is also the signal `harness start` and the update handoff wait on to know the bind succeeded.- try { writeFileSync(PID_FILE, String(process.pid) + '\n') } catch { /* best effort */ }+ // Under harnessd the master claims it, for itself, when this core says it is bound.+ if (coreLink.supervised) {+ coreLink.bound(hookPort)+ coreLink.startHeartbeat()+ } else {+ try { writeFileSync(PID_FILE, String(process.pid) + '\n') } catch { /* best effort */ }+ }// The one thing a handoff that happens before start-up finishes has to release: the port has no// fallback, so a successor that cannot bind it is a daemon that does not come up (see bootHandoff).daemonBoot.hookServer = hookServer@@ -6370,10 +6412,11 @@ async function runForeground(session: AuthSession | null): Promise<void> {// Keep the log file under its cap. This daemon writes it through an inherited stdout fd, so a size// check on a timer is the only place that can see it grow — `prepareLogFile` at spawn time alone// would let a long-lived, chatty daemon run unbounded between restarts.- const logTrimTimer = setInterval(() => {+ // A core run by harnessd leaves this to its master, which outlives it (harnessd/master.ts).+ const logTrimTimer = coreLink.supervised ? undefined : setInterval(() => {if (trimLogFile(LOG_FILE)) console.log(`[log] ${tildify(LOG_FILE)} hit its size cap — dropped the oldest half`)}, LOG_CHECK_INTERVAL_MS)- logTrimTimer.unref?.() // never hold the event loop open for log upkeep+ logTrimTimer?.unref?.() // never hold the event loop open for log upkeep// Signed out, the backend is not dialed at all. The socket would only meet a missing session and back// off forever, one log line at a time; a sign-in RESTARTS this process with the session in hand@@ -6441,6 +6484,12 @@ async function runForeground(session: AuthSession | null): Promise<void> {await backend.stop() // graceful WS close → releases the Redis machine-owner claimawait new Promise((r) => setTimeout(r, 1000)) // grace before the same-machine reclaim+ if (coreLink.supervised) {+ // Everything above is released; the master starts the new bundle as soon as this exits and+ // rolls back to the .prev bytes if it does not come up and stay up (harnessd/supervisor.ts).+ console.log(`[update] handing ${newVersion} to harnessd`)+ process.exit(CORE_EXIT_UPDATE)+ }const sinceOffset = existsSync(LOG_FILE) ? statSync(LOG_FILE).size : 0const child = spawnDaemonChild({ ADAPTER_UPDATED_TO: newVersion })handoffChild = child@@ -6500,6 +6549,9 @@ async function runForeground(session: AuthSession | null): Promise<void> {handoffChild = null})+ /** Stopping for good — removed from the account, or connected from elsewhere: tell harnessd's master,+ * which restarts any other exit (harnessd/protocol.ts). */+ const forGood = (reason: string): boolean => reason === 'revoked' || reason === 'busy'const shutdown = async (signal: string): Promise<void> => {console.log(`\n[cli] ${signal} — shutting down`)// Mid-handoff everything below has already been torn down once, and the daemon that matters is@@ -6546,10 +6598,13 @@ async function runForeground(session: AuthSession | null): Promise<void> {autonomousDeviceDirect?.stop()await backend.stop()try { if (readPid() === process.pid) rmSync(PID_FILE, { force: true }) } catch { /* ignore */ }- process.exit(0)+ process.exit(coreLink.supervised && forGood(signal) ? CORE_EXIT_STOP : 0)}process.on('SIGINT', () => void shutdown('SIGINT'))process.on('SIGTERM', () => void shutdown('SIGTERM'))+ // A core whose master is gone stops, so nothing is left holding the port for a master that is not+ // there to restart it.+ coreLink.onMasterGone(() => void shutdown('the harnessd master is gone'))process.on('exit', () => { shutdownVoiceRouter() })// A machine revocation or invalid SSO refresh ends this adapter session permanently.@@ -6858,6 +6913,15 @@ async function runForeground(session: AuthSession | null): Promise<void> {if (env.CABLE_DISABLE) console.log('[cable] disabled (CABLE_DISABLE=true) — the serial port is left alone')else cable.start()+ // Last: every handler is wired and the restored agents are confirmed, so requests that arrived while+ // starting — a client reconnecting the moment the port answered, the backend's first frames — are+ // answered now, in order, by the handlers meant to answer them (see BackendSocket.openRequests).+ // Only the end-to-end harness sets this: a start-up that hangs after binding, for the master's deadline.+ if (process.env.HARNESSD_TEST_HOLD_READY === '1') await new Promise<never>(() => {})+ backend.openRequests()+ daemonBoot.openRequests = null+ coreLink.ready()+ console.log('[cli] ready')}// ── info block ───────────────────────────────────────────────────────────────────────────────────@@ -7021,7 +7085,10 @@ async function spawnDaemon(session: AuthSession | null, runtimeNode: string | nuconst logFd = openSync(LOG_FILE, 'a')// The daemon starts on the managed runtime straight away rather than inheriting this process's// interpreter and waiting for some later restart to adopt it.- const child = spawn(runtimeNode ?? process.execPath, [SCRIPT_PATH, '__run'], {+ // harnessd: a master that keeps the daemon's core running (harnessd/supervisor.ts). HARNESS_NO_MASTER=1+ // starts the core on its own, as before, for a machine where the master itself is in question.+ const entry = process.env.HARNESS_NO_MASTER === '1' ? '__run' : '__harnessd'+ const child = spawn(runtimeNode ?? process.execPath, [SCRIPT_PATH, entry], {detached: true,env: { ...process.env },stdio: ['ignore', logFd, logFd],@@ -7949,6 +8016,8 @@ async function status(): Promise<void> {// A daemon whose start-up failed is alive and answering, but nothing on this machine works. Say so// in the one line a person reads, rather than leaving it looking like an ordinary slow start.const safeMode = alive ? readSafeModeMarker(env.ADAPTER_DATA_DIR, isAlive) : null+ // A core that cannot answer — restarting, starting, crash-looping — is described by its master.+ const master = alive && daemonStatus == null ? describeMasterStatus(readStatusFile(HARNESSD_STATUS_FILE, pid)) : nullprintInfoBlock({// The backend link is the daemon's own business, so `status` is where it is read — `start` no// longer waits to see it, and a daemon with no backend is still serving every local agent.@@ -7956,6 +8025,8 @@ async function status(): Promise<void> {// and `machine: not signed in` in place of the whole block hid a running daemon and its agents.status: !alive? '○ stopped'+ : master+ ? master: safeMode? `◍ safe mode · start-up failed on v${safeMode.version} — waiting for a fixed build (${safeMode.error.split('\n')[0]})`: !signedIn@@ -8085,7 +8156,7 @@ const onError = (err: unknown): never => {* `harness status` say what happened. `harness stop` still works throughout — it kills by pid.*/const enterSafeMode = (err: unknown): void => {- const disposition = safeModeDisposition(err, { selfPid: process.pid, readPid, isAlive })+ const disposition = safeModeDisposition(err, { selfPid: process.pid, masterPid: coreLink.masterPid, readPid, isAlive })if (!disposition.stay) {console.error(`[safe-mode] not staying up — ${disposition.reason}`)onError(err)@@ -8097,6 +8168,8 @@ const enterSafeMode = (err: unknown): void => {writeSafeModeMarker(env.ADAPTER_DATA_DIR, { pid: process.pid, version: VERSION, at: Date.now(), error: detail })daemonBoot.safeMode = disposition.reasondaemonBoot.markNotReady?.(disposition.reason)+ // Requests queued behind a start-up that will not finish are answered now, by whatever is wired.+ daemonBoot.openRequests?.()const leave = (why: string, code: number): never => {clearSafeModeMarker(env.ADAPTER_DATA_DIR)@@ -8106,6 +8179,13 @@ const enterSafeMode = (err: unknown): void => {}process.on('SIGINT', () => leave('SIGINT — leaving safe mode', 0))process.on('SIGTERM', () => leave('SIGTERM — leaving safe mode', 0))+ // Up, though not ready: the master must neither give up waiting for a bind nor take it for hung,+ // or the updater that can fix this build would never get its chance. It hears why, and rolls back an+ // update whose first core ends up here.+ coreLink.bound(daemonPort())+ coreLink.ready(disposition.reason)+ coreLink.startHeartbeat()+ coreLink.onMasterGone(() => leave('the harnessd master is gone — leaving safe mode', 0))// The bound control port is a ref'd handle and holds the loop on its own. Without one — the bind// itself was what failed, or we never got that far — take the port for the status alone, so the app@@ -8179,6 +8259,18 @@ switch (cmd) {case 'join':console.error('`harness join` has been removed. Run `harness login`, then `harness start`.')process.exit(1)+ case '__harnessd': // internal: the master `harness start` launches; it runs and supervises `__run`+ runMaster({+ nodePath: process.execPath,+ execArgv: process.execArgv,+ scriptPath: SCRIPT_PATH,+ pidFile: PID_FILE,+ statusFile: HARNESSD_STATUS_FILE,+ logFile: LOG_FILE,+ restoreUpdate: () => restoreUpdate(env.ADAPTER_CLI_DIR),+ confirmUpdate: () => confirmUpdate(env.ADAPTER_CLI_DIR),+ })+ breakcase '__run': // internal: the detached daemon child reads the durable SSO session — or runs without one// NOT `onError`: a daemon that dies here can never be updated. See `enterSafeMode`.runForeground(readAuthSession()).catch(enterSafeMode)diff --git a/cli/src/harnessd/coreLink.spec.ts b/cli/src/harnessd/coreLink.spec.tsnew file mode 100644index 000000000..f750f8e5d--- /dev/null+++ b/cli/src/harnessd/coreLink.spec.ts@@ -0,0 +1,127 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { connectToMaster, processChannel, processLoopDelay, type LoopDelay, type MasterChannel } from './coreLink.js'+import { HARNESSD_PROTOCOL, type CoreMessage } from './protocol.js'++class FakeChannel implements MasterChannel {+ readonly sent: CoreMessage[] = []+ private messageListeners: Array<(message: unknown) => void> = []+ private disconnect: Array<() => void> = []+ rss = 100+ parentPid = 4242+ send?: (message: CoreMessage) => unknown = (message) => { this.sent.push(message) }+ once(_event: 'disconnect', listener: () => void): void { this.disconnect.push(listener) }+ on(_event: 'message', listener: (message: unknown) => void): void { this.messageListeners.push(listener) }+ memoryUsage(): { rss: number; heapUsed: number } { return { rss: this.rss, heapUsed: this.rss / 2 } }+ say(message: unknown): void { for (const listener of this.messageListeners) listener(message) }+ leave(): void { for (const listener of this.disconnect) listener() }+}++const supervised = { HARNESSD_SUPERVISED: '1' }++/** Loop delays that read back as given, one per beat, and remember being stopped. */+const delays = (...values: number[]) => {+ const made: Array<LoopDelay & { stopped: boolean }> = []+ return {+ made,+ factory: () => {+ const queue = [...values]+ const delay = { stopped: false, take: () => queue.shift() ?? 0, stop: () => { delay.stopped = true } }+ made.push(delay)+ return delay+ },+ }+}++describe('connectToMaster', () => {+ beforeEach(() => vi.useFakeTimers())+ afterEach(() => vi.useRealTimers())++ it('is inert without a master: nothing sent, nothing listened for', () => {+ const channel = new FakeChannel()+ for (const link of [connectToMaster(channel, {}), connectToMaster({ ...channel, send: undefined, once: channel.once.bind(channel), on: channel.on.bind(channel), memoryUsage: () => channel.memoryUsage(), parentPid: 1 }, supervised)]) {+ expect(link.supervised).toBe(false)+ expect(link.masterPid).toBeNull()+ link.bound(1)+ link.ready()+ link.startHeartbeat()+ const gone = vi.fn()+ link.onMasterGone(gone)+ channel.leave()+ vi.advanceTimersByTime(60_000)+ expect(gone).not.toHaveBeenCalled()+ expect(link.status()).toBeNull()+ link.close()+ }+ expect(channel.sent).toEqual([])+ })++ it('says it is bound, then ready, then beats with its memory and loop delay until closed', () => {+ const channel = new FakeChannel()+ const loop = delays(4, 250)+ const link = connectToMaster(channel, supervised, 1_000, loop.factory)+ expect(link.supervised).toBe(true)+ expect(link.masterPid).toBe(4242)+ // Reparented once the master is gone: the master it had is still the one it names.+ channel.parentPid = 1+ expect(link.masterPid).toBe(4242)+ link.bound(18473)+ link.ready()+ link.startHeartbeat()+ link.startHeartbeat()+ channel.rss = 200+ vi.advanceTimersByTime(1_000)+ link.close()+ vi.advanceTimersByTime(5_000)+ link.close()+ expect(channel.sent).toEqual([+ { type: 'harnessd:bound', protocol: HARNESSD_PROTOCOL, port: 18473 },+ { type: 'harnessd:ready' },+ { type: 'harnessd:heartbeat', rssBytes: 100, heapUsedBytes: 50, loopDelayMs: 4 },+ { type: 'harnessd:heartbeat', rssBytes: 200, heapUsedBytes: 100, loopDelayMs: 250 },+ ])+ expect(loop.made).toHaveLength(1)+ expect(loop.made[0].stopped).toBe(true)+ })++ it('says why when start-up gave way to safe mode', () => {+ const channel = new FakeChannel()+ connectToMaster(channel, supervised).ready('no tmux')+ expect(channel.sent).toEqual([{ type: 'harnessd:ready', safeMode: 'no tmux' }])+ })++ it('keeps what the master says about itself, and hears when the master goes', () => {+ const channel = new FakeChannel()+ const link = connectToMaster(channel, supervised)+ channel.say({ type: 'other' })+ expect(link.status()).toBeNull()+ const status = { state: 'running', corePid: 7, restarts: 2, lastExit: 'code 1', protocol: 1 } as const+ channel.say({ type: 'harnessd:status', status })+ expect(link.status()).toEqual(status)+ const gone = vi.fn()+ link.onMasterGone(gone)+ channel.leave()+ expect(gone).toHaveBeenCalledOnce()+ })++ it('survives a send on a channel the master already closed', () => {+ const channel = new FakeChannel()+ channel.send = () => { throw new Error('ERR_IPC_CHANNEL_CLOSED') }+ const link = connectToMaster(channel, supervised)+ expect(() => link.bound(1)).not.toThrow()+ })++ it('uses this process by default, which no master started', () => {+ expect(connectToMaster().supervised).toBe(false)+ expect(processChannel.memoryUsage().rss).toBeGreaterThan(0)+ expect(processChannel.parentPid).toBe(process.ppid)+ })++ it('measures this process\'s loop delay, in whole milliseconds, from nothing after each reading', () => {+ vi.useRealTimers()+ const delay = processLoopDelay()+ const first = delay.take()+ expect(Number.isInteger(first) && first >= 0).toBe(true)+ expect(delay.take()).toBeGreaterThanOrEqual(0)+ delay.stop()+ })+})diff --git a/cli/src/harnessd/coreLink.ts b/cli/src/harnessd/coreLink.tsnew file mode 100644index 000000000..055c0463e--- /dev/null+++ b/cli/src/harnessd/coreLink.ts@@ -0,0 +1,118 @@+/**+ * The core's half of harnessd's spawn channel (see ./protocol.ts and ./supervisor.ts).+ *+ * Inert unless a harnessd master started this process: a core run on its own (`harness __run`, the+ * test harness) behaves exactly as before.+ */+import { monitorEventLoopDelay } from 'node:perf_hooks'+import { HARNESSD_PROTOCOL, isMasterMessage, type CoreMessage } from './protocol.js'+import type { SupervisorStatus } from './supervisor.js'++/** How much the event loop was held since the last beat: the longest pause, in ms. */+export interface LoopDelay {+ /** The longest the loop was held since the last call, and start counting again. */+ take(): number+ stop(): void+}++/** The event-loop delay monitor this process runs while it beats. */+export function processLoopDelay(): LoopDelay {+ const histogram = monitorEventLoopDelay({ resolution: 20 })+ histogram.enable()+ return {+ take: () => {+ // Nanoseconds; 0 before the first sample.+ const longest = Math.round(histogram.max / 1e6)+ histogram.reset()+ return longest+ },+ stop: () => histogram.disable(),+ }+}++/** How often a core tells its master it is alive. The master's patience is several of these. */+export const HEARTBEAT_INTERVAL_MS = 5_000++export interface MasterChannel {+ send?: (message: CoreMessage) => unknown+ once(event: 'disconnect', listener: () => void): unknown+ on(event: 'message', listener: (message: unknown) => void): unknown+ memoryUsage(): { rss: number; heapUsed: number }+ /** The process that started this one: the master, when it holds the channel. */+ readonly parentPid: number+}++export interface CoreLink {+ /** Started by a master, with the channel to it still open. */+ readonly supervised: boolean+ /** The master's pid, as the pid file names the daemon; null without one. Read once, at start: a+ * core whose master died is handed to launchd, and must not then name it as its master. */+ readonly masterPid: number | null+ /** The control port is bound: the master may now tell everyone the daemon is up. */+ bound(port: number): void+ /** Start-up is done and requests are served — or it gave way to safe mode, and why: the master stops+ * waiting for it either way, and rolls back an update whose first core ends up in safe mode. */+ ready(safeMode?: string): void+ /** Tell the master, every `HEARTBEAT_INTERVAL_MS`, that this core is alive and how big it is. */+ startHeartbeat(): void+ /** The master is gone. A core without one stops, so nothing is left holding the port. */+ onMasterGone(listener: () => void): void+ /** What the master last said about itself (restarts, the last exit), for `/api/status`. */+ status(): SupervisorStatus | null+ close(): void+}++/** This process's own channel: present only when it was spawned with one (`stdio` 'ipc'). */+export const processChannel: MasterChannel = {+ send: process.send?.bind(process),+ once: process.once.bind(process),+ on: process.on.bind(process),+ memoryUsage: () => process.memoryUsage(),+ parentPid: process.ppid,+}++export function connectToMaster(+ channel: MasterChannel = processChannel,+ env: NodeJS.ProcessEnv = process.env,+ intervalMs = HEARTBEAT_INTERVAL_MS,+ loopDelay: () => LoopDelay = processLoopDelay,+): CoreLink {+ const supervised = env.HARNESSD_SUPERVISED === '1' && typeof channel.send === 'function'+ let heartbeat: ReturnType<typeof setInterval> | null = null+ let delay: LoopDelay | null = null+ let status: SupervisorStatus | null = null+ // A send on a channel the master closed throws; the master is gone and `onMasterGone` says so.+ const send = (message: CoreMessage): void => {+ if (!supervised) return+ try { channel.send!(message) } catch { /* closing */ }+ }+ if (supervised) {+ channel.on('message', (message) => { if (isMasterMessage(message)) status = message.status })+ }+ return {+ supervised,+ masterPid: supervised ? channel.parentPid : null,+ bound: (port) => send({ type: 'harnessd:bound', protocol: HARNESSD_PROTOCOL, port }),+ ready: (safeMode) => send(safeMode === undefined ? { type: 'harnessd:ready' } : { type: 'harnessd:ready', safeMode }),+ startHeartbeat: () => {+ if (!supervised || heartbeat) return+ delay = loopDelay()+ const monitor = delay+ const beat = (): void => {+ const usage = channel.memoryUsage()+ send({ type: 'harnessd:heartbeat', rssBytes: usage.rss, heapUsedBytes: usage.heapUsed, loopDelayMs: monitor.take() })+ }+ beat()+ heartbeat = setInterval(beat, intervalMs)+ heartbeat.unref()+ },+ onMasterGone: (listener) => { if (supervised) channel.once('disconnect', listener) },+ status: () => status,+ close: () => {+ if (heartbeat) clearInterval(heartbeat)+ heartbeat = null+ delay?.stop()+ delay = null+ },+ }+}diff --git a/cli/src/harnessd/master.spec.ts b/cli/src/harnessd/master.spec.tsnew file mode 100644index 000000000..777f4ea69--- /dev/null+++ b/cli/src/harnessd/master.spec.ts@@ -0,0 +1,319 @@+import { EventEmitter } from 'node:events'+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { spawn, type ChildProcess } from 'node:child_process'+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import {+ coreExecArgv, coreHandle, describeMasterStatus, heapLimitInArgv, masterDefaults, onProcessSignal, processExit,+ readStatusFile, runMaster, supervisorOptions, trimLogEvery, writeStatusFile, type MasterStatusFile,+} from './master.js'+import { DEFAULT_SUPERVISOR_OPTIONS } from './supervisor.js'++describe('supervisorOptions', () => {+ it('reads valid overrides and keeps the defaults for anything unset or invalid', () => {+ expect(supervisorOptions({})).toEqual(DEFAULT_SUPERVISOR_OPTIONS)+ expect(supervisorOptions({+ HARNESSD_BIND_TIMEOUT_MS: '100', HARNESSD_READY_TIMEOUT_MS: '150', HARNESSD_HEARTBEAT_TIMEOUT_MS: '200',+ HARNESSD_STOP_GRACE_MS: '300', HARNESSD_INITIAL_BACKOFF_MS: '0', HARNESSD_MAX_BACKOFF_MS: '50',+ HARNESSD_BACKOFF_RESET_MS: '0', HARNESSD_HEAP_LIMIT_MIB: '512', HARNESSD_HEAP_RESTART_PERCENT: '90',+ HARNESSD_RSS_LIMIT_MIB: '0', HARNESSD_UPDATE_PROBATION_MS: '10', HARNESSD_CRASH_LOOP_CRASHES: '5',+ HARNESSD_CRASH_LOOP_WINDOW_MS: '1000',+ })).toEqual({+ bindTimeoutMs: 100, readyTimeoutMs: 150, heartbeatTimeoutMs: 200, stopGraceMs: 300, initialBackoffMs: 0,+ maxBackoffMs: 50, backoffResetMs: 0, heapLimitMiB: 512, heapRestartPercent: 90, rssLimitMiB: 0,+ updateProbationMs: 10, crashLoopCrashes: 5, crashLoopWindowMs: 1000,+ })+ expect(supervisorOptions({+ HARNESSD_BIND_TIMEOUT_MS: '0', HARNESSD_HEARTBEAT_TIMEOUT_MS: 'soon', HARNESSD_RSS_LIMIT_MIB: '-1',+ HARNESSD_HEAP_RESTART_PERCENT: '101', HARNESSD_CRASH_LOOP_CRASHES: '0',+ })).toEqual(DEFAULT_SUPERVISOR_OPTIONS)+ })++ it('takes the heap limit from the flags the master was given, unless the environment names one', () => {+ expect(supervisorOptions({}, ['--import', 'tsx', '--max-old-space-size=256']).heapLimitMiB).toBe(256)+ expect(supervisorOptions({ HARNESSD_HEAP_LIMIT_MIB: '2048' }, ['--max-old-space-size=256']).heapLimitMiB).toBe(2048)+ })+})++describe('the core\'s flags', () => {+ it('finds the last heap limit given, in either spelling', () => {+ expect(heapLimitInArgv([])).toBeNull()+ expect(heapLimitInArgv(['--max-old-space-size=100', '--max_old_space_size=200', '--inspect'])).toBe(200)+ expect(heapLimitInArgv(['--max-old-space-size=lots'])).toBeNull()+ })++ it('gives the core exactly the heap limit its budget is a share of, or V8\'s own when that is 0', () => {+ expect(coreExecArgv(['--import', 'tsx', '--max-old-space-size=256'], 512)).toEqual(['--import', 'tsx', '--max-old-space-size=512'])+ expect(coreExecArgv(['--max_old_space_size=256'], 0)).toEqual([])+ })+})++describe('the status file', () => {+ let dir: string+ beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'harnessd-status-')) })+ afterEach(() => rmSync(dir, { recursive: true, force: true }))+ const status = (over: Partial<MasterStatusFile> = {}): MasterStatusFile => ({+ state: 'running', corePid: 7, restarts: 0, lastExit: null, lastExitReason: null, safeMode: null,+ protocol: 2, since: 1, masterPid: 42, ...over,+ })++ it('is written whole and read back only for the master that wrote it', () => {+ const file = join(dir, 'harnessd-status.json')+ writeStatusFile(file, status())+ expect(readStatusFile(file, 42)).toEqual(status())+ expect(readStatusFile(file, 43)).toBeNull()+ expect(readStatusFile(join(dir, 'none.json'), 42)).toBeNull()+ writeFileSync(file, '{"masterPid": 42}')+ expect(readStatusFile(file, 42)).toBeNull()+ expect(existsSync(`${file}.42.tmp`)).toBe(false)+ })++ it('never stops the master over a disk it cannot write', () => {+ expect(() => writeStatusFile(join(dir, 'missing', 'harnessd-status.json'), status())).not.toThrow()+ })++ it('says what the master knows when the core cannot answer', () => {+ expect(describeMasterStatus(null)).toBeNull()+ expect(describeMasterStatus(status())).toBeNull()+ expect(describeMasterStatus(status({ state: 'restarting', restarts: 3, lastExit: 'signal SIGKILL', lastExitReason: 'hung' })))+ .toBe('◍ restarting its core · restart 3 · last core ended because it hung (signal SIGKILL)')+ expect(describeMasterStatus(status({ state: 'starting' }))).toBe('● starting')+ expect(describeMasterStatus(status({ state: 'listening', restarts: 1, lastExit: 'code 1', lastExitReason: 'crashed' })))+ .toBe('● starting · restart 1 · last core ended because it crashed (code 1)')+ expect(describeMasterStatus(status({ safeMode: 'crash-loop' })))+ .toBe('◍ safe mode · the core kept crashing — waiting for a fixed build')+ expect(describeMasterStatus(status({ safeMode: 'no tmux', lastExit: 'code 7', lastExitReason: 'mystery' as never })))+ .toBe('◍ safe mode · the core could not start (no tmux) — waiting for a fixed build · last core ended because mystery (code 7)')+ })+})++describe('the process defaults', () => {+ it('exit through process.exit and listen for signals on the process', () => {+ const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never)+ let calls: unknown[][] = []+ try { processExit(3); calls = exit.mock.calls.map((call) => [...call]) } finally { exit.mockRestore() }+ expect(calls).toEqual([[3]])+ const listener = vi.fn()+ onProcessSignal('SIGHUP', listener)+ try { process.emit('SIGHUP') } finally { process.removeListener('SIGHUP', listener) }+ expect(listener).toHaveBeenCalledOnce()+ })+})++describe('trimLogEvery', () => {+ beforeEach(() => vi.useFakeTimers())+ afterEach(() => vi.useRealTimers())++ it('trims the log on its interval until stopped, and does nothing without one', () => {+ const trimmed: string[] = []+ const stop = trimLogEvery('/logs/harness.log', 1_000, (file) => { trimmed.push(file); return false })+ vi.advanceTimersByTime(2_999)+ expect(trimmed).toEqual(['/logs/harness.log', '/logs/harness.log'])+ stop()+ vi.advanceTimersByTime(5_000)+ expect(trimmed).toHaveLength(2)+ expect(() => trimLogEvery(undefined)()).not.toThrow()+ })++ it('uses the daemon log trim by default', () => {+ const dir = mkdtempSync(join(tmpdir(), 'harnessd-trim-'))+ try {+ const file = join(dir, 'harness.log')+ writeFileSync(file, 'x'.repeat(11 * 1024 * 1024))+ const stop = trimLogEvery(file, 1_000)+ vi.advanceTimersByTime(1_000)+ stop()+ expect(readFileSync(file).length).toBeLessThan(11 * 1024 * 1024)+ } finally { rmSync(dir, { recursive: true, force: true }) }+ })+})++describe('masterDefaults', () => {+ it('takes what the config gives, and this process for the rest', () => {+ const base = { nodePath: 'n', execArgv: [], scriptPath: 's', pidFile: 'p', restoreUpdate: () => {}, confirmUpdate: () => {} }+ expect(masterDefaults(base)).toEqual({ env: process.env, exit: processExit, onSignal: onProcessSignal })+ const exit = () => {}+ const onSignal = () => {}+ expect(masterDefaults({ ...base, env: { A: '1' }, exit, onSignal })).toEqual({ env: { A: '1' }, exit, onSignal })+ })+})++describe('coreHandle', () => {+ const fake = () => {+ const child = new EventEmitter() as EventEmitter & { pid: number; send: (m: unknown) => void; kill: (s: string) => void }+ child.pid = 4242+ child.send = vi.fn()+ child.kill = vi.fn()+ return child+ }++ it('reports one exit however the child ends, a failed spawn included', () => {+ for (const end of [(c: EventEmitter) => { c.emit('exit', 3, null); c.emit('error', new Error('late')) }, (c: EventEmitter) => { c.emit('error', new Error('ENOENT')); c.emit('exit', 1, null) }]) {+ const child = fake()+ const handle = coreHandle(child as unknown as ChildProcess)+ const exits: unknown[] = []+ handle.onExit((code, signal) => exits.push([code, signal]))+ end(child)+ expect(exits).toHaveLength(1)+ }+ })++ it('passes messages and signals through, and swallows them for a child that is gone', () => {+ const child = fake()+ const handle = coreHandle(child as unknown as ChildProcess)+ const messages: unknown[] = []+ handle.onMessage((message) => messages.push(message))+ child.emit('message', { type: 'x' })+ handle.send({ type: 'harnessd:status', status: { state: 'running', corePid: 1, restarts: 0, lastExit: null, lastExitReason: null, safeMode: null, protocol: 2, since: 0 } })+ handle.kill('SIGTERM')+ expect(handle.pid).toBe(4242)+ expect(messages).toEqual([{ type: 'x' }])+ expect(child.send).toHaveBeenCalledOnce()+ expect(child.kill).toHaveBeenCalledWith('SIGTERM')+ child.send = () => { throw new Error('closed') }+ child.kill = () => { throw new Error('ESRCH') }+ expect(() => { handle.send({ type: 'harnessd:status', status: {} as never }); handle.kill('SIGKILL') }).not.toThrow()+ })+})++// A real master over a real child: a few lines of JavaScript that behave like a core.+describe('runMaster', () => {+ let dir: string+ beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'harnessd-master-')) })+ afterEach(() => rmSync(dir, { recursive: true, force: true }))++ const until = async (what: string, test: () => boolean, ms = 10_000) => {+ const deadline = Date.now() + ms+ while (!test()) {+ if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`)+ await new Promise((resolve) => setTimeout(resolve, 20))+ }+ }++ it('starts the core, claims the pid file when it binds, restarts it after a crash, and stops it on a signal', async () => {+ const pidFile = join(dir, 'adapter.pid')+ const runs = join(dir, 'runs')+ const core = join(dir, 'core.cjs')+ writeFileSync(core, `+ const { appendFileSync } = require('node:fs')+ appendFileSync(${JSON.stringify(runs)}, process.env.HARNESSD_RESTARTS + '\\n')+ if (process.argv[2] !== '__run') process.exit(9)+ process.send({ type: 'harnessd:bound', protocol: 1, port: 1 })+ setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1 }), 50)+ process.on('SIGTERM', () => process.exit(0))+ if (process.env.HARNESSD_RESTARTS === '0') setTimeout(() => process.exit(1), 100)+ `)+ const signals = new Map<string, () => void>()+ const exits: number[] = []+ const updates: string[] = []+ const supervisor = runMaster({+ nodePath: process.execPath,+ execArgv: [],+ scriptPath: core,+ pidFile,+ restoreUpdate: () => updates.push('restore'),+ confirmUpdate: () => updates.push('confirm'),+ env: { ...process.env, HARNESSD_INITIAL_BACKOFF_MS: '10' },+ exit: (code) => exits.push(code),+ onSignal: (signal, listener) => signals.set(signal, listener),+ })+ expect([...signals.keys()]).toEqual(['SIGTERM', 'SIGINT', 'SIGHUP'])+ await until('the pid file', () => existsSync(pidFile))+ expect(readFileSync(pidFile, 'utf8')).toBe(`${process.pid}\n`)+ await until('a restart', () => existsSync(runs) && readFileSync(runs, 'utf8') === '0\n1\n')+ await until('the restarted core to bind', () => supervisor.status().state === 'running')+ signals.get('SIGTERM')!()+ await until('the master to finish', () => exits.length > 0)+ expect(exits).toEqual([0])+ expect(existsSync(pidFile)).toBe(false)+ expect(updates).toEqual([])+ })++ it('records its status as it goes, and starts the core with the heap limit its budget is a share of', async () => {+ const pidFile = join(dir, 'adapter.pid')+ const statusFile = join(dir, 'harnessd-status.json')+ const flags = join(dir, 'flags')+ const core = join(dir, 'core.cjs')+ writeFileSync(core, `+ require('node:fs').writeFileSync(${JSON.stringify(flags)}, JSON.stringify(process.execArgv))+ process.send({ type: 'harnessd:bound', protocol: 2, port: 1 })+ process.send({ type: 'harnessd:ready' })+ setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)+ process.on('SIGTERM', () => process.exit(0))+ `)+ const exits: number[] = []+ const signals = new Map<string, () => void>()+ runMaster({+ nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile, statusFile, logFile: join(dir, 'harness.log'),+ restoreUpdate: () => {}, confirmUpdate: () => {},+ env: { ...process.env, HARNESSD_HEAP_LIMIT_MIB: '300' },+ exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),+ })+ await until('the core to be ready', () => readStatusFile(statusFile, process.pid)?.state === 'running')+ expect(JSON.parse(readFileSync(flags, 'utf8'))).toEqual(['--max-old-space-size=300'])+ signals.get('SIGTERM')!()+ await until('the master to finish', () => exits.length > 0)+ expect(readStatusFile(statusFile, process.pid)).toMatchObject({ state: 'stopped', lastExitReason: 'stopped' })+ })++ it.each([+ ['is gone', (pidFile: string) => rmSync(pidFile, { force: true })],+ ['holds no number', (pidFile: string) => writeFileSync(pidFile, 'garbage\n')],+ ])('finishes cleanly when its pid file %s by the time it leaves', async (_, disturb) => {+ const pidFile = join(dir, 'adapter.pid')+ const core = join(dir, 'core.cjs')+ writeFileSync(core, `process.send({ type: 'harnessd:bound', protocol: 1, port: 1 }); setTimeout(() => process.exit(0), 100)`)+ const exits: number[] = []+ runMaster({+ nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile,+ restoreUpdate: () => {}, confirmUpdate: () => {},+ exit: (code) => exits.push(code), onSignal: () => {},+ })+ await until('the pid file', () => existsSync(pidFile))+ disturb(pidFile)+ await until('the master to finish', () => exits.length > 0)+ expect(exits).toEqual([0])+ })++ it('leaves a pid file that is not its own alone', async () => {+ const pidFile = join(dir, 'adapter.pid')+ const core = join(dir, 'core.cjs')+ writeFileSync(core, `+ process.send({ type: 'harnessd:bound', protocol: 1, port: 1 })+ setTimeout(() => { require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, '1\\n'); process.exit(0) }, 50)+ `)+ const exits: number[] = []+ runMaster({+ nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile,+ restoreUpdate: () => {}, confirmUpdate: () => {},+ exit: (code) => exits.push(code), onSignal: () => {},+ })+ await until('the master to finish', () => exits.length > 0)+ expect(readFileSync(pidFile, 'utf8')).toBe('1\n')+ })++ // The defaults act on the process itself, so they are exercised in a real one: a master on its own,+ // whose core cannot even be spawned, that a SIGTERM must still stop cleanly.+ it('uses this process for exit and signals by default', async () => {+ const script = join(dir, 'master.mts')+ writeFileSync(script, `+ import { runMaster } from ${JSON.stringify(join(__dirname, 'master.ts'))}+ runMaster({+ nodePath: '/nonexistent/node', execArgv: [], scriptPath: '/nonexistent/core.js',+ pidFile: ${JSON.stringify(join(dir, 'adapter.pid'))}, restoreUpdate: () => {}, confirmUpdate: () => {},+ env: { ...process.env, HARNESSD_INITIAL_BACKOFF_MS: '20' },+ })+ console.log('READY')+ `)+ const child = spawn(process.execPath, ['--import', 'tsx', script], { cwd: join(__dirname, '../..'), stdio: ['ignore', 'pipe', 'pipe'] })+ let output = ''+ child.stdout.on('data', (chunk) => { output += chunk })+ child.stderr.on('data', (chunk) => { output += chunk })+ const exited = new Promise<number | null>((resolve) => child.once('exit', (code) => resolve(code)))+ await until('the master to start', () => output.includes('READY') && output.includes('restarting'), 20_000)+ child.kill('SIGTERM')+ expect(await exited).toBe(0)+ expect(output).toContain('[harnessd] SIGTERM — stopping')+ })+})diff --git a/cli/src/harnessd/master.ts b/cli/src/harnessd/master.tsnew file mode 100644index 000000000..87652f00e--- /dev/null+++ b/cli/src/harnessd/master.ts@@ -0,0 +1,197 @@+/**+ * `harness __harnessd`: the master process `harness start` launches (see ./supervisor.ts for what it+ * does). This file is only its wiring to the operating system: the core child, the pid file, the status+ * file, the log's size, signals.+ */+import { spawn, type ChildProcess } from 'node:child_process'+import { readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'+import { performance } from 'node:perf_hooks'+import { trimLogFile } from '../lib/log.js'+import type { MasterMessage } from './protocol.js'+import { DEFAULT_SUPERVISOR_OPTIONS, Supervisor, type CoreHandle, type SupervisorOptions, type SupervisorStatus } from './supervisor.js'++/** How often the master keeps the daemon's log under its cap. */+export const LOG_TRIM_INTERVAL_MS = 60_000++export interface MasterConfig {+ /** The node binary and flags the core runs with. */+ nodePath: string+ execArgv: string[]+ /** The CLI entry (`cli.js`, or `src/cli.ts` under tsx). */+ scriptPath: string+ pidFile: string+ /** Where the master records its status for `harness status` (see `writeStatusFile`). */+ statusFile?: string+ /** The log the master and its core write, kept under its cap by the master. */+ logFile?: string+ restoreUpdate(): void+ confirmUpdate(): void+ env?: NodeJS.ProcessEnv+ /** Defaults to `process.exit`. */+ exit?: (code: number) => void+ /** Where SIGTERM, SIGINT and SIGHUP are listened for; defaults to this process. */+ onSignal?: (signal: NodeJS.Signals, listener: () => void) => void+}++/** The defaults `runMaster` acts on this process with. */+export const processExit = (code: number): void => { process.exit(code) }+export const onProcessSignal = (signal: NodeJS.Signals, listener: () => void): void => { process.on(signal, listener) }++/** What a config leaves out, taken from this process. Pure: choosing a default does not act on one. */+export function masterDefaults(config: MasterConfig): Required<Pick<MasterConfig, 'env' | 'exit' | 'onSignal'>> {+ return { env: config.env ?? process.env, exit: config.exit ?? processExit, onSignal: config.onSignal ?? onProcessSignal }+}++/** The heap limit `--max-old-space-size` sets in these flags, MiB; null when none does. */+export function heapLimitInArgv(execArgv: readonly string[]): number | null {+ for (let i = execArgv.length - 1; i >= 0; i--) {+ const match = /^--max[-_]old[-_]space[-_]size=(\d+)$/.exec(execArgv[i])+ if (match) return Number(match[1])+ }+ return null+}++/**+ * Supervisor timings and budgets from the environment (for tests and support), and the flags the core+ * runs with. Anything unset or invalid keeps its default. The heap limit is one number in both places:+ * `HARNESSD_HEAP_LIMIT_MIB` when set, else a `--max-old-space-size` the master itself was given, else+ * the default — and the core is given exactly the limit its budget is a share of.+ */+export function supervisorOptions(env: NodeJS.ProcessEnv, execArgv: readonly string[] = []): SupervisorOptions {+ const read = (name: string, fallback: number, min: number, max = Infinity): number => {+ const value = Number(env[name])+ return env[name] !== undefined && Number.isFinite(value) && value >= min && value <= max ? value : fallback+ }+ const d = DEFAULT_SUPERVISOR_OPTIONS+ return {+ bindTimeoutMs: read('HARNESSD_BIND_TIMEOUT_MS', d.bindTimeoutMs, 1),+ readyTimeoutMs: read('HARNESSD_READY_TIMEOUT_MS', d.readyTimeoutMs, 1),+ heartbeatTimeoutMs: read('HARNESSD_HEARTBEAT_TIMEOUT_MS', d.heartbeatTimeoutMs, 1),+ stopGraceMs: read('HARNESSD_STOP_GRACE_MS', d.stopGraceMs, 1),+ initialBackoffMs: read('HARNESSD_INITIAL_BACKOFF_MS', d.initialBackoffMs, 0),+ maxBackoffMs: read('HARNESSD_MAX_BACKOFF_MS', d.maxBackoffMs, 0),+ backoffResetMs: read('HARNESSD_BACKOFF_RESET_MS', d.backoffResetMs, 0),+ heapLimitMiB: read('HARNESSD_HEAP_LIMIT_MIB', heapLimitInArgv(execArgv) ?? d.heapLimitMiB, 0),+ heapRestartPercent: read('HARNESSD_HEAP_RESTART_PERCENT', d.heapRestartPercent, 1, 100),+ rssLimitMiB: read('HARNESSD_RSS_LIMIT_MIB', d.rssLimitMiB, 0),+ updateProbationMs: read('HARNESSD_UPDATE_PROBATION_MS', d.updateProbationMs, 0),+ crashLoopCrashes: read('HARNESSD_CRASH_LOOP_CRASHES', d.crashLoopCrashes, 1),+ crashLoopWindowMs: read('HARNESSD_CRASH_LOOP_WINDOW_MS', d.crashLoopWindowMs, 0),+ }+}++/** The flags the core runs with: the master's own, with the heap limit its budget is a share of. */+export function coreExecArgv(execArgv: readonly string[], heapLimitMiB: number): string[] {+ const rest = execArgv.filter((flag) => !/^--max[-_]old[-_]space[-_]size=/.test(flag))+ return heapLimitMiB > 0 ? [...rest, `--max-old-space-size=${heapLimitMiB}`] : rest+}++/**+ * Record the master's status for `harness status`, which reads it when no core can answer — one that+ * is crash-looping, restarting, or in safe mode. Written whole and renamed into place, so a reader+ * never sees half of it.+ */+export function writeStatusFile(file: string, status: SupervisorStatus & { masterPid: number }): void {+ const temp = `${file}.${status.masterPid}.tmp`+ try {+ writeFileSync(temp, `${JSON.stringify(status)}\n`)+ renameSync(temp, file)+ } catch {+ // Best effort: a full disk must not stop the master from supervising.+ try { rmSync(temp, { force: true }) } catch { /* nothing more to do */ }+ }+}++export type MasterStatusFile = SupervisorStatus & { masterPid: number }++/** Keep `file` under its cap from now on; returns what stops it. Nothing to trim, nothing to stop. */+export function trimLogEvery(file: string | undefined, ms = LOG_TRIM_INTERVAL_MS, trim: (file: string) => boolean = trimLogFile): () => void {+ if (!file) return () => {}+ const timer = setInterval(() => trim(file), ms)+ return () => clearInterval(timer)+}++/** The master's status file, if it is there and was written by `masterPid`; null otherwise. */+export function readStatusFile(file: string, masterPid: number | null): MasterStatusFile | null {+ try {+ const status = JSON.parse(readFileSync(file, 'utf8')) as Partial<MasterStatusFile>+ if (typeof status.state !== 'string' || status.masterPid !== masterPid) return null+ return status as MasterStatusFile+ } catch {+ return null+ }+}++const REASONS: Record<string, string> = {+ crashed: 'it crashed', hung: 'it hung', 'did-not-bind': 'it did not start listening',+ 'not-ready': 'it did not finish starting', memory: 'it outgrew its memory budget', update: 'an update',+ stopped: 'it was stopped',+}++/** What `harness status` says when the core cannot answer for itself; null when the master has nothing+ * to add (a core is up, or nothing is known). */+export function describeMasterStatus(status: MasterStatusFile | null): string | null {+ if (!status) return null+ const last = status.lastExitReason ? ` · last core ended because ${REASONS[status.lastExitReason] ?? status.lastExitReason} (${status.lastExit})` : ''+ if (status.safeMode) return `◍ safe mode · the core ${status.safeMode === 'crash-loop' ? 'kept crashing' : `could not start (${status.safeMode})`} — waiting for a fixed build${last}`+ if (status.state === 'restarting') return `◍ restarting its core · restart ${status.restarts}${last}`+ if (status.state === 'starting' || status.state === 'listening') return `● starting${status.restarts ? ` · restart ${status.restarts}` : ''}${last}`+ return null+}++/** A child process as the supervisor sees it. A spawn that fails reports as an exit. */+export function coreHandle(child: ChildProcess): CoreHandle {+ let exited = false+ const exits: Array<(code: number | null, signal: NodeJS.Signals | null) => void> = []+ const exit = (code: number | null, signal: NodeJS.Signals | null): void => {+ if (exited) return+ exited = true+ for (const listener of exits) listener(code, signal)+ }+ child.on('exit', exit)+ child.on('error', () => exit(1, null))+ return {+ pid: child.pid,+ send: (message: MasterMessage) => { try { child.send(message) } catch { /* the core is going */ } },+ kill: (signal) => { try { child.kill(signal) } catch { /* already gone */ } },+ onMessage: (listener) => { child.on('message', listener) },+ onExit: (listener) => { exits.push(listener) },+ }+}++export function runMaster(config: MasterConfig): Supervisor {+ const { env, exit, onSignal } = masterDefaults(config)+ process.title = 'harnessd'+ const readPid = (): number | null => {+ try { return Number.parseInt(readFileSync(config.pidFile, 'utf8').trim(), 10) || null } catch { return null }+ }+ const options = supervisorOptions(env, config.execArgv)+ const execArgv = coreExecArgv(config.execArgv, options.heapLimitMiB)+ // Trimmed here rather than by the core: the master outlives every core, and two trimmers rewriting+ // one file in place would race.+ const stopTrimming = trimLogEvery(config.logFile)+ const supervisor = new Supervisor({+ spawnCore: (extra) => coreHandle(spawn(config.nodePath, [...execArgv, config.scriptPath, '__run'], {+ env: { ...env, ...extra },+ stdio: ['ignore', 'inherit', 'inherit', 'ipc'],+ })),+ now: () => performance.now(),+ wallClock: () => Date.now(),+ writeStatus: (status) => { if (config.statusFile) writeStatusFile(config.statusFile, { ...status, masterPid: process.pid }) },+ setTimer: (run, ms) => setTimeout(run, ms),+ clearTimer: (timer) => clearTimeout(timer as ReturnType<typeof setTimeout>),+ claimPidFile: () => writeFileSync(config.pidFile, `${process.pid}\n`),+ releasePidFile: () => { if (readPid() === process.pid) rmSync(config.pidFile, { force: true }) },+ restoreUpdate: config.restoreUpdate,+ confirmUpdate: config.confirmUpdate,+ log: (line) => console.log(`${new Date().toISOString().replace('T', ' ').slice(0, 23)} ${line}`),+ exit: (code) => {+ stopTrimming()+ exit(code)+ },+ }, options)+ for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP'] as const) onSignal(signal, () => supervisor.stop(signal))+ supervisor.start()+ return supervisor+}+diff --git a/cli/src/harnessd/protocol.spec.ts b/cli/src/harnessd/protocol.spec.tsnew file mode 100644index 000000000..448f34414--- /dev/null+++ b/cli/src/harnessd/protocol.spec.ts@@ -0,0 +1,33 @@+import { describe, expect, it } from 'vitest'+import { HARNESSD_PROTOCOL, isCoreMessage, isMasterMessage } from './protocol.js'++describe('the spawn channel protocol', () => {+ it.each([+ [{ type: 'harnessd:bound', protocol: HARNESSD_PROTOCOL, port: 18473 }, true],+ [{ type: 'harnessd:bound', protocol: 1.5, port: 18473 }, false],+ [{ type: 'harnessd:bound', protocol: 1, port: '18473' }, false],+ [{ type: 'harnessd:ready' }, true],+ [{ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1 }, true],+ [{ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 12 }, true],+ [{ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: '12' }, false],+ [{ type: 'harnessd:heartbeat', rssBytes: '1', heapUsedBytes: 1 }, false],+ [{ type: 'harnessd:heartbeat', rssBytes: 1 }, false],+ [{ type: 'harnessd:status', status: {} }, false],+ [{ type: 'other' }, false],+ [null, false],+ ['harnessd:bound', false],+ ])('a core message %j is %s', (message, valid) => {+ expect(isCoreMessage(message)).toBe(valid)+ })++ it.each([+ [{ type: 'harnessd:status', status: { state: 'running' } }, true],+ [{ type: 'harnessd:status', status: null }, false],+ [{ type: 'harnessd:status', status: 'running' }, false],+ [{ type: 'harnessd:bound', protocol: 1, port: 1 }, false],+ [undefined, false],+ [7, false],+ ])('a master message %j is %s', (message, valid) => {+ expect(isMasterMessage(message)).toBe(valid)+ })+})diff --git a/cli/src/harnessd/protocol.ts b/cli/src/harnessd/protocol.tsnew file mode 100644index 000000000..46de53fdb--- /dev/null+++ b/cli/src/harnessd/protocol.ts@@ -0,0 +1,62 @@+/**+ * What harnessd's master and its core say to each other over the spawn channel (Node IPC).+ *+ * Small on purpose. The core says when it is bound, when it is ready and that it is alive; the master+ * tells it its own status. Everything else the daemon does goes through the core's local API, never+ * through here.+ *+ * Messages are only ever added, and a side ignores what it does not know: during an update an older+ * master supervises a newer core, and after a rollback a newer master supervises an older one. Each+ * core states its protocol in `bound`, and the master holds it to what that version can say — a+ * protocol 1 core has no `ready`.+ */+import type { SupervisorStatus } from './supervisor.js'++/** 2: `ready`, the event-loop delay on the heartbeat, and the exit-code contract below. */+export const HARNESSD_PROTOCOL = 2++/** The exit code a core uses to be restarted at once on the bundle now on disk (a staged update). */+export const CORE_EXIT_UPDATE = 75++/**+ * The exit code a core uses to stop for good — this computer was removed from its account, or it is+ * connected from another one — so the master stops with it. Any other exit is a crash, 0 included: Node+ * also exits 0 when its event loop runs out of work, and a SIGTERM from outside must not take the daemon+ * down. (Protocol 1 cores stopped for good with 0.)+ */+export const CORE_EXIT_STOP = 78++export type CoreMessage =+ /** The control port is bound: from here on the daemon answers, though start-up is not done. */+ | { type: 'harnessd:bound'; protocol: number; port: number }+ /** Start-up is done and requests are being served (protocol 2) — or, with `safeMode`, start-up gave+ * way to safe mode: the updater alone runs, waiting for a fix. */+ | { type: 'harnessd:ready'; safeMode?: string }+ /** Sent every few seconds; a core that stops sending is hung. `loopDelayMs`: the longest the event+ * loop was held since the last one (protocol 2). */+ | { type: 'harnessd:heartbeat'; rssBytes: number; heapUsedBytes: number; loopDelayMs?: number }++export type MasterMessage =+ | { type: 'harnessd:status'; status: SupervisorStatus }++export function isCoreMessage(value: unknown): value is CoreMessage {+ if (!value || typeof value !== 'object') return false+ const message = value as Record<string, unknown>+ switch (message.type) {+ case 'harnessd:bound':+ return Number.isInteger(message.protocol) && Number.isInteger(message.port)+ case 'harnessd:ready':+ return message.safeMode === undefined || typeof message.safeMode === 'string'+ case 'harnessd:heartbeat':+ return typeof message.rssBytes === 'number' && typeof message.heapUsedBytes === 'number'+ && (message.loopDelayMs === undefined || typeof message.loopDelayMs === 'number')+ default:+ return false+ }+}++export function isMasterMessage(value: unknown): value is MasterMessage {+ if (!value || typeof value !== 'object') return false+ const message = value as Record<string, unknown>+ return message.type === 'harnessd:status' && !!message.status && typeof message.status === 'object'+}diff --git a/cli/src/harnessd/supervisor.spec.ts b/cli/src/harnessd/supervisor.spec.tsnew file mode 100644index 000000000..f61d523fe--- /dev/null+++ b/cli/src/harnessd/supervisor.spec.ts@@ -0,0 +1,475 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { CORE_EXIT_STOP, CORE_EXIT_UPDATE, HARNESSD_PROTOCOL, type MasterMessage } from './protocol.js'+import { DEFAULT_SUPERVISOR_OPTIONS, Supervisor, type CoreHandle, type SupervisorOptions, type SupervisorStatus } from './supervisor.js'++const MIB = 1024 * 1024++class FakeCore implements CoreHandle {+ readonly sent: MasterMessage[] = []+ readonly kills: NodeJS.Signals[] = []+ private messageListeners: Array<(message: unknown) => void> = []+ private exitListeners: Array<(code: number | null, signal: NodeJS.Signals | null) => void> = []+ constructor(readonly pid: number | undefined, readonly env: Record<string, string>) {}+ send(message: MasterMessage): void { this.sent.push(message) }+ kill(signal: NodeJS.Signals): void { this.kills.push(signal) }+ onMessage(listener: (message: unknown) => void): void { this.messageListeners.push(listener) }+ onExit(listener: (code: number | null, signal: NodeJS.Signals | null) => void): void { this.exitListeners.push(listener) }+ say(message: unknown): void { for (const listener of this.messageListeners) listener(message) }+ bind(protocol = HARNESSD_PROTOCOL): void { this.say({ type: 'harnessd:bound', protocol, port: 18473 }) }+ ready(safeMode?: string): void { this.say(safeMode === undefined ? { type: 'harnessd:ready' } : { type: 'harnessd:ready', safeMode }) }+ /** Bound and ready: up, the way a healthy start-up ends. */+ up(): void { this.bind(); this.ready() }+ beat(rssBytes = 100 * MIB, heapUsedBytes = rssBytes / 2): void { this.say({ type: 'harnessd:heartbeat', rssBytes, heapUsedBytes, loopDelayMs: 3 }) }+ exit(code: number | null, signal: NodeJS.Signals | null = null): void { for (const listener of this.exitListeners) listener(code, signal) }+}++const options: SupervisorOptions = {+ ...DEFAULT_SUPERVISOR_OPTIONS,+ bindTimeoutMs: 10_000,+ readyTimeoutMs: 8_000,+ heartbeatTimeoutMs: 6_000,+ stopGraceMs: 2_000,+ initialBackoffMs: 500,+ maxBackoffMs: 4_000,+ backoffResetMs: 20_000,+ heapLimitMiB: 1_024,+ heapRestartPercent: 75,+ rssLimitMiB: 2_048,+ updateProbationMs: 5_000,+ crashLoopCrashes: 3,+ crashLoopWindowMs: 60_000,+}++describe('Supervisor', () => {+ let cores: FakeCore[]+ let calls: string[]+ let lines: string[]+ let exited: number[]+ let statuses: SupervisorStatus[]+ let pidOf: (index: number) => number | undefined+ const core = () => cores[cores.length - 1]++ const make = (overrides: Partial<SupervisorOptions> = {}) => new Supervisor({+ spawnCore: (env) => {+ const next = new FakeCore(pidOf(cores.length), env)+ cores.push(next)+ return next+ },+ now: () => performance.now(),+ wallClock: () => Date.now(),+ setTimer: (run, ms) => setTimeout(run, ms),+ clearTimer: (timer) => clearTimeout(timer as ReturnType<typeof setTimeout>),+ claimPidFile: () => calls.push('claim'),+ releasePidFile: () => calls.push('release'),+ restoreUpdate: () => calls.push('restore'),+ confirmUpdate: () => calls.push('confirm'),+ writeStatus: (status) => statuses.push(status),+ log: (line) => lines.push(line),+ exit: (code) => exited.push(code),+ }, { ...options, ...overrides })++ /** Let time pass for a running core, beating as a healthy one does. */+ const live = (ms: number) => { for (let left = ms; left > 0; left -= 1_000) { vi.advanceTimersByTime(Math.min(1_000, left)); core().beat() } }+ /** Crash the running core and let its replacement start. */+ const crash = (code: number | null = 1, signal: NodeJS.Signals | null = null) => { core().exit(code, signal); vi.runOnlyPendingTimers() }++ beforeEach(() => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date', 'performance'] })+ cores = []+ calls = []+ lines = []+ exited = []+ statuses = []+ pidOf = (index) => 1000 + index+ })+ afterEach(() => vi.useRealTimers())++ it('starts one core, claims the pid file once it is bound, and runs it once it is ready', () => {+ const supervisor = make()+ expect(new Supervisor({ wallClock: () => 7 } as never).status())+ .toEqual({ state: 'idle', corePid: null, restarts: 0, lastExit: null, lastExitReason: null, safeMode: null, protocol: HARNESSD_PROTOCOL, since: 7 })+ supervisor.start()+ supervisor.start()+ expect(cores).toHaveLength(1)+ expect(core().env).toEqual({ HARNESSD_SUPERVISED: '1', HARNESSD_RESTARTS: '0' })+ expect(supervisor.status()).toMatchObject({ state: 'starting', corePid: 1000 })+ core().bind()+ expect(calls).toEqual(['claim'])+ expect(supervisor.status().state).toBe('listening')+ core().bind()+ expect(calls).toEqual(['claim'])+ vi.advanceTimersByTime(1_000)+ core().ready()+ expect(supervisor.status()).toEqual({+ state: 'running', corePid: 1000, restarts: 0, lastExit: null, lastExitReason: null, safeMode: null,+ protocol: HARNESSD_PROTOCOL, since: Date.now(),+ })+ expect(lines.at(-1)).toBe('[harnessd] core ready (pid 1000)')+ // Every change is written for `harness status`, and told to the bound core.+ expect(statuses.map((status) => status.state)).toEqual(['starting', 'listening', 'running'])+ expect(core().sent.map((message) => message.status.state)).toEqual(['listening', 'running'])+ core().ready()+ expect(statuses).toHaveLength(3)+ })++ it('ignores what is not a core message, and anything but a bind before the bind', () => {+ const supervisor = make()+ supervisor.start()+ core().say(null)+ core().say({ type: 'other' })+ core().beat()+ core().ready()+ vi.advanceTimersByTime(options.bindTimeoutMs - 1)+ expect(core().kills).toEqual([])+ expect(supervisor.status().state).toBe('starting')+ })++ it('kills a core that never binds and starts another after the backoff', () => {+ pidOf = (index) => (index === 1 ? undefined : 1000 + index)+ const supervisor = make()+ supervisor.start()+ vi.advanceTimersByTime(options.bindTimeoutMs)+ expect(core().kills).toEqual(['SIGKILL'])+ core().exit(null, 'SIGKILL')+ expect(supervisor.status()).toMatchObject({ state: 'restarting', restarts: 1, lastExit: 'signal SIGKILL', lastExitReason: 'did-not-bind' })+ vi.advanceTimersByTime(499)+ expect(cores).toHaveLength(1)+ vi.advanceTimersByTime(1)+ expect(cores).toHaveLength(2)+ expect(core().env).toEqual({ HARNESSD_SUPERVISED: '1', HARNESSD_RESTARTS: '1', HARNESSD_LAST_EXIT: 'signal SIGKILL' })+ expect(lines.at(-1)).toBe('[harnessd] core started (pid ?) · restart 1')+ core().bind()+ expect(lines.at(-1)).toBe(`[harnessd] core bound (pid ?, protocol ${HARNESSD_PROTOCOL})`)+ core().ready()+ expect(lines.at(-1)).toBe('[harnessd] core ready (pid ?)')+ })++ it('kills a core that binds but never finishes starting', () => {+ const supervisor = make()+ supervisor.start()+ core().bind()+ // Its heartbeats go on — an event loop that turns — and still it never says it is ready.+ for (let i = 0; i < 7; i++) { vi.advanceTimersByTime(1_000); core().beat() }+ expect(core().kills).toEqual([])+ vi.advanceTimersByTime(1_000)+ expect(core().kills).toEqual(['SIGKILL'])+ expect(lines.at(-1)).toBe(`[harnessd] core bound but not ready within ${options.readyTimeoutMs} ms — killing it`)+ core().exit(null, 'SIGKILL')+ expect(supervisor.status()).toMatchObject({ restarts: 1, lastExitReason: 'not-ready' })+ })++ it('kills a bound core whose heartbeats stop: it is hung', () => {+ const supervisor = make()+ supervisor.start()+ core().up()+ vi.advanceTimersByTime(5_000)+ core().beat()+ vi.advanceTimersByTime(5_999)+ expect(core().kills).toEqual([])+ vi.advanceTimersByTime(1)+ expect(core().kills).toEqual(['SIGKILL'])+ core().exit(null, 'SIGKILL')+ expect(supervisor.status().lastExitReason).toBe('hung')+ })++ it('backs off crash after crash, up to its cap, and starts over after a good run', () => {+ const supervisor = make({ crashLoopCrashes: 100 })+ supervisor.start()+ const delays: number[] = []+ for (let i = 0; i < 5; i++) {+ const before = Date.now()+ core().exit(1)+ const spawned = cores.length+ while (cores.length === spawned) vi.advanceTimersByTime(100)+ delays.push(Date.now() - before)+ }+ expect(delays).toEqual([500, 1000, 2000, 4000, 4000])+ core().up()+ live(options.backoffResetMs)+ const before = Date.now()+ crash()+ expect(Date.now() - before).toBe(500)+ expect(supervisor.status()).toMatchObject({ restarts: 6, lastExitReason: 'crashed' })+ })++ describe('what an exit means', () => {+ it('stops for good with a core that says so (78), releasing the pid file it claimed', () => {+ const supervisor = make()+ supervisor.start()+ core().up()+ core().exit(CORE_EXIT_STOP)+ expect(exited).toEqual([0])+ expect(calls).toEqual(['claim', 'release'])+ expect(supervisor.status()).toMatchObject({ state: 'stopped', lastExit: `code ${CORE_EXIT_STOP}`, lastExitReason: 'stopped' })+ expect(lines.at(-1)).toBe(`[harnessd] core stopped for good (code ${CORE_EXIT_STOP}) — stopping`)+ })++ it('restarts a core that exits 0 — an emptied event loop, or a SIGTERM from outside', () => {+ const supervisor = make()+ supervisor.start()+ core().up()+ crash(0)+ expect(exited).toEqual([])+ expect(cores).toHaveLength(2)+ expect(supervisor.status()).toMatchObject({ restarts: 1, lastExit: 'code 0', lastExitReason: 'crashed' })+ // One that never even bound is held to this protocol too.+ crash(0)+ expect(cores).toHaveLength(3)+ })++ it('holds a core from before `ready` to what it could say: running once bound, stopped for good by 0', () => {+ const supervisor = make()+ supervisor.start()+ core().bind(1)+ expect(supervisor.status().state).toBe('running')+ live(options.readyTimeoutMs * 2)+ expect(core().kills).toEqual([])+ core().exit(0)+ expect(exited).toEqual([0])+ })++ it('restarts a protocol 1 core that the master itself was restarting, whatever its exit', () => {+ make({ rssLimitMiB: 512 }).start()+ core().bind(1)+ core().beat(600 * MIB)+ core().exit(0)+ vi.runOnlyPendingTimers()+ expect(exited).toEqual([])+ expect(cores).toHaveLength(2)+ })+ })++ describe('memory', () => {+ it('restarts a core past its share of the heap limit, cleanly, before V8 would abort it', () => {+ const supervisor = make()+ supervisor.start()+ core().up()+ core().beat(100 * MIB, 767 * MIB)+ expect(core().kills).toEqual([])+ core().beat(100 * MIB, 769 * MIB)+ core().beat(100 * MIB, 900 * MIB)+ expect(core().kills).toEqual(['SIGTERM'])+ expect(lines.at(-1)).toBe('[harnessd] core its heap is at 769 MiB, past 75% of its 1024 MiB limit — restarting it')+ core().exit(0)+ expect(supervisor.status().lastExitReason).toBe('memory')+ })++ it('restarts a core over its resident budget, by SIGTERM, then SIGKILL if it lingers, backing off', () => {+ const supervisor = make({ crashLoopCrashes: 100 })+ supervisor.start()+ core().up()+ core().beat(2_047 * MIB, 10 * MIB)+ expect(core().kills).toEqual([])+ core().beat(2_049 * MIB, 10 * MIB)+ expect(core().kills).toEqual(['SIGTERM'])+ vi.advanceTimersByTime(options.stopGraceMs)+ expect(core().kills).toEqual(['SIGTERM', 'SIGKILL'])+ core().exit(0)+ expect(exited).toEqual([])+ vi.advanceTimersByTime(options.initialBackoffMs - 1)+ expect(cores).toHaveLength(1)+ vi.advanceTimersByTime(1)+ expect(cores).toHaveLength(2)+ expect(supervisor.status()).toMatchObject({ lastExit: 'code 0', lastExitReason: 'memory' })+ // Over budget again at once: it backs off like a crash rather than restarting as fast as it binds.+ core().up()+ core().beat(4_000 * MIB, 10 * MIB)+ core().exit(0)+ vi.advanceTimersByTime(options.initialBackoffMs * 2 - 1)+ expect(cores).toHaveLength(2)+ vi.advanceTimersByTime(1)+ expect(cores).toHaveLength(3)+ })++ it('does not check memory with budgets of 0', () => {+ make({ rssLimitMiB: 0, heapLimitMiB: 0 }).start()+ core().up()+ core().beat(64 * 1024 * MIB, 64 * 1024 * MIB)+ expect(core().kills).toEqual([])+ })+ })++ describe('crash loops', () => {+ it('starts the core in safe mode after enough crashes close together, and a normal one after', () => {+ const supervisor = make()+ supervisor.start()+ crash()+ vi.advanceTimersByTime(options.crashLoopWindowMs) // the first crash ages out+ crash()+ crash()+ expect(core().env.HARNESSD_SAFE_MODE).toBeUndefined()+ crash()+ expect(lines).toContain('[harnessd] core crashed 3 times in 1 min — starting it in safe mode')+ expect(core().env.HARNESSD_SAFE_MODE).toBe('crash-loop')+ expect(lines.at(-1)).toMatch(/· safe mode: crash-loop$/)+ expect(supervisor.status().safeMode).toBe('crash-loop')+ core().bind()+ core().ready('harnessd saw this core crash again and again')+ expect(supervisor.status()).toMatchObject({ state: 'running', safeMode: 'harnessd saw this core crash again and again' })+ // Safe mode ran out without a fix: a normal core gets another try, and its crashes count afresh.+ crash()+ expect(core().env.HARNESSD_SAFE_MODE).toBeUndefined()+ expect(supervisor.status().safeMode).toBeNull()+ crash()+ expect(core().env.HARNESSD_SAFE_MODE).toBeUndefined()+ })++ it('leaves safe mode at once for an update', () => {+ make().start()+ crash(); crash(); crash()+ expect(core().env.HARNESSD_SAFE_MODE).toBe('crash-loop')+ core().bind()+ core().ready('waiting for a fix')+ core().exit(CORE_EXIT_UPDATE)+ vi.advanceTimersByTime(0)+ expect(core().env.HARNESSD_SAFE_MODE).toBeUndefined()+ })++ it('reports a core that fell into safe mode on its own, and gives it a normal try after', () => {+ const supervisor = make()+ supervisor.start()+ core().bind()+ core().ready('start-up failed: no tmux')+ expect(supervisor.status().safeMode).toBe('start-up failed: no tmux')+ crash()+ expect(supervisor.status().safeMode).toBeNull()+ expect(core().env.HARNESSD_SAFE_MODE).toBeUndefined()+ })+ })++ it('stops the core and itself on request, and kills the core on a second request', () => {+ const supervisor = make()+ supervisor.start()+ core().up()+ supervisor.stop('SIGTERM')+ expect(core().kills).toEqual(['SIGTERM'])+ expect(supervisor.status().state).toBe('stopping')+ supervisor.stop('SIGTERM')+ expect(core().kills).toEqual(['SIGTERM', 'SIGKILL'])+ core().bind()+ core().beat(64 * 1024 * MIB)+ expect(supervisor.status().state).toBe('stopping')+ core().exit(null, 'SIGKILL')+ expect(exited).toEqual([0])+ expect(calls).toEqual(['claim', 'release'])+ expect(supervisor.status()).toMatchObject({ state: 'stopped', lastExitReason: 'stopped' })+ supervisor.stop('again')+ expect(exited).toEqual([0])+ })++ it('stops while a core is still starting, without waiting for it to be ready', () => {+ const supervisor = make()+ supervisor.start()+ core().bind()+ supervisor.stop('SIGTERM')+ vi.advanceTimersByTime(options.readyTimeoutMs)+ expect(core().kills).toEqual(['SIGTERM', 'SIGKILL'])+ core().ready()+ expect(supervisor.status().state).toBe('stopping')+ })++ it('binds a core that was asked to stop before it bound, without calling it running', () => {+ const supervisor = make()+ supervisor.start()+ supervisor.stop('SIGTERM')+ core().bind()+ expect(supervisor.status().state).toBe('stopping')+ })++ it('stops at once when there is no core: never started, or waiting to restart one', () => {+ const idle = make()+ idle.stop('SIGTERM')+ expect(exited).toEqual([0])+ const waiting = make()+ waiting.start()+ core().exit(1)+ waiting.stop('SIGTERM')+ vi.runAllTimers()+ expect(cores).toHaveLength(1)+ expect(exited).toEqual([0, 0])+ expect(calls).toEqual([])+ })++ it('kills a core that takes too long to stop', () => {+ const supervisor = make()+ supervisor.start()+ supervisor.stop('SIGTERM')+ vi.advanceTimersByTime(options.stopGraceMs)+ expect(core().kills).toEqual(['SIGTERM', 'SIGKILL'])+ })++ it('pays no attention to a core it has already replaced', () => {+ make().start()+ const first = core()+ crash()+ first.bind()+ first.exit(1)+ expect(calls).toEqual([])+ expect(cores).toHaveLength(2)+ })++ describe('updates', () => {+ it('restarts at once onto the new bundle and keeps it once it has stayed up, from ready', () => {+ const supervisor = make()+ supervisor.start()+ core().up()+ core().exit(CORE_EXIT_UPDATE)+ expect(lines.at(-1)).toBe(`[harnessd] core exited (code ${CORE_EXIT_UPDATE}) for an update — restarting`)+ vi.advanceTimersByTime(0)+ expect(cores).toHaveLength(2)+ core().bind()+ vi.advanceTimersByTime(options.updateProbationMs)+ expect(calls).toEqual(['claim'])+ core().ready()+ vi.advanceTimersByTime(options.updateProbationMs - 1)+ expect(calls).toEqual(['claim'])+ vi.advanceTimersByTime(1)+ expect(calls).toEqual(['claim', 'confirm'])+ expect(supervisor.status()).toMatchObject({ state: 'running', restarts: 1, lastExitReason: 'update' })+ core().exit(1)+ expect(calls).toEqual(['claim', 'confirm'])+ })++ it('judges a protocol 1 core\'s update from its bind', () => {+ make().start()+ core().up()+ core().exit(CORE_EXIT_UPDATE)+ vi.advanceTimersByTime(0)+ core().bind(1)+ vi.advanceTimersByTime(options.updateProbationMs)+ expect(calls).toEqual(['claim', 'confirm'])+ })++ it.each([+ ['never binds', (c: FakeCore) => c.exit(1)],+ ['crashes while on probation', (c: FakeCore) => { c.up(); vi.advanceTimersByTime(1000); c.exit(null, 'SIGABRT') }],+ ['starts in safe mode', (c: FakeCore) => { c.bind(); c.ready('start-up failed'); expect(c.kills).toEqual(['SIGKILL']); c.exit(null, 'SIGKILL') }],+ ])('rolls the bundle back when the updated core %s', (_, fail) => {+ const supervisor = make()+ supervisor.start()+ core().up()+ core().exit(CORE_EXIT_UPDATE)+ vi.advanceTimersByTime(0)+ fail(core())+ expect(calls).toEqual(['claim', 'restore'])+ vi.advanceTimersByTime(0)+ expect(cores).toHaveLength(3)+ expect(supervisor.status().restarts).toBe(2)+ core().up()+ vi.runOnlyPendingTimers()+ expect(calls).toEqual(['claim', 'restore'])+ })++ it('leaves the update unconfirmed when stopped while on probation', () => {+ const supervisor = make()+ supervisor.start()+ core().exit(CORE_EXIT_UPDATE)+ vi.advanceTimersByTime(0)+ core().up()+ supervisor.stop('SIGTERM')+ core().exit(0)+ vi.runAllTimers()+ expect(calls).toEqual(['claim', 'release'])+ })+ })+})diff --git a/cli/src/harnessd/supervisor.ts b/cli/src/harnessd/supervisor.tsnew file mode 100644index 000000000..8065414bc--- /dev/null+++ b/cli/src/harnessd/supervisor.ts@@ -0,0 +1,401 @@+/**+ * harnessd's master: keeps the core running, and nothing else.+ *+ * The core is the process that owns sessions, terminals and turns. The master starts it, restarts it+ * when it crashes, hangs, never finishes starting or outgrows its memory budget, puts it in safe mode+ * when it keeps crashing, and stops it when asked. It holds no sessions, opens no network connection+ * and contains no feature code, so there is almost nothing in it that can fail — which is the point:+ * the daemon comes back even when the desktop app is not running to restart it.+ *+ * The core and the master talk over the spawn channel (`./protocol.ts`). Everything that touches the+ * operating system is injected (`SupervisorDeps`), so every decision here is tested without one.+ */+import {+ CORE_EXIT_STOP,+ CORE_EXIT_UPDATE,+ HARNESSD_PROTOCOL,+ isCoreMessage,+ type CoreMessage,+ type MasterMessage,+} from './protocol.js'++export interface CoreHandle {+ readonly pid: number | undefined+ send(message: MasterMessage): void+ kill(signal: NodeJS.Signals): void+ onMessage(listener: (message: unknown) => void): void+ onExit(listener: (code: number | null, signal: NodeJS.Signals | null) => void): void+}++export interface SupervisorDeps {+ /** Start a core with these extra environment variables. */+ spawnCore(env: Record<string, string>): CoreHandle+ /** A monotonic clock, in ms: durations only, untouched by sleep and clock changes. */+ now(): number+ /** The wall clock, in ms: what the status says things happened at. */+ wallClock(): number+ setTimer(run: () => void, ms: number): unknown+ clearTimer(timer: unknown): void+ /** Claim the pid file for the master — the signal `harness start` waits on. */+ claimPidFile(): void+ /** Remove the pid file if it is still the master's. */+ releasePidFile(): void+ /** Put the previous bundle back (`selfUpdate.restore`): the update it replaced failed. */+ restoreUpdate(): void+ /** Drop the previous bundle (`selfUpdate.confirm`): the update came up and stayed up. */+ confirmUpdate(): void+ /** Record the status where `harness status` reads it when no core can answer. */+ writeStatus(status: SupervisorStatus): void+ log(line: string): void+ exit(code: number): void+}++export interface SupervisorOptions {+ /** A core that has not said it is bound by then is killed and started again. */+ bindTimeoutMs: number+ /** A bound core that has not said it is ready by then is killed and started again (protocol 2 on). */+ readyTimeoutMs: number+ /** A bound core that sends no heartbeat for this long is hung: killed and started again. */+ heartbeatTimeoutMs: number+ /** How long a core gets to stop after SIGTERM before SIGKILL. */+ stopGraceMs: number+ /** Restart delay: starts here, doubles per crash, caps at `maxBackoffMs`. */+ initialBackoffMs: number+ maxBackoffMs: number+ /** A core that stayed up this long earns the next crash the initial delay again. */+ backoffResetMs: number+ /** The V8 heap limit the core runs with, MiB (`--max-old-space-size`); 0 leaves V8's own. */+ heapLimitMiB: number+ /** Past this share of `heapLimitMiB` the core is restarted cleanly, before V8 aborts it. */+ heapRestartPercent: number+ /** Resident memory past which a core is restarted, MiB — buffers V8 does not count included; 0: off. */+ rssLimitMiB: number+ /** How long a core started on a new bundle must stay up, from ready, before the update is kept. */+ updateProbationMs: number+ /** This many crashes inside `crashLoopWindowMs` start the next core in safe mode. */+ crashLoopCrashes: number+ crashLoopWindowMs: number+}++export const DEFAULT_SUPERVISOR_OPTIONS: SupervisorOptions = {+ bindTimeoutMs: 60_000,+ readyTimeoutMs: 120_000,+ heartbeatTimeoutMs: 30_000,+ // Inside `harness stop`'s own 3 s grace, so the master stops its core and exits before that SIGKILL.+ stopGraceMs: 2_500,+ initialBackoffMs: 500,+ maxBackoffMs: 30_000,+ backoffResetMs: 60_000,+ heapLimitMiB: 4_096,+ heapRestartPercent: 75,+ rssLimitMiB: 6_144,+ updateProbationMs: 30_000,+ crashLoopCrashes: 3,+ crashLoopWindowMs: 300_000,+}++/** `listening`: bound, still starting. `running`: ready (a protocol 1 core is running once bound). */+export type SupervisorState = 'idle' | 'starting' | 'listening' | 'running' | 'restarting' | 'stopping' | 'stopped'++/** Why the last core ended. */+export type ExitReason = 'crashed' | 'hung' | 'did-not-bind' | 'not-ready' | 'memory' | 'update' | 'stopped'++export interface SupervisorStatus {+ state: SupervisorState+ corePid: number | null+ restarts: number+ lastExit: string | null+ lastExitReason: ExitReason | null+ /** Why the core runs in safe mode, or null. */+ safeMode: string | null+ protocol: number+ /** When the state last changed, wall clock ms. */+ since: number+}++type TimerName = 'bindTimer' | 'readyTimer' | 'heartbeatTimer' | 'killTimer' | 'restartTimer' | 'probationTimer'++const describeExit = (code: number | null, signal: NodeJS.Signals | null): string =>+ signal ? `signal ${signal}` : `code ${code}`+const MIB = 1024 * 1024++export class Supervisor {+ private state: SupervisorState = 'idle'+ private since: number+ private core: CoreHandle | null = null+ private bound = false+ /** The protocol the running core stated in `bound`; the master's own until it does. */+ private coreProtocol = HARNESSD_PROTOCOL+ private upAt = 0+ private claimed = false+ private restarts = 0+ private backoff: number+ private lastExit: string | null = null+ private lastExitReason: ExitReason | null = null+ /** Why the master is ending the core it is running, if it is: decides what its exit means. */+ private ending: 'stop' | 'restart' | null = null+ /** Why the master killed the running core, when it did. */+ private killReason: ExitReason | null = null+ /** A core exited for an update: the next one runs the new bundle, on probation until it proves it. */+ private update: 'pending' | 'probation' | null = null+ /** Crashes inside the crash-loop window, on the monotonic clock. */+ private crashes: number[] = []+ /** Set while the core is started in safe mode, with the reason. */+ private safeMode: string | null = null+ /** The safe mode the running core says it is in, whoever chose it. */+ private coreSafeMode: string | null = null+ private bindTimer: unknown = null+ private readyTimer: unknown = null+ private probationTimer: unknown = null+ private heartbeatTimer: unknown = null+ private killTimer: unknown = null+ private restartTimer: unknown = null++ constructor(private readonly deps: SupervisorDeps, private readonly options: SupervisorOptions = DEFAULT_SUPERVISOR_OPTIONS) {+ this.backoff = options.initialBackoffMs+ this.since = deps.wallClock()+ }++ status(): SupervisorStatus {+ return {+ state: this.state,+ corePid: this.core?.pid ?? null,+ restarts: this.restarts,+ lastExit: this.lastExit,+ lastExitReason: this.lastExitReason,+ safeMode: this.coreSafeMode ?? this.safeMode,+ protocol: HARNESSD_PROTOCOL,+ since: this.since,+ }+ }++ start(): void {+ if (this.state !== 'idle') return+ this.spawn()+ }++ /** Stop the core and the master. A second call while stopping kills the core outright. */+ stop(reason: string): void {+ if (this.state === 'stopped') return+ if (this.state === 'stopping') {+ this.core?.kill('SIGKILL')+ return+ }+ this.deps.log(`[harnessd] ${reason} — stopping`)+ this.setState('stopping')+ this.clearTimer('restartTimer')+ if (!this.core) { this.finish(0); return }+ this.end('stop', 'SIGTERM')+ }++ private setState(state: SupervisorState): void {+ this.state = state+ this.since = this.deps.wallClock()+ this.publish()+ }++ /** The status to the status file, and to a bound core for its `/api/status`. */+ private publish(): void {+ const status = this.status()+ this.deps.writeStatus(status)+ if (this.bound) this.core?.send({ type: 'harnessd:status', status })+ }++ private spawn(): void {+ this.bound = false+ this.coreProtocol = HARNESSD_PROTOCOL+ this.coreSafeMode = null+ this.ending = null+ this.killReason = null+ const env: Record<string, string> = { HARNESSD_SUPERVISED: '1', HARNESSD_RESTARTS: String(this.restarts) }+ if (this.lastExit) env.HARNESSD_LAST_EXIT = this.lastExit+ if (this.safeMode) env.HARNESSD_SAFE_MODE = this.safeMode+ const core = this.deps.spawnCore(env)+ this.core = core+ core.onMessage((message) => { if (this.core === core && isCoreMessage(message)) this.onMessage(core, message) })+ core.onExit((code, signal) => { if (this.core === core) this.onExit(code, signal) })+ this.deps.log(`[harnessd] core started (pid ${core.pid ?? '?'})${this.restarts ? ` · restart ${this.restarts}` : ''}${this.safeMode ? ` · safe mode: ${this.safeMode}` : ''}`)+ this.setState(this.restarts === 0 ? 'starting' : 'restarting')+ this.armTimer('bindTimer', () => this.kill(core, 'did-not-bind', `did not bind within ${this.options.bindTimeoutMs} ms`), this.options.bindTimeoutMs)+ }++ private onMessage(core: CoreHandle, message: CoreMessage): void {+ switch (message.type) {+ case 'harnessd:bound': {+ if (this.bound) return+ this.bound = true+ this.coreProtocol = message.protocol+ this.upAt = this.deps.now()+ this.clearTimer('bindTimer')+ if (!this.claimed) { this.deps.claimPidFile(); this.claimed = true }+ this.deps.log(`[harnessd] core bound (pid ${core.pid ?? '?'}, protocol ${message.protocol})`)+ this.watchHeartbeat()+ // A core from before `ready` is running once bound, and its update is judged from there.+ const readiness = message.protocol >= 2+ if (readiness) {+ this.armTimer('readyTimer', () => this.kill(core, 'not-ready', `bound but not ready within ${this.options.readyTimeoutMs} ms`), this.options.readyTimeoutMs)+ }+ if (this.state !== 'stopping') this.setState(readiness ? 'listening' : 'running')+ if (!readiness) this.beginProbation()+ return+ }+ case 'harnessd:ready':+ if (!this.bound || this.state !== 'listening') return+ this.clearTimer('readyTimer')+ if (message.safeMode !== undefined && this.update) {+ // The new bundle could not start: that is the update failing, however long it stays up.+ this.kill(core, 'crashed', `started in safe mode on the new bundle (${message.safeMode})`)+ return+ }+ this.upAt = this.deps.now()+ this.coreSafeMode = message.safeMode ?? null+ this.deps.log(`[harnessd] core ready (pid ${core.pid ?? '?'})${message.safeMode === undefined ? '' : ` · in safe mode: ${message.safeMode}`}`)+ this.setState('running')+ this.beginProbation()+ return+ case 'harnessd:heartbeat': {+ if (!this.bound) return+ this.watchHeartbeat()+ if (this.ending) return+ const heapBudget = this.options.heapLimitMiB * MIB * this.options.heapRestartPercent / 100+ if (heapBudget && message.heapUsedBytes > heapBudget) {+ this.restartForMemory(`its heap is at ${Math.round(message.heapUsedBytes / MIB)} MiB, past ${this.options.heapRestartPercent}% of its ${this.options.heapLimitMiB} MiB limit`)+ } else if (this.options.rssLimitMiB && message.rssBytes > this.options.rssLimitMiB * MIB) {+ this.restartForMemory(`it is using ${Math.round(message.rssBytes / MIB)} MiB, over its ${this.options.rssLimitMiB} MiB budget`)+ }+ return+ }+ }+ }++ private restartForMemory(why: string): void {+ this.deps.log(`[harnessd] core ${why} — restarting it`)+ this.killReason = 'memory'+ this.end('restart', 'SIGTERM')+ }++ private beginProbation(): void {+ if (this.update !== 'pending') return+ this.update = 'probation'+ this.armTimer('probationTimer', () => {+ this.probationTimer = null+ this.update = null+ this.deps.confirmUpdate()+ this.deps.log('[harnessd] the update stayed up — keeping it')+ }, this.options.updateProbationMs)+ }++ private onExit(code: number | null, signal: NodeJS.Signals | null): void {+ const exit = describeExit(code, signal)+ const reason = this.killReason ?? (code === CORE_EXIT_UPDATE ? 'update' : 'crashed')+ const wasSafe = this.safeMode ?? this.coreSafeMode+ for (const timer of ['bindTimer', 'readyTimer', 'heartbeatTimer', 'killTimer', 'probationTimer'] as const) this.clearTimer(timer)+ this.core = null+ this.bound = false+ this.coreSafeMode = null+ if (this.state === 'stopping') {+ this.deps.log(`[harnessd] core stopped (${exit})`)+ this.lastExit = exit+ this.lastExitReason = 'stopped'+ this.finish(0)+ return+ }+ this.lastExit = exit+ this.lastExitReason = reason+ this.restarts+++ if (this.update) {+ // The core on the new bundle did not come up, or did not stay up: the bundle before it did.+ this.update = null+ this.deps.restoreUpdate()+ this.deps.log(`[harnessd] the updated core failed (${exit}) — rolled back to the previous bundle; restarting`)+ this.scheduleSpawn(0)+ return+ }+ // A deliberate end: signed out for good, removed from the account, connected from elsewhere. A core+ // started again would only end itself again; the master goes with it. Protocol 1 cores said so with 0.+ const forGood = signal === null && this.ending !== 'restart' && (code === CORE_EXIT_STOP || (code === 0 && this.coreProtocol < 2))+ if (forGood) {+ this.deps.log(`[harnessd] core stopped for good (${exit}) — stopping`)+ this.lastExitReason = 'stopped'+ this.finish(0)+ return+ }+ if (reason === 'update') {+ // A new bundle, and a clean slate: whatever was crashing may be what it fixes.+ this.update = 'pending'+ this.crashes = []+ this.safeMode = null+ this.deps.log(`[harnessd] core exited (${exit}) for an update — restarting`)+ this.scheduleSpawn(0)+ return+ }+ if (wasSafe) {+ // Safe mode ran its course without a fix arriving: a normal core gets another try.+ this.crashes = []+ this.safeMode = null+ } else {+ const now = this.deps.now()+ this.crashes = [...this.crashes.filter((at) => now - at < this.options.crashLoopWindowMs), now]+ if (this.crashes.length >= this.options.crashLoopCrashes) {+ this.safeMode = 'crash-loop'+ this.deps.log(`[harnessd] core crashed ${this.crashes.length} times in ${Math.round(this.options.crashLoopWindowMs / 60_000)} min — starting it in safe mode`)+ }+ }+ if (this.upAt && this.deps.now() - this.upAt >= this.options.backoffResetMs) this.backoff = this.options.initialBackoffMs+ // A memory restart backs off like a crash: a core over budget from the start would otherwise be+ // restarted as fast as it can bind. Only an update restarts at once.+ const delay = this.backoff+ this.backoff = Math.min(this.backoff * 2, this.options.maxBackoffMs)+ this.upAt = 0+ this.deps.log(`[harnessd] core exited (${exit}, ${reason}) — restarting in ${delay} ms`)+ this.scheduleSpawn(delay)+ }++ private scheduleSpawn(delay: number): void {+ this.setState('restarting')+ this.armTimer('restartTimer', () => { this.restartTimer = null; this.spawn() }, delay)+ }++ /** Kill a core that broke a promise — to bind, to be ready, to beat — and remember which. */+ private kill(core: CoreHandle, reason: ExitReason, what: string): void {+ this.deps.log(`[harnessd] core ${what} — killing it`)+ this.killReason = reason+ core.kill('SIGKILL')+ }++ /** End the running core: SIGTERM (or SIGKILL), and SIGKILL if it outlives the grace. */+ private end(why: 'stop' | 'restart', signal: NodeJS.Signals): void {+ const core = this.core!+ this.ending = why+ this.clearTimer('heartbeatTimer')+ this.clearTimer('readyTimer')+ core.kill(signal)+ this.armTimer('killTimer', () => {+ this.deps.log(`[harnessd] core outlived its ${this.options.stopGraceMs} ms to stop — killing it`)+ core.kill('SIGKILL')+ }, this.options.stopGraceMs)+ }++ private watchHeartbeat(): void {+ const core = this.core!+ this.armTimer('heartbeatTimer', () => this.kill(core, 'hung', `sent no heartbeat for ${this.options.heartbeatTimeoutMs} ms — it is hung;`), this.options.heartbeatTimeoutMs)+ }++ private finish(code: number): void {+ this.clearTimer('restartTimer')+ this.clearTimer('probationTimer')+ this.setState('stopped')+ if (this.claimed) this.deps.releasePidFile()+ this.deps.exit(code)+ }++ private armTimer(name: TimerName, run: () => void, ms: number): void {+ this.clearTimer(name)+ this[name] = this.deps.setTimer(run, ms)+ }++ private clearTimer(name: TimerName): void {+ if (this[name] !== null) this.deps.clearTimer(this[name])+ this[name] = null+ }+}diff --git a/cli/src/lib/daemonSafeMode.spec.ts b/cli/src/lib/daemonSafeMode.spec.tsindex 6f9aa3638..44555f7b1 100644--- a/cli/src/lib/daemonSafeMode.spec.ts+++ b/cli/src/lib/daemonSafeMode.spec.ts@@ -4,7 +4,7 @@ import { tmpdir } from 'node:os'import { join } from 'node:path'import {clearSafeModeMarker, readSafeModeMarker, runBootHandoff, safeModeDisposition,- safeModeFile, safeModeStatusBody, writeSafeModeMarker, type BootHandoffDeps,+ safeModeFile, safeModeStatusBody, SafeModeRequest, writeSafeModeMarker, type BootHandoffDeps,} from './daemonSafeMode.js'let dir = ''@@ -31,6 +31,20 @@ describe('safeModeDisposition', () => {expect(safeModeDisposition(new Error('boom'), { selfPid: 100, readPid: () => 100, isAlive: () => true }).stay).toBe(true)expect(safeModeDisposition(new Error('boom'), { selfPid: 100, readPid: () => 200, isAlive: () => false }).stay).toBe(true)})++ // Under harnessd the pid file is the master's. Leaving because of it was why safe mode never held+ // under a master: the core exited, and the master restarted it into the same failure.+ it('stays when the pid file names the harnessd master running this core', () => {+ expect(safeModeDisposition(new Error('boom'), { selfPid: 100, masterPid: 200, readPid: () => 200, isAlive: () => true }).stay).toBe(true)+ expect(safeModeDisposition(new Error('boom'), { selfPid: 100, masterPid: 300, readPid: () => 200, isAlive: () => true }).stay).toBe(false)+ })++ it('stays, saying why, when the master asked for safe mode', () => {+ expect(safeModeDisposition(new SafeModeRequest('crash-loop'), { ...nobody, masterPid: 1 }))+ .toEqual({ stay: true, reason: 'harnessd saw this core crash again and again — started in safe mode' })+ const other = new SafeModeRequest('a corrupt registry')+ expect([other.name, other.why, other.message]).toEqual(['SafeModeRequest', 'a corrupt registry', 'harnessd asked for safe mode: a corrupt registry'])+ })})describe('safeModeStatusBody', () => {diff --git a/cli/src/lib/daemonSafeMode.ts b/cli/src/lib/daemonSafeMode.tsindex 575a86e0a..cf5c46399 100644--- a/cli/src/lib/daemonSafeMode.ts+++ b/cli/src/lib/daemonSafeMode.ts@@ -29,6 +29,14 @@ export interface SafeModeDisposition {reason: string}+/** Thrown at the top of start-up when harnessd's master asks for safe mode: the core kept crashing. */+export class SafeModeRequest extends Error {+ constructor(readonly why: string) {+ super(why === 'crash-loop' ? 'harnessd saw this core crash again and again — started in safe mode' : `harnessd asked for safe mode: ${why}`)+ this.name = 'SafeModeRequest'+ }+}+/*** Whether staying alive is the right answer for THIS failure.*@@ -38,14 +46,15 @@ export interface SafeModeDisposition {*/export function safeModeDisposition(error: unknown,- deps: { selfPid: number; readPid: () => number | null; isAlive: (pid: number) => boolean },+ deps: { selfPid: number; masterPid?: number | null; readPid: () => number | null; isAlive: (pid: number) => boolean },): SafeModeDisposition {const message = error instanceof Error ? `${error.message}` : String(error)if (/EADDRINUSE|address already in use/i.test(message)) {return { stay: false, reason: 'the control port belongs to another daemon' }}const owner = deps.readPid()- if (owner !== null && owner !== deps.selfPid && deps.isAlive(owner)) {+ // Under harnessd the pid file names the master that runs this core: that daemon is this one.+ if (owner !== null && owner !== deps.selfPid && owner !== deps.masterPid && deps.isAlive(owner)) {return { stay: false, reason: `another daemon (pid ${owner}) owns this machine` }}return { stay: true, reason: message || 'start-up failed' }diff --git a/cli/src/lib/selfUpdate.spec.ts b/cli/src/lib/selfUpdate.spec.tsindex 6f310f5a5..df2d605fa 100644--- a/cli/src/lib/selfUpdate.spec.ts+++ b/cli/src/lib/selfUpdate.spec.ts@@ -1,9 +1,10 @@-import { mkdtempSync, readFileSync, rmSync } from 'fs'+import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'fs'import { tmpdir } from 'os'import { join } from 'path'import { createHash } from 'crypto'import { afterEach, describe, expect, it, vi } from 'vitest'-import { canary, isLocalDevBuild, msUntilSlot, semverGt, shouldAutoUpdate, stage, startSelfUpdater } from './selfUpdate.js'+import { canary, confirm, downloadVerified, fetchManifest, isLocalDevBuild, msUntilSlot, rejectedVersions, restore, semverGt, shouldAutoUpdate, stage, startSelfUpdater } from './selfUpdate.js'+import { SpawnLockBusyError } from './daemonSpawnLock.js'let dirs: string[] = []@@ -256,3 +257,179 @@ describe('startSelfUpdater', () => {expect(events).toEqual(['lock', 'unlock'])})})++describe('a rolled-back version', () => {+ afterEach(() => {+ vi.unstubAllGlobals()+ for (const dir of dirs) rmSync(dir, { recursive: true, force: true })+ dirs = []+ })++ it('is remembered when the update is rolled back, and forgotten when one is kept', () => {+ const dir = tempDir()+ stage(dir, Buffer.from('v1'), Buffer.from('n1'))+ stage(dir, Buffer.from('v2'), Buffer.from('n2'), '2.0.0')+ expect(JSON.parse(readFileSync(join(dir, 'update-pending.json'), 'utf8'))).toMatchObject({ version: '2.0.0' })+ restore(dir)+ expect(readFileSync(join(dir, 'cli.js'), 'utf8')).toBe('v1')+ expect(rejectedVersions(dir)).toEqual(['2.0.0'])+ expect(existsSync(join(dir, 'update-pending.json'))).toBe(false)+ // Nothing pending: a rollback with nothing staged remembers nothing new.+ restore(dir)+ expect(rejectedVersions(dir)).toEqual(['2.0.0'])+ // Installed on purpose and kept: no longer rejected.+ stage(dir, Buffer.from('v2'), Buffer.from('n2'), '2.0.0')+ confirm(dir)+ expect(rejectedVersions(dir)).toEqual([])+ expect(existsSync(join(dir, 'update-pending.json'))).toBe(false)+ stage(dir, Buffer.from('v3'), Buffer.from('n3'), '3.0.0')+ confirm(dir)+ expect(rejectedVersions(dir)).toEqual([])+ })++ it('keeps the last ten, each once, and reads anything else as none', () => {+ const dir = tempDir()+ for (const version of ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '3']) {+ stage(dir, Buffer.from('x'), Buffer.from('y'), version)+ restore(dir)+ }+ expect(rejectedVersions(dir)).toEqual(['2', '4', '5', '6', '7', '8', '9', '10', '11', '3'])+ writeFileSync(join(dir, 'update-rejected.json'), '{"not":"a list"}')+ expect(rejectedVersions(dir)).toEqual([])+ writeFileSync(join(dir, 'update-rejected.json'), '["1.0.0", 7, null]')+ expect(rejectedVersions(dir)).toEqual(['1.0.0'])+ })++ it('is never staged again by the background updater, which says so once and waits for a newer build', async () => {+ const cli = Buffer.from('#!/usr/bin/env node\nconsole.log("x")\n')+ const notify = Buffer.from('export {}\n')+ const sha = (b: Buffer): string => createHash('sha256').update(b).digest('hex')+ let offered = '9.9.9'+ vi.stubGlobal('fetch', async (url: string) => {+ if (url.endsWith('/metadata.json')) {+ return new Response(JSON.stringify({ adapter: {+ version: offered,+ cli: { url: 'https://updates.test/cli.js', sha256: sha(cli) },+ notify: { url: 'https://updates.test/notify.mjs', sha256: sha(notify) },+ } }))+ }+ return new Response(url.endsWith('cli.js') ? cli : notify)+ })+ const dir = tempDir()+ writeFileSync(join(dir, 'update-rejected.json'), '["9.9.9"]')+ const logs: string[] = []+ const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { logs.push(String(line)) })+ const staged: string[] = []+ try {+ const poller = startSelfUpdater({+ currentVersion: '1.0.0', url: 'https://updates.test/metadata.json', key: 'adapter', dir, intervalMs: 10,+ onStaged: (version) => { staged.push(version) },+ })+ await new Promise((resolve) => setTimeout(resolve, 80))+ expect(staged).toEqual([])+ expect(logs.filter((line) => line.includes('9.9.9 was rolled back on this machine'))).toHaveLength(1)+ offered = '9.9.10'+ for (let i = 0; i < 100 && !staged.length; i++) await new Promise((resolve) => setTimeout(resolve, 10))+ poller.stop()+ expect(staged).toEqual(['9.9.10'])+ expect(JSON.parse(readFileSync(join(dir, 'update-pending.json'), 'utf8'))).toMatchObject({ version: '9.9.10' })+ } finally { log.mockRestore() }+ })+})++describe('the updater when things go wrong', () => {+ const sha = (b: Buffer): string => createHash('sha256').update(b).digest('hex')+ const notify = Buffer.from('export {}\n')+ afterEach(() => {+ vi.unstubAllGlobals()+ vi.restoreAllMocks()+ for (const dir of dirs) rmSync(dir, { recursive: true, force: true })+ dirs = []+ })++ function serve(cli: Buffer, onDownload: () => void = () => {}): void {+ vi.stubGlobal('fetch', async (url: string) => {+ if (url.endsWith('/metadata.json')) {+ return new Response(JSON.stringify({ adapter: {+ version: '9.9.9',+ cli: { url: 'https://updates.test/cli.js', sha256: sha(cli) },+ notify: { url: 'https://updates.test/notify.mjs', sha256: sha(notify) },+ } }))+ }+ onDownload()+ return new Response(url.endsWith('cli.js') ? cli : notify)+ })+ }++ it('never calls an unreadable version newer', () => {+ expect(semverGt('soon', '1.0.0')).toBe(false)+ expect(semverGt('1.0.0', 'v-next')).toBe(false)+ })++ it('reads an unreachable manifest as none, and refuses a download that fails or does not match', async () => {+ vi.stubGlobal('fetch', async (url: string) => {+ if (url.endsWith('/metadata.json')) return new Response('', { status: 503 })+ if (url.endsWith('/gone.js')) return new Response('', { status: 404 })+ return new Response('the wrong bytes')+ })+ expect(await fetchManifest('https://updates.test/metadata.json', 'adapter')).toBeNull()+ await expect(downloadVerified({ url: 'https://updates.test/gone.js', sha256: 'x' })).rejects.toThrow('HTTP 404')+ await expect(downloadVerified({ url: 'https://updates.test/cli.js', sha256: sha(Buffer.from('the right bytes')) }))+ .rejects.toThrow('sha256 mismatch')+ })++ it('fails the canary of a build that will not run, or that it cannot even write down', () => {+ const broken = Buffer.from('#!/usr/bin/env node\nprocess.exit(3)\n')+ const dir = tempDir()+ expect(canary(broken, dir)).toBe(false)+ expect(readdirSync(dir), 'the canary cleans up after itself').toEqual([])+ const file = join(dir, 'not-a-folder')+ writeFileSync(file, '')+ expect(canary(Buffer.from('console.log(1)\n'), file)).toBe(false)+ })++ it('skips a build that fails its canary, says why, and stages nothing', async () => {+ serve(Buffer.from('#!/usr/bin/env node\nprocess.exit(3)\n'))+ vi.spyOn(console, 'log').mockImplementation(() => {})+ const dir = tempDir()+ const errors: string[] = []+ let poller: { stop(): void } | undefined+ // Stopped from the log line itself, so no later check is left running when the test ends.+ vi.spyOn(console, 'error').mockImplementation((line: string) => { errors.push(String(line)); poller?.stop() })+ const staged: string[] = []+ poller = startSelfUpdater({+ currentVersion: '1.0.0', url: 'https://updates.test/metadata.json', key: 'adapter', dir, intervalMs: 10,+ onStaged: (version) => { staged.push(version) },+ })+ await vi.waitFor(() => expect(errors).toHaveLength(1), { timeout: 5_000 })+ expect(errors[0]).toContain('canary failed for the new build')+ expect(staged).toEqual([])+ expect(readdirSync(dir)).toEqual([])+ })++ it('waits out a busy spawn lock with the build it already verified, and reports any other failure', async () => {+ let downloads = 0+ serve(Buffer.from('#!/usr/bin/env node\nconsole.log("x")\n'), () => { downloads++ })+ const logs: string[] = []+ vi.spyOn(console, 'log').mockImplementation((line: string) => { logs.push(String(line)) })+ vi.spyOn(console, 'error').mockImplementation((line: string) => { logs.push(String(line)) })+ let attempts = 0+ const staged: string[] = []+ startSelfUpdater({+ currentVersion: '1.0.0', url: 'https://updates.test/metadata.json', key: 'adapter', dir: tempDir(), intervalMs: 10,+ withLock: async (fn) => {+ attempts+++ if (attempts === 1) throw new SpawnLockBusyError(null)+ if (attempts === 2) throw new Error('disk full')+ if (attempts === 3) throw 'lock folder vanished' // not an Error: still logged, still retried+ return fn()+ },+ onStaged: (version) => { staged.push(version) },+ })+ // Staging stops the updater for good, so nothing is left running.+ await vi.waitFor(() => expect(staged).toEqual(['9.9.9']), { timeout: 5_000 })+ expect(downloads, 'downloaded once: later checks reuse the bytes already verified').toBe(2)+ expect(logs).toContain('[update] 9.9.9 is ready but the daemon spawn lock is held by an unknown process — trying again next check')+ expect(logs.filter((line) => line === '[update] check failed (will retry):')).toHaveLength(2)+ })+})diff --git a/cli/src/lib/selfUpdate.ts b/cli/src/lib/selfUpdate.tsindex 59d0318d3..ac7d1f250 100644--- a/cli/src/lib/selfUpdate.ts+++ b/cli/src/lib/selfUpdate.ts@@ -14,7 +14,7 @@*/import { createHash } from 'crypto'-import { copyFileSync, existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'fs'+import { copyFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'fs'import { spawnSync } from 'child_process'import { join } from 'path'import { SpawnLockBusyError, describeSpawnLockFailure } from './daemonSpawnLock.js'@@ -35,6 +35,33 @@ const CLI = 'cli.js'const NOTIFY = 'notify.mjs'const PACKAGE = 'package.json'const RUNTIME_PACKAGE = `${JSON.stringify({ type: 'module' })}\n`+/** The version a staged update put in place, until it is kept (`confirm`) or rolled back (`restore`). */+const PENDING = 'update-pending.json'+/** Versions this machine rolled back. The background updater never stages them again; a newer build,+ * or `harness update` on purpose, moves past them. */+const REJECTED = 'update-rejected.json'+const REJECTED_KEPT = 10++function readJson(file: string): unknown {+ try { return JSON.parse(readFileSync(file, 'utf8')) } catch { return null }+}++function writeJson(file: string, value: unknown): void {+ const tmp = `${file}.tmp`+ writeFileSync(tmp, `${JSON.stringify(value)}\n`)+ renameSync(tmp, file)+}++function pendingVersion(dir: string): string | null {+ const pending = readJson(join(dir, PENDING)) as { version?: unknown } | null+ return typeof pending?.version === 'string' ? pending.version : null+}++/** The versions this machine rolled back, oldest first. */+export function rejectedVersions(dir: string): string[] {+ const rejected = readJson(join(dir, REJECTED))+ return Array.isArray(rejected) ? rejected.filter((version): version is string => typeof version === 'string') : []+}/** Strict semver-greater on the `X.Y.Z` core (ignores pre-release/build). Unparseable → false, so a* malformed manifest or the `0.0.0-dev` dev sentinel never triggers a downgrade/oscillation.@@ -121,9 +148,10 @@ export function canary(cliBuf: Buffer, dir: string): boolean {}}-/** Atomically swap the new bytes into `dir`, backing up the current files to `.prev` for rollback.+/** Atomically swap the new bytes into `dir`, backing up the current files to `.prev` for rollback, and+ * note `version` as pending until it is kept or rolled back.* (Verify-in-memory first ⇒ we only ever write bytes we already trust; write-tmp+rename ⇒ no torn file.) */-export function stage(dir: string, cliBuf: Buffer, notifyBuf: Buffer): void {+export function stage(dir: string, cliBuf: Buffer, notifyBuf: Buffer, version?: string): void {mkdirSync(dir, { recursive: true })writeFileSync(join(dir, PACKAGE), RUNTIME_PACKAGE)const swap = (name: string, buf: Buffer): void => {@@ -135,21 +163,37 @@ export function stage(dir: string, cliBuf: Buffer, notifyBuf: Buffer): void {}swap(CLI, cliBuf)swap(NOTIFY, notifyBuf)+ if (version) writeJson(join(dir, PENDING), { version, at: Date.now() })}-/** Roll a failed update back to the `.prev` bytes (called by the supervisor when the new build crashes). */+/**+ * Roll a failed update back to the `.prev` bytes (called by the supervisor when the new build crashes),+ * and remember the version that failed. Without that, the restored daemon's updater found the same+ * build in the manifest a minute later and staged it again: a release that passed its canary but+ * crashed the daemon restarted every machine once a minute until a fix was published.+ */export function restore(dir: string): void {for (const name of [CLI, NOTIFY]) {const prev = join(dir, `${name}.prev`)if (existsSync(prev)) { try { renameSync(prev, join(dir, name)) } catch { /* ignore */ } }}+ const failed = pendingVersion(dir)+ try {+ if (failed) writeJson(join(dir, REJECTED), [...rejectedVersions(dir).filter((version) => version !== failed), failed].slice(-REJECTED_KEPT))+ rmSync(join(dir, PENDING), { force: true })+ } catch { /* the rollback itself is what matters */ }}-/** Drop the `.prev` backups once the new build is confirmed healthy. */+/** Drop the `.prev` backups once the new build is confirmed healthy; a version kept is not rejected. */export function confirm(dir: string): void {for (const name of [CLI, NOTIFY]) {try { rmSync(join(dir, `${name}.prev`), { force: true }) } catch { /* ignore */ }}+ const kept = pendingVersion(dir)+ try {+ if (kept && rejectedVersions(dir).includes(kept)) writeJson(join(dir, REJECTED), rejectedVersions(dir).filter((version) => version !== kept))+ rmSync(join(dir, PENDING), { force: true })+ } catch { /* the new build runs either way */ }}export interface Poller { stop(): void }@@ -206,18 +250,19 @@ export function startSelfUpdater(opts: {// the lock was busy (a `harness start` or `harness update` mid-flight) the next tick should try the// swap again, not the whole download.let verified: { version: string; cliBuf: Buffer; notifyBuf: Buffer } | null = null+ // Said once per version: the check repeats every interval.+ let toldRejected: string | null = nullconst now = opts.now ?? Date.nowconst stop = (): void => { if (timer) { clearTimeout(timer); timer = null } }const schedule = (): void => {- if (done) returnstop()timer = setTimeout(() => void tick(), msUntilSlot(now(), opts.slotSecond, opts.intervalMs))timer.unref?.()}+ // No `done` check here or in `schedule`: `done` is set only next to `stop()`, so no tick fires after it.const tick = async (): Promise<void> => {- if (done) return// Booked before the work, from the clock: the next tick is on the next slot whatever this one// costs; a tick that stages calls `stop()` below and cancels it. Booked even when this slot is// skipped because the previous check is still running (a download on a slow link can outlast a@@ -228,6 +273,11 @@ export function startSelfUpdater(opts: {try {const entry = await fetchManifest(opts.url, opts.key)if (!entry || !shouldAutoUpdate(entry.version, opts.currentVersion)) return+ if (rejectedVersions(opts.dir).includes(entry.version)) {+ if (toldRejected !== entry.version) console.log(`[update] ${entry.version} was rolled back on this machine — waiting for a newer build (\`harness update\` installs it anyway)`)+ toldRejected = entry.version+ return+ }if (verified?.version !== entry.version) {console.log(`[update] newer build available: ${opts.currentVersion} → ${entry.version}`)const cliBuf = await downloadVerified(entry.cli)@@ -239,20 +289,19 @@ export function startSelfUpdater(opts: {// Downloaded and verified OUTSIDE the lock (that can take a while on a slow link and touches// nothing shared); swapped and handed off INSIDE it.await withLock(async () => {- stage(opts.dir, ready.cliBuf, ready.notifyBuf)+ stage(opts.dir, ready.cliBuf, ready.notifyBuf, ready.version)done = truestop()console.log(`[update] staged ${ready.version} — restarting now`)await opts.onStaged(ready.version)+ }).catch((err: unknown) => {+ // A busy lock is not a failed check: the bytes are good and waiting, and the next tick tries+ // the swap again. Say so, or a minute of "check failed" reads as the updater being broken.+ if (!(err instanceof SpawnLockBusyError)) throw err+ console.log(`[update] ${ready.version} is ready but the daemon spawn lock is ${describeSpawnLockFailure(err)} — trying again next check`)})} catch (err) {- // A busy lock is not a failed check: the bytes are good and waiting, and the next tick tries- // the swap again. Say so, or a minute of "check failed" reads as the updater being broken.- if (err instanceof SpawnLockBusyError) {- console.log(`[update] ${verified?.version ?? 'a new build'} is ready but the daemon spawn lock is ${describeSpawnLockFailure(err)} — trying again next check`)- } else {- console.error('[update] check failed (will retry):', err instanceof Error ? err.message : err)- }+ console.error('[update] check failed (will retry):', err instanceof Error ? err.message : err)} finally {checking = false}diff --git a/cli/vitest.harnessd.config.ts b/cli/vitest.harnessd.config.tsnew file mode 100644index 000000000..523f9ac60--- /dev/null+++ b/cli/vitest.harnessd.config.ts@@ -0,0 +1,19 @@+import { defineConfig } from 'vitest/config'+import base from './vitest.config.js'++/** harnessd's master, protocol and core link: the daemon's supervisor, held to 100% in every file. */+export default defineConfig({+ test: {+ ...base.test,+ include: ['src/harnessd/**/*.spec.ts'],+ coverage: {+ enabled: true,+ provider: 'v8',+ include: ['src/harnessd/**/*.ts'],+ exclude: ['src/harnessd/**/*.spec.ts'],+ reporter: ['text', 'json-summary'],+ reportsDirectory: 'coverage/harnessd',+ thresholds: { perFile: true, statements: 100, branches: 100, functions: 100, lines: 100 },+ },+ },+})diff --git a/docs/design/2026-10-03-harnessd.md b/docs/design/2026-10-03-harnessd.mdnew file mode 100644index 000000000..0a64a4ebf--- /dev/null+++ b/docs/design/2026-10-03-harnessd.md@@ -0,0 +1,628 @@+# harnessd++harnessd is the Harness daemon rebuilt around one rule: **the process that keeps your agents+reachable never fails because of a feature.** It is a small supervisor, a lean core that owns+sessions, terminals and turns, and separate service processes for everything else. The `harness`+command and every app stay clients, the way `docker` is a client of `dockerd`.++This document records the design requested on October 3, 2026, and what has been built against it.+It is the one document the rebuild is reviewed against. It changes as decisions land, and says which+ones are still open.++## Why++The daemon today is one Node process that does everything. Its startup is a single 5,200-line+function in [`cli.ts`](../../cli/src/cli.ts); [`backendSocket.ts`](../../cli/src/backendSocket.ts)+routes 54 request types for local and remote clients alike through about 30 callbacks that+`cli.ts` assigns after construction. Nearly every subsystem is constructed on every start. The+only gates are environment flags, sign-in, a connected device, or a state directory existing.++That shape turns any feature's bug into everybody's outage:++- **2026-10-03.** Opening a pane read the agent's whole transcript into memory. An 803 MB Codex+ rollout took 1.9 GB of heap on its own; the daemon died at its 4 GB limit seconds after every+ start, and every pane on the machine dropped about every ten seconds+ ([PR #705](https://github.com/autonomous-ai/openharness/pull/705)). Nothing restarted it but the+ desktop app running `harness start` again, up to a minute later.+- Optional features run inside the process that holds the sessions: the USB dial's `ioreg` probe+ every 2 s whether or not a dial was ever plugged in, warm `claude`/`codex` recap workers that the+ default recap mode never uses, the search indexer over every transcript, one viewer process per+ harness, headless Chrome for shared viewers, a 15 s backend poll for tab channels, and a+ 10-minute check that keeps the managed `grid` on its pin.+- Work grows with history, not activity. Besides attach, whole histories are read by a client's+ `session_get` (before it windows) and `sessions_list` (to count lines), every engine's recap but+ Claude Code's and Codex's, the Cursor watcher, the first search pass, Change agent's handoff, the+ Codex resume repair, and the database engines' first load: OpenCode, Kilo, Hermes and Devin read+ every message of a session in one synchronous query on the core's event loop.++[PR #684](https://github.com/autonomous-ai/openharness/pull/684) moved Companions and Memory out+as a library. harnessd finishes that direction for the whole daemon and gives the extracted parts+somewhere to run.++## Rules++1. **Stability first.** No feature runs in the process that owns sessions. A service can crash,+ hang or leak without dropping a pane, losing a question or blocking input.+2. **Work follows activity, never history.** No hot path (attach, a request, a turn ending) may+ read in proportion to a conversation's length. Reads are bounded and streamed; caches have+ budgets; queues have limits.+3. **Nothing waits on a service.** The core starts, serves and stops without any service. An+ unavailable service fails only its own capability, with a retryable error.+4. **One contract, narrow authority.** Services use the same authorized local API clients use,+ with a scoped role. No unrestricted extension RPC; every addition to the contract is reviewed+ on its own ([CONTRIBUTING.md](../../CONTRIBUTING.md)).+5. **Port proven logic.** The normalizers, tmux handling and transport carry hundreds of measured+ fixes. The architecture is rebuilt; that logic moves with its tests.+6. **No flag day.** The desktop app, web, phone, `hn`, engine hooks, the device and the backend+ keep working throughout. Protocol changes are additive.+7. **Learn from the daemons that work.** Where a well-run daemon has solved a problem we have, we+ take its answer unless we can say why ours differs ([What other daemons do](#appendix-what-other-daemons-do)).++## Processes++```text+ desktop app · hn · `harness` · engine hooks phone · web · other machines+ │ │+ │ unix socket + 127.0.0.1:18473 relay server │+┌─┴─────────────────────────────────────────────────────────────────────────────────────────┴────┐+│ │ harnessd supervised by launchd / systemd (planned) │ │+│ │ │ │+│ │ ┌────────────────────────────────────────────────────────────────────────────────┐ │ │+│ │ │ MASTER · the supervisor │ │ │+│ │ │ does start · readiness and heartbeat watch · memory budget │ │ │+│ │ │ restart with backoff · safe mode on a crash loop · updates │ │ │+│ │ │ status file · log caps │ │ │+│ │ │ never feature code · network · sockets │ │ │+│ │ └────────────────────────────────────────────────────────────────────────────────┘ │ │+│ │ │ runs and watches it; restarts it in about a second │ │+│ │ ┌────────────────────────────────────────────────────────────────────────────────┐ │ │+│ │ │ CORE · the part that must never go down │ │ │+│ │ │ agents registry · create / fork / restart / resume / stop / close │ │ │+│ │ │ terminals tmux backend · discovery (5 s) · live streams · reboot restore │ │ │+│ │ │ engines hooks server for 13 engines · launch commands │ │ │+│ │ │ transcripts attach from the end · live tail · history pages │ │ │+│ │ │ turns working / idle · model chips · activity · recap excerpts │ │ │+│ │ │ input what you type, delivered to the pane, confirmed │ │ │+│ │ │ questions agent waiting on you · your answer back │ │ │+│ └─►│ local API the one door · request gate until start-up is done │ │ │+│ └────────────────────────────────────────────────────────────────────────────────┘ │ │+│ │ local API, service role: the same door, narrower rights │ │+│ ┌────────────────────────────────────────────────────────────────────────────────┐ │ │+│ │ SERVICES · their own processes, light ones sharing a host │ │ │+│ │ relay backend link · sign-in · E2EE pairing · trust group · P2P │◄────┘ │+│ │ sharing · view other machines · machine list (self-hostable) │ │+│ │ devices driver host: the round unit (USB) · Wi-Fi device · more types │ │+│ │ models Grid · local models (Ollama, LM Studio, llama.cpp) · API keys │ │+│ │ search session index across all engines │ │+│ │ viewers harness viewers · Store installs · headless Chrome (shares) │ │+│ │ teams tab channels · mailbox · orchestrator │ │+│ │ workspaces worktrees · git and PRs · project folders · file previews │ │+│ │ companion characters · growth ┐ a package since #684; │ │+│ │ memory coding memory · recall ┘ nothing starts them today │ │+│ │ light host usage · machine resources · command bar │ │+│ └────────────────────────────────────────────────────────────────────────────────┘ │+│ │+└────────────────────────────────────────────────────────────────────────────────────────────────┘+ your agents live in tmux: they keep running through every restart above+```++**The master** ([`harnessd/supervisor.ts`](../../cli/src/harnessd/supervisor.ts)) is the process+`harness start` launches (`harness __harnessd`). It claims the pid file, keeps its children running+and stops them. It contains no feature code, opens no network connection and holds no sockets, so+there is almost nothing in it that can fail. It is generic: it knows a service by its manifest+entry, not by what it does.++**The core** owns everything a session needs and nothing else, and it binds the listening sockets+itself. Passing them through the master would buy a refused connection's worth of time during a+core restart, which every client already retries, at the price of complexity in the one process that+must stay simple; on Linux, systemd socket activation can hold them later without the master doing+anything (see [Supervision](#supervision)). If the core crashes, hangs or outgrows its memory budget,+the master starts a new one. Clients reconnect as they do today; agents keep running throughout,+because they live in tmux, not in the daemon. A core whose master is gone stops, so nothing is left+holding the port.++**Services** are separate processes, each started from the same bundle as `harnessd <service>`+with its own process title. Each connects to the core's local socket like any other client, under+the service role, and reconnects with backoff when the core restarts. A service never writes to+tmux, never reads the registry file, and never holds a credential the core can hold for it.+Isolating a feature costs a process: an idle Node process is about 40 MiB, and one that loads+today's 4.4 MB bundle peaks near 125 MiB. So the risky services — relay, devices, viewers, search,+models — get their own processes, and the light ones share one host.++## What the core owns++| Today | harnessd |+| --- | --- |+| Registry, stopped agents, session sync | core |+| tmux backend, terminal streams, discovery, restore after reboot | core |+| Engine hooks (`/api/hook/*`) and hook installation | core |+| Transcript tailing, normalizers, turn lifecycle, activity, model chips | core: the transcript service |+| Recap excerpts (`turn_summary`, `agent_recent`): the last answer, cut at turn end | core, with the turns |+| Input queue, delivery confirmation, questions on the pane | core |+| Agent create, fork, restart, retarget, resume, stop, close, purge, checkpoints | core |+| Worktrees, project folders and pre-trust, git and PRs, file and media preview | workspaces |+| Local socket and loopback API, `/api/local-ws` | core |+| Backend link, sign-in token, control-plane proxy, machine list, E2EE, pairing, trust group and device key log, peers, remote relay pool, P2P, sharing | relay |+| USB dial, cable fleet, device link, Wi-Fi device, firmware push, dial↔window bridge, voice router | devices |+| Grid access, local models, fleet RPC, API connections | models |+| Session search, external sessions, open sessions | search |+| Harness viewers, verdict watcher, Store installs, interactive and shared viewers (headless Chrome) | viewers |+| Tab channels, team service and mailbox, orchestrator | teams |+| Token and account usage, machine resources, the command bar | light host |+| The worktree sweep (deletes idle worktrees on a timer) | removed: cleanup becomes an app action ([#711](https://github.com/autonomous-ai/openharness/issues/711)) |+| Companion, Memory | their own services, off until used |+| CLI self-update, `hn` update, log trim | master |+| Recap model mode (`SUMMARY_MODE=model`) and its warm worker pool | removed |+| Requests kept only to refuse retired features (pair brain, the creatures desk) | removed once no shipped app sends them |++Questions stay in the core because they are session state: a question answered through one+interface must resolve for every other one. A recap excerpt is a few hundred characters cut from+the turn the core just finished, so it stays with the turns; the model that once rewrote it is gone.++## Services++### Relay — anyone can host one++The relay service connects this machine to its phone, the web and other machines through a relay+server. Harness runs one (`harness-api.autonomous.ai`), and the server's code is in this repository+(`backend/`), so anyone can run their own and point their machines and apps at it.++- **End-to-end encryption makes the server untrusted by design.** Sessions travel sealed between+ paired devices (CPace pairing, the trust group); the relay forwards frames it cannot read. Trust+ comes from pairing, not from whoever runs the server.+- **The address is a setting everywhere.** The daemon reads it from `BACKEND_WS_URL` and `WEB_URL`+ today; it becomes a `harness` setting and a field in the desktop app, and the phone's QR+ onboarding carries it, so a phone paired to a self-hosted relay never asks Harness for anything.+- **Sign-in is pluggable.** A Harness account is the default. A self-hosted relay can use its own+ accounts (OIDC) or none: paired device keys are already an identity the trust group verifies.+- **Features are capabilities the relay advertises.** Forwarding, presence and pairing rendezvous+ are the minimum. Desk sync, the Store, tab channels, shares and the device key log are optional,+ and the apps hide what a relay does not offer instead of failing.+- **The protocol is documented** — `adapter-ws` for machines, `web-ws`, `device-ws` and+ `observer-ws` for clients — with versioned frames, and TURN for peer-to-peer terminals is+ configurable.++### Devices — a host for drivers++The round unit is the first device type, not the only one. The devices service hosts one driver+per device type: `usb-dial` (the round unit over USB serial), `autonomous-device` (Wi-Fi and LAN),+and whatever comes next. A driver finds its devices, owns their transport — one owner per physical+port — and speaks to the core only through the local API: agent, turn and question events in;+input, answers and agent actions out. Firmware push, the dial↔window bridge and the voice router are+shared utilities drivers use. Adding a device type adds a driver and its manifest capabilities;+the core does not change.++### Models++Grid is how Harness runs local AI models with Codex, Claude Code and the other engines, from the+app. It is essential, so it starts by default, but it runs as its own service: a failed model+download or a crashed local model server must never take sessions down with it.++### Workspaces++Everything about the folder an agent works in: worktrees (created with their branch, the ignored files+`.worktreeinclude` names copied in), project folders and their pre-trust, git and PR status, file and+media previews. It runs on the machine that holds the folders, because a phone or another computer+can start an agent there, but not in the core: preparing a folder fetches, checks out and copies, and+is slow and can fail. A client asks workspaces for a folder, then creates the agent in it; the core+only ever receives a path. Cleaning up is the person's decision, made in the app with a preview —+workspaces lists what is left and removes what it is asked to, to the Trash+([#711](https://github.com/autonomous-ai/openharness/issues/711)).++### Companion and Memory++Both are a package since #684, and nothing starts them today. In harnessd they are two services+that are off until used, and Memory never depends on Companion.++## The local API++The core serves the contract documented in [cli.md](../cli.md#automation): a `machine_select`+frame, then request frames with a `requestId` answered by `<type>_result`, events for agents,+turns and activity, and the binary terminal channel. Nothing about it changes for existing+clients.++Services use the same socket and frames with three narrow additions, each reviewed on its own:++- **A service role in `machine_select`** (`role: "service"`, its name and a per-boot token the+ master passes it). The token proves the process is the one the master started; it grants only+ the capabilities its manifest entry lists.+- **Request ownership.** A service declares the request types it answers (`search` answers+ `session_search` and `session_tail`). The core routes those requests to it and returns its+ answer. If the service is down, the client gets `SERVICE_UNAVAILABLE` with `retryable: true`+ instead of a hang. Security checks (owner-only, sealed-only, local-only) run in the core before+ any routing, so moving a handler can never widen who may call it.+- **Scoped reads.** Services that need more than events get bounded reads, never a file path:+ `transcript_read` returns at most a window of one agent's transcript from an offset, and only+ to a service whose manifest grants it.++Events reach services through the same subscription clients use, with bounded queues. A slow+subscriber's queue is cut at its limit and it is sent `resync`; the core never blocks on it.++Clients and the daemon agree on a version range: an app newer or older than the daemon is told+so, the way the Docker CLI negotiates its API version and `tailscale` warns when the daemon differs.++## Supervision++Built for the core ([`harnessd/`](../../cli/src/harnessd/), spawn protocol 2): readiness apart from+binding, with a start-up deadline; the heartbeat watch; the exit-code contract; exit reasons;+crash-loop safe mode; the heap and resident budgets; the status file; log trimming in the master; and+updates with probation from ready and rollback — a broken update that falls into safe mode rolls back+too. `HARNESS_NO_MASTER=1` runs the core on its own. What follows is the design the master is held to;+the parts still to build say so.++- **Ready is not bound.** The core says `bound` when its port is listening and `ready` when start-up+ is done and the request gate opens. Start-up has a deadline: a core that binds but is not ready in+ time is killed and counted as a crash, because a start-up that hangs after binding otherwise looks+ healthy for ever while every request waits at the gate. Probation for an update counts from+ `ready`. A core from before `ready` (protocol 1, after a rollback) is held to `bound` alone.+- **Alive is more than a timer.** The heartbeat proves the event loop turns; it also carries the+ longest event-loop delay since the last one, and next the age of the oldest waiting request, so a+ core that is slow to the point of useless is seen before it is hung. Every child's last exit is+ recorded with its reason: crashed, hung, did not bind, not ready, memory, update, stopped.+- **Exit codes are a contract.** 75 restarts at once onto the bundle on disk (an update). 78 means+ stop for good — this computer was removed from the account, or it is connected from another one —+ and the master stops with it. Any other exit, 0 included, is a crash and is restarted: Node also+ exits 0 when its event loop runs out of work, and an outside SIGTERM to the core must not take the+ daemon down.+- **Backoff that forgives, and a way out of a loop.** Restarts back off from 0.5 s to 30 s and the+ delay resets after a minute up, timed on a monotonic clock so sleep and clock changes do not skew+ it. Three crashes inside five minutes start the core in safe mode — the updater alone, so a+ published fix can still arrive — instead of the same crash again. When safe mode's own time runs+ out (15 minutes, counted while the machine is awake) the master tries a normal core again, and an+ update leaves safe mode at once. A core that falls into safe mode by itself, because its start-up+ threw, tells the master so; the master reports it, and rolls back an update whose first core does+ this. Chrome and VS Code break their loops the same way.+- **Memory: a clean restart before a fatal one.** The core runs with an explicit V8 heap limit, and+ the master restarts it cleanly when its heap passes 75% of that limit, or its resident memory+ passes a larger ceiling that also covers buffers V8 does not count. With the two limits equal, V8's+ own out-of-memory abort — which skips every cleanup — would come first, as it did on October 3.+- **Status without the core.** The master writes a status file (each child's state, restarts, last+ exit and its reason, safe mode) atomically on every change, so `harness status` can say what+ happened when nobody can ask a core that is crash-looping.+- **Logs have caps.** The master trims the daemon's log, which a supervised core no longer does+ itself; per-process files come next with their own caps, written by the master as each child's+ stdio so no line is lost in a pipe.+- **Ordered stop, deadlines owned by the master.** Services, then the core, then the master, each+ with a deadline the master enforces; `harness stop` waits for the master instead of killing it+ after a fixed three seconds. A fast stop leaves agents running in tmux; that is the only kind.+- **Updates.** The core's updater stages a bundle and exits 75; the master restarts the core on it,+ on probation from `ready`, and rolls back if it fails. Next: when the master's own code changed,+ it re-executes itself in the gap between the old core's exit and the new core's start, as+ Syncthing's monitor does — re-executing with a core running would cost that core its channel.+ Bundles move to versioned directories behind one pointer, so a swap is one rename, and a version+ that was rolled back is not offered again.+- **Something supervises the supervisor.** On macOS a launchd user agent (registered through+ `SMAppService`, which asks the user once) and on Linux a systemd user unit run the master in the+ foreground and restart it; the desktop app then watches instead of respawning, so there is one+ supervisor per machine. tmux must stay out of the daemon's process group and cgroup — a systemd+ stop kills everything in the unit's cgroup by default, and launchd kills a dead job's process+ group — so agents keep running when the daemon stops, the way containerd shims keep containers.+- **Versioned spawn protocol.** Master and core may differ during an update. Messages are only ever+ added, unknown ones are ignored, and each side states its protocol in its first message.++For services the master reads a manifest bundled with the release:++```json+{ "name": "devices", "args": ["service", "devices"], "enabled": "auto",+ "memoryMiB": 256, "capabilities": ["events:agents", "events:turns", "requests:dial_*"] }+```++- **Start.** The core first, then enabled services, without waiting on any of them. `enabled` is+ on, off, or auto (a probe the service runs itself and exits cleanly from when it has nothing to+ do, such as no device ever paired).+- **Restart.** Exponential backoff from 1 s to 60 s. After five crashes in ten minutes a service+ is parked as failed and reported, never respawned in a tight loop, and retried slowly and after+ every update. The core is always restarted.+- **Budgets.** Every child runs with an explicit heap limit from its manifest and the memory rules+ above.+- **Lifetime.** Children watch their spawn channel and exit when the master goes, so a crash+ never leaves orphans holding the socket, the serial port or a pid file.++## Bounded work++Every read on a hot path is bounded by what it serves, never by a conversation's length. Built:++- **Attach** walks back from the end of a transcript to the last turn opener and streams forward+ from there ([`attachTranscript.ts`](../../cli/src/lib/attachTranscript.ts), merged in #705). It+ reaches past the opener only for the few older records the chips, a continuing `/goal` and calls+ still answering need, at most 64 MiB. A session attached again while it is tailed hands its tail+ over byte-exact.+- **History pages** (`session_get`) are read by walking back from their cursor to the turn they+ start in, holding only the page, at most 32 MiB; a line index kept as the file grows counts lines+ (`sessions_list`) and finds Codex's line cursors ([`transcriptPages.ts`](../../cli/src/lib/transcriptPages.ts)).+- **The live tail** reads what it fell behind on a chunk at a time, and a file rewritten in place+ with more history than one batch should hold is attached again from its end instead of replayed.++Claude Code and Codex are done. The order for the rest is OpenCode, Grok, pi, then every other+engine; database engines follow the same rule with a query for the last turn instead of the whole+message list, and their reads move off the event loop. A test fails the build if core code makes a+whole-file read (`tailFile(…, Infinity)`, `readFile` on a transcript, an unbounded+`createReadStream`) of an agent's history.++## Names and files++| Thing | Today | harnessd |+| --- | --- | --- |+| Daemon process | `node cli.js __run` | `harnessd` (master), `harnessd core`, `harnessd <service>` |+| Command | `harness` | `harness`, unchanged: `harness start`, `stop`, `status`, `logs` manage harnessd |+| State | `~/.harness/cli/data/` | unchanged until the core is stable; then `~/.harness/harnessd/` with a staged, verified move and the old path kept for rollback |+| Socket | `~/.harness/cli/data/daemon-<port>.sock` | unchanged path, bound by the core |+| Loopback | `127.0.0.1:18473` | unchanged |+| Status | `/api/status`, from the core | the core's, plus the master's status file beside the pid file |+| Logs | `~/.harness/cli/data/harness.log` | trimmed by the master; later one capped file per process |++`harnessd` joins `harness` and `hn` as a command name in the+[Naming System](../naming-system.md); it is not a product name. Prose says "the Harness daemon".++## Compatibility++| Client | What must keep working |+| --- | --- |+| Desktop app | `LocalCliDiscovery` finds the socket and port, runs `harness start`, reads the pid file; the owner check in `process_responsibility.dart` sees the master as the daemon. |+| Web and phone | Through the relay, now terminated in the relay service: same frames, same E2EE type sets and keys. |+| `hn` | The same local frames and terminal channel. |+| Engine hooks | `notify.mjs` posts to the loopback port as today. |+| Harness devices | The same cable protocol, now spoken by the device's driver, with one owner of the serial port at a time. |+| Backend | The same `adapter-ws` connection, from the relay service. |+| Older installs | An auto-update from a daemon before harnessd starts the new core the old way, without a master, which works unchanged; the master arrives at the next `harness start`. Next: such a core hands itself to a master. A downgrade stops cleanly: the old `harness stop` signals the master named in the pid file. |++## Rollout++Each step ships on its own, keeps every client working, and can be turned back by configuration+while it beds in.++Done so far:++- **The end-to-end harness** (`cli/e2e`, `npm run test:e2e`): the real daemon under a throwaway home,+ a private tmux server, fake Claude Code and Codex engines that bind through the real hooks, and a+ client that speaks the desktop's protocol. It reproduced the October 3 crash on `main` and found+ three more defects, now fixed and kept as tests.+- **The master** at 100% coverage, tested end to end: crash, hang, a start-up that hangs after+ binding, a SIGTERM from outside, a crash loop into safe mode and out again, killed master, clean+ stop inside `harness stop`'s grace, memory budget with backoff, and the status file throughout.+- **The request gate** (`BackendSocket.holdRequests`/`openRequests`): requests that arrive while the+ core is still starting wait, each connection's in order, instead of being refused or dropped.+- **Bounded reads for Claude Code and Codex:** attach (#705, merged), history pages and line counts,+ and the live tail, each proven under a 256 MiB heap against a 300 MiB transcript that runs `main`+ out of memory.+- **The recap model mode is gone,** with its warm worker pool: a recap is the answer's own excerpt.+- **Updates that fail stay failed.** A build the master rolls back is remembered and never staged+ again; a newer build moves past it, and `harness update` installs it on purpose. Before, a release+ that passed its canary but crashed the core restarted every machine on it once a minute. Tested+ end to end on real release bundles served from a local manifest (`e2e/update.e2e.ts`): one kept,+ one rolled back and left alone, then a newer one kept.++Next, in order:++1. **Removals:** requests kept only to refuse retired features, once no shipped app sends them.+2. **Bounded reads for every other engine,** in the order above, each with a conformance suite.+3. **Core boundary, in process.** Turn `runForeground` into a composition root over explicit+ modules: registry, terminals, transcripts, input, questions, local API. Replace the callback+ properties with a typed core API. No behaviour change; the existing suites pass unmodified.+ The plan is below.+4. **Services out of process,** one at a time, each behind its manifest entry and able to fall back+ to in-process for one release: devices first, then relay (with the self-hosting settings),+ models, workspaces, search, viewers, teams, the light host, Companion and Memory.+5. **Platform supervision** (launchd agent, systemd user unit) and the master re-executing itself on+ update.+6. **State directory move,** last, once nothing else is changing.++### The core boundary, step by step++`runForeground` is 5,306 lines. About 4,600 of them move into modules under `cli/src/core/`. What+stays, about 700 lines, is the composition root: the guards and updaters that must run first, the+terminal backend, `BackendSocket` and its gate, the hook server and pid file, the boot sequence+(restore, then the first reconcile), restart-for-update and shutdown, and `openRequests`, `ready`+last. `startupOrder.spec.ts` already pins those in place.++Each module is a factory, `createX(deps)`, that returns closures, the pattern+`createStopAgentService` already uses.++| Module | What moves into it |+|---|---|+| `terminals/control` | Control leases, capture, submit, type, keys, pinning |+| `terminals/bootstrap` | The tmux backend, terminals, the host theme, pane titles |+| `agents/events` | Session sync and announcements |+| `transcripts/normalizers` | The 14 per-engine normalizer and reader maps, as one table |+| `transcripts/attach`, `ingest` | Attach, the attach tracker, watcher wiring, line ingest |+| `turns/funnel` | The one place session events fan out from |+| `turns/*` | Heartbeats, recaps, stop hooks, runtime profiles, reconcile timers |+| `input` | The input controller and the pane writer lock |+| `questions` | The question watcher, answers, open questions |+| `agents/*` | Forget, bind, discovery, launch; create, fork, retarget, restart, resume, stop, close, purge |+| `engines/hooks` | Hook handlers and hook installs |+| `api` | `CoreApi`, `CorePorts`, and binding `BackendSocket` |++Two typed interfaces replace the 53 callback assignments on `BackendSocket`:++- **`CoreApi`, services to core:** agents (list, resolve, create, fork), input (submit, cancel),+ questions (answer, open), recaps (recent turns, asks), runtime profiles (models, set), and events+ (agent, question, turn, stream, delivery). Each member is an existing request or event. There is+ no generic `call(name, args)`.+- **`CorePorts`, core to services, in process only:** the synchronous calls the core makes into+ features today: audience (which devices are watching), teams, devices, search, models, viewers+ and workspaces. Each port is a contract decision to make before its service leaves the process.++The order. Each step moves code verbatim and keeps the old names as aliases, so call sites and the+start-up lint stay as they are; each is checked by `tsc`, the unit suites and the e2e suite.++1. A spec that pins the start-up order: requests held from construction, every port bound before+ `openRequests`, then `ready`, then `[cli] ready`.+2. Terminal control.+3. Session events.+4. The normalizer table.+5. Input, with the pane writer lock.+6. Questions.+7. Heartbeats, activity and recaps.+8. The event funnel.+9. Attach, ingest and the stop hooks.+10. Agent discovery and binding.+11. The agent lifecycle.+12. Engine hooks.+13. Services in process, each behind `start(core, ports)`.+14. One `bindBackendSocket` for all 53 assignments.++Moving the request switch's case bodies into a file per owner comes after.++What has to survive the move exactly:++- **The funnel's order.** Each event goes to the backend, then search, input, teams, the pane+ writer, devices, the heartbeat and the question watcher, in that order. Recaps come after the+ loop, and the devices stream comes last and never on replay. A bus that notified subscribers+ afterwards would reorder frames.+- **The input, pane writer and turns cycle.** Every pane write goes through the device input lock,+ which calls back into input, which re-enters the funnel. They move together.+- **Start-up windows.** Hooks arrive before the request gate opens and are not gated. Stopped-agent+ frames are published during restore, outside the gate, so their providers must be bound before+ restore. Relay callbacks are bound after `connect()` today, which is safe only because nothing+ awaits in between; they move before it.+- **Log lines that tests read:** `[cli] ready`, `transcript rewritten in place — attaching it+ again`, `[backend] machine busy`.++Found while mapping, and fixed on their own rather than inside a move:++- Kilo's reader is skipped on cancel and on shutdown, so a cancelled Kilo agent can keep showing+ as working.+- `BackendSocket.onLocalClient` is never wired.++## Testing++The bar for the core is total: 100% statements, branches, functions and lines, enforced in its+vitest configuration, as the orchestrator, sharing and resume suites already do.++- **Engine conformance, for every agent we support.** Each engine has a suite that checks attach+ and history pages against the whole-history reads they replace at every cut point, its turn+ lifecycle, questions and model chips, on fixtures, plus an opt-in run against a machine's own+ transcripts (`ATTACH_REAL_TRANSCRIPTS=1` today for Claude Code and Codex), plus the real-CLI+ checks for installed engines (`test:tmux-real`).+- **End to end, out of process.** The harness boots the master, core and services under a throwaway+ home and a private tmux server, drives them with a client that speaks the desktop's protocol, and+ runs scripted engines that write transcripts and call hooks: create, stream, type, deliver,+ question, answer, turn end, recap, restart, resume, stop.+- **Failure.** Kill, hang and overload each service and the core under load: crash loops, a start-up+ that hangs after binding, a heap that grows to its limit. Panes, questions and input must stay+ within an agreed baseline, and nothing respawns in a loop. Corrupt each state file, fill the disk,+ stall a subscriber.+- **Budgets as tests.** The regression from October 3 is a permanent test: a transcript larger than+ the heap, attached, paged and rewritten in place, and 50 panes attaching at once, under a fixed+ memory and time budget. A soak run holds a desk of agents for hours and fails on memory growth.+- **The apps.** The desktop app against harnessd in an isolated profile, never against the+ person's own harnesses.++## Open decisions++- **Platform supervision by default.** Whether the launchd agent and systemd unit are installed on+ every install, which on macOS asks the user once (Login Items).+- **tmux server.** Agents on the user's default tmux server, or a dedicated one that `tmux+ kill-server` cannot reach.+- **Service defaults.** Which services start on a new install and which wait to be needed (a device+ paired, sharing used, a viewer installed), and which light ones share the host.+- **Safe mode.** Whether three crashes in five minutes is the right trigger, and what a degraded core+ keeps beyond the updater.+- **A healthy update.** Ready plus a number of crash-free minutes, or a first successful attach.+- **The relay protocol's stability promise,** once self-hosting is documented.+- **State directory.** Whether moving to `~/.harness/harnessd/` is worth its migration at all.++## Appendix: what other daemons do++Studied on October 4, 2026, from each project's own documentation and source. The decisions above+cite what they take from here.++### Erlang/OTP supervisors++- **A restart strategy for each group.** `one_for_one` restarts only the failed child; `one_for_all` and `rest_for_one` also restart its dependents ([OTP](https://www.erlang.org/doc/system/sup_princ.html)).+- **Intensity and period.** More than MaxR restarts in MaxT seconds and the supervisor stops its children and itself; its parent decides what happens next. Default: 1 restart in 5 s.+- **Limits multiply across levels:** "do not set the restart intensities to the same values on all levels".+- **`transient` children** are restarted only after an abnormal exit. Children stop in reverse start order, each with its own timeout.++### systemd++- **Exit codes are a contract** (`RestartForceExitStatus=`, `SuccessExitStatus=`). Exponential restart delay arrived in version 254, but it does not reset after a healthy run ([systemd.service](https://man7.org/linux/man-pages/man5/systemd.service.5.html), [#43813](https://github.com/systemd/systemd/issues/43813)).+- **Start limit.** By default a unit started more than 5 times in 10 s is no longer restarted automatically ([systemd.unit](https://man7.org/linux/man-pages/man5/systemd.unit.5.html), [defaults](https://man7.org/linux/man-pages/man5/systemd-system.conf.5.html)).+- **Readiness and liveness are separate signals.** `READY=1` means start-up is done. `WATCHDOG=1` is sent from the event loop every half `WatchdogSec`; a missed one means SIGABRT ([sd_notify](https://man7.org/linux/man-pages/man3/sd_notify.3.html), [watchdog](http://0pointer.de/blog/projects/watchdog.html)).+- **Sockets outlive the service.** "If a service dies its listening socket stays around, not losing a single message" ([socket activation](http://0pointer.de/blog/projects/socket-activation.html)). `FDSTORE=1` keeps other fds across restarts ([sd_listen_fds](https://man7.org/linux/man-pages/man3/sd_listen_fds.3.html)).+- **Memory limits.** `MemoryHigh=` slows a unit down and `MemoryMax=` triggers the OOM killer inside it ([resource-control](https://man7.org/linux/man-pages/man5/systemd.resource-control.5.html)). User units can use both ([[email protected]](https://github.com/systemd/systemd/blob/main/units/user%40.service.in)). By default, stopping a unit kills everything in its cgroup ([systemd.kill](https://man7.org/linux/man-pages/man5/systemd.kill.5.html)).++### launchd++- **User agents** run while their user is logged in, and they must not daemonize: "If you do, launchd thinks your process has died" ([Apple](https://developer.apple.com/library/archive/documentation/MacOSX/Conceptual/BPSystemStartup/Chapters/CreatingLaunchdJobs.html)).+- **`KeepAlive`** can restart a job after a clean exit, a crash, or both. By default a job is spawned at most once every 10 s (`ThrottleInterval`) ([launchd.plist](https://keith.github.io/xcode-man-pages/launchd.plist.5.html), Apple's man page).+- **Sockets** are created by launchd, and the job collects them with `launch_activate_socket(3)`.+- **`ProcessType` Background** jobs get resource limits; Interactive jobs do not. By default launchd kills a dead job's process group.+- **`SMAppService`** (macOS 13+) registers an agent shipped inside the app bundle, "subject to user approval" ([SMAppService](https://developer.apple.com/documentation/servicemanagement/smappservice), [agent](https://developer.apple.com/documentation/servicemanagement/smappservice/agent(plistname:))).++### nginx++- **The master does the privileged setup** (config, ports) and manages workers, which "do all of the work" ([NGINX](https://blog.nginx.org/blog/inside-nginx-how-we-designed-for-performance-scale)).+- **Signals control it.** TERM is a fast stop, QUIT a graceful one, HUP reloads, USR1 reopens logs and USR2 loads a new binary ([control](https://nginx.org/en/docs/control.html)). A graceful stop has a time limit ([worker_shutdown_timeout](https://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout)).+- **Reload validates first.** If the new config fails, nginx "rolls back changes and continues to work with old configuration".+- **An upgrade keeps the old master as the rollback.** It starts the new binary but keeps its own listening sockets open, so it can bring its workers back if the new one misbehaves.++### PostgreSQL postmaster++- **The supervisor keeps away from shared state.** It "as a rule does not touch" shared memory, so a backend crash can't take it down ([postmaster.c](https://github.com/postgres/postgres/blob/master/src/backend/postmaster/postmaster.c)).+- **One crash resets everything.** The postmaster stops the other children and reinitializes ([restart_after_crash](https://www.postgresql.org/docs/current/runtime-config-error-handling.html)).+- **Children watch a pipe** to notice when the postmaster dies. There are three stop modes: smart, fast and immediate ([shutdown](https://www.postgresql.org/docs/current/server-shutdown.html)).+- **The OOM killer must not pick the supervisor.** The postmaster gets `oom_score_adj` −1000 and its children 0 ([docs](https://www.postgresql.org/docs/current/kernel-resources.html)).++### Syncthing++- **A monitor process restarts the real one.** Exit code 3 means restart and 4 means upgrade ([docs](https://docs.syncthing.net/users/syncthing.html)).+- **Loop guard.** After 4 restarts within 60 s, the monitor gives up ([monitor.go](https://github.com/syncthing/syncthing/blob/main/cmd/syncthing/monitor.go)).+- **Upgrade.** When the child exits with "upgrade", the monitor `exec`s the new binary, which starts the new child. Panics go to a panic log; logs rotate by size and count.+- **Under systemd:** `--no-restart`, `RestartForceExitStatus=3 4`, and `StartLimitBurst=4` in 60 s ([unit](https://github.com/syncthing/syncthing/blob/main/etc/linux-systemd/user/syncthing.service)).++### Docker and containerd++- **Live-restore:** "containers remain running if the daemon becomes unavailable". It only works across patch releases, and a full 64K log FIFO blocks containers ([live-restore](https://docs.docker.com/engine/daemon/live-restore/)).+- **Shims are the containers' parents.** containerd reconnects to them at boot and cleans up any it can no longer reach ([runtime v2](https://github.com/containerd/containerd/blob/main/docs/runtime-v2.md)).+- **The unit files let workloads survive:** `KillMode=process` ("kill only the docker process, not all processes in the cgroup"), a negative `OOMScoreAdjust`, and socket activation ([docker.service](https://github.com/moby/moby/blob/master/contrib/init/systemd/docker.service), [containerd.service](https://github.com/containerd/containerd/blob/main/containerd.service)).+- **Version skew is planned for:** the API version is negotiated ([Engine API](https://docs.docker.com/reference/api/engine/)), and unknown shim extensions are ignored.+- **Logs:** the `local` driver caps them at 100 MB per container ([local](https://docs.docker.com/engine/logging/drivers/local/)); `json-file` does not ([json-file](https://docs.docker.com/engine/logging/drivers/json-file/)).++### Chrome++- **The network service runs in its own process**, which is "preferred for isolation and stability". After a crash it is restarted and clients reconnect ([network service](https://chromium.googlesource.com/chromium/src/+/main/services/network/README.md)).+- **It degrades instead of looping.** About 3 GPU-process crashes switch Chrome to a more stable mode, and one crash is forgiven every 5 minutes ([fallback](https://chromium.googlesource.com/chromium/src/+/main/content/browser/gpu/fallback.md), [gpu_process_host.cc](https://chromium.googlesource.com/chromium/src/+/main/content/browser/gpu/gpu_process_host.cc)).+- **The number of processes depends on available memory** ([process model](https://chromium.googlesource.com/chromium/src/+/main/docs/process_model_and_site_isolation.md)).++### VS Code++- **Extensions run together in a separate extension host process** (one per local, web or remote host), each loaded only when needed ([docs](https://vscode-docs.readthedocs.io/en/stable/extensions/our-approach/), [extension host](https://code.visualstudio.com/api/advanced-topics/extension-host)).+- **Crash budget:** automatic restarts until 3 crashes in 5 minutes, then a Restart button ([source](https://github.com/microsoft/vscode/blob/main/src/vs/workbench/services/extensions/common/abstractExtensionService.ts)).+- **Terminals live in a pty host process** that survives window reloads ([1.54](https://code.visualstudio.com/updates/v1_54)): a 5 s heartbeat, a second timer against false alarms after sleep, and capped restarts (`MaxRestarts = 5`) ([terminal.ts](https://github.com/microsoft/vscode/blob/main/src/vs/platform/terminal/common/terminal.ts), [ptyHostService.ts](https://github.com/microsoft/vscode/blob/main/src/vs/platform/terminal/node/ptyHostService.ts)).+- **Agent sessions live in one Agent Host** and are "not tied to the lifetime of the window"; clients replay missed actions on reconnect ([agent host](https://code.visualstudio.com/docs/agents/concepts/agent-host), [blog](https://code.visualstudio.com/blogs/2026/08/26/agent-host-architecture)).++### tmux++- **One server owns all sessions.** Clients connect to it over a socket, and `-L` or `-S` runs a separate server ([tmux(1)](https://man.openbsd.org/tmux.1)).+- **The server daemonizes** (`daemon(3)`), leaving its launcher's process group ([proc.c](https://github.com/tmux/tmux/blob/master/proc.c)).+- **The protocol version is strict.** A client built for a different version gets "protocol version mismatch" ([client.c](https://github.com/tmux/tmux/blob/master/client.c)).++### PM2++- **Backoff that forgives:** delays grow to 15 s and reset after 30 s of uptime ([restart strategies](https://pm2.keymetrics.io/docs/usage/restart-strategies/)). After 16 unstable restarts the app is marked "errored" ([ecosystem](https://pm2.keymetrics.io/docs/usage/application-declaration/)). Memory is checked every 30 s.+- **The app says when it is ready.** `wait_ready` waits for `process.send('ready')`. Stopping sends SIGINT, then SIGKILL after 1.6 s ([signals](https://pm2.keymetrics.io/docs/usage/signals-clean-restart/)).+- **Zero-downtime reload** in cluster mode, where the Node cluster primary owns the port and accepts connections for the workers ([cluster](https://pm2.keymetrics.io/docs/usage/cluster-mode/), [Node](https://nodejs.org/api/cluster.html)).+- **Logs don't rotate by default** ([logs](https://pm2.keymetrics.io/docs/usage/log-management/)). `pm2 startup` leaves boot to systemd or launchd ([startup](https://pm2.keymetrics.io/docs/usage/startup/)).++### s6, runit, daemontools++- **One tiny supervisor per service**, waiting at least a second between restarts ([s6-supervise](https://skarnet.org/software/s6/s6-supervise.html), [runsv](http://smarden.org/runit/runsv.8.html), [supervise](https://cr.yp.to/daemontools/supervise.html)).+- **State is kept in files.** The supervisor writes `supervise/status`, which `svstat` reads.+- **Readiness is reported explicitly.** The daemon writes a newline to `notification-fd`, because started is not the same as ready ([notifywhenup](https://skarnet.org/software/s6/notifywhenup.html)).+- **The supervision tree owns the log pipe**, so restarting the service or its logger loses nothing ([s6-svscan](https://skarnet.org/software/s6/s6-svscan.html)). `s6-log` rotates by size and count ([s6-log](https://skarnet.org/software/s6/s6-log.html)).++### Tailscale (tailscaled)++- **Supervised by systemd:** `Type=notify`, `Restart=on-failure`, a private state directory ([unit](https://github.com/tailscale/tailscale/blob/main/cmd/tailscaled/tailscaled.service)).+- **State is written atomically:** write a temp file, sync it, rename it ([atomicfile](https://github.com/tailscale/tailscale/blob/main/atomicfile/atomicfile.go)).+- **The local API checks the connecting process's user** ([safesocket](https://github.com/tailscale/tailscale/blob/main/safesocket/safesocket.go)), and the CLI warns when its version differs from the daemon's ([cli.go](https://github.com/tailscale/tailscale/blob/main/cmd/tailscale/cli/cli.go)).+- **Saying ready too early is a bug.** `READY=1` once went out before the daemon could actually serve ([#11504](https://github.com/tailscale/tailscale/issues/11504)).++### Caddy, HAProxy, Kubernetes++- **Atomic reload.** "If the new config fails for any reason, the old config is rolled back into place without downtime" ([Caddy](https://caddyserver.com/docs/api)).+- **The master re-executes itself.** HAProxy's master re-execs on SIGUSR2, and with `-x` a new process takes over the old one's listening sockets so it doesn't miss new connections ([HAProxy](https://docs.haproxy.org/3.0/management.html)).+- **Three probes:** a failed liveness probe restarts the container, a failed readiness probe stops its traffic, and a startup probe holds both off until start-up finishes. Dependency checks belong in readiness ([probes](https://kubernetes.io/docs/concepts/configuration/liveness-readiness-startup-probes/)).+- **CrashLoopBackOff:** 10 s doubling to 5 minutes, reset after 10 crash-free minutes; 30 s to stop ([pod lifecycle](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/)).diff --git a/docs/naming-system.md b/docs/naming-system.mdindex cbf4d409c..a366da32e 100644--- a/docs/naming-system.md+++ b/docs/naming-system.md@@ -28,7 +28,9 @@ applicable design system, including the [desktop Design System](../desktop/desig| Open source project and repository | **OpenHarness** | Use for the source project and repository identity; the product interface says Harness. |Use **Harness** in prose. Wordmarks and ASCII artwork may use their established-lettering. `hn` and `harness` are commands, not additional product brands.+lettering. `hn` and `harness` are commands, and `harnessd` is the name of the+daemon's process (the way `dockerd` is Docker's); none of them is an additional+product brand. Prose says "the Harness daemon".Keep vendor names intact: **Claude Code**, **Codex**, **OpenCode**, and **pi**.The roadmap can be explained in one sentence: