The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
import { EventEmitter } from 'node:events'
import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { spawn, type ChildProcess } from 'node:child_process'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { placeNode } from '../testing/placeNode.js'
import {
LOG_TRIM_INTERVAL_MS, coreExecArgv, coreHandle, describeMasterStatus, heapLimitInArgv, masterDefaults, onProcessSignal, probeMaster, processExecve, processExit,
readStatusFile, runMaster, supervisorOptions, trimLogEvery, writeStatusFile, type MasterStatusFile,
} from './master.js'
import { ignoreLogWriteErrors } from '../lib/log.js'
import { LEAN_CORE_ENTRY, folderFingerprint } from './leanBundle.js'
import {
PROBE_ANSWER, PROBE_HANDOVER, RESUME_ENV, decodeResume, encodeResume, fingerprint, readHandoverClaimant, readMarker, takeBackHandover, takeHandover, writeMarker,
} from './reexec.js'
import { HARNESSD_PROTOCOL, LEAN_CORE_SCRIPT_ENV } from './protocol.js'
import { DEFAULT_SUPERVISOR_OPTIONS, type ResumeState } from './supervisor.js'
describe('supervisorOptions', () => {
it('reads valid overrides and keeps the defaults for anything unset or invalid', () => {
expect(supervisorOptions({})).toEqual(DEFAULT_SUPERVISOR_OPTIONS)
expect(supervisorOptions({
HARNESSD_BIND_TIMEOUT_MS: '100', HARNESSD_READY_TIMEOUT_MS: '150', HARNESSD_HEARTBEAT_TIMEOUT_MS: '1000',
HARNESSD_STOP_GRACE_MS: '300', HARNESSD_INITIAL_BACKOFF_MS: '0', HARNESSD_MAX_BACKOFF_MS: '50',
HARNESSD_BACKOFF_RESET_MS: '0', HARNESSD_HEAP_LIMIT_MIB: '512', HARNESSD_HEAP_RESTART_PERCENT: '90',
HARNESSD_RSS_LIMIT_MIB: '0', HARNESSD_UPDATE_PROBATION_MS: '10', HARNESSD_CRASH_LOOP_CRASHES: '5',
HARNESSD_CRASH_LOOP_WINDOW_MS: '1000',
})).toEqual({
bindTimeoutMs: 100, readyTimeoutMs: 150, heartbeatTimeoutMs: 1000, stopGraceMs: 300, initialBackoffMs: 0,
maxBackoffMs: 50, backoffResetMs: 0, heapLimitMiB: 512, heapRestartPercent: 90, rssLimitMiB: 0,
updateProbationMs: 10, crashLoopCrashes: 5, crashLoopWindowMs: 1000,
})
expect(supervisorOptions({
HARNESSD_BIND_TIMEOUT_MS: '0', HARNESSD_HEARTBEAT_TIMEOUT_MS: 'soon', HARNESSD_RSS_LIMIT_MIB: '-1',
HARNESSD_HEAP_RESTART_PERCENT: '101', HARNESSD_CRASH_LOOP_CRASHES: '0',
})).toEqual(DEFAULT_SUPERVISOR_OPTIONS)
})
it('allows no hang timeout under a second, where a GC pause would read as a hang', () => {
expect(supervisorOptions({ HARNESSD_HEARTBEAT_TIMEOUT_MS: '999' }).heartbeatTimeoutMs).toBe(DEFAULT_SUPERVISOR_OPTIONS.heartbeatTimeoutMs)
expect(supervisorOptions({ HARNESSD_HEARTBEAT_TIMEOUT_MS: '1000' }).heartbeatTimeoutMs).toBe(1000)
})
it('takes the heap limit from the flags the master was given, unless the environment names one', () => {
expect(supervisorOptions({}, ['--import', 'tsx', '--max-old-space-size=256']).heapLimitMiB).toBe(256)
expect(supervisorOptions({ HARNESSD_HEAP_LIMIT_MIB: '2048' }, ['--max-old-space-size=256']).heapLimitMiB).toBe(2048)
})
})
describe('the core\'s flags', () => {
it('finds the last heap limit given, in either spelling', () => {
expect(heapLimitInArgv([])).toBeNull()
expect(heapLimitInArgv(['--max-old-space-size=100', '--max_old_space_size=200', '--inspect'])).toBe(200)
expect(heapLimitInArgv(['--max-old-space-size=lots'])).toBeNull()
})
it('gives the core exactly the heap limit its budget is a share of, or V8\'s own when that is 0', () => {
expect(coreExecArgv(['--import', 'tsx', '--max-old-space-size=256'], 512))
.toEqual(['--import', 'tsx', '--max-semi-space-size=4', '--max-old-space-size=512'])
expect(coreExecArgv(['--max_old_space_size=256'], 0)).toEqual(['--max-semi-space-size=4'])
})
it('keeps a young-generation size the master was given', () => {
expect(coreExecArgv(['--max_semi_space_size=8'], 512)).toEqual(['--max_semi_space_size=8', '--max-old-space-size=512'])
})
})
describe('the status file', () => {
let dir: string
beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'harnessd-status-')) })
afterEach(() => rmSync(dir, { recursive: true, force: true }))
const status = (over: Partial<MasterStatusFile> = {}): MasterStatusFile => ({
state: 'running', corePid: 7, restarts: 0, lastExit: null, lastExitReason: null, safeMode: null,
protocol: 2, since: 1, masterVersion: null, reexecs: 0, masterPid: 42, ...over,
})
it('is written whole and read back only for the master that wrote it', () => {
const file = join(dir, 'harnessd-status.json')
writeStatusFile(file, status())
expect(readStatusFile(file, 42)).toEqual(status())
expect(readStatusFile(file, 43)).toBeNull()
expect(readStatusFile(join(dir, 'none.json'), 42)).toBeNull()
writeFileSync(file, '{"masterPid": 42}')
expect(readStatusFile(file, 42)).toBeNull()
expect(existsSync(`${file}.42.tmp`)).toBe(false)
})
it('never stops the master over a disk it cannot write', () => {
expect(() => writeStatusFile(join(dir, 'missing', 'harnessd-status.json'), status())).not.toThrow()
})
it('says what the master knows when the core cannot answer', () => {
expect(describeMasterStatus(null)).toBeNull()
expect(describeMasterStatus(status())).toBeNull()
expect(describeMasterStatus(status({ state: 'restarting', restarts: 3, lastExit: 'signal SIGKILL', lastExitReason: 'hung' })))
.toBe('◍ restarting its core · restart 3 · last core ended because it hung (signal SIGKILL)')
expect(describeMasterStatus(status({ state: 'starting' }))).toBe('● starting')
expect(describeMasterStatus(status({ state: 'listening', restarts: 1, lastExit: 'code 1', lastExitReason: 'crashed' })))
.toBe('● starting · restart 1 · last core ended because it crashed (code 1)')
expect(describeMasterStatus(status({ safeMode: 'crash-loop' })))
.toBe('◍ safe mode · the core kept crashing — waiting for a fixed build')
expect(describeMasterStatus(status({ safeMode: 'no tmux', lastExit: 'code 7', lastExitReason: 'mystery' as never })))
.toBe('◍ safe mode · the core could not start (no tmux) — waiting for a fixed build · last core ended because mystery (code 7)')
})
})
describe('the process defaults', () => {
it('exit through process.exit and listen for signals on the process', () => {
const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never)
let calls: unknown[][] = []
try { processExit(3); calls = exit.mock.calls.map((call) => [...call]) } finally { exit.mockRestore() }
expect(calls).toEqual([[3]])
const listener = vi.fn()
onProcessSignal('SIGHUP', listener)
try { process.emit('SIGHUP') } finally { process.removeListener('SIGHUP', listener) }
expect(listener).toHaveBeenCalledOnce()
})
})
describe('trimLogEvery', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => vi.useRealTimers())
it('trims the log on its interval until stopped, and does nothing without one', () => {
const trimmed: string[] = []
const stop = trimLogEvery('/logs/harness.log', 1_000, (file) => { trimmed.push(file); return false })
vi.advanceTimersByTime(2_999)
expect(trimmed).toEqual(['/logs/harness.log', '/logs/harness.log'])
stop()
vi.advanceTimersByTime(5_000)
expect(trimmed).toHaveLength(2)
expect(() => trimLogEvery(undefined)()).not.toThrow()
})
it('uses the daemon log trim by default', () => {
const dir = mkdtempSync(join(tmpdir(), 'harnessd-trim-'))
try {
const file = join(dir, 'harness.log')
writeFileSync(file, 'x'.repeat(11 * 1024 * 1024))
const stop = trimLogEvery(file, 1_000)
vi.advanceTimersByTime(1_000)
stop()
expect(readFileSync(file).length).toBeLessThan(11 * 1024 * 1024)
} finally { rmSync(dir, { recursive: true, force: true }) }
})
})
describe('masterDefaults', () => {
it('takes what the config gives, and this process for the rest', () => {
const base = { nodePath: 'n', execArgv: [], scriptPath: 's', pidFile: 'p', restoreUpdate: () => {}, confirmUpdate: () => {} }
expect(masterDefaults(base)).toEqual({ env: process.env, exit: processExit, onSignal: onProcessSignal, execve: expect.any(Function) })
const exit = () => {}
const onSignal = () => {}
const execve = () => {}
expect(masterDefaults({ ...base, env: { A: '1' }, exit, onSignal, execve })).toEqual({ env: { A: '1' }, exit, onSignal, execve })
expect(masterDefaults({ ...base, execve: null }).execve).toBeNull()
})
it('replaces the process through process.execve where this Node has it, and has no way to where it has not', () => {
expect(processExecve({})).toBeNull()
const calls: unknown[][] = []
const proc = { execve(this: unknown, ...args: unknown[]) { calls.push([this === proc, ...args]) } }
processExecve(proc)!('/node', ['/node', 'cli.js'], { A: '1' })
expect(calls).toEqual([[true, '/node', ['/node', 'cli.js'], { A: '1' }]])
// The Node these tests run on has it (22.15 and 23.11 on).
expect(processExecve()).toEqual(expect.any(Function))
})
})
describe('the probe a re-executing master asks a bundle', () => {
it('answers when the master builds, from the state it would be handed or from none', () => {
const said: string[] = []
expect(probeMaster({ env: {}, execArgv: [], version: '9.9.9' }, (line) => said.push(line))).toBe(0)
const resume: ResumeState = { restarts: 1, lastExit: 'code 75', lastExitReason: 'update', update: 'pending', claimed: true, reexecs: 1, unproven: 1 }
expect(probeMaster({ env: { [RESUME_ENV]: encodeResume(resume) }, execArgv: [], version: '9.9.9' }, (line) => said.push(line))).toBe(0)
// Saying it takes a handover (Windows, ./reexec.ts): a master from before would run beside the one handing over.
const answer = `${PROBE_ANSWER} · protocol ${HARNESSD_PROTOCOL} · v9.9.9 · ${PROBE_HANDOVER}`
expect(said).toEqual([answer, answer])
})
it('refuses a state it cannot read, and says so on stdout by default', () => {
const log = vi.spyOn(console, 'log').mockImplementation(() => {})
try {
expect(probeMaster({ env: { [RESUME_ENV]: '{"restarts":"many"}' }, execArgv: [], version: '9.9.9' })).toBe(1)
expect(log).toHaveBeenCalledWith('harnessd-probe failed: the state it would be handed is not one this master can read')
} finally { log.mockRestore() }
// A failure that is not an Error is reported all the same: here, the answer itself could not be said.
const said: string[] = []
let first = true
const say = (line: string) => { if (first) { first = false; throw 'stdout closed' } said.push(line) }
expect(probeMaster({ env: { HARNESSD_SERVICES: 'search' }, execArgv: [], version: '' }, say)).toBe(1)
expect(said).toEqual(['harnessd-probe failed: stdout closed'])
})
})
describe('coreHandle', () => {
const fake = () => {
const child = new EventEmitter() as EventEmitter & { pid: number; send: (m: unknown) => void; kill: (s: string) => void }
child.pid = 4242
child.send = vi.fn()
child.kill = vi.fn()
return child
}
it('reports one exit however the child ends, a failed spawn included', () => {
for (const end of [(c: EventEmitter) => { c.emit('exit', 3, null); c.emit('error', new Error('late')) }, (c: EventEmitter) => { c.emit('error', new Error('ENOENT')); c.emit('exit', 1, null) }]) {
const child = fake()
const handle = coreHandle(child as unknown as ChildProcess)
const exits: unknown[] = []
handle.onExit((code, signal) => exits.push([code, signal]))
end(child)
expect(exits).toHaveLength(1)
}
})
it('keeps the real update exit when an IPC send fails as the core is leaving', async () => {
// Found by QA after a quiet-machine run: CI rolled back a core exiting for an update when a
// status send lost its IPC channel. Node reports that send error on the child without a callback.
const child = fake()
child.send = (_message: unknown, callback?: (error: Error | null) => void) => {
queueMicrotask(() => {
const error = Object.assign(new Error('Channel closed'), { code: 'ERR_IPC_CHANNEL_CLOSED' })
if (callback) callback(error)
else child.emit('error', error)
})
}
const handle = coreHandle(child as unknown as ChildProcess)
const exits: unknown[] = []
handle.onExit((code, signal) => exits.push([code, signal]))
handle.send({ type: 'harnessd:status', status: {} as never })
await Promise.resolve()
expect(exits).toEqual([])
child.emit('exit', 75, null)
expect(exits).toEqual([[75, null]])
})
it('passes messages and signals through, and swallows them for a child that is gone', () => {
const child = fake()
const handle = coreHandle(child as unknown as ChildProcess)
const messages: unknown[] = []
handle.onMessage((message) => messages.push(message))
child.emit('message', { type: 'x' })
handle.send({ type: 'harnessd:status', status: { state: 'running', corePid: 1, restarts: 0, lastExit: null, lastExitReason: null, safeMode: null, protocol: 2, since: 0, masterVersion: null, reexecs: 0 } })
handle.kill('SIGTERM')
expect(handle.pid).toBe(4242)
expect(messages).toEqual([{ type: 'x' }])
expect(child.send).toHaveBeenCalledOnce()
expect(child.kill).toHaveBeenCalledWith('SIGTERM')
child.send = () => { throw new Error('closed') }
child.kill = () => { throw new Error('ESRCH') }
expect(() => { handle.send({ type: 'harnessd:status', status: {} as never }); handle.kill('SIGKILL') }).not.toThrow()
})
})
// A real master over a real child: a few lines of JavaScript that behave like a core.
describe('a log that cannot grow', () => {
// The master's stdout is the daemon's log file, on the disk everything else is on. A size limit stands
// in for a full disk: the write fails the same way (EFBIG for ENOSPC), and the process must live on.
const script = (guard: boolean) => `
${guard ? `import { ignoreLogWriteErrors } from ${JSON.stringify(new URL('../lib/log.ts', import.meta.url).pathname)}\nignoreLogWriteErrors()` : ''}
const line = 'x'.repeat(1500)
let n = 0
const timer = setInterval(() => { console.log(line); if (++n === 30) { clearInterval(timer); process.exit(0) } }, 5)
`
const run = (guard: boolean): Promise<number | null> => {
const dir = mkdtempSync(join(tmpdir(), 'harnessd-full-log-'))
const file = join(dir, 'master.mts')
writeFileSync(file, script(guard))
const cli = new URL('../..', import.meta.url).pathname
const child = spawn('/bin/sh', ['-c', `ulimit -f 4; exec "$0" --import tsx "$1" > "$2" 2>&1`, process.execPath, file, join(dir, 'harness.log')], { cwd: cli, stdio: 'ignore' })
return new Promise((resolve) => child.once('exit', (code) => { rmSync(dir, { recursive: true, force: true }); resolve(code) }))
}
it('has its failed writes dropped', () => {
const stream = new EventEmitter()
ignoreLogWriteErrors([stream])
expect(() => stream.emit('error', Object.assign(new Error('ENOSPC: no space left on device, write'), { code: 'ENOSPC' }))).not.toThrow()
})
it('is not what stops the master: a write that fails is dropped', async () => {
expect(await run(false)).toBe(1)
expect(await run(true)).toBe(0)
}, 30_000)
})
// Every test here starts real Node processes one after another (a core, services, probes) and waits on
// what they write or say. A Node starts in about 50 ms on a quiet machine and in seconds at a load of 50
// to 60, where vitest's 5 s for a whole test ran out before the conditions did.
describe('runMaster', { timeout: 60_000 }, () => {
let dir: string
// runMaster titles the process it runs in: here, the test worker.
const title = process.title
beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'harnessd-master-')) })
afterEach(() => { rmSync(dir, { recursive: true, force: true }); process.title = title })
/** Wait for [test] to hold. The deadline only names what never came, well before the test's own. */
/** What the children appended to [path] so far, one JSON line each: whole lines only. Polled while they
* run, so a line still being appended is not read half-written (and thrown on as JSON). */
const jsonLines = <T = Record<string, any>>(path: string): T[] =>
existsSync(path) ? readFileSync(path, 'utf8').split('\n').slice(0, -1).map((line) => JSON.parse(line) as T) : []
const until = async (what: string, test: () => boolean, ms = 30_000) => {
const deadline = Date.now() + ms
while (!test()) {
if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`)
await new Promise((resolve) => setTimeout(resolve, 20))
}
}
it('starts the core through a link named after it, so Activity Monitor tells it from node', async () => {
const runtime = join(dir, 'runtime')
const bin = join(runtime, 'node-v1', 'bin')
mkdirSync(bin, { recursive: true })
const node = join(bin, 'node')
placeNode(node, [join(runtime, 'node-v1', 'libexec', 'harnessd')])
const ran = join(dir, 'ran')
const core = join(dir, 'core.cjs')
writeFileSync(core, `
// The services run this script too (as \`__service <name>\`); each records its own. Written whole, then
// renamed: the test waits for the file to exist, and once read it empty, a service caught between the
// write's create and its bytes on a loaded machine.
const fs = require('node:fs'), mine = ${JSON.stringify(ran)} + (process.argv[3] ?? '')
fs.writeFileSync(mine + '.tmp', process.execPath)
fs.renameSync(mine + '.tmp', mine)
process.send({ type: 'harnessd:bound', protocol: ${HARNESSD_PROTOCOL}, port: 1 })
process.send({ type: 'harnessd:ready' })
// A device: the core asks for the devices' process, which runs only on demand.
if (process.argv[2] === '__run') process.send({ type: 'harnessd:want', service: 'wifi' })
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
process.on('SIGTERM', () => process.exit(0))
`)
const exits: number[] = []
const signals = new Map<string, () => void>()
const supervisor = runMaster({
nodePath: node, runtimeDir: runtime, execArgv: [], scriptPath: core, pidFile: join(dir, 'adapter.pid'),
restoreUpdate: () => {}, confirmUpdate: () => {},
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
await until('the core and a service to be running', () => supervisor.status().state === 'running' && existsSync(ran) && existsSync(`${ran}search`))
await until('the devices\' process the core asked for', () => existsSync(`${ran}devices,wifi,machineAddons`))
const libexec = join(realpathSync(runtime), 'node-v1', 'libexec', 'harnessd')
expect(readFileSync(ran, 'utf8')).toBe(join(libexec, 'harnessd-core'))
expect(readFileSync(`${ran}search`, 'utf8')).toBe(join(libexec, 'harnessd-search'))
expect(readFileSync(`${ran}devices,wifi,machineAddons`, 'utf8')).toBe(join(libexec, 'harnessd-devices'))
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
})
it('starts the core, claims the pid file when it binds, restarts it after a crash, and stops it on a signal', async () => {
const pidFile = join(dir, 'adapter.pid')
const runs = join(dir, 'runs')
const core = join(dir, 'core.cjs')
writeFileSync(core, `
const { appendFileSync } = require('node:fs')
appendFileSync(${JSON.stringify(runs)}, process.env.HARNESSD_RESTARTS + '\\n')
if (process.argv[2] !== '__run') process.exit(9)
process.send({ type: 'harnessd:bound', protocol: 1, port: 1 })
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1 }), 50)
process.on('SIGTERM', () => process.exit(0))
if (process.env.HARNESSD_RESTARTS === '0') setTimeout(() => process.exit(1), 100)
`)
const signals = new Map<string, () => void>()
const exits: number[] = []
const updates: string[] = []
const supervisor = runMaster({
nodePath: process.execPath,
execArgv: [],
scriptPath: core,
pidFile,
restoreUpdate: () => updates.push('restore'),
confirmUpdate: () => updates.push('confirm'),
env: { ...process.env, HARNESSD_SERVICES: 'none', HARNESSD_INITIAL_BACKOFF_MS: '10' },
exit: (code) => exits.push(code),
onSignal: (signal, listener) => signals.set(signal, listener),
})
expect([...signals.keys()]).toEqual(['SIGTERM', 'SIGINT', 'SIGHUP'])
await until('the pid file', () => existsSync(pidFile))
expect(readFileSync(pidFile, 'utf8')).toBe(`${process.pid}\n`)
await until('a restart', () => existsSync(runs) && readFileSync(runs, 'utf8') === '0\n1\n')
await until('the restarted core to bind', () => supervisor.status().state === 'running')
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
expect(exits).toEqual([0])
expect(existsSync(pidFile)).toBe(false)
expect(updates).toEqual([])
})
it('carries on, without a pid file, when the disk will not take one', async () => {
const log = vi.spyOn(console, 'log').mockImplementation(() => {})
const pidFile = join(dir, 'no-such-folder', 'adapter.pid')
const core = join(dir, 'core.cjs')
writeFileSync(core, `
process.send({ type: 'harnessd:bound', protocol: 2, port: 1 })
process.send({ type: 'harnessd:ready' })
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
process.on('SIGTERM', () => process.exit(0))
`)
const exits: number[] = []
const signals = new Map<string, () => void>()
const supervisor = runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile,
restoreUpdate: () => {}, confirmUpdate: () => {},
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
try {
await until('the core to be running', () => supervisor.status().state === 'running')
expect(log.mock.calls.map((call) => String(call[0]))).toContainEqual(expect.stringContaining('[harnessd] could not write the pid file: ENOENT'))
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
expect(exits).toEqual([0])
} finally { log.mockRestore() }
})
it('records its status as it goes, and starts the core with the heap limit its budget is a share of', async () => {
const pidFile = join(dir, 'adapter.pid')
const statusFile = join(dir, 'harnessd-status.json')
const flags = join(dir, 'flags')
const core = join(dir, 'core.cjs')
writeFileSync(core, `
require('node:fs').writeFileSync(${JSON.stringify(flags)}, JSON.stringify(process.execArgv))
process.send({ type: 'harnessd:bound', protocol: 2, port: 1 })
process.send({ type: 'harnessd:ready' })
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
process.on('SIGTERM', () => process.exit(0))
`)
const exits: number[] = []
const signals = new Map<string, () => void>()
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile, statusFile, logFile: join(dir, 'harness.log'),
restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'none', HARNESSD_HEAP_LIMIT_MIB: '300' },
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
await until('the core to be ready', () => readStatusFile(statusFile, process.pid)?.state === 'running')
expect(JSON.parse(readFileSync(flags, 'utf8'))).toEqual(['--max-semi-space-size=4', '--max-old-space-size=300'])
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
expect(readStatusFile(statusFile, process.pid)).toMatchObject({ state: 'stopped', lastExitReason: 'stopped' })
})
it('says in its status file when launchd or systemd runs it, so `harness stop` asks the platform', async () => {
const pidFile = join(dir, 'adapter.pid')
const statusFile = join(dir, 'harnessd-status.json')
const logged = join(dir, 'master.log')
const core = join(dir, 'core.cjs')
writeFileSync(core, `
process.send({ type: 'harnessd:bound', protocol: 2, port: 1 })
process.send({ type: 'harnessd:ready' })
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
process.on('SIGTERM', () => process.exit(0))
`)
const exits: number[] = []
const signals = new Map<string, () => void>()
const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { writeFileSync(logged, `${line}\n`, { flag: 'a' }) })
try {
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile, statusFile,
restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'none', HARNESSD_PLATFORM: 'launchd' },
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
await until('the core to be ready', () => readStatusFile(statusFile, process.pid)?.state === 'running')
expect(readStatusFile(statusFile, process.pid)).toMatchObject({ platform: 'launchd' })
expect(readFileSync(logged, 'utf8')).toContain('[harnessd] run by launchd, which starts this master again if it dies')
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
} finally { log.mockRestore() }
expect(exits).toEqual([0])
})
it('runs the services it is told to beside the core, with one token for both, restarts them, and stops them with it', async () => {
const pidFile = join(dir, 'adapter.pid')
const seen = join(dir, 'seen')
const script = join(dir, 'daemon.cjs')
writeFileSync(script, `
const { appendFileSync } = require('node:fs')
const role = process.argv[2] === '__service' ? 'service:' + process.argv[3] : process.argv[2]
appendFileSync(${JSON.stringify(seen)}, JSON.stringify({ role, token: process.env.HARNESSD_SERVICE_TOKEN, name: process.env.HARNESSD_SERVICE, restarts: process.env.HARNESSD_RESTARTS, flags: process.execArgv, processes: process.env.HARNESSD_SERVICE_PROCESSES, services: process.env.HARNESSD_SERVICES }) + '\\n')
if (role === '__run') { process.send({ type: 'harnessd:bound', protocol: 2, port: 1 }); process.send({ type: 'harnessd:ready' }) }
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
process.on('SIGTERM', () => process.exit(0))
if (role === 'service:search' && process.env.HARNESSD_RESTARTS === '0') setTimeout(() => process.exit(1), 100)
`)
const exits: number[] = []
const signals = new Map<string, () => void>()
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: script, pidFile,
restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'search,unknown,monitor,usage', HARNESSD_SERVICE_INITIAL_BACKOFF_MS: '10' },
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
const lines = (): Array<Record<string, any>> => jsonLines(seen)
await until('the crashed service to be restarted', () => lines().filter((line) => line.role === 'service:search').length === 2)
// The light services share one process, the edge host, which runs each of those named.
await until('the edge host to start', () => lines().some((line) => line.role === 'service:usage,monitor'))
expect(lines().find((line) => line.role === 'service:usage,monitor')).toMatchObject({ name: 'edge', flags: ['--max-semi-space-size=4', '--max-old-space-size=384'] })
const core = lines().find((line) => line.role === '__run')!
const services = lines().filter((line) => line.role === 'service:search')
expect(services.map((service) => service.restarts)).toEqual(['0', '1'])
expect(services[0]).toMatchObject({ name: 'search', flags: ['--max-semi-space-size=4', '--max-old-space-size=1024'] })
expect(services[0].token).toMatch(/^[0-9a-f]{48}$/)
expect(core.token).toBe(services[0].token)
// The core is told exactly what runs out here, in a form a core from before the list reads too.
expect(core).toMatchObject({ processes: 'search,usage,monitor', services: 'search,usage,monitor' })
expect(lines().some((line) => line.role === 'service:unknown')).toBe(false)
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
expect(exits).toEqual([0])
})
it('runs the updater when told to, whatever HARNESSD_SERVICES names, and has the core hand over for what it stages', async () => {
const pidFile = join(dir, 'adapter.pid')
const seen = join(dir, 'seen')
const script = join(dir, 'daemon.cjs')
writeFileSync(script, `
const { appendFileSync } = require('node:fs')
const role = process.argv[2] === '__service' ? 'service:' + process.argv[3] : process.argv[2]
const say = (what) => appendFileSync(${JSON.stringify(seen)}, JSON.stringify({ role, what, restarts: process.env.HARNESSD_RESTARTS }) + '\\n')
say('started')
if (role === '__run') {
process.send({ type: 'harnessd:bound', protocol: 3, port: 1 }); process.send({ type: 'harnessd:ready' })
process.on('message', (message) => { if (message.type === 'harnessd:update') { say('asked for ' + message.version); process.exit(75) } })
}
if (role === 'service:updater') say('judges pty ' + process.env.HARNESSD_JUDGES_PTY)
if (role === 'service:updater' && process.env.HARNESSD_RESTARTS === '0') setTimeout(() => process.send({ type: 'harnessd:staged', version: '9.9.9' }), 100)
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
process.on('SIGTERM', () => process.exit(0))
`)
const exits: number[] = []
const signals = new Map<string, () => void>()
const lines: string[] = []
const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { lines.push(line) })
try {
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: script, pidFile, updater: true,
restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'none', HARNESSD_UPDATE_PROBATION_MS: '50' },
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
const said = (): Array<Record<string, string>> => jsonLines(seen)
await until('the core to hand over and a new one to start', () => said().filter((line) => line.role === '__run' && line.what === 'started').length === 2)
expect(said().filter((line) => line.role.startsWith('service:') && line.what === 'started').map((line) => line.role)).toEqual(['service:updater'])
expect(said()).toContainEqual({ role: '__run', what: 'asked for 9.9.9', restarts: '0' })
// The updater hears that this master judges a terminal host it stages (Windows).
expect(said()).toContainEqual({ role: 'service:updater', what: 'judges pty 1', restarts: '0' })
expect(lines.some((line) => line.endsWith('[harnessd] the updater staged 9.9.9 — asking the core to hand over'))).toBe(true)
expect(lines.some((line) => line.endsWith('[harnessd] core exited (code 75) for an update — restarting'))).toBe(true)
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
} finally { log.mockRestore() }
})
it('hands over for a staged build only when the updater staged it, never another service', async () => {
const pidFile = join(dir, 'adapter.pid')
const seen = join(dir, 'seen')
const script = join(dir, 'daemon.cjs')
// Search says it staged a build, as only the updater may: the core must not be asked to hand over.
writeFileSync(script, `
const { appendFileSync } = require('node:fs')
const role = process.argv[2] === '__service' ? 'service:' + process.argv[3] : process.argv[2]
const say = (what) => appendFileSync(${JSON.stringify(seen)}, JSON.stringify({ role, what }) + '\\n')
say('started')
if (role === '__run') {
process.send({ type: 'harnessd:bound', protocol: 4, port: 1 }); process.send({ type: 'harnessd:ready' })
process.on('message', (message) => { if (message.type === 'harnessd:update') { say('asked for ' + message.version); process.exit(75) } })
}
if (role === 'service:search') setTimeout(() => { process.send({ type: 'harnessd:staged', version: '6.6.6' }); say('staged') }, 100)
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
process.on('SIGTERM', () => process.exit(0))
`)
const exits: number[] = []
const signals = new Map<string, () => void>()
const lines: string[] = []
const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { lines.push(line) })
try {
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: script, pidFile, updater: true,
restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'search' },
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
const said = (): Array<Record<string, string>> => jsonLines(seen)
await until('search to say it staged a build', () => said().some((line) => line.role === 'service:search' && line.what === 'staged'))
// Long enough for a handover the master had asked for to reach the core.
await new Promise((resolve) => setTimeout(resolve, 500))
expect(said().filter((line) => line.role === '__run')).toEqual([{ role: '__run', what: 'started' }])
expect(lines.some((line) => line.includes('asking the core to hand over'))).toBe(false)
expect(lines.some((line) => line.endsWith('[harnessd] service search said it staged 6.6.6, which only the updater may — ignored'))).toBe(true)
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
} finally { log.mockRestore() }
})
it('starts the services and the core from the lean bundle when given one, telling the core which CLI it runs for', async () => {
// The lean bundle cli.js carries (./leanBundle.ts): each service, and the core, parses its own code, not
// the CLI's. A core started from it is told the cli.js it came from, its CLI to everything it hands on.
const pidFile = join(dir, 'adapter.pid')
const seen = join(dir, 'seen')
const body = (who: string) => `
const { appendFileSync } = require('node:fs')
const role = process.argv[2] === '__service' ? 'service:' + process.argv[3] : process.argv[2]
appendFileSync(${JSON.stringify(seen)}, JSON.stringify({ role, from: ${JSON.stringify(who)}, cli: process.env.${LEAN_CORE_SCRIPT_ENV} ?? null }) + '\\n')
if (role === '__run') { process.send({ type: 'harnessd:bound', protocol: 2, port: 1 }); process.send({ type: 'harnessd:ready' }) }
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
process.on('SIGTERM', () => process.exit(0))
`
const cli = join(dir, 'cli.cjs')
const lean = join(dir, 'lean.cjs')
writeFileSync(cli, body('cli'))
writeFileSync(lean, body('lean'))
// The core's own entry, beside the services'.
writeFileSync(join(dir, LEAN_CORE_ENTRY), body('lean core').replace("require('node:fs')", "await import('node:fs')"))
const lines: string[] = []
const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { lines.push(line) })
const exits: number[] = []
const signals = new Map<string, () => void>()
try {
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: cli, serviceScriptPath: lean, pidFile,
restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'search' },
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
const roles = (): Array<Record<string, string>> => jsonLines(seen)
await until('the core and the service', () => roles().length >= 2)
expect(roles().sort((a, b) => a.role.localeCompare(b.role))).toEqual([{ role: '__run', from: 'lean core', cli }, { role: 'service:search', from: 'lean', cli: null }])
expect(lines.some((line) => line.endsWith(`[harnessd] services run from ${lean}`))).toBe(true)
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
} finally { log.mockRestore() }
})
it('starts the core from cli.js once it has died twice from the lean bundle before it beat, and judges it the same', async () => {
// A lean core that cannot start costs two quick restarts, never the daemon (./leanServices.ts).
const pidFile = join(dir, 'adapter.pid')
const seen = join(dir, 'seen')
const cli = join(dir, 'cli.cjs')
const lean = join(dir, 'lean.cjs')
writeFileSync(cli, `
require('node:fs').appendFileSync(${JSON.stringify(seen)}, 'cli\\n')
process.send({ type: 'harnessd:bound', protocol: 2, port: 1 }); process.send({ type: 'harnessd:ready' })
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
process.on('SIGTERM', () => process.exit(0))
`)
// The services' entry is fine; the core's, beside it, dies as it starts.
writeFileSync(lean, '')
writeFileSync(join(dir, LEAN_CORE_ENTRY), `(await import('node:fs')).appendFileSync(${JSON.stringify(seen)}, 'lean\\n'); process.exit(1)`)
const lines: string[] = []
const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { lines.push(line) })
const exits: number[] = []
const signals = new Map<string, () => void>()
try {
const supervisor = runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: cli, serviceScriptPath: lean, pidFile,
restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'none', HARNESSD_INITIAL_BACKOFF_MS: '0' },
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
await until('a core from cli.js, running', () => supervisor.status().state === 'running')
expect(readFileSync(seen, 'utf8').trim().split('\n')).toEqual(['lean', 'lean', 'cli'])
expect(lines.some((line) => line.endsWith(`[harnessd] the core died 2 times from the lean bundle before it beat: it starts from ${cli} from now on`))).toBe(true)
// Two crashes, as any two: not yet the three in a row that start it in safe mode.
expect(supervisor.status().safeMode).toBeNull()
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
} finally { log.mockRestore() }
})
it('starts a service from cli.js once its lean bundle is gone, or cli.js is no longer the bundle it came from', async () => {
// A master lives for weeks: a lean folder gone from under it failed every restart of every service
// with MODULE_NOT_FOUND, and after an update it did not re-execute on, the services ran the old lean
// code against the new core. The lean bundle is only an optimisation (./leanServices.ts).
const pidFile = join(dir, 'adapter.pid')
const seen = join(dir, 'seen')
// The test ends each service process itself, once the world has changed, and the master starts it
// again at once, from wherever it may. Ended by SIGTERM, as the master ends one, which is never the
// lean bundle's fault: a service that ended itself on a timer could end before its master had read
// its first beat, on a loaded machine, and two of those in a row sent it to cli.js for good.
const body = (who: string) => `
const { appendFileSync } = require('node:fs')
const role = process.argv[2] === '__service' ? 'service:' + process.argv[3] : process.argv[2]
appendFileSync(${JSON.stringify(seen)}, JSON.stringify({ role, from: ${JSON.stringify(who)}, pid: process.pid }) + '\\n')
if (role === '__run') { process.send({ type: 'harnessd:bound', protocol: 2, port: 1 }); process.send({ type: 'harnessd:ready' }) }
process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 })
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
if (role === '__run') process.on('SIGTERM', () => process.exit(0))
`
const cli = join(dir, 'cli.cjs')
const leanDir = join(dir, 'lean')
const lean = join(leanDir, 'harnessd.mjs')
writeFileSync(cli, body('cli'))
mkdirSync(leanDir)
writeFileSync(lean, body('lean').replace("require('node:fs')", "await import('node:fs')"))
const lines: string[] = []
const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { lines.push(line) })
const exits: number[] = []
const signals = new Map<string, () => void>()
const services = (): Array<{ from: string; pid: number }> => jsonLines<{ role: string; from: string; pid: number }>(seen)
.filter((line) => line.role === 'service:search')
/** End the service running now, and wait for the master to start the next: where it started it from. */
const restart = async (): Promise<string> => {
const before = services()
process.kill(before.at(-1)!.pid, 'SIGTERM')
await until('search started again', () => services().length > before.length)
return services().at(-1)!.from
}
try {
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: cli, serviceScriptPath: lean, leanFingerprint: folderFingerprint(leanDir)!, pidFile,
restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'search', HARNESSD_SERVICE_INITIAL_BACKOFF_MS: '0', HARNESSD_SERVICE_MAX_BACKOFF_MS: '0', HARNESSD_SERVICE_PARK_CRASHES: '1000' },
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
await until('a service', () => services().length > 0)
expect(services()[0]!.from).toBe('lean')
rmSync(leanDir, { recursive: true })
expect(await restart()).toBe('cli')
expect(lines.some((line) => line.endsWith(`[harnessd] the lean bundle ${lean} cannot be used (it is gone): the core and the services start from ${cli}`))).toBe(true)
// Back as it was, and then cli.js replaced by an update this master runs no code of.
mkdirSync(leanDir)
writeFileSync(lean, body('lean').replace("require('node:fs')", "await import('node:fs')"))
expect(await restart()).toBe('lean')
writeFileSync(cli, `${body('cli')}\n// the next release\n`)
expect(await restart()).toBe('cli')
expect(lines.some((line) => line.endsWith(`[harnessd] the lean bundle ${lean} cannot be used (${cli} is no longer the bundle it came from): the core and the services start from ${cli}`))).toBe(true)
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
} finally { log.mockRestore() }
})
it.each([
['is gone', (pidFile: string) => rmSync(pidFile, { force: true })],
['holds no number', (pidFile: string) => writeFileSync(pidFile, 'garbage\n')],
])('finishes cleanly when its pid file %s by the time it leaves', async (_, disturb) => {
const pidFile = join(dir, 'adapter.pid')
const core = join(dir, 'core.cjs')
writeFileSync(core, `process.send({ type: 'harnessd:bound', protocol: 1, port: 1 }); setTimeout(() => process.exit(0), 100)`)
const exits: number[] = []
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile,
restoreUpdate: () => {}, confirmUpdate: () => {},
exit: (code) => exits.push(code), onSignal: () => {},
})
await until('the pid file', () => existsSync(pidFile))
disturb(pidFile)
await until('the master to finish', () => exits.length > 0)
expect(exits).toEqual([0])
})
it('leaves a pid file that is not its own alone', async () => {
const pidFile = join(dir, 'adapter.pid')
const servicesFile = join(dir, 'services')
const core = join(dir, 'core.cjs')
writeFileSync(core, `
if (process.argv[2] === '__run') require('node:fs').writeFileSync(${JSON.stringify(servicesFile)}, process.env.HARNESSD_SERVICE_PROCESSES)
process.send({ type: 'harnessd:bound', protocol: 1, port: 1 })
setTimeout(() => { require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, '1\\n'); process.exit(0) }, 50)
`)
const exits: number[] = []
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile,
// Found by QA on a quiet machine: each default service ran this same writer. A service stopped
// between truncate and write left an empty pid file, so this cleanup check raced its fixture.
env: { ...process.env, HARNESSD_SERVICES: 'none' },
restoreUpdate: () => {}, confirmUpdate: () => {},
exit: (code) => exits.push(code), onSignal: () => {},
})
await until('the master to finish', () => exits.length > 0)
expect(readFileSync(pidFile, 'utf8')).toBe('1\n')
expect(readFileSync(servicesFile, 'utf8')).toBe('')
})
it('starts the core from the build it is told is current, read before every start, and from its own without one (Windows: cli\\current)', async () => {
const ran = join(dir, 'ran')
const write = (name: string) => {
const file = join(dir, name)
writeFileSync(file, `
require('node:fs').appendFileSync(${JSON.stringify(ran)}, ${JSON.stringify(name)} + '\\n')
process.send({ type: 'harnessd:bound', protocol: ${HARNESSD_PROTOCOL}, port: 1 })
process.send({ type: 'harnessd:ready' })
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1 }), 50)
process.on('SIGTERM', () => process.exit(0))
if (process.env.HARNESSD_RESTARTS === '0') setTimeout(() => process.exit(1), 100)
`)
return file
}
const own = write('own.cjs')
let current: string | null = null
const signals = new Map<string, () => void>()
const exits: number[] = []
const supervisor = runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: own, pidFile: join(dir, 'adapter.pid'), restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'none', HARNESSD_INITIAL_BACKOFF_MS: '10' },
currentScript: () => current,
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
await until('the first core', () => existsSync(ran))
current = write('second.cjs')
await until('the core started again', () => readFileSync(ran, 'utf8').split('\n').length >= 3)
expect(readFileSync(ran, 'utf8')).toBe('own.cjs\nsecond.cjs\n')
await until('running', () => supervisor.status().state === 'running')
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
})
it('starts the services and the core from the current build when it has one, never from the lean bundle', async () => {
// Windows: cli\current names the build. A service process runs from it too (spawnService's currentScript branch),
// and is not recorded as a lean start; the core is told no lean CLI.
const pidFile = join(dir, 'adapter.pid')
const seen = join(dir, 'seen')
const body = (who: string) => `
const { appendFileSync } = require('node:fs')
const role = process.argv[2] === '__service' ? 'service:' + process.argv[3] : process.argv[2]
appendFileSync(${JSON.stringify(seen)}, JSON.stringify({ role, from: ${JSON.stringify(who)}, cli: process.env.${LEAN_CORE_SCRIPT_ENV} ?? null }) + '\\n')
if (role === '__run') { process.send({ type: 'harnessd:bound', protocol: ${HARNESSD_PROTOCOL}, port: 1 }); process.send({ type: 'harnessd:ready' }) }
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
process.on('SIGTERM', () => process.exit(0))
`
const own = join(dir, 'cli.cjs')
const lean = join(dir, 'lean.cjs')
const current = join(dir, 'current.cjs')
writeFileSync(own, body('own'))
writeFileSync(lean, body('lean'))
writeFileSync(join(dir, LEAN_CORE_ENTRY), body('lean core').replace("require('node:fs')", "await import('node:fs')"))
writeFileSync(current, body('current'))
const exits: number[] = []
const signals = new Map<string, () => void>()
const log = vi.spyOn(console, 'log').mockImplementation(() => {})
try {
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: own, serviceScriptPath: lean, pidFile,
restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'search' }, currentScript: () => current,
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
const roles = (): Array<Record<string, string>> => jsonLines(seen)
await until('the core and the service', () => roles().length >= 2)
expect(roles().sort((a, b) => a.role.localeCompare(b.role))).toEqual([
{ role: '__run', from: 'current', cli: null }, { role: 'service:search', from: 'current', cli: null },
])
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
} finally { log.mockRestore() }
})
it('on Windows, stops its core and its services by message, and stops everything when the core asks, writing its children\'s output where told', async () => {
const heard = join(dir, 'heard')
const serviceUp = join(dir, 'service-up')
const output = join(dir, 'output.log')
const core = join(dir, 'core.cjs')
writeFileSync(core, `
const fs = require('node:fs')
const who = process.argv[2] === '__service' ? 'service ' : ''
process.on('SIGTERM', () => { fs.appendFileSync(${JSON.stringify(heard)}, who + 'SIGTERM\\n'); process.exit(0) })
process.on('message', (m) => { if (m.type === 'harnessd:shutdown') { fs.appendFileSync(${JSON.stringify(heard)}, who + 'shutdown\\n'); process.exit(0) } })
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
if (who) fs.writeFileSync(${JSON.stringify(serviceUp)}, '')
else {
console.log('the core speaks')
process.send({ type: 'harnessd:bound', protocol: ${HARNESSD_PROTOCOL}, port: 1 })
process.send({ type: 'harnessd:ready' })
// harness stop, once the service listens for the master's word.
const asking = setInterval(() => { if (fs.existsSync(${JSON.stringify(serviceUp)})) { clearInterval(asking); process.send({ type: 'harnessd:stop' }) } }, 20)
}
`)
const fd = openSync(output, 'a')
const exits: number[] = []
const log = vi.spyOn(console, 'log').mockImplementation(() => {})
try {
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile: join(dir, 'adapter.pid'), restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'search' }, stopByMessage: true, output: fd,
exit: (code) => exits.push(code), onSignal: () => {},
})
await until('the master to finish', () => exits.length > 0)
} finally { log.mockRestore(); closeSync(fd) }
expect(exits).toEqual([0])
expect(readFileSync(heard, 'utf8').split('\n').filter(Boolean).sort()).toEqual(['service shutdown', 'shutdown'])
expect(readFileSync(output, 'utf8')).toContain('the core speaks')
})
describe('handing over, where there is no exec (Windows)', () => {
const marker = () => join(dir, 'harnessd-reexec.json')
/** A core that binds, is ready and beats, or one that never binds. */
const coreScript = (binds = true) => {
const file = join(dir, 'core.cjs')
writeFileSync(file, `
${binds ? `process.send({ type: 'harnessd:bound', protocol: ${HARNESSD_PROTOCOL}, port: 1 })
process.send({ type: 'harnessd:ready' })` : ''}
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1 }), 50)
process.on('SIGTERM', () => process.exit(0))
`)
return file
}
const handedTo = (token: string, update: 'pending' | null = null, from = 999999): ResumeState => ({
restarts: 0, lastExit: null, lastExitReason: null, update, claimed: true, reexecs: 1, unproven: 1, handover: { from, token },
})
/** The log trimmer's interval (`trimLogEvery`): started by a master that serves, never by one that leaves. */
const trimmers = (spy: { mock: { calls: unknown[][] } }) => spy.mock.calls.filter(([, ms]) => ms === LOG_TRIM_INTERVAL_MS).length
it('takes over from the master that handed over: the handover and the pid file this process\'s from its start', async () => {
const pidFile = join(dir, 'adapter.pid')
const core = coreScript()
writeFileSync(pidFile, '999999\n')
// The marker names the master that handed over, gone by now: no failed re-execution to roll back for all that.
const handed = { pid: 999999, from: 'a', to: fingerprint(core)!, at: Date.now(), handover: 'tok' }
writeMarker(marker(), handed)
// Decisions of handovers over and done, swept; this one's kept.
takeBackHandover(marker(), 'old', 999999)
const signals = new Map<string, () => void>()
const exits: number[] = []
const updates: string[] = []
const lines: string[] = []
const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { lines.push(line) })
const intervals = vi.spyOn(globalThis, 'setInterval')
try {
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile, reexecMarkerFile: marker(), logFile: join(dir, 'daemon.log'),
restoreUpdate: () => updates.push('restore'), confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'none', [RESUME_ENV]: encodeResume(handedTo('tok')) },
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
expect(readHandoverClaimant(marker(), 'tok')).toBe(process.pid)
expect(readFileSync(pidFile, 'utf8')).toBe(`${process.pid}\n`)
// Left as it was: the master that handed over reads it until it leaves.
expect(readMarker(marker())).toEqual(handed)
expect(readdirSync(dir).filter((name) => name.endsWith('.handover'))).toEqual(['harnessd-reexec.json.tok.handover'])
expect(lines.some((line) => line.endsWith('[harnessd] took over from the master before it (pid 999999)'))).toBe(true)
expect(lines.some((line) => line.includes('master re-executed'))).toBe(false)
expect(trimmers(intervals)).toBe(1)
await until('its core to come up, which ends the marker', () => !existsSync(marker()))
expect(updates).toEqual([])
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
expect(existsSync(pidFile)).toBe(false)
} finally { log.mockRestore(); intervals.mockRestore() }
})
it('takes the marker of the handover it took with it when stopped before its first core is up', async () => {
// The marker names the master that handed over: matched by the handover's token, not by the pid.
const pidFile = join(dir, 'adapter.pid')
const core = coreScript(false)
writeMarker(marker(), { pid: process.ppid, from: 'a', to: fingerprint(core)!, at: Date.now(), handover: 'tok' })
const signals = new Map<string, () => void>()
const exits: number[] = []
const supervisor = runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile, reexecMarkerFile: marker(), restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'none', [RESUME_ENV]: encodeResume(handedTo('tok')) },
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
await until('its core', () => supervisor.status().corePid !== null)
signals.get('SIGTERM')!()
await until('the master to finish', () => exits.length > 0)
expect(readMarker(marker())).toBeNull()
expect(existsSync(pidFile)).toBe(false)
})
it('leaves at once, starting nothing and touching neither the pid file nor the marker, when it may not take over', async () => {
const pidFile = join(dir, 'adapter.pid')
const ran = join(dir, 'ran')
const core = join(dir, 'core.cjs')
writeFileSync(core, `require('node:fs').writeFileSync(${JSON.stringify(ran)}, '')`)
const unwritable = join(dir, 'missing', 'adapter.pid')
const now = Date.now()
const cases: Array<{ why: string; token: string; marker: Record<string, unknown> | null; pidFile?: string; before?: () => void }> = [
{ why: 'no marker', token: 'a', marker: null },
{ why: 'another handover\'s marker', token: 'b', marker: { handover: 'other' } },
{ why: 'past its cutoff', token: 'c', marker: { handover: 'c', at: 1 } },
{ why: 'taken back', token: 'd', marker: { handover: 'd' }, before: () => takeBackHandover(marker(), 'd', 999999) },
{ why: 'a pid file that will not write', token: 'e', marker: { handover: 'e' }, pidFile: unwritable },
]
const lines: string[] = []
const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { lines.push(line) })
// Two trimmers would rewrite the one log at once: the master that handed over keeps its own.
const intervals = vi.spyOn(globalThis, 'setInterval')
try {
for (const { why, token, marker: over, pidFile: file = pidFile, before } of cases) {
rmSync(marker(), { force: true })
const written = over ? { pid: 999999, from: 'a', to: fingerprint(core)!, at: now, ...over } : null
if (written) writeMarker(marker(), written as never)
before?.()
const exits: number[] = []
const updates: string[] = []
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile: file, reexecMarkerFile: marker(), logFile: join(dir, 'daemon.log'),
restoreUpdate: () => updates.push('restore'), confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'none', [RESUME_ENV]: encodeResume(handedTo(token, 'pending')) },
exit: (code) => exits.push(code), onSignal: () => {},
})
expect(exits, why).toEqual([0])
expect(updates, why).toEqual([])
expect(readMarker(marker()), why).toEqual(written)
expect(existsSync(file), why).toBe(false)
}
expect(lines.filter((line) => line.endsWith('[harnessd] the master that started this one (pid 999999) stopped waiting for it — leaving'))).toHaveLength(4)
expect(lines.some((line) => line.includes('[harnessd] could not take over from pid 999999 (ENOENT'))).toBe(true)
// Handed over to, with no marker file to take over at all.
const alone: number[] = []
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile, restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'none', [RESUME_ENV]: encodeResume(handedTo('f')) },
exit: (code) => alone.push(code), onSignal: () => {},
})
expect(alone).toEqual([0])
// Refused right after it wrote the pid file (here the pid file is the marker, which that write ends): it serves
// nothing. The master that handed over finds the pid file naming a process that ended, and stays.
writeMarker(marker(), { pid: 999999, from: 'a', to: fingerprint(core)!, at: now, handover: 'g' })
const late: number[] = []
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile: marker(), reexecMarkerFile: marker(), restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'none', [RESUME_ENV]: encodeResume(handedTo('g')) },
exit: (code) => late.push(code), onSignal: () => {},
})
expect(late).toEqual([0])
expect(lines.some((line) => line.endsWith('[harnessd] the master that started this one (pid 999999) stopped waiting for it as this one took over — leaving'))).toBe(true)
// That master gone: the pid file names this process, which leaves, and so a process that ended.
expect(readFileSync(marker(), 'utf8')).toBe(`${process.pid}\n`)
// That master still running, having settled on the pid file before this one overwrote it: the file is its again,
// or `harness start --if-down` would take the machine for one with no master and start a second.
writeMarker(marker(), { pid: process.ppid, from: 'a', to: fingerprint(core)!, at: now, handover: 'h' })
const back: number[] = []
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: core, pidFile: marker(), reexecMarkerFile: marker(), restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'none', [RESUME_ENV]: encodeResume(handedTo('h', null, process.ppid)) },
exit: (code) => back.push(code), onSignal: () => {},
})
expect(back).toEqual([0])
expect(readFileSync(marker(), 'utf8')).toBe(`${process.ppid}\n`)
expect(trimmers(intervals)).toBe(0)
} finally { log.mockRestore(); intervals.mockRestore() }
await new Promise((resolve) => setTimeout(resolve, 300))
expect(existsSync(ran)).toBe(false)
expect(existsSync(pidFile)).toBe(false)
})
/** A master on [own] whose current build is another, which hands over through a launcher [launch] plays. */
const handingOver = (launch: (env: NodeJS.ProcessEnv, sleeper: number, stop: () => void) => void) => {
const pidFile = join(dir, 'adapter.pid')
const own = join(dir, 'own.cjs')
const next = join(dir, 'next.cjs')
writeFileSync(own, '// this master\'s bundle')
writeFileSync(next, `if (process.argv[2] === '__harnessd-probe') { console.log(${JSON.stringify(`${PROBE_ANSWER} · ${PROBE_HANDOVER}`)}); process.exit(0) }`)
const sleeper = spawn(process.execPath, ['-e', 'setTimeout(() => {}, 60000)'], { stdio: 'ignore' })
const launched: NodeJS.ProcessEnv[] = []
const exits: number[] = []
const signals = new Map<string, () => void>()
const env = { ...process.env, HARNESSD_SERVICES: 'none' }
runMaster({
nodePath: process.execPath, execArgv: [], scriptPath: own, pidFile, reexecMarkerFile: marker(), restoreUpdate: () => {}, confirmUpdate: () => {},
env, currentScript: () => next, execve: null,
handOver: {
lock: async () => () => {},
launch: async (handed) => { launched.push(handed); launch(handed, sleeper.pid!, () => signals.get('SIGTERM')!()); return null },
readPid: () => Number(readFileSync(pidFile, 'utf8')) || null,
alive: (pid) => pid === sleeper.pid,
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
token: () => 'tok',
},
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
})
return { pidFile, next, sleeper, launched, exits, env }
}
it('hands over through the launcher when the bundle on disk is another, and leaves once the new master holds the pid file', async () => {
const master = handingOver((_, sleeper) => {
// The new master: takes the handover, then the pid file.
expect(takeHandover(marker(), 'tok', sleeper)).toBe(true)
writeFileSync(master.pidFile, `${sleeper}\n`)
})
try {
await until('the old master to leave', () => master.exits.length > 0)
expect(master.exits).toEqual([0])
expect(decodeResume(master.launched[0]![RESUME_ENV])?.handover).toEqual({ from: process.pid, token: 'tok' })
// ./AGENTS.md rule 11: this master's own environment, and the state handed on.
expect(Object.keys(master.launched[0]!).sort()).toEqual([...Object.keys(master.env), RESUME_ENV].sort())
expect(readMarker(marker())).toMatchObject({ pid: process.pid, to: fingerprint(master.next), handover: 'tok' })
expect(readFileSync(master.pidFile, 'utf8')).toBe(`${master.sleeper.pid}\n`)
} finally { master.sleeper.kill() }
})
it('stopped as it hands over, leaves the pid file and the marker to a new master that took over, and takes them back otherwise', async () => {
// Taken by the new master, which writes the pid file next: neither is this master's to remove.
const taken = handingOver((_, sleeper, stop) => { takeHandover(marker(), 'tok', sleeper); stop() })
try {
await until('the master to finish', () => taken.exits.length > 0)
expect(taken.exits).toEqual([0])
expect(readFileSync(taken.pidFile, 'utf8')).toBe(`${process.pid}\n`)
expect(readMarker(marker())).toMatchObject({ handover: 'tok' })
} finally { taken.sleeper.kill() }
rmSync(`${marker()}.tok.handover`)
// Not taken: taken back, and both go with this master.
const back = handingOver((_, __, stop) => stop())
try {
await until('the master to finish', () => back.exits.length > 0)
expect(back.exits).toEqual([0])
expect(existsSync(back.pidFile)).toBe(false)
expect(readMarker(marker())).toBeNull()
} finally { back.sleeper.kill() }
})
})
// The defaults act on the process itself, so they are exercised in a real one: a master on its own,
// whose core cannot even be spawned, that a SIGTERM must still stop cleanly.
it('uses this process for exit and signals by default', async () => {
const script = join(dir, 'master.mts')
writeFileSync(script, `
import { runMaster } from ${JSON.stringify(join(__dirname, 'master.ts'))}
runMaster({
nodePath: '/nonexistent/node', execArgv: [], scriptPath: '/nonexistent/core.js',
pidFile: ${JSON.stringify(join(dir, 'adapter.pid'))}, restoreUpdate: () => {}, confirmUpdate: () => {},
env: { ...process.env, HARNESSD_SERVICES: 'none', HARNESSD_INITIAL_BACKOFF_MS: '20' },
})
console.log('READY')
`)
const child = spawn(process.execPath, ['--import', 'tsx', script], { cwd: join(__dirname, '../..'), stdio: ['ignore', 'pipe', 'pipe'] })
let output = ''
child.stdout.on('data', (chunk) => { output += chunk })
child.stderr.on('data', (chunk) => { output += chunk })
const exited = new Promise<number | null>((resolve) => child.once('exit', (code) => resolve(code)))
await until('the master to start', () => output.includes('READY') && output.includes('restarting'), 20_000)
child.kill('SIGTERM')
expect(await exited).toBe(0)
expect(output).toContain('[harnessd] SIGTERM — stopping')
})
describe('re-executing on the bundle on disk', () => {
const DEAD_PID = 2_000_000_000
/** One script plays the bundle: as a core it binds, is ready and beats; as a probe it answers, or not. */
const bundle = (core: string, probe = `console.log(${JSON.stringify(PROBE_ANSWER)} + ' · test'); process.exit(0)`) => {
const file = join(dir, 'bundle.cjs')
writeFileSync(file, `
const fs = require('node:fs')
if (process.argv[2] === '__harnessd-probe') { ${probe} }
else {
process.send({ type: 'harnessd:bound', protocol: 2, port: 1 })
process.send({ type: 'harnessd:ready' })
setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
process.on('SIGTERM', () => process.exit(0))
${core}
}
`)
return file
}
const start = (scriptPath: string, over: Record<string, unknown> = {}) => {
const execs: Array<{ file: string; args: string[]; env: NodeJS.ProcessEnv }> = []
const exits: number[] = []
const calls: string[] = []
const signals = new Map<string, () => void>()
const supervisor = runMaster({
nodePath: process.execPath, execArgv: [], scriptPath, pidFile: join(dir, 'adapter.pid'), statusFile: join(dir, 'harnessd-status.json'),
restoreUpdate: () => calls.push('restore'), confirmUpdate: () => calls.push('confirm'),
env: { ...process.env, HARNESSD_SERVICES: 'none', HARNESSD_INITIAL_BACKOFF_MS: '10', HARNESSD_UPDATE_PROBATION_MS: '100' },
exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
version: '1.0.0', reexecMarkerFile: join(dir, 'harnessd-reexec.json'),
execve: (file: string, args: string[], env: NodeJS.ProcessEnv) => { execs.push({ file, args, env }) },
...over,
})
const stop = async () => { signals.get('SIGTERM')!(); await until('the master to finish', () => exits.length > 0) }
return { supervisor, execs, exits, calls, stop }
}
const marker = () => join(dir, 'harnessd-reexec.json')
it('replaces itself with the bundle its core staged before it exited for the update, leaving its marker and handing on its state', async () => {
// A newer bundle written over this one, then the update exit: what the core's updater does.
const script = bundle(`if (process.env.HARNESSD_RESTARTS === '0') setTimeout(() => { fs.appendFileSync(__filename, '\\n// a newer build\\n'); process.exit(75) }, 100)`)
const own = fingerprint(script)
const master = start(script)
await until('the master to re-execute', () => master.execs.length > 0)
expect(master.execs[0].file).toBe(process.execPath)
expect(master.execs[0].args).toEqual([process.execPath, script, '__harnessd'])
expect(decodeResume(master.execs[0].env[RESUME_ENV])).toEqual({ restarts: 1, lastExit: 'code 75', lastExitReason: 'update', update: 'pending', claimed: true, reexecs: 1, unproven: 1 })
expect(readMarker(marker())).toEqual({ pid: process.pid, from: own, to: fingerprint(script), at: expect.any(Number) })
expect(master.supervisor.status()).toMatchObject({ state: 'restarting', corePid: null, masterVersion: '1.0.0' })
await master.stop()
expect(master.exits).toEqual([0])
})
it('rolls an update back when the new bundle\'s master does not answer its probe, and keeps its own code', async () => {
const script = bundle(`if (process.env.HARNESSD_RESTARTS === '0') setTimeout(() => { fs.appendFileSync(__filename, '\\n// a newer build\\n'); process.exit(75) }, 100)`,
`console.error('Unknown command: ' + process.argv[2]); process.exit(1)`)
const master = start(script)
await until('the rollback', () => master.calls.includes('restore'))
await until('a core on the bundle restored', () => master.supervisor.status().state === 'running')
expect(master.execs).toEqual([])
expect(readMarker(marker())).toBeNull()
await master.stop()
})
it('carries on as itself, without its marker, when the exec fails', async () => {
const script = bundle(`if (process.env.HARNESSD_RESTARTS === '0') setTimeout(() => { fs.appendFileSync(__filename, '\\n// a newer build\\n'); process.exit(75) }, 100)`)
const master = start(script, { execve: () => { throw new Error('E2BIG') } })
await until('a core on the new bundle, under this master', () => master.supervisor.status().state === 'running' && master.supervisor.status().restarts === 1)
expect(readMarker(marker())).toBeNull()
await master.stop()
})
it('abandons a probe still running when it is stopped, and replaces nothing', async () => {
const script = bundle(`if (process.env.HARNESSD_RESTARTS === '0') setTimeout(() => { fs.appendFileSync(__filename, '\\n// a newer build\\n'); process.exit(75) }, 100)`,
`fs.writeFileSync(${JSON.stringify(join(dir, 'probing.tmp'))}, String(process.pid)); fs.renameSync(${JSON.stringify(join(dir, 'probing.tmp'))}, ${JSON.stringify(join(dir, 'probing'))}); setInterval(() => {}, 1000)`)
const master = start(script)
// Written whole, then renamed into place: the file seen created but not yet written read as pid 0,
// and `kill(0, 0)` asks after this process's own group, which is always there. The probe was never
// seen gone (a full unit run under 12 busy loops, load 87).
await until('the probe to be running', () => existsSync(join(dir, 'probing')))
const probe = Number(readFileSync(join(dir, 'probing'), 'utf8'))
expect(probe).toBeGreaterThan(0)
await master.stop()
await until('the probe to be gone', () => { try { process.kill(probe, 0); return false } catch { return true } })
expect(master.execs).toEqual([])
})
it('carries on as the master it re-executed from, without handing that state to its children', async () => {
const seen = join(dir, 'seen')
const script = bundle(`fs.appendFileSync(${JSON.stringify(seen)}, JSON.stringify({ restarts: process.env.HARNESSD_RESTARTS, resume: process.env[${JSON.stringify(RESUME_ENV)}] ?? null }) + '\\n')`)
const resume: ResumeState = { restarts: 3, lastExit: 'code 75', lastExitReason: 'update', update: 'pending', claimed: true, reexecs: 1, unproven: 1 }
const lines: string[] = []
const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { lines.push(line) })
try {
const master = start(script, { env: { ...process.env, HARNESSD_SERVICES: 'none', HARNESSD_UPDATE_PROBATION_MS: '100', [RESUME_ENV]: encodeResume(resume) } })
await until('the update to be kept', () => master.calls.includes('confirm'))
expect(JSON.parse(readFileSync(seen, 'utf8').trim().split('\n')[0])).toEqual({ restarts: '3', resume: null })
expect(readStatusFile(join(dir, 'harnessd-status.json'), process.pid)).toMatchObject({ masterVersion: '1.0.0', reexecs: 1, restarts: 3 })
expect(lines.some((line) => line.endsWith(`[harnessd] master re-executed (pid ${process.pid}) · now v1.0.0`))).toBe(true)
await master.stop()
// Without a version it says so without one.
const plain = start(script, { version: undefined, env: { ...process.env, HARNESSD_SERVICES: 'none', [RESUME_ENV]: encodeResume(resume) } })
await until('its core', () => plain.supervisor.status().state === 'running')
expect(lines.some((line) => line.endsWith(`[harnessd] master re-executed (pid ${process.pid})`))).toBe(true)
await plain.stop()
} finally { log.mockRestore() }
})
it('judges the bundle on disk against the one its lean bundle was read from, when it runs on one', async () => {
// A master re-executed on the lean bundle is handed the sha256 of the cli.js it was read from: the
// code it runs is that bundle's, whatever is on disk by the time it starts.
const script = bundle('')
const lines: string[] = []
const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { lines.push(line) })
try {
const same = start(script, { serviceScriptPath: join(dir, 'lean.mjs'), bundleFingerprint: fingerprint(script) })
await until('its core', () => same.supervisor.status().state === 'running')
expect(same.execs).toEqual([])
expect(lines.some((line) => line.endsWith(`[harnessd] services run from ${join(dir, 'lean.mjs')}, as this master does`))).toBe(true)
await same.stop()
const older = start(script, { bundleFingerprint: 'the bundle before' })
await until('the master to re-execute on the bundle on disk', () => older.execs.length > 0)
expect(older.execs[0].args).toEqual([process.execPath, script, '__harnessd'])
await older.stop()
} finally { log.mockRestore() }
})
it('rolls back a re-execution that never brought a core up, then moves onto the bundle restored', async () => {
const script = bundle('')
writeMarker(marker(), { pid: DEAD_PID, from: 'the one before', to: fingerprint(script)!, at: 1 })
const master = start(script, { restoreUpdate: () => { writeFileSync(script, readFileSync(script, 'utf8') + '\n// the build before\n') } })
await until('the master to re-execute on the bundle restored', () => master.execs.length > 0)
expect(readMarker(marker())).toMatchObject({ pid: process.pid, to: fingerprint(script) })
await master.stop()
})
it('judges, when it starts fresh, an update on disk that no master kept or rolled back: kept once its core stays up, rolled back when it crashes', async () => {
const lines: string[] = []
const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { lines.push(line) })
try {
const good = bundle('')
const asked: Array<string | null> = []
const kept = start(good, { reexecMarkerFile: undefined, unjudgedUpdate: (fingerprintOnDisk: string | null) => { asked.push(fingerprintOnDisk); return '2.0.0' } })
await until('the update to be kept', () => kept.calls.includes('confirm'))
expect(asked).toEqual([fingerprint(good)])
expect(lines.some((line) => line.endsWith('[harnessd] the bundle on disk is 2.0.0, an update no master kept or rolled back — its first core is on probation'))).toBe(true)
await kept.stop()
const crashing = start(bundle('process.exit(3)'), { reexecMarkerFile: undefined, unjudgedUpdate: () => '2.0.1' })
await until('the update to be rolled back', () => crashing.calls.includes('restore'))
expect(crashing.calls).toEqual(['restore'])
await crashing.stop()
// Nothing unjudged: a core like any other.
const plain = start(bundle(''), { reexecMarkerFile: undefined, unjudgedUpdate: () => null })
await until('its core', () => plain.supervisor.status().state === 'running')
await new Promise((resolve) => setTimeout(resolve, 200))
expect(plain.calls).toEqual([])
await plain.stop()
} finally { log.mockRestore() }
})
it('asks whether the bundle on disk after a recovery is unjudged, not the bundle it started from', async () => {
// A re-execution that never came up is rolled back at start; with no room to remember the build,
// its pending note stays and names the bundle this master started from. That is not the bundle its
// first core runs: the build before, put back, is no update to judge.
const lines: string[] = []
const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { lines.push(line) })
try {
const script = bundle('')
const startedFrom = fingerprint(script)!
writeMarker(marker(), { pid: DEAD_PID, from: 'the one before', to: startedFrom, at: 1 })
const asked: Array<string | null> = []
const master = start(script, {
restoreUpdate: () => { writeFileSync(script, readFileSync(script, 'utf8') + '\n// the build before\n') },
unjudgedUpdate: (onDisk: string | null) => { asked.push(onDisk); return onDisk === startedFrom ? '2.0.0' : null },
})
await until('the master to re-execute on the bundle restored', () => master.execs.length > 0)
expect(asked).toEqual([fingerprint(script)])
expect(asked[0]).not.toBe(startedFrom)
expect(lines.some((line) => line.includes('an update no master kept or rolled back'))).toBe(false)
expect(decodeResume(master.execs[0].env[RESUME_ENV])?.update).toBeNull()
await master.stop()
} finally { log.mockRestore() }
})
it('judges the newer build a core on probation staged, instead of rolling it back as a failure', async () => {
// Its first core stages a build and exits for the update; the next, on probation, stages another before it is kept.
const script = bundle(`if (process.env.HARNESSD_RESTARTS !== '2') setTimeout(() => { fs.appendFileSync(__filename, '\\n// a newer build\\n'); process.exit(75) }, process.env.HARNESSD_RESTARTS === '0' ? 100 : 20)`)
// Its staging 20 ms after it starts must come inside its probation. A loaded machine can leave a
// process waiting for the CPU longer than the 100 ms the other tests give it, never 3 s.
const master = start(script, { reexecMarkerFile: undefined, env: { ...process.env, HARNESSD_SERVICES: 'none', HARNESSD_INITIAL_BACKOFF_MS: '10', HARNESSD_UPDATE_PROBATION_MS: '3000' } })
await until('the newest build to be kept', () => master.calls.includes('confirm'))
expect(master.calls).toEqual(['confirm'])
expect(master.supervisor.status()).toMatchObject({ state: 'running', restarts: 2 })
await master.stop()
})
it('starts its services once it carries on, never while it replaces itself before its first core', async () => {
// A rollback found at start moves this master onto the bundle restored before it starts anything.
// Its services were started beside that move: they ran through the probe and were cut off by the
// exec, children no one reaps, and a probe that refused the bundle started them a second time.
const started = (lines: string[]) => lines.filter((line) => line.includes('[harnessd] service search started'))
const lines: string[] = []
const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { lines.push(line) })
const env = { ...process.env, HARNESSD_INITIAL_BACKOFF_MS: '10', HARNESSD_SERVICES: 'search' }
const restore = (file: string) => () => { writeFileSync(file, readFileSync(file, 'utf8') + '\n// the build before\n') }
try {
const script = bundle('')
writeMarker(marker(), { pid: DEAD_PID, from: 'the one before', to: fingerprint(script)!, at: 1 })
let atExec: string[] | null = null
const moving = start(script, { env, restoreUpdate: restore(script), execve: () => { atExec = started(lines) } })
await until('the master to re-execute on the bundle restored', () => atExec !== null)
expect(atExec).toEqual([])
await moving.stop()
// The bundle restored refuses its probe: this master carries on as itself, with each service once.
lines.length = 0
const refusing = bundle('', `console.error('Unknown command: ' + process.argv[2]); process.exit(1)`)
writeMarker(marker(), { pid: DEAD_PID, from: 'the one before', to: fingerprint(refusing)!, at: 1 })
const staying = start(refusing, { env, restoreUpdate: restore(refusing) })
await until('a core on the bundle restored', () => staying.supervisor.status().state === 'running')
expect(started(lines)).toHaveLength(1)
await staying.stop()
} finally { log.mockRestore() }
})
it('takes its own marker with it when it is stopped before its first core is up, and leaves another\'s', async () => {
// Stopped between its exec and its first core, a re-executed master left its marker, and the next
// start took that for a re-execution that never came up: a good update rolled back, its version
// rejected (recoverFailedReexec, selfUpdate.restore).
const script = join(dir, 'unbound.cjs')
writeFileSync(script, `setInterval(() => {}, 1000); process.on('SIGTERM', () => process.exit(0))`)
const resume = encodeResume({ restarts: 1, lastExit: 'code 75', lastExitReason: 'update', update: 'pending', claimed: true, reexecs: 1, unproven: 1 })
writeMarker(marker(), { pid: process.pid, from: 'the one before', to: fingerprint(script)!, at: 1 })
const own = start(script, { env: { ...process.env, [RESUME_ENV]: resume } })
await until('its core', () => own.supervisor.status().corePid !== null)
await own.stop()
expect(readMarker(marker())).toBeNull()
expect(own.calls).toEqual([])
const another = { pid: process.ppid, from: 'the one before', to: fingerprint(script)!, at: 1 }
writeMarker(marker(), another)
const beside = start(script)
await until('its core', () => beside.supervisor.status().corePid !== null)
await beside.stop()
expect(readMarker(marker())).toEqual(another)
})
it('removes its marker once its core is up, and leaves alone what it does not judge', async () => {
const script = bundle('')
writeMarker(marker(), { pid: process.pid, from: 'the one before', to: fingerprint(script)!, at: 1 })
const master = start(script)
await until('its marker to go', () => !existsSync(marker()))
expect(master.calls).toEqual([])
await master.stop()
})
})
})