Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

The master starts an update only when the updater staged it (#964)

Deed03750d4f533
diff --git a/cli/src/harnessd/AGENTS.md b/cli/src/harnessd/AGENTS.md
index 7a8bf2651..1daf741e4 100644
--- a/cli/src/harnessd/AGENTS.md
+++ b/cli/src/harnessd/AGENTS.md
@@ -41,7 +41,9 @@ It keeps the core and the services running, and nothing else.
9. **Updates are the master's to run, never the core's.** The updater runs in a process the master starts
(`UPDATER_HOST`, `../services/updaterProcess.ts`): it downloads, verifies, canaries and stages a build,
then says so (`harnessd:staged`); the master asks the core to hand over (`harnessd:update`) and judges
- the new core on probation, as before. The master itself still makes no network call. Every core this
+ the new core on probation, as before. Only the updater's process (`UPDATER_PROCESS`) can start this, or
+ exit 75 to be restarted at once. From any other service, `harnessd:staged` is logged once and ignored,
+ and exit 75 counts as a crash. The master itself still makes no network call. Every core this
master starts hears `HARNESSD_UPDATES=master`; a core under a master from before that runs the updater
beside itself, in its own process (`../core/updaterBeside.ts`).
10. **A process on demand starts only when the core asks** (`onDemand` in `SERVICE_HOSTS`, `services.ts`).
diff --git a/cli/src/harnessd/master.spec.ts b/cli/src/harnessd/master.spec.ts
index 2a2e07596..6f538e6f4 100644
--- a/cli/src/harnessd/master.spec.ts
+++ b/cli/src/harnessd/master.spec.ts
@@ -529,6 +529,47 @@ describe('runMaster', { timeout: 60_000 }, () => {
} finally { log.mockRestore() }
})
+ it('hands over for a staged build only when the updater staged it, never another service', async () => {
+ const pidFile = join(dir, 'adapter.pid')
+ const seen = join(dir, 'seen')
+ const script = join(dir, 'daemon.cjs')
+ // Search says it staged a build, as only the updater may: the core must not be asked to hand over.
+ writeFileSync(script, `
+ const { appendFileSync } = require('node:fs')
+ const role = process.argv[2] === '__service' ? 'service:' + process.argv[3] : process.argv[2]
+ const say = (what) => appendFileSync(${JSON.stringify(seen)}, JSON.stringify({ role, what }) + '\\n')
+ say('started')
+ if (role === '__run') {
+ process.send({ type: 'harnessd:bound', protocol: 4, port: 1 }); process.send({ type: 'harnessd:ready' })
+ process.on('message', (message) => { if (message.type === 'harnessd:update') { say('asked for ' + message.version); process.exit(75) } })
+ }
+ if (role === 'service:search') setTimeout(() => { process.send({ type: 'harnessd:staged', version: '6.6.6' }); say('staged') }, 100)
+ setInterval(() => process.send({ type: 'harnessd:heartbeat', rssBytes: 1, heapUsedBytes: 1, loopDelayMs: 0 }), 50)
+ process.on('SIGTERM', () => process.exit(0))
+ `)
+ const exits: number[] = []
+ const signals = new Map<string, () => void>()
+ const lines: string[] = []
+ const log = vi.spyOn(console, 'log').mockImplementation((line: string) => { lines.push(line) })
+ try {
+ runMaster({
+ nodePath: process.execPath, execArgv: [], scriptPath: script, pidFile, updater: true,
+ restoreUpdate: () => {}, confirmUpdate: () => {},
+ env: { ...process.env, HARNESSD_SERVICES: 'search' },
+ exit: (code) => exits.push(code), onSignal: (signal, listener) => signals.set(signal, listener),
+ })
+ const said = (): Array<Record<string, string>> => existsSync(seen) ? readFileSync(seen, 'utf8').trim().split('\n').map((line) => JSON.parse(line)) : []
+ await until('search to say it staged a build', () => said().some((line) => line.role === 'service:search' && line.what === 'staged'))
+ // Long enough for a handover the master had asked for to reach the core.
+ await new Promise((resolve) => setTimeout(resolve, 500))
+ expect(said().filter((line) => line.role === '__run')).toEqual([{ role: '__run', what: 'started' }])
+ expect(lines.some((line) => line.includes('asking the core to hand over'))).toBe(false)
+ expect(lines.some((line) => line.endsWith('[harnessd] service search said it staged 6.6.6, which only the updater may — ignored'))).toBe(true)
+ signals.get('SIGTERM')!()
+ await until('the master to finish', () => exits.length > 0)
+ } finally { log.mockRestore() }
+ })
+
it('starts the services and the core from the lean bundle when given one, telling the core which CLI it runs for', async () => {
// The lean bundle cli.js carries (./leanBundle.ts): each service, and the core, parses its own code, not
// the CLI's. A core started from it is told the cli.js it came from, its CLI to everything it hands on.
diff --git a/cli/src/harnessd/master.ts b/cli/src/harnessd/master.ts
index a44f699eb..7e045efee 100644
--- a/cli/src/harnessd/master.ts
+++ b/cli/src/harnessd/master.ts
@@ -16,7 +16,7 @@ import { LEAN_CORE_SCRIPT_ENV, type MasterMessage } from './protocol.js'
import {
PROBE_ANSWER, RESUME_ENV, createReexec, decodeResume, fingerprint, readMarker, recoverFailedReexec, removeMarker, runProbe, writeMarker,
} from './reexec.js'
-import { SERVICE_HOSTS, ServiceSupervisor, UPDATER_HOST, serviceOptions, serviceProcessesEnv, serviceSpecs } from './services.js'
+import { SERVICE_HOSTS, ServiceSupervisor, UPDATER_HOST, UPDATER_PROCESS, serviceOptions, serviceProcessesEnv, serviceSpecs } from './services.js'
import {
DEFAULT_SUPERVISOR_OPTIONS, Supervisor, type CoreHandle, type SupervisorDeps, type SupervisorOptions, type SupervisorStatus,
} from './supervisor.js'
@@ -298,7 +298,7 @@ export function runMaster(config: MasterConfig): Supervisor {
})
const specs = serviceSpecs(env, SERVICE_HOSTS)
// The updater beside them, whatever HARNESSD_SERVICES says: the core neither routes to it nor runs it.
- const processes = config.updater ? [...specs, { name: 'updater', ...UPDATER_HOST }] : specs
+ const processes = config.updater ? [...specs, { name: UPDATER_PROCESS, ...UPDATER_HOST }] : specs
let supervisor: Supervisor | null = null
const services = new ServiceSupervisor(processes, {
spawnService: (spec, extra) => {
diff --git a/cli/src/harnessd/services.spec.ts b/cli/src/harnessd/services.spec.ts
index 1d5b5f3f3..5b2c3a9e5 100644
--- a/cli/src/harnessd/services.spec.ts
+++ b/cli/src/harnessd/services.spec.ts
@@ -105,6 +105,27 @@ describe('ServiceSupervisor', () => {
expect(staged).toEqual(['9.9.9'])
})
+ it('takes a staged build, or a request to start again at once, from the updater alone', () => {
+ // A staged build restarts the core. From any other service it is a bug or worse: said once, nothing done.
+ staged.length = 0
+ const supervisor = make([search], { parkCrashes: 2 })
+ supervisor.start()
+ latest().beat()
+ latest().say({ type: 'harnessd:staged', version: '6.6.6' })
+ latest().say({ type: 'harnessd:staged', version: '6.6.7' })
+ expect(staged).toEqual([])
+ expect(lines.filter((line) => line.includes('staged'))).toEqual(['[harnessd] service search said it staged 6.6.6, which only the updater may — ignored'])
+ // Its exit 75 is a crash like any other: restarted after the backoff, and parked when it keeps on.
+ latest().exit(75)
+ expect(supervisor.status()[0]).toMatchObject({ state: 'restarting', lastExit: 'code 75', lastExitReason: 'crashed' })
+ vi.advanceTimersByTime(0)
+ expect(children).toHaveLength(1)
+ vi.advanceTimersByTime(options.initialBackoffMs)
+ expect(children).toHaveLength(2)
+ latest().exit(75)
+ expect(supervisor.status()[0].state).toBe('parked')
+ })
+
it('restarts a crashed service with a backoff that doubles, caps, and is forgiven after a long run', () => {
const supervisor = make([search], { parkCrashes: 99 })
supervisor.start()
diff --git a/cli/src/harnessd/services.ts b/cli/src/harnessd/services.ts
index f556c02db..f2b208a6f 100644
--- a/cli/src/harnessd/services.ts
+++ b/cli/src/harnessd/services.ts
@@ -114,6 +114,7 @@ class Service {
private killTimer: unknown = null
private restartTimer: unknown = null
private onStopped: Array<() => void> = []
+ private refusedStaged = false
constructor(
readonly spec: ServiceSpec,
@@ -181,6 +182,8 @@ class Service {
private onMessage(child: CoreHandle, message: unknown): void {
if (isUpdaterMessage(message)) {
+ // It restarts the core: from any other service than the updater, a bug or worse, said once and ignored.
+ if (this.spec.name !== UPDATER_PROCESS) { if (!this.refusedStaged) this.deps.log(`[harnessd] service ${this.spec.name} said it staged ${message.version}, which only the updater may — ignored`); this.refusedStaged = true; return }
this.deps.log(`[harnessd] service ${this.spec.name} staged ${message.version}`)
this.deps.staged?.(message.version)
return
@@ -216,9 +219,10 @@ class Service {
this.stopped()
return
}
- if (code === SERVICE_EXIT_RESTART && !this.killReason) {
- // Asked for: the updater, once it has staged a build, so that it next runs as that build (a lean
- // bundle that no longer matches cli.js is not used, ./leanServices.ts). Not a crash, and not delayed.
+ if (code === SERVICE_EXIT_RESTART && this.spec.name === UPDATER_PROCESS && !this.killReason) {
+ // Asked for: the updater, once it has staged a build, so that it next runs as that build (a lean bundle
+ // that no longer matches cli.js is not used, ./leanServices.ts). Not a crash, and not delayed; from any
+ // other service, 75 is a crash, or a loop of them would never be parked.
this.lastExitReason = 'restart'
this.restarts++
this.deps.log(`[harnessd] service ${this.spec.name} asked to start again (${exit}) — restarting`)
@@ -404,6 +408,7 @@ export const KNOWN_SERVICES: readonly string[] = Object.values(SERVICE_HOSTS).fl
* it when it runs the installed copy with updates on (masterProcess.ts). It holds one download at a time.
*/
export const UPDATER_HOST: ServiceHostSpec = { services: ['updater'], heapLimitMiB: 256, rssLimitMiB: 512 }
+export const UPDATER_PROCESS = 'updater' // the name the master runs it under: the one whose staged build counts
/** Service timings from the environment (for tests and support); anything unset or invalid keeps its default. */
export function serviceOptions(env: NodeJS.ProcessEnv): ServiceSupervisorOptions {