Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: introduce daemon lifecycle orchestration with spawn locking, graceful termination, and robust self-update handling.

0xken153dc149e7dc8f
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index 93de8cca6..023ab191a 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -38,6 +38,11 @@ import { DeviceFleet } from './device/deviceFleet.js'
import { registry, projectDisplayName, type RegisteredSession } from './lib/registry.js'
import { installAmpPlugin, installCodexHooks, installCommandCodeHooks, installCursorHooks, installDevinHooks, installGrokHooks, installAgyHooks, installCopilotHooks, installHermesHooks, installKiloPlugin, installOpencodePlugin, installPiExtension, installSessionHooks } from './lib/hooks.js'
import { PID_FILE, daemonPort, isAlive, readPid } from './lib/daemonState.js'
+import {
+ BIND_WAIT_MS, connectFailure, defaultLaunchDeps, removePidFileIf, waitForBind, waitForReady,
+} from './lib/daemonLaunch.js'
+import { SpawnLockBusyError, describeSpawnLockFailure, describeSpawnLockOwner, withSpawnLock } from './lib/daemonSpawnLock.js'
+import { stopDaemonProcess } from './lib/daemonStop.js'
import { ensureTmuxOnPath, requireTmuxAvailable } from './lib/tmuxOnPath.js'
import { flashCommand } from './lib/flash.js'
import { readOrMintComputerId } from './lib/computerIdentity.js'
@@ -780,15 +785,29 @@ async function startCommand(foreground: boolean, repair: boolean = false): Promi
// and self-updates on its own) and on a dev/repo build; never blocks start if the check fails.
// Run alongside resolveComputerMachine() rather than before it — the two touch disjoint resources
// (bundle files vs. the auth-session file/a machine-resolve POST), so there is nothing to serialize.
+ //
+ // Under the spawn lock from the staging onwards: the running daemon's own updater swaps the same
+ // cli.js/.prev, and two writers there can leave `.prev` holding the NEW bytes — which is what a
+ // later rollback would then "restore". The lock is re-entrant, so `launch` below just joins it.
if (!foreground) {
- const [v] = await Promise.all([
- stageLatestBundle((m) => console.log(m)),
- resolveComputerMachine(),
- ])
- if (v) console.log(` ✓ updated to v${v} — connecting on the new build`)
- } else {
- await resolveComputerMachine()
+ await withSpawnLock('start', async () => {
+ const [v] = await Promise.all([
+ stageLatestBundle((m) => console.log(m)),
+ resolveComputerMachine(),
+ ])
+ if (v) console.log(` ✓ updated to v${v} — connecting on the new build`)
+ await launch(foreground, repair)
+ }, {
+ onWaiting: (owner) => console.log(` the daemon is ${describeSpawnLockOwner(owner)} — waiting for it to finish…`),
+ }).catch((error: unknown) => {
+ if (!(error instanceof SpawnLockBusyError)) throw error
+ console.error(`\n✗ Could not start: the daemon spawn lock is ${describeSpawnLockFailure(error)}.`)
+ console.error(' check harness status · stop it harness stop')
+ process.exit(1)
+ })
+ return
}
+ await resolveComputerMachine()
await launch(foreground, repair)
}
@@ -859,27 +878,40 @@ async function updateCommand(force: boolean): Promise<void> {
// A newer build exists. Stop the running daemon FIRST so its own background updater can't race our
// staging on the .prev/.tmp files, then swap the bytes and bring it back up on the new build.
- const running = readPid()
- const wasRunning = !!(running && isAlive(running))
- const relaunch = async (): Promise<void> => {
- if (!readAuthSession()) { console.log(' (not signed in — run `harness login`, then `harness start`.)'); process.exit(0) }
- await new Promise((r) => setTimeout(r, 1000)) // grace for the backend to release the one-machine claim
- await launch(false) // spawns a fresh daemon on the new bytes, prints status, and exits
- }
- if (wasRunning) { console.log(' stopping the running adapter…'); await stopDaemonProcess() }
-
- console.log(`▸ Updating v${VERSION} → v${entry.version}…`)
- let ok = false
- try { ok = await downloadCanaryStage(entry, resolve(env.ADAPTER_CLI_DIR), (m) => console.log(m)) }
- catch (e) { console.error(`✗ Update failed: ${e instanceof Error ? e.message : e}`); ok = false }
- if (!ok) {
- if (wasRunning) await relaunch() // staging failed → bring the OLD build back so `update` never leaves it down
+ //
+ // The whole stop → stage → relaunch sequence runs under the spawn lock. Between the stop and the
+ // relaunch there is no pid file for several seconds, and anything that spawns `harness start` on
+ // "no daemon" (the desktop app does, every few seconds) used to land a second child in that gap.
+ const staged = entry
+ await withSpawnLock('update', async () => {
+ const running = readPid()
+ const wasRunning = !!(running && isAlive(running))
+ const relaunch = async (): Promise<void> => {
+ if (!readAuthSession()) { console.log(' (not signed in — run `harness login`, then `harness start`.)'); process.exit(0) }
+ await new Promise((r) => setTimeout(r, 1000)) // grace for the backend to release the one-machine claim
+ await launch(false) // spawns a fresh daemon on the new bytes, prints status, and exits
+ }
+ if (wasRunning) { console.log(' stopping the running adapter…'); await stopDaemonProcess() }
+
+ console.log(`▸ Updating v${VERSION} → v${staged.version}…`)
+ let ok = false
+ try { ok = await downloadCanaryStage(staged, resolve(env.ADAPTER_CLI_DIR), (m) => console.log(m)) }
+ catch (e) { console.error(`✗ Update failed: ${e instanceof Error ? e.message : e}`); ok = false }
+ if (!ok) {
+ if (wasRunning) await relaunch() // staging failed → bring the OLD build back so `update` never leaves it down
+ process.exit(1)
+ }
+ console.log(` ✓ installed v${staged.version}`)
+ if (wasRunning) { await relaunch(); return }
+ console.log(`✓ Updated to v${staged.version}. Run \`harness start\` to connect.`)
+ process.exit(0)
+ }, {
+ onWaiting: (owner) => console.log(` the daemon is ${describeSpawnLockOwner(owner)} — waiting for it to finish…`),
+ }).catch((error: unknown) => {
+ if (!(error instanceof SpawnLockBusyError)) throw error
+ console.error(`\n✗ Could not update: the daemon spawn lock is ${describeSpawnLockFailure(error)}. Try again in a moment.`)
process.exit(1)
- }
- console.log(` ✓ installed v${entry.version}`)
- if (wasRunning) { await relaunch(); return }
- console.log(`✓ Updated to v${entry.version}. Run \`harness start\` to connect.`)
- process.exit(0)
+ })
}
/**
@@ -893,9 +925,9 @@ async function updateCommand(force: boolean): Promise<void> {
* long-lived credential is not left behind in silence.
*/
async function logout(): Promise<void> {
- const pid = readPid()
- if (pid && isAlive(pid)) { try { process.kill(pid, 'SIGTERM') } catch { /* ignore */ } }
- rmSync(PID_FILE, { force: true })
+ // Through the lock-taking stop, not an inline kill: a logout that lands mid-handoff would otherwise
+ // SIGTERM the OLD daemon, leave the new one coming up, and then delete the session under it.
+ await stopDaemonProcess()
clearAuthSession()
rmSync(MACHINE_NAME_FILE, { force: true })
console.log('Signed out. Run `harness login`, then `harness start`, to reconnect this computer.')
@@ -929,12 +961,8 @@ async function runForeground(session: AuthSession): Promise<void> {
let discoveryReady = false
let discoveryError: string | null = null
- // Claim the pid file for OURSELVES, first thing. It used to be written by whoever spawned us — the
- // update-restart's parent, after supervising the handover — so a parent that died mid-handover (a
- // `harness stop` landing in that window is enough) left a daemon nothing could manage: `status` said
- // stopped, `stop` had no pid to signal, and `join` walked into "port already in use" against the very
- // daemon it could not see. A process that is running is the only honest author of its own pid.
- try { writeFileSync(PID_FILE, String(process.pid) + '\n') } catch { /* best effort */ }
+ // The pid file is claimed further down, the moment the control port is bound — not here, and not
+ // by whoever spawned us. See the comment at that claim.
// Last-resort net: a stray throw in ANY long-lived callback (a malformed JSONL line, a hostile backend
// frame, a timer) must NEVER take the daemon down — there is no supervisor. Log it and keep running.
@@ -2441,6 +2469,14 @@ async function runForeground(session: AuthSession): Promise<void> {
return { status: res.status, body: json }
}
+ // Update-handoff state, declared here — ahead of the /api/status handler that reads `restarting` —
+ // rather than beside the updater that writes it, so the closure never reaches a `let` in its TDZ.
+ let restarting = false
+ // The child a handoff is supervising, so a signal that lands mid-handoff can take it down with us
+ // rather than leaving two daemons — see shutdown(). Cleared the moment the handoff is CONFIRMED:
+ // from then on that child is the daemon, and a signal must not take it down with the old one.
+ let handoffChild: ReturnType<typeof spawn> | null = null
+
const { server: hookServer, port: hookPort } = await startHookServer(env.PORT, {
onAutonomousDeviceRequest: async (method, target, body) => {
if (!autonomousDeviceService) return { status: 503, body: { error: { code: 'UNAVAILABLE', message: 'Autonomous device service is starting' } } }
@@ -2766,6 +2802,11 @@ async function runForeground(session: AuthSession): Promise<void> {
deviceTransportConnected: backend.hasCommander(),
deviceE2eeConnected: backend.deviceE2eeConnected(),
uptimeSec: Math.round((Date.now() - startedAt) / 1000),
+ pid: process.pid,
+ startedAt,
+ // True for the few hundred ms between an update being staged and this server closing for the
+ // handoff. Informational: nothing should build readiness on a field the server stops serving.
+ restarting,
discoveryReady,
discoveryError,
fingerprint: backend.e2eeFingerprint(),
@@ -2811,6 +2852,14 @@ async function runForeground(session: AuthSession): Promise<void> {
onMachineDelete: (machineId) => proxyBackend('DELETE', `/api/machines/${encodeURIComponent(machineId)}`),
onAuthMe: () => proxyBackend('GET', '/api/auth/me'),
})
+ // Claim the pid file for OURSELVES, and only now that the control port is bound. It used to be
+ // written by whoever spawned us — so a parent that died mid-handover left a daemon nothing could
+ // manage — and then, for a while, by us at the top of this function, before the bind — so a child
+ // that LOST the port to a sibling still left a file naming itself, a corpse, over the winner. A
+ // process that is running AND holds the port is the only honest author of its own pid; that claim
+ // is also the signal `harness start` and the update handoff wait on to know the bind succeeded.
+ try { writeFileSync(PID_FILE, String(process.pid) + '\n') } catch { /* best effort */ }
+ console.log(`[cli] daemon pid ${process.pid} · v${VERSION}${process.env.ADAPTER_UPDATED_TO ? ' · updated' : ''} · listening on 127.0.0.1:${hookPort}`)
// Same on-disk identity `harness remote-password set`/`link connect` use (E2eeStore.init() is
// idempotent per file, so a separate in-memory instance here just reads the one this machine
// already has).
@@ -3778,10 +3827,13 @@ async function runForeground(session: AuthSession): Promise<void> {
// The cost is real and accepted: a turn streaming at that moment loses the rest of its events, and its
// clients see no turn_end for it until the new daemon re-attaches the session and the next turn runs.
let updater: Poller | null = null
- let restarting = false
// Hand off to a freshly-spawned daemon running the just-swapped cli.js, then SUPERVISE it and roll
// back to the .prev bytes if it fails to come up. NOT launch() — that refuses while a daemon is alive.
+ //
+ // Runs under the spawn lock for its whole length (the updater's `withLock` wraps the staging and
+ // this together), so no `harness start` can spawn into the seconds where the port is free and the
+ // pid file names nothing.
const restartForUpdate = async (newVersion: string): Promise<void> => {
if (restarting) return
restarting = true
@@ -3830,39 +3882,51 @@ async function runForeground(session: AuthSession): Promise<void> {
const sinceOffset = existsSync(LOG_FILE) ? statSync(LOG_FILE).size : 0
const child = spawnDaemon({ ADAPTER_UPDATED_TO: newVersion })
+ handoffChild = child
let childExited = false
child.on('exit', () => { childExited = true })
- // KEEP on connected/unreachable/busy (new build RAN); ROLL BACK only on an early exit or a `fatal`
- // (bad bundle / can't bind). unreachable = backend transient, not a bad build → don't bounce.
- const ready = await waitForReady(sinceOffset, 30_000)
- if (!childExited && ready.state !== 'fatal') {
- try { writeFileSync(PID_FILE, String(child.pid) + '\n') } catch { /* ignore */ }
+ // Two phases. First the child has to BIND the port — it claims the pid file itself at that
+ // moment, and nothing else writes that file any more. A child that exits or stalls before then
+ // is a bad build (or a port it could not take): roll back at once instead of burning the whole
+ // connect window on it. Then, bound, wait for the backend: KEEP on connected/unreachable/busy
+ // (the new build RAN), ROLL BACK only on `fatal`. unreachable = backend transient, not a bad build.
+ const bind = await waitForBind(child.pid ?? -1, () => childExited, BIND_WAIT_MS, launchDeps)
+ const ready = bind === 'bound' ? await waitForReady(sinceOffset, 30_000, launchDeps) : null
+ if (bind === 'bound' && !childExited && ready?.state !== 'fatal') {
+ // Confirmed: it is the daemon now. Let go of it BEFORE anything else — a SIGTERM landing between
+ // here and the exit below must not take it down with us (see shutdown()).
+ handoffChild = null
child.unref()
confirmUpdate(env.ADAPTER_CLI_DIR) // drop the .prev backups
console.log(`[update] now running ${newVersion} (pid ${child.pid})`)
process.exit(0)
}
- console.error(`[update] new build failed to start (${childExited ? 'exited' : ready.state}) — rolling back`)
+ console.error(`[update] new build failed to start (${bind !== 'bound' ? bind : childExited ? 'exited' : ready?.state}) — rolling back`)
try { if (child.pid) process.kill(child.pid, 'SIGKILL') } catch { /* ignore */ }
+ // A killed child cannot remove its own pid file; do it for it — but only once it is actually
+ // dead (SIGKILL is asynchronous, and a child mid-bind could still write the file after our
+ // removal) and only if it is still ITS file.
+ if (child.pid) {
+ const gone = Date.now() + 2_000
+ while (Date.now() < gone && isAlive(child.pid)) await new Promise((r) => setTimeout(r, 50))
+ }
+ removePidFileIf(child.pid)
restoreUpdate(env.ADAPTER_CLI_DIR) // restore .prev → cli.js/notify.mjs
const good = spawnDaemon({})
- try { writeFileSync(PID_FILE, String(good.pid) + '\n') } catch { /* ignore */ }
+ handoffChild = good
+ let goodExited = false
+ good.on('exit', () => { goodExited = true })
+ // Hold the lock — and this process — until the rollback child has bound too. Exiting the moment it
+ // is spawned would free the lock while the port is still unclaimed, which is the window this whole
+ // arrangement exists to close. Nothing to do if it fails: the .prev bytes were the build that was
+ // running a minute ago, and `harness start` can be tried by hand.
+ const goodBind = await waitForBind(good.pid ?? -1, () => goodExited, BIND_WAIT_MS, launchDeps)
+ if (goodBind !== 'bound') console.error(`[update] rollback build did not come up either (${goodBind}) — run harness start`)
good.unref()
process.exit(0)
}
- // Staged → restart, right now. `restartForUpdate` already latches on `restarting`, so a second call
- // (e.g. the poller staging again before teardown finishes) is a no-op rather than two daemons.
- const applyUpdate = (version: string): void => {
- // If the restart handoff itself throws/rejects (I/O fault during teardown), don't let it become an
- // unhandledRejection — log, un-latch `restarting`, and stay on the current build until the next poll.
- void restartForUpdate(version).catch((err) => {
- console.error('[update] restart failed — staying on current build:', err instanceof Error ? err.message : err)
- restarting = false
- })
- }
-
// Self-update ONLY manages the INSTALLED copy (`~/.harness/cli/cli.js`). A dev/repo run — `tsx`
// (`npm run dev`) OR `node dist/cli.js` from the checkout — must NEVER self-update: it would swap
// the published bundle into ~/.harness/cli and restart, hijacking the version you're developing.
@@ -3877,7 +3941,19 @@ async function runForeground(session: AuthSession): Promise<void> {
key: env.ADAPTER_UPDATE_KEY,
dir: env.ADAPTER_CLI_DIR,
intervalMs: env.ADAPTER_UPDATE_CHECK_MS,
- onStaged: (v) => applyUpdate(v),
+ // The lock spans the byte swap AND the handoff it triggers, as one critical section: a
+ // `harness start` that lands between the two would otherwise stage over our .prev, and one
+ // that lands during the handoff would spawn a second daemon.
+ withLock: (fn) => withSpawnLock('handoff', fn, {
+ onWaiting: (owner) => console.log(`[update] waiting — the daemon is ${describeSpawnLockOwner(owner)}`),
+ }),
+ onStaged: (v) => restartForUpdate(v).catch((err) => {
+ // If the restart handoff itself throws/rejects (I/O fault during teardown), don't let it
+ // become an unhandledRejection — log, un-latch `restarting`, and stay on the current build.
+ console.error('[update] restart failed — staying on current build:', err instanceof Error ? err.message : err)
+ restarting = false
+ handoffChild = null
+ }),
})
console.log(`[update] self-update on · v${VERSION} · every ${Math.round(env.ADAPTER_UPDATE_CHECK_MS / 1000)}s`)
} else if (!env.ADAPTER_UPDATE_DISABLE) {
@@ -3886,6 +3962,19 @@ async function runForeground(session: AuthSession): Promise<void> {
const shutdown = async (signal: string): Promise<void> => {
console.log(`\n[cli] ${signal} — shutting down`)
+ // Mid-handoff everything below has already been torn down once, and the daemon that matters is
+ // the child being supervised. Take it down with us and leave — a second teardown of closed servers
+ // is noise, and a child left running would be a daemon nothing manages.
+ if (restarting) {
+ const child = handoffChild // null once the handoff was confirmed — that daemon stays up
+ if (child?.pid) {
+ console.log(`[cli] ${signal} during an update handoff — stopping the new daemon (pid ${child.pid}) too`)
+ try { process.kill(child.pid, 'SIGTERM') } catch { /* ignore */ }
+ removePidFileIf(child.pid)
+ }
+ try { if (readPid() === process.pid) rmSync(PID_FILE, { force: true }) } catch { /* ignore */ }
+ process.exit(0)
+ }
// Release the serial port first. It is exclusive, and a daemon that exits still holding it makes
// esptool fail in a way that reads exactly like dead hardware.
void cableRef?.stop()
@@ -4187,65 +4276,9 @@ function printInfoBlock(opts: {
console.log('')
}
-// 'fatal' = misconfig; retrying is pointless → kill the daemon + error out.
-// 'unreachable' = transient (backend deploying / 5xx / slow / not up yet) → the daemon keeps retrying
-// in the background and connects on its own, so DON'T kill it — just report the state.
-// 'deauth' = the saved credential is invalid/revoked (401/403) → clear it and ask for a fresh token.
-// 'busy' = this machine is already connected from another computer (409) → keep token, stop, inform.
-interface ReadyResult { state: 'connected' | 'deauth' | 'fatal' | 'unreachable' | 'busy'; detail?: string }
-
-/** Classify a backend connection error from the log tail: a human reason + fatal (won't self-heal) vs
- * transient (will), and `deauth` for 401/403 (the token is no longer valid). null = no signal yet. */
-function connectFailure(tail: string): { detail: string; fatal: boolean; deauth?: boolean; busy?: boolean } | null {
- // The daemon logs this marker when the backend rejected us with 409 (machine held by another computer).
- if (tail.includes('[backend] machine busy')) {
- return { detail: 'this machine is already connected from another computer', fatal: true, busy: true }
- }
- // The daemon couldn't bind the (fixed) hook port → another adapter is almost certainly already
- // running. Fatal: don't sit on "retrying"; tell the user how to find/stop the other one.
- if (/EADDRINUSE|already in use/.test(tail)) {
- return { detail: `hook port ${env.PORT} is already in use — is another machine daemon running? (harness stop, or lsof -ti :${env.PORT} | xargs kill)`, fatal: true }
- }
- const m = tail.match(/Unexpected server response: (\d+)/)
- if (m) {
- const code = Number(m[1])
- // 401/403 = this computer's machine was deleted/revoked (or a bad token) → deauth: clear + re-join.
- // 409 = one machine per machine: already connected from another computer → busy (keep token, stop).
- // 404 = wrong backend / route not deployed → misconfig (fatal, don't wipe the token).
- // 5xx (esp. 502/503/504) = gateway up but the app is deploying/restarting → transient, keep retrying.
- if (code === 409) return { detail: 'this machine is already connected from another computer', fatal: true, busy: true }
- const deauth = code === 401 || code === 403
- return { detail: `backend returned HTTP ${code}`, fatal: deauth || code === 404, deauth }
- }
- if (/ENOTFOUND|EAI_AGAIN|getaddrinfo/.test(tail)) return { detail: 'host not found (DNS)', fatal: true }
- if (/certificate|CERT_|self-signed/i.test(tail)) return { detail: 'TLS certificate error', fatal: true }
- if (/ECONNREFUSED/.test(tail)) return { detail: 'connection refused', fatal: false } // backend not up yet → retry
- // A 1006 close with no HTTP code — generic "couldn't reach it right now"; transient.
- if (/disconnected \(close 1006\)/.test(tail)) return { detail: 'cannot reach backend', fatal: false }
- return null
-}
-
-/** Poll the log until the adapter connects, hits a FATAL error, or the window ends. Transient errors
- * don't short-circuit — the connection may recover within the window (e.g. a 502 during a deploy). */
-async function waitForReady(sinceOffset: number, timeoutMs = 8000): Promise<ReadyResult> {
- const deadline = Date.now() + timeoutMs
- let lastTransient: string | undefined
- while (Date.now() < deadline) {
- let tail = ''
- // Slice the raw BYTES from sinceOffset (a Buffer byte length) THEN decode. The log is full of
- // multi-byte glyphs (→ · ─ ● …), so decoding first and slicing the STRING by that byte count
- // overshoots (byte length > char length) and returns "" — the classic false "no connection".
- try { tail = readFileSync(LOG_FILE).subarray(sinceOffset).toString('utf-8') } catch { /* not yet */ }
- if (tail.includes('[backend] connected')) return { state: 'connected' }
- const fail = connectFailure(tail)
- if (fail?.busy) return { state: 'busy', detail: fail.detail }
- if (fail?.deauth) return { state: 'deauth', detail: fail.detail }
- if (fail?.fatal) return { state: 'fatal', detail: fail.detail }
- if (fail) lastTransient = fail.detail // remember, but keep waiting — it may connect on a retry
- await new Promise((r) => setTimeout(r, 250))
- }
- return { state: 'unreachable', detail: lastTransient ?? `no connection within ${timeoutMs / 1000}s` }
-}
+/** The log-tail readiness classifier and the two-phase wait live in lib/daemonLaunch.ts — see there.
+ * `launchDeps` binds them to this process's log file and port. */
+const launchDeps = defaultLaunchDeps(LOG_FILE, daemonPort())
// ── daemon start / stop / status ───────────────────────────────────────────────────────────────
@@ -4271,6 +4304,26 @@ async function launch(foreground: boolean, repair: boolean = false): Promise<voi
return
}
+ // ONE spawner at a time. The desktop app re-runs `harness start` every few seconds while the daemon
+ // looks down — which it does for the length of an update handoff, or of `harness update` — and a
+ // second child racing the first for the fixed port is how an orphan ends up holding it. Waiting is
+ // the right answer: when the holder finishes, the pid file names a live daemon and the check below
+ // says "already running", which is exactly what the caller wanted to hear.
+ try {
+ await withSpawnLock('start', () => spawnDaemon(session, runtimeNode), {
+ onWaiting: (owner) => console.log(` the daemon is ${describeSpawnLockOwner(owner)} — waiting for it to finish…`),
+ })
+ } catch (error) {
+ if (!(error instanceof SpawnLockBusyError)) throw error
+ console.error(`\n✗ Could not start: the daemon spawn lock is ${describeSpawnLockFailure(error)}.`)
+ console.error(' check harness status · stop it harness stop')
+ console.error(` logs ${tildify(LOG_FILE)}`)
+ process.exit(1)
+ }
+}
+
+/** The part of `launch` that runs under the spawn lock: check, spawn, wait for bind, wait for connect. */
+async function spawnDaemon(session: AuthSession, runtimeNode: string | null): Promise<void> {
const running = readPid()
if (running && isAlive(running)) {
console.log(`machine already running (pid ${running}) — it auto-reconnects.`)
@@ -4289,16 +4342,31 @@ async function launch(foreground: boolean, repair: boolean = false): Promise<voi
env: { ...process.env },
stdio: ['ignore', logFd, logFd],
})
- writeFileSync(PID_FILE, String(child.pid) + '\n')
+ let childExited = false
+ child.on('exit', () => { childExited = true })
+ child.on('error', () => { childExited = true })
child.unref()
- const ready = await waitForReady(logOffset, CONNECT_WAIT_MS)
+ // The pid file is NOT written here. The child claims it itself, once — and only once — it has bound
+ // the control port (see runForeground); that claim is the bind signal waited on below. A spawner
+ // writing it first meant a child that lost the port left a file naming a corpse.
+ const bind = await waitForBind(child.pid ?? -1, () => childExited, BIND_WAIT_MS, launchDeps)
+ if (bind !== 'bound') {
+ const fail = connectFailure(launchDeps.readLogSlice(logOffset), daemonPort())
+ if (bind === 'timeout') { try { if (child.pid) process.kill(child.pid, 'SIGTERM') } catch { /* ignore */ } }
+ const detail = fail?.detail ?? (bind === 'exited' ? 'the daemon exited during startup' : `the daemon did not bind within ${BIND_WAIT_MS / 1000}s`)
+ console.error(`\n✗ ${detail}`)
+ console.error(` logs ${tildify(LOG_FILE)}`)
+ process.exit(1)
+ }
+
+ const ready = await waitForReady(logOffset, CONNECT_WAIT_MS, launchDeps)
// DEAUTH (401/403): the saved credential is no longer valid — this computer was removed from the machine
// (or the token is stale). Wipe it, stop the daemon, and signal the caller to ask for a fresh token.
if (ready.state === 'deauth') {
try { if (child.pid) process.kill(child.pid, 'SIGTERM') } catch { /* ignore */ }
- rmSync(PID_FILE, { force: true })
+ removePidFileIf(child.pid)
clearAuthSession()
return
}
@@ -4308,7 +4376,7 @@ async function launch(foreground: boolean, repair: boolean = false): Promise<voi
// failure, not a retry loop). The user stops the other machine first, then runs `harness start` here.
if (ready.state === 'busy') {
try { if (child.pid) process.kill(child.pid, 'SIGTERM') } catch { /* ignore */ }
- rmSync(PID_FILE, { force: true })
+ removePidFileIf(child.pid)
console.log('\n ℹ This machine is already connected from another computer.')
console.log(' Only one machine per machine — stop the adapter on that machine first,')
console.log(' then run `harness start` here again.')
@@ -4319,7 +4387,7 @@ async function launch(foreground: boolean, repair: boolean = false): Promise<voi
// to fix. (Not 401/403 — those are handled as deauth above.)
if (ready.state === 'fatal') {
try { if (child.pid) process.kill(child.pid, 'SIGTERM') } catch { /* ignore */ }
- rmSync(PID_FILE, { force: true })
+ removePidFileIf(child.pid)
// A hook-port clash isn't a backend problem — it's a duplicate/leftover adapter. Report it as such.
if (ready.detail?.startsWith('hook port')) {
console.error(`\n✗ ${ready.detail}`)
@@ -4358,20 +4426,6 @@ async function launch(foreground: boolean, repair: boolean = false): Promise<voi
process.exit(0)
}
-async function stopDaemonProcess(): Promise<{ pid: number | null; stopped: boolean }> {
- const pid = readPid()
- if (!pid || !isAlive(pid)) {
- rmSync(PID_FILE, { force: true })
- return { pid: null, stopped: false }
- }
- try { process.kill(pid, 'SIGTERM') } catch { /* already gone */ }
- const deadline = Date.now() + 3000
- while (Date.now() < deadline && isAlive(pid)) await new Promise((r) => setTimeout(r, 150))
- if (isAlive(pid)) { try { process.kill(pid, 'SIGKILL') } catch { /* ignore */ } }
- rmSync(PID_FILE, { force: true })
- return { pid, stopped: true }
-}
-
/** `harness stop` — SIGTERM the background adapter, SIGKILL if it lingers. */
async function stop(): Promise<void> {
const r = await stopDaemonProcess()
@@ -4390,6 +4444,7 @@ function clearAdapterState(): void {
for (const name of [
'token',
'adapter.pid',
+ 'adapter.spawn.lock',
'harness.log',
'machine.log', // pre-rename names — still cleared so a reset leaves nothing behind
'adapter.log',
@@ -5017,6 +5072,9 @@ if (cmd === 'version' || cmd === '--version' || cmd === '-v') { console.log(VERS
const onError = (err: unknown): never => {
console.error('Failed to start adapter:', err)
+ // A daemon that claimed the pid file (port bound) and then failed to finish starting must not leave
+ // that file naming a corpse — the next `harness start` would refuse on it. Only ours, though.
+ removePidFileIf(process.pid)
process.exit(1)
}
diff --git a/cli/src/hookServer.spec.ts b/cli/src/hookServer.spec.ts
index e0ec6baca..4c15e3fd1 100644
--- a/cli/src/hookServer.spec.ts
+++ b/cli/src/hookServer.spec.ts
@@ -159,3 +159,14 @@ describe('chooseHookAgent', () => {
expect(chooseHookAgent(['a', 'b'], ['c'])).toEqual({ agent: null, reason: 'ambiguous' })
})
})
+
+describe('/api/status', () => {
+ it('serves whatever the daemon reports — including the pid the desktop uses to tell daemons apart', async () => {
+ const { base } = await start({
+ onStatus: () => ({ pid: process.pid, connected: false, restarting: false, discoveryReady: true }),
+ })
+ const response = await fetch(`${base}/api/status`)
+ expect(response.status).toBe(200)
+ expect(await response.json()).toEqual({ pid: process.pid, connected: false, restarting: false, discoveryReady: true })
+ })
+})
diff --git a/cli/src/lib/daemonLaunch.spec.ts b/cli/src/lib/daemonLaunch.spec.ts
new file mode 100644
index 000000000..b801a2b94
--- /dev/null
+++ b/cli/src/lib/daemonLaunch.spec.ts
@@ -0,0 +1,122 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
+import type { LaunchDeps } from './daemonLaunch.js'
+
+let dataDir = ''
+
+async function load() {
+ vi.resetModules()
+ process.env.ADAPTER_DATA_DIR = dataDir
+ return import('./daemonLaunch.js')
+}
+
+/** Deterministic deps: a virtual clock that advances on every sleep, a log that tests append to. */
+function fakeDeps(overrides: Partial<LaunchDeps> = {}): LaunchDeps & { log: string[]; clock: { now: number } } {
+ const clock = { now: 1_000_000 }
+ const log: string[] = []
+ return {
+ log,
+ clock,
+ readPid: () => null,
+ isAlive: () => false,
+ readLogSlice: () => log.join('\n'),
+ now: () => clock.now,
+ // Advance the virtual clock and yield a real macrotask, so a test can change state "between" polls.
+ sleep: (ms) => new Promise((r) => { clock.now += ms; setImmediate(r) }),
+ port: 18473,
+ ...overrides,
+ }
+}
+
+describe('connectFailure', () => {
+ it('classifies the log tail', async () => {
+ const { connectFailure } = await load()
+ expect(connectFailure('', 1)).toBeNull()
+ expect(connectFailure('[backend] machine busy', 1)).toMatchObject({ busy: true, fatal: true })
+ expect(connectFailure('Error: listen EADDRINUSE 127.0.0.1:18473', 18473)).toMatchObject({ fatal: true })
+ expect(connectFailure('Error: listen EADDRINUSE 127.0.0.1:18473', 18473)?.detail).toContain('hook port 18473')
+ expect(connectFailure('Unexpected server response: 401', 1)).toMatchObject({ deauth: true, fatal: true })
+ expect(connectFailure('Unexpected server response: 409', 1)).toMatchObject({ busy: true })
+ expect(connectFailure('Unexpected server response: 404', 1)).toMatchObject({ fatal: true, deauth: false })
+ expect(connectFailure('Unexpected server response: 502', 1)).toMatchObject({ fatal: false })
+ expect(connectFailure('getaddrinfo ENOTFOUND x', 1)).toMatchObject({ fatal: true })
+ expect(connectFailure('connect ECONNREFUSED', 1)).toMatchObject({ fatal: false })
+ })
+})
+
+describe('waitForReady', () => {
+ it('returns connected as soon as the marker lands', async () => {
+ const { waitForReady } = await load()
+ const deps = fakeDeps()
+ const p = waitForReady(0, 10_000, deps)
+ deps.log.push('[cli] dialing', '[backend] connected → wss://x')
+ await expect(p).resolves.toEqual({ state: 'connected' })
+ })
+
+ it('keeps waiting through transient failures and reports the last one on timeout', async () => {
+ const { waitForReady } = await load()
+ const deps = fakeDeps()
+ deps.log.push('connect ECONNREFUSED')
+ await expect(waitForReady(0, 2_000, deps)).resolves.toEqual({ state: 'unreachable', detail: 'connection refused' })
+ })
+
+ it('short-circuits on fatal, deauth and busy', async () => {
+ const { waitForReady } = await load()
+ for (const [line, state] of [
+ ['Unexpected server response: 404', 'fatal'],
+ ['Unexpected server response: 403', 'deauth'],
+ ['[backend] machine busy', 'busy'],
+ ] as const) {
+ const deps = fakeDeps()
+ deps.log.push(line)
+ expect((await waitForReady(0, 10_000, deps)).state).toBe(state)
+ }
+ })
+})
+
+describe('waitForBind', () => {
+ it('resolves bound once the pid file names the child', async () => {
+ const { waitForBind } = await load()
+ let pid: number | null = null
+ const deps = fakeDeps({ readPid: () => pid })
+ const p = waitForBind(42, () => false, 60_000, deps)
+ pid = 41 // a stale file naming someone else is not our bind
+ await new Promise((r) => setImmediate(r))
+ await new Promise((r) => setImmediate(r))
+ expect(deps.clock.now).toBeGreaterThan(1_000_000) // polled at least once while it was 41
+ pid = 42
+ await expect(p).resolves.toBe('bound')
+ })
+
+ it('resolves exited when the child dies first', async () => {
+ const { waitForBind } = await load()
+ let gone = false
+ const deps = fakeDeps()
+ const p = waitForBind(42, () => gone, 60_000, deps)
+ gone = true
+ await expect(p).resolves.toBe('exited')
+ })
+
+ it('resolves timeout when nothing happens', async () => {
+ const { waitForBind } = await load()
+ await expect(waitForBind(42, () => false, 5_000, fakeDeps())).resolves.toBe('timeout')
+ })
+})
+
+describe('removePidFileIf', () => {
+ beforeEach(() => { dataDir = mkdtempSync(join(tmpdir(), 'adapter-launch-')) })
+ afterEach(() => { rmSync(dataDir, { recursive: true, force: true }); delete process.env.ADAPTER_DATA_DIR })
+
+ it('removes the file only when it names the given pid', async () => {
+ const { removePidFileIf } = await load()
+ const { PID_FILE } = await import('./daemonState.js')
+ writeFileSync(PID_FILE, '777\n')
+ expect(removePidFileIf(778)).toBe(false)
+ expect(existsSync(PID_FILE)).toBe(true) // another daemon's record survives our failed child
+ expect(removePidFileIf(undefined)).toBe(false)
+ expect(removePidFileIf(777)).toBe(true)
+ expect(existsSync(PID_FILE)).toBe(false)
+ })
+})
diff --git a/cli/src/lib/daemonLaunch.ts b/cli/src/lib/daemonLaunch.ts
new file mode 100644
index 000000000..96ec4731b
--- /dev/null
+++ b/cli/src/lib/daemonLaunch.ts
@@ -0,0 +1,132 @@
+/**
+ * The parts of starting a daemon that decide whether it WORKED — pulled out of cli.ts so they can be
+ * tested without a backend, a bundle, or a real child.
+ *
+ * Two phases, and the split is the point. A spawned child first has to BIND the fixed control port;
+ * only then does it try the backend. The old single wait watched the log for "[backend] connected"
+ * and, when that never came inside ten seconds, gave up with "unreachable" — leaving a child that was
+ * still booting (a slow login shell, tmux adoption) with no pid file and nothing guarding it, which is
+ * exactly when a second `harness start` would spawn a competitor for the port.
+ *
+ * The bind signal is the pid file itself: the daemon writes its own pid there ONLY after the port is
+ * bound (see runForeground), so `readPid() === child.pid` is something only a bound child can
+ * produce. No parsing, no shared-log ambiguity.
+ */
+
+import { readFileSync, rmSync } from 'fs'
+import { PID_FILE, isAlive, readPid } from './daemonState.js'
+
+export interface LaunchDeps {
+ readPid: () => number | null
+ isAlive: (pid: number) => boolean
+ /** The daemon log from a byte offset — decoded AFTER slicing, see waitForReady. */
+ readLogSlice: (sinceOffset: number) => string
+ now: () => number
+ sleep: (ms: number) => Promise<void>
+ /** The fixed control port, for the EADDRINUSE message. */
+ port: number
+}
+
+export function defaultLaunchDeps(logFile: string, port: number): LaunchDeps {
+ return {
+ readPid,
+ isAlive,
+ readLogSlice: (sinceOffset) => {
+ // Slice the raw BYTES from sinceOffset (a Buffer byte length) THEN decode. The log is full of
+ // multi-byte glyphs (→ · ─ ● …), so decoding first and slicing the STRING by that byte count
+ // overshoots (byte length > char length) and returns "" — the classic false "no connection".
+ try { return readFileSync(logFile).subarray(sinceOffset).toString('utf-8') } catch { return '' }
+ },
+ now: () => Date.now(),
+ sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
+ port,
+ }
+}
+
+// 'fatal' = misconfig; retrying is pointless → kill the daemon + error out.
+// 'unreachable' = transient (backend deploying / 5xx / slow / not up yet) → the daemon keeps retrying
+// in the background and connects on its own, so DON'T kill it — just report the state.
+// 'deauth' = the saved credential is invalid/revoked (401/403) → clear it and ask for a fresh token.
+// 'busy' = this machine is already connected from another computer (409) → keep token, stop, inform.
+export interface ReadyResult { state: 'connected' | 'deauth' | 'fatal' | 'unreachable' | 'busy'; detail?: string }
+
+export interface ConnectFailure { detail: string; fatal: boolean; deauth?: boolean; busy?: boolean }
+
+/** Classify a backend connection error from the log tail: a human reason + fatal (won't self-heal) vs
+ * transient (will), and `deauth` for 401/403 (the token is no longer valid). null = no signal yet. */
+export function connectFailure(tail: string, port: number): ConnectFailure | null {
+ // The daemon logs this marker when the backend rejected us with 409 (machine held by another computer).
+ if (tail.includes('[backend] machine busy')) {
+ return { detail: 'this machine is already connected from another computer', fatal: true, busy: true }
+ }
+ // The daemon couldn't bind the (fixed) hook port → another adapter is almost certainly already
+ // running. Fatal: don't sit on "retrying"; tell the user how to find/stop the other one.
+ if (/EADDRINUSE|already in use/.test(tail)) {
+ return { detail: `hook port ${port} is already in use — is another machine daemon running? (harness stop, or lsof -ti :${port} | xargs kill)`, fatal: true }
+ }
+ const m = tail.match(/Unexpected server response: (\d+)/)
+ if (m) {
+ const code = Number(m[1])
+ // 401/403 = this computer's machine was deleted/revoked (or a bad token) → deauth: clear + re-join.
+ // 409 = one machine per machine: already connected from another computer → busy (keep token, stop).
+ // 404 = wrong backend / route not deployed → misconfig (fatal, don't wipe the token).
+ // 5xx (esp. 502/503/504) = gateway up but the app is deploying/restarting → transient, keep retrying.
+ if (code === 409) return { detail: 'this machine is already connected from another computer', fatal: true, busy: true }
+ const deauth = code === 401 || code === 403
+ return { detail: `backend returned HTTP ${code}`, fatal: deauth || code === 404, deauth }
+ }
+ if (/ENOTFOUND|EAI_AGAIN|getaddrinfo/.test(tail)) return { detail: 'host not found (DNS)', fatal: true }
+ if (/certificate|CERT_|self-signed/i.test(tail)) return { detail: 'TLS certificate error', fatal: true }
+ if (/ECONNREFUSED/.test(tail)) return { detail: 'connection refused', fatal: false } // backend not up yet → retry
+ // A 1006 close with no HTTP code — generic "couldn't reach it right now"; transient.
+ if (/disconnected \(close 1006\)/.test(tail)) return { detail: 'cannot reach backend', fatal: false }
+ return null
+}
+
+/** Poll the log until the adapter connects, hits a FATAL error, or the window ends. Transient errors
+ * don't short-circuit — the connection may recover within the window (e.g. a 502 during a deploy). */
+export async function waitForReady(sinceOffset: number, timeoutMs: number, deps: LaunchDeps): Promise<ReadyResult> {
+ const deadline = deps.now() + timeoutMs
+ let lastTransient: string | undefined
+ while (deps.now() < deadline) {
+ const tail = deps.readLogSlice(sinceOffset)
+ if (tail.includes('[backend] connected')) return { state: 'connected' }
+ const fail = connectFailure(tail, deps.port)
+ if (fail?.busy) return { state: 'busy', detail: fail.detail }
+ if (fail?.deauth) return { state: 'deauth', detail: fail.detail }
+ if (fail?.fatal) return { state: 'fatal', detail: fail.detail }
+ if (fail) lastTransient = fail.detail // remember, but keep waiting — it may connect on a retry
+ await deps.sleep(250)
+ }
+ return { state: 'unreachable', detail: lastTransient ?? `no connection within ${timeoutMs / 1000}s` }
+}
+
+/** How long a spawned child gets to bind the control port. Generous on purpose: this covers a cold
+ * login shell and tmux adoption, and nothing else can spawn while the caller holds the spawn lock. */
+export const BIND_WAIT_MS = 60_000
+
+export type BindResult = 'bound' | 'exited' | 'timeout'
+
+/** Wait for the child to claim the pid file — which it does only once the port is bound. */
+export async function waitForBind(
+ childPid: number,
+ exited: () => boolean,
+ timeoutMs: number,
+ deps: LaunchDeps,
+): Promise<BindResult> {
+ const deadline = deps.now() + timeoutMs
+ for (;;) {
+ if (deps.readPid() === childPid) return 'bound'
+ if (exited()) return 'exited'
+ if (deps.now() >= deadline) return 'timeout'
+ await deps.sleep(250)
+ }
+}
+
+/** Remove the pid file only if it still names `pid`. A spawner that deletes unconditionally after ITS
+ * child failed can erase the record of the daemon that actually won the port. */
+export function removePidFileIf(pid: number | undefined, deps: Pick<LaunchDeps, 'readPid'> = { readPid }): boolean {
+ if (pid === undefined || deps.readPid() !== pid) return false
+ try { rmSync(PID_FILE, { force: true }) } catch { /* ignore */ }
+ return true
+}
diff --git a/cli/src/lib/daemonSpawnLock.spec.ts b/cli/src/lib/daemonSpawnLock.spec.ts
new file mode 100644
index 000000000..c5833ea01
--- /dev/null
+++ b/cli/src/lib/daemonSpawnLock.spec.ts
@@ -0,0 +1,208 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { spawn, spawnSync } from 'node:child_process'
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
+
+let dataDir = ''
+
+async function loadLock() {
+ vi.resetModules()
+ process.env.ADAPTER_DATA_DIR = dataDir
+ return import('./daemonSpawnLock.js')
+}
+
+const lockDir = () => join(dataDir, 'adapter.spawn.lock')
+const ownerFile = () => join(lockDir(), 'owner.json')
+
+/** A real process that stays alive until killed — a lock owner whose liveness is not faked. */
+function sleeper(): ReturnType<typeof spawn> {
+ return spawn(process.execPath, ['-e', 'setTimeout(() => {}, 100000)'], { stdio: 'ignore' })
+}
+
+const waitFor = async (predicate: () => boolean, ms = 3000): Promise<void> => {
+ const deadline = Date.now() + ms
+ while (!predicate()) {
+ if (Date.now() > deadline) throw new Error('condition not met in time')
+ await new Promise((r) => setTimeout(r, 20))
+ }
+}
+
+describe('daemon spawn lock', () => {
+ beforeEach(() => {
+ dataDir = mkdtempSync(join(tmpdir(), 'adapter-spawn-lock-'))
+ })
+
+ afterEach(() => {
+ rmSync(dataDir, { recursive: true, force: true })
+ delete process.env.ADAPTER_DATA_DIR
+ })
+
+ it('acquires, records this process as the owner, and releases', async () => {
+ const lock = await loadLock()
+ const release = await lock.acquireSpawnLock('start')
+ expect(existsSync(ownerFile())).toBe(true)
+ const owner = lock.readSpawnLockOwner()
+ expect(owner?.pid).toBe(process.pid)
+ expect(owner?.purpose).toBe('start')
+ release()
+ expect(existsSync(lockDir())).toBe(false)
+ expect(lock.readSpawnLockOwner()).toBeNull()
+ })
+
+ it('is re-entrant within one process and drops the lock only with the outermost release', async () => {
+ const lock = await loadLock()
+ const outer = await lock.acquireSpawnLock('update')
+ const inner = await lock.acquireSpawnLock('start') // update → launch
+ expect(lock.readSpawnLockOwner()?.purpose).toBe('update') // the outer section's identity holds
+ inner()
+ expect(existsSync(lockDir())).toBe(true)
+ outer()
+ expect(existsSync(lockDir())).toBe(false)
+ })
+
+ it('withSpawnLock releases on a throw', async () => {
+ const lock = await loadLock()
+ await expect(lock.withSpawnLock('handoff', async () => { throw new Error('boom') })).rejects.toThrow('boom')
+ expect(existsSync(lockDir())).toBe(false)
+ })
+
+ it('waits for a live holder and reports it once, then acquires when it lets go', async () => {
+ const lock = await loadLock()
+ const holder = sleeper()
+ await waitFor(() => !!holder.pid)
+ mkdirSync(lockDir(), { mode: 0o700 })
+ writeFileSync(ownerFile(), JSON.stringify({
+ pid: holder.pid, startMarker: '', token: 'held-by-child', purpose: 'handoff', since: Date.now(),
+ }), { mode: 0o600 })
+
+ const waiting: unknown[] = []
+ const acquired = lock.acquireSpawnLock('start', { waitMs: 5000, onWaiting: (o) => waiting.push(o) })
+ // Generous: every poll reads the holder's start marker through `ps`, which a loaded box can
+ // take a good fraction of a second over.
+ await waitFor(() => waiting.length > 0, 2000)
+ expect(waiting).toHaveLength(1)
+ expect((waiting[0] as { pid: number }).pid).toBe(holder.pid)
+ expect(readFileSync(ownerFile(), 'utf8')).toContain('held-by-child') // still theirs
+
+ holder.kill('SIGKILL')
+ const release = await acquired
+ expect(lock.readSpawnLockOwner()?.pid).toBe(process.pid)
+ release()
+ })
+
+ it('gives up with SpawnLockBusyError carrying the owner when the holder outlives the wait', async () => {
+ const lock = await loadLock()
+ const holder = sleeper()
+ await waitFor(() => !!holder.pid)
+ mkdirSync(lockDir(), { mode: 0o700 })
+ writeFileSync(ownerFile(), JSON.stringify({
+ pid: holder.pid, startMarker: '', token: 't', purpose: 'update', since: Date.now(),
+ }), { mode: 0o600 })
+ try {
+ await expect(lock.acquireSpawnLock('start', { waitMs: 300 })).rejects.toMatchObject({
+ name: 'SpawnLockBusyError',
+ owner: { pid: holder.pid, purpose: 'update' },
+ })
+ expect(existsSync(ownerFile())).toBe(true)
+ } finally {
+ holder.kill('SIGKILL')
+ }
+ })
+
+ it('reclaims a lock whose owner is dead', async () => {
+ const lock = await loadLock()
+ const dead = spawnSync(process.execPath, ['-e', 'process.exit(0)'])
+ expect(dead.status).toBe(0)
+ mkdirSync(lockDir(), { mode: 0o700 })
+ writeFileSync(ownerFile(), JSON.stringify({
+ pid: 999_999_999, startMarker: '', token: 'stale', purpose: 'start', since: Date.now() - 60_000,
+ }), { mode: 0o600 })
+ const release = await lock.acquireSpawnLock('start', { waitMs: 1000 })
+ expect(lock.readSpawnLockOwner()?.pid).toBe(process.pid)
+ release()
+ })
+
+ it('reclaims a lock whose pid has been reused by another process generation', async () => {
+ const lock = await loadLock()
+ mkdirSync(lockDir(), { mode: 0o700 })
+ writeFileSync(ownerFile(), JSON.stringify({
+ pid: process.pid, startMarker: 'different-process-generation', token: 'stale', purpose: 'start', since: Date.now(),
+ }), { mode: 0o600 })
+ const release = await lock.acquireSpawnLock('start', { waitMs: 1000 })
+ expect(lock.readSpawnLockOwner()?.token).not.toBe('stale')
+ release()
+ })
+
+ it('reclaims an ownerless directory once it is clearly debris, not one being created', async () => {
+ const lock = await loadLock()
+ mkdirSync(lockDir(), { mode: 0o700 })
+ // Fresh and ownerless: someone is between mkdir and the O_EXCL write — do not touch it yet.
+ await expect(lock.acquireSpawnLock('start', { waitMs: 250 })).rejects.toMatchObject({ name: 'SpawnLockBusyError', owner: null })
+ // Age it past the debris threshold.
+ const old = new Date(Date.now() - 10_000)
+ const { utimesSync } = await import('fs')
+ utimesSync(lockDir(), old, old)
+ const release = await lock.acquireSpawnLock('start', { waitMs: 1000 })
+ expect(lock.readSpawnLockOwner()?.pid).toBe(process.pid)
+ release()
+ })
+
+ it('refuses a lock directory with an unsafe mode rather than reading it', async () => {
+ const lock = await loadLock()
+ mkdirSync(lockDir(), { mode: 0o777 })
+ const { chmodSync } = await import('fs')
+ chmodSync(lockDir(), 0o777)
+ writeFileSync(ownerFile(), JSON.stringify({ pid: 1, startMarker: '', token: 't', purpose: 'start', since: 0 }), { mode: 0o600 })
+ expect(() => lock.readSpawnLockOwner()).toThrow(/unsafe/)
+ })
+
+ it('gives up AT ONCE, naming the path, when something that is not a lock sits at the lock path', async () => {
+ // A plain file — from a stray `touch`, a bad restore, whatever. Waiting 45s would never turn it
+ // into a directory, and a stop that could not get past it would be a stop that does not work.
+ const lock = await loadLock()
+ writeFileSync(lockDir(), 'not a directory')
+ const started = Date.now()
+ await expect(lock.acquireSpawnLock('start', { waitMs: 45_000 })).rejects.toMatchObject({
+ name: 'SpawnLockBusyError',
+ owner: null,
+ reason: expect.stringContaining('remove it by hand'),
+ })
+ expect(Date.now() - started).toBeLessThan(2_000)
+ expect(lock.describeSpawnLockFailure(new lock.SpawnLockBusyError(null, 'why'))).toBe('why')
+ })
+
+ it('serializes two real spawners racing for the lock', async () => {
+ // Each child takes the lock, appends "<pid> in", sleeps, appends "<pid> out". Serialized, the
+ // trace is in/out/in/out; a broken lock interleaves in/in.
+ const trace = join(dataDir, 'trace.log')
+ const script = join(dataDir, 'racer.mts') // .mts: the temp dir has no package.json, so .ts would run as CJS
+ const lockModule = join(process.cwd(), 'src', 'lib', 'daemonSpawnLock.ts')
+ writeFileSync(script, `
+ import { appendFileSync } from 'fs'
+ import { withSpawnLock } from ${JSON.stringify(lockModule)}
+ await withSpawnLock('start', async () => {
+ appendFileSync(${JSON.stringify(trace)}, process.pid + ' in\\n')
+ await new Promise((r) => setTimeout(r, 400))
+ appendFileSync(${JSON.stringify(trace)}, process.pid + ' out\\n')
+ })
+ `)
+ const tsx = join(process.cwd(), 'node_modules', '.bin', 'tsx')
+ const run = () => new Promise<number | null>((resolve) => {
+ const c = spawn(tsx, [script], { env: { ...process.env, ADAPTER_DATA_DIR: dataDir }, stdio: 'inherit' })
+ c.on('exit', (code) => resolve(code))
+ })
+ const codes = await Promise.all([run(), run(), run()])
+ expect(codes).toEqual([0, 0, 0])
+ const lines = readFileSync(trace, 'utf8').trim().split('\n')
+ expect(lines).toHaveLength(6)
+ for (let i = 0; i < lines.length; i += 2) {
+ const [inPid, inWord] = lines[i].split(' ')
+ const [outPid, outWord] = lines[i + 1].split(' ')
+ expect(inWord).toBe('in')
+ expect(outWord).toBe('out')
+ expect(outPid).toBe(inPid)
+ }
+ expect(existsSync(lockDir())).toBe(false)
+ }, 30_000)
+})
diff --git a/cli/src/lib/daemonSpawnLock.ts b/cli/src/lib/daemonSpawnLock.ts
new file mode 100644
index 000000000..4bd506e1c
--- /dev/null
+++ b/cli/src/lib/daemonSpawnLock.ts
@@ -0,0 +1,283 @@
+/**
+ * ONE process may spawn, hand off, or stop the daemon at a time.
+ *
+ * Three things used to do it with no coordination: `harness start` (the desktop app re-runs it every
+ * few seconds whenever the daemon looks down), the daemon's own self-update handoff, and
+ * `harness update`. The control port is fixed, so two spawners in the same second is a coin toss for
+ * which child binds — and the loser, having already written its pid, either left `adapter.pid`
+ * naming a corpse or deleted the file that named the winner. That is the orphan on :18473 that
+ * `harness stop` cannot see and every later `harness start` trips over.
+ *
+ * Same shape as the registry lock (a directory created with mkdir + an `owner.json` written O_EXCL,
+ * reclaimed only when the owning process is provably gone), with three differences it needs:
+ * - waiting is ASYNC — the daemon takes this lock on its own event loop during a handoff, and the
+ * registry's `Atomics.wait` would freeze it;
+ * - it is RE-ENTRANT within a process — `harness update` calls `launch()`, `harness start` stages
+ * the bundle and then calls `launch()`, and the updater's tick runs the whole handoff — each of
+ * those is one critical section, not two;
+ * - it is released on `process.exit`, because the CLI exits from a dozen places inside the section.
+ *
+ * There is deliberately NO age-based staleness. A holder is a holder for as long as its process is
+ * alive — a handoff supervises the new build for up to a minute and a download on a slow link can
+ * take longer — and the only thing an age bound would buy is letting a second spawner in while the
+ * first is still working, which is the bug this exists to prevent. The escape hatch is
+ * `harness stop`, which waits its turn and then proceeds regardless.
+ */
+
+import {
+ closeSync,
+ constants,
+ fsyncSync,
+ lstatSync,
+ mkdirSync,
+ openSync,
+ readFileSync,
+ rmSync,
+ writeFileSync,
+} from 'fs'
+import { randomUUID } from 'crypto'
+import { join } from 'path'
+import { env } from '../config/env.js'
+import { lockOwnerAlive, processStartMarker } from './processLiveness.js'
+import { secureStateDirectory } from './secureState.js'
+
+export type SpawnLockPurpose = 'start' | 'update' | 'handoff' | 'stop'
+
+export interface SpawnLockOwner {
+ pid: number
+ startMarker: string
+ token: string
+ purpose: SpawnLockPurpose
+ since: number
+}
+
+export const SPAWN_LOCK_DIR = join(env.ADAPTER_DATA_DIR, 'adapter.spawn.lock')
+const OWNER_FILE = 'owner.json'
+const POLL_MS = 100
+export const SPAWN_LOCK_WAIT_MS = 45_000
+/** A directory with no owner.json is a crash between mkdir and the O_EXCL write — a window of
+ * microseconds. Anything older than this with no owner is debris, not a lock. */
+const OWNERLESS_STALE_MS = 5_000
+
+/**
+ * The lock could not be taken. `owner` names a live holder when there is one; `reason` is set instead
+ * when the lock cannot be taken for a reason waiting will not fix — something that is not a lock this
+ * CLI made sits at the lock path — so the message can tell a human what to remove.
+ */
+export class SpawnLockBusyError extends Error {
+ constructor(readonly owner: SpawnLockOwner | null, readonly reason?: string) {
+ super(
+ reason ?? (owner ? `daemon spawn lock is held (${describeSpawnLockOwner(owner)})` : 'daemon spawn lock is busy'),
+ )
+ this.name = 'SpawnLockBusyError'
+ }
+}
+
+/** What to say about a failed acquire, for a command that has to explain itself and move on. */
+export function describeSpawnLockFailure(error: unknown): string {
+ if (error instanceof SpawnLockBusyError) {
+ if (error.reason) return error.reason
+ return error.owner ? `held — the daemon is ${describeSpawnLockOwner(error.owner)}` : 'held by an unknown process'
+ }
+ return `unusable (${error instanceof Error ? error.message : String(error)})`
+}
+
+export function describeSpawnLockOwner(owner: SpawnLockOwner): string {
+ const verb: Record<SpawnLockPurpose, string> = {
+ start: 'being started', update: 'being updated', handoff: 'restarting for an update', stop: 'being stopped',
+ }
+ const secs = Math.max(0, Math.round((Date.now() - owner.since) / 1000))
+ return `${verb[owner.purpose]} by pid ${owner.pid} for ${secs}s`
+}
+
+let held: { token: string; purpose: SpawnLockPurpose; depth: number } | null = null
+let exitHookInstalled = false
+
+function ownerPath(): string { return join(SPAWN_LOCK_DIR, OWNER_FILE) }
+
+function uid(): number | null {
+ return typeof process.getuid === 'function' ? process.getuid() : null
+}
+
+/**
+ * Read the owner record, refusing anything an attacker could have planted (a symlink, another
+ * account's directory, a loose mode). Returns null when there is no readable owner — which callers
+ * must NOT take as "free": it is also what a lock mid-creation looks like.
+ */
+export function readSpawnLockOwner(): SpawnLockOwner | null {
+ const me = uid()
+ try {
+ const dir = lstatSync(SPAWN_LOCK_DIR)
+ if (!dir.isDirectory() || dir.isSymbolicLink() || (me !== null && dir.uid !== me) || (dir.mode & 0o777) !== 0o700) {
+ throw new Error(`daemon spawn lock ${SPAWN_LOCK_DIR} has an unsafe owner, mode, or type`)
+ }
+ const file = lstatSync(ownerPath())
+ if (!file.isFile() || file.isSymbolicLink() || (me !== null && file.uid !== me) || (file.mode & 0o777) !== 0o600) {
+ throw new Error(`daemon spawn lock owner ${ownerPath()} has an unsafe owner, mode, or type`)
+ }
+ const raw = JSON.parse(readFileSync(ownerPath(), 'utf8')) as Partial<Record<keyof SpawnLockOwner, unknown>>
+ const pid = Number(raw.pid)
+ if (!Number.isSafeInteger(pid) || pid <= 0 || typeof raw.token !== 'string' || !raw.token) return null
+ return {
+ pid,
+ startMarker: typeof raw.startMarker === 'string' ? raw.startMarker : '',
+ token: raw.token,
+ purpose: isPurpose(raw.purpose) ? raw.purpose : 'start',
+ since: Number.isFinite(Number(raw.since)) ? Number(raw.since) : 0,
+ }
+ } catch (error) {
+ if (error instanceof Error && error.message.startsWith('daemon spawn lock')) throw error
+ return null
+ }
+}
+
+function isPurpose(value: unknown): value is SpawnLockPurpose {
+ return value === 'start' || value === 'update' || value === 'handoff' || value === 'stop'
+}
+
+/** Create the lock for this process. Returns the token, or null when someone else holds it. */
+function tryCreate(purpose: SpawnLockPurpose): string | null {
+ secureStateDirectory(env.ADAPTER_DATA_DIR)
+ const token = randomUUID()
+ let created = false
+ try {
+ mkdirSync(SPAWN_LOCK_DIR, { mode: 0o700 })
+ created = true
+ const fd = openSync(ownerPath(), constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600)
+ try {
+ const owner: SpawnLockOwner = {
+ pid: process.pid, startMarker: processStartMarker(process.pid) ?? '', token, purpose, since: Date.now(),
+ }
+ writeFileSync(fd, JSON.stringify(owner))
+ fsyncSync(fd)
+ } finally { closeSync(fd) }
+ return token
+ } catch (error) {
+ if (created) releaseOwnedBy(token)
+ if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error
+ return null
+ }
+}
+
+function releaseOwnedBy(token: string): void {
+ try {
+ const saved = JSON.parse(readFileSync(ownerPath(), 'utf8')) as { token?: unknown }
+ if (saved.token === token) rmSync(SPAWN_LOCK_DIR, { recursive: true, force: true })
+ } catch {
+ // A directory we created but never got to write an owner into is ours to remove; anything else
+ // belongs to someone.
+ try {
+ lstatSync(ownerPath())
+ } catch (probe) {
+ if ((probe as NodeJS.ErrnoException).code === 'ENOENT') rmSync(SPAWN_LOCK_DIR, { recursive: true, force: true })
+ }
+ }
+}
+
+/** Remove a lock whose owner is gone — re-read first so a lock that changed hands meanwhile survives. */
+function reclaimIfStale(owner: SpawnLockOwner): boolean {
+ if (lockOwnerAlive(owner.pid, owner.startMarker)) return false
+ try {
+ const current = readSpawnLockOwner()
+ if (current && current.pid === owner.pid && current.startMarker === owner.startMarker
+ && current.token === owner.token && !lockOwnerAlive(owner.pid, owner.startMarker)) {
+ rmSync(SPAWN_LOCK_DIR, { recursive: true, force: true })
+ return true
+ }
+ } catch { /* changed or vanished under us; let the loop look again */ }
+ return false
+}
+
+function reclaimIfOwnerless(): boolean {
+ try {
+ const dir = lstatSync(SPAWN_LOCK_DIR)
+ try { lstatSync(ownerPath()); return false } catch { /* no owner file */ }
+ if (Date.now() - dir.mtimeMs > OWNERLESS_STALE_MS) {
+ rmSync(SPAWN_LOCK_DIR, { recursive: true, force: true })
+ return true
+ }
+ } catch { /* gone already */ }
+ return false
+}
+
+function releaseSync(): void {
+ const mine = held
+ held = null
+ if (mine) releaseOwnedBy(mine.token)
+}
+
+function installExitHook(): void {
+ if (exitHookInstalled) return
+ exitHookInstalled = true
+ process.once('exit', releaseSync)
+}
+
+const sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms))
+
+/**
+ * Take the lock, waiting up to `waitMs` for a live holder to finish. Resolves to the release
+ * function. Re-entrant: a process that already holds it gets a nested handle immediately, and the
+ * lock is dropped when the outermost handle releases.
+ *
+ * `onWaiting` fires once, the first time a live holder is seen, so a command can say who it is
+ * waiting for rather than sitting silent for most of a minute.
+ */
+export async function acquireSpawnLock(
+ purpose: SpawnLockPurpose,
+ opts: { waitMs?: number; onWaiting?: (owner: SpawnLockOwner) => void } = {},
+): Promise<() => void> {
+ if (held) {
+ held.depth += 1
+ return release
+ }
+ const waitMs = opts.waitMs ?? SPAWN_LOCK_WAIT_MS
+ const deadline = Date.now() + waitMs
+ let announced = false
+ for (;;) {
+ const token = tryCreate(purpose)
+ if (token) {
+ held = { token, purpose, depth: 1 }
+ installExitHook()
+ return release
+ }
+ let owner: SpawnLockOwner | null
+ try {
+ owner = readSpawnLockOwner()
+ } catch (error) {
+ // A symlink, another account's directory, a plain file where the directory should be: not a
+ // lock this CLI made, and not one waiting will ever turn into one. Say what is there, now —
+ // 45 seconds of silence before the same answer helps nobody.
+ throw new SpawnLockBusyError(
+ null,
+ `not a lock this CLI made: ${SPAWN_LOCK_DIR} — remove it by hand (${error instanceof Error ? error.message : error})`,
+ )
+ }
+ if (owner ? reclaimIfStale(owner) : reclaimIfOwnerless()) continue
+ if (owner && !announced) {
+ announced = true
+ opts.onWaiting?.(owner)
+ }
+ if (Date.now() >= deadline) throw new SpawnLockBusyError(owner)
+ await sleep(POLL_MS)
+ }
+}
+
+function release(): void {
+ if (!held) return
+ held.depth -= 1
+ if (held.depth > 0) return
+ releaseSync()
+}
+
+export async function withSpawnLock<T>(
+ purpose: SpawnLockPurpose,
+ fn: () => Promise<T>,
+ opts: { waitMs?: number; onWaiting?: (owner: SpawnLockOwner) => void } = {},
+): Promise<T> {
+ const done = await acquireSpawnLock(purpose, opts)
+ try {
+ return await fn()
+ } finally {
+ done()
+ }
+}
diff --git a/cli/src/lib/daemonStop.spec.ts b/cli/src/lib/daemonStop.spec.ts
new file mode 100644
index 000000000..21651499c
--- /dev/null
+++ b/cli/src/lib/daemonStop.spec.ts
@@ -0,0 +1,76 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
+import type { StopDeps } from './daemonStop.js'
+
+let dataDir = ''
+
+async function load() {
+ vi.resetModules()
+ process.env.ADAPTER_DATA_DIR = dataDir
+ return import('./daemonStop.js')
+}
+
+function fakeDeps(state: { pid: number | null; alive: Set<number>; killed: string[]; locked: number; released: number }, overrides: Partial<StopDeps> = {}): StopDeps {
+ const clock = { now: 0 }
+ return {
+ readPid: () => state.pid,
+ isAlive: (pid) => state.alive.has(pid),
+ kill: (pid, signal) => {
+ state.killed.push(`${pid}:${signal}`)
+ if (signal === 'SIGKILL') state.alive.delete(pid)
+ },
+ sleep: async (ms) => { clock.now += ms },
+ now: () => clock.now,
+ lock: async () => { state.locked += 1; return () => { state.released += 1 } },
+ warn: () => {},
+ ...overrides,
+ }
+}
+
+describe('stopDaemonProcess', () => {
+ beforeEach(() => { dataDir = mkdtempSync(join(tmpdir(), 'adapter-stop-')) })
+ afterEach(() => { rmSync(dataDir, { recursive: true, force: true }); delete process.env.ADAPTER_DATA_DIR })
+
+ it('takes the spawn lock, SIGTERMs, escalates to SIGKILL, and clears the pid file it stopped', async () => {
+ const { stopDaemonProcess } = await load()
+ const { PID_FILE } = await import('./daemonState.js')
+ writeFileSync(PID_FILE, '500\n')
+ const state = { pid: 500, alive: new Set([500]), killed: [] as string[], locked: 0, released: 0 }
+ const r = await stopDaemonProcess(fakeDeps(state))
+ expect(r).toEqual({ pid: 500, stopped: true })
+ expect(state.killed).toEqual(['500:SIGTERM', '500:SIGKILL'])
+ expect(state.locked).toBe(1)
+ expect(state.released).toBe(1)
+ expect(existsSync(PID_FILE)).toBe(false)
+ })
+
+ it('does not delete a pid file that a different daemon wrote meanwhile', async () => {
+ const { stopDaemonProcess } = await load()
+ const { PID_FILE } = await import('./daemonState.js')
+ writeFileSync(PID_FILE, '500\n')
+ const state = { pid: 500, alive: new Set([500]), killed: [] as string[], locked: 0, released: 0 }
+ const deps = fakeDeps(state, {
+ kill: (pid, signal) => {
+ state.killed.push(`${pid}:${signal}`)
+ state.alive.delete(pid)
+ state.pid = 501 // the file now names someone else
+ writeFileSync(PID_FILE, '501\n')
+ },
+ })
+ await stopDaemonProcess(deps)
+ expect(state.killed).toEqual(['500:SIGTERM'])
+ expect(existsSync(PID_FILE)).toBe(true)
+ })
+
+ it('clears a pid file naming a dead process and reports nothing stopped', async () => {
+ const { stopDaemonProcess } = await load()
+ const { PID_FILE } = await import('./daemonState.js')
+ writeFileSync(PID_FILE, '9\n')
+ const state = { pid: 9, alive: new Set<number>(), killed: [] as string[], locked: 0, released: 0 }
+ await expect(stopDaemonProcess(fakeDeps(state))).resolves.toEqual({ pid: null, stopped: false })
+ expect(existsSync(PID_FILE)).toBe(false)
+ expect(state.released).toBe(1)
+ })
+})
diff --git a/cli/src/lib/daemonStop.ts b/cli/src/lib/daemonStop.ts
new file mode 100644
index 000000000..23b7c973a
--- /dev/null
+++ b/cli/src/lib/daemonStop.ts
@@ -0,0 +1,74 @@
+/**
+ * `harness stop`'s core, shared by `stop`, `reset`, `logout`, `update` and `flash`.
+ *
+ * Takes the spawn lock first, so a stop can never land in the middle of an update handoff — the one
+ * window where "the daemon" is two processes and killing the pid on file leaves the other one
+ * running. It waits its turn, but not forever: stop is the escape hatch for everything else here, so
+ * a holder that has been busy for longer than the wait is reported and then overridden.
+ */
+
+import { rmSync } from 'fs'
+import { PID_FILE, isAlive, readPid } from './daemonState.js'
+import { acquireSpawnLock, describeSpawnLockFailure, describeSpawnLockOwner, SPAWN_LOCK_WAIT_MS } from './daemonSpawnLock.js'
+
+export interface StopDeps {
+ readPid: () => number | null
+ isAlive: (pid: number) => boolean
+ kill: (pid: number, signal: NodeJS.Signals) => void
+ sleep: (ms: number) => Promise<void>
+ now: () => number
+ /** Acquire the spawn lock; resolves to a release. Injected so the stop sequence is testable alone. */
+ lock: () => Promise<() => void>
+ warn: (message: string) => void
+}
+
+export function defaultStopDeps(): StopDeps {
+ return {
+ readPid,
+ isAlive,
+ kill: (pid, signal) => process.kill(pid, signal),
+ sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
+ now: () => Date.now(),
+ lock: async () => {
+ try {
+ return await acquireSpawnLock('stop', {
+ waitMs: SPAWN_LOCK_WAIT_MS,
+ onWaiting: (owner) => console.log(` the daemon is ${describeSpawnLockOwner(owner)} — waiting for it to finish…`),
+ })
+ } catch (error) {
+ // ANY failure to take the lock — a holder that outlived the wait, a directory that is not a
+ // lock at all — ends the same way: stop proceeds. It is the escape hatch for everything else
+ // here, and an escape hatch that can be locked is not one.
+ console.warn(` ! the daemon spawn lock is ${describeSpawnLockFailure(error)} — stopping anyway`)
+ return () => {}
+ }
+ },
+ warn: (m) => console.warn(m),
+ }
+}
+
+export const STOP_GRACE_MS = 3_000
+
+/** SIGTERM the daemon named by the pid file, SIGKILL if it lingers. Returns what was stopped. */
+export async function stopDaemonProcess(deps: StopDeps = defaultStopDeps()): Promise<{ pid: number | null; stopped: boolean }> {
+ const release = await deps.lock()
+ try {
+ const pid = deps.readPid()
+ if (!pid || !deps.isAlive(pid)) {
+ // A pid file naming nothing is debris from a crash; nobody else can be relying on it.
+ try { rmSync(PID_FILE, { force: true }) } catch { /* ignore */ }
+ return { pid: null, stopped: false }
+ }
+ try { deps.kill(pid, 'SIGTERM') } catch { /* already gone */ }
+ const deadline = deps.now() + STOP_GRACE_MS
+ while (deps.now() < deadline && deps.isAlive(pid)) await deps.sleep(150)
+ if (deps.isAlive(pid)) { try { deps.kill(pid, 'SIGKILL') } catch { /* ignore */ } }
+ // Only if it is still OUR daemon's file: a SIGTERM'd daemon removes its own pid on the way out
+ // (shutdown), and a daemon that came up meanwhile — impossible under the lock, but cheap to
+ // respect — must not lose its record to us.
+ if (deps.readPid() === pid) { try { rmSync(PID_FILE, { force: true }) } catch { /* ignore */ } }
+ return { pid, stopped: true }
+ } finally {
+ release()
+ }
+}
diff --git a/cli/src/lib/processLiveness.ts b/cli/src/lib/processLiveness.ts
new file mode 100644
index 000000000..e6c01f7c3
--- /dev/null
+++ b/cli/src/lib/processLiveness.ts
@@ -0,0 +1,42 @@
+/**
+ * "Is the process that wrote this lock still the process that wrote it?" — shared by every on-disk
+ * lock the CLI keeps (registry.json.lock, adapter.spawn.lock).
+ *
+ * A bare pid is not enough: pids are recycled, so a crashed owner's pid can belong to an unrelated
+ * process by the time anyone checks. The start marker pins the process GENERATION — Linux exposes it
+ * in /proc/<pid>/stat (field 22, starttime in clock ticks); elsewhere `ps -o lstart` gives it to the
+ * second, which is coarse but has never collided in practice. A marker that cannot be read at all
+ * degrades to "alive if the pid exists", which is what Windows gets.
+ */
+
+import { readFileSync } from 'fs'
+import { execFileSync } from 'node:child_process'
+
+export function processExists(pid: number): boolean {
+ if (!Number.isSafeInteger(pid) || pid <= 0) return false
+ try { process.kill(pid, 0); return true } catch (error) {
+ return (error as NodeJS.ErrnoException).code === 'EPERM'
+ }
+}
+
+export function processStartMarker(pid: number): string | null {
+ if (!Number.isSafeInteger(pid) || pid <= 0) return null
+ try {
+ const stat = readFileSync(`/proc/${pid}/stat`, 'utf8')
+ const fields = stat.slice(stat.lastIndexOf(')') + 2).trim().split(/\s+/)
+ if (fields[19]) return `linux:${fields[19]}`
+ } catch { /* non-Linux or exited process; use ps below */ }
+ try {
+ const started = execFileSync('ps', ['-p', String(pid), '-o', 'lstart='], {
+ encoding: 'utf8', timeout: 1_000,
+ }).trim()
+ return started ? `ps:${started}` : null
+ } catch { return null }
+}
+
+export function lockOwnerAlive(pid: number, startMarker: string): boolean {
+ if (!processExists(pid)) return false
+ if (!startMarker) return true
+ const current = processStartMarker(pid)
+ return current === null || current === startMarker
+}
diff --git a/cli/src/lib/registry.ts b/cli/src/lib/registry.ts
index 3842c07ad..109ded03c 100644
--- a/cli/src/lib/registry.ts
+++ b/cli/src/lib/registry.ts
@@ -29,7 +29,6 @@ import {
statSync,
writeFileSync,
} from 'fs'
-import { execFileSync } from 'node:child_process'
import { randomUUID } from 'crypto'
import { join, basename, dirname, relative } from 'path'
import { hostname, uptime } from 'os'
@@ -40,6 +39,7 @@ import type { GridAssignment } from './gridAssignment.js'
import { commandcodeTranscriptPath } from '../engines/commandcode/transcript.js'
import { agyTranscriptPath } from '../engines/agy/session.js'
import { copilotTranscriptPath } from '../engines/copilot/session.js'
+import { lockOwnerAlive, processStartMarker } from './processLiveness.js'
import { hardenPrivateStateFileIfPresent, readPrivateStateFile, secureStateDirectory } from './secureState.js'
import { mergeTerminalRuntimes, processIdentityKey, terminalPlacementKey, terminalRouteKey } from './terminalRuntime.js'
import type { HookTerminalHint, ProcessIdentity, TerminalRuntimeRef } from './terminalTypes.js'
@@ -180,35 +180,6 @@ function sleepSync(ms: number): void {
Atomics.wait(view, 0, 0, ms)
}
-function processExists(pid: number): boolean {
- if (!Number.isSafeInteger(pid) || pid <= 0) return false
- try { process.kill(pid, 0); return true } catch (error) {
- return (error as NodeJS.ErrnoException).code === 'EPERM'
- }
-}
-
-function processStartMarker(pid: number): string | null {
- if (!Number.isSafeInteger(pid) || pid <= 0) return null
- try {
- const stat = readFileSync(`/proc/${pid}/stat`, 'utf8')
- const fields = stat.slice(stat.lastIndexOf(')') + 2).trim().split(/\s+/)
- if (fields[19]) return `linux:${fields[19]}`
- } catch { /* non-Linux or exited process; use ps below */ }
- try {
- const started = execFileSync('ps', ['-p', String(pid), '-o', 'lstart='], {
- encoding: 'utf8', timeout: 1_000,
- }).trim()
- return started ? `ps:${started}` : null
- } catch { return null }
-}
-
-function lockOwnerAlive(pid: number, startMarker: string): boolean {
- if (!processExists(pid)) return false
- if (!startMarker) return true
- const current = processStartMarker(pid)
- return current === null || current === startMarker
-}
-
function removeRegistryLockOwnedBy(token: string): void {
try {
const saved = JSON.parse(readFileSync(join(LOCK_DIR, 'owner.json'), 'utf8')) as { token?: unknown }
diff --git a/cli/src/lib/selfUpdate.spec.ts b/cli/src/lib/selfUpdate.spec.ts
index 2619c1038..8fc1e7407 100644
--- a/cli/src/lib/selfUpdate.spec.ts
+++ b/cli/src/lib/selfUpdate.spec.ts
@@ -1,8 +1,9 @@
import { mkdtempSync, readFileSync, rmSync } from 'fs'
import { tmpdir } from 'os'
import { join } from 'path'
-import { afterEach, describe, expect, it } from 'vitest'
-import { canary, isLocalDevBuild, semverGt, shouldAutoUpdate, stage } from './selfUpdate.js'
+import { createHash } from 'crypto'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { canary, isLocalDevBuild, semverGt, shouldAutoUpdate, stage, startSelfUpdater } from './selfUpdate.js'
let dirs: string[] = []
@@ -58,3 +59,85 @@ describe('shouldAutoUpdate', () => {
expect(shouldAutoUpdate('0.1.55', '0.1.56')).toBe(false)
})
})
+
+describe('startSelfUpdater', () => {
+ const cliSource = Buffer.from('#!/usr/bin/env node\nimport { createRequire } from "module";\nconsole.log(createRequire(import.meta.url) ? "9.9.9" : "nope")\n')
+ const notifySource = Buffer.from('export {}\n')
+ const sha = (b: Buffer): string => createHash('sha256').update(b).digest('hex')
+
+ afterEach(() => {
+ vi.unstubAllGlobals()
+ for (const dir of dirs) rmSync(dir, { recursive: true, force: true })
+ dirs = []
+ })
+
+ function serveUpdate(): void {
+ vi.stubGlobal('fetch', async (url: string) => {
+ if (url === 'https://updates.test/metadata.json') {
+ return new Response(JSON.stringify({
+ adapter: {
+ version: '9.9.9',
+ cli: { url: 'https://updates.test/cli.js', sha256: sha(cliSource) },
+ notify: { url: 'https://updates.test/notify.mjs', sha256: sha(notifySource) },
+ },
+ }))
+ }
+ if (url === 'https://updates.test/cli.js') return new Response(cliSource)
+ if (url === 'https://updates.test/notify.mjs') return new Response(notifySource)
+ return new Response('', { status: 404 })
+ })
+ }
+
+ it('swaps the bytes and awaits onStaged inside ONE withLock section', async () => {
+ serveUpdate()
+ const dir = tempDir()
+ const events: string[] = []
+ let stagedResolve: () => void = () => {}
+ const stagedDone = new Promise<void>((r) => { stagedResolve = r })
+ const poller = startSelfUpdater({
+ currentVersion: '1.0.0',
+ url: 'https://updates.test/metadata.json',
+ key: 'adapter',
+ dir,
+ intervalMs: 60_000,
+ withLock: async (fn) => {
+ events.push('lock')
+ try { return await fn() } finally { events.push('unlock') }
+ },
+ onStaged: async (v) => {
+ events.push(`staged:${v}`)
+ expect(readFileSync(join(dir, 'cli.js'))).toEqual(cliSource) // swapped BEFORE the handoff runs
+ await new Promise((r) => setTimeout(r, 30)) // the handoff takes time…
+ events.push('handoff-done')
+ stagedResolve()
+ },
+ })
+ await stagedDone
+ await new Promise((r) => setTimeout(r, 10))
+ poller.stop()
+ expect(events).toEqual(['lock', 'staged:9.9.9', 'handoff-done', 'unlock'])
+ })
+
+ it('releases the lock when onStaged throws', async () => {
+ serveUpdate()
+ const dir = tempDir()
+ const events: string[] = []
+ let sawThrow: () => void = () => {}
+ const thrown = new Promise<void>((r) => { sawThrow = r })
+ const poller = startSelfUpdater({
+ currentVersion: '1.0.0',
+ url: 'https://updates.test/metadata.json',
+ key: 'adapter',
+ dir,
+ intervalMs: 60_000,
+ withLock: async (fn) => {
+ events.push('lock')
+ try { return await fn() } finally { events.push('unlock'); sawThrow() }
+ },
+ onStaged: async () => { throw new Error('teardown I/O fault') },
+ })
+ await thrown
+ poller.stop()
+ expect(events).toEqual(['lock', 'unlock'])
+ })
+})
diff --git a/cli/src/lib/selfUpdate.ts b/cli/src/lib/selfUpdate.ts
index 956c78ca3..9419099ff 100644
--- a/cli/src/lib/selfUpdate.ts
+++ b/cli/src/lib/selfUpdate.ts
@@ -17,6 +17,7 @@ import { createHash } from 'crypto'
import { copyFileSync, existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'fs'
import { spawnSync } from 'child_process'
import { join } from 'path'
+import { SpawnLockBusyError, describeSpawnLockFailure } from './daemonSpawnLock.js'
import { managedNodePath } from './nodeRuntime.js'
export interface FileRef {
@@ -166,11 +167,21 @@ export function startSelfUpdater(opts: {
key: string
dir: string
intervalMs: number
- onStaged: (version: string) => void
+ /** Awaited: the section from the byte swap through whatever `onStaged` does (the daemon's restart
+ * handoff) is ONE critical section, and the lock `withLock` takes must outlive all of it. */
+ onStaged: (version: string) => void | Promise<void>
+ /** Wrap the swap + `onStaged` in a mutual exclusion with every other process that writes the
+ * bundle or spawns the daemon. Default: none (tests, and callers that hold their own). */
+ withLock?: <T>(fn: () => Promise<T>) => Promise<T>
}): Poller {
let checking = false
let done = false
let timer: NodeJS.Timeout | null = null
+ const withLock = opts.withLock ?? ((fn) => fn())
+ // The bytes of a build we have already downloaded, verified and canaried, kept across ticks: when
+ // the lock was busy (a `harness start` or `harness update` mid-flight) the next tick should try the
+ // swap again, not the whole download.
+ let verified: { version: string; cliBuf: Buffer; notifyBuf: Buffer } | null = null
const stop = (): void => { if (timer) { clearInterval(timer); timer = null } }
@@ -180,17 +191,31 @@ export function startSelfUpdater(opts: {
try {
const entry = await fetchManifest(opts.url, opts.key)
if (!entry || !shouldAutoUpdate(entry.version, opts.currentVersion)) return
- console.log(`[update] newer build available: ${opts.currentVersion} → ${entry.version}`)
- const cliBuf = await downloadVerified(entry.cli)
- const notifyBuf = await downloadVerified(entry.notify)
- if (!canary(cliBuf, opts.dir)) { console.error('[update] canary failed for the new build — skipping'); return }
- stage(opts.dir, cliBuf, notifyBuf)
- done = true
- stop()
- console.log(`[update] staged ${entry.version} — restarting now`)
- opts.onStaged(entry.version)
+ if (verified?.version !== entry.version) {
+ console.log(`[update] newer build available: ${opts.currentVersion} → ${entry.version}`)
+ const cliBuf = await downloadVerified(entry.cli)
+ const notifyBuf = await downloadVerified(entry.notify)
+ if (!canary(cliBuf, opts.dir)) { console.error('[update] canary failed for the new build — skipping'); return }
+ verified = { version: entry.version, cliBuf, notifyBuf }
+ }
+ const ready = verified
+ // Downloaded and verified OUTSIDE the lock (that can take a while on a slow link and touches
+ // nothing shared); swapped and handed off INSIDE it.
+ await withLock(async () => {
+ stage(opts.dir, ready.cliBuf, ready.notifyBuf)
+ done = true
+ stop()
+ console.log(`[update] staged ${ready.version} — restarting now`)
+ await opts.onStaged(ready.version)
+ })
} catch (err) {
- console.error('[update] check failed (will retry):', err instanceof Error ? err.message : err)
+ // A busy lock is not a failed check: the bytes are good and waiting, and the next tick tries
+ // the swap again. Say so, or a minute of "check failed" reads as the updater being broken.
+ if (err instanceof SpawnLockBusyError) {
+ console.log(`[update] ${verified?.version ?? 'a new build'} is ready but the daemon spawn lock is ${describeSpawnLockFailure(err)} — trying again next check`)
+ } else {
+ console.error('[update] check failed (will retry):', err instanceof Error ? err.message : err)
+ }
} finally {
checking = false
}