Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: add daemon safe mode and boot handoff mechanism for safer updates

0xken15929fd1ce6334
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index d881d18dc..94131a96c 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -28,7 +28,7 @@ import { readFileSync, writeFileSync, mkdirSync, openSync, existsSync, rmSync, s
import { join, resolve } from 'path'
import { fileURLToPath } from 'url'
import { spawn } from 'child_process'
-import { createServer } from 'http'
+import { createServer, type Server } from 'http'
import { createInterface, emitKeypressEvents } from 'readline'
import { homedir, hostname } from 'os'
import { env } from './config/env.js'
@@ -50,12 +50,13 @@ import { registry, projectDisplayName, sessionDisplayTitle, type RegisteredSessi
import { engineSessionTitle } from './lib/sessionTitle.js'
import { installAmpPlugin, installCodexHooks, installCommandCodeHooks, installCursorHooks, installDevinHooks, installGrokHooks, installAgyHooks, installCopilotHooks, installHermesHooks, installKiloPlugin, installOpencodePlugin, installPiExtension, installSessionHooks } from './lib/hooks.js'
import { PID_FILE, daemonPort, isAlive, isDaemonRunning, readPid } from './lib/daemonState.js'
+import { clearSafeModeMarker, readSafeModeMarker, runBootHandoff, safeModeDisposition, safeModeStatusBody, writeSafeModeMarker } from './lib/daemonSafeMode.js'
import {
BIND_WAIT_MS, connectFailure, defaultLaunchDeps, removePidFileIf, waitForBind, waitForReady,
} from './lib/daemonLaunch.js'
import { SpawnLockBusyError, describeSpawnLockBusyPlainly, describeSpawnLockFailure, describeSpawnLockOwner, withSpawnLock } from './lib/daemonSpawnLock.js'
import { stopDaemonProcess } from './lib/daemonStop.js'
-import { ensureTmuxOnPath, requireTmuxAvailable } from './lib/tmuxOnPath.js'
+import { ensureTmuxOnPath } from './lib/tmuxOnPath.js'
import { flashCommand } from './lib/flash.js'
import { readOrMintComputerId } from './lib/computerIdentity.js'
import { awaitLoginCallback, extractCallbackParams, LOGIN_TIMEOUT_MESSAGE } from './lib/loginCallback.js'
@@ -433,6 +434,82 @@ function tildify(p: string): string {
/** The currently-running script — dist/cli.js when built, src/cli.ts under tsx. */
const SCRIPT_PATH = fileURLToPath(import.meta.url)
+/**
+ * Start the daemon that succeeds this one, on whatever bytes are in `~/.harness/cli` right now.
+ *
+ * Extracted from `restartForUpdate`'s own closure so the update handoff, the rollback respawn and the
+ * BOOT handoff below all spawn the same way. Not to be confused with the module's `spawnDaemon`: that
+ * one serves `harness start`, reads the pid file, finds THIS daemon in it and exits — called from
+ * inside the daemon it would quietly do nothing and lose the update.
+ *
+ * `managedNodePath()` is re-read here rather than captured at boot, so a runtime provisioned during
+ * this process's lifetime is the one the next daemon runs on.
+ */
+function spawnDaemonChild(extraEnv: Record<string, string>): ReturnType<typeof spawn> {
+ prepareLogFile(LOG_FILE, LEGACY_LOG_FILE) // before the fd, so the caller's sinceOffset sees one size
+ const fd = openSync(LOG_FILE, 'a')
+ const child = spawn(managedNodePath(), [SCRIPT_PATH, '__run'], {
+ detached: true, env: { ...process.env, ...extraEnv }, stdio: ['ignore', fd, fd],
+ })
+ // A spawn failure (e.g. EMFILE) emits 'error' on the child; with no listener that is an
+ // uncaughtException. Catch it so a failed restart can't take the daemon that asked for it down.
+ child.on('error', (e) => console.error('[update] daemon spawn error:', e instanceof Error ? e.message : e))
+ return child
+}
+
+/**
+ * What a staged update does while the daemon is still starting up — and the little the boot needs to
+ * know about itself to do it.
+ *
+ * The self-updater is started in `runForeground`'s prologue, before anything that can throw or hang,
+ * because a daemon that cannot finish booting is a daemon that can never be fixed: there is no
+ * supervisor, and the desktop app only re-runs `harness start` on the same broken bytes, once a
+ * minute, for ever. Its `onStaged` therefore has to mean something LONG before `restartForUpdate`
+ * exists — hence the indirection: `applyStagedUpdate` is `bootHandoff` until the body has built
+ * everything `restartForUpdate` tears down, and is swapped for it at that one line.
+ */
+const daemonBoot: {
+ updater: Poller | null
+ /** The hook server, once bound — the only thing a mid-boot handoff has to release. */
+ hookServer: Server | null
+ /** Set by the body so a failed boot can flip its own `/api/status` to not-ready. */
+ markNotReady: ((reason: string) => void) | null
+ /** Why this daemon is in safe mode, or null while it is healthy. Read by `/api/status`. */
+ safeMode: string | null
+ handingOff: boolean
+ applyStagedUpdate: (version: string) => void | Promise<void>
+} = { updater: null, hookServer: null, markNotReady: null, safeMode: null, handingOff: false, applyStagedUpdate: bootHandoff }
+
+/**
+ * Hand the machine to a newer build without finishing start-up.
+ *
+ * SYNCHRONOUS END TO END, and that is the whole safety argument: never awaiting means the half-built
+ * `runForeground` body cannot interleave between the port closing and the exit, so it can never
+ * reach the code that would bind the port the successor is about to take, and two daemons are
+ * impossible by construction. That is also why it does not supervise the child the way
+ * `restartForUpdate` does — waiting would leave this process running alongside the new one for up to
+ * a minute, both reconciling tmux and writing the registry.
+ *
+ * It spawns rather than merely exiting because on a machine with no desktop app nothing else would
+ * ever start the successor, and even with one the next spawn window is up to ~70s away.
+ */
+function bootHandoff(version: string): void {
+ if (daemonBoot.handingOff) return
+ daemonBoot.handingOff = true
+ runBootHandoff(VERSION, version, {
+ // The hook port has no fallback: a successor that cannot bind it is a daemon that does not come up.
+ closeServer: () => {
+ try { (daemonBoot.hookServer as unknown as { closeAllConnections?: () => void } | null)?.closeAllConnections?.() } catch { /* already gone */ }
+ try { daemonBoot.hookServer?.close() } catch { /* already gone */ }
+ },
+ // Only if it still names us — a no-op when start-up never got as far as claiming it.
+ removePidFile: () => { removePidFileIf(process.pid) },
+ spawn: (extraEnv) => spawnDaemonChild(extraEnv),
+ exit: (code) => process.exit(code),
+ log: (message) => console.log(message),
+ })
+}
+
/** This computer's identity — see lib/computerIdentity.ts. Sent on connect so the backend can enforce
* one machine per computer, and used by `harness start` to reconnect to the machine already
* bound to this box instead of minting a second one. */
@@ -1292,15 +1369,15 @@ async function statBirthMs(path: string): Promise<number> {
/** The daemon body: hooks + watcher + process discovery + backend socket. */
async function runForeground(session: AuthSession | null): Promise<void> {
installTimestampedConsole() // daemon-only: every harness.log line gets a wall-clock timestamp
- const savedApis = new ApiConnections(env.ADAPTER_DATA_DIR)
- const prepareApiTools = (cwd: string | null | undefined, engine: string): void => {
- if (!cwd) return
- try { prepareApiInstructions(savedApis, cwd, engine) }
- catch { console.warn('[apis] Tool instructions could not be added. Saved connections remain available through harness api.') }
- }
const startedAt = Date.now()
+ // Set when the restore pass could not run. The reconciler reads it at call time (its deps are built
+ // long before this is decided) and keeps rows it would otherwise retire — see `onRemoved`.
+ let restoreDegraded = false
let discoveryReady = false
let discoveryError: string | null = null
+ // How a boot that failed AFTER this server bound turns its own status not-ready: the app reads
+ // `discoveryReady: false` as "alive, not ready" and stops respawning over it (`enterSafeMode`).
+ daemonBoot.markNotReady = (reason) => { discoveryReady = false; discoveryError = reason }
// The pid file is claimed further down, the moment the control port is bound — not here, and not
// by whoever spawned us. See the comment at that claim.
@@ -1315,6 +1392,59 @@ async function runForeground(session: AuthSession | null): Promise<void> {
console.error('[fatal-guard] uncaughtException:', err instanceof Error ? (err.stack ?? err.message) : err)
})
+ // ── THE UPDATER GOES FIRST. Everything below this point can throw, hang, or wait on a vendor file,
+ // a port, or tmux — and a daemon that never finishes starting is a daemon that can never be fixed:
+ // there is no supervisor, and the desktop app only re-runs `harness start` on the same broken bytes.
+ // Started here, a published fix lands on its own however badly the rest of the boot goes.
+ //
+ // `onStaged` is one indirection on purpose: `restartForUpdate` does not exist yet and must not move
+ // (it tears down two dozen subsystems declared further down). Until it is ready, a staged update is
+ // applied by `bootHandoff`, which hands the machine over without finishing start-up.
+ // Update-handoff state, declared here — ahead of the /api/status handler that reads `restarting` —
+ // rather than beside the updater that writes it, so the closure never reaches a `let` in its TDZ.
+ let restarting = false
+ // The child a handoff is supervising, so a signal that lands mid-handoff can take it down with us
+ // rather than leaving two daemons — see shutdown(). Cleared the moment the handoff is CONFIRMED:
+ // from then on that child is the daemon, and a signal must not take it down with the old one.
+ let handoffChild: ReturnType<typeof spawn> | null = null
+
+ // Self-update ONLY manages the INSTALLED copy (`~/.harness/cli/cli.js`). A dev/repo run — `tsx`
+ // (`npm run dev`) OR `node dist/cli.js` from the checkout — must NEVER self-update: it would swap
+ // the published bundle into ~/.harness/cli and restart, hijacking the version you're developing.
+ // Match by inode so symlinks/realpath don't fool it; fall back to a path compare.
+ const installedCli = join(env.ADAPTER_CLI_DIR, 'cli.js')
+ let isInstalledCopy = SCRIPT_PATH === installedCli
+ try { isInstalledCopy = statSync(SCRIPT_PATH).ino === statSync(installedCli).ino } catch { /* keep path compare */ }
+ if (isInstalledCopy && !env.ADAPTER_UPDATE_DISABLE) {
+ daemonBoot.updater = startSelfUpdater({
+ currentVersion: VERSION,
+ url: env.ADAPTER_UPDATE_URL,
+ key: env.ADAPTER_UPDATE_KEY,
+ dir: env.ADAPTER_CLI_DIR,
+ intervalMs: env.ADAPTER_UPDATE_CHECK_MS,
+ slotSecond: env.ADAPTER_UPDATE_SLOT_SEC,
+ // The lock spans the byte swap AND the handoff it triggers, as one critical section: a
+ // `harness start` that lands between the two would otherwise stage over our .prev, and one
+ // that lands during the handoff would spawn a second daemon.
+ withLock: (fn) => withSpawnLock('handoff', fn, {
+ onWaiting: (owner) => console.log(`[update] waiting — the daemon is ${describeSpawnLockOwner(owner)}`),
+ }),
+ onStaged: (v) => daemonBoot.applyStagedUpdate(v),
+ })
+ const slotted = env.ADAPTER_UPDATE_SLOT_SEC >= 0 && 60_000 % env.ADAPTER_UPDATE_CHECK_MS === 0
+ console.log(`[update] self-update on · v${VERSION} · every ${Math.round(env.ADAPTER_UPDATE_CHECK_MS / 1000)}s`
+ + (slotted ? ` at :${String(env.ADAPTER_UPDATE_SLOT_SEC % 60).padStart(2, '0')}` : ''))
+ } else if (!env.ADAPTER_UPDATE_DISABLE) {
+ console.log(`[update] self-update off · running a dev/repo build (v${VERSION}), not the installed copy`)
+ }
+
+ const savedApis = new ApiConnections(env.ADAPTER_DATA_DIR)
+ const prepareApiTools = (cwd: string | null | undefined, engine: string): void => {
+ if (!cwd) return
+ try { prepareApiInstructions(savedApis, cwd, engine) }
+ catch { console.warn('[apis] Tool instructions could not be added. Saved connections remain available through harness api.') }
+ }
+
// The managed grid follows its pin on EVERY daemon start — this one, and the restart a self-update
// ends in — not only on `--repair`: the pin is expected to move, and a machine installed last month
// has to notice. Not awaited here: a download must never hold the control port back, and every grid
@@ -1366,9 +1496,19 @@ async function runForeground(session: AuthSession | null): Promise<void> {
// `loginShellEnvPromise` is awaited later, near the existing `[env]` log line.
const tmuxPathPromise = terminalConfig.backends.includes('tmux') ? ensureTmuxOnPath() : null
const loginShellEnvPromise = warmLoginShellEnvironment()
+ // Missing tmux is a STATE, not a reason to refuse to start. The daemon already models a machine
+ // without it — `tmuxBackend` is null whenever the config omits tmux, every caller tests it, and the
+ // create/restart/resume paths answer `TMUX_UNAVAILABLE` — so it can still serve its status, the
+ // local socket, the backend link and its updater, and say what is missing. Refusing instead left a
+ // machine whose PATH lost tmux with a daemon that could not start and therefore could not be fixed.
+ let tmuxUnavailable: string | null = null
if (tmuxPathPromise) {
- const tmuxPath = requireTmuxAvailable(await tmuxPathPromise)
- if (tmuxPath.state === 'adopted') {
+ const tmuxPath = await tmuxPathPromise
+ if (tmuxPath.state === 'absent') {
+ tmuxUnavailable = tmuxPath.reason
+ console.error(`[tmux] unavailable: ${tmuxPath.reason} · install tmux and verify \`tmux -V\`,`
+ + ' then restart — agents cannot be created or restored until then')
+ } else if (tmuxPath.state === 'adopted') {
console.log(`[tmux] not on the daemon PATH · adopted ${tmuxPath.from} from the user's login shell`)
}
}
@@ -1376,7 +1516,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
// sent, or its stock dark palette until it says otherwise. Read through a closure so a change
// reaches sessions created after it without rebuilding the backend.
let hostTheme: HostTheme = loadHostTheme() ?? DEFAULT_HOST_THEME
- const tmuxBackend = terminalConfig.backends.includes('tmux') ? new TmuxBackend(() => hostTheme) : null
+ const tmuxBackend = terminalConfig.backends.includes('tmux') && !tmuxUnavailable ? new TmuxBackend(() => hostTheme) : null
const herdrTargets = await resolveHerdrTargets()
activeHerdrSessions = herdrTargets.map((target) => target.sessionName)
const resolvedHerdrPaths = herdrTargets.flatMap((target) => target.state === 'available' ? [target.endpoint.socketPath] : [])
@@ -3307,6 +3447,16 @@ async function runForeground(session: AuthSession | null): Promise<void> {
announceSession(agent)
},
onRemoved: (agent, reason) => {
+ // A pane absent because RESTORE never ran is not a pane the person closed. Retiring it here
+ // would archive a row whose tmux pane was simply never rebuilt, and the person would have to
+ // Open each one by hand; keeping it dormant leaves the next daemon — the fixed one — something
+ // to restore.
+ if (restoreDegraded) {
+ console.log(`[discovery] ${sid(agent.agentId)} kept · restore did not run this boot · ${reason}`)
+ registry.setActive(agent.agentId, false)
+ announceSession(agent)
+ return
+ }
console.log(`[discovery] ${sid(agent.agentId)} removed · ${reason}`)
forgetSession(agent.agentId, { force: true })
},
@@ -3482,13 +3632,6 @@ async function runForeground(session: AuthSession | null): Promise<void> {
return { status: 200, body: withStaleMarker(cached.body, cached.fetchedAt) }
}
- // Update-handoff state, declared here — ahead of the /api/status handler that reads `restarting` —
- // rather than beside the updater that writes it, so the closure never reaches a `let` in its TDZ.
- let restarting = false
- // The child a handoff is supervising, so a signal that lands mid-handoff can take it down with us
- // rather than leaving two daemons — see shutdown(). Cleared the moment the handoff is CONFIRMED:
- // from then on that child is the daemon, and a signal must not take it down with the old one.
- let handoffChild: ReturnType<typeof spawn> | null = null
const { server: hookServer, port: hookPort } = await startHookServer(env.PORT, {
onCommandBar: commandBarService,
@@ -3826,7 +3969,10 @@ async function runForeground(session: AuthSession | null): Promise<void> {
// handoff. Informational: nothing should build readiness on a field the server stops serving.
restarting,
discoveryReady,
- discoveryError,
+ discoveryError: discoveryError ?? (tmuxUnavailable ? `tmux unavailable: ${tmuxUnavailable}` : null),
+ // Present only when start-up failed and this daemon is holding the machine open for its
+ // updater. Clients key on `discoveryReady`; this says WHY, in one word, for a person reading it.
+ ...(daemonBoot.safeMode ? { safeMode: true } : {}),
// Agents whose history is being read right now, and how many wait their turn. Normally empty or
// gone in a second; one that stays here names the store that is slow, which no other field does.
attaching: attaches.attaching(),
@@ -3892,6 +4038,9 @@ async function runForeground(session: AuthSession | null): Promise<void> {
// process that is running AND holds the port is the only honest author of its own pid; that claim
// is also the signal `harness start` and the update handoff wait on to know the bind succeeded.
try { writeFileSync(PID_FILE, String(process.pid) + '\n') } catch { /* best effort */ }
+ // The one thing a handoff that happens before start-up finishes has to release: the port has no
+ // fallback, so a successor that cannot bind it is a daemon that does not come up (see bootHandoff).
+ daemonBoot.hookServer = hookServer
console.log(`[cli] daemon pid ${process.pid} · v${VERSION}${process.env.ADAPTER_UPDATED_TO ? ' · updated' : ''} · listening on 127.0.0.1:${hookPort}`)
// Same on-disk identity `harness remote-password set`/`link connect` use (E2eeStore.init() is
// idempotent per file, so a separate in-memory instance here just reads the one this machine
@@ -4143,28 +4292,37 @@ async function runForeground(session: AuthSession | null): Promise<void> {
localClient: () => ({ kind: 'desktop', name: terminalHintMachineName().slice(0, 64), machineId: backend.machineId }),
})
// Install both CLI hooks with the port the local server actually bound.
+ //
+ // One vendor at a time, each behind its own guard: these write into thirteen different settings
+ // files owned by thirteen different CLIs, and one that is malformed, read-only or mid-write is not
+ // a reason for the other twelve to go uninstalled — let alone for the daemon not to come up.
+ const hookStep = (vendor: string, install: () => void): void => {
+ try { install() } catch (error) {
+ console.warn(`[hooks] ${vendor} install skipped · ${error instanceof Error ? error.message : error}`)
+ }
+ }
if (!env.DISABLE_HOOK_INSTALL) {
- installSessionHooks(hookPort)
- installCodexHooks(hookPort)
- installCursorHooks(hookPort)
- installOpencodePlugin(hookPort)
+ hookStep('claude', () => installSessionHooks(hookPort))
+ hookStep('codex', () => installCodexHooks(hookPort))
+ hookStep('cursor', () => installCursorHooks(hookPort))
+ hookStep('opencode', () => installOpencodePlugin(hookPort))
// The `grid` CLI the Grid harness shells out to, for a machine that signed in before this
// existed or whose sign-in could not fetch it. In the background: a download must not hold
// the daemon's own start, and nothing here waits on it.
void ensureGridInstalled().then((result) => {
if (result.status !== 'present') console.log(`[grid] ${result.message}`)
})
- installKiloPlugin(hookPort)
- installPiExtension(hookPort)
+ hookStep('kilo', () => installKiloPlugin(hookPort))
+ hookStep('pi', () => installPiExtension(hookPort))
// A self-update refreshes plugin files here; running engine processes pick them up according to each
// vendor's own plugin reload lifecycle.
- installAmpPlugin(hookPort)
- installHermesHooks(hookPort)
- installDevinHooks(hookPort)
- installCommandCodeHooks(hookPort)
- installGrokHooks(hookPort)
- installAgyHooks(hookPort)
- installCopilotHooks(hookPort)
+ hookStep('amp', () => installAmpPlugin(hookPort))
+ hookStep('hermes', () => installHermesHooks(hookPort))
+ hookStep('devin', () => installDevinHooks(hookPort))
+ hookStep('commandcode', () => installCommandCodeHooks(hookPort))
+ hookStep('grok', () => installGrokHooks(hookPort))
+ hookStep('agy', () => installAgyHooks(hookPort))
+ hookStep('copilot', () => installCopilotHooks(hookPort))
}
backend.setDashboardPort(hookPort) // surfaced to the web (e2e_status) so it can link here to approve
console.log(`[cli] local dashboard → http://127.0.0.1:${hookPort}`)
@@ -4407,6 +4565,10 @@ async function runForeground(session: AuthSession | null): Promise<void> {
if (saved && !isTerminalEngine(saved.engine)) retainExitedSession(entry, true)
}
if (tmuxBackend) {
+ // Best effort, like the cwd repair above it: panes that cannot be rebuilt cost this boot its
+ // tiles, not the daemon. `restoreDegraded` then stops discovery retiring the rows whose panes
+ // restore never got to, so the next daemon can put them back.
+ try {
const backend = tmuxBackend
let paneInventory: ReturnType<typeof listTmuxPanes> | null = null
const summary = await restoreAgents({
@@ -4491,12 +4653,23 @@ async function runForeground(session: AuthSession | null): Promise<void> {
console.log(`[restore] restored ${summary.restored.length} · skipped ${summary.skipped.length} · failed ${summary.failed.length}`
+ (registry.rebootedSinceLastRun ? ' · after reboot' : ''))
}
+ } catch (error) {
+ restoreDegraded = true
+ console.warn(`[restore] skipped · ${error instanceof Error ? error.message : error}`
+ + ' · agents keep their rows and come back on the next start')
+ }
}
// Every DSH agent the registry kept gets its viewer and verdict watch back — restored or not, an
// agent whose pane is still up is still that harness.
for (const session of registry.list()) if (session.dsh) attachDsh(session)
await agentReconciler.start(env.TERMINAL_RECONCILE_INTERVAL_MS ?? env.TMUX_REAP_INTERVAL_MS)
- for (const task of await loadCursorPendingTasks(env.ADAPTER_DATA_DIR)) {
+ // A file lock and a JSON parse, neither of which is worth the daemon: an unreadable queue means no
+ // pending Cursor tasks this boot, not no daemon.
+ const pendingCursorTasks = await loadCursorPendingTasks(env.ADAPTER_DATA_DIR).catch((error) => {
+ console.warn(`[cursor] pending tasks skipped · ${error instanceof Error ? error.message : error}`)
+ return []
+ })
+ for (const task of pendingCursorTasks) {
onCursorTaskStart(task.sessionId, task.toolUseId, task.input)
}
@@ -5494,7 +5667,6 @@ async function runForeground(session: AuthSession | null): Promise<void> {
//
// The cost is real and accepted: a turn streaming at that moment loses the rest of its events, and its
// clients see no turn_end for it until the new daemon re-attaches the session and the next turn runs.
- let updater: Poller | null = null
// Hand off to a freshly-spawned daemon running the just-swapped cli.js, then SUPERVISE it and roll
// back to the .prev bytes if it fails to come up. NOT launch() — that refuses while a daemon is alive.
@@ -5507,7 +5679,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
restarting = true
console.log(`[update] applying ${VERSION} → ${newVersion} — restarting daemon`)
registry.flush()
- updater?.stop()
+ daemonBoot.updater?.stop()
agentReconciler.stop()
clearInterval(logTrimTimer)
clearInterval(runtimeReconcileTimer)
@@ -5538,23 +5710,8 @@ async function runForeground(session: AuthSession | null): Promise<void> {
await backend.stop() // graceful WS close → releases the Redis machine-owner claim
await new Promise((r) => setTimeout(r, 1000)) // grace before the same-machine reclaim
- const spawnDaemon = (extraEnv: Record<string, string>): ReturnType<typeof spawn> => {
- prepareLogFile(LOG_FILE, LEGACY_LOG_FILE) // before the fd + the sinceOffset below, so both see one size
- const fd = openSync(LOG_FILE, 'a')
- // Serves the update restart AND the rollback respawn. managedNodePath() is re-read here rather
- // than captured at boot, so a runtime provisioned during this process's lifetime is the one the
- // next daemon runs on.
- const c = spawn(managedNodePath(), [SCRIPT_PATH, '__run'], {
- detached: true, env: { ...process.env, ...extraEnv }, stdio: ['ignore', fd, fd],
- })
- // A spawn failure (e.g. EMFILE) emits 'error' on the child; with no listener that is an
- // uncaughtException. Catch it so a failed update-restart can't take the old daemon down.
- c.on('error', (e) => console.error('[update] daemon spawn error:', e instanceof Error ? e.message : e))
- return c
- }
-
const sinceOffset = existsSync(LOG_FILE) ? statSync(LOG_FILE).size : 0
- const child = spawnDaemon({ ADAPTER_UPDATED_TO: newVersion })
+ const child = spawnDaemonChild({ ADAPTER_UPDATED_TO: newVersion })
handoffChild = child
let childExited = false
child.on('exit', () => { childExited = true })
@@ -5586,7 +5743,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
}
removePidFileIf(child.pid)
restoreUpdate(env.ADAPTER_CLI_DIR) // restore .prev → cli.js/notify.mjs
- const good = spawnDaemon({})
+ const good = spawnDaemonChild({})
handoffChild = good
let goodExited = false
good.on('exit', () => { goodExited = true })
@@ -5600,41 +5757,17 @@ async function runForeground(session: AuthSession | null): Promise<void> {
process.exit(0)
}
- // Self-update ONLY manages the INSTALLED copy (`~/.harness/cli/cli.js`). A dev/repo run — `tsx`
- // (`npm run dev`) OR `node dist/cli.js` from the checkout — must NEVER self-update: it would swap
- // the published bundle into ~/.harness/cli and restart, hijacking the version you're developing.
- // Match by inode so symlinks/realpath don't fool it; fall back to a path compare.
- const installedCli = join(env.ADAPTER_CLI_DIR, 'cli.js')
- let isInstalledCopy = SCRIPT_PATH === installedCli
- try { isInstalledCopy = statSync(SCRIPT_PATH).ino === statSync(installedCli).ino } catch { /* keep path compare */ }
- if (isInstalledCopy && !env.ADAPTER_UPDATE_DISABLE) {
- updater = startSelfUpdater({
- currentVersion: VERSION,
- url: env.ADAPTER_UPDATE_URL,
- key: env.ADAPTER_UPDATE_KEY,
- dir: env.ADAPTER_CLI_DIR,
- intervalMs: env.ADAPTER_UPDATE_CHECK_MS,
- slotSecond: env.ADAPTER_UPDATE_SLOT_SEC,
- // The lock spans the byte swap AND the handoff it triggers, as one critical section: a
- // `harness start` that lands between the two would otherwise stage over our .prev, and one
- // that lands during the handoff would spawn a second daemon.
- withLock: (fn) => withSpawnLock('handoff', fn, {
- onWaiting: (owner) => console.log(`[update] waiting — the daemon is ${describeSpawnLockOwner(owner)}`),
- }),
- onStaged: (v) => restartForUpdate(v).catch((err) => {
- // If the restart handoff itself throws/rejects (I/O fault during teardown), don't let it
- // become an unhandledRejection — log, un-latch `restarting`, and stay on the current build.
- console.error('[update] restart failed — staying on current build:', err instanceof Error ? err.message : err)
- restarting = false
- handoffChild = null
- }),
- })
- const slotted = env.ADAPTER_UPDATE_SLOT_SEC >= 0 && 60_000 % env.ADAPTER_UPDATE_CHECK_MS === 0
- console.log(`[update] self-update on · v${VERSION} · every ${Math.round(env.ADAPTER_UPDATE_CHECK_MS / 1000)}s`
- + (slotted ? ` at :${String(env.ADAPTER_UPDATE_SLOT_SEC % 60).padStart(2, '0')}` : ''))
- } else if (!env.ADAPTER_UPDATE_DISABLE) {
- console.log(`[update] self-update off · running a dev/repo build (v${VERSION}), not the installed copy`)
- }
+ // The handoff handler stops being `bootHandoff` HERE, and not a line earlier: everything
+ // `restartForUpdate` tears down — the hook server, the reconciler, the three interval timers, the
+ // watcher, the backend socket — exists by now. A straight-line assignment, never a wait: if the
+ // body never reaches this line the handler stays `bootHandoff`, and the fix still lands.
+ daemonBoot.applyStagedUpdate = (v) => restartForUpdate(v).catch((err) => {
+ // If the restart handoff itself throws/rejects (I/O fault during teardown), don't let it become
+ // an unhandledRejection — log, un-latch `restarting`, and stay on the current build.
+ console.error('[update] restart failed — staying on current build:', err instanceof Error ? err.message : err)
+ restarting = false
+ handoffChild = null
+ })
const shutdown = async (signal: string): Promise<void> => {
console.log(`\n[cli] ${signal} — shutting down`)
@@ -5655,7 +5788,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
// esptool fail in a way that reads exactly like dead hardware.
void cableRef?.stop()
deviceLinkRef?.stop()
- updater?.stop()
+ daemonBoot.updater?.stop()
agentReconciler.stop()
clearInterval(logTrimTimer)
clearInterval(runtimeReconcileTimer)
@@ -6793,6 +6926,9 @@ async function status(): Promise<void> {
const session = readAuthSession()
const daemonStatus = alive ? await runningDaemonStatus() : null
if (!alive) registry.load()
+ // A daemon whose start-up failed is alive and answering, but nothing on this machine works. Say so
+ // in the one line a person reads, rather than leaving it looking like an ordinary slow start.
+ const safeMode = alive ? readSafeModeMarker(env.ADAPTER_DATA_DIR, isAlive) : null
printInfoBlock({
// The backend link is the daemon's own business, so `status` is where it is read — `start` no
// longer waits to see it, and a daemon with no backend is still serving every local agent.
@@ -6800,6 +6936,8 @@ async function status(): Promise<void> {
// and `machine: not signed in` in place of the whole block hid a running daemon and its agents.
status: !alive
? '○ stopped'
+ : safeMode
+ ? `◍ safe mode · start-up failed on v${safeMode.version} — waiting for a fixed build (${safeMode.error.split('\n')[0]})`
: !session
? '● running · this computer only (not signed in)'
: daemonStatus == null
@@ -6879,6 +7017,73 @@ const onError = (err: unknown): never => {
process.exit(1)
}
+/**
+ * The daemon's start-up threw. STAY UP anyway, running nothing but the updater.
+ *
+ * Exiting here is what made one bad build unrecoverable: nothing supervises this process, the desktop
+ * app answers a dead port by running `harness start` again — the same bytes, about once a minute, for
+ * ever — and the updater that could have fixed it lives most of the way down a body that never
+ * finished. The updater is started in the prologue now (see `runForeground`), so by the time this
+ * runs it is already polling; all this has to do is keep the process alive long enough for a
+ * published fix to land, and tell everyone what state the machine is in.
+ *
+ * Three ways it earns its keep, in order: the bound control port answers `discoveryReady: false`, so
+ * the app reads the machine as not-ready instead of dead and STOPS respawning; the pid file stays
+ * ours, so `harness start` is a cheap no-op rather than a zombie factory; and the marker file lets
+ * `harness status` say what happened. `harness stop` still works throughout — it kills by pid.
+ */
+const enterSafeMode = (err: unknown): void => {
+ const disposition = safeModeDisposition(err, { selfPid: process.pid, readPid, isAlive })
+ if (!disposition.stay) {
+ console.error(`[safe-mode] not staying up — ${disposition.reason}`)
+ onError(err)
+ }
+ const detail = err instanceof Error ? (err.stack ?? err.message) : String(err)
+ console.error('Failed to start adapter:', err)
+ console.error(`[safe-mode] staying up on v${VERSION} with the updater only — a published fix will be`
+ + ' applied on its own. Nothing else on this machine works until then.')
+ writeSafeModeMarker(env.ADAPTER_DATA_DIR, { pid: process.pid, version: VERSION, at: Date.now(), error: detail })
+ daemonBoot.safeMode = disposition.reason
+ daemonBoot.markNotReady?.(disposition.reason)
+
+ const leave = (why: string, code: number): never => {
+ clearSafeModeMarker(env.ADAPTER_DATA_DIR)
+ removePidFileIf(process.pid)
+ console.log(`[safe-mode] ${why}`)
+ process.exit(code)
+ }
+ process.on('SIGINT', () => leave('SIGINT — leaving safe mode', 0))
+ process.on('SIGTERM', () => leave('SIGTERM — leaving safe mode', 0))
+
+ // The bound control port is a ref'd handle and holds the loop on its own. Without one — the bind
+ // itself was what failed, or we never got that far — take the port for the status alone, so the app
+ // still reads not-ready rather than down. A port we cannot take at all leaves only a ticking clock.
+ if (!daemonBoot.hookServer) {
+ const status = createServer((req, res) => {
+ const body = safeModeStatusBody({
+ version: VERSION, pid: process.pid, startedAt: Date.now(),
+ computerId: computerId(), error: disposition.reason,
+ })
+ res.writeHead(200, { 'content-type': 'application/json' })
+ res.end(JSON.stringify(body))
+ })
+ status.on('error', (e) => {
+ console.error(`[safe-mode] could not serve status on ${env.PORT}: ${e instanceof Error ? e.message : e}`)
+ // A ref'd timer, unlike the updater's: something has to hold the event loop open.
+ setInterval(() => console.log(`[safe-mode] still waiting for a fixed build · v${VERSION}`), 10 * 60_000)
+ })
+ status.listen(env.PORT, '127.0.0.1')
+ }
+
+ // Bounded on purpose. A cause that has since cleared — tmux not yet on PATH after a reboot, a lock
+ // file, a port held for a moment — would otherwise leave the machine wedged in a state nobody
+ // respawns over, because not-ready is exactly what stops the app trying again.
+ if (env.ADAPTER_SAFE_MODE_MS > 0) {
+ setTimeout(() => leave(`no fix arrived within ${Math.round(env.ADAPTER_SAFE_MODE_MS / 60_000)}m — letting a clean start try`, 1),
+ env.ADAPTER_SAFE_MODE_MS).unref?.()
+ }
+}
+
switch (cmd) {
case 'orchestrator':
orchestratorCommand(rest).then(code => { process.exitCode = code }).catch(onError)
@@ -6904,7 +7109,8 @@ switch (cmd) {
console.error('`harness join` has been removed. Run `harness login`, then `harness start`.')
process.exit(1)
case '__run': // internal: the detached daemon child reads the durable SSO session — or runs without one
- runForeground(readAuthSession()).catch(onError)
+ // NOT `onError`: a daemon that dies here can never be updated. See `enterSafeMode`.
+ runForeground(readAuthSession()).catch(enterSafeMode)
break
case 'autonomous-device':
runAutonomousDeviceCommand(rest, env.ADAPTER_DATA_DIR, env.PORT).then(code => { process.exitCode = code }).catch(onError)
diff --git a/cli/src/config/env.ts b/cli/src/config/env.ts
index 0bf2f2999..f73fd84b7 100644
--- a/cli/src/config/env.ts
+++ b/cli/src/config/env.ts
@@ -396,6 +396,9 @@ const envSchema = z.object({
ADAPTER_UPDATE_SLOT_SEC: z.string().default('45').transform(Number),
// Set 'true' to disable self-update entirely.
ADAPTER_UPDATE_DISABLE: z.string().default('false').transform((v) => v === 'true'),
+ /** How long a daemon whose start-up failed stays up serving nothing but its updater, before it
+ * gives a clean process a turn. ~15 update slots; `0` keeps it up for ever. */
+ ADAPTER_SAFE_MODE_MS: z.string().default('900000').transform(Number),
// Install dir holding the packaged cli.js + notify.mjs that the self-updater swaps in place.
ADAPTER_CLI_DIR: z.string().default(adapterCliDir),
// Where the managed Node runtime lives. Read (never written) by this process: the hook command
diff --git a/cli/src/lib/daemonSafeMode.spec.ts b/cli/src/lib/daemonSafeMode.spec.ts
new file mode 100644
index 000000000..6f9aa3638
--- /dev/null
+++ b/cli/src/lib/daemonSafeMode.spec.ts
@@ -0,0 +1,106 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import {
+ clearSafeModeMarker, readSafeModeMarker, runBootHandoff, safeModeDisposition,
+ safeModeFile, safeModeStatusBody, writeSafeModeMarker, type BootHandoffDeps,
+} from './daemonSafeMode.js'
+
+let dir = ''
+beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'harness-safe-mode-')) })
+afterEach(() => { rmSync(dir, { recursive: true, force: true }) })
+
+describe('safeModeDisposition', () => {
+ const nobody = { selfPid: 100, readPid: () => null, isAlive: () => false }
+
+ it('stays up for a start-up that simply failed', () => {
+ expect(safeModeDisposition(new Error('Cannot access \'p4\' before initialization'), nobody))
+ .toEqual({ stay: true, reason: 'Cannot access \'p4\' before initialization' })
+ expect(safeModeDisposition('tmux is required but unavailable', nobody).stay).toBe(true)
+ })
+
+ it('leaves when another daemon owns the machine — a loser must not hold port, pid and update slot', () => {
+ expect(safeModeDisposition(new Error('listen EADDRINUSE: address already in use 127.0.0.1:18473'), nobody))
+ .toMatchObject({ stay: false })
+ expect(safeModeDisposition(new Error('boom'), { selfPid: 100, readPid: () => 200, isAlive: () => true }))
+ .toMatchObject({ stay: false, reason: 'another daemon (pid 200) owns this machine' })
+ })
+
+ it('stays when the pid file names us, or names a corpse', () => {
+ expect(safeModeDisposition(new Error('boom'), { selfPid: 100, readPid: () => 100, isAlive: () => true }).stay).toBe(true)
+ expect(safeModeDisposition(new Error('boom'), { selfPid: 100, readPid: () => 200, isAlive: () => false }).stay).toBe(true)
+ })
+})
+
+describe('safeModeStatusBody', () => {
+ // ⚠️ Cross-language contract: `desktop/lib/ws/local_cli_discovery.dart` reads exactly this field
+ // and this value to decide "alive but not ready" — which is what stops it respawning every minute.
+ it('says not-ready in the one field the desktop keys on', () => {
+ const body = safeModeStatusBody({ version: '0.3.5', pid: 7, startedAt: 1, computerId: 'c1', error: 'tmux missing' })
+ expect(body.discoveryReady).toBe(false)
+ expect(body).toMatchObject({ safeMode: true, connected: false, discoveryError: 'tmux missing', computerId: 'c1' })
+ })
+})
+
+describe('the safe-mode marker', () => {
+ it('round-trips, and a marker left by a dead process reads as none', () => {
+ writeSafeModeMarker(dir, { pid: 42, version: '0.3.5', at: 5, error: 'boom' })
+ expect(readSafeModeMarker(dir, () => true)).toEqual({ pid: 42, version: '0.3.5', at: 5, error: 'boom' })
+ expect(readSafeModeMarker(dir, () => false)).toBeNull()
+ clearSafeModeMarker(dir)
+ expect(readSafeModeMarker(dir, () => true)).toBeNull()
+ })
+
+ it('a corrupt marker is no marker, never a crash', () => {
+ writeFileSync(safeModeFile(dir), '{ not json')
+ expect(readSafeModeMarker(dir, () => true)).toBeNull()
+ expect(readFileSync(safeModeFile(dir), 'utf-8')).toBe('{ not json')
+ })
+})
+
+describe('runBootHandoff', () => {
+ function harness(over: Partial<BootHandoffDeps> = {}) {
+ const calls: string[] = []
+ let env: Record<string, string> = {}
+ let exited: number | null = null
+ const deps: BootHandoffDeps = {
+ closeServer: () => calls.push('close'),
+ removePidFile: () => calls.push('removePid'),
+ spawn: (extra) => { calls.push('spawn'); env = extra; return { pid: 999, unref: () => calls.push('unref') } },
+ exit: ((code: number) => { calls.push(`exit:${code}`); exited = code; return undefined as never }),
+ log: () => {},
+ ...over,
+ }
+ return { deps, calls, env: () => env, exited: () => exited }
+ }
+
+ it('releases the port, then the pid file, then spawns the successor and leaves', () => {
+ const h = harness()
+ runBootHandoff('0.3.5', '0.3.6', h.deps)
+ expect(h.calls).toEqual(['close', 'removePid', 'spawn', 'unref', 'exit:0'])
+ expect(h.env()).toEqual({ ADAPTER_UPDATED_TO: '0.3.6' })
+ })
+
+ it('has already exited by the time it returns — the property that makes two daemons impossible', () => {
+ // Nothing is awaited, so the half-built boot cannot interleave between closing the port and the
+ // exit, and can never reach the code that would bind the port the successor is taking.
+ const h = harness()
+ runBootHandoff('0.3.5', '0.3.6', h.deps)
+ expect(h.exited()).toBe(0) // no await, no tick: true immediately after the call returns
+ })
+
+ it('a start-up that never bound or claimed anything still hands off', () => {
+ const h = harness({ closeServer: () => {}, removePidFile: () => {} })
+ runBootHandoff('0.3.5', '0.3.6', h.deps)
+ expect(h.calls).toEqual(['spawn', 'unref', 'exit:0'])
+ })
+
+ it('says which pid took over, so the log names the successor', () => {
+ const lines: string[] = []
+ const h = harness({ log: (m) => lines.push(m) })
+ runBootHandoff('0.3.5', '0.3.6', h.deps)
+ expect(lines[0]).toContain('0.3.5 → 0.3.6')
+ expect(lines[1]).toContain('pid 999')
+ })
+})
diff --git a/cli/src/lib/daemonSafeMode.ts b/cli/src/lib/daemonSafeMode.ts
new file mode 100644
index 000000000..575a86e0a
--- /dev/null
+++ b/cli/src/lib/daemonSafeMode.ts
@@ -0,0 +1,143 @@
+/**
+ * What a daemon does when its own start-up throws.
+ *
+ * Dying was the old answer, and on a machine that kept throwing it was a trap: there is no
+ * supervisor, the desktop app only re-runs `harness start` on the same broken bytes — about once a
+ * minute, for ever — and the self-updater lives 87% of the way down `runForeground`, so a boot that
+ * never finishes can never be fixed. Measured on a person's Mac at v0.3.5: two boots, one stack, no
+ * way out but deleting the registry by hand.
+ *
+ * So a daemon whose boot failed STAYS UP with nothing but its updater running, and says so. The
+ * decisions that have nothing to do with process control live here, where they can be tested.
+ */
+import { existsSync, readFileSync, rmSync } from 'node:fs'
+import { join } from 'node:path'
+import { atomicWriteJson } from './registry.js'
+
+/** Written while a daemon is in safe mode; removed on the way out. `harness status` reads it. */
+export const SAFE_MODE_FILE_NAME = 'adapter.safe-mode'
+
+export interface SafeModeMarker {
+ pid: number
+ version: string
+ at: number
+ error: string
+}
+
+export interface SafeModeDisposition {
+ stay: boolean
+ reason: string
+}
+
+/**
+ * Whether staying alive is the right answer for THIS failure.
+ *
+ * Two shapes where it is not, and both are the same mistake: a second daemon that lost a race has no
+ * business holding a port, a pid file and an update slot the winner already owns. Without this rule
+ * every lost race would leave an immortal process behind, and they would accumulate.
+ */
+export function safeModeDisposition(
+ error: unknown,
+ deps: { selfPid: number; readPid: () => number | null; isAlive: (pid: number) => boolean },
+): SafeModeDisposition {
+ const message = error instanceof Error ? `${error.message}` : String(error)
+ if (/EADDRINUSE|address already in use/i.test(message)) {
+ return { stay: false, reason: 'the control port belongs to another daemon' }
+ }
+ const owner = deps.readPid()
+ if (owner !== null && owner !== deps.selfPid && deps.isAlive(owner)) {
+ return { stay: false, reason: `another daemon (pid ${owner}) owns this machine` }
+ }
+ return { stay: true, reason: message || 'start-up failed' }
+}
+
+/**
+ * The status a safe-mode daemon serves.
+ *
+ * ⚠️ CROSS-LANGUAGE CONTRACT. `discoveryReady: false` is what tells the desktop app this daemon is
+ * alive but not ready (`desktop/lib/ws/local_cli_discovery.dart` — `notReady`, which it never
+ * respawns over), and that single field is what stops the once-a-minute restart storm. It has to be
+ * a 200: a 5xx also reads as not-ready there, but `runningDaemonStatus()` treats a non-ok response
+ * as no daemon at all, and `harness start` would then mistake safe mode for a healthy machine.
+ */
+export function safeModeStatusBody(info: {
+ version: string
+ pid: number
+ startedAt: number
+ computerId: string
+ error: string
+}): Record<string, unknown> {
+ return {
+ safeMode: true,
+ discoveryReady: false,
+ discoveryError: info.error,
+ connected: false,
+ version: info.version,
+ pid: info.pid,
+ startedAt: info.startedAt,
+ computerId: info.computerId,
+ }
+}
+
+export function safeModeFile(dataDir: string): string {
+ return join(dataDir, SAFE_MODE_FILE_NAME)
+}
+
+export function writeSafeModeMarker(dataDir: string, marker: SafeModeMarker): void {
+ try { atomicWriteJson(safeModeFile(dataDir), marker) } catch { /* a marker is a courtesy, not the state */ }
+}
+
+export function clearSafeModeMarker(dataDir: string): void {
+ try { rmSync(safeModeFile(dataDir), { force: true }) } catch { /* ignore */ }
+}
+
+export interface BootHandoffDeps {
+ /** The bound control port, if start-up ever got that far. */
+ closeServer: () => void
+ removePidFile: () => void
+ spawn: (extraEnv: Record<string, string>) => { pid?: number; unref: () => void }
+ exit: (code: number) => never
+ log: (message: string) => void
+}
+
+/**
+ * Hand the machine to a newer build without finishing start-up.
+ *
+ * SYNCHRONOUS END TO END, and that is the whole safety argument: never awaiting means the half-built
+ * boot cannot interleave between the port closing and the exit, so it can never reach the code that
+ * would bind the port the successor is about to take. Two daemons are impossible by construction.
+ * That is also why it does not supervise the child the way a normal update restart does — waiting
+ * would leave this process running alongside the new one for up to a minute, both reconciling tmux
+ * and writing the registry.
+ *
+ * It spawns rather than merely exiting because nothing supervises a daemon: on a machine with no
+ * desktop app nothing else would ever start the successor.
+ */
+export function runBootHandoff(from: string, to: string, deps: BootHandoffDeps): void {
+ deps.log(`[update] ${from} → ${to} staged during start-up — handing off without finishing boot`)
+ deps.closeServer()
+ deps.removePidFile()
+ const child = deps.spawn({ ADAPTER_UPDATED_TO: to })
+ child.unref()
+ deps.log(`[update] boot handoff → pid ${child.pid ?? '?'} · this process is leaving`)
+ deps.exit(0)
+}
+
+/** The marker, or null — including for one left behind by a process that is no longer running. */
+export function readSafeModeMarker(
+ dataDir: string,
+ isAlive: (pid: number) => boolean,
+): SafeModeMarker | null {
+ const path = safeModeFile(dataDir)
+ try {
+ if (!existsSync(path)) return null
+ const raw = JSON.parse(readFileSync(path, 'utf-8')) as Partial<SafeModeMarker>
+ if (typeof raw.pid !== 'number' || !isAlive(raw.pid)) return null
+ return {
+ pid: raw.pid,
+ version: typeof raw.version === 'string' ? raw.version : 'unknown',
+ at: typeof raw.at === 'number' ? raw.at : 0,
+ error: typeof raw.error === 'string' ? raw.error : 'start-up failed',
+ }
+ } catch { return null }
+}
diff --git a/cli/src/lib/restoreAgents.ts b/cli/src/lib/restoreAgents.ts
index 30ba4d449..e68ee97bb 100644
--- a/cli/src/lib/restoreAgents.ts
+++ b/cli/src/lib/restoreAgents.ts
@@ -139,7 +139,11 @@ export async function restoreAgents(deps: RestoreAgentsDeps): Promise<RestoreSum
const summary: RestoreSummary = { restored: [], skipped: [], failed: [] }
const missing: Array<{ entry: RegisteredSession; runtime: TmuxRuntimeRef }> = []
+ // Per row, because a survey that gives up on the first bad one gives up on every row behind it —
+ // one pane whose `tmux list-panes` timed out, or one archive that could not be written, and the
+ // whole desk comes back empty. A row that cannot be surveyed is reported and the rest go on.
for (const entry of deps.registry.list()) {
+ try {
const runtime = tmuxRuntime(entry)
if (!runtime) { summary.skipped.push({ agentId: entry.agentId, reason: 'no tmux pane' }); continue }
if (entry.launch?.state === 'failed') { summary.skipped.push({ agentId: entry.agentId, reason: 'last launch failed' }); continue }
@@ -208,6 +212,11 @@ export async function restoreAgents(deps: RestoreAgentsDeps): Promise<RestoreSum
continue
}
missing.push({ entry, runtime })
+ } catch (error) {
+ const reason = error instanceof Error ? error.message : String(error)
+ summary.failed.push({ agentId: entry.agentId, reason })
+ deps.log(`[restore] ${entry.engine} · agent ${entry.agentId} · could not be surveyed · ${reason}`)
+ }
}
if (!missing.length) return summary
diff --git a/cli/src/lib/selfUpdate.spec.ts b/cli/src/lib/selfUpdate.spec.ts
index 0b6cee948..6f310f5a5 100644
--- a/cli/src/lib/selfUpdate.spec.ts
+++ b/cli/src/lib/selfUpdate.spec.ts
@@ -145,6 +145,35 @@ describe('startSelfUpdater', () => {
expect(events).toEqual(['lock', 'staged:9.9.9', 'handoff-done', 'unlock'])
})
+ it('is already stopped when onStaged runs — a handler that defers loses the updater for good', async () => {
+ // `done = true` and `stop()` happen BEFORE `onStaged`, so a handler that returns without handing
+ // the machine over leaves no timer and no way back. This is why the daemon's boot-time handler
+ // always takes over and exits rather than waiting for start-up to finish (`runBootHandoff`).
+ serveUpdate()
+ const dir = tempDir()
+ let staged = 0
+ let checks = 0
+ const counted = globalThis.fetch as typeof fetch
+ vi.stubGlobal('fetch', async (...args: Parameters<typeof fetch>) => {
+ if (String(args[0]).endsWith('metadata.json')) checks++
+ return counted(...args)
+ })
+ const poller = startSelfUpdater({
+ currentVersion: '1.0.0',
+ url: 'https://updates.test/metadata.json',
+ key: 'adapter',
+ dir,
+ intervalMs: 20,
+ onStaged: () => { staged++ }, // deliberately does NOT exit or restart
+ })
+ await vi.waitFor(() => expect(staged).toBe(1))
+ const after = checks
+ await new Promise((r) => setTimeout(r, 120)) // six intervals' worth
+ poller.stop()
+ expect(staged).toBe(1)
+ expect(checks, 'the poller never ticks again once a staged build has been handed over').toBe(after)
+ })
+
it('does not check on start; the first check lands on the slot, the next on the following one', async () => {
vi.useFakeTimers()
try {
diff --git a/cli/src/lib/terminalAgentReconciler.spec.ts b/cli/src/lib/terminalAgentReconciler.spec.ts
index e846407d0..51c878840 100644
--- a/cli/src/lib/terminalAgentReconciler.spec.ts
+++ b/cli/src/lib/terminalAgentReconciler.spec.ts
@@ -455,6 +455,34 @@ describe('verified process adoption', () => {
})
})
+describe('start()', () => {
+ it('keeps scanning when the opening pass fails — the interval is armed before it runs', async () => {
+ // Awaiting first meant one bad probe left discovery unscheduled for the life of the daemon, and
+ // rejected the caller's start-up on the way: no agents, no liveness, `discoveryReady` never true.
+ vi.useFakeTimers()
+ try {
+ let pass = 0
+ const scan = vi.fn(async () => {
+ pass++
+ if (pass === 1) throw new Error('ps timed out')
+ return probe([])
+ })
+ const reconciler = new TerminalAgentReconciler({
+ current: () => [], backends: [], backendOrder: ['tmux'], herdrSessionOrder: [],
+ onDiscovered: vi.fn(), onObserved: vi.fn(), onDormant: vi.fn(), onRemoved: vi.fn(),
+ probe: scan,
+ })
+ await expect(reconciler.start(5_000)).resolves.toBeUndefined()
+ expect(scan).toHaveBeenCalledTimes(1)
+ await vi.advanceTimersByTimeAsync(5_000)
+ expect(scan).toHaveBeenCalledTimes(2)
+ reconciler.stop()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+})
+
describe('a terminal pane (engine `terminal`)', () => {
const live = [{ instanceId: 'tmux:default', result: { state: 'available' as const, roots: [{ runtime: tmux, rootPid: 1, cwd: '/work' }] } }]
function terminal(overrides: Partial<RegisteredSession> = {}): RegisteredSession {
diff --git a/cli/src/lib/terminalAgentReconciler.ts b/cli/src/lib/terminalAgentReconciler.ts
index 93241e4ca..6f8232a58 100644
--- a/cli/src/lib/terminalAgentReconciler.ts
+++ b/cli/src/lib/terminalAgentReconciler.ts
@@ -114,10 +114,20 @@ export class TerminalAgentReconciler {
constructor(private readonly deps: TerminalAgentReconcilerDeps) {}
+ /**
+ * Arm the interval FIRST, then run the opening pass.
+ *
+ * The other way round — await, then schedule — meant a first pass that threw left discovery
+ * unscheduled for the life of the daemon: no new agents, no liveness, `discoveryReady` never true,
+ * and the caller's own start-up rejected on top of it. Neither is worth one bad probe. The opening
+ * pass is reported and dropped; the interval retries it a few seconds later.
+ */
async start(intervalMs: number): Promise<void> {
- await this.trigger()
this.timer = setInterval(() => { void this.trigger() }, intervalMs)
this.timer.unref?.()
+ await this.trigger().catch((error) => {
+ console.warn(`[discovery] first pass failed, retrying on the interval · ${error instanceof Error ? error.message : error}`)
+ })
}
stop(): void {
diff --git a/cli/src/lib/tmuxOnPath.ts b/cli/src/lib/tmuxOnPath.ts
index d6db975ba..9a1956cad 100644
--- a/cli/src/lib/tmuxOnPath.ts
+++ b/cli/src/lib/tmuxOnPath.ts
@@ -33,7 +33,15 @@ export type TmuxPathOutcome =
export type AvailableTmuxPathOutcome = Exclude<TmuxPathOutcome, { state: 'absent' }>
-/** Refuse to start a daemon that cannot create terminals. */
+/**
+ * Refuse to start over a tmux that cannot be found.
+ *
+ * ⚠️ NOT used by the daemon any more. A daemon that will not start is a daemon that cannot be fixed
+ * — its own updater lives inside the boot it never finishes — and a machine that merely lost tmux
+ * from its PATH does not need that. `runForeground` records the reason and runs without a tmux
+ * backend instead, which every caller already handles (`TMUX_UNAVAILABLE`). Kept for callers that
+ * genuinely have nothing to do without terminals, and for its tests.
+ */
export function requireTmuxAvailable(outcome: TmuxPathOutcome): AvailableTmuxPathOutcome {
if (outcome.state === 'absent') {
throw new Error(
diff --git a/cli/src/startupOrder.spec.ts b/cli/src/startupOrder.spec.ts
index 6a50748a3..cf60a56b5 100644
--- a/cli/src/startupOrder.spec.ts
+++ b/cli/src/startupOrder.spec.ts
@@ -21,6 +21,31 @@ import { join } from 'node:path'
*/
const SOURCE = readFileSync(join(import.meta.dirname, 'cli.ts'), 'utf-8')
+/** `runForeground`'s own body. Other functions are indented the same way; their locals are not ours. */
+function runForegroundBody(source: string): { text: string; from: number } {
+ const from = source.indexOf('async function runForeground(')
+ expect(from, 'cli.ts still defines runForeground').toBeGreaterThan(-1)
+ const end = source.indexOf('\n}\n', from)
+ return { text: source.slice(from, end < 0 ? source.length : end), from }
+}
+
+/** Where each local of `runForeground` is initialised, by absolute index into the source. */
+function localsOf(source: string): Map<string, number> {
+ const { text, from } = runForegroundBody(source)
+ const declaredAt = new Map<string, number>()
+ for (const match of text.matchAll(/^ {2}(?:const|let) ([A-Za-z_$][\w$]*)\s*[:=]/gm)) {
+ if (!declaredAt.has(match[1])) declaredAt.set(match[1], from + match.index)
+ }
+ return declaredAt
+}
+
+/** Identifiers a block REFERENCES: member names and the block's own declarations are not references. */
+function referencedIn(body: string): string[] {
+ const own = new Set([...body.matchAll(/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)/g)].map(match => match[1]))
+ const outer = body.replace(/\.\s*[A-Za-z_$][\w$]*/g, '')
+ return [...new Set(outer.match(/[A-Za-z_$][\w$]*/g) ?? [])].filter(name => !own.has(name))
+}
+
/** The source with comments blanked, so a name mentioned in prose is not read as a reference.
* Strings are left alone on purpose: blanking them needs a tokenizer to survive the backticks and
* regex literals this file is full of, and a word inside a string cannot match a local's name. */
@@ -30,39 +55,93 @@ function code(text: string): string {
.replace(/(^|[^:])\/\/[^\n]*/g, (match, lead: string) => lead + ' '.repeat(match.length - lead.length))
}
-/** The object literal passed to `name(`, by brace balance. */
-function callArgument(source: string, name: string): { body: string; at: number } {
- const at = source.indexOf(`await ${name}({`)
- expect(at, `${name} is still called as \`await ${name}({\` in cli.ts`).toBeGreaterThan(-1)
+/** The object literal passed to `call`, by brace balance. */
+function callArgument(source: string, call: string): { body: string; at: number } {
+ const at = source.indexOf(call)
+ expect(at, `cli.ts still contains \`${call}\``).toBeGreaterThan(-1)
let depth = 0
for (let i = source.indexOf('{', at); i < source.length; i++) {
if (source[i] === '{') depth++
else if (source[i] === '}' && --depth === 0) return { body: source.slice(at, i + 1), at }
}
- throw new Error(`unbalanced braces after ${name}(`)
+ throw new Error(`unbalanced braces after ${call}`)
}
/** Every call whose dependencies run DURING start-up, before `runForeground` has finished its body. */
-const STARTUP_CALLS = ['repairClaudeCwd', 'restoreAgents']
+const STARTUP_CALLS = ['await repairClaudeCwd({', 'await restoreAgents({', 'startSelfUpdater({']
+
+/** What the prologue is allowed to do before the updater is running: nothing that can throw. */
+const PROLOGUE_CALLS = new Set([
+ 'installTimestampedConsole', 'startSelfUpdater', 'statSync', 'join', 'String', 'Number', 'Date',
+ 'withSpawnLock', 'describeSpawnLockOwner',
+])
+const KEYWORDS = new Set(['if', 'for', 'while', 'switch', 'catch', 'return', 'typeof', 'new', 'await', 'function'])
describe('cli.ts start-up order', () => {
- it.each(STARTUP_CALLS)('every local %s reaches is initialised before it runs', (name) => {
+ it.each(STARTUP_CALLS)('every local `%s` reaches is initialised before it runs', (name) => {
const source = code(SOURCE)
const { body, at } = callArgument(source, name)
- // Locals of `runForeground` are the two-space-indented declarations; anything else is a module
- // constant, an import, or a nested scope, none of which can be in a dead zone here.
- const declaredAt = new Map<string, number>()
- for (const match of source.matchAll(/^ {2}(?:const|let) ([A-Za-z_$][\w$]*)\s*[:=]/gm)) {
- if (!declaredAt.has(match[1])) declaredAt.set(match[1], match.index)
- }
- // A name the closures declare themselves shadows anything outside them, however late that is.
- const own = new Set([...body.matchAll(/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)/g)].map(match => match[1]))
- // `entry.agentId` names a property, not the `const agentId` three thousand lines down; the same
- // goes for every other member access in these closures.
- const referenced = body.replace(/\.\s*[A-Za-z_$][\w$]*/g, '')
- const late = [...new Set(referenced.match(/[A-Za-z_$][\w$]*/g) ?? [])]
- .filter(used => !own.has(used) && (declaredAt.get(used) ?? -1) > at)
- .sort()
+ const declaredAt = localsOf(source)
+ const late = referencedIn(body).filter(used => (declaredAt.get(used) ?? -1) > at).sort()
expect(late, `declare these above the \`${name}\` call — see this file’s header`).toEqual([])
})
+
+ // ── The invariant the whole safe-boot design rests on ───────────────────────────────────────────
+ // A daemon that cannot finish starting can only be fixed by its own updater, and the updater can
+ // only run if start-up reached it. So it goes first, and everything that can throw or hang goes
+ // after. These three say so in the order they are worth reading.
+
+ it('starts the updater before anything that can throw or hang', () => {
+ const source = code(SOURCE)
+ const updater = source.indexOf('startSelfUpdater({')
+ expect(updater, 'cli.ts still starts the self-updater').toBeGreaterThan(-1)
+ for (const risky of [
+ 'requireTmuxAvailable(', // throws outright when tmux is missing
+ 'await startHookServer(', // EADDRINUSE on a fixed port with no fallback
+ 'installSessionHooks(', // 13 vendor settings files, any of which can be unreadable
+ 'await restoreAgents({', // tmux, the registry, and the closures a bad edit puts in a dead zone
+ 'await agentReconciler.start(',
+ 'loadCursorPendingTasks(', // a file lock that can hang, not just throw
+ ]) {
+ const at = source.indexOf(risky)
+ if (at < 0) continue // renamed or gone; the TDZ lint above still covers what remains
+ expect(at, `\`${risky}\` must come after startSelfUpdater({ — a crash there would strand the machine`)
+ .toBeGreaterThan(updater)
+ }
+ })
+
+ it('keeps the prologue itself incapable of throwing', () => {
+ // Stronger than listing today's hazards: this is what catches the NEXT line somebody adds on top.
+ const source = code(SOURCE)
+ const from = source.indexOf('async function runForeground(')
+ const to = source.indexOf('startSelfUpdater({')
+ const prologue = source.slice(source.indexOf('{', from), to)
+ expect(prologue.includes('\n await '), 'no await may precede the updater — a hang there is unrecoverable').toBe(false)
+ expect(/\n\s*throw /.test(prologue), 'no throw may precede the updater').toBe(false)
+ const called = [...prologue.replace(/\.\s*[A-Za-z_$][\w$]*\s*\(/g, ' ').matchAll(/([A-Za-z_$][\w$]*)\s*\(/g)]
+ .map(match => match[1])
+ .filter(name => !KEYWORDS.has(name))
+ expect([...new Set(called.filter(name => !PROLOGUE_CALLS.has(name)))].sort(),
+ 'only calls that cannot fail belong above the updater; move this below it, or add it to PROLOGUE_CALLS with a reason')
+ .toEqual([])
+ })
+
+ it('only swaps in the full restart handler once everything it tears down exists', () => {
+ // `bootHandoff` hands the machine over without finishing start-up; `restartForUpdate` tears down
+ // two dozen subsystems and may only be armed after every one of them is built.
+ const source = code(SOURCE)
+ const flip = source.indexOf('daemonBoot.applyStagedUpdate = ')
+ expect(flip, 'the handoff handler is still swapped in cli.ts').toBeGreaterThan(-1)
+ const { body } = callArgument(source, 'const restartForUpdate = ')
+ const declaredAt = localsOf(source)
+ const late = referencedIn(body).filter(used => (declaredAt.get(used) ?? -1) > flip).sort()
+ expect(late, 'these are torn down by restartForUpdate but declared after it is armed').toEqual([])
+ })
+
+ it('the daemon arm survives its own start-up failure', () => {
+ const source = code(SOURCE)
+ const arm = source.slice(source.indexOf("case '__run'"), source.indexOf("case '__run'") + 400)
+ expect(arm).toContain('catch(enterSafeMode)')
+ expect(arm, 'a daemon that exits here can never be updated').not.toContain('catch(onError)')
+ })
})