Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

32.7 KB
Raw Blame
/**
 * harnessd's master: keeps the core running, and nothing else.
 *
 * The core is the process that owns sessions, terminals and turns. The master starts it, restarts it
 * when it crashes, hangs, never finishes starting or outgrows its memory budget, puts it in safe mode
 * when it keeps crashing, and stops it when asked. It holds no sessions, opens no network connection
 * and contains no feature code, so there is almost nothing in it that can fail — which is the point:
 * the daemon comes back even when the desktop app is not running to restart it.
 *
 * The core and the master talk over the spawn channel (`./protocol.ts`). Everything that touches the
 * operating system is injected (`SupervisorDeps`), so every decision here is tested without one.
 */
import {
  CORE_EXIT_STOP,
  CORE_EXIT_UPDATE,
  HARNESSD_PROTOCOL,
  describeStaged,
  heartbeatGraceMs,
  isCoreMessage,
  type CoreMessage,
  type MasterMessage,
  type StagedUpdate,
} from './protocol.js'

export interface CoreHandle {
  readonly pid: number | undefined
  send(message: MasterMessage): void
  kill(signal: NodeJS.Signals): void
  onMessage(listener: (message: unknown) => void): void
  onExit(listener: (code: number | null, signal: NodeJS.Signals | null) => void): void
}

export interface SupervisorDeps {
  /** Start a core with these extra environment variables. */
  spawnCore(env: Record<string, string>): CoreHandle
  /** A monotonic clock, in ms: durations only, untouched by sleep and clock changes. */
  now(): number
  /** The wall clock, in ms: what the status says things happened at. */
  wallClock(): number
  setTimer(run: () => void, ms: number): unknown
  clearTimer(timer: unknown): void
  /** Claim the pid file for the master — the signal `harness start` waits on. */
  claimPidFile(): void
  /** Remove the pid file if it is still the master's. */
  releasePidFile(): void
  /** Put the previous bundle back (`selfUpdate.restore`): the update it replaced failed. */
  restoreUpdate(): void
  /** Drop the previous bundle (`selfUpdate.confirm`): the update came up and stayed up. */
  confirmUpdate(): void
  /** Record the status where `harness status` reads it when no core can answer. */
  writeStatus(status: SupervisorStatus): void
  log(line: string): void
  exit(code: number): void
  /**
   * Replace this master with the bundle on disk before a core is started on it, carrying `state`
   * (./reexec.ts): asked in the gap before every core start, when no core is running, so the swap
   * orphans nothing. `proceed` is called when it did not replace itself, and why. Left out, never.
   */
  reexec?(state: ResumeState, proceed: (outcome: ReexecOutcome) => void): void
  /** A core is up: ready, or bound for a protocol 1 core. A re-executed master has proved itself. */
  coreUp?(): void
  /**
   * The bundle on disk now (its sha256, ./reexec.ts `fingerprint`), or null when it cannot be read: a
   * core judged on an update that exits for another has staged a newer bundle only if this changed
   * since it started. Left out, such an exit is the update failing, as it was before.
   */
  bundle?(): string | null
  /** Start the process that runs [service] on demand (./services.ts `ServiceSupervisor.want`): the core asked for it. */
  want?(service: string): void
  /**
   * The core bound speaking an older protocol than this master (./services.ts `ServiceSupervisor.unasked`): it
   * never asks for a process that became on demand after its protocol, and expects it to run, as it did.
   */
  unasked?(protocol: number): void
  /**
   * Stop the core with `harnessd:shutdown` (./protocol.ts) rather than SIGTERM. Windows: Node's `kill` there is
   * TerminateProcess, whatever the signal, and a core ended that way never flushes its registry or closes its
   * servers. SIGKILL after the grace is TerminateProcess on the handle this master holds, so it can only end its
   * own child. Left out, signals, as on macOS and Linux.
   */
  stopByMessage?: boolean
  /**
   * The core asked to stop everything (`harnessd:stop`: `harness stop` reached it over its pipe). The master stops
   * its services beside the core, as on a signal (./master.ts); left out, the supervisor stops the core itself.
   */
  stopRequested?(why: string): void
}

/**
 * What a master hands the one that replaces it in its own process (./reexec.ts). The crash times and
 * the backoff are left behind: they are on the old process's monotonic clock, which a new one restarts.
 */
export interface ResumeState {
  restarts: number
  lastExit: string | null
  lastExitReason: ExitReason | null
  /** `pending`: the core about to start runs an update, on probation once it is ready. */
  update: 'pending' | null
  /** The pid file is this process's already. */
  claimed: boolean
  /** How many times this process has re-executed. */
  reexecs: number
  /** Of those, how many since a core last came up: a master that keeps re-executing without one is
   *  in a loop, and stops (./reexec.ts `REEXEC_LIMIT`). */
  unproven: number
  /** Windows, which has no exec: the master that handed over to this process, and the token its marker carries
   *  (./reexec.ts `HandOverDeps`). This process claims the pid file and the marker as it starts (./master.ts). */
  handover?: { from: number; token: string }
}

/**
 * Why a master did not replace itself: the bundle on disk is its own code (`same`); it could not or
 * would not, for a reason that is not the bundle's (`kept`); or the bundle's master did not answer its
 * probe (`refused`), which fails an update to it.
 */
export type ReexecOutcome = 'same' | 'kept' | 'refused'

/** Who this master is: its code's version, and what the master it replaced handed on. */
export interface SupervisorIdentity {
  version?: string
  resume?: ResumeState | null
  /**
   * The bundle this master starts on is an update no master kept or rolled back (the version it names,
   * `selfUpdate.unjudgedUpdate`): the master judging it died first (a crash, a kill, a power cut). Its
   * first core is on probation, as it would have been. Before, a fresh master ran it unwatched, and a
   * build whose core crashed at start crash-looped for good (e2e/updateHostile.e2e.ts). Ignored when
   * `resume` is given: a re-executed master carries its own.
   */
  unjudgedUpdate?: string | null
}

export interface SupervisorOptions {
  /** A core that has not said it is bound by then is killed and started again. */
  bindTimeoutMs: number
  /** A bound core that has not said it is ready by then is killed and started again (protocol 2 on). */
  readyTimeoutMs: number
  /** A bound core that sends no heartbeat for this long is hung: killed and started again. */
  heartbeatTimeoutMs: number
  /** How long a core gets to stop after SIGTERM before SIGKILL. */
  stopGraceMs: number
  /** Restart delay: starts here, doubles per crash, caps at `maxBackoffMs`. */
  initialBackoffMs: number
  maxBackoffMs: number
  /** A core that stayed up this long earns the next crash the initial delay again. */
  backoffResetMs: number
  /** The V8 heap limit the core runs with, MiB (`--max-old-space-size`); 0 leaves V8's own. */
  heapLimitMiB: number
  /** Past this share of `heapLimitMiB` the core is restarted cleanly, before V8 aborts it. */
  heapRestartPercent: number
  /** Resident memory past which a core is restarted, MiB — buffers V8 does not count included; 0: off. */
  rssLimitMiB: number
  /** How long a core started on a new bundle must stay up, from ready, before the update is kept. */
  updateProbationMs: number
  /** This many crashes inside `crashLoopWindowMs` start the next core in safe mode. */
  crashLoopCrashes: number
  crashLoopWindowMs: number
}

export const DEFAULT_SUPERVISOR_OPTIONS: SupervisorOptions = {
  bindTimeoutMs: 60_000,
  readyTimeoutMs: 120_000,
  heartbeatTimeoutMs: 30_000,
  // Inside `harness stop`'s own 3 s grace, so the master stops its core and exits before that SIGKILL.
  stopGraceMs: 2_500,
  initialBackoffMs: 500,
  maxBackoffMs: 30_000,
  backoffResetMs: 60_000,
  heapLimitMiB: 4_096,
  heapRestartPercent: 75,
  rssLimitMiB: 6_144,
  updateProbationMs: 30_000,
  crashLoopCrashes: 3,
  crashLoopWindowMs: 300_000,
}

/** `listening`: bound, still starting. `running`: ready (a protocol 1 core is running once bound). */
export type SupervisorState = 'idle' | 'starting' | 'listening' | 'running' | 'restarting' | 'stopping' | 'stopped'

/** Why the last core ended. */
export type ExitReason = 'crashed' | 'hung' | 'did-not-bind' | 'not-ready' | 'memory' | 'update' | 'stopped'

export interface SupervisorStatus {
  state: SupervisorState
  corePid: number | null
  restarts: number
  lastExit: string | null
  lastExitReason: ExitReason | null
  /** Why the core runs in safe mode, or null. */
  safeMode: string | null
  protocol: number
  /** When the state last changed, wall clock ms. */
  since: number
  /** The version of the code this master runs, when it knows it. */
  masterVersion: string | null
  /** How many times this master's process has replaced itself with a newer or restored bundle. */
  reexecs: number
}

type TimerName = 'bindTimer' | 'readyTimer' | 'heartbeatTimer' | 'killTimer' | 'restartTimer' | 'probationTimer' | 'handOverTimer'

/**
 * How long a core asked to hand over for an update (`harnessd:update`) has before the master stops it all
 * the same: its teardown gives up after 15 s (core/updateHandoff.ts) and a grace of 1 s follows, so this
 * is a core that did not hear, or did not act. Its exit then counts as the update's.
 */
export const HAND_OVER_GRACE_MS = 45_000

/**
 * A child's exit code as its platform means it. Windows' is a DWORD, which Node gives unsigned: a process ended by
 * TerminateProcess(-1) (Stop-Process, .NET's Process.Kill) showed as `code 4294967295`, and an NTSTATUS crash as a
 * ten-digit number nobody looks up. Signed, they read as Windows' own tools print them. A Unix code is under 256.
 */
export const signedExitCode = (code: number | null): number | null =>
  code !== null && code > 0x7fff_ffff ? code - 0x1_0000_0000 : code

/** How a child ended, for the log and the status: `signal SIGKILL`, `code 1`, `code -1`. */
export const describeExit = (code: number | null, signal: NodeJS.Signals | null): string =>
  signal ? `signal ${signal}` : `code ${signedExitCode(code)}`

/**
 * Whether why a core could not start names the disk rather than the build: no space left, a quota
 * spent, a file system mounted read-only. Read off the reason a core gives for its safe mode, which is
 * the error it failed on (`listen ENOSPC: no space left on device …` for its socket's claim).
 */
export function namesTheDisk(reason: string): boolean {
  return /\b(ENOSPC|EDQUOT|EROFS)\b|no space left on device|disk quota exceeded|read-only file system/i.test(reason)
}
const MIB = 1024 * 1024

export class Supervisor {
  private state: SupervisorState = 'idle'
  private since: number
  private core: CoreHandle | null = null
  private bound = false
  /** The protocol the running core stated in `bound`; the master's own until it does. */
  private coreProtocol = HARNESSD_PROTOCOL
  private upAt = 0
  private claimed = false
  private restarts = 0
  private backoff: number
  private lastExit: string | null = null
  private lastExitReason: ExitReason | null = null
  /** Why the master is ending the core it is running, if it is: decides what its exit means. */
  private ending: 'stop' | 'restart' | null = null
  /** Why the master killed the running core, when it did. */
  private killReason: ExitReason | null = null
  /** The core on an update could not start for want of room on the disk: its end is no verdict. */
  private noRoom = false
  /** A core exited for an update: the next one runs the new bundle, on probation until it proves it. */
  private update: 'pending' | 'probation' | null = null
  /** Crashes inside the crash-loop window, on the monotonic clock. */
  private crashes: number[] = []
  /** Set while the core is started in safe mode, with the reason. */
  private safeMode: string | null = null
  /** The safe mode the running core says it is in, whoever chose it. */
  private coreSafeMode: string | null = null
  private bindTimer: unknown = null
  private readyTimer: unknown = null
  private probationTimer: unknown = null
  private heartbeatTimer: unknown = null
  private killTimer: unknown = null
  private restartTimer: unknown = null
  private handOverTimer: unknown = null
  private readonly masterVersion: string | null
  private reexecs = 0
  private unproven = 0
  /** The bundle on disk when the running core was started (`deps.bundle`). */
  private spawnedOn: string | null = null
  /** The updater staged a terminal host while the running core ran (`updateStaged`). */
  private ptyStagedSinceSpawn = false

  constructor(
    private readonly deps: SupervisorDeps,
    private readonly options: SupervisorOptions = DEFAULT_SUPERVISOR_OPTIONS,
    identity: SupervisorIdentity = {},
  ) {
    this.backoff = options.initialBackoffMs
    this.since = deps.wallClock()
    this.masterVersion = identity.version ?? null
    const resume = identity.resume
    if (resume) {
      // The master this process was a moment ago, carried on: its core exited for an update (or was
      // rolled back), and the one about to start is judged as it would have judged it.
      this.restarts = resume.restarts
      this.lastExit = resume.lastExit
      this.lastExitReason = resume.lastExitReason
      this.update = resume.update
      this.claimed = resume.claimed
      this.reexecs = resume.reexecs
      this.unproven = resume.unproven
    } else if (identity.unjudgedUpdate) {
      this.update = 'pending'
    }
  }

  status(): SupervisorStatus {
    return {
      state: this.state,
      corePid: this.core?.pid ?? null,
      restarts: this.restarts,
      lastExit: this.lastExit,
      lastExitReason: this.lastExitReason,
      safeMode: this.coreSafeMode ?? this.safeMode,
      protocol: HARNESSD_PROTOCOL,
      since: this.since,
      masterVersion: this.masterVersion,
      reexecs: this.reexecs,
    }
  }

  start(): void {
    if (this.state !== 'idle') return
    this.next()
  }

  /** What this master hands on if it replaces itself now. */
  private resumeState(): ResumeState {
    return {
      restarts: this.restarts, lastExit: this.lastExit, lastExitReason: this.lastExitReason,
      update: this.update === 'pending' ? 'pending' : null, claimed: this.claimed, reexecs: this.reexecs, unproven: this.unproven,
    }
  }

  /**
   * Start the next core, in the one moment no core is running: first, if the bundle on disk is not
   * this master's code (an update the core staged, a rollback that put the previous one back, a bundle
   * replaced by hand), the master replaces itself with it and the new master starts the core. A master
   * left running old code supervised every core after it with that code until something restarted it.
   */
  private next(): void {
    if (!this.deps.reexec) { this.spawn(); return }
    this.deps.reexec(this.resumeState(), (outcome) => {
      // Stopped while it was deciding: the stop has finished this master already.
      if (this.state === 'stopping' || this.state === 'stopped') return
      if (outcome === 'refused' && this.update === 'pending') {
        // A bundle whose master cannot start is an update that failed, as one whose core cannot is.
        this.update = null
        this.deps.restoreUpdate()
        this.deps.log('[harnessd] the new bundle\'s master would not start — rolled back to the previous bundle; restarting')
        this.next()
        return
      }
      this.spawn()
    })
  }

  /**
   * The updater (services/updaterProcess.ts) staged a CLI build, a terminal host (Windows) or both on disk.
   * The core used to stage a build itself and exit 75; now the master asks it to (`harnessd:update`), and its
   * exit is the update's, judged as before: one probation for whatever was staged, which the master's own
   * rollback and keep cover both of (lib/windows/ptyUpdate.ts `rollBackStaged`, `keepStaged`). A core that
   * does not leave within `HAND_OVER_GRACE_MS` is stopped, and that exit counts as the update's too. With no
   * core running (a restart's backoff, safe mode's), the next one starts on the new bundle at once, on
   * probation, with a clean slate: whatever was crashing may be what it fixes.
   */
  updateStaged(staged: StagedUpdate): void {
    if (this.state === 'stopping' || this.state === 'stopped') return
    const what = describeStaged(staged)
    const core = this.core
    if (!core) {
      this.update = 'pending'
      this.crashes = []
      this.safeMode = null
      this.deps.log(`[harnessd] the updater staged ${what} — the next core starts on it`)
      // Waiting out a backoff: no longer. Otherwise the next core is already being started (a
      // re-execution deciding), and starts on the bundle on disk.
      if (this.restartTimer !== null) this.scheduleSpawn(0)
      return
    }
    if (staged.ptyVersion) this.ptyStagedSinceSpawn = true
    if (this.handOverTimer !== null) return
    this.deps.log(`[harnessd] the updater staged ${what} — asking the core to hand over`)
    // A core of any version takes the message only with a version, which it says in its log: for a host alone, the
    // host's, named as one.
    const version = staged.version ?? what
    core.send({ type: 'harnessd:update', version })
    this.armTimer('handOverTimer', () => {
      this.handOverTimer = null
      this.deps.log(`[harnessd] core did not hand over for ${version} within ${HAND_OVER_GRACE_MS} ms — stopping it`)
      this.killReason = 'update'
      this.end('restart', 'SIGTERM')
    }, HAND_OVER_GRACE_MS)
  }

  /** Stop the core and the master. A second call while stopping kills the core outright. */
  stop(reason: string): void {
    if (this.state === 'stopped') return
    if (this.state === 'stopping') {
      this.core?.kill('SIGKILL')
      return
    }
    this.deps.log(`[harnessd] ${reason} — stopping`)
    this.setState('stopping')
    this.clearTimer('restartTimer')
    if (!this.core) { this.finish(0); return }
    this.end('stop', 'SIGTERM')
  }

  private setState(state: SupervisorState): void {
    this.state = state
    this.since = this.deps.wallClock()
    this.publish()
  }

  /** The status to the status file, and to a bound core for its `/api/status`. */
  private publish(): void {
    const status = this.status()
    this.deps.writeStatus(status)
    if (this.bound) this.core?.send({ type: 'harnessd:status', status })
  }

  private spawn(): void {
    this.bound = false
    this.coreProtocol = HARNESSD_PROTOCOL
    this.coreSafeMode = null
    this.ending = null
    this.killReason = null
    // The core is told how long a silence the master allows, and beats well inside it (systemd passes
    // WATCHDOG_USEC the same way): a timeout shorter than the core's own beat would kill a healthy core.
    // HARNESSD_JUDGES_SUPERSEDED: this master judges a build its core on probation stages on its own
    // (#807). A master from before rolls back on any exit during probation, the update's own included,
    // and its core must not stage then: see `stageWhileJudged` in lib/selfUpdate.ts.
    // HARNESSD_UPDATES: updates are this master's to run (services/updaterProcess.ts), not the core's. A core
    // under a master from before it starts the updater beside itself (core/updaterBeside.ts).
    const env: Record<string, string> = {
      HARNESSD_SUPERVISED: '1', HARNESSD_RESTARTS: String(this.restarts), HARNESSD_WATCHDOG_MS: String(this.options.heartbeatTimeoutMs),
      HARNESSD_JUDGES_SUPERSEDED: '1', HARNESSD_UPDATES: 'master',
    }
    if (this.lastExit) env.HARNESSD_LAST_EXIT = this.lastExit
    if (this.safeMode) env.HARNESSD_SAFE_MODE = this.safeMode
    this.spawnedOn = this.deps.bundle?.() ?? null
    this.ptyStagedSinceSpawn = false
    const core = this.deps.spawnCore(env)
    this.core = core
    core.onMessage((message) => { if (this.core === core && isCoreMessage(message)) this.onMessage(core, message) })
    core.onExit((code, signal) => { if (this.core === core) this.onExit(code, signal) })
    this.deps.log(`[harnessd] core started (pid ${core.pid ?? '?'})${this.restarts ? ` · restart ${this.restarts}` : ''}${this.safeMode ? ` · safe mode: ${this.safeMode}` : ''}`)
    this.setState(this.restarts === 0 ? 'starting' : 'restarting')
    this.armTimer('bindTimer', () => this.kill(core, 'did-not-bind', `did not bind within ${this.options.bindTimeoutMs} ms`), this.options.bindTimeoutMs)
  }

  private onMessage(core: CoreHandle, message: CoreMessage): void {
    switch (message.type) {
      case 'harnessd:bound': {
        if (this.bound) return
        this.bound = true
        this.coreProtocol = message.protocol
        this.upAt = this.deps.now()
        this.clearTimer('bindTimer')
        if (!this.claimed) { this.deps.claimPidFile(); this.claimed = true }
        this.deps.log(`[harnessd] core bound (pid ${core.pid ?? '?'}, protocol ${message.protocol})`)
        if (message.protocol < HARNESSD_PROTOCOL) this.deps.unasked?.(message.protocol)
        this.watchHeartbeat()
        // A core from before `ready` is running once bound, and its update is judged from there.
        const readiness = message.protocol >= 2
        if (readiness) {
          this.armTimer('readyTimer', () => this.kill(core, 'not-ready', `bound but not ready within ${this.options.readyTimeoutMs} ms`), this.options.readyTimeoutMs)
        }
        if (this.state !== 'stopping') this.setState(readiness ? 'listening' : 'running')
        if (!readiness) this.up()
        return
      }
      case 'harnessd:ready':
        if (!this.bound || this.state !== 'listening') return
        this.clearTimer('readyTimer')
        if (message.safeMode !== undefined && this.update && namesTheDisk(message.safeMode)) {
          // A disk too full to start on says nothing of the build: the one before would fail the same
          // way, and a rollback that frees the room only for the updater to stage the build again is a
          // loop. Before, the good build was rolled back and remembered as bad (e2e/updateHostile.e2e.ts).
          // Started again, on the same bundle and still on trial, once there may be room.
          this.noRoom = true
          this.kill(core, 'not-ready', `could not start on the new bundle for want of room on the disk (${message.safeMode}) — not a verdict on the build`)
          return
        }
        if (message.safeMode !== undefined && this.update) {
          // The new bundle could not start: that is the update failing, however long it stays up.
          this.kill(core, 'crashed', `started in safe mode on the new bundle (${message.safeMode})`)
          return
        }
        this.upAt = this.deps.now()
        this.coreSafeMode = message.safeMode ?? null
        this.deps.log(`[harnessd] core ready (pid ${core.pid ?? '?'})${message.safeMode === undefined ? '' : ` · in safe mode: ${message.safeMode}`}`)
        this.setState('running')
        this.up()
        return
      case 'harnessd:want':
        // Before its bind too: a core signed in asks for the gateway as it starts, so that the relay comes up
        // beside it, as it did when the gateway started with every other process. A core too old to ask sends
        // none, before or after.
        this.deps.want?.(message.service)
        return
      case 'harnessd:stop': {
        const why = 'the core asked to stop (harness stop)'
        if (this.deps.stopRequested) this.deps.stopRequested(why)
        else this.stop(why)
        return
      }
      case 'harnessd:heartbeat': {
        if (!this.bound) return
        this.watchHeartbeat()
        if (this.ending) return
        const heapBudget = this.options.heapLimitMiB * MIB * this.options.heapRestartPercent / 100
        if (heapBudget && message.heapUsedBytes > heapBudget) {
          this.restartForMemory(`its heap is at ${Math.round(message.heapUsedBytes / MIB)} MiB, past ${this.options.heapRestartPercent}% of its ${this.options.heapLimitMiB} MiB limit`)
        } else if (this.options.rssLimitMiB && message.rssBytes > this.options.rssLimitMiB * MIB) {
          this.restartForMemory(`it is using ${Math.round(message.rssBytes / MIB)} MiB, over its ${this.options.rssLimitMiB} MiB budget`)
        }
        return
      }
    }
  }

  /** A core is up: its update goes on probation, and a re-executed master has proved itself. */
  private up(): void {
    this.beginProbation()
    this.unproven = 0
    this.deps.coreUp?.()
  }

  private restartForMemory(why: string): void {
    this.deps.log(`[harnessd] core ${why} — restarting it`)
    this.killReason = 'memory'
    this.end('restart', 'SIGTERM')
  }

  private beginProbation(): void {
    if (this.update !== 'pending') return
    this.update = 'probation'
    this.armTimer('probationTimer', () => {
      this.probationTimer = null
      // The core on probation has staged a newer build and is on its way out for it (a teardown can
      // take seconds). Kept now, the newer build's pending note and the backup to roll it back to were
      // deleted with this one's, and it ran on probation with nothing to restore. Left on probation, its
      // exit for the update is the superseded one, and the newer build is judged on its own.
      if (this.stagedSinceSpawn()) {
        this.deps.log('[harnessd] the update stayed up, but has staged a newer build — leaving that one to be judged')
        return
      }
      this.update = null
      this.deps.confirmUpdate()
      this.deps.log('[harnessd] the update stayed up — keeping it')
    }, this.options.updateProbationMs)
  }

  private onExit(code: number | null, signal: NodeJS.Signals | null): void {
    const exit = describeExit(code, signal)
    const reason = this.killReason ?? (code === CORE_EXIT_UPDATE ? 'update' : 'crashed')
    const wasSafe = this.safeMode ?? this.coreSafeMode
    for (const timer of ['bindTimer', 'readyTimer', 'heartbeatTimer', 'killTimer', 'probationTimer', 'handOverTimer'] as const) this.clearTimer(timer)
    this.core = null
    this.bound = false
    this.coreSafeMode = null
    if (this.state === 'stopping') {
      this.deps.log(`[harnessd] core stopped (${exit})`)
      this.lastExit = exit
      this.lastExitReason = 'stopped'
      this.finish(0)
      return
    }
    this.lastExit = exit
    this.lastExitReason = reason
    this.restarts++
    // A core still being judged on an update that exits for another has staged a newer bundle: the
    // build it ran came up far enough to find the next and stage it, and the next is judged on its own.
    // Read as the update failing, the newer build was rolled back and remembered as bad on every machine
    // that found it while the one before was on probation (e2e/updateHostile.e2e.ts: a fix published
    // moments after the release it fixes).
    const superseded = this.update !== null && reason === 'update' && this.stagedSinceSpawn()
    const noRoom = this.noRoom
    this.noRoom = false
    if (this.update && noRoom) {
      // Still on trial: the next core on this bundle is judged on it, from ready. Backed off like a
      // crash, without counting as one: safe mode would not make room either.
      const delay = this.backoff
      this.backoff = Math.min(this.backoff * 2, this.options.maxBackoffMs)
      this.deps.log(`[harnessd] the updated core had no room to start (${exit}) — keeping the update on trial; trying again in ${delay} ms`)
      this.scheduleSpawn(delay)
      return
    }
    if (this.update && !superseded) {
      // The core on the new bundle did not come up, or did not stay up: the bundle before it did.
      this.update = null
      this.deps.restoreUpdate()
      this.deps.log(`[harnessd] the updated core failed (${exit}) — rolled back to the previous bundle; restarting`)
      this.scheduleSpawn(0)
      return
    }
    // A deliberate end: signed out for good, removed from the account, connected from elsewhere. A core
    // started again would only end itself again; the master goes with it. Protocol 1 cores said so with 0.
    const forGood = signal === null && this.ending !== 'restart' && (code === CORE_EXIT_STOP || (code === 0 && this.coreProtocol < 2))
    if (forGood) {
      this.deps.log(`[harnessd] core stopped for good (${exit}) — stopping`)
      this.lastExitReason = 'stopped'
      this.finish(0)
      return
    }
    if (reason === 'update') {
      // A new bundle, and a clean slate: whatever was crashing may be what it fixes.
      this.update = 'pending'
      this.crashes = []
      this.safeMode = null
      this.deps.log(superseded
        ? `[harnessd] the updated core staged a newer build before it was kept (${exit}) — restarting onto that one`
        : `[harnessd] core exited (${exit}) for an update — restarting`)
      this.scheduleSpawn(0)
      return
    }
    if (wasSafe) {
      // Safe mode ran its course without a fix arriving: a normal core gets another try.
      this.crashes = []
      this.safeMode = null
    } else {
      const now = this.deps.now()
      this.crashes = [...this.crashes.filter((at) => now - at < this.options.crashLoopWindowMs), now]
      if (this.crashes.length >= this.options.crashLoopCrashes) {
        this.safeMode = 'crash-loop'
        this.deps.log(`[harnessd] core crashed ${this.crashes.length} times in ${Math.round(this.options.crashLoopWindowMs / 60_000)} min — starting it in safe mode`)
      }
    }
    if (this.upAt && this.deps.now() - this.upAt >= this.options.backoffResetMs) this.backoff = this.options.initialBackoffMs
    // A memory restart backs off like a crash: a core over budget from the start would otherwise be
    // restarted as fast as it can bind. Only an update restarts at once.
    const delay = this.backoff
    this.backoff = Math.min(this.backoff * 2, this.options.maxBackoffMs)
    this.upAt = 0
    this.deps.log(`[harnessd] core exited (${exit}, ${reason}) — restarting in ${delay} ms`)
    this.scheduleSpawn(delay)
  }

  /** Whether something newer was staged since the running core was started: the bundle on disk changed
   *  (`deps.bundle`), or the updater staged a terminal host, which leaves the bundle as it was. */
  private stagedSinceSpawn(): boolean {
    if (this.ptyStagedSinceSpawn) return true
    const now = this.deps.bundle?.() ?? null
    return now !== null && now !== this.spawnedOn
  }

  private scheduleSpawn(delay: number): void {
    this.setState('restarting')
    this.armTimer('restartTimer', () => { this.restartTimer = null; this.next() }, delay)
  }

  /** Kill a core that broke a promise — to bind, to be ready, to beat — and remember which. */
  private kill(core: CoreHandle, reason: ExitReason, what: string): void {
    this.deps.log(`[harnessd] core ${what} — killing it`)
    this.killReason = reason
    core.kill('SIGKILL')
  }

  /** End the running core: SIGTERM (or SIGKILL), and SIGKILL if it outlives the grace. On Windows the SIGTERM is
   *  `harnessd:shutdown` (`stopByMessage`). */
  private end(why: 'stop' | 'restart', signal: NodeJS.Signals): void {
    const core = this.core!
    this.ending = why
    this.clearTimer('heartbeatTimer')
    this.clearTimer('readyTimer')
    // Every caller ends a core with SIGTERM; a second stop is SIGKILL through `stop`, never through here.
    if (this.deps.stopByMessage) core.send({ type: 'harnessd:shutdown' })
    else core.kill(signal)
    this.armTimer('killTimer', () => {
      this.deps.log(`[harnessd] core outlived its ${this.options.stopGraceMs} ms to stop — killing it`)
      core.kill('SIGKILL')
    }, this.options.stopGraceMs)
  }

  /**
   * ⚠️ A silence the master slept through is not a hang. Ctrl-Z on a daemon run in a terminal, a
   * paused virtual machine or a swap storm stops the master with its core, and on resume every timer
   * that fell due fires at once, before the master reads the beat the resumed core sends a moment
   * later. Measured end to end (`e2e/paused.e2e.ts`): paused for 40 s, the core was killed as hung the
   * instant it resumed, and the message an app had sent into its socket was lost with it. So a silence
   * that runs out gets one beat's time more, counted from now, while this master is running.
   */
  private watchHeartbeat(): void {
    const core = this.core!
    const timeout = this.options.heartbeatTimeoutMs
    const grace = heartbeatGraceMs(timeout)
    this.armTimer('heartbeatTimer', () => {
      this.deps.log(`[harnessd] core sent no heartbeat for ${timeout} ms — giving it ${grace} ms more, in case this master was paused too`)
      this.armTimer('heartbeatTimer', () => this.kill(core, 'hung', `sent no heartbeat for ${timeout + grace} ms — it is hung;`), grace)
    }, timeout)
  }

  private finish(code: number): void {
    this.clearTimer('restartTimer')
    this.clearTimer('probationTimer')
    this.setState('stopped')
    if (this.claimed) this.deps.releasePidFile()
    this.deps.exit(code)
  }

  private armTimer(name: TimerName, run: () => void, ms: number): void {
    this.clearTimer(name)
    this[name] = this.deps.setTimer(run, ms)
  }

  private clearTimer(name: TimerName): void {
    if (this[name] !== null) this.deps.clearTimer(this[name])
    this[name] = null
  }
}