Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
34.2 KB
Raw Blame
/**
 * ChildWorkerSupervisor — Pure spawn-and-respawn core for child workers.
 *
 * Extracted from MinionSupervisor (src/core/minions/supervisor.ts) so it can
 * be reused by both MinionSupervisor (standalone `gbrain jobs supervisor`
 * daemon) and the autopilot command (src/commands/autopilot.ts). Pre-fix
 * those two had separate spawn loops with different crash-counting bugs;
 * this class is the single source of truth.
 *
 * RESPONSIBILITIES:
 *   - Spawn the child process (with optional tini wrapper for zombie reaping).
 *   - Await exit, classify the exit code, decide whether to respawn.
 *   - Track crash count and trip max_crashes for real failures (code != 0).
 *   - Track clean-restart budget for code=0 exits and apply backoff when
 *     the watchdog-drain rate exceeds the budget (D2 in plan).
 *   - Emit lifecycle events via injected callback.
 *
 * NON-RESPONSIBILITIES (these stay in the composing class):
 *   - PID file locking.
 *   - Signal handlers (SIGTERM/SIGINT).
 *   - process.exit() — composer decides what to do on max_crashes.
 *   - Health checks (DB probing, queue depth).
 *   - Audit-log writing (composer's onEvent decides where it lands).
 *
 * EXIT CLASSIFIER (post-D1/D2):
 *
 *   code === 0  -> crashCount UNCHANGED (preserves flap detection across
 *                  mixed exit sequences). Record clean-restart timestamp.
 *                  If clean-restart budget exceeded -> emit health_warn +
 *                  apply backoff. Else emit ms:0 backoff (immediate restart).
 *
 *   code !== 0  AND runDuration > stableRunResetMs -> crashCount = 1
 *                  (stable run forgives prior crash history).
 *
 *   code !== 0  AND runDuration <= stableRunResetMs -> crashCount++
 *                  (escalating exponential backoff).
 */

import { spawn, type ChildProcess } from '../spawn.ts';
import { existsSync, mkdtempSync, realpathSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { basename, isAbsolute, join, resolve as resolvePath } from 'node:path';
import { buildSpawnInvocation, detectTini } from './spawn-helpers.ts';
import { classifyWorkerExit } from './exit-classification.ts';
import { calculateBackoffMs } from './supervisor.ts';
import { WORKER_EXIT_CONFIGURATION, WORKER_EXIT_RSS_WATCHDOG } from './worker-exit-codes.ts';
import type { OwnerProcessingState, OwnerProcessingStatus } from './processing-state.ts';
import { killProcessGroup, observeTiniChildProcessGroups } from './job-isolation.ts';
import { CWD_ENV_QUARANTINED_MARKER } from '../cli-preflight.ts';

export type ChildSupervisorEvent =
  | { kind: 'worker_spawned'; pid: number; tini: boolean }
  | { kind: 'worker_startup_timeout'; timeoutMs: number }
  | { kind: 'worker_spawn_failed'; error: string; phase: 'sync' | 'async'; errnoCode?: string }
  | {
      kind: 'worker_exited';
      /** The exited child's pid (absent when the spawn never produced a process). */
      pid?: number;
      code: number | null;
      signal: NodeJS.Signals | null;
      runDurationMs: number;
      likelyCause: string;
      crashCount: number;
    }
  | {
      kind: 'backoff';
      ms: number;
      crashCount: number;
      reason: 'clean_exit' | 'crash' | 'budget_exceeded' | 'rss_watchdog' | 'wedge_restart';
    }
  | {
      kind: 'health_warn';
      reason: 'clean_restart_budget_exceeded' | 'rss_watchdog_loop' | 'crash_budget_degraded';
      count: number;
      /** Present for window-scoped warnings (budget/watchdog loops). */
      windowMs?: number;
      /** Present for crash_budget_degraded: the soft budget that was crossed. */
      max?: number;
    };

export interface ChildWorkerSupervisorOpts {
  processingState?: OwnerProcessingState;
  onConfigurationBlocked?: (status: OwnerProcessingStatus | null) => void;
  _startupReadyTimeoutMs?: number;
  _startupKillGraceMs?: number;
  /** Path to the gbrain CLI binary. */
  cliPath: string;
  /** Worker argv after cliPath (e.g. ['jobs', 'work', '--max-rss', '2048']). */
  args: string[];
  /** Child env. Defaults to a clone of process.env. */
  env?: NodeJS.ProcessEnv;
  /**
   * Soft crash budget. issue #1994 (#2227 tail): crossing this NO LONGER
   * permanently gives up. Instead the supervisor enters DEGRADED mode — it
   * keeps respawning with capped exponential backoff (60s cap) and emits a
   * loud `crash_budget_degraded` health_warn — so a transient DB-pooler outage
   * that trips the counter self-heals when the DB returns (the stable-run reset
   * clears crashCount once a respawn runs > stableRunResetMs) instead of
   * wedging the queue until a human restart.
   */
  maxCrashes: number;
  /**
   * Hard ceiling: permanently give up (fire onMaxCrashesExceeded) ONLY after
   * this many consecutive crashes — the runaway backstop for a genuinely
   * unrecoverable hot crash-loop that the stable-run reset never escapes.
   * Default: maxCrashes * HARD_STOP_CRASH_MULTIPLIER. Set to 0 to disable
   * permanent give-up entirely (retry-forever-with-backoff).
   */
  hardStopMaxCrashes?: number;
  /** Stable-run reset window: code != 0 after this duration resets crashCount to 1. Default 5 min. */
  stableRunResetMs?: number;

  /**
   * D2 clean-restart budget. Caps the rate of code=0 restarts so the
   * supervisor cannot tight-loop when the worker exits cleanly forever
   * (e.g. macOS RSS fallback path always over-threshold). When the count
   * of clean restarts inside `cleanRestartWindowMs` exceeds this number,
   * emit `health_warn` and apply `cleanRestartBudgetBackoffMs` before
   * the next spawn. Default 10.
   */
  cleanRestartBudget?: number;
  /** Sliding-window size for budget tracking. Default 60 seconds. */
  cleanRestartWindowMs?: number;
  /** Backoff applied when budget is exceeded. Default 1 second. */
  cleanRestartBudgetBackoffMs?: number;

  /**
   * v0.42.5.0 (issue #1678) — RSS-watchdog loop breaker, cause-keyed and
   * INDEPENDENT of crashCount/max_crashes. A watchdog drain
   * (WORKER_EXIT_RSS_WATCHDOG) means the worker hit its memory cap, not that
   * the code is defective — so it does NOT count toward max_crashes (that
   * would stop ALL job processing, worse than slow-looping). Instead we
   * always apply `watchdogBackoffMs` (so an instant-OOM-on-startup can't
   * hot-loop) and, when more than `watchdogLoopBudget` watchdog exits land
   * inside `watchdogLoopWindowMs`, emit a loud `rss_watchdog_loop` health_warn
   * so the operator sees "raise --max-rss" instead of chasing a phantom
   * connection/lock failure. NOTE: the stable-run reset that defeats
   * max_crashes for >5-min runs is exactly why a SEPARATE window is required
   * here — routing watchdog exits through the crash path would never trip.
   */
  watchdogLoopBudget?: number;
  /** Sliding window for the watchdog-loop breaker. Default 10 minutes. */
  watchdogLoopWindowMs?: number;
  /** Backoff applied after every watchdog drain. Default 30 seconds. */
  watchdogBackoffMs?: number;

  /**
   * Test-only override: minimum backoff in ms between child respawns.
   * Tests pass `1` to make crash-loops finish in < 1s. Not exposed via CLI.
   * @internal
   */
  _backoffFloorMs?: number;

  /** Lifecycle event callback. Composer routes these to its own log/audit channels. */
  onEvent: (event: ChildSupervisorEvent) => void;
  /** Called when crashCount reaches maxCrashes. Composer decides what to do (process.exit, shutdown, etc.). */
  onMaxCrashesExceeded: (count: number, max: number) => void;
  /** Accessor for the composer's stopping flag; loop exits when this returns true. */
  isStopping: () => boolean;

  /**
   * Optional fenced maintenance hook run immediately before EVERY child spawn,
   * including crash/watchdog respawns. The composer owns error handling AND
   * bounding: a rejection propagates out of run() with no crash accounting or
   * respawn, and the await is not isStopping-checked — so composers must
   * try/catch and time-bound the hook themselves (MinionSupervisor wraps its
   * recovery hook in a 30s race and spawns on failure). Reserve a bare
   * rethrowing hook for genuine spawn-blocking safety preconditions.
   */
  beforeSpawn?: () => Promise<void>;

  /** Test seed for the clean-restart window. Defaults to Date.now. @internal */
  _now?: () => number;
}

/** issue #1994: how many multiples of the soft crash budget before the hard
 *  permanent-give-up backstop fires. 10× the default soft budget of 10 = 100
 *  consecutive crashes (each within the stable-run window) before we conclude
 *  it's a genuine code bug and stop. A transient outage recovers long before. */
export const HARD_STOP_CRASH_MULTIPLIER = 10;
export const WORKER_STARTUP_READY_TIMEOUT_MS = 120_000;

const DEFAULTS = {
  stableRunResetMs: 5 * 60 * 1000,
  cleanRestartBudget: 10,
  cleanRestartWindowMs: 60_000,
  cleanRestartBudgetBackoffMs: 1_000,
  watchdogLoopBudget: 3,
  watchdogLoopWindowMs: 10 * 60 * 1000,
  watchdogBackoffMs: 30_000,
} as const;

export class ChildWorkerSupervisor {
  private readonly opts: ChildWorkerSupervisorOpts;
  private readonly tiniPath: string;
  private _crashCount = 0;
  private _lastExitCode: number | null = null;
  private _lastExit: { code: number | null; signal: NodeJS.Signals | null } | null = null;
  private _cleanRestartTimestamps: number[] = [];
  /** Sliding window of RSS-watchdog exit timestamps (issue #1678). Separate
   *  from crashCount so the >5-min stable-run reset can't defeat the breaker. */
  private _watchdogExitTimestamps: number[] = [];
  private _child: ChildProcess | null = null;
  private _inBackoff = false;
  private _lastStartTime = 0;
  /** issue #1801: set by restartCurrentChild() before a deliberate wedge
   *  SIGTERM so the exit handler skips crash accounting. Cleared on exit. */
  private _intentionalRestart = false;
  /** issue #1801: carried from the exit handler to applyBackoff so a wedge
   *  self-heal respawns immediately (ms:0) instead of paying crash backoff. */
  private _lastWasIntentionalRestart = false;
  private _configurationBlocked = false;
  private wakeBackoff: (() => void) | null = null;

  constructor(opts: ChildWorkerSupervisorOpts) {
    this.opts = opts;
    this.tiniPath = detectTini();
  }

  /** Read-only state surfaces for the composing class's health checks. */
  get childAlive(): boolean {
    return this._child !== null && this._child.exitCode === null;
  }
  get inBackoff(): boolean {
    return this._inBackoff;
  }
  get crashCount(): number {
    return this._crashCount;
  }
  /** #5062: how the most recent child exited, recorded on the shutdown path
   *  too, so a composer can tell a clean drain (code 0) from a forced or
   *  failed stop. */
  get lastExit(): { code: number | null; signal: NodeJS.Signals | null } | null {
    return this._lastExit;
  }
  get configurationBlocked(): boolean {
    if (this.opts.processingState?.blocked) this.markConfigurationBlocked();
    return this._configurationBlocked;
  }

  private markConfigurationBlocked(): void {
    if (this._configurationBlocked) return;
    this._configurationBlocked = true;
    this.wakeBackoff?.();
    this.opts.processingState?.block();
    this.opts.onConfigurationBlocked?.(this.opts.processingState?.snapshot ?? null);
  }
  /** Whether tini was detected at construction. Used by tests + worker_spawned event payload. */
  get isTiniDetected(): boolean {
    return this.tiniPath !== '';
  }

  /**
   * Send a signal to the live child (no-op if none). Used by composers'
   * shutdown paths. Idempotent — `kill('SIGTERM')` on a dead child is a no-op.
   */
  killChild(signal: NodeJS.Signals): void {
    // Gate on LIVENESS, not `.killed`. Node's `child.killed` flips true the
    // moment a signal has been *sent*, not when the process exits — so a guard
    // of `!this._child.killed` makes a follow-up SIGKILL (after an ignored
    // SIGTERM) a silent no-op. That bug bit both the wedge escalation AND the
    // existing shutdown() drain (issue #1801, Codex #1). Checking exitCode /
    // signalCode === null means "still running," so SIGKILL actually lands on a
    // process that ignored SIGTERM.
    const child = this._child;
    if (child && child.exitCode === null && child.signalCode === null) {
      try {
        child.kill(signal);
      } catch {
        /* already dead */
      }
    }
  }

  /**
   * Wait for the current child to exit, bounded by `timeoutMs`. No-op if no
   * child is running. Used by composers' graceful-shutdown drains.
   *
   * Handles the already-exited case: if the child terminated between
   * `killChild('SIGTERM')` and this call (common on fast SIGTERM
   * responders), Node's `'exit'` event has already fired and a late
   * `once('exit', ...)` listener would never resolve. We probe
   * `child.exitCode !== null` first and short-circuit so clean shutdown
   * is sub-second instead of waiting out the full `timeoutMs`.
   */
  awaitChildExit(timeoutMs: number): Promise<void> {
    if (!this._child) return Promise.resolve();
    return this._awaitExit(this._child, timeoutMs);
  }

  /**
   * awaitChildExit, bound to a SPECIFIC captured child reference rather than
   * `this._child`. The wedge-restart path (issue #1801) must wait on the child
   * it SIGTERMed, not whatever `this._child` points at later — by the time the
   * grace window elapses, `run()` may have respawned a fresh child, and a
   * `this._child`-based wait/kill would hit the replacement (Codex #2).
   */
  private _awaitExit(child: ChildProcess, timeoutMs: number): Promise<void> {
    // Already exited? `exitCode` becomes non-null once Node has seen the
    // child terminate. `signalCode` is the symmetric flag for kill-signal
    // termination — checked too so a SIGKILLed child also short-circuits.
    if (child.exitCode !== null || child.signalCode !== null) {
      return Promise.resolve();
    }
    return new Promise<void>((resolve) => {
      let settled = false;
      const onExit = () => {
        if (settled) return;
        settled = true;
        resolve();
      };
      child.once('exit', onExit);
      setTimeout(() => {
        if (settled) return;
        settled = true;
        child.removeListener('exit', onExit);
        resolve();
      }, timeoutMs);
    });
  }

  /**
   * Intentional restart of the CURRENT child — the supervisor's wedge watchdog
   * (issue #1801) calls this when a worker is alive but making no forward
   * progress (dead pool, stuck handler). Captures the live child reference,
   * SIGTERMs it, waits up to `graceMs` on THAT captured ref, then SIGKILLs THAT
   * ref if it ignored SIGTERM. Operating on the captured reference (never
   * `this._child`) closes the race where `run()` respawns a fresh child during
   * the grace window (Codex #2). The exit is flagged intentional so it does NOT
   * count toward crashCount / max_crashes (Codex #3) — a deliberate self-heal,
   * not a worker defect; the caller bounds repetition via its wedge-restart
   * loop budget. `run()` respawns a fresh worker (and a fresh DB pool) on exit.
   */
  async restartCurrentChild(graceMs: number): Promise<void> {
    if (this.configurationBlocked) return;
    const child = this._child;
    if (!child) return;
    this._intentionalRestart = true;
    // SIGTERM the captured child (liveness-gated, same fix as killChild).
    if (child.exitCode === null && child.signalCode === null) {
      try {
        child.kill('SIGTERM');
      } catch {
        /* already dead */
      }
    }
    await this._awaitExit(child, graceMs);
    // Escalate to SIGKILL only if the CAPTURED child is still alive — never the
    // respawned one.
    if (child.exitCode === null && child.signalCode === null) {
      try {
        child.kill('SIGKILL');
      } catch {
        /* already dead */
      }
    }
  }

  /**
   * Run the spawn-and-respawn loop. Resolves when:
   *   1. composer.isStopping() returns true, OR
   *   2. crashCount reaches the HARD ceiling (after firing onMaxCrashesExceeded).
   *
   * issue #1994 (#2227 tail): crossing the SOFT budget (`maxCrashes`) no longer
   * stops the supervisor. It enters degraded mode — keep respawning with capped
   * exponential backoff (60s cap, so it's a paced retry, not a hot loop) and
   * announce loudly — so a transient DB-pooler outage that trips the counter
   * recovers on its own (a respawn that runs > stableRunResetMs resets
   * crashCount to 1) instead of permanently wedging the queue. Permanent
   * give-up fires only at the much-higher hard ceiling: the runaway backstop
   * for a genuinely unrecoverable hot crash-loop.
   */
  async run(): Promise<void> {
    const statusTimer = this.opts.processingState
      ? setInterval(() => { void this.configurationBlocked; }, 100)
      : null;
    try {
      const hardStop = this.opts.hardStopMaxCrashes ??
        this.opts.maxCrashes * HARD_STOP_CRASH_MULTIPLIER;
      let degradedAnnounced = false;
      while (!this.opts.isStopping() && !this.configurationBlocked) {
        try {
          await this.opts.beforeSpawn?.();
        } catch (error) {
          if (this.configurationBlocked) return;
          throw error;
        }
        if (this.opts.isStopping() || this.configurationBlocked) return;
        await this.spawnOnce();

        if (this.configurationBlocked || this.opts.isStopping()) return;

        // Hard ceiling: permanent give-up (the runaway backstop). hardStop <= 0
        // disables it entirely (retry-forever-with-backoff for deployments that
        // would rather never auto-stop a recoverable supervisor).
        if (hardStop > 0 && this._crashCount >= hardStop) {
          this.opts.processingState?.waiting('stopped');
          this.opts.onMaxCrashesExceeded(this._crashCount, hardStop);
          return;
        }

        // Soft budget crossed → degraded mode. Announce once per degradation
        // episode; re-arm after a stable-run reset drops us back under budget.
        if (this._crashCount >= this.opts.maxCrashes) {
          if (!degradedAnnounced) {
            degradedAnnounced = true;
            this.opts.onEvent({
              kind: 'health_warn',
              reason: 'crash_budget_degraded',
              count: this._crashCount,
              max: this.opts.maxCrashes,
            });
          }
        } else {
          degradedAnnounced = false;
        }

        await this.applyBackoff();
      }
    } finally {
      if (statusTimer) clearInterval(statusTimer);
    }
  }

  /** Single spawn lifecycle: spawn -> await exit -> classify. */
  private spawnOnce(): Promise<void> {
    return new Promise<void>((resolve) => {
      if (this.opts.isStopping() || this.configurationBlocked) {
        resolve();
        return;
      }

      const env: NodeJS.ProcessEnv = { ...(this.opts.env ?? process.env) };
      this._lastStartTime = this.now();
      const originalCwd = process.cwd();
      const args = [...this.opts.args];
      if (this.opts.processingState && ['bun', 'bun.exe', 'node', 'node.exe'].includes(basename(this.opts.cliPath)) &&
          args[0] && !args[0].startsWith('-') && !isAbsolute(args[0]) && existsSync(resolvePath(originalCwd, args[0]))) {
        args[0] = resolvePath(originalCwd, args[0]);
      }

      const { cmd: spawnCmd, args: spawnArgs } = buildSpawnInvocation(
        this.tiniPath,
        this.opts.processingState && !isAbsolute(this.opts.cliPath) && /[/\\]/.test(this.opts.cliPath)
          ? resolvePath(this.opts.cliPath) : this.opts.cliPath,
        args,
      );

      let child: ChildProcess;
      let neutralCwd: string | undefined;
      let startupTimer: ReturnType<typeof setInterval> | undefined;
      let killTimer: ReturnType<typeof setTimeout> | undefined;
      let startupTimedOut = false;
      const processGroups = new Set<number>();
      const cleanup = () => {
        if (startupTimer) clearInterval(startupTimer);
        if (killTimer) clearTimeout(killTimer);
        if (neutralCwd) {
          try { rmSync(neutralCwd, { recursive: true, force: true }); } catch {}
        }
      };
      try {
        if (this.opts.processingState) {
          Object.assign(env, this.opts.processingState.prepareChild());
          neutralCwd = realpathSync(mkdtempSync(join(tmpdir(), 'gbrain-worker-')));
          env[CWD_ENV_QUARANTINED_MARKER] = JSON.stringify({ cwd: originalCwd, neutral: neutralCwd });
        }
        child = spawn(spawnCmd, spawnArgs, {
          stdio: 'inherit',
          env,
          ...(neutralCwd ? { cwd: neutralCwd } : {}),
        });
      } catch (err: unknown) {
        cleanup();
        this.opts.processingState?.workerExited();
        // Synchronous spawn error (e.g. invalid cliPath shape). Count as a crash.
        this.opts.onEvent({
          kind: 'worker_spawn_failed',
          error: err instanceof Error ? err.message : String(err),
          phase: 'sync',
        });
        this._crashCount++;
        this._lastExitCode = null;
        resolve();
        return;
      }

      this._child = child;

      if (this.opts.processingState) {
        const deadlineMs = this.opts._startupReadyTimeoutMs ?? WORKER_STARTUP_READY_TIMEOUT_MS;
        const deadline = this.now() + deadlineMs;
        const signalStartupChild = (signal: 'SIGTERM' | 'SIGKILL') => {
          if (child.pid && this.tiniPath) observeTiniChildProcessGroups(child.pid, processGroups);
          for (const group of processGroups) killProcessGroup(group, signal);
          if (!this.tiniPath || processGroups.size === 0 || signal === 'SIGTERM') {
            try { child.kill(signal); } catch {}
          }
        };
        startupTimer = setInterval(() => {
          if (this.configurationBlocked || this.opts.isStopping() || this.opts.processingState!.snapshot.processing_ready) {
            clearInterval(startupTimer);
            return;
          }
          if (this.now() < deadline) return;
          clearInterval(startupTimer);
          if (!this.opts.processingState!.waiting('startup_timeout')) return;
          startupTimedOut = true;
          this.opts.onEvent({ kind: 'worker_startup_timeout', timeoutMs: deadlineMs });
          signalStartupChild('SIGTERM');
          killTimer = setTimeout(() => signalStartupChild('SIGKILL'), this.opts._startupKillGraceMs ?? 35_000);
        }, Math.min(100, Math.max(1, deadlineMs)));
      }

      this.opts.onEvent({
        kind: 'worker_spawned',
        pid: child.pid ?? -1,
        tini: this.tiniPath !== '',
      });

      // Settle-once guard shared by the 'exit' path (the child ran) and the
      // spawn-failure path (the child never became a process).
      let settled = false;
      let spawnErrored = false;

      /**
       * The child never launched — ENOENT/EACCES on `cliPath`, or a target the
       * OS refuses to execute (e.g. a `.sh` on Windows). Node and Bun emit
       * 'error' and then 'close' for such a spawn but NEVER 'exit', so the
       * 'exit' handler below can never settle this promise.
       *
       * Without this path `run()` awaits a promise that can never resolve: the
       * supervisor wedges forever on the FIRST bad spawn — no respawn, no crash
       * accounting, no give-up — which is the exact opposite of the bounded-retry
       * contract this class exists to provide. (The comment that used to live
       * here claimed 'exit' fires after 'error'; it does not.)
       *
       * A worker that can never start is a crash: increment `_crashCount` so it
       * pays the normal exponential backoff in applyBackoff() and is ultimately
       * bounded by `hardStopMaxCrashes`, the same as any other permanent misconfig.
       */
      const settleSpawnFailure = () => {
        if (settled) return;
        settled = true;
        cleanup();
        this._child = null;
        this._intentionalRestart = false;

        if (this.opts.isStopping()) {
          resolve();
          return;
        }

        const runDuration = this.now() - this._lastStartTime;
        this._lastExitCode = null;
        this._crashCount++;

        this.opts.onEvent({
          kind: 'worker_exited',
          pid: child.pid,
          code: null,
          signal: null,
          runDurationMs: runDuration,
          likelyCause: 'spawn_failed',
          crashCount: this._crashCount,
        });

        resolve();
      };

      // Async spawn errors (ENOENT, EACCES).
      child.on('error', (err) => {
        spawnErrored = true;
        this.opts.onEvent({
          kind: 'worker_spawn_failed',
          error: err.message,
          phase: 'async',
          errnoCode: (err as NodeJS.ErrnoException).code,
        });
        // No pid means the OS never created a process, so no 'exit' is coming.
        // Settle now instead of awaiting an event that can never fire.
        if (child.pid === undefined) settleSpawnFailure();
      });

      // Belt-and-braces for a failed spawn that did get a pid (platform-dependent
      // EACCES shapes). Gated on `spawnErrored` so a normal run — which never
      // emits 'error' — can't have its classified 'exit' path pre-empted by
      // 'close', whose ordering relative to 'exit' is not guaranteed.
      child.on('close', () => {
        if (spawnErrored) settleSpawnFailure();
      });

      child.on('exit', (code, signal) => {
        if (settled) return;
        settled = true;
        cleanup();
        this._child = null;
        this._lastExit = { code, signal };

        if (code === WORKER_EXIT_CONFIGURATION) {
          this.opts.processingState?.workerExited();
          this._lastExitCode = code;
          this.markConfigurationBlocked();
          this.opts.onEvent({ kind: 'worker_exited', pid: child.pid, code, signal: signal ?? null,
            runDurationMs: this.now() - this._lastStartTime, likelyCause: 'configuration_blocked', crashCount: this._crashCount });
          resolve();
          return;
        }
        this.opts.processingState?.workerExited();
        if (this.configurationBlocked) {
          resolve();
          return;
        }

        if (this.opts.isStopping()) {
          resolve();
          return;
        }

        const runDuration = this.now() - this._lastStartTime;

        // D1: code=0 is a clean exit (watchdog drain, graceful stop, etc.).
        // Don't touch crashCount — preserves flap detection across mixed
        // exit sequences. D2: record the clean-restart timestamp for budget
        // tracking and prune entries outside the sliding window. Routes
        // through the shared `classifyWorkerExit` helper so doctor.ts and
        // jobs.ts (audit-log consumers) read the same rule.
        this._lastExitCode = startupTimedOut ? 1 : code;
        if (startupTimedOut) {
          this._intentionalRestart = false;
          this._crashCount++;
        } else if (this._intentionalRestart) {
          // issue #1801: deliberate wedge self-heal (restartCurrentChild). Leave
          // crashCount UNTOUCHED — like the RSS watchdog — so a recurring wedge
          // never trips max_crashes and kills the daemon. A SIGTERM exit has
          // code=null, which classifyWorkerExit() (correctly) treats as a crash,
          // so without this guard the wedge restart would increment crashCount
          // (Codex #3). The supervisor's wedgeRestartLoopBudget bounds repetition.
          this._intentionalRestart = false;
          this._lastWasIntentionalRestart = true;
        } else if (code === WORKER_EXIT_RSS_WATCHDOG) {
          // issue #1678: RSS-watchdog drain. NOT a code defect — leave
          // crashCount untouched so it never trips max_crashes (which would
          // stop ALL job processing). Tracked in its own window so the
          // breaker survives the >5-min stable-run reset that defeats the
          // generic crash path.
          const nowMs = this.now();
          this._watchdogExitTimestamps.push(nowMs);
          const windowMs = this.opts.watchdogLoopWindowMs ?? DEFAULTS.watchdogLoopWindowMs;
          const cutoff = nowMs - windowMs;
          this._watchdogExitTimestamps = this._watchdogExitTimestamps.filter(
            (t) => t > cutoff,
          );
        } else if (classifyWorkerExit({ code }) === 'clean_exit') {
          const nowMs = this.now();
          this._cleanRestartTimestamps.push(nowMs);
          const windowMs = this.opts.cleanRestartWindowMs ?? DEFAULTS.cleanRestartWindowMs;
          const cutoff = nowMs - windowMs;
          this._cleanRestartTimestamps = this._cleanRestartTimestamps.filter(
            (t) => t > cutoff,
          );
        } else {
          const resetMs = this.opts.stableRunResetMs ?? DEFAULTS.stableRunResetMs;
          if (runDuration > resetMs) {
            // Stable-run reset: forgive prior crash history.
            this._crashCount = 1;
          } else {
            this._crashCount++;
          }
        }

        // Likely-cause heuristic, kept verbatim from MinionSupervisor.
        let likelyCause: string;
        if (startupTimedOut) {
          likelyCause = 'startup_readiness_timeout';
        } else if (this._lastWasIntentionalRestart) {
          // issue #1801: a deliberate wedge restart. Label it as such even
          // though the kill signal was SIGTERM/SIGKILL, so the audit summary
          // (supervisor-audit.ts CLEAN_EXIT_CAUSES) counts it as a self-heal,
          // not a crash.
          likelyCause = 'wedge_restart';
        } else if (signal === 'SIGKILL') {
          likelyCause = 'oom_or_external_kill';
        } else if (signal === 'SIGTERM') {
          likelyCause = 'graceful_shutdown';
        } else if (code === WORKER_EXIT_RSS_WATCHDOG) {
          likelyCause = 'rss_watchdog';
        } else if (code === 1) {
          likelyCause = 'runtime_error';
        } else if (code === 0) {
          likelyCause = 'clean_exit';
        } else {
          likelyCause = 'unknown';
        }

        this.opts.onEvent({
          kind: 'worker_exited',
          pid: child.pid,
          code: code ?? null,
          signal: signal ?? null,
          runDurationMs: runDuration,
          likelyCause,
          crashCount: this._crashCount,
        });

        resolve();
      });
    });
  }

  /** Compute and apply backoff based on the most recent exit classifier. */
  private async applyBackoff(): Promise<void> {
    if (this._lastWasIntentionalRestart) {
      // issue #1801: a deliberate wedge restart. Respawn immediately (a fresh
      // pool is the whole point of the self-heal) — no crash backoff, no
      // budget accounting (the supervisor's wedgeRestartLoopBudget owns that).
      this._lastWasIntentionalRestart = false;
      this.opts.onEvent({
        kind: 'backoff',
        ms: 0,
        crashCount: this._crashCount,
        reason: 'wedge_restart',
      });
      return;
    }

    if (this._lastExitCode === 0) {
      // D2: check the clean-restart budget. If exceeded, emit health_warn
      // and apply a fixed cooldown so the next spawn isn't instant. This
      // bounds the worst case on platforms where Diff 2's RssAnon helper
      // falls back to VmRSS (macOS, kernel <4.5, restricted containers).
      const budget = this.opts.cleanRestartBudget ?? DEFAULTS.cleanRestartBudget;
      const windowMs = this.opts.cleanRestartWindowMs ?? DEFAULTS.cleanRestartWindowMs;
      if (this._cleanRestartTimestamps.length > budget) {
        this.opts.onEvent({
          kind: 'health_warn',
          reason: 'clean_restart_budget_exceeded',
          count: this._cleanRestartTimestamps.length,
          windowMs,
        });
        const cooldown =
          this.opts._backoffFloorMs !== undefined
            ? this.opts._backoffFloorMs
            : this.opts.cleanRestartBudgetBackoffMs ?? DEFAULTS.cleanRestartBudgetBackoffMs;
        this.opts.onEvent({
          kind: 'backoff',
          ms: Math.round(cooldown),
          crashCount: this._crashCount,
          reason: 'budget_exceeded',
        });
        this._inBackoff = true;
        try {
          await this.sleep(cooldown);
        } finally {
          this._inBackoff = false;
        }
        return;
      }
      // Within budget — immediate restart.
      this.opts.onEvent({
        kind: 'backoff',
        ms: 0,
        crashCount: this._crashCount,
        reason: 'clean_exit',
      });
      return;
    }

    // issue #1678: RSS-watchdog drain. Always back off (so an instant-OOM on
    // startup can't hot-loop) and, when more than `watchdogLoopBudget` drains
    // land inside the window, emit the loud `rss_watchdog_loop` alert. crashCount
    // is untouched (the worker is fine; the cap is too low), so this branch
    // never trips max_crashes — the workload keeps running, just paced + loud.
    if (this._lastExitCode === WORKER_EXIT_RSS_WATCHDOG) {
      const count = this._watchdogExitTimestamps.length;
      const budget = this.opts.watchdogLoopBudget ?? DEFAULTS.watchdogLoopBudget;
      const windowMs = this.opts.watchdogLoopWindowMs ?? DEFAULTS.watchdogLoopWindowMs;
      if (count > budget) {
        this.opts.onEvent({
          kind: 'health_warn',
          reason: 'rss_watchdog_loop',
          count,
          windowMs,
        });
      }
      const watchdogBackoff =
        this.opts._backoffFloorMs !== undefined
          ? this.opts._backoffFloorMs
          : this.opts.watchdogBackoffMs ?? DEFAULTS.watchdogBackoffMs;
      this.opts.onEvent({
        kind: 'backoff',
        ms: Math.round(watchdogBackoff),
        crashCount: this._crashCount,
        reason: 'rss_watchdog',
      });
      this._inBackoff = true;
      try {
        await this.sleep(watchdogBackoff);
      } finally {
        this._inBackoff = false;
      }
      return;
    }

    // code != 0: exponential backoff scaled by crashCount-1 (retry-attempt
    // index). On first crash: crashCount=1, exponent=0 -> 1s. After stable-
    // run reset: crashCount=1 again -> 1s fresh cycle.
    const backoff =
      this.opts._backoffFloorMs !== undefined
        ? this.opts._backoffFloorMs
        : calculateBackoffMs(this._crashCount - 1);

    this.opts.onEvent({
      kind: 'backoff',
      ms: Math.round(backoff),
      crashCount: this._crashCount,
      reason: 'crash',
    });

    this._inBackoff = true;
    try {
      await this.sleep(backoff);
    } finally {
      this._inBackoff = false;
    }
  }

  private sleep(ms: number): Promise<void> {
    if (this.configurationBlocked) return Promise.resolve();
    this.opts.processingState?.waiting('retry_backoff', this.now() + ms);
    return new Promise<void>((resolve) => {
      const finish = () => {
        clearTimeout(timer);
        this.wakeBackoff = null;
        resolve();
      };
      const timer = setTimeout(finish, ms);
      this.wakeBackoff = finish;
    });
  }

  private now(): number {
    return this.opts._now ? this.opts._now() : Date.now();
  }
}