Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
86.4 KB
Raw Blame
import { assertNoUnreviewedJobs, authorizeJobExecution, withSubmissionAuthority } from './submission-authority.ts';
/**
 * MinionWorker — Concurrent in-process job worker with BullMQ-inspired patterns.
 *
 * Processes up to `concurrency` jobs simultaneously using a Promise pool.
 * Each job gets its own AbortController, lock renewal timer, and isolated state.
 *
 * Usage:
 *   const worker = new MinionWorker(engine);
 *   worker.register('sync', async (job) => { ... });
 *   worker.register('embed', async (job) => { ... });
 *   await worker.start(); // polls until SIGTERM
 */

import { existsSync } from 'fs';
import { autopilotOperatorPauseMarkerPath, autopilotPaused, autopilotPausedMarkerPath } from '../autopilot-paths.ts';
import type { BrainEngine } from '../engine.ts';
import type {
  MinionJob, MinionJobContext, MinionHandler, MinionWorkerOpts,
  MinionQueueOpts, TokenUpdate,
} from './types.ts';
import {
  JobDeferredError,
  UnrecoverableError,
  ABORT_REASON_LOCK_RENEWAL_FAILED,
  ABORT_REASON_LOCK_LOST,
} from './types.ts';
import { MinionQueue } from './queue.ts';
import { runWaitingTtlTick, ttlNoticeGraceMs } from './admission.ts';
import { withChatPhase } from '../ai/chat-usage.ts';
import { calculateBackoff } from './backoff.ts';
import { RateLeaseUnavailableError } from './handlers/subagent.ts';
import { leaseFullBackoffMs } from './rate-leases.ts';
import { logLeasePressure } from './lease-pressure-audit.ts';
import {
  runLockRenewalTick,
  resolveLockRenewalKnobs,
  renewalIntervalFor,
  type LockRenewalDeps,
  type LockRenewalState,
  type TickResult,
} from './lock-renewal-tick.ts';
import { clampLockDurationMs } from './handler-timeouts.ts';
import {
  runDbProbe,
  assertWorkerDbReadiness,
  getConnectionRouting,
  DIRECT_PROBE_TIMEOUT_MS,
  type DbProbeResult,
  type PoolDiagnostics,
} from './db-probe.ts';
import { buildJobContext } from './job-context.ts';
import { runWithJobSpend } from './spend-authorization.ts';
import {
  runJobInChild,
  ChildSpawnInfraError,
  ChildWorkerShutdownError,
  ChildNotClaimedError,
} from './child-job-runner.ts';
import { lockRenewalAudit } from '../audit/lock-renewal-audit.ts';
import { loadavg, cpus } from 'os';
import { monitorEventLoopDelay } from 'perf_hooks';
import { isRetryableConnError, isServerRefusingConnections, isVanishedSocketError } from '../retry-matcher.ts';
import { reconnectAfterConnectionError as reconnectEngineAfterConnError } from './reconnect.ts';
import { LocalConfigurationError, isLocalConfigurationError } from './configuration-error.ts';

/** GST-2529: how often the worker asks whether a watchdog-marked job is past its wait (never less often than the wait itself). */
const SUSPECT_PROBE_EVERY_MS = 5_000;

/**
 * Abort reasons that signal infrastructure failure (PgBouncer outage,
 * connection drop, lock reclaimed by another worker) — NOT a job
 * defect. executeJob's catch block consults this set and SKIPS failJob
 * for these reasons, letting the stall detector requeue the row
 * cleanly without burning an attempt or dead-lettering the job.
 *
 * Codex C6 absorption (D8a): pre-v0.41.22.2, a PgBouncer blip during a
 * long-running job would lock-renewal-abort → handler throws → failJob
 * burns an attempt. That's wrong direction: the job's fine; the
 * infrastructure stumbled. Stall-detector reclaim is the correct path.
 *
 * Exported so tests can pin the named-constant contract (a future edit
 * to this set is a deliberate two-line change, not a silent regression).
 */
export const INFRASTRUCTURE_ABORT_REASONS = new Set<string>([
  ABORT_REASON_LOCK_RENEWAL_FAILED,
  ABORT_REASON_LOCK_LOST,
]);
import { randomUUID } from 'crypto';
import { EventEmitter } from 'events';
import { evaluateQuietHours, type QuietHoursConfig } from './quiet-hours.ts';
import { endedByShutdown, releaseUndrainedClaims, settleShutdownInterruptedJob } from './worker-shutdown.ts';
import { isWatchdogSuspect, settleWatchdogAbort } from './watchdog-attribution.ts';
import { IdleWaiter } from './idle-wait.ts';
import { jobOutcomeResult, jobResultOutcome } from './job-outcome.ts';
import { DEFAULT_LONG_JOB_NAMES, DEFAULT_SHORT_JOB_RESERVED_SLOTS, claimExclusions } from './lanes.ts';
import { jobRefusal, refusedJobResult } from './refusal.ts';
import { readFileSync } from 'fs';
import { parseRssFromProcStatus } from './proc-rss.ts';
import { withEngineScope } from '../engine-scope.ts';
import { startWakeDrain } from '../wakeshift/drain.ts';

export { parseRssFromProcStatus };

/**
 * Read accurate RSS from /proc/self/status (RssAnon + RssShmem).
 *
 * `process.memoryUsage().rss` returns VmRSS which includes file-backed mmap'd
 * pages (e.g. git packfiles). On a 96K-page brain repo, git operations can
 * inflate VmRSS to 7GB+ while actual heap usage is ~100MB. The kernel reclaims
 * file-backed pages under memory pressure — they're cache, not real usage.
 *
 * RssAnon = anonymous pages (heap, stack, anonymous mmap). RssShmem = shared
 * anonymous pages (IPC, tmpfs). Their sum is the non-file-backed resident
 * memory used for **per-process leak detection** — exactly the metric a leak
 * watchdog wants. It is NOT a full container-OOM metric: cgroup memory
 * pressure includes page cache, so a sibling container holding the page
 * cache hot can OOM us even at low anon+shmem. Use cgroup-aware monitoring
 * for that scenario; this helper is for the worker's own leak guard.
 *
 * Falls back to process.memoryUsage().rss on non-Linux, missing /proc, or
 * kernels older than 4.5 that don't expose RssAnon/RssShmem.
 *
 * `readStatus` is injectable for tests — production callers use the default,
 * which reads `/proc/self/status`.
 */
export function getAccurateRss(
  readStatus: () => string = () => readFileSync('/proc/self/status', 'utf8'),
): number {
  try {
    const status = readStatus();
    const parsed = parseRssFromProcStatus(status);
    if (parsed !== null) return parsed;
  } catch {
    // Non-Linux or /proc unavailable
  }
  return process.memoryUsage().rss;
}

/** Reason payload emitted with `'unhealthy'` when self-health-check trips.
 *  CLI layer (jobs.ts:work) subscribes and decides whether to call process.exit.
 *  `verdict` (issue #6) distinguishes local pool starvation from a genuinely
 *  unreachable server so operators stop debugging the wrong layer; absent on
 *  engines without the probe's disambiguation lane. */
export type UnhealthyReason =
  | { reason: 'client_misconfigured'; error: LocalConfigurationError }
  | {
      reason: 'db_dead';
      consecutiveFailures: number;
      message: string;
      verdict?: 'pool_starved' | 'server_unreachable' | 'unknown';
    }
  | { reason: 'stalled'; waitingCount: number; idleMinutes: number }
  | { reason: 'child_spawn_failing'; consecutiveFailures: number; message: string };

/**
 * Read the quiet_hours JSONB column off a MinionJob, if present. The
 * column was added in schema migration v12; older rows + versions of
 * MinionJob that don't include the field return null.
 */
function readQuietHoursConfig(job: MinionJob): QuietHoursConfig | null {
  const cfg = (job as MinionJob & { quiet_hours?: unknown }).quiet_hours;
  if (!cfg || typeof cfg !== 'object') return null;
  return cfg as unknown as QuietHoursConfig;
}

/** Per-job in-flight state (isolated per job, not shared on the worker). */
interface InFlightJob {
  job: MinionJob;
  lockToken: string;
  lockTimer: ReturnType<typeof setInterval>;
  abort: AbortController;
  promise: Promise<void>;
  /** Another job ran beside it, or it registered after a drain began: an RSS watchdog drain requeues it uncharged and marked (watchdog-attribution.ts). */
  shared: boolean;
}

export interface ConfigurationReleaseResult {
  jobId: number | null;
  lockToken: string;
  outcome: 'released' | 'no_op' | 'unconfirmed';
}

interface OwnedExecution {
  job: MinionJob;
  lockToken: string;
  stopped: boolean;
  abort?: AbortController;
  promise?: Promise<void>;
}

/** Type-safe `on('unhealthy', ...)` for callers. */
export interface MinionWorker {
  on(event: 'ready', listener: () => void): this;
  emit(event: 'ready'): boolean;
  on(event: 'unhealthy', listener: (info: UnhealthyReason) => void): this;
  emit(event: 'unhealthy', info: UnhealthyReason): boolean;
}

export class MinionWorker extends EventEmitter {
  private queue: MinionQueue;
  private handlers = new Map<string, MinionHandler>();
  private running = false;
  /** Log the pause/resume transition once each, not every poll. */
  private pausedByMarkerAnnounced = false;
  /** The idle poll wait a sync-queue wake ends early (minions/idle-wait.ts). */
  readonly idleWaiter = new IdleWaiter();
  private wakeDrain: { stop(): void } | null = null;
  private inFlight = new Map<number, InFlightJob>();
  private executions = new Map<string, OwnedExecution>();
  private pendingClaim: { lockToken: string; promise: Promise<MinionJob | null> } | null = null;
  /** GST-2529: when the next overdue watchdog-suspect probe is due, and whether the last one found such a job. */
  private suspectProbeDueAt = 0;
  private suspectOverdue = false;
  private _configurationError: LocalConfigurationError | null = null;
  private configurationDeadline = 0;
  private configurationDrainFinished = false;
  private configurationBlocked = new AbortController();
  private _configurationReleaseResults: ConfigurationReleaseResult[] = [];
  private workerId = randomUUID();

  /** Fires only on worker process SIGTERM/SIGINT. Handlers that need to run
   *  shutdown-specific cleanup (e.g. shell handler's SIGTERM→SIGKILL sequence on
   *  its child) subscribe via `ctx.shutdownSignal`. Separated from the per-job
   *  abort controller so non-shell handlers don't get cancelled mid-flight on
   *  deploy restart — they still get the full 30s cleanup race instead. */
  private shutdownAbort = new AbortController();

  /** Cumulative jobs that finished (success or failure). Used in watchdog log lines. */
  private jobsCompleted = 0;
  /** Idempotency latch for gracefulShutdown — per-job and periodic check sites can race.
   *  Holds the reason so launchJob can abort a claim that committed before the shutdown fired. */
  private gracefulShutdownReason: string | null = null;
  drainForced = false; // #5062: the shutdown drain expired and running claims were handed back (`jobs work` exits 17)
  /**
   * Set true when the RSS watchdog (not a normal SIGTERM) initiated the
   * drain. The CLI handler (src/commands/jobs.ts case 'work') reads this
   * AFTER start() resolves and exits the process with
   * WORKER_EXIT_RSS_WATCHDOG so the supervisor can classify the drain as
   * `rss_watchdog` instead of a clean exit. The worker deliberately does
   * NOT set process.exitCode itself — that would leak a non-zero code into
   * embedding hosts (tests, other process owners) that call start()/stop()
   * in-process. Ownership of process exit stays with the CLI, same as the
   * engine-disconnect boundary.
   */
  private _rssWatchdogTriggered = false;
  /** Peak observed RSS (MB) this process lifetime — surfaced in the watchdog
   *  drain line and the 80% soft-warn so operators can size --max-rss. */
  private _peakRssMb = 0;
  /** Latch so the 80%-of-cap soft-warn fires once per crossing, not every check. */
  private _softWarnFired = false;
  /**
   * Circuit breaker for deterministic child-bootstrap failures (red-team
   * finding): a spawn failure releases the job with no attempt burned, the
   * stall sweeper requeues it, the same worker re-claims — an infinite
   * claim/release loop the stall detector cannot see (every settle refreshes
   * the progress clock). After CHILD_SPAWN_FAIL_EXIT_AFTER consecutive
   * spawn-class failures we emit 'unhealthy' so the process manager restarts
   * the worker (and the supervisor's crash budget takes over if the child
   * CLI stays broken).
   */
  private _consecutiveChildSpawnFailures = 0;
  private static readonly CHILD_SPAWN_FAIL_EXIT_AFTER = 3;

  private opts: Required<MinionWorkerOpts>;

  /**
   * Event-loop-delay histogram (CDX-1/R2-9, issue #4145): the DIRECT
   * measurement of local starvation, sampled at eviction time and RESET
   * on every successful lock renewal so a sample attributes to the
   * window since the last success — not process lifetime. The histogram
   * is deliberately WORKER-global, not per-job: the event loop is one
   * shared resource, and ANY job's successful renewal proves the loop was
   * healthy enough to process a round-trip at that moment — a legitimate
   * truncation of the starvation window even for a sibling job that
   * evicts moments later (its per-job discriminator is tick lateness,
   * which IS per-job). Null when the runtime doesn't ship
   * `monitorEventLoopDelay` (fail-open: eviction logs omit eld fields).
   */
  private eldHistogram: ReturnType<typeof monitorEventLoopDelay> | null = null;
  /** Core count cached once — pairs with raw loadavg in eviction telemetry. */
  private readonly cpuCores: number;

  constructor(
    private engine: BrainEngine,
    opts?: MinionWorkerOpts & MinionQueueOpts,
  ) {
    super();
    this.queue = new MinionQueue(engine, {
      maxSpawnDepth: opts?.maxSpawnDepth,
      maxAttachmentBytes: opts?.maxAttachmentBytes,
    });
    let cores = 0;
    try { cores = cpus().length; } catch { /* telemetry best-effort */ }
    this.cpuCores = cores;
    try {
      if (typeof monitorEventLoopDelay === 'function') {
        // Created DISABLED: start() enables and stop() disables, so a
        // constructed-but-never-started worker (setup failures, probe
        // instances) never holds a native sampling timer.
        this.eldHistogram = monitorEventLoopDelay({ resolution: 20 });
      }
    } catch { this.eldHistogram = null; /* fail-open */ }
    this.opts = {
      queue: opts?.queue ?? 'default',
      concurrency: opts?.concurrency ?? 1,
      lockDuration: opts?.lockDuration ?? 30000,
      stalledInterval: opts?.stalledInterval ?? 30000,
      maxStalledCount: opts?.maxStalledCount ?? 1,
      pollInterval: opts?.pollInterval ?? 5000,
      maxRssMb: opts?.maxRssMb ?? 0,
      getRss: opts?.getRss ?? getAccurateRss,
      rssCheckInterval: opts?.rssCheckInterval ?? 60000,
      healthCheckInterval: opts?.healthCheckInterval ?? 60000,
      stallWarnAfterMs: opts?.stallWarnAfterMs ?? 5 * 60_000,
      stallExitAfterMs: opts?.stallExitAfterMs ?? 10 * 60_000,
      dbFailExitAfter: opts?.dbFailExitAfter ?? 3,
      dbProbeTimeoutMs: opts?.dbProbeTimeoutMs ?? 10_000,
      jobIsolation: opts?.jobIsolation ?? 'inline',
      childCliInvocation: opts?.childCliInvocation ?? null,
      childTiniPath: opts?.childTiniPath ?? '',
      longJobNames: opts?.longJobNames ?? DEFAULT_LONG_JOB_NAMES,
      shortJobReservedSlots: opts?.shortJobReservedSlots ?? DEFAULT_SHORT_JOB_RESERVED_SLOTS,
      watchdogSuspectWaitMs: opts?.watchdogSuspectWaitMs ?? 60_000,
      onJobSettled: opts?.onJobSettled ?? (() => {}),
    };
    // Process isolation contract: 'process' without a resolved child CLI
    // invocation would silently execute handlers INLINE while the evict path
    // believed it was isolated (predicate mismatch — red-team finding). The
    // CLI layer always resolves + validates the invocation; library callers
    // must too. Loud construction throw, same discipline as the stall
    // thresholds below.
    if (this.opts.jobIsolation === 'process' && this.opts.childCliInvocation == null) {
      throw new Error(
        "MinionWorkerOpts: jobIsolation 'process' requires childCliInvocation " +
        '(resolve it via resolveChildCliInvocation and validate it exists before constructing the worker).',
      );
    }
    // Stall thresholds contract: exit MUST be strictly greater than warn.
    // If exit <= warn, the warn-then-exit semantics break: a single tick at
    // idle > warn would set stallWarningSince and the subsequent tick at
    // idle > exit could fire immediately without giving operators visibility.
    // Reject misconfigurations at construction time so the failure mode is
    // a loud throw on startup rather than a quiet contract violation.
    if (this.opts.stallExitAfterMs <= this.opts.stallWarnAfterMs) {
      throw new Error(
        `MinionWorkerOpts: stallExitAfterMs (${this.opts.stallExitAfterMs}) must be > ` +
        `stallWarnAfterMs (${this.opts.stallWarnAfterMs}). ` +
        `The contract is "warn first, exit later" — they cannot fire on the same tick.`,
      );
    }
  }

  /**
   * Read-only handler lookup. `gbrain jobs run-child` registers the builtin
   * handlers against a throwaway worker (registerBuiltinHandlers's existing
   * contract) and resolves the one it needs through this accessor.
   */
  getHandler(name: string): MinionHandler | undefined {
    return this.handlers.get(name);
  }

  /** Register a handler for a job type. */
  register(name: string, handler: MinionHandler): void {
    this.handlers.set(name, handler);
  }

  /** Get registered handler names (used by claim query). */
  get registeredNames(): string[] {
    return Array.from(this.handlers.keys());
  }

  /**
   * True when the RSS watchdog drained this worker (vs a normal SIGTERM
   * shutdown). The CLI handler reads this after `start()` resolves to set
   * the distinct WORKER_EXIT_RSS_WATCHDOG process exit code. See the field
   * comment on `_rssWatchdogTriggered` for the ownership rationale.
   */
  get rssWatchdogTriggered(): boolean {
    return this._rssWatchdogTriggered;
  }

  get configurationError(): LocalConfigurationError | null {
    return this._configurationError;
  }

  get configurationReleaseResults(): readonly ConfigurationReleaseResult[] {
    return this._configurationReleaseResults;
  }

  /** GST-2529: at most every min(wait, 5 s), ask whether a marked job is past its wait; a failed probe errs toward the drain. */
  private async probeOverdueSuspect(): Promise<void> {
    if (Date.now() < this.suspectProbeDueAt) return;
    this.suspectProbeDueAt = Date.now() + Math.min(this.opts.watchdogSuspectWaitMs, SUSPECT_PROBE_EVERY_MS);
    this.suspectOverdue = await this.queue.hasOverdueWatchdogSuspect(this.opts.queue, this.registeredNames, this.opts.watchdogSuspectWaitMs)
      .catch(() => true);
  }

  private probeDatabase(): Promise<DbProbeResult> {
    const dualPool = getConnectionRouting(this.engine)?.isDualPoolActive?.() === true;
    const getDiag = (this.engine as { getPoolDiagnostics?: () => PoolDiagnostics | null }).getPoolDiagnostics;
    return runDbProbe({
      probeRead: async signal => { await this.engine.executeRaw('SELECT 1', undefined, { signal }); },
      ...(dualPool ? {
        probeDirect: async (signal: AbortSignal) => { await this.engine.executeRawDirect('SELECT 1', undefined, { signal }); },
      } : {}),
      ...(typeof getDiag === 'function' ? { getDiagnostics: () => getDiag.call(this.engine) } : {}),
      timeoutMs: this.opts.dbProbeTimeoutMs,
      directTimeoutMs: DIRECT_PROBE_TIMEOUT_MS,
    });
  }

  blockForConfiguration(error: LocalConfigurationError): void {
    if (this._configurationError) return;
    this._configurationError = error;
    this.configurationDeadline = performance.now() + 30_000;
    this.running = false;
    this.configurationBlocked.abort(error);
    if (!this.shutdownAbort.signal.aborted) this.shutdownAbort.abort(error);
    for (const execution of this.executions.values()) execution.abort?.abort(error);
    this.emit('unhealthy', { reason: 'client_misconfigured', error });
  }

  private async drainConfiguration(): Promise<void> {
    const controller = new AbortController();
    const remaining = Math.max(0, this.configurationDeadline - performance.now());
    if (remaining === 0) controller.abort();
    const timer = setTimeout(() => controller.abort(), remaining);
    const pending = new Map(this.executions);
    const releasing = new Set<string>();
    let claimDone = !this.pendingClaim;
    const claim = this.pendingClaim;
    const collectClaim = claim?.promise.then(job => {
      claimDone = true;
      if (job) {
        const execution = this.executions.get(claim.lockToken);
        if (execution) pending.set(claim.lockToken, execution);
      }
    }, () => {
      claimDone = true;
      if (!this.configurationDrainFinished) {
        this._configurationReleaseResults.push({ jobId: null, lockToken: claim.lockToken, outcome: 'unconfirmed' });
      }
    });
    try {
      while (!controller.signal.aborted && (pending.size > 0 || !claimDone)) {
        for (const [token, execution] of pending) {
          if (!execution.stopped || releasing.has(token)) continue;
          if (releasing.size >= 4) break;
          releasing.add(token);
          void this.queue.releaseConfigurationJob(execution.job.id, token, controller.signal).then(outcome => {
            if (this.configurationDrainFinished) return;
            this._configurationReleaseResults.push({ jobId: execution.job.id, lockToken: token, outcome });
            pending.delete(token);
            releasing.delete(token);
            this.executions.delete(token);
          });
        }
        if (pending.size > 0 || !claimDone) {
          await new Promise(resolve => setTimeout(resolve, Math.min(10, Math.max(0, this.configurationDeadline - performance.now()))));
        }
      }
      for (const [token, execution] of pending) {
        this._configurationReleaseResults.push({ jobId: execution.job.id, lockToken: token, outcome: 'unconfirmed' });
      }
      if (!claimDone && claim) this._configurationReleaseResults.push({ jobId: null, lockToken: claim.lockToken, outcome: 'unconfirmed' });
      void collectClaim;
    } finally {
      this.configurationDrainFinished = true;
      controller.abort();
      clearTimeout(timer);
      for (const entry of this.inFlight.values()) clearInterval(entry.lockTimer);
    }
  }

  /** Emit 'unhealthy' with a no-listener fallback. The default contract is
   *  fail-stop: pre-EventEmitter-refactor behavior was process.exit(1) inside
   *  the timer; the refactor moved that responsibility to the CLI subscriber.
   *  But direct API consumers without a listener would see emit() become a
   *  no-op AND `healthExited=true` permanently disabling monitoring — a
   *  silent regression. Solution: if no one subscribed, log and exit
   *  ourselves so the worker dies and the PM restarts it. Subscribers
   *  override this default by adding a listener before start(). */
  private emitUnhealthy(info: UnhealthyReason): void {
    if (info.reason === 'client_misconfigured') {
      this.blockForConfiguration(info.error);
      return;
    }
    if (this.configurationError) return;
    if (this.listenerCount('unhealthy') === 0) {
      const detail = info.reason === 'db_dead'
        ? `DB unreachable (${info.consecutiveFailures} probes): ${info.message}`
        : info.reason === 'child_spawn_failing'
          ? `job-child spawn failing (${info.consecutiveFailures} consecutive): ${info.message}`
          : `worker stalled (${info.waitingCount} waiting, ${info.idleMinutes}m idle)`;
      console.error(
        `[health] FATAL: ${detail}. No 'unhealthy' listener registered; ` +
        `defaulting to process.exit(1) for process-manager restart.`,
      );
      process.exit(1);
    }
    this.emit('unhealthy', info);
  }

  /** Start the worker loop. Blocks until stopped. */
  async start(): Promise<void> {
    if (this.configurationError) return;
    if (this.handlers.size === 0) {
      throw new Error('No handlers registered. Call worker.register(name, handler) before start().');
    }

    this.running = true;
    if (this.engine.kind === 'postgres') {
      while (this.running) {
        try {
          await assertWorkerDbReadiness(this.engine);
          break;
        } catch (error) {
          if (isLocalConfigurationError(error)) {
            this.blockForConfiguration(error);
            return;
          }
        }
        await new Promise(resolve => setTimeout(resolve, Math.max(100, Math.min(1000, this.opts.pollInterval))));
      }
    }
    if (!this.running) return;
    try {
      await this.queue.ensureSchema();
      await assertNoUnreviewedJobs(this.engine);
    } catch (error) {
      if (!isLocalConfigurationError(error)) throw error;
      this.blockForConfiguration(error);
      return;
    }
    if (!this.running) return;
    try {
      this.emit('ready');
    } catch (error) {
      this.running = false;
      throw error;
    }
    // R2-9 lifecycle: (re-)enable the event-loop-delay histogram for this
    // run; stop() disables it so embedding hosts / test suites that cycle
    // start()/stop() don't leak a ~50Hz native sampling timer per instance.
    try { this.eldHistogram?.enable(); } catch { /* fail-open */ }
    this.wakeDrain ??= startWakeDrain(this.engine); // Wakeshift: the 5 s wake drainer (wakeshift/drain.ts), a no-op tick while off

    // Graceful shutdown. Fires shutdownAbort so handlers subscribed to
    // `ctx.shutdownSignal` (currently: shell handler) can run their own cleanup
    // BEFORE the 30s cleanup race expires. Non-shell handlers ignore shutdown
    // and keep running — they get the full 30s window.
    const shutdown = () => this.requestShutdown();
    process.on('SIGTERM', shutdown);
    process.on('SIGINT', shutdown);

    // Stall + timeout detection on interval. Order matters: handleStalled FIRST
    // so a stalled job (lock_until expired) gets requeued before handleTimeouts'
    // `lock_until > now()` guard would skip it. Stall → retry, timeout → dead.
    const stalledTimer = setInterval(async () => {
      if (this.configurationError) return;
      // issue #1720: a dead pool used to spray "Stall detection error: write
      // CONNECTION_CLOSED ..." every tick forever — this interval was the only
      // background loop without the #1491-style reconnect. Rebuild the
      // worker-owned pool AT MOST ONCE per tick, shared across the three
      // sweeps: a dead pool fails all three, and one rebuild is enough (three
      // back-to-back connect attempts would just add pooler pressure).
      let reconnectedThisTick = false;
      const recoverConnection = async (site: string, e: unknown): Promise<void> => {
        if (isLocalConfigurationError(e)) {
          this.blockForConfiguration(e);
          return;
        }
        if (reconnectedThisTick || !isRetryableConnError(e)) return;
        reconnectedThisTick = true;
        await this.reconnectAfterConnectionError(site, e);
      };
      try {
        const { requeued, dead } = await this.queue.handleStalled();
        if (requeued.length > 0) console.log(`Stall detector: requeued ${requeued.length} jobs`);
        if (dead.length > 0) console.log(`Stall detector: dead-lettered ${dead.length} jobs`);
      } catch (e) {
        console.error('Stall detection error:', e instanceof Error ? e.message : String(e));
        await recoverConnection('handleStalled', e);
      }
      if (this.configurationError) return;
      try {
        const timedOut = await this.queue.handleTimeouts();
        if (timedOut.length > 0) console.log(`Timeout detector: dead-lettered ${timedOut.length} jobs (timeout exceeded)`);
      } catch (e) {
        console.error('Timeout detection error:', e instanceof Error ? e.message : String(e));
        await recoverConnection('handleTimeouts', e);
      }
      if (this.configurationError) return;
      try {
        const wallClockTimedOut = await this.queue.handleWallClockTimeouts(this.opts.lockDuration);
        if (wallClockTimedOut.length > 0) {
          console.log(`Wall-clock detector: dead-lettered ${wallClockTimedOut.length} jobs (wall-clock timeout exceeded)`);
        }
      } catch (e) {
        console.error('Wall-clock timeout detection error:', e instanceof Error ? e.message : String(e));
        await recoverConnection('handleWallClockTimeouts', e);
      }
      if (this.configurationError) return;
      // 4th sweep: waiting-TTL (admission control). Warn-before-act (user
      // requirement D1A) lives in runWaitingTtlTick (admission.ts): the first
      // tick counts + stamps the notice timestamp, sweeping starts only after
      // the grace window elapses. The gate is engine-state (not process
      // state) because the worker restarts via self-upgrade/systemd without
      // ever running the CLI's runPostUpgrade banner; the notice must precede
      // the cancellation on EVERY channel, and the worker log is the daemon
      // channel.
      try {
        const tick = await runWaitingTtlTick(this.engine, this.queue);
        if (tick.phase === 'notice') {
          console.log(
            `⚠ Waiting-TTL is now active: ${tick.affected ?? 0} queued job(s) currently exceed their TTL and ` +
            `will be cancelled after a ${Math.round(ttlNoticeGraceMs() / 60_000)}min grace window. ` +
            `Tune: gbrain config set minions.ttl_waiting_hours.<name> <hours|0>.`,
          );
        } else if (tick.phase === 'swept' && tick.cancelled > 0) {
          const breakdown = Object.entries(tick.by_name).map(([n, c]) => `${n}: ${c}`).join(', ');
          console.log(
            `Waiting-TTL: cancelled ${tick.cancelled} job(s) that waited past their TTL (${breakdown}). ` +
            `Tune: gbrain config set minions.ttl_waiting_hours.<name> <hours|0>. See 'gbrain jobs stats'.`,
          );
        }
      } catch (e) {
        console.error('Waiting-TTL sweep error:', e instanceof Error ? e.message : String(e));
        await recoverConnection('handleWaitingTTL', e);
      }
    }, this.opts.stalledInterval);

    // Periodic RSS watchdog — closes the production-freeze regression where
    // all concurrency slots are wedged with zero job completions, so the
    // per-job check in executeJob().finally() never fires. Disabled when
    // maxRssMb is 0 (default for bare `gbrain jobs work`; supervisor sets 2048).
    let rssTimer: ReturnType<typeof setInterval> | null = null;
    if (this.opts.maxRssMb > 0) {
      rssTimer = setInterval(() => {
        this.checkMemoryLimit('periodic');
      }, this.opts.rssCheckInterval);
    }

    // Self-health-check. Catches two failure modes that leave the process alive
    // but non-functional:
    //   1. DB connection death (Supabase/PgBouncer drops, network blip) — runs
    //      ALWAYS (incl. under a supervisor), because it's the only signal for
    //      "MY pool is dead" and the supervisor watches a different connection
    //      (issue #1801, fix #2). Disabled only when healthCheckInterval is 0.
    //   2. Worker stall (event loop alive but not claiming/completing jobs) —
    //      runs only when NOT supervised (GBRAIN_SUPERVISED=1); under a
    //      supervisor the progress watchdog owns forward-progress detection.
    //
    // On failure, emits an `'unhealthy'` event with a structured reason. The
    // CLI layer (`src/commands/jobs.ts:work`) subscribes and decides whether to
    // call process.exit. Library code never calls process.exit directly so
    // MinionWorker stays embeddable in non-CLI contexts (tests, other hosts).
    //
    // Timer pattern: recursive setTimeout with a `running` flag, not setInterval.
    // setInterval queues callbacks even when the prior is still awaiting; on a
    // hung DB probe that piles up overlapping async checks racing on
    // `consecutiveDbFailures`. The recursive pattern guarantees one tick at a time.
    const isSupervisedChild = process.env.GBRAIN_SUPERVISED === '1';
    let healthTimer: ReturnType<typeof setTimeout> | null = null;
    // issue #1801 (fix #2): the DB-liveness probe (part 1) runs EVEN under a
    // supervisor — it's the worker's own "is MY pool dead" signal, and the
    // supervisor watches a DIFFERENT connection, so it cannot see this worker's
    // dead pool. A supervised worker whose pool dies now self-exits (db_dead →
    // process.exit(1) via the jobs.ts listener) and the supervisor respawns it
    // with a fresh pool in ~3 min — faster than, and orthogonal to, the
    // supervisor's 15-min progress watchdog (which backstops NON-DB wedges).
    // Stall detection (part 2) STAYS gated to non-supervised: the supervisor's
    // progress watchdog now owns forward-progress, so the worker's own stall
    // detector would double-act.
    if (this.opts.healthCheckInterval > 0) {
      let consecutiveDbFailures = 0;
      let lastKnownCompleted = this.jobsCompleted;
      let lastCompletionTime = Date.now();
      let stallWarningSince: number | null = null;
      let healthRunning = false;
      let healthExited = false;

      // DB liveness probe with pool-starvation disambiguation (issue #6).
      // The probe body lives in db-probe.ts (hermetically tested,
      // lock-renewal-tick pattern); this is the thin adapter. Both probes
      // carry an AbortSignal — a hung probe is CANCELLED (slot released),
      // never abandoned. The direct-lane probe runs ONLY when dual-pool is
      // genuinely active (a kill-switched executeRawDirect would probe the
      // same starved read pool twice and fake a verdict).
      const runHealthCheck = async (): Promise<void> => {
        if (healthRunning || !this.running || healthExited) return;
        healthRunning = true;
        try {
          // --- 1. DB liveness probe ---
          const probe = await this.probeDatabase();
          if (this.configurationError) return;
          if (!probe.ok && probe.verdict === 'client_misconfigured') {
            this.blockForConfiguration(probe.configurationError);
            return;
          }
          if (probe.ok) {
            consecutiveDbFailures = 0;
          } else {
            consecutiveDbFailures++;
            console.error(
              `[health] DB probe failed (${consecutiveDbFailures}/${this.opts.dbFailExitAfter}): ${probe.detail}`,
            );
            if (consecutiveDbFailures >= this.opts.dbFailExitAfter) {
              console.error(
                `[health] DB probe failed ${this.opts.dbFailExitAfter} consecutive times ` +
                `(verdict: ${probe.verdict}). Emitting 'unhealthy' for process-manager restart.`,
              );
              healthExited = true;
              this.emitUnhealthy({
                reason: 'db_dead',
                consecutiveFailures: consecutiveDbFailures,
                message: probe.detail,
                verdict: probe.verdict,
              });
            }
            return; // Skip stall check when DB is flaky
          }

          // --- 2. Stall detection (NON-supervised only) ---
          // Under a supervisor, the supervisor's progress watchdog (issue #1801)
          // owns forward-progress detection; running the worker's own stall
          // detector too would double-act (and both emitting 'unhealthy' +
          // supervisor SIGTERM race). Bare `gbrain jobs work` keeps it.
          if (!isSupervisedChild) {
          if (this.jobsCompleted > lastKnownCompleted || autopilotPaused()) { // #7966: a pause holds claims, so its ticks restart the clock
            lastKnownCompleted = this.jobsCompleted;
            lastCompletionTime = Date.now();
            stallWarningSince = null;
          }

          const idleMs = Date.now() - lastCompletionTime;

          // Only check for stalls when no jobs are in-flight and it's been a while
          if (idleMs > this.opts.stallWarnAfterMs && this.inFlight.size === 0) {
            try {
              // Filter by registered handler names so a worker that doesn't
              // claim a particular job-name doesn't false-positive when those
              // jobs accumulate in `waiting`. Only counts work THIS worker would
              // actually have claimed.
              const handlerNames = this.registeredNames;
              const rows = handlerNames.length === 0
                ? [] as { cnt: string }[]
                : await this.engine.executeRaw<{ cnt: string }>(
                    `SELECT count(*)::text AS cnt FROM minion_jobs
                     WHERE status = 'waiting'
                       AND queue = $1
                       AND name = ANY($2::text[])`,
                    [this.opts.queue, handlerNames],
                  );
              const waiting = parseInt(rows[0]?.cnt ?? '0', 10);
              const idleMinutes = Math.round(idleMs / 60_000);
              if (waiting > 0) {
                // Two thresholds, both measured from `lastCompletionTime` (NOT
                // from when the warning fired). With defaults (warn=5min,
                // exit=10min), the first warning fires at idle=5min and the
                // unhealthy emit fires at idle=10min — matching the contract
                // documented in MinionWorkerOpts.
                if (!stallWarningSince) {
                  stallWarningSince = Date.now();
                  console.warn(
                    `[health] Possible stall: ${waiting} waiting job(s) for ` +
                    `registered handlers, 0 in-flight, ${idleMinutes}m since last completion`,
                  );
                } else if (idleMs > this.opts.stallExitAfterMs) {
                  console.error(
                    `[health] Worker stalled for ${Math.round(this.opts.stallExitAfterMs / 60_000)}+ ` +
                    `minutes with ${waiting} waiting job(s). Emitting 'unhealthy' for process-manager restart.`,
                  );
                  healthExited = true;
                  this.emitUnhealthy({
                    reason: 'stalled',
                    waitingCount: waiting,
                    idleMinutes,
                  });
                }
              } else {
                stallWarningSince = null; // Queue empty (for our handlers) — not stalled, just idle
              }
            } catch {
              // DB query failed — the liveness probe above will catch persistent failures
            }
          } else {
            stallWarningSince = null;
          }
          } // end stall detection (NON-supervised only)
        } finally {
          healthRunning = false;
          if (this.running && !healthExited) {
            healthTimer = setTimeout(runHealthCheck, this.opts.healthCheckInterval);
          }
        }
      };

      // First tick scheduled after one interval so newly-started workers have
      // a chance to do real work before the stall clock starts ticking.
      healthTimer = setTimeout(runHealthCheck, this.opts.healthCheckInterval);
    }

    const runLoop = async () => {
      while (this.running) {
        // Promote delayed jobs
        try {
          await this.queue.promoteDelayed();
        } catch (e) {
          if (isLocalConfigurationError(e)) {
            this.blockForConfiguration(e);
            break;
          }
          const msg = e instanceof Error ? e.message : String(e);
          console.error('Promotion error:', msg);
          // issue #1491: a retryable pool/connection loss during promotion used
          // to be logged and ignored, leaving the worker in a repeated
          // "Promotion error: No database connection" loop until a later path
          // happened to reconnect or crash. Promotion is a standalone UPDATE
          // from delayed→waiting, so after a connection failure we can safely
          // rebuild the worker-owned pool before continuing to claim work.
          if (isRetryableConnError(e)) {
            await this.reconnectAfterConnectionError('promoteDelayed', e);
          }
        }

        await this.probeOverdueSuspect();
        if (!this.running) break;

        // Claim jobs up to concurrency limit — unless the system-wide pause
        // marker is parked. `gbrain migrate` quiesces writers for the copy
        // window; the marker stops the autopilot dispatch loop, and gating
        // the CLAIM here extends that fence to queued jobs (an already
        // in-flight job finishes and is waited on by the migrate drain).
        // Checked at claim time only: one existsSync per poll tick.
        if (existsSync(autopilotPausedMarkerPath()) || existsSync(autopilotOperatorPauseMarkerPath())) {
          if (!this.pausedByMarkerAnnounced) {
            console.log('[worker] pause marker present — not claiming new jobs until it clears.');
            this.pausedByMarkerAnnounced = true;
          }
          await new Promise(resolve => setTimeout(resolve, this.opts.pollInterval));
          continue;
        }
        if (this.pausedByMarkerAnnounced) {
          console.log('[worker] pause marker cleared — resuming job claims.');
          this.pausedByMarkerAnnounced = false;
        }
        if (this.inFlight.size < this.opts.concurrency && ![...this.inFlight.values()].some(f => isWatchdogSuspect(f.job))) {
          const lockToken = `${this.workerId}:${randomUUID()}`;
          this.idleWaiter.claimStarted();
          let job: MinionJob | null;
          try {
            const claim = this.queue.claim(
              lockToken,
              this.opts.lockDuration,
              this.opts.queue,
              this.registeredNames,
              { excludeWatchdogSuspects: this.inFlight.size > 0, suspectWaitMs: this.suspectOverdue ? this.opts.watchdogSuspectWaitMs : undefined, excludeNames: claimExclusions(this.opts, [...this.inFlight.values()].map(f => f.job.name)) },
            ).then(claimed => {
              if (claimed) {
                this.executions.set(lockToken, { job: claimed, lockToken, stopped: true });
                if (this.configurationDrainFinished) {
                  const previous = this._configurationReleaseResults.find(result => result.lockToken === lockToken);
                  if (previous) previous.jobId = claimed.id;
                  else this._configurationReleaseResults.push({ jobId: claimed.id, lockToken, outcome: 'unconfirmed' });
                  this.executions.delete(lockToken);
                }
              }
              return claimed;
            });
            this.pendingClaim = { lockToken, promise: claim };
            job = await claim;
            this.pendingClaim = null;
          } catch (e) {
            if (isLocalConfigurationError(e)) {
              this.blockForConfiguration(e);
              break;
            }
            this.pendingClaim = null;
            // issue #1678 (Codex #1): a reaped pooler socket / nulled instance
            // pool throws a retryable conn error here. Blind-retrying claim is
            // UNSAFE — if the UPDATE...RETURNING committed but the connection
            // died before the row reached us, a retry would claim a SECOND
            // job (invisible active job, no renewal, later stall). So instead:
            // reconnect once and let the NEXT poll tick re-claim against a live
            // pool. Non-retryable errors propagate (real bug → PM restart). A vanished
            // local socket or a server refusing connections is the same lost connection (#9540).
            if (!isRetryableConnError(e) && !isVanishedSocketError(e) && !isServerRefusingConnections(e)) throw e;
            const msg = e instanceof Error ? e.message : String(e);
            console.error(`[worker] claim hit a connection error; reconnecting, retry on next tick: ${msg}`);
            await this.reconnectAfterConnectionError('claim', e);
            await new Promise(resolve => setTimeout(resolve, this.opts.pollInterval));
            continue;
          }

          if (job) {
            if (this.configurationError) break;
            // Post-claim fence re-check: the pre-claim marker check above
            // races migrate's marker write — this claim may have committed
            // after migrate's drain probe counted zero active jobs. A job
            // claimed into that window is released back (delayed, un-run)
            // instead of executed; the poll loop then parks on the marker.
            if (existsSync(autopilotPausedMarkerPath()) || existsSync(autopilotOperatorPauseMarkerPath())) {
              console.log(`[worker] pause marker appeared after claim — releasing ${job.name} (id=${job.id}) un-run.`);
              await this.releaseClaimForPause(job, lockToken);
              this.executions.delete(lockToken);
              continue;
            }
            // Quiet-hours gate: evaluated at claim time, not dispatch.
            // Config lives on the job record (jsonb column added in
            // schema migration v12). Worker releases the job back to the
            // queue on 'defer' or marks it cancelled on 'skip'.
            const quietCfg = readQuietHoursConfig(job);
            const verdict = evaluateQuietHours(quietCfg);
            if (verdict !== 'allow') {
              await this.handleQuietHoursDefer(job, lockToken, verdict);
              this.executions.delete(lockToken);
            } else {
              this.launchJob(job, lockToken);
            }
          } else if (this.inFlight.size === 0) {
            // No jobs and nothing in flight, poll (a sync-queue wake ends the wait early)
            await this.idleWaiter.wait(this.opts.pollInterval);
          } else {
            // Jobs are running but no new ones available, brief pause before re-checking
            await new Promise(resolve => setTimeout(resolve, 100));
          }
        } else {
          // At concurrency limit, wait briefly before re-checking for free slots
          await new Promise(resolve => setTimeout(resolve, 100));
        }
      }
    };
    const blocked = new Promise<void>(resolve => {
      if (this.configurationBlocked.signal.aborted) resolve();
      else this.configurationBlocked.signal.addEventListener('abort', () => resolve(), { once: true });
    });
    try {
      await Promise.race([runLoop(), blocked]);
    } finally {
      clearInterval(stalledTimer);
      if (rssTimer) clearInterval(rssTimer);
      if (healthTimer) clearTimeout(healthTimer); // recursive setTimeout pattern
      process.removeListener('SIGTERM', shutdown);
      process.removeListener('SIGINT', shutdown);

      // Graceful shutdown: wait for all in-flight jobs with timeout
      if (this.configurationError) {
        await this.drainConfiguration();
      } else if (this.executions.size > 0) {
        console.log(`Waiting for ${this.executions.size} in-flight job(s) to finish (30s timeout)...`);
        const pending = Array.from(this.executions.values()).map(f => f.promise);
        await Promise.race([
          Promise.allSettled(pending),
          new Promise(resolve => setTimeout(resolve, 30000)),
        ]);
        if (this.configurationError) await this.drainConfiguration();
        else if (this.shutdownAbort.signal.aborted) this.drainForced = await releaseUndrainedClaims(this.engine, this.executions.values(), this.opts.jobIsolation === 'process');
      }

      // The worker does NOT disconnect the engine: it doesn't own the
      // engine's lifecycle. The caller (CLI handler at src/commands/jobs.ts
      // case 'work', or a test fixture) is responsible for disconnect when
      // it has finished using the engine. Earlier wave's experiment of
      // calling engine.disconnect() here violated ownership and broke
      // every test that shared a single engine across multiple
      // worker.start() / worker.stop() cycles (PGLiteEngine kills its
      // single _db connection; PostgresEngine.disconnect was non-idempotent
      // and clobbered the global db singleton on the second call). The
      // pool-slot-release intent is now handled in the CLI handler which
      // does own the engine.
      console.log('Minion worker stopped.');
    }
  }

  /**
   * Called when a claimed job falls inside its quiet-hours window. The
   * claim already set status='active' and held the lock; we reverse the
   * state transition (defer) or cancel outright (skip).
   *
   * 'defer' → status='waiting', lock cleared, delay_until bumped ahead by
   *   15 minutes so the same job doesn't immediately re-claim. Jobs will
   *   naturally pick up again once `now` exits the quiet window.
   * 'skip' → status='cancelled', final_status='skipped_quiet_hours'. The
   *   event is dropped.
   */
  /**
   * Release a just-claimed job back to the queue un-run because the
   * system-wide pause marker appeared between our pre-claim check and the
   * claim committing. Same conditional-release SQL shape as the quiet-hours
   * defer, but with a short delay: the poll loop parks on the marker, so the
   * job re-enters waiting and is picked up as soon as the pause clears.
   */
  private async releaseClaimForPause(job: MinionJob, lockToken: string): Promise<void> {
    try {
      await this.engine.executeRaw(
        `UPDATE minion_jobs
         SET status = 'delayed', lock_token = NULL, lock_until = NULL,
             delay_until = now() + interval '1 minute',
             updated_at = now()
         WHERE id = $1 AND lock_token = $2`,
        [job.id, lockToken],
      );
    } catch (e) {
      // Fail-open: if the release UPDATE itself fails, the claim lock simply
      // expires and the stall detector requeues the row — slower, same end
      // state, and never a reason to crash the worker.
      console.error(`[worker] pause release failed for job ${job.id}: ${e instanceof Error ? e.message : String(e)}`);
    }
  }

  private async handleQuietHoursDefer(job: MinionJob, lockToken: string, verdict: 'skip' | 'defer'): Promise<void> {
    try {
      if (verdict === 'skip') {
        // Route through MinionQueue.cancelJob so parent jobs in waiting-children
        // see the cancellation and roll up correctly. A direct status='cancelled'
        // UPDATE strands parents forever (no inbox, no dependency resolution).
        // Release our lock first so cancelJob's descendant walk sees a clean state.
        await this.engine.executeRaw(
          `UPDATE minion_jobs SET lock_token = NULL, lock_until = NULL, updated_at = now()
           WHERE id = $1 AND lock_token = $2`,
          [job.id, lockToken],
        );
        try {
          await this.queue.cancelJob(job.id);
        } catch {
          // cancelJob best-effort — if the parent rollup path errors, we still
          // want the job out of 'active' rather than re-claimed on next tick.
          await this.engine.executeRaw(
            `UPDATE minion_jobs
             SET status = 'cancelled', error_text = 'skipped_quiet_hours', updated_at = now()
             WHERE id = $1 AND status NOT IN ('completed','failed','dead','cancelled')`,
            [job.id],
          );
        }
        console.log(`Quiet-hours skip: ${job.name} (id=${job.id})`);
      } else {
        // Defer: release back to delayed, push delay ~15 minutes to avoid
        // immediate re-claim loops when the claim query re-runs.
        await this.engine.executeRaw(
          `UPDATE minion_jobs
           SET status = 'delayed', lock_token = NULL, lock_until = NULL,
               delay_until = now() + interval '15 minutes',
               updated_at = now()
           WHERE id = $1 AND lock_token = $2`,
          [job.id, lockToken],
        );
        console.log(`Quiet-hours defer: ${job.name} (id=${job.id}) → retry after 15m`);
      }
    } catch (e) {
      console.error(`handleQuietHoursDefer error for job ${job.id}:`, e instanceof Error ? e.message : String(e));
    }
  }

  /** #5062: signal-driven shutdown entry (SIGTERM/SIGINT, or the `jobs work` signal owner). Idempotent. */
  requestShutdown(): void {
    if (this.running) console.log('Minion worker shutting down...');
    this.running = false;
    this.idleWaiter.wake('stop');
    if (!this.shutdownAbort.signal.aborted) this.shutdownAbort.abort(new Error('shutdown'));
  }

  /** Stop the worker gracefully. */
  stop(): void {
    this.running = false;
    this.idleWaiter.wake('stop');
    this.wakeDrain?.stop(); this.wakeDrain = null;
    try { this.eldHistogram?.disable(); } catch { /* fail-open */ }
  }

  /**
   * Rebuild the worker-owned DB pool after a retryable connection failure.
   * Shared with the inline child drain (#2050) via minions/reconnect.ts.
   */
  private async reconnectAfterConnectionError(site: string, error: unknown): Promise<void> {
    await reconnectEngineAfterConnError(this.engine, site, error);
  }

  /** RSS watchdog. Called from the per-job finally and the periodic timer.
   *  Idempotent: returns early if already not running or already shut down.
   *  When threshold is exceeded, hands off to gracefulShutdown(). */
  private checkMemoryLimit(source: 'post-job' | 'periodic'): void {
    if (this.opts.maxRssMb <= 0) return;
    if (!this.running) return;
    if (this.gracefulShutdownReason !== null) return;

    let rss = 0;
    try {
      rss = this.opts.getRss();
    } catch {
      // process.memoryUsage() effectively cannot throw, but be safe.
      return;
    }
    const rssMb = Math.round(rss / (1024 * 1024));
    if (rssMb > this._peakRssMb) this._peakRssMb = rssMb;

    // Names of the jobs in flight when memory crested — the diagnostic an
    // operator needs to know WHICH job kind is the memory hog.
    const inFlightKinds = Array.from(this.inFlight.values()).map(f => f.job.name);

    // 80%-of-cap soft warn: fires once per crossing (re-arms once RSS drops
    // back under the line) so operators get a heads-up BEFORE the kill rather
    // than a silent death. Cheap: one extra comparison per check.
    const softLine = Math.floor(this.opts.maxRssMb * 0.8);
    if (rssMb < this.opts.maxRssMb) {
      if (rssMb >= softLine && !this._softWarnFired) {
        this._softWarnFired = true;
        const ts = new Date().toISOString().slice(11, 19);
        console.warn(
          `[watchdog ${ts}] approaching cap: rss=${rssMb}MB (${Math.round((rssMb / this.opts.maxRssMb) * 100)}% of ${this.opts.maxRssMb}MB) ` +
          `peak=${this._peakRssMb}MB in_flight=${inFlightKinds.join(',') || 'none'} — next overshoot will drain. ` +
          `Raise --max-rss if this job kind legitimately needs more.`,
        );
      } else if (rssMb < softLine) {
        this._softWarnFired = false;
      }
      return;
    }

    this._rssWatchdogTriggered = true;
    const ts = new Date().toISOString().slice(11, 19);
    console.warn(
      `[watchdog ${ts}] rss=${rssMb}MB threshold=${this.opts.maxRssMb}MB peak=${this._peakRssMb}MB ` +
      `jobs_completed=${this.jobsCompleted} in_flight=${inFlightKinds.join(',') || 'none'} source=${source} — draining ` +
      `(raise --max-rss if this is legitimate working set, not a leak)`,
    );
    this.gracefulShutdown('watchdog');
  }

  /** Trigger a unified-style graceful shutdown. Fires shutdownAbort + per-job
   *  aborts + running=false in that order so:
   *  1. Shell handlers (and anything subscribed to ctx.shutdownSignal) start
   *     their cleanup sequence (SIGTERM → 5s grace → SIGKILL on children).
   *  2. Cooperative handlers see ctx.signal.aborted and bail instead of
   *     waiting out the 30s drain.
   *  3. Main loop exits at the top of the next iteration.
   *  The existing 30s drain in start()'s finally then backstops genuinely
   *  uninterruptible work. A claim whose UPDATE committed but whose result has
   *  not reached the poll loop is not in `inFlight` yet; launchJob aborts it. */
  private gracefulShutdown(reason: string): void {
    if (this.gracefulShutdownReason !== null) return;
    this.gracefulShutdownReason = reason;
    if (!this.shutdownAbort.signal.aborted) {
      this.shutdownAbort.abort(new Error(reason));
    }
    for (const entry of this.inFlight.values()) {
      if (!entry.abort.signal.aborted) {
        entry.abort.abort(new Error(reason));
      }
    }
    this.running = false;
  }

  /**
   * Launch a job as an independent in-flight promise.
   *
   * v0.41.22.2 hardening — the lock-renewal cathedral wave (closes the
   * production unhandledRejection crash class + 4 codex outside-voice
   * gaps). The renewal timer now wraps a pure `runLockRenewalTick`
   * call from `src/core/minions/lock-renewal-tick.ts` rather than
   * inlining `setInterval(async () => { await renewLock(...) })` —
   * which would let any throw escape to `process.on('unhandledRejection')`
   * and crash the worker (the v0.41.22.1 bug).
   *
   * State machine guarded by:
   *   - `cancelled` flag set in the finally block so an in-flight
   *     renewLock that resolves after the job ended bails cleanly (D1)
   *   - `tickInFlight` re-entrancy guard so overlapping ticks during a
   *     PgBouncer stall don't pile concurrent connection acquisitions
   *     on an already-saturated pool
   *   - `Promise.race(renewLock, timeoutPromise)` inside the tick so a
   *     hung connection can't wedge the re-entrancy guard forever (D6 / codex C3)
   *   - time-based abort (`Date.now() - lastSuccessfulRenewalAt >=
   *     lockDuration - safetyMargin`) so we voluntarily release BEFORE
   *     the stall detector can reclaim the row (D6 / codex C2)
   *
   * Universal grace-eviction (D8b / codex C7): the 30s force-evict
   * safety net fires for ANY abort reason, not just `job.timeout_ms`.
   * Handlers that ignore AbortSignal won't wedge the inFlight slot
   * forever on lock-renewal aborts.
   *
   * Second unhandledRejection vector (D7 / codex C5): the stored
   * `executeJob(...).finally(...)` promise gets an explicit `.catch()`
   * so an unhandled rejection inside the finally/catch chain (e.g.,
   * `failJob` throwing during the same DB outage) can't propagate to
   * the process-level handler and crash the daemon.
   */
  /**
   * One formatter for the classified abort telemetry (R2-7) so the
   * should_abort warn and the 30s-later grace-evict line can never drift
   * apart (they briefly did: `load1:` vs `load1_at_abort:`).
   */
  private formatAbortMeta(meta: Extract<TickResult, { kind: 'lock_lost' | 'should_abort' }>): string {
    if (meta.kind === 'lock_lost') {
      return `cause: ${meta.cause}, via: ${meta.via}`;
    }
    return `cause: ${meta.cause}, since_last_success_ms: ${Math.round(meta.sinceLastSuccessMs)}, ` +
      `tick_lateness_ms: ${Math.round(meta.latenessMs)}, overlap_skips: ${meta.overlapSkips}` +
      `${meta.load1 !== undefined ? `, load1: ${meta.load1.toFixed(2)}/${meta.cores} cores` : ''}`;
  }

  /**
   * Event-loop-delay sample for eviction log lines (CDX-1). The histogram
   * resets on every successful renewal (R2-9 via deps.onRenewalSuccess),
   * so these numbers attribute to the window since the last success —
   * i.e. exactly the window in which renewal was failing. ns→ms. Empty
   * string when the runtime lacks the histogram or sampling throws
   * (fail-open: never let telemetry break the eviction path).
   */
  private formatEvictionTelemetry(): string {
    const h = this.eldHistogram;
    if (h === null) return '';
    try {
      const p99Ms = Math.round(h.percentile(99) / 1e6);
      const maxMs = Math.round(h.max / 1e6);
      return ` [event_loop_delay since last renewal: p99 ${p99Ms}ms, max ${maxMs}ms]`;
    } catch {
      return '';
    }
  }

  private launchJob(job: MinionJob, lockToken: string): void {
    if (this.configurationError) return;
    const abort = new AbortController();
    const execution: OwnedExecution = { job, lockToken, stopped: false, abort };
    this.executions.set(lockToken, execution);

    // --- D1: cancellation flag for the in-flight renewal IIFE ---
    let cancelled = false;
    // --- re-entrancy guard for overlapping ticks during PgBouncer stalls ---
    let tickInFlight = false;
    // --- R2-7: the tick's final result, stashed at abort time so the ---
    // --- grace-evict log (which fires 30s LATER) can report cause/    ---
    // --- lateness/load instead of just the Error string.              ---
    let abortMeta: Extract<TickResult, { kind: 'lock_lost' | 'should_abort' }> | null = null;

    // R2-4: ALL elapsed-time arithmetic in the renewal state machine runs
    // on a monotonic clock — a wall-clock jump (NTP step, DST bug) must
    // never evict or indefinitely defer. Date.now stays only in log/audit
    // timestamps (the audit writer stamps its own `ts`).
    const monotonicNow = () => performance.now();

    // --- D3: pure-function lock renewal ---
    // #4145: the EFFECTIVE lease is per-job (claim stamped it from the
    // handler map / explicit submit; NULL = worker default). The cadence
    // clamps to 60s so a 300s lease renews 5x per window (matching the
    // cycle refresher's multiple-chances philosophy) instead of the bare
    // lease/2 = every 150s; leases ≤120s keep the legacy /2 exactly.
    // Defense-in-depth: re-clamp the row value at consumption. The exposed
    // submit surfaces already clamp, but the claim COALESCE trusts the row
    // and the DB CHECK only enforces > 0 — a writer that bypasses add()
    // (direct SQL repair, foreign tooling) could otherwise stamp a 1ms
    // lease (renewal-storm setInterval) or a ~25-day one (weeks-long
    // dead-worker pin).
    const effectiveLockMs = job.lock_duration_ms != null
      ? clampLockDurationMs(job.lock_duration_ms)
      : this.opts.lockDuration;
    const renewalIntervalMs = renewalIntervalFor(effectiveLockMs);
    const knobs = resolveLockRenewalKnobs(process.env, effectiveLockMs, renewalIntervalMs);
    const renewalState: LockRenewalState = {
      jobId: job.id,
      jobName: job.name,
      lockToken,
      lockDurationMs: effectiveLockMs,
      knobs,
      lastSuccessfulRenewalAt: monotonicNow(),
      consecutiveFailures: 0,
      cancelled: () => cancelled,
      intervalMs: renewalIntervalMs,
      lastTickFiredAt: monotonicNow(),
      overlapSkips: 0,
    };
    // issue #1678 (Codex #2): hand the tick a bounded reconnect-once hook when
    // the engine owns a pool that a transaction-mode pooler can reap. Postgres
    // exposes reconnect(); PGLite (no pooler) doesn't, so the hook is absent
    // and the tick keeps its legacy no-reconnect behavior.
    const engineReconnect = (this.engine as { reconnect?: (ctx?: { error?: unknown }) => Promise<void> }).reconnect;
    const renewalDeps: LockRenewalDeps = {
      renewLock: async (id, tok, dur, opts) => {
        try {
          return await this.queue.renewLock(id, tok, dur, opts);
        } catch (error) {
          if (isLocalConfigurationError(error)) {
            cancelled = true;
            this.blockForConfiguration(error);
          }
          throw error;
        }
      },
      audit: lockRenewalAudit,
      // R2-4: monotonic — see monotonicNow above.
      now: monotonicNow,
      setTimeout: (cb, ms) => globalThis.setTimeout(cb, ms),
      // Forward the tick's classified error (CODEX impl review #2) so a pooler
      // reap during lock renewal is audited as reap_detected, not reconnect_other.
      ...(engineReconnect ? { reconnect: (ctx?: { error?: unknown }) => engineReconnect.call(this.engine, ctx) } : {}),
      // Issue #4145 telemetry: raw loadavg + cached cores (CEO-F2: the tick
      // try/catches every call), and the R2-9 histogram reset-on-success.
      loadSnapshot: () => ({ load1: loadavg()[0], cores: this.cpuCores }),
      onRenewalSuccess: () => { try { this.eldHistogram?.reset(); } catch { /* fail-open */ } },
    };

    const lockTimer = setInterval(() => {
      if (tickInFlight) {
        // Overlap skip (CDX-13): a prior tick is still awaiting its
        // renewal call. Count it — this is NOT a missed interval (those
        // coalesce and show up as tick LATENESS instead) — and ADVANCE the
        // lateness baseline: this callback fired on schedule, so the next
        // executed tick must not book the skipped window as event-loop
        // lateness (that would misclassify a slow DB call as starvation).
        renewalState.overlapSkips += 1;
        renewalState.lastTickFiredAt = monotonicNow();
        return;
      }
      tickInFlight = true;
      void runLockRenewalTick(renewalDeps, renewalState)
        .then((result) => {
          if (cancelled) return;
          switch (result.kind) {
            case 'ok':
            case 'cancelled':
              return;
            case 'lock_lost':
              if (!abort.signal.aborted) {
                abortMeta = result;
                console.warn(
                  `Lock lost for job ${job.id}, aborting execution ` +
                  `(${this.formatAbortMeta(result)}${this.formatEvictionTelemetry()})`,
                );
                clearInterval(lockTimer);
                abort.abort(new Error('lock-lost'));
              }
              return;
            case 'should_abort':
              if (!abort.signal.aborted) {
                abortMeta = result;
                // Issue #4145 request 3: the one line that saves the 8h of
                // forensics — WHY renewal failed + was the loop starved.
                console.warn(
                  `Lock renewal failed for job ${job.id} (${job.name}); aborting ` +
                  `(${this.formatAbortMeta(result)}${this.formatEvictionTelemetry()})`,
                );
                clearInterval(lockTimer);
                abort.abort(new Error(result.reason));
              }
              return;
          }
        })
        .catch((err) => {
          // Belt-and-suspenders. runLockRenewalTick's own try/catch
          // should make this unreachable, but a stray throw from the
          // .then handler itself (console.warn EPIPE on a piped worker
          // for instance) would otherwise propagate to
          // unhandledRejection and crash the daemon — the exact bug
          // class this whole wave exists to close.
          const msg = err instanceof Error ? err.message : String(err);
          console.error(`[worker] runLockRenewalTick post-handler error: ${msg}`);
        })
        .finally(() => {
          tickInFlight = false;
        });
    }, renewalIntervalMs);

    // --- D8b: universal grace-eviction timer ---
    // Fires for ANY abort reason (not just job.timeout_ms). Without
    // this generalization, lock-renewal aborts could leave the inFlight
    // slot wedged forever if the handler ignores AbortSignal.
    let graceTimer: ReturnType<typeof setTimeout> | null = null;
    abort.signal.addEventListener('abort', () => {
      // Avoid scheduling a second grace timer if abort fires again
      // (e.g., timeout + lock-renewal-failed close to each other).
      if (graceTimer != null) return;
      graceTimer = setTimeout(() => {
        if (this.inFlight.has(job.id)) {
          const reason = abort.signal.reason instanceof Error
            ? abort.signal.reason.message
            : String(abort.signal.reason);
          // R2-7: abortMeta carries the tick's classified cause + starvation
          // telemetry captured AT abort time — the Error string alone would
          // make this line read like an orphan leak (the #4145 forensics trap).
          const meta = abortMeta === null ? '' : ` (${this.formatAbortMeta(abortMeta)})`;
          console.warn(
            `Job ${job.id} (${job.name}) did not exit within 30s of abort (reason: ${reason}).${meta} ` +
            `Force-evicting from inFlight to unblock worker. ` +
            (this.configurationError
              ? 'The handler is still running; configuration blocking prevents new claims.'
              : 'The handler is still running but the worker will claim new jobs.') +
            this.formatEvictionTelemetry()
          );
          clearInterval(lockTimer);
          // R2-1 generation-safety: delete ONLY our own execution's entry.
          // After a force-evict, this job id can be requeued and re-claimed
          // by THIS worker while the old handler is still alive — an
          // unconditional delete-by-id would then remove the NEW
          // execution's entry (concurrency undercount, lost tracking).
          // The lockToken is minted per claim, so it is the generation.
          if (this.inFlight.get(job.id)?.lockToken === lockToken) {
            this.inFlight.delete(job.id);
          }
          // D8a: don't failJob if the abort was infrastructure. The stall
          // detector will reclaim the row cleanly: a lock-renewal abort
          // now fires only after the at-deadline VERIFY either returned
          // fenced-false (row already reclaimed) or stayed unreachable
          // past hardEvictMs (lease long expired) — see #4145
          // verify-before-evict in lock-renewal-tick.ts. Isolation mode:
          // also skip — the group SIGKILL already fired and executeJob's
          // own recording follows; a competing evict failJob('dead') could
          // dead-letter a job with attempts remaining (adversarial-review P3).
          if (!this.configurationError && !INFRASTRUCTURE_ABORT_REASONS.has(reason) && this.opts.jobIsolation !== 'process') {
            this.queue.failJob(
              job.id,
              lockToken,
              'handler ignored abort signal (force-evicted)',
              'dead',
            ).catch(() => {});
          }
        }
      }, 30_000);
    });

    // Per-job wall-clock timeout (timer-armed only if `timeout_ms` was
    // set on the job; the grace-evict pattern above now lives outside
    // this branch). The delay derives from the claim-time `timeout_at`
    // stamp when present so this timer, the DB sweeper (handleTimeouts),
    // and the handler-visible `deadlineAtMs` all agree on ONE absolute
    // deadline instead of three clocks started at slightly different
    // instants.
    let timeoutTimer: ReturnType<typeof setTimeout> | null = null;
    if (job.timeout_ms != null) {
      const delayMs = job.timeout_at != null
        ? Math.max(0, job.timeout_at.getTime() - Date.now())
        : job.timeout_ms;
      timeoutTimer = setTimeout(() => {
        if (!abort.signal.aborted) {
          console.warn(`Job ${job.id} (${job.name}) hit per-job timeout (${job.timeout_ms}ms), aborting`);
          abort.abort(new Error('timeout'));
        }
      }, delayMs);
    }

    const promise = this.executeJob(job, lockToken, abort, lockTimer)
      .catch(error => {
        if (isLocalConfigurationError(error)) this.blockForConfiguration(error);
        throw error;
      })
      .finally(() => {
        // D1: signal in-flight IIFE to bail at its next checkpoint so
        // a renewLock resolution that lands after the job ended
        // doesn't write a misleading audit event or abort an
        // already-dead controller.
        cancelled = true;
        clearInterval(lockTimer);
        if (timeoutTimer) clearTimeout(timeoutTimer);
        if (graceTimer) clearTimeout(graceTimer);
        // R2-1 generation-safety: a force-evicted execution's finally can
        // fire long after the same job id was re-claimed by this worker.
        // Only delete the entry if it is still OURS (token = generation).
        if (this.inFlight.get(job.id)?.lockToken === lockToken) {
          this.inFlight.delete(job.id);
        }
        if (!this.configurationError || this.configurationDrainFinished) this.executions.delete(lockToken);
        this.jobsCompleted += 1;
        this.checkMemoryLimit('post-job');
        try { this.opts.onJobSettled(job); } catch { /* the terminal hook never affects the job */ }
      })
      // D7 / codex C5: close the SECOND unhandledRejection vector. If
      // executeJob's catch path throws (e.g., failJob's executeRaw
      // throws during the same DB outage that caused lock renewal to
      // fail), the rejection would otherwise escape to
      // process.on('unhandledRejection') and crash the daemon.
      .catch((err) => {
        const msg = err instanceof Error ? err.message : String(err);
        console.error(`[worker] executeJob unhandled error for job ${job.id} (${job.name}): ${msg}`);
        try {
          lockRenewalAudit.logExecuteJobRejected(job.id, job.name, err);
        } catch { /* audit best-effort */ }
      });

    for (const entry of this.inFlight.values()) entry.shared = true;
    this.inFlight.set(job.id, { job, lockToken, lockTimer, abort, promise, shared: this.inFlight.size > 0 || this.gracefulShutdownReason !== null });
    execution.promise = promise;
    if (this.gracefulShutdownReason !== null && !abort.signal.aborted) abort.abort(new Error(this.gracefulShutdownReason));
  }

  /** A handler deferral (JobDeferredError): back to delayed, no attempt counted. */
  private async deferJob(job: MinionJob, lockToken: string, err: JobDeferredError): Promise<void> {
    const deferred = await this.queue.deferJob(job.id, lockToken, `deferred (${err.reason}): ${err.message}`, err.retryInMs);
    if (!deferred) console.warn(`Job ${job.id} deferral dropped (lock token mismatch)`);
    else console.log(`Job ${job.id} (${job.name}) deferred (${err.reason}) for ${Math.round(err.retryInMs / 1000)}s (no attempt burned)`);
  }

  private async executeJob(
    job: MinionJob,
    lockToken: string,
    abort: AbortController,
    lockTimer: ReturnType<typeof setInterval>,
  ): Promise<void> {
    const handler = this.handlers.get(job.name);
    if (!handler) {
      await this.queue.failJob(job.id, lockToken, `No handler for job type '${job.name}'`, 'dead');
      return;
    }

    // issue #5: 'process' isolation runs the handler in a SIGKILL-able child;
    // the parent keeps claim/renewal and ALL result recording below — this
    // branch swaps ONLY the execution engine. When isolated, the parent-side
    // context is skipped entirely (the child builds its own against its own
    // engine; building it here would be dead work holding closures). The
    // constructor guarantees childCliInvocation != null whenever
    // jobIsolation === 'process', so this predicate matches the evict guard.
    const isolated = this.opts.jobIsolation === 'process';

    // Build job context with per-job AbortSignal + shared shutdown signal.
    // Most handlers only care about `signal` (timeout / cancel / lock-loss).
    // `shutdownSignal` is separate: fires only on worker process SIGTERM/SIGINT.
    // Handlers that need to run cleanup before worker exit (shell handler's
    // SIGTERM→5s→SIGKILL on its child) subscribe to shutdownSignal too.
    // Builder shared with `gbrain jobs run-child` (job-context.ts) so the
    // process-isolation child wires the exact same DB-backed callbacks.
    const context: MinionJobContext | null = isolated
      ? null
      : buildJobContext(
          this.engine,
          this.queue,
          job,
          lockToken,
          abort.signal,
          this.shutdownAbort.signal,
        );

    let childExecutionStopped = false;
    try {
      const authority = await authorizeJobExecution(this.engine, job);
      if (this.configurationError) {
        const execution = this.executions.get(lockToken);
        if (execution) execution.stopped = true;
        return;
      }
      const result = isolated
        ? await runJobInChild({
            jobId: job.id, spendAuthorized: job.spend_authorization != null,
            jobName: job.name,
            lockToken,
            abortSignal: abort.signal,
            shutdownSignal: this.shutdownAbort.signal,
            invocation: this.opts.childCliInvocation as { cmd: string; argsPrefix: string[] },
            tiniPath: this.opts.childTiniPath, maxRssMb: this.opts.maxRssMb,
            onExecutionStopped: () => {
              childExecutionStopped = true;
              const execution = this.executions.get(lockToken);
              if (execution) execution.stopped = true;
            },
            onConfigurationError: error => {
              this.blockForConfiguration(error);
            },
          })
        // #4218: attribute every gateway.chat() the handler makes to this
        // job so chat_usage_log rows carry `phase = 'job:<name>'`.
        : await withEngineScope(this.engine, () => withSubmissionAuthority(authority, () => withChatPhase(`job:${job.name}`, () => runWithJobSpend(this.engine, job, context as MinionJobContext, handler)), abort.signal));

      // The child spawned and ran — the spawn path is healthy again.
      this._consecutiveChildSpawnFailures = 0;

      clearInterval(lockTimer);

      const stored = result != null ? (typeof result === 'object' ? result as Record<string, unknown> : { value: result }) : undefined;
      const verdict = jobResultOutcome(stored);
      const completed = verdict.outcome === 'failed'
        ? await this.queue.failJob(job.id, lockToken, verdict.reason!, 'failed', 0, jobOutcomeResult(stored, verdict))
        : await this.queue.completeJob(job.id, lockToken, jobOutcomeResult(stored, verdict));
      const execution = this.executions.get(lockToken);
      if (execution) execution.stopped = true;

      if (!completed) {
        console.warn(`Job ${job.id} completion dropped (lock token mismatch, job was reclaimed)`);
        return;
      }
      if (verdict.outcome === 'failed') console.log(`Job ${job.id} (${job.name}) finished failed: ${verdict.reason}`);
      // resolveParent is folded into queue.completeJob() (same transaction as
      // status flip + token rollup + child_done), so a process crash here can't
      // strand the parent in waiting-children.
    } catch (err) {
      clearInterval(lockTimer);
      const execution = this.executions.get(lockToken);
      if (execution) execution.stopped = err instanceof ChildWorkerShutdownError
        ? err.executionStopped === true
        : !isolated || childExecutionStopped;
      if (isLocalConfigurationError(err)) this.blockForConfiguration(err);
      if (this.configurationError) return;

      // If the per-job abort fired, derive the reason from signal.reason (set
      // by whichever site aborted: 'timeout' / 'cancel' / 'lock-lost'). We call
      // failJob unconditionally — the DB match on status='active' + lock_token
      // makes it idempotent: if another path (handleTimeouts, cancelJob, stall)
      // already flipped status, our call no-ops cleanly. The prior silent-return
      // left jobs stranded in 'active' until a secondary sweep, breaking
      // timeout/cancel contracts downstream callers rely on.
      let errorText: string, abortReason: string | null = null;
      if (abort.signal.aborted) {
        abortReason = abort.signal.reason instanceof Error
          ? abort.signal.reason.message
          : String(abort.signal.reason || 'aborted');
        errorText = `aborted: ${abortReason}`;
      } else {
        errorText = err instanceof Error ? err.message : String(err);
      }
      if (abortReason === 'watchdog' && await settleWatchdogAbort(this.queue, this.engine, job, lockToken, this.inFlight.get(job.id)?.shared !== false, execution?.stopped !== false)) return;

      // v0.41.22.2 (D8a / codex C6): infrastructure aborts (lock-renewal-failed,
      // lock-lost) are NOT job defects — they're connection / coordination
      // failures the stall detector will reclaim cleanly. Calling failJob here
      // would burn an attempt or dead-letter the job for what's really a
      // PgBouncer blip; that's a worse outcome than the v0.41.22.1 crash it
      // replaces. The lock has already expired (lock-renewal-failed only fires
      // after lockDuration - safetyMargin elapsed without renewal), so the
      // stall detector will pick the row up on its next poll and another
      // worker will claim it cleanly.
      if (abortReason !== null && INFRASTRUCTURE_ABORT_REASONS.has(abortReason)) {
        console.log(
          `Job ${job.id} (${job.name}) released after infrastructure abort (${abortReason}); ` +
          `stall detector will requeue (no attempt burned)`,
        );
        return;
      }

      // Any error that ISN'T a spawn failure proves the spawn path works —
      // keep the breaker's "consecutive" semantics honest.
      if (!(err instanceof ChildSpawnInfraError)) {
        this._consecutiveChildSpawnFailures = 0;
      }

      // issue #5 process isolation — two more infrastructure classes, same
      // release semantics as the block above (lock expires once launchJob's
      // finally clears the renewal timer; the stall sweeper requeues):
      //   - spawn failure: an ops misconfiguration (bad child CLI path) must
      //     not burn attempts job-by-job until the queue dead-letters. The
      //     CLI layer also fail-fast validates the invocation at startup.
      //   - worker shutdown: a routine deploy killed the child before it
      //     could report (codex-2 #7); burning an attempt per deploy would
      //     dead-letter long jobs after a few releases.
      if (err instanceof ChildSpawnInfraError) {
        console.error(
          `Job ${job.id} (${job.name}) released after child spawn failure — ` +
          `check the worker's child CLI configuration: ${errorText} (no attempt burned)`,
        );
        this._consecutiveChildSpawnFailures += 1;
        if (this._consecutiveChildSpawnFailures >= MinionWorker.CHILD_SPAWN_FAIL_EXIT_AFTER) {
          console.error(
            `[isolation] ${this._consecutiveChildSpawnFailures} consecutive child spawn/bootstrap ` +
            `failures — the child CLI is deterministically broken. Emitting 'unhealthy' for ` +
            `process-manager restart instead of looping claim/release forever.`,
          );
          this.emitUnhealthy({
            reason: 'child_spawn_failing',
            consecutiveFailures: this._consecutiveChildSpawnFailures,
            message: errorText,
          });
        }
        return;
      }
      if (err instanceof ChildWorkerShutdownError || (!isolated && endedByShutdown(err, this.shutdownAbort.signal, abort.signal))) {
        return settleShutdownInterruptedJob(this.engine, job, lockToken, errorText, !(err instanceof ChildWorkerShutdownError) || err.executionStopped === true);
      }
      if (err instanceof ChildNotClaimedError) {
        // The child proved the claim is gone (reclaimed/cancelled before the
        // handler ran). The token-fenced failJob would no-op anyway — return
        // without burning anything against a claim we no longer hold.
        console.log(`Job ${job.id} (${job.name}): ${errorText}`);
        return;
      }

      // v0.41 Bug 2: lease-full bounces don't burn attempts.
      //
      // Pre-v0.41 every non-`UnrecoverableError` routed to `delayed` with
      // exponential backoff BUT still incremented `attempts_made`. After 3
      // lease-full bounces the job hit `max_attempts` and dead-lettered
      // with message `rate lease "..." full (N/M)` — operators saw a
      // "dead" job and assumed real failure. The field-report dead-letter
      // loop is exactly this path.
      //
      // Detect `RateLeaseUnavailableError` BEFORE the attempts-exhaustion
      // gate and route through `queue.releaseLeaseFullJob` which mirrors
      // `failJob` minus the `attempts_made` increment. Audit row to
      // `minion_lease_pressure_log` so operators see pressure live in
      // `gbrain doctor` + `gbrain jobs stats lease_pressure`.
      if (err instanceof JobDeferredError) return this.deferJob(job, lockToken, err);
      const isLeaseFull = err instanceof RateLeaseUnavailableError;
      if (isLeaseFull) {
        const leaseErr = err as RateLeaseUnavailableError;
        // 1-3s jittered backoff (shared with the inline drain — one curve,
        // no silent desync). Not the exponential curve — this is "yield the
        // slot, try again soon", not "give up after a few tries."
        // #4310: a caller-suggested delay (the global-LLM-halt cooldown's
        // remaining window) wins over the short lease bounce.
        const leaseBackoffMs = leaseErr.retryInMs ?? leaseFullBackoffMs();
        const released = await this.queue.releaseLeaseFullJob(
          job.id, lockToken, errorText, leaseBackoffMs,
        );
        if (!released) {
          console.warn(`Job ${job.id} lease-full release dropped (lock token mismatch)`);
          return;
        }
        // Audit row write is best-effort — never blocks the bypass path.
        // Denormalized columns persist past `gbrain jobs prune` so post-NULL
        // forensic queries still see context (Eng D8 / codex pass-3 #7).
        await logLeasePressure(this.engine, {
          job_id: job.id,
          lease_key: leaseErr.key,
          active_at_bounce: leaseErr.active,
          max_concurrent: Number.isFinite(leaseErr.max) ? leaseErr.max : -1,
          queue_name: job.queue,
          job_name: job.name,
          // Best-effort context — populated when we can. The worker doesn't
          // always know the model at catch time (model is resolved inside
          // the handler), so leave NULL when unavailable. The doctor check's
          // aggregate queries handle NULL gracefully.
          model: null,
          provider: null,
          root_owner_id: job.parent_job_id ?? null,
        });
        console.log(
          `Job ${job.id} (${job.name}) lease-full, re-queuing in ${Math.round(leaseBackoffMs)}ms (no attempt burned)`,
        );
        return;
      }

      const refusal = jobRefusal(err), isUnrecoverable = refusal !== null || err instanceof UnrecoverableError; // #7239: refusals end now
      const attemptsExhausted = job.attempts_made + 1 >= job.max_attempts;

      let newStatus: 'delayed' | 'failed' | 'dead';
      if (isUnrecoverable || attemptsExhausted) {
        newStatus = 'dead';
      } else {
        newStatus = 'delayed';
      }

      const backoffMs = newStatus === 'delayed' ? calculateBackoff({
        backoff_type: job.backoff_type,
        backoff_delay: job.backoff_delay,
        backoff_jitter: job.backoff_jitter,
        attempts_made: job.attempts_made + 1,
      }) : 0;

      // issue #1720: failJob can itself throw during the same DB outage that
      // failed the job. Pre-fix the rejection escaped to launchJob's .catch
      // and the ORIGINAL job error was never logged anywhere — the recording
      // error masked it. Log the original FIRST (it must survive no matter
      // what), then reconnect + retry the recording once. If it still fails,
      // leave the row to the stall detector: the lock has stopped renewing,
      // so handleStalled requeues it cleanly on a live pool (the D8a path).
      let failed: MinionJob | null;
      try {
        failed = await this.queue.failJob(job.id, lockToken, errorText, newStatus, backoffMs, refusal ? refusedJobResult(refusal) : undefined);
      } catch (recordErr) {
        const recordMsg = recordErr instanceof Error ? recordErr.message : String(recordErr);
        console.error(
          `Job ${job.id} (${job.name}) failed with: ${errorText} — and recording the failure threw: ${recordMsg}`,
        );
        if (!isRetryableConnError(recordErr)) throw recordErr;
        await this.reconnectAfterConnectionError('failJob', recordErr);
        try {
          failed = await this.queue.failJob(job.id, lockToken, errorText, newStatus, backoffMs, refusal ? refusedJobResult(refusal) : undefined);
        } catch (retryErr) {
          if (isLocalConfigurationError(retryErr)) {
            this.blockForConfiguration(retryErr);
            return;
          }
          console.error(
            `Job ${job.id} (${job.name}) failure-recording retry also failed ` +
            `(${retryErr instanceof Error ? retryErr.message : String(retryErr)}); ` +
            `leaving the row for the stall detector to requeue after lock expiry`,
          );
          return;
        }
      }
      if (!failed) {
        console.warn(`Job ${job.id} failure dropped (lock token mismatch)`);
        return;
      }
      // Parent-failure hook (fail_parent / remove_dep / ignore / continue) is
      // folded into queue.failJob() in the same transaction as the child status
      // flip + remove_on_fail delete. Worker stays out of multi-statement
      // crash-window territory.

      if (newStatus === 'delayed') {
        console.log(`Job ${job.id} (${job.name}) failed, retrying in ${Math.round(backoffMs)}ms (attempt ${job.attempts_made + 1}/${job.max_attempts})`);
      } else {
        console.log(`Job ${job.id} (${job.name}) ${refusal ? `refused (${refusal.code}), not retried` : 'permanently failed'}: ${errorText}`);
      }
    }
  }
}