Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
121 KB
Raw Blame
/**
 * MinionQueue — Postgres-native job queue inspired by BullMQ.
 *
 * Usage:
 *   const queue = new MinionQueue(engine);
 *   const job = await queue.add('sync', { full: true });
 *   const status = await queue.getJob(job.id);
 *   await queue.prune({ olderThan: new Date(Date.now() - 30 * 86400000) });
 */

import { APPLICATION_AUTHORITY, LEGACY_AUTHORITY_COLUMN, assertNoUnreviewedJobs, authorizeJobExecution, currentSubmissionAuthority, parseSubmissionAuthority, type SubmissionAuthority } from './submission-authority.ts';
import type { BrainEngine } from '../engine.ts';
import type {
  MinionJob, MinionJobInput, MinionJobStatus, InboxMessage, TokenUpdate,
  MinionQueueOpts, ChildDoneMessage, ChildOutcome, Attachment, AttachmentInput,
} from './types.ts';
import { rowToMinionJob, rowToInboxMessage, rowToAttachment, jobActorSql, NO_JOB_ACTOR, type JobActor } from './types.ts';
import { WATCHDOG_SUSPECT_MARK, markedJobSql, releaseWatchdogClaim, requeueCoRunnerStalls, unmarkedJobSql } from './watchdog-attribution.ts';
export { WATCHDOG_SUSPECT_MARK } from './watchdog-attribution.ts';
import { coalesceOnIdempotencyKey, decideCoalesce, insertOrCoalesce } from './idempotency-coalesce.ts';
import { withSubmitterLabel } from './job-insight.ts';
import { adoptSpendAuthorization, legacyDefaultClaimSetSql, legacyDefaultClaimParams, type SpendAuthorization } from './spend-authorization.ts';
import { validateAttachment } from './attachments.ts';
import type { QueueFamilyStats } from './queue-families.ts';
import { isProtectedJobName, LEASE_BEACON_QUEUE } from './protected-names.ts';
import { jsonbSafeJson } from '../text-safe.ts';
import { assertEmbedBackfillQueueAdmission } from './embed-backfill-admission.ts';
import { lockDelegatedSubmission, checkDelegatedCapacity, prepareDelegatedReplay, admitDelegatedRetry } from './delegated-admission.ts';
import {
  computeParamHash,
  resolveAdmissionPolicy,
  resolveTtlNames,
  QueueQuotaExceededError,
  PARAM_HASH_EXCLUDED_KEYS,
  TTL_REASON_PREFIX,
} from './admission.ts';
import {
  defaultTimeoutMsFor, HANDLER_DEFAULT_TIMEOUT_MS,
  defaultLockDurationMsFor, HANDLER_DEFAULT_LOCK_DURATION_MS, clampLockDurationMs, WALL_CLOCK_THRESHOLD_MS_SQL,
} from './handler-timeouts.ts';
import {
  withRetry, BULK_RETRY_OPTS, resolveBulkRetryOpts, computeNextDelay,
  isRetryableConnError,
} from '../retry.ts';
import {
  logBatchRetry as auditLogBatchRetry,
  logBatchExhausted as auditLogBatchExhausted,
} from '../audit/batch-retry-audit.ts';
import { retryOnStalePlan } from '../retry-matcher.ts';
/** Trusted 4th argument, kept outside user-spread job options. */
export interface TrustedSubmitOpts {
  submissionAuthority?: SubmissionAuthority;
  /** Allow PROTECTED_JOB_NAMES; CLI or operation-local callers only. */
  allowProtectedSubmit?: boolean;
  /** Allow PGLite embed-backfill only for an explicit inline-worker caller. */
  allowPgliteInlineWorker?: boolean;
  /** Authenticated submit_agent identity, never read from agent job parameters. */
  delegatedClientId?: string;
  /** Consent-gated CLI producers only: the user's spend authorization (spend-authorization.ts). */
  spendAuthorization?: SpendAuthorization;
}

const MIGRATION_VERSION = 7;

const DEFAULT_MAX_SPAWN_DEPTH = 5;
export const DREAM_INLINE_PRIVATE_QUEUE_PREFIX = 'dream-inline-';
export const DEFAULT_PRIVATE_QUEUE_LEASE_MS = 10 * 60 * 1000;
/**
 * Machine-readable reason family for private-queue reconciliation, mirroring
 * the waiting-TTL prefix convention: every reconcile cancellation is stamped
 * `<prefix>: <detail>` by reconcilePrivateQueue itself, so jobs-stats/doctor
 * surfaces can LIKE-match the family without chasing per-call-site strings.
 */
export const PRIVATE_QUEUE_RECONCILE_REASON_PREFIX = 'private_queue_reconciled';

/**
 * Stall-sweep reclaim grace (#4145, CDX-7): don't reclaim a row whose
 * `lock_until` lapsed within the last N ms. When a CPU-starved worker's
 * event loop unblocks, its coalesced renewal tick and the stall sweep
 * fire in the same burst — if the sweep's UPDATE lands first it steals
 * the OWNER'S live job. The grace is a HEAD-START for the owner's
 * recovery renewal, not a guarantee: it only covers starvation bursts
 * shorter than the grace, and a healthy second worker's sweep still
 * wins beyond it. Minion analog of `GBRAIN_LOCK_STEAL_GRACE_SECONDS`
 * (db-lock.ts), adapted because minion_jobs has no last_refreshed_at.
 *
 * Cost: dead-worker recovery becomes lock_until + grace + up to
 * stalledInterval. Env `GBRAIN_MINION_STALL_RECLAIM_GRACE_MS` (0 allowed
 * — restores the exact legacy reclaim predicate).
 */
export const DEFAULT_STALL_RECLAIM_GRACE_MS = 15_000;

const _warnedGraceEnv = new Set<string>();

export function _resetStallGraceWarningsForTests(): void {
  _warnedGraceEnv.clear();
}

export function resolveStallReclaimGraceMs(
  env: Record<string, string | undefined> = process.env,
): number {
  const raw = env.GBRAIN_MINION_STALL_RECLAIM_GRACE_MS;
  if (raw === undefined || raw.trim() === '') return DEFAULT_STALL_RECLAIM_GRACE_MS;
  // Unlike the lock-renewal knobs, 0 is a VALID value here (legacy reclaim).
  if (!/^\d+$/.test(raw.trim())) {
    if (!_warnedGraceEnv.has(raw)) {
      _warnedGraceEnv.add(raw);
      process.stderr.write(
        `[minions] env GBRAIN_MINION_STALL_RECLAIM_GRACE_MS=${JSON.stringify(raw)} is not a non-negative integer; ` +
        `falling back to default ${DEFAULT_STALL_RECLAIM_GRACE_MS}\n`,
      );
    }
    return DEFAULT_STALL_RECLAIM_GRACE_MS;
  }
  const n = Number(raw.trim());
  // Cap at 10 minutes: an absurd digit string (Number → huge/Infinity)
  // would otherwise push the sweep cutoff to -infinity and silently
  // disable stalled-job recovery altogether.
  const MAX_GRACE_MS = 600_000;
  if (n > MAX_GRACE_MS) {
    if (!_warnedGraceEnv.has(raw)) {
      _warnedGraceEnv.add(raw);
      process.stderr.write(
        `[minions] env GBRAIN_MINION_STALL_RECLAIM_GRACE_MS=${JSON.stringify(raw)} exceeds the ${MAX_GRACE_MS}ms cap; clamping\n`,
      );
    }
    return MAX_GRACE_MS;
  }
  return n;
}
const DEFAULT_MAX_ATTACHMENT_BYTES = 5 * 1024 * 1024; // 5 MiB

const TERMINAL_STATUSES = ['completed', 'failed', 'dead', 'cancelled'] as const;
const NON_TERMINAL_STATUSES: MinionJobStatus[] = ['waiting', 'active', 'delayed', 'waiting-children', 'paused'];
/**
 * Literal IN-list derived from the constant above, for the recovery queries:
 * the partial indexes idx_minion_jobs_private_queue_* embed these statuses in
 * their WHERE predicate, and a parameterized `= ANY($n)` under a generic plan
 * cannot be proven to imply it — literals keep the queries index-eligible.
 * Values are internal constants, never user input.
 */
const NON_TERMINAL_SQL_LIST = NON_TERMINAL_STATUSES.map(st => `'${st}'`).join(', ');

export function isDreamInlinePrivateQueue(queueName: string): boolean {
  return queueName.startsWith(DREAM_INLINE_PRIVATE_QUEUE_PREFIX);
}

export interface PrivateQueueRecoveryResult {
  scanned_queues: number;
  cancelled_queues: number;
  cancelled_jobs: number;
  skipped_live_queues: number;
  skipped_unowned_queues: number;
  skipped_non_orphan_queues: number;
}

/** Audit payload deferred from inside the submission transaction. */
type CoalesceAuditEvent = {
  queue: string; name: string; returned_job_id: number;
  waiting_count?: number; max_waiting?: number;
  pending_count?: number; max_pending?: number;
  /** Set when the coalesce matched on an identical payload hash (admission). */
  param_hash?: string;
};

/** Shared cap-hit coalesce return for the backpressure guards: hydrate the
 *  existing row, stamp the non-persisted `coalesced` marker, hand the audit
 *  payload to the caller's sink, return. Both maxWaiting and maxPending route
 *  through here so the coalesce contract cannot drift between them.
 *
 *  The sink DEFERS the audit write to after the transaction commits: the
 *  audit append is filesystem I/O, and doing it while holding the advisory
 *  lock + a pool connection would let a hung audit volume serialize every
 *  submission for the scope (adversarial-review finding). */
async function coalesceReturn(
  tx: BrainEngine,
  row: Record<string, unknown>,
  audit: Omit<CoalesceAuditEvent, 'returned_job_id'>,
  sink: (ev: CoalesceAuditEvent) => void,
  authority: SubmissionAuthority,
): Promise<MinionJob> {
  if (await decideCoalesce(tx, row, authority) !== 'coalesce') throw new Error(`job ${String(row.id)} is ${String(row.status)} and cannot be a coalesce target`);
  const coalesced = rowToMinionJob(row);
  coalesced.coalesced = true;
  sink({ ...audit, returned_job_id: coalesced.id });
  return coalesced;
}


/** SQL true for a job with the watchdog mark (`mark`) whose last requeue is at least `waitMs` milliseconds old (GST-2529). */
function overdueSuspectSql(mark: string, waitMs: string): string {
  return `(${markedJobSql(mark)} AND updated_at <= now() - (${waitMs}::double precision * interval '1 millisecond'))`;
}

export class MinionQueue {
  readonly maxSpawnDepth: number;
  readonly maxAttachmentBytes: number;

  constructor(private engine: BrainEngine, opts: MinionQueueOpts = {}) {
    this.maxSpawnDepth = opts.maxSpawnDepth ?? DEFAULT_MAX_SPAWN_DEPTH;
    this.maxAttachmentBytes = opts.maxAttachmentBytes ?? DEFAULT_MAX_ATTACHMENT_BYTES;
  }

  /** Verify minion_jobs table exists (migration v5+). Call before first operation. */
  async ensureSchema(): Promise<void> {
    const ver = await this.engine.getConfig('version');
    const current = parseInt(ver || '1', 10);
    if (current < MIGRATION_VERSION) {
      throw new Error(
        `minion_jobs table not found (schema version ${current}, need ${MIGRATION_VERSION}). Run 'gbrain init' to apply migrations.`
      );
    }
  }

  /**
   * Submit a new job.
   *
   * Wrapped in engine.transaction(): when parent_job_id is set, takes
   * SELECT ... FOR UPDATE on the parent so concurrent submissions serialize
   * on the cap check. Without this, two concurrent submissions could both
   * see count = N-1 and both insert, blowing max_children.
   *
   * Child status is 'waiting' (or 'delayed') — claimable. Parent is flipped
   * to 'waiting-children' atomically. Idempotency_key dedups via PG unique
   * partial index; same key returns the existing row (no second insert).
   */
  async add(
    name: string,
    data?: Record<string, unknown>,
    opts?: Partial<MinionJobInput>,
    trusted?: TrustedSubmitOpts,
  ): Promise<MinionJob> {
    // Normalize first so the protected-name check and the insert use the same
    // canonical form. Without the trim-before-check, `queue.add(' shell ', ...)`
    // would evade the guard and insert a job literally named 'shell'.
    if (currentSubmissionAuthority() && currentSubmissionAuthority()!.kind !== 'application') throw new Error('Remote jobs cannot submit descendant jobs');
    // #7240: Postgres refuses a lone UTF-16 surrogate or U+0000 anywhere in a JSONB value, which failed the whole insert.
    if (data !== undefined) data = jsonbSafeJson(data);
    const authority = parseSubmissionAuthority(trusted?.submissionAuthority ?? APPLICATION_AUTHORITY);
    if (!authority) throw new Error('Unsupported submission authority');
    const delegatedClientId = authority.kind === 'remote_agent' ? authority.principal.id : trusted?.delegatedClientId;
    if (authority.kind === 'remote_agent' && trusted?.delegatedClientId !== undefined && trusted.delegatedClientId !== delegatedClientId) {
      throw new Error('Delegated submission identity differs from its authority');
    }
    const jobName = (name || '').trim();
    if (authority.kind !== 'application') await authorizeJobExecution(this.engine, { name: jobName, data: data ?? {}, submission_authority: authority });
    if (jobName.length === 0) {
      throw new Error('Job name cannot be empty');
    }
    assertEmbedBackfillQueueAdmission(this.engine, jobName, data, trusted);
    if (isProtectedJobName(jobName) && !trusted?.allowProtectedSubmit) {
      throw new Error(
        `protected job name '${jobName}' requires CLI or operation-local submitter ` +
        `(pass {allowProtectedSubmit: true} as the 4th arg to MinionQueue.add)`,
      );
    }
    // v0.38 (S1.7 + D6) — capability-based gate replaces the v0.31.12 Anthropic
    // pin. The subagent loop now routes through `gateway.toolLoop()` so any
    // provider whose recipe declares tool calling AND supports_subagent_loop
    // works. Refuse-at-submit when the requested model cannot run a tool loop
    // (no tools, loop declared unsupported, or unknown provider). The handler
    // (`subagent.ts`) does a defense-in-depth check at dispatch time too.
    if (jobName === 'subagent' && data && typeof data === 'object') {
      const submittedModel = (data as { model?: unknown }).model;
      if (typeof submittedModel === 'string' && submittedModel.length > 0) {
        const { classifyCapabilities } = await import('../ai/capabilities.ts');
        const verdict = classifyCapabilities(submittedModel);
        if (verdict === 'unusable:no_tools') {
          throw new Error(
            `subagent job rejected: data.model "${submittedModel}" lacks native tool calling. ` +
            `The subagent loop dispatches brain ops via tool calls — without tool support the loop has no way to run. ` +
            `Pick a provider that supports tools (anthropic, openai, google, litellm, deepseek, groq, together, azure-openai).`,
          );
        }
        if (verdict === 'unusable:no_subagent_loop') {
          throw new Error(
            `subagent job rejected: data.model "${submittedModel}" comes from a provider whose recipe declares ` +
            `supports_subagent_loop: false — its tool_call_ids are not stable enough across crashes/replays ` +
            `to drive the subagent loop. ` +
            `Pick a provider whose recipe declares supports_subagent_loop: true (e.g. anthropic, openai, google, deepseek, groq).`,
          );
        }
        if (verdict === 'unknown') {
          const { listRecipes } = await import('../ai/recipes/index.ts');
          const known = listRecipes().map((r) => r.id).join(', ');
          throw new Error(
            `subagent job rejected: data.model "${submittedModel}" references an unknown provider. ` +
            `Use format provider:model where provider matches a recipe in src/core/ai/recipes/. ` +
            `Known providers: ${known}.`,
          );
        }
        // 'degraded:no_caching' and 'degraded:no_parallel' pass through — the
        // gateway prints a once-per-(source, model) cost warning at first
        // dispatch. 'ok' passes through silently.
      }
    }
    await this.ensureSchema();

    const childStatus: MinionJobStatus = opts?.delay ? 'delayed' : 'waiting';
    const delayUntil = opts?.delay ? new Date(Date.now() + opts.delay) : null;
    const maxSpawnDepth = opts?.max_spawn_depth ?? this.maxSpawnDepth;

    // Admission policy (param-coalescing + name-global quota). Resolved
    // OUTSIDE the transaction (60s in-process cache; fail-open to defaults).
    // Parented submits never coalesce: fanout children belong to their
    // parent's bookkeeping/aggregator — returning some other child would
    // corrupt child_done accounting. opts.coalesce_params overrides per call.
    const policy = await resolveAdmissionPolicy(this.engine, jobName);
    // An EMPTY payload (after excluding the hash key itself) carries no
    // dedupe signal — two no-param submits are more likely distinct
    // placeholder/scaffolding jobs than a runaway producer (which always
    // carries a prompt). Never coalesce those.
    // A caller-supplied idempotency_key also disables param-coalescing:
    // producer-owned idempotency is the STRONGER contract ("this exact key
    // maps to this exact row"), and a param-coalesce hit would return a row
    // the key was never registered against — a later same-key submit would
    // then insert fresh and run the work twice (adversarial-review finding).
    const hashablePayload = Object.keys(data ?? {}).some(k => !PARAM_HASH_EXCLUDED_KEYS.has(k));
    const coalesceActive =
      authority.kind === 'application' &&
      (opts?.coalesce_params ?? policy.coalesceParams) &&
      !opts?.parent_job_id &&
      !opts?.idempotency_key &&
      hashablePayload &&
      childStatus === 'waiting';
    let paramHash: string | null = null;
    if (coalesceActive) {
      // Execution options are part of the coalescing IDENTITY (codex re-review
      // P1): identical payloads with different timeout/priority/attempt/
      // quiet-hours semantics are NOT the same job — coalescing them would
      // silently hand the second submitter the first's execution contract.
      // Only DEFINED options fold in (conservative: an explicit value never
      // coalesces onto an implicit-default row; forward-safe like any hash
      // input change — old rows just stop matching).
      const optIdentity: Record<string, unknown> = {};
      for (const k of ['priority', 'timeout_ms', 'max_attempts', 'quiet_hours', 'lock_duration_ms', 'delay_ms', 'max_stalled'] as const) {
        const v = (opts as Record<string, unknown> | undefined)?.[k];
        if (v !== undefined) optIdentity[k] = v;
      }
      paramHash = computeParamHash(
        Object.keys(optIdentity).length > 0
          ? { ...(data ?? {}), __opts_identity: optIdentity }
          : ((data ?? {}) as Record<string, unknown>),
      );
      // Clone rather than mutate the caller's object; the hash rides in the
      // payload (the __-prefixed embedded-metadata convention) so the SQL
      // match needs no DDL and `jobs get` shows what matched.
      data = { ...(data ?? {}), __param_hash: paramHash };
    }

    // Set inside the transaction by a cap-hit coalesce; flushed AFTER commit
    // so audit filesystem I/O never runs while holding the advisory lock.
    let coalesceAudit: CoalesceAuditEvent | null = null;

    const result = await this.engine.transaction(async (tx) => {
      // Client lock spans grant validation, capacity check and insertion.
      const delegatedLimit = await lockDelegatedSubmission(tx, delegatedClientId, jobName, data);
      // 1. Idempotency fast path — if a row already exists for this key, return it
      //    without doing any other work. The unique partial index guarantees
      //    no second row can be inserted with the same non-null key.
      //
      //    Dead/cancelled jobs represent permanently-failed work whose
      //    idempotency slot must be freed so a fresh attempt can be inserted.
      //    We NULL the key (preserving the row, with the released key in its
      //    data for the dream breaker) and fall through to the INSERT below.
      if (opts?.idempotency_key) {
        const existing = await coalesceOnIdempotencyKey(tx, opts.idempotency_key, authority);
        if (existing) return existing;
      }

      // 1a. Param-coalescing (admission): an identical parentless submit —
      // same (name, queue, payload hash, incl. __owner_client_id so owner
      // lanes never cross) — returns the newest matching WAITING row instead
      // of inserting a duplicate. Honest-dispatch contract holds (coalesced:
      // true), and unlike the cap-hit coalesce below, returning this row to a
      // result-consumer is semantically exact: identical params ⇒ identical
      // result. Waiting-only by design (a RUNNING identical job does not
      // suppress a re-run; maxPending exists for single-flight callers).
      // Age-bounded to ttl/2: coalescing onto a nearly-TTL-expired row would
      // silently kill the fresh intent an hour later (round-2 V7).
      if (coalesceActive && paramHash) {
        const admissionQueue = opts?.queue ?? 'default';
        // Lock-census (PR6 D5): INTENTIONALLY not source-keyed — admission identity is (name, queue, param-hash); any source in the payload is already folded into paramHash.
        await tx.executeRaw(
          `SELECT pg_advisory_xact_lock(hashtext('minion_admission:' || $1 || ':' || $2 || ':' || $3))`,
          [jobName, admissionQueue, paramHash]
        );
        const ttlHours = policy.ttlWaitingHours;
        // updated_at, matching the TTL sweep's key: a requeued row has a
        // fresh TTL window and is a legitimate coalesce target again.
        const ageCond = ttlHours != null
          ? `AND updated_at > now() - ($4 * interval '1 hour')`
          : '';
        const matchParams: unknown[] = [jobName, admissionQueue, paramHash];
        if (ttlHours != null) matchParams.push(ttlHours / 2);
        const match = await tx.executeRaw<Record<string, unknown>>(
          `SELECT *, ${LEGACY_AUTHORITY_COLUMN} FROM minion_jobs
            WHERE name = $1 AND queue = $2 AND status = 'waiting'
              AND parent_job_id IS NULL
              AND data->>'__param_hash' = $3
              ${ageCond}
            ORDER BY created_at DESC, id DESC
            LIMIT 1`,
          matchParams
        );
        if (match.length > 0) {
          return coalesceReturn(tx, match[0], {
            queue: admissionQueue,
            name: jobName,
            param_hash: paramHash,
          }, ev => { coalesceAudit = ev; }, authority);
        }
      }

      await checkDelegatedCapacity(tx, delegatedClientId, delegatedLimit);

      // 1a2. Name-global waiting quota (admission; config-only, no shipped
      // default — user decision D2C). Counts the name across ALL queues:
      // fanout producers use per-run private queues (dream-inline-*), so a
      // queue-scoped count would reset per run and never bind. REJECTION,
      // not coalesce — quota-coalescing would hand result-consumers an
      // unrelated row. The name-global advisory lock below serializes
      // check+insert across concurrent submitters (adversarial finding:
      // without it, N parallel distinct-payload submits each observed
      // capacity and inserted, so overshoot was bounded only by attacker
      // concurrency — defeating the DoS backstop). Serialization cost only
      // applies to names with a quota configured, i.e. the runaway ones.
      if (policy.quotaMaxWaiting != null) {
        // Lock-census (PR6 D5): INTENTIONALLY cross-source AND cross-queue — the quota is a name-global DoS backstop; scoping the key would let per-source submitters overshoot it in parallel.
        await tx.executeRaw(
          `SELECT pg_advisory_xact_lock(hashtext('minion_quota:' || $1))`,
          [jobName]
        );
        const quotaRows = await tx.executeRaw<{ count: string }>(
          `SELECT count(*)::text AS count FROM minion_jobs WHERE name = $1 AND status = 'waiting'`,
          [jobName]
        );
        const waitingTotal = parseInt(quotaRows[0]?.count ?? '0', 10);
        if (waitingTotal >= policy.quotaMaxWaiting) {
          throw new QueueQuotaExceededError(jobName, waitingTotal, policy.quotaMaxWaiting);
        }
      }

      // 1b. Submission-time backpressure for high-frequency named jobs.
      // Two guards share the advisory-lock machinery but differ in what they
      // count and how they scope:
      //   - maxWaiting (rate cap): counts status='waiting' only. Source scope
      //     is NULL-as-wildcard — a submission with no source key counts ALL
      //     rows for (name, queue). Intentional; existing callers rely on it.
      //   - maxPending (single-flight): counts waiting rows PLUS live-lock
      //     active rows (lock_until > now()). An expired-lock active belongs
      //     to a dead/blocked worker and must NOT suppress dispatch — the
      //     fresh waiting row keeps feeding the waitingClaimable>0 wedge
      //     detectors (supervisor watchdog, jobs stats) that a suppressed
      //     queue would otherwise starve. Source scope is EXACT (NULL matches
      //     only NULL-source rows), so a legacy no-source dispatch can never
      //     coalesce into an arbitrary per-source row.
      //
      // Correctness: two concurrent submitters could both see count < cap and
      // both insert, violating the cap. `pg_advisory_xact_lock` keyed on
      // (name, queue, source) serializes concurrent count+insert decisions
      // for the SAME scope while leaving other scopes fully parallel; both
      // guards share the key namespace so maxWaiting and maxPending
      // submitters for one scope serialize against each other. The lock
      // releases on txn commit/rollback automatically — no cleanup path to
      // leak.
      //
      // Queue scope: the filters include `queue=$2` so a waiting
      // 'autopilot-cycle' in queue 'default' does NOT suppress submissions
      // to queue 'shell' with the same name (pre-D2 cross-queue bleed).
      //
      // Engine compatibility: PGLite (WASM Postgres 17) supports
      // pg_advisory_xact_lock, so this works on both engines without branching.
      if (opts?.maxWaiting !== undefined || opts?.maxPending !== undefined) {
        const backpressureQueue = opts?.queue ?? 'default';
        // Multi-source scope: jobs of the same (name, queue) but different
        // source are independent workstreams (per-source sync/cycle). Counting
        // them together made a waiting default-source sync swallow every other
        // source's freshness sync. Both payload spellings are read: sync/
        // webhook payloads carry camelCase sourceId; per-source autopilot
        // payloads carry snake_case source_id.
        const d = data as Record<string, unknown> | undefined;
        const bpSourceId = typeof d?.sourceId === 'string' ? d.sourceId as string
          : typeof d?.source_id === 'string' ? d.source_id as string
          : null;
        // Lock-census (PR6 D5): COMPLIANT — key already folds the payload source (coalesce($3,'')); the NULL-source wildcard arm is the documented maxWaiting contract above.
        await tx.executeRaw(
          `SELECT pg_advisory_xact_lock(hashtext('minion_maxwaiting:' || $1 || ':' || $2 || ':' || coalesce($3, '')))`,
          [jobName, backpressureQueue, bpSourceId]
        );
        const scopeExact = `COALESCE(data->>'sourceId', data->>'source_id') IS NOT DISTINCT FROM $3`;
        const scopeWildcard = `($3::text IS NULL OR ${scopeExact})`;

        // maxPending first: the stricter, in-flight-aware guard.
        if (opts?.maxPending !== undefined) {
          const maxPending = Math.max(1, Math.floor(opts.maxPending));
          const pendingCond = `(status = 'waiting' OR (status = 'active' AND lock_until > now()))`;
          const pendingCountRows = await tx.executeRaw<{ count: string }>(
            `SELECT count(*)::text AS count
             FROM minion_jobs
             WHERE name = $1 AND queue = $2 AND ${pendingCond}
               AND ${scopeExact}`,
            [jobName, backpressureQueue, bpSourceId]
          );
          const pendingCount = parseInt(pendingCountRows[0]?.count ?? '0', 10);
          if (pendingCount >= maxPending) {
            const existingPending = await tx.executeRaw<Record<string, unknown>>(
              `SELECT *, ${LEGACY_AUTHORITY_COLUMN} FROM minion_jobs
               WHERE name = $1 AND queue = $2 AND ${pendingCond}
                 AND ${scopeExact}
               ORDER BY CASE WHEN status = 'waiting' THEN 0 ELSE 1 END, created_at DESC, id DESC
               LIMIT 1`,
              [jobName, backpressureQueue, bpSourceId]
            );
            if (existingPending.length > 0) {
              return coalesceReturn(tx, existingPending[0], {
                queue: backpressureQueue,
                name: jobName,
                pending_count: pendingCount,
                max_pending: maxPending,
              }, ev => { coalesceAudit = ev; }, authority);
            }
          }
        }

        if (opts?.maxWaiting !== undefined) {
          const maxWaiting = Math.max(1, Math.floor(opts.maxWaiting));
          const waitingCountRows = await tx.executeRaw<{ count: string }>(
            `SELECT count(*)::text AS count
             FROM minion_jobs
             WHERE name = $1 AND queue = $2 AND status = 'waiting'
               AND ${scopeWildcard}`,
            [jobName, backpressureQueue, bpSourceId]
          );
          const waitingCount = parseInt(waitingCountRows[0]?.count ?? '0', 10);
          if (waitingCount >= maxWaiting) {
            const existingWaiting = await tx.executeRaw<Record<string, unknown>>(
              `SELECT *, ${LEGACY_AUTHORITY_COLUMN} FROM minion_jobs
               WHERE name = $1 AND queue = $2 AND status = 'waiting'
                 AND ${scopeWildcard}
               ORDER BY created_at DESC, id DESC
               LIMIT 1`,
              [jobName, backpressureQueue, bpSourceId]
            );
            if (existingWaiting.length > 0) {
              return coalesceReturn(tx, existingWaiting[0], {
                queue: backpressureQueue,
                name: jobName,
                waiting_count: waitingCount,
                max_waiting: maxWaiting,
              }, ev => { coalesceAudit = ev; }, authority);
            }
          }
        }
      }

      // 2. Parent lock + depth/cap validation
      let depth = 0;
      if (opts?.parent_job_id) {
        const parentRows = await tx.executeRaw<Record<string, unknown>>(
          `SELECT * FROM minion_jobs WHERE id = $1 FOR UPDATE`,
          [opts.parent_job_id]
        );
        if (parentRows.length === 0) {
          throw new Error(`parent_job_id ${opts.parent_job_id} not found`);
        }
        const parent = rowToMinionJob(parentRows[0]);
        if (parent.submission_authority && parent.submission_authority.kind !== 'application') throw new Error('Remote jobs cannot have descendants');
        if (!NON_TERMINAL_STATUSES.includes(parent.status)) throw new Error(`parent job ${opts.parent_job_id} is ${parent.status}; a finished job cannot get new children`);

        depth = parent.depth + 1;
        if (depth > maxSpawnDepth) {
          throw new Error(`spawn depth ${depth} exceeds maxSpawnDepth ${maxSpawnDepth}`);
        }

        if (parent.max_children !== null) {
          const countRows = await tx.executeRaw<{ count: string }>(
            `SELECT count(*)::text as count FROM minion_jobs
             WHERE parent_job_id = $1 AND status NOT IN ('completed','failed','dead','cancelled')`,
            [opts.parent_job_id]
          );
          const live = parseInt(countRows[0]?.count ?? '0', 10);
          if (live >= parent.max_children) {
            throw new Error(`parent ${opts.parent_job_id} already has ${live} live children (max_children=${parent.max_children})`);
          }
        }
      }

      // 3. Insert child. Use ON CONFLICT for idempotency; if a concurrent submit
      //    raced past the fast-path SELECT, the unique index catches it here.
      //    quiet_hours + stagger_key always present (null fallback; schema
      //    stores NULL). max_stalled is conditional: provided values get
      //    clamped to [1, 100] and included in the INSERT; omitted values
      //    skip the column so the schema DEFAULT (5 as of v0.14.1) kicks in.
      //    Keeps the app layer from hardcoding the schema default constant.
      //
      //    Footgun note (codex iter 3): threading max_stalled on INSERT only is
      //    deliberate. An idempotency-key hit returns the EXISTING row via the
      //    fast-path SELECT above — we do NOT UPDATE max_stalled on a re-submit,
      //    because letting a second submitter mutate the first submitter's
      //    durability semantics is a nasty surprise.
      const hasMaxStalled = opts?.max_stalled !== undefined && opts.max_stalled !== null;
      const clampedMaxStalled = hasMaxStalled
        ? Math.max(1, Math.min(100, Math.floor(opts!.max_stalled as number)))
        : null;

      const privateQueueLeaseUntil = opts?.private_queue_lease_ms != null
        ? new Date(Date.now() + Math.max(1, Math.floor(opts.private_queue_lease_ms))).toISOString()
        : null;

      const baseCols = `name, queue, status, priority, data, max_attempts, backoff_type,
            backoff_delay, backoff_jitter, delay_until, parent_job_id, on_child_fail,
            depth, max_children, timeout_ms, lock_duration_ms, remove_on_complete, remove_on_fail, idempotency_key,
            quiet_hours, stagger_key, private_queue_owner_job_id, private_queue_owner_token, private_queue_lease_until, submission_authority, spend_authorization`;
      const baseVals = `$1, $2, $3, $4, $5::jsonb, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20::jsonb, $21`;
      const baseValsWithOwner = `${baseVals}, $22, $23, $24, $25::jsonb, $26::text::jsonb`;
      const cols = hasMaxStalled ? `${baseCols}, max_stalled` : baseCols;
      const vals = hasMaxStalled ? `${baseValsWithOwner}, $27` : baseValsWithOwner;

      const insertSql = opts?.idempotency_key
        ? `INSERT INTO minion_jobs (${cols})
           VALUES (${vals})
           ON CONFLICT (idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING
           RETURNING *`
        : `INSERT INTO minion_jobs (${cols})
           VALUES (${vals})
           RETURNING *`;

      const params: unknown[] = [
        jobName,
        opts?.queue ?? 'default',
        childStatus,
        opts?.priority ?? 0,
        withSubmitterLabel(data, authority.kind === 'application'),
        opts?.max_attempts ?? 3,
        opts?.backoff_type ?? 'exponential',
        opts?.backoff_delay ?? 1000,
        opts?.backoff_jitter ?? 0.2,
        delayUntil?.toISOString() ?? null,
        opts?.parent_job_id ?? null,
        opts?.on_child_fail ?? 'fail_parent',
        depth,
        opts?.max_children ?? null,
        // #1737: long handlers (subagent, embed-backfill, autopilot-cycle) get a
        // sane long wall-clock default stamped at submit when the caller didn't
        // pass one, so they aren't killed mid-progress by the short null-default.
        opts?.timeout_ms ?? defaultTimeoutMsFor(jobName),
        // #4145: same three-layer pattern for the lock lease. Explicit input
        // is clamped to [5s,1h]; absent → handler map default; NULL row =
        // worker-global lockDuration at claim. INSERT-only (see the
        // max_stalled footgun note above): an idempotency-key re-submit
        // never mutates the first submitter's lease.
        opts?.lock_duration_ms != null
          ? clampLockDurationMs(opts.lock_duration_ms)
          : defaultLockDurationMsFor(jobName),
        opts?.remove_on_complete ?? false,
        opts?.remove_on_fail ?? false,
        opts?.idempotency_key ?? null,
        opts?.quiet_hours ?? null,
        opts?.stagger_key ?? null,
        opts?.private_queue_owner_job_id ?? null,
        opts?.private_queue_owner_token ?? null,
        privateQueueLeaseUntil,
        authority,
        trusted?.spendAuthorization ? JSON.stringify(trusted.spendAuthorization) : null,
      ];
      if (hasMaxStalled) params.push(clampedMaxStalled);

      // ON CONFLICT DO NOTHING returns 0 rows: a concurrent submit won the
      // race, and its row goes through the same coalesce rule (third path).
      const outcome = await insertOrCoalesce(tx, insertSql, params, opts?.idempotency_key, authority);
      if ('coalesced' in outcome) return outcome.coalesced;

      const child = rowToMinionJob(outcome.inserted);

      // 4. Flip parent to waiting-children if this is a fresh child insert.
      //    Only transition from non-terminal, non-already-waiting-children states.
      if (opts?.parent_job_id) {
        await tx.executeRaw(
          `UPDATE minion_jobs SET status = 'waiting-children', updated_at = now()
           WHERE id = $1 AND status IN ('waiting','active','delayed')`,
          [opts.parent_job_id]
        );
      }

      return child;
    });

    // Deferred audit flush — after commit, advisory lock released, connection
    // returned to the pool. A hung/slow audit volume degrades only this one
    // submission's latency, never the queue.
    if (coalesceAudit) {
      try {
        const { logBackpressureCoalesce } = await import('./backpressure-audit.ts');
        logBackpressureCoalesce(coalesceAudit);
      } catch { /* audit failures never block submission */ }
    }

    return trusted?.spendAuthorization && result.coalesced ? adoptSpendAuthorization(this.engine, result, trusted.spendAuthorization) : result;
  }

  /** Get a job by ID. Returns null if not found. */
  async getJob(id: number): Promise<MinionJob | null> {
    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      'SELECT * FROM minion_jobs WHERE id = $1',
      [id]
    );
    return rows.length > 0 ? rowToMinionJob(rows[0]) : null;
  }

  /** List jobs with optional filters. */
  async getJobs(opts?: {
    status?: MinionJobStatus;
    queue?: string;
    name?: string;
    limit?: number;
    offset?: number;
    /**
     * #4098 — agent-lane ownership fence: restrict to jobs whose
     * `data->>'__owner_client_id'` equals this OAuth client id (the JSONB
     * predicate submit_agent stamps at enqueue). SQL-side WHERE, never a
     * post-fetch JS filter, so a fenced list can't leak foreign rows.
     */
    ownerClientId?: string;
  }): Promise<MinionJob[]> {
    const conditions: string[] = [];
    const params: unknown[] = [];
    let idx = 1;

    if (opts?.status) {
      conditions.push(`status = $${idx++}`);
      params.push(opts.status);
    }
    if (opts?.queue) {
      conditions.push(`queue = $${idx++}`);
      params.push(opts.queue);
    }
    if (opts?.name) {
      conditions.push(`name = $${idx++}`);
      params.push(opts.name);
    }
    if (opts?.ownerClientId) {
      conditions.push(`data->>'__owner_client_id' = $${idx++}`);
      params.push(opts.ownerClientId);
    }

    const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
    const limit = opts?.limit ?? 50;
    const offset = opts?.offset ?? 0;

    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `SELECT * FROM minion_jobs ${where} ORDER BY created_at DESC LIMIT $${idx++} OFFSET $${idx}`,
      [...params, limit, offset]
    );
    return rows.map(rowToMinionJob);
  }

  /** Remove a job. Only terminal statuses can be removed. */
  async removeJob(id: number): Promise<boolean> {
    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `DELETE FROM minion_jobs WHERE id = $1 AND status IN ('completed', 'dead', 'cancelled', 'failed') RETURNING id`,
      [id]
    );
    return rows.length > 0;
  }

  /**
   * Cancel a job and cascade-kill all descendants in one statement.
   *
   * Honest scope: this is BullMQ-style best-effort cancel. The recursive CTE
   * snapshots the parent_job_id chain at statement start. A descendant
   * re-parented BEFORE the cancel call is excluded; one re-parented DURING
   * the call may still get cancelled (cancel wins if seen in the snapshot).
   * Re-parented descendants whose parent_job_id is NULL'd by
   * removeChildDependency naturally fall out of the recursive walk.
   *
   * Active descendants get lock_token = NULL — same path pause uses, so the
   * worker's renewLock will fail next tick and AbortController fires.
   *
   * Returns the *root* (the job matching id), not an arbitrary descendant.
   */
  async cancelJob(id: number, opts?: { ownerClientId?: string; ownerActor?: JobActor }): Promise<MinionJob | null> {
    const cancelled = await this.cancelJobs([id], { rootStatuses: NON_TERMINAL_STATUSES, ...(opts?.ownerClientId ? { ownerClientId: opts.ownerClientId, ownerActor: opts.ownerActor } : {}) });
    const root = cancelled.find(j => j.id === id);
    return root ?? null;
  }

  /**
   * Terminalize every non-terminal job in one private, parent-owned queue.
   *
   * Dream phases drain `dream-inline-*` queues themselves; the shared worker
   * intentionally never claims them.  A phase therefore owns cleanup too:
   * every return/throw/timeout path calls this from `finally`.  The method is
   * idempotent and routes through cancelJobs() so child_done messages,
   * descendant cancellation, active-job aborts, and aggregator unblocking all
   * retain the queue's normal bookkeeping semantics.
   */
  async reconcilePrivateQueue(queueName: string, reason: string): Promise<MinionJob[]> {
    if (!isDreamInlinePrivateQueue(queueName)) {
      throw new Error(`refusing to reconcile non-private queue '${queueName}'`);
    }
    // Enforce the machine-readable family at the single choke point so the
    // call sites (phase finally ×2, supervisor/worker spawn, cycle start,
    // startup recovery) can never drift apart.
    if (!reason.startsWith(PRIVATE_QUEUE_RECONCILE_REASON_PREFIX)) {
      reason = `${PRIVATE_QUEUE_RECONCILE_REASON_PREFIX}: ${reason}`;
    }
    const rows = await this.engine.executeRaw<{ id: number }>(
      `SELECT id FROM minion_jobs
        WHERE queue = $1
          AND status IN ('waiting','active','delayed','waiting-children','paused')
        ORDER BY id`,
      [queueName],
    );
    if (rows.length === 0) return [];
    return this.cancelJobs(rows.map(r => r.id), {
      reason,
      rootStatuses: ['waiting', 'active', 'delayed', 'waiting-children', 'paused'],
    });
  }

  /**
   * Renew the owner lease on still-non-terminal jobs in one private queue.
   * The owner token prevents a stale phase finally/keepalive from extending a
   * successor queue that reused the same queue name in a test or fixture.
   */
  async renewPrivateQueueLease(
    queueName: string,
    ownerToken: string,
    leaseMs = DEFAULT_PRIVATE_QUEUE_LEASE_MS,
  ): Promise<number> {
    if (!isDreamInlinePrivateQueue(queueName)) {
      throw new Error(`refusing to renew non-private queue '${queueName}'`);
    }
    if (ownerToken.length === 0) {
      throw new Error('private queue owner token cannot be empty');
    }
    // GREATEST: a renewal may only EXTEND the lease. A default-horizon (10min)
    // renewal racing a creation-time horizon sized to the phase's wait timeout
    // must never shrink the window a healthy long run still needs — shrinking
    // is what would let startup recovery cancel a LIVE ownerless queue.
    const rows = await this.engine.executeRaw<{ id: number }>(
      `UPDATE minion_jobs
          SET private_queue_lease_until = GREATEST(
                COALESCE(private_queue_lease_until, to_timestamp(0)),
                now() + ($3::text::interval)
              ),
              updated_at = now()
        WHERE queue = $1
          AND private_queue_owner_token = $2
          AND status IN (${NON_TERMINAL_SQL_LIST})
        RETURNING id`,
      [queueName, ownerToken, `${Math.max(1, Math.floor(leaseMs))} milliseconds`],
    );
    return rows.length;
  }

  /**
   * Throttled keepalive closure for a phase that owns a private queue: renews
   * the lease at most once per `throttleMs`, then runs `onRenewed` (the cycle
   * lock refresh rides it). The throttle gates BOTH — 1-5s drain polls cost
   * one UPDATE per half-minute, not per poll — and the order is fixed
   * (renew, then onRenewed) so a lock-refresh failure can't starve the lease.
   */
  makeThrottledLeaseRenewer(
    queueName: string,
    ownerToken: string,
    onRenewed?: () => Promise<void> | void,
    throttleMs = 30_000,
  ): () => Promise<void> {
    let lastRenewalAtMs = 0;
    return async () => {
      const nowMs = Date.now();
      if (nowMs - lastRenewalAtMs < throttleMs) return;
      lastRenewalAtMs = nowMs;
      await this.renewPrivateQueueLease(queueName, ownerToken);
      if (onRenewed) await onRenewed();
    };
  }

  /**
   * Startup/supervisor crash recovery for metadata-backed private dream queues.
   * Cancels only queues that are provably orphaned:
   *   - no live child lock in the private queue;
   *   - explicit owner metadata exists; and
   *   - the owner job is terminal/missing OR the renewable lease has expired.
   *
   * Legacy `dream-inline-*` rows with no owner metadata are left untouched and
   * remain a Doctor/manual-retriage concern.
   */
  async reconcileOrphanedPrivateQueues(opts: {
    reason?: string;
    maxQueues?: number;
  } = {}): Promise<PrivateQueueRecoveryResult> {
    await assertNoUnreviewedJobs(this.engine);
    const result: PrivateQueueRecoveryResult = {
      scanned_queues: 0,
      cancelled_queues: 0,
      cancelled_jobs: 0,
      skipped_live_queues: 0,
      skipped_unowned_queues: 0,
      skipped_non_orphan_queues: 0,
    };
    const maxQueues = Math.max(1, Math.floor(opts.maxQueues ?? 100));
    // HAVING metadata: legacy unowned queues are NEVER recoverable by this
    // lane (doctor/retriage owns them), so they must not occupy the LIMIT
    // window — a backlog of them would otherwise starve every newer orphan
    // out of the scan forever (the incident's exact accumulation shape).
    const queues = await this.engine.executeRaw<{ queue: string }>(
      `SELECT queue
         FROM minion_jobs
        WHERE queue LIKE '${DREAM_INLINE_PRIVATE_QUEUE_PREFIX}%'
          AND status IN (${NON_TERMINAL_SQL_LIST})
        GROUP BY queue
        HAVING bool_or(private_queue_owner_job_id IS NOT NULL OR private_queue_lease_until IS NOT NULL)
        ORDER BY min(created_at), queue
        LIMIT $1`,
      [maxQueues],
    );
    result.scanned_queues = queues.length;
    for (const q of queues) {
      const verdict = await this.classifyPrivateQueueForRecovery(q.queue);
      if (verdict === 'live') {
        result.skipped_live_queues++;
        continue;
      }
      // Defensive only: the scan's HAVING bool_or(owner/lease) excludes
      // metadata-less queues at the SQL level, so this arm is unreachable
      // today — kept in case the scan predicate ever loosens.
      if (verdict === 'unowned') {
        result.skipped_unowned_queues++;
        continue;
      }
      if (verdict === 'not_orphan') {
        result.skipped_non_orphan_queues++;
        continue;
      }
      // Re-verify immediately before cancelling: a child claimed (or a lease
      // renewed) between the first classify and this point flips the verdict
      // to live — the cancel itself must never run on a stale verdict.
      const recheck = await this.classifyPrivateQueueForRecovery(q.queue);
      if (recheck !== 'orphan') {
        result.skipped_non_orphan_queues++;
        continue;
      }
      const cancelled = await this.reconcilePrivateQueue(
        q.queue,
        opts.reason ?? 'startup recovery: orphaned dream-inline private queue',
      );
      if (cancelled.length > 0) {
        result.cancelled_queues++;
        result.cancelled_jobs += cancelled.length;
      }
    }
    return result;
  }

  /**
   * PUBLIC (doctor shares this verdict): the orphaned_private_queue check
   * buckets its candidates through the SAME classifier recovery uses, so the
   * check's advertised remediation and recovery's actual behavior can't drift.
   */
  async classifyPrivateQueueForRecovery(
    queueName: string,
  ): Promise<'orphan' | 'live' | 'unowned' | 'not_orphan'> {
    const rows = await this.engine.executeRaw<{
      active_healthy: string | number;
      owner_ids: unknown;
      metadata_rows: string | number;
      live_owner_rows: string | number;
      nonterminal_owner_rows: string | number;
      max_lease_until: string | null;
      future_lease_rows: string | number;
      recently_touched: string | number;
    }>(
      `WITH q AS (
         SELECT *
           FROM minion_jobs
          WHERE queue = $1
            AND status IN (${NON_TERMINAL_SQL_LIST})
       ),
       owner_ids AS (
         SELECT DISTINCT private_queue_owner_job_id AS id
           FROM q
          WHERE private_queue_owner_job_id IS NOT NULL
       ),
       owners AS (
         SELECT o.id, m.status, m.lock_until
           FROM owner_ids o
           LEFT JOIN minion_jobs m ON m.id = o.id
       )
       SELECT
         count(*) FILTER (WHERE q.status = 'active' AND q.lock_until > now()) AS active_healthy,
         COALESCE(jsonb_agg(DISTINCT q.private_queue_owner_job_id) FILTER (WHERE q.private_queue_owner_job_id IS NOT NULL), '[]'::jsonb) AS owner_ids,
         count(*) FILTER (WHERE q.private_queue_owner_job_id IS NOT NULL OR q.private_queue_lease_until IS NOT NULL) AS metadata_rows,
         (SELECT count(*) FROM owners WHERE status = 'active' AND lock_until > now()) AS live_owner_rows,
         (SELECT count(*) FROM owners WHERE status IN ('waiting','active','delayed','waiting-children','paused')) AS nonterminal_owner_rows,
         max(q.private_queue_lease_until)::text AS max_lease_until,
         count(*) FILTER (WHERE q.private_queue_lease_until > now()) AS future_lease_rows,
         count(*) FILTER (WHERE q.updated_at > now() - interval '120 seconds') AS recently_touched
       FROM q`,
      [queueName],
    );
    const r = rows[0];
    // Defensive only: the FROM q aggregate always yields exactly one row.
    if (!r) return 'not_orphan';
    if (Number(r.active_healthy ?? 0) > 0 || Number(r.live_owner_rows ?? 0) > 0) {
      return 'live';
    }
    if (Number(r.metadata_rows ?? 0) === 0) return 'unowned';
    // Freshness guard BEFORE the owner-terminal fast path: any row touched in
    // the last 2 minutes (a 30s-cadence lease renewal, a claim, a child_done)
    // means SOMETHING is actively working this queue — even when the owner
    // job row reads terminal (stall-swept owner whose drain loop survived a
    // host suspend, the laptop-sleep shape). Never cancel under a live toucher;
    // a genuinely crashed queue goes untouched and classifies orphan on the
    // next pass ≤2 minutes later.
    if (Number(r.recently_touched ?? 0) > 0) return 'live';
    const ownerIds = Array.isArray(r.owner_ids)
      ? r.owner_ids
      : (typeof r.owner_ids === 'string' ? JSON.parse(r.owner_ids) : []);
    const ownerTerminal = ownerIds.length > 0 && Number(r.nonterminal_owner_rows ?? 0) === 0;
    if (ownerTerminal) return 'orphan';
    if (Number(r.future_lease_rows ?? 0) > 0) return 'live';
    const leaseExpired = r.max_lease_until !== null && new Date(r.max_lease_until).getTime() <= Date.now();
    return leaseExpired ? 'orphan' : 'not_orphan';
  }

  /**
   * Batch variant of cancelJob: cancels every root id AND its descendants in
   * ONE transaction, with the full bookkeeping the single-id path carries
   * (child_done inbox messages + aggregator-parent resolution). Callers that
   * cancel in bulk (the waiting-TTL sweep) MUST use this — a raw set-based
   * UPDATE would skip that bookkeeping and wedge parents in waiting-children
   * forever (the exact wedge class this wave fights).
   *
   * `opts.reason` is written to error_text (COALESCE-preserved when a row
   * already carries one). The single-id UPDATE never wrote error_text, so
   * every surface keyed on a reason prefix (jobs stats, doctor) would
   * silently report zero without this parameter.
   */
  async cancelJobs(ids: number[], opts?: { reason?: string; rootStatuses?: MinionJobStatus[]; ownerClientId?: string; ownerActor?: JobActor }): Promise<MinionJob[]> {
    if (ids.length === 0) return [];
    // opts.rootStatuses re-checks each ROOT id's status ATOMICALLY inside the
    // cancel UPDATE's CTE seed. The waiting-TTL sweep passes ['waiting'] to
    // close its SELECT→cancel race: claim() and the sweep both target the
    // oldest waiting rows, so without this a job claimed between the sweep's
    // SELECT and this UPDATE would be cancelled while ACTIVE (lock_token
    // NULLed under the running handler). Operator cancels omit it — killing
    // an active job is exactly what `jobs cancel` means.
    //
    // opts.ownerClientId (#4098) fences the CTE SEED on
    // `data->>'__owner_client_id'`: an agent-scoped caller can cancel only
    // roots it owns. Descendants of an owned root cascade regardless of their
    // own data payload — the recursion follows the owned root, which is the
    // semantic the delegating agent expects (its job tree, not per-row tags).
    const rootStatuses = opts?.rootStatuses ?? null;
    const ownerClientId = opts?.ownerClientId ?? null;
    return this.engine.transaction(async (tx) => {
      const rows = await tx.executeRaw<Record<string, unknown>>(
        `WITH RECURSIVE descendants AS (
          SELECT id, 0 AS d FROM minion_jobs
           WHERE id = ANY($1::int[])
             AND ($3::text[] IS NULL OR status = ANY($3::text[]))
             AND ($4::text IS NULL OR (data->>'__owner_client_id' = $4::text AND ${jobActorSql(5)}))
          UNION ALL
          SELECT m.id, descendants.d + 1
            FROM minion_jobs m
            JOIN descendants ON m.parent_job_id = descendants.id
            WHERE descendants.d < 100
        )
        UPDATE minion_jobs SET
          status = 'cancelled',
          lock_token = NULL,
          lock_until = NULL,
          -- Reason stamps ROOT ids only: a descendant is bookkeeping-cancelled
          -- because its parent went away, not because IT hit the caller's
          -- reason (e.g. a waiting-TTL child would otherwise carry a factually
          -- false 'waited > Nh' text AND inflate the LIKE-prefix stats the
          -- alerting surfaces count).
          error_text = CASE WHEN id = ANY($1::int[]) THEN COALESCE($2, error_text) ELSE error_text END,
          finished_at = now(),
          updated_at = now()
         WHERE id IN (SELECT id FROM descendants)
           AND status IN ('waiting','active','delayed','waiting-children','paused')
         RETURNING *`,
        [ids, opts?.reason ?? null, rootStatuses, ownerClientId, ...(opts?.ownerActor ?? NO_JOB_ACTOR)]
      );
      if (rows.length === 0) return [];

      // v0.15: emit child_done(outcome='cancelled') for every cancelled row
      // that had a parent. Without this, an aggregator waiting for N
      // child_done messages hangs forever when a child is cancelled (codex
      // iteration 3). Also unblock any aggregator parents whose last
      // non-terminal child we just cancelled.
      const parentIds = new Set<number>();
      for (const r of rows) {
        const childId = r.id as number;
        const parentJobId = r.parent_job_id as number | null;
        const name = r.name as string;
        // Skip the root if it's the caller's cancel target AND has no parent.
        // Descendants whose parent got cancelled in the same sweep still
        // benefit from the inbox message — their parent exits waiting-children
        // via the resolve sweep below even though the parent is itself
        // cancelled (EXISTS guard on inbox INSERT handles it).
        if (parentJobId == null) continue;
        parentIds.add(parentJobId);
        const childDone: ChildDoneMessage = {
          type: 'child_done',
          child_id: childId,
          job_name: name,
          result: null,
          outcome: 'cancelled',
          error: 'cancelled',
        };
        await tx.executeRaw(
          `INSERT INTO minion_inbox (job_id, sender, payload)
           SELECT $1, 'minions', $2::jsonb
           WHERE EXISTS (
             SELECT 1 FROM minion_jobs
             WHERE id = $1 AND status NOT IN ('completed','failed','dead','cancelled')
           )`,
          [parentJobId, childDone]
        );
      }

      // Resolve any non-cancelled aggregator parents sitting on
      // waiting-children whose last open child we just cancelled.
      for (const parentId of parentIds) {
        await tx.executeRaw(
          `UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
           WHERE id = $1 AND status = 'waiting-children'
             AND NOT EXISTS (
               SELECT 1 FROM minion_jobs
               WHERE parent_job_id = $1
                 AND status NOT IN ('completed', 'failed', 'dead', 'cancelled')
             )`,
          [parentId]
        );
      }

      return rows.map(rowToMinionJob);
    });
  }

  /**
   * Waiting-TTL sweep (admission control, run from the worker's maintenance
   * interval): cancel jobs still WAITING past their per-name TTL, via
   * cancelJobs() so descendants cancel and aggregator parents resolve.
   *
   * `maxPerTick` bounds one tick's work (default 500) so the first
   * post-upgrade tick against a large backlog can't stall the maintenance
   * loop — the backlog drains over a few ticks. Oldest-first so FIFO
   * fairness of what REMAINS is preserved.
   *
   * NOTE (warn-before-act, user requirement D1A): the worker gates this
   * sweep behind the one-time `minions.ttl_notice_shown` flag — tick 1
   * counts + warns, sweeping starts on tick 2. The gate lives in worker.ts
   * (the channel where the notice prints); this method just sweeps.
   */
  async handleWaitingTTL(opts?: { maxPerTick?: number }): Promise<{ cancelled: number; by_name: Record<string, number> }> {
    await assertNoUnreviewedJobs(this.engine);
    const maxPerTick = Math.max(1, Math.floor(opts?.maxPerTick ?? 500));
    const ttlNames = await resolveTtlNames(this.engine);
    const by_name: Record<string, number> = {};
    let cancelled = 0;
    for (const [name, hours] of ttlNames) {
      if (cancelled >= maxPerTick) break;
      const budget = maxPerTick - cancelled;
      // Keyed on updated_at (last state transition), NOT created_at: every
      // path that RETURNS a row to 'waiting' — handleStalled's requeue (sweep
      // #1 of the SAME maintenance tick), retryJob, promoteDelayed, the
      // parent-unblock flips — bumps updated_at but not created_at. Keying on
      // created_at would cancel a job the same tick just requeued it, with a
      // "waited > Nh" reason that is factually false. For rows that sat
      // untouched in 'waiting' the two timestamps are equivalent, so the
      // backlog-drain semantics are unchanged.
      const stale = await this.engine.executeRaw<{ id: number }>(
        `SELECT id FROM minion_jobs
          WHERE name = $1 AND status = 'waiting'
            AND updated_at < now() - ($2 * interval '1 hour')
          ORDER BY updated_at ASC
          LIMIT $3`,
        [name, hours, budget]
      );
      if (stale.length === 0) continue;
      const reason =
        `${TTL_REASON_PREFIX}: waited > ${hours}h in queue ` +
        `(minions.ttl_waiting_hours.${name}; set 0 to disable)`;
      // rootStatuses:['waiting'] re-checks atomically inside the cancel — a
      // job CLAIMED between the SELECT above and this UPDATE must not be
      // cancelled mid-run (both the claimer and this sweep target the oldest
      // waiting rows, so the race is systematic, not incidental).
      const swept = await this.cancelJobs(stale.map(r => r.id), { reason, rootStatuses: ['waiting'] });
      // Count only the requested roots — descendants of a swept parent are
      // bookkeeping, not TTL victims of their own.
      const rootIds = new Set(stale.map(r => r.id));
      const rootCount = swept.filter(j => rootIds.has(j.id)).length;
      by_name[name] = (by_name[name] ?? 0) + rootCount;
      cancelled += rootCount;
    }
    return { cancelled, by_name };
  }

  /**
   * Re-queue a failed or dead job for retry.
   *
   * #2783: an explicit `jobs retry` is an operator asserting "run this
   * fresh" — so it clears `started_at` (re-stamped on re-claim via
   * `claim()`'s `COALESCE(started_at, now())`, `queue.ts:620`) and resets
   * `attempts_made`/`attempts_started` to 0. Without this, `started_at`
   * kept the ORIGINAL first-claim time, so `handleWallClockTimeouts()`
   * (anchored on `now() - started_at`, `queue.ts:729-749`) could measure
   * from long before the retry — a retry issued more than `timeout_ms * 2`
   * after the original claim was dead-lettered again in under a second,
   * with `attempts_made` already past `max_attempts`. This made retry
   * useless for exactly the case it exists for: recovering work after an
   * outage that outlasted the job's timeout.
   *
   * Also resets `stalled_counter` (Codex review): `handleStalled()`
   * dead-letters once `stalled_counter + 1 >= max_stalled` (`queue.ts:1190`).
   * A job dead-lettered BY stall exhaustion, left un-reset, would hit that
   * same threshold on its very first lock expiry after retry — a job
   * killed by 3 stalls doesn't get a fresh stall budget, contradicting
   * "run this fresh" the same way the unreset attempt counters did.
   */
  async retryJob(id: number): Promise<MinionJob | null> {
    return this.engine.transaction(async tx => {
      const [prior] = await tx.executeRaw<Record<string, unknown>>('SELECT * FROM minion_jobs WHERE id = $1', [id]);
      if (!prior) return null;
      await authorizeJobExecution(tx, rowToMinionJob(prior));
      if (!await admitDelegatedRetry(tx, id)) return null;
      const rows = await tx.executeRaw<Record<string, unknown>>(
        `UPDATE minion_jobs SET status = 'waiting', error_text = NULL,
          lock_token = NULL, lock_until = NULL, delay_until = NULL,
          finished_at = NULL, started_at = NULL, attempts_made = 0,
          attempts_started = 0, stalled_counter = 0, updated_at = now()
         WHERE id = $1 AND status IN ('failed', 'dead')
         RETURNING *`,
        [id]
      );
      return rows.length > 0 ? rowToMinionJob(rows[0]) : null;
    });
  }

  /** Prune old jobs in terminal statuses. Returns count of deleted rows. */
  async prune(opts?: { olderThan?: Date; status?: MinionJobStatus[]; dryRun?: boolean }): Promise<number> {
    const statuses = opts?.status ?? ['completed', 'dead', 'cancelled'];
    const olderThan = opts?.olderThan ?? new Date(Date.now() - 30 * 86400000);

    // #2712: dryRun counts the would-be-pruned rows without deleting.
    // Silent-ignoring a safety flag on a delete path is data loss.
    if (opts?.dryRun) {
      const rows = await this.engine.executeRaw<{ count: string }>(
        `SELECT count(*)::text as count FROM minion_jobs
         WHERE status = ANY($1) AND updated_at < $2`,
        [statuses, olderThan.toISOString()]
      );
      return parseInt(rows[0]?.count ?? '0', 10);
    }

    // A completed dream synthesis child is the record that its transcript was
    // synthesized. Archive the key before the row goes, or the next cycle
    // pays to synthesize the transcript again.
    const rows = await this.engine.executeRaw<{ count: string }>(
      `WITH pruned AS (
         DELETE FROM minion_jobs
         WHERE status = ANY($1) AND updated_at < $2
         RETURNING name, status, data, idempotency_key, finished_at
       ), archived AS (
         INSERT INTO dream_synthesis_completions (source_id, idempotency_key, completed_at)
         SELECT COALESCE(NULLIF(data->>'source_id', ''), 'default'), idempotency_key, COALESCE(finished_at, now())
           FROM pruned
          WHERE name = 'subagent' AND status = 'completed' AND idempotency_key LIKE 'dream:synth%'
         ON CONFLICT DO NOTHING
       )
       SELECT count(*)::text as count FROM pruned`,
      [statuses, olderThan.toISOString()]
    );
    return parseInt(rows[0]?.count ?? '0', 10);
  }

  /** Get job statistics. */
  async getStats(opts?: { since?: Date; queue?: string }): Promise<{
    by_status: Record<string, number>;
    /**
     * Per-type window stats. `total` counts rows CREATED in the window
     * (intake). The `drained_*` fields count rows that reached a terminal
     * status IN the window (`finished_at >= since`) regardless of when they
     * were created — the true outflow. They are split by terminal status
     * because a naive combined "drain" number self-inflates on TTL/manual
     * cancellations while zero useful work happens; divergence alerting
     * compares intake against drained_completed. `waiting_now` and
     * `oldest_waiting_minutes` are point-in-time (not windowed).
     */
    by_type: Array<{ name: string; total: number; completed: number; failed: number; dead: number; avg_duration_ms: number | null;
      drained_completed: number; drained_failed: number; drained_dead: number; drained_cancelled: number;
      waiting_now: number; oldest_waiting_minutes: number | null }>;
    queue_health: { waiting: number; active: number; stalled: number };
    by_queue: QueueFamilyStats[];
    /**
     * issue #1801 — QUEUE-SCOPED wedge signature for the `jobs stats` WEDGED
     * line. by_status/by_type/queue_health above stay GLOBAL (dashboard
     * overview); this block is scoped to one queue (default 'default') because
     * a wedge is per-queue — a healthy worker on one queue must not mask a
     * wedged one (Codex #14/#15). `active_healthy` counts only live-lock active
     * rows so an expired-lock row (worker died mid-job) does NOT mask the wedge.
     */
    wedge: {
      queue: string;
      active_healthy: number;
      waiting: number;
      last_completed_at: string | null;
      minutes_since_completion: number | null;
    };
  }> {
    const since = opts?.since ?? new Date(Date.now() - 86400000);
    const wedgeQueue = opts?.queue ?? 'default';

    // Status counts
    const statusRows = await this.engine.executeRaw<{ status: string; count: string }>(
      `SELECT status, count(*)::text as count FROM minion_jobs GROUP BY status`
    );
    const by_status: Record<string, number> = {};
    for (const r of statusRows) by_status[r.status] = parseInt(r.count, 10);

    // Type breakdown (within time window)
    const typeRows = await this.engine.executeRaw<Record<string, unknown>>(
      `SELECT name,
        count(*)::text as total,
        count(*) FILTER (WHERE status = 'completed')::text as completed,
        count(*) FILTER (WHERE status = 'failed')::text as failed,
        count(*) FILTER (WHERE status = 'dead')::text as dead,
        avg(EXTRACT(EPOCH FROM (finished_at - started_at)) * 1000) FILTER (WHERE finished_at IS NOT NULL AND started_at IS NOT NULL) as avg_duration_ms
       FROM minion_jobs WHERE created_at >= $1
       GROUP BY name ORDER BY total DESC`,
      [since.toISOString()]
    );
    // True per-type outflow: rows that reached a terminal status IN the
    // window, keyed on finished_at (a row created last week and finished
    // today drained today). Split by status — cancellations (incl. the
    // waiting-TTL sweep) are outflow but not useful work.
    const drainRows = await this.engine.executeRaw<Record<string, unknown>>(
      `SELECT name,
        count(*) FILTER (WHERE status = 'completed')::text AS d_completed,
        count(*) FILTER (WHERE status = 'failed')::text AS d_failed,
        count(*) FILTER (WHERE status = 'dead')::text AS d_dead,
        count(*) FILTER (WHERE status = 'cancelled')::text AS d_cancelled
       FROM minion_jobs
       WHERE finished_at IS NOT NULL AND finished_at >= $1
         AND status IN ('completed','failed','dead','cancelled')
       GROUP BY name`,
      [since.toISOString()]
    );
    const drainByName = new Map(drainRows.map(r => [r.name as string, r]));

    // Point-in-time per-type waiting depth + oldest wait age.
    const depthRows = await this.engine.executeRaw<Record<string, unknown>>(
      `SELECT name,
        count(*)::text AS waiting_now,
        EXTRACT(EPOCH FROM (now() - min(created_at)))::text AS oldest_waiting_seconds
       FROM minion_jobs WHERE status = 'waiting'
       GROUP BY name`,
      []
    );
    const depthByName = new Map(depthRows.map(r => [r.name as string, r]));

    // Union of names so a type with waiting rows but zero window intake (or
    // vice versa) still gets a row — divergence alerting needs both sides.
    const typeNames = new Set<string>([
      ...typeRows.map(r => r.name as string),
      ...drainRows.map(r => r.name as string),
      ...depthRows.map(r => r.name as string),
    ]);
    const typeByName = new Map(typeRows.map(r => [r.name as string, r]));
    const by_type = [...typeNames].map(name => {
      const r = typeByName.get(name);
      const d = drainByName.get(name);
      const w = depthByName.get(name);
      const oldest = w?.oldest_waiting_seconds != null ? Number(w.oldest_waiting_seconds) : null;
      return {
        name,
        total: r ? parseInt(r.total as string, 10) : 0,
        completed: r ? parseInt(r.completed as string, 10) : 0,
        failed: r ? parseInt(r.failed as string, 10) : 0,
        dead: r ? parseInt(r.dead as string, 10) : 0,
        avg_duration_ms: r?.avg_duration_ms != null ? Math.round(r.avg_duration_ms as number) : null,
        drained_completed: d ? parseInt(d.d_completed as string, 10) : 0,
        drained_failed: d ? parseInt(d.d_failed as string, 10) : 0,
        drained_dead: d ? parseInt(d.d_dead as string, 10) : 0,
        drained_cancelled: d ? parseInt(d.d_cancelled as string, 10) : 0,
        waiting_now: w ? parseInt(w.waiting_now as string, 10) : 0,
        oldest_waiting_minutes: oldest != null && Number.isFinite(oldest) ? Math.round(oldest / 60) : null,
      };
    }).sort((a, b) => b.total - a.total);

    // Queue health: stalled = active with expired lock
    const stalledRows = await this.engine.executeRaw<{ count: string }>(
      `SELECT count(*)::text as count FROM minion_jobs WHERE status = 'active' AND lock_until < now()`
    );
    const stalled = parseInt(stalledRows[0]?.count ?? '0', 10);

    // issue #1801 — queue-scoped wedge signature (one query, one queue).
    const wedgeRows = await this.engine.executeRaw<{
      active_healthy: string;
      waiting: string;
      last_completed: string | null;
    }>(
      `SELECT
         count(*) FILTER (WHERE status = 'active' AND lock_until > now())::text AS active_healthy,
         count(*) FILTER (WHERE status = 'waiting')::text AS waiting,
         max(updated_at) FILTER (WHERE status = 'completed')::text AS last_completed
       FROM minion_jobs
       WHERE queue = $1`,
      [wedgeQueue],
    );
    const wr = wedgeRows[0] ?? { active_healthy: '0', waiting: '0', last_completed: null };
    const wedgeLastCompleted = wr.last_completed ? new Date(wr.last_completed) : null;

    return {
      by_status,
      by_type,
      queue_health: {
        waiting: by_status['waiting'] ?? 0,
        active: by_status['active'] ?? 0,
        stalled,
      },
      by_queue: await (await import('./queue-families.ts')).queueFamilyStats(this.engine, since),
      wedge: {
        queue: wedgeQueue,
        active_healthy: parseInt(wr.active_healthy ?? '0', 10),
        waiting: parseInt(wr.waiting ?? '0', 10),
        last_completed_at: wr.last_completed,
        minutes_since_completion: wedgeLastCompleted
          ? Math.round((Date.now() - wedgeLastCompleted.getTime()) / 60_000)
          : null,
      },
    };
  }

  /**
   * Claim the next waiting job for a worker. Token-fenced, filters by registered names.
   *
   * Sets timeout_at = now() + timeout_ms when the job has a per-job deadline,
   * so handleTimeouts() can dead-letter expired jobs without rereading timeout_ms.
   *
   * Claim-time budget fallback: rows inserted before the submit-time stamping
   * (or by any writer that bypasses add()) carry timeout_ms = NULL and used to
   * fall through to the minutes-scale null-default wall-clock sweep — a 30-min
   * handler died at ~5 min purely because of WHEN its row was inserted. The
   * COALESCE below resolves HANDLER_DEFAULT_TIMEOUT_MS at claim as the durable
   * invariant (the v128 migration is the one-shot repair for rows already in
   * flight). Names outside the map stay NULL — exactly today's behavior.
   * Postgres evaluates SET expressions against the OLD row, so the timeout_at
   * CASE must repeat the COALESCE rather than reference the assigned column.
   * The map binds as a RAW object (never JSON.stringify into ::jsonb — the
   * postgres.js double-encode trap; PGLite hides it, real PG does not).
   */
  /** `excludeWatchdogSuspects`: a worker with jobs in flight never claims a job marked by a shared watchdog drain (watchdog-attribution.ts).
   *  `suspectWaitMs` (GST-2529): once a marked job this worker could claim has waited that long since its requeue, a worker with
   *  jobs in flight claims nothing, so it drains, and one with nothing in flight takes that job first, so it runs alone and soon.
   *  `excludeNames`: skip these job names (the worker's long-job lane reservation, lanes.ts). */
  async claim(lockToken: string, lockDurationMs: number, queue: string, registeredNames: string[], opts: { excludeWatchdogSuspects?: boolean; suspectWaitMs?: number; excludeNames?: string[] } = {}): Promise<MinionJob | null> {
    if (registeredNames.length === 0) return null;
    await assertNoUnreviewedJobs(this.engine);

    // Direct (session-mode) pool: claim opens the lock that renewLock then
    // heartbeats. Both must live on a connection the transaction-mode pooler
    // won't recycle mid-hold, or the lock orphans and the worker wedges.
    //
    // #4145: lock_duration_ms resolves row → handler map ($6, RAW object —
    // same double-encode rule as $5) → worker default ($2), is STAMPED onto
    // the row (durable, like timeout_ms), and lock_until derives from the
    // same COALESCE (OLD-row semantics: repeat the expression, don't
    // reference the assigned column). Both the stamp and lock_until are
    // CASE-clamped to the [5s,1h] bound IN SQL (row/map resolution only —
    // the worker-default fallback $2 is operator-configured, not row data,
    // and tests/short-lived workers legitimately use sub-5s leases): the exposed
    // submit surfaces clamp already, but a bypass-written row (direct SQL
    // repair, foreign tooling) must not grant a ~24-day lease to a worker
    // that crashes before its first renewal (or a 1ms one that thrashes).
    const excludeNames = opts.excludeNames?.length ? opts.excludeNames : null;
    const params: unknown[] = [lockToken, lockDurationMs, queue, registeredNames, HANDLER_DEFAULT_TIMEOUT_MS, HANDLER_DEFAULT_LOCK_DURATION_MS, ...legacyDefaultClaimParams()];
    const param = (value: unknown) => `$${params.push(value)}`;
    const mark = opts.excludeWatchdogSuspects || opts.suspectWaitMs !== undefined ? param(WATCHDOG_SUSPECT_MARK) : '';
    const overdue = opts.suspectWaitMs === undefined ? null : overdueSuspectSql(mark, param(opts.suspectWaitMs));
    const suspects = !opts.excludeWatchdogSuspects ? '' : ` AND ${unmarkedJobSql(mark)}${overdue ? ` AND NOT EXISTS (SELECT 1 FROM minion_jobs s WHERE s.queue = $3
           AND s.status = 'waiting' AND s.submission_authority IS NOT NULL AND s.name = ANY($4) AND ${overdue})` : ''}`;
    const names = excludeNames ? ` AND NOT (name = ANY(${param(excludeNames)}))` : '';
    const rows = await this.engine.executeRawDirect<Record<string, unknown>>(
      `UPDATE minion_jobs SET
        status = 'active',
        claim_generation = claim_generation + 1,
        ${legacyDefaultClaimSetSql('$7', '$8')},
        lock_token = $1,
        lock_until = now() + ((CASE WHEN COALESCE(lock_duration_ms, ($6::jsonb ->> name)::int) IS NULL THEN $2
                                    ELSE LEAST(GREATEST(COALESCE(lock_duration_ms, ($6::jsonb ->> name)::int), 5000), 3600000) END)::double precision * interval '1 millisecond'),
        lock_duration_ms = CASE WHEN COALESCE(lock_duration_ms, ($6::jsonb ->> name)::int) IS NULL THEN NULL
                                ELSE LEAST(GREATEST(COALESCE(lock_duration_ms, ($6::jsonb ->> name)::int), 5000), 3600000) END,
        timeout_ms = COALESCE(timeout_ms, ($5::jsonb ->> name)::int),
        timeout_at = CASE WHEN COALESCE(timeout_ms, ($5::jsonb ->> name)::int) IS NOT NULL
                          THEN now() + (COALESCE(timeout_ms, ($5::jsonb ->> name)::int)::double precision * interval '1 millisecond')
                          ELSE NULL END,
        attempts_started = attempts_started + 1,
        started_at = COALESCE(started_at, now()),
        updated_at = now()
       WHERE id = (
         SELECT id FROM minion_jobs
         WHERE queue = $3 AND status = 'waiting' AND submission_authority IS NOT NULL AND name = ANY($4)${suspects}${names}
         ORDER BY ${overdue && !opts.excludeWatchdogSuspects ? `${overdue} DESC, ` : ''}priority ASC, created_at ASC
         FOR UPDATE SKIP LOCKED
         LIMIT 1
       )
       RETURNING *`,
      params,
    );
    return rows.length > 0 ? rowToMinionJob(rows[0]) : null;
  }

  /** GST-2529: whether a marked job this worker could claim has waited `suspectWaitMs` since its requeue. The worker asks at most every
   *  few seconds and passes `suspectWaitMs` to `claim` only when one has, so a claim with no such job keeps the claim index's order. */
  async hasOverdueWatchdogSuspect(queue: string, registeredNames: string[], suspectWaitMs: number): Promise<boolean> {
    if (registeredNames.length === 0) return false;
    const rows = await this.engine.executeRaw<{ overdue: boolean }>(
      `SELECT EXISTS (SELECT 1 FROM minion_jobs WHERE queue = $1 AND status = 'waiting' AND submission_authority IS NOT NULL AND name = ANY($2)
         AND ${overdueSuspectSql('$3', '$4')}) AS overdue`,
      [queue, registeredNames, WATCHDOG_SUSPECT_MARK, suspectWaitMs],
    );
    return rows[0]?.overdue === true;
  }

  /** A co-runner of a watchdog drain back to waiting, marked, no attempt burned (token-fenced). */
  releaseWatchdogClaim(id: number, lockToken: string, errorText: string): Promise<boolean> { return releaseWatchdogClaim(this.engine, id, lockToken, errorText); }

  /**
   * Dead-letter active jobs whose timeout_at has passed.
   *
   * The lock_until > now() guard is critical: a stalled job (lock_until < now)
   * is being requeued by handleStalled, NOT timed out terminally. Stall →
   * retry, timeout → dead. Order in worker loop: handleStalled() before
   * handleTimeouts() to give stall recovery first crack.
   *
   * Honest scope: 1-tick TOCTOU window remains. A job whose lock_until
   * expires between handleStalled and handleTimeouts may miss this tick
   * but will be caught the next one (after re-claim). Never double-handled.
   */
  async handleTimeouts(): Promise<MinionJob[]> {
    await assertNoUnreviewedJobs(this.engine);
    return this.engine.transaction(async (tx) => {
      // #1737: count the timed-out run as a spent attempt (terminal, no retry).
      // Safe against double-count: the worker sweep runs handleStalled ->
      // handleTimeouts -> handleWallClockTimeouts sequentially and awaited, and
      // each guards on `status = 'active'`, so the first to set status='dead'
      // excludes the row from the later sweeps.
      //
      // W0 (D5.12): candidates are discovered with a plain read, PARENTS are
      // locked first in sorted order (matching failJob's parent-before-child
      // order), and the child UPDATE re-checks every predicate under a
      // SKIP LOCKED subselect — see killJobs() for the shared tail.
      const candidates = await tx.executeRaw<{ id: number; parent_job_id: number | null }>(
        `SELECT id, parent_job_id FROM minion_jobs
          WHERE status = 'active' AND queue <> '${LEASE_BEACON_QUEUE}'
            AND timeout_at IS NOT NULL
            AND timeout_at < now()
            AND lock_until > now()`
      );
      if (candidates.length === 0) return [];
      await this.lockParentsOrdered(tx, candidates);
      const rows = await tx.executeRaw<Record<string, unknown>>(
        `UPDATE minion_jobs SET
          status = 'dead',
          error_text = 'timeout exceeded',
          attempts_made = attempts_made + 1,
          lock_token = NULL,
          lock_until = NULL,
          finished_at = now(),
          updated_at = now()
         WHERE id IN (
           SELECT id FROM minion_jobs
            WHERE id = ANY($1::bigint[])
              AND status = 'active'
              AND timeout_at IS NOT NULL
              AND timeout_at < now()
              AND lock_until > now()
            FOR UPDATE SKIP LOCKED
         )
         RETURNING *`,
        [candidates.map(c => c.id)]
      );
      await this.killJobs(tx, rows, 'timeout', 'timeout exceeded');
      return rows.map(rowToMinionJob);
    });
  }

  /**
   * W0 fix-wave (Tier-1 #4, D5.12): the ONE parent-notification tail shared
   * by every reaper that terminally kills active jobs. Pre-fix this ~45-line
   * block was hand-copied between handleTimeouts and handleWallClockTimeouts
   * (differing only in the error string), and handleStalled's dead-letter
   * branch had NO copy at all — a child that died via max-stall left its
   * aggregator parent in 'waiting-children' forever (the exact hang the v0.15
   * comment says was fixed for timeouts).
   *
   * Runs inside the caller's transaction, AFTER the child transitions.
   * Callers must have locked the parents first via lockParentsOrdered() —
   * parents-before-children is the queue-wide lock order (failJob locks the
   * parent before touching the child), so the reapers can never deadlock
   * against a concurrent failJob/completeJob.
   *
   * Emits child_done(outcome) to each non-terminal parent's inbox, then flips
   * any 'waiting-children' parent whose last open child we just killed back
   * to 'waiting'.
   */
  private async killJobs(
    tx: Pick<BrainEngine, 'executeRaw'>,
    rows: Array<Record<string, unknown>>,
    outcome: ChildOutcome,
    errorText: string,
  ): Promise<void> {
    const parentIds = new Set<number>();
    for (const r of rows) {
      const parentJobId = r.parent_job_id as number | null;
      if (parentJobId == null) continue;
      parentIds.add(parentJobId);
      const childDone: ChildDoneMessage = {
        type: 'child_done',
        child_id: r.id as number,
        job_name: r.name as string,
        result: null,
        outcome,
        error: errorText,
      };
      await tx.executeRaw(
        `INSERT INTO minion_inbox (job_id, sender, payload)
         SELECT $1, 'minions', $2::jsonb
         WHERE EXISTS (
           SELECT 1 FROM minion_jobs
           WHERE id = $1 AND status NOT IN ('completed','failed','dead','cancelled')
         )`,
        [parentJobId, childDone]
      );
    }

    // Unblock any aggregator parents whose last open child we just killed.
    for (const parentId of [...parentIds].sort((a, b) => a - b)) {
      await tx.executeRaw(
        `UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
         WHERE id = $1 AND status = 'waiting-children'
           AND NOT EXISTS (
             SELECT 1 FROM minion_jobs
             WHERE parent_job_id = $1
               AND status NOT IN ('completed', 'failed', 'dead', 'cancelled')
           )`,
        [parentId]
      );
    }
  }

  /**
   * W0 (D5.12): take parent row locks in ASCENDING id order before any child
   * transition. Matches failJob's parent-first order so the three reapers and
   * failJob can never deadlock each other on parent/child lock acquisition.
   */
  private async lockParentsOrdered(
    tx: Pick<BrainEngine, 'executeRaw'>,
    candidates: Array<{ parent_job_id: number | null }>,
  ): Promise<void> {
    const parentIds = [...new Set(
      candidates.map(c => c.parent_job_id).filter((p): p is number => p != null),
    )].sort((a, b) => a - b);
    if (parentIds.length === 0) return;
    await tx.executeRaw(
      `SELECT id FROM minion_jobs WHERE id = ANY($1::bigint[]) ORDER BY id FOR UPDATE`,
      [parentIds]
    );
  }

  /**
   * Dead-letter active jobs that exceed a wall-clock runtime threshold,
   * regardless of lock state. This catches jobs stuck while still holding
   * DB resources (e.g. blocked on file locks) where stall sweeps skip rows.
   *
   * Threshold (ms):
   *   timeout_ms set   -> timeout_ms * 2
   *   timeout_ms null  -> 2 * lockDurationMs * max_stalled
   */
  async handleWallClockTimeouts(lockDurationMs: number): Promise<MinionJob[]> {
    await assertNoUnreviewedJobs(this.engine);
    return this.engine.transaction(async (tx) => {
      // W0 (D5.12): same parents-first discover/lock/kill shape as
      // handleTimeouts; shared tail in killJobs().
      const candidates = await tx.executeRaw<{ id: number; parent_job_id: number | null }>(
        `SELECT id, parent_job_id FROM minion_jobs
          WHERE status = 'active' AND queue <> '${LEASE_BEACON_QUEUE}'
            AND started_at IS NOT NULL
            AND EXTRACT(EPOCH FROM (now() - started_at)) * 1000 > ${WALL_CLOCK_THRESHOLD_MS_SQL}`,
        [lockDurationMs]
      );
      if (candidates.length === 0) return [];
      await this.lockParentsOrdered(tx, candidates);
      const rows = await tx.executeRaw<Record<string, unknown>>(
        `UPDATE minion_jobs SET
          status = 'dead',
          error_text = 'wall-clock timeout exceeded',
          attempts_made = attempts_made + 1,
          lock_token = NULL,
          lock_until = NULL,
          finished_at = now(),
          updated_at = now()
         WHERE id IN (
           SELECT id FROM minion_jobs
            WHERE id = ANY($2::bigint[])
              AND status = 'active'
              AND started_at IS NOT NULL
              AND EXTRACT(EPOCH FROM (now() - started_at)) * 1000 >
                CASE
                  WHEN timeout_ms IS NOT NULL THEN timeout_ms * 2
                  ELSE COALESCE(lock_duration_ms, $1)::double precision * 2 * GREATEST(max_stalled, 1)
                END
            FOR UPDATE SKIP LOCKED
         )
         RETURNING *`,
        [lockDurationMs, candidates.map(c => c.id)]
      );
      await this.killJobs(tx, rows, 'timeout', 'wall-clock timeout exceeded');
      return rows.map(rowToMinionJob);
    });
  }

  /**
   * Complete a job (token-fenced). All side effects atomic in one transaction:
   *   1. UPDATE child to 'completed' with result
   *   2. Roll up token counts to parent (skipped if parent is terminal)
   *   3. Insert child_done message into parent's inbox (skipped if parent terminal)
   *   4. Resolve parent (flip waiting-children → waiting if all kids done)
   *   5. If remove_on_complete, DELETE the child row (cascades inbox + attachments)
   *
   * Returns the completed job (the in-memory snapshot before any delete), or
   * null if the lock_token mismatched (e.g., reclaimed mid-completion).
   *
   * The fold-in of resolveParent eliminates the crash window where a process
   * died between completeJob and worker's prior post-call resolveParent,
   * stranding the parent in waiting-children forever.
   */
  async completeJob(id: number, lockToken: string, result?: Record<string, unknown>): Promise<MinionJob | null> {
    return retryOnStalePlan(() => this.engine.transaction(async (tx) => {
      // Peek at parent_job_id before the UPDATE so we can lock the parent row
      // FIRST. Without this SELECT FOR UPDATE, two siblings completing
      // concurrently each see the other as still active (pre-commit snapshot
      // under read-committed), neither flips the parent, and the parent is
      // stuck in waiting-children forever.
      const peek = await tx.executeRaw<{ parent_job_id: number | null }>(
        `SELECT parent_job_id FROM minion_jobs WHERE id = $1`,
        [id]
      );
      const parentId = peek[0]?.parent_job_id ?? null;
      if (parentId) {
        await tx.executeRaw(
          `SELECT id FROM minion_jobs WHERE id = $1 FOR UPDATE`,
          [parentId]
        );
      }

      const rows = await tx.executeRaw<Record<string, unknown>>(
        `UPDATE minion_jobs SET status = 'completed', result = $1::jsonb,
          finished_at = now(), lock_token = NULL, lock_until = NULL, updated_at = now()
         WHERE id = $2 AND status = 'active' AND lock_token = $3
         RETURNING *`,
        [result ?? null, id, lockToken]
      );
      if (rows.length === 0) return null;

      const completed = rowToMinionJob(rows[0]);

      if (completed.parent_job_id) {
        // Roll up token counts. Guarded against parent already being terminal.
        if (completed.tokens_input > 0 || completed.tokens_output > 0 || completed.tokens_cache_read > 0) {
          await tx.executeRaw(
            `UPDATE minion_jobs SET
              tokens_input = tokens_input + $1,
              tokens_output = tokens_output + $2,
              tokens_cache_read = tokens_cache_read + $3,
              updated_at = now()
             WHERE id = $4 AND status NOT IN ('completed', 'failed', 'dead', 'cancelled')`,
            [completed.tokens_input, completed.tokens_output, completed.tokens_cache_read, completed.parent_job_id]
          );
        }

        // Auto-post child_done into parent's inbox. EXISTS guard skips if parent
        // was deleted or hit a terminal state mid-flight (no FK violation, no
        // contradiction with the token rollup guard).
        const childDone: ChildDoneMessage = {
          type: 'child_done',
          child_id: completed.id,
          job_name: completed.name,
          result: result ?? null,
          outcome: 'complete',
        };
        await tx.executeRaw(
          `INSERT INTO minion_inbox (job_id, sender, payload)
           SELECT $1, 'minions', $2::jsonb
           WHERE EXISTS (
             SELECT 1 FROM minion_jobs
             WHERE id = $1 AND status NOT IN ('completed','failed','dead','cancelled')
           )`,
          [completed.parent_job_id, childDone]
        );

        // Fold-in resolveParent: flip parent to waiting once all children are
        // in ANY terminal state. Terminal set includes 'failed' so a failed
        // child with on_child_fail='continue'/'ignore' doesn't strand the
        // parent in waiting-children forever (v0.15 aggregator fix).
        await tx.executeRaw(
          `UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
           WHERE id = $1 AND status = 'waiting-children'
             AND NOT EXISTS (
               SELECT 1 FROM minion_jobs
               WHERE parent_job_id = $1
                 AND status NOT IN ('completed', 'failed', 'dead', 'cancelled')
             )`,
          [completed.parent_job_id]
        );
      }

      // remove_on_complete cleanup AFTER all parent-side bookkeeping.
      // The child_done we just inserted lives in the *parent's* inbox row,
      // so it survives the child cascade-delete.
      if (completed.remove_on_complete) {
        await tx.executeRaw(
          `DELETE FROM minion_jobs WHERE id = $1`,
          [completed.id]
        );
      }

      return completed;
    }));
  }

  /**
   * Fail a job (token-fenced). All side effects atomic in one transaction:
   *   1. UPDATE child to 'delayed' (retry) | 'failed' | 'dead'; `result` (a handler result that reported a failure, #7188) is kept
   *   2. If terminal AND parent_job_id, run on_child_fail policy:
   *      - 'fail_parent' → mark parent 'failed' (via failParent SQL)
   *      - 'remove_dep'  → null out parent_job_id (via removeChildDependency SQL)
   *      - 'ignore' / 'continue' → no parent action
   *   3. If remove_on_fail AND terminal, DELETE the child row (parent hook
   *      already ran in this txn using in-memory state, so child deletion is safe)
   *
   * Folding the parent hook into this transaction eliminates the crash window
   * where a process died between failJob and worker's prior post-call hook,
   * leaving the parent stuck in waiting-children.
   */
  async failJob(
    id: number,
    lockToken: string,
    errorText: string,
    newStatus: 'delayed' | 'failed' | 'dead',
    backoffMs?: number,
    result?: Record<string, unknown>,
  ): Promise<MinionJob | null> {
    return retryOnStalePlan(() => this.engine.transaction(async (tx) => {
      // Lock the parent row first so concurrent sibling completions/failures
      // serialize on the parent — same race fix as completeJob.
      const peek = await tx.executeRaw<{ parent_job_id: number | null }>(
        `SELECT parent_job_id FROM minion_jobs WHERE id = $1`,
        [id]
      );
      const parentId = peek[0]?.parent_job_id ?? null;
      if (parentId) {
        await tx.executeRaw(
          `SELECT id FROM minion_jobs WHERE id = $1 FOR UPDATE`,
          [parentId]
        );
      }

      const rows = await tx.executeRaw<Record<string, unknown>>(
        `UPDATE minion_jobs SET
          status = $1, error_text = $2, attempts_made = attempts_made + 1,
          stacktrace = COALESCE(stacktrace, '[]'::jsonb) || to_jsonb($3::text),
          delay_until = CASE WHEN $1 = 'delayed' THEN now() + ($4::double precision * interval '1 millisecond') ELSE NULL END,
          finished_at = CASE WHEN $1 IN ('failed', 'dead') THEN now() ELSE NULL END,
          started_at = CASE WHEN $1 = 'delayed' THEN NULL ELSE started_at END,
          result = COALESCE($7::jsonb, result),
          lock_token = NULL, lock_until = NULL, updated_at = now()
         WHERE id = $5 AND status = 'active' AND lock_token = $6
         RETURNING *`,
        [newStatus, errorText, errorText, backoffMs ?? 0, id, lockToken, result ?? null]
      );
      if (rows.length === 0) return null;

      const failed = rowToMinionJob(rows[0]);
      const terminal = newStatus === 'failed' || newStatus === 'dead';

      // Parent hook on terminal failure.
      if (terminal && failed.parent_job_id) {
        // v0.15: emit child_done(outcome='failed') BEFORE any parent-terminal
        // update. Insertion order matters because `completeJob`'s inbox-write
        // EXISTS guard skips writes once the parent is 'failed' — if we let
        // the fail_parent UPDATE run first, this inbox row would be dropped
        // for aggregator-style parents that still want to count it (codex).
        const childDone: ChildDoneMessage = {
          type: 'child_done',
          child_id: failed.id,
          job_name: failed.name,
          result: null,
          outcome: newStatus === 'dead' ? 'dead' : 'failed',
          error: errorText,
        };
        await tx.executeRaw(
          `INSERT INTO minion_inbox (job_id, sender, payload)
           SELECT $1, 'minions', $2::jsonb
           WHERE EXISTS (
             SELECT 1 FROM minion_jobs
             WHERE id = $1 AND status NOT IN ('completed','failed','dead','cancelled')
           )`,
          [failed.parent_job_id, childDone]
        );

        if (failed.on_child_fail === 'fail_parent') {
          await tx.executeRaw(
            `UPDATE minion_jobs SET status = 'failed', lock_token = NULL, lock_until = NULL,
              error_text = $1, finished_at = now(), updated_at = now()
             WHERE id = $2 AND status = 'waiting-children'`,
            [`child job ${failed.id} failed: ${errorText}`, failed.parent_job_id]
          );
        } else if (failed.on_child_fail === 'remove_dep') {
          await tx.executeRaw(
            `UPDATE minion_jobs SET parent_job_id = NULL, updated_at = now() WHERE id = $1`,
            [failed.id]
          );
          // After dropping the dep, try to resolve the parent if all OTHER
          // kids are terminal. Terminal set includes 'failed' (v0.15).
          await tx.executeRaw(
            `UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
             WHERE id = $1 AND status = 'waiting-children'
               AND NOT EXISTS (
                 SELECT 1 FROM minion_jobs
                 WHERE parent_job_id = $1
                   AND status NOT IN ('completed', 'failed', 'dead', 'cancelled')
               )`,
            [failed.parent_job_id]
          );
        } else {
          // 'ignore' / 'continue': parent stays in waiting-children waiting on
          // siblings. With v0.15 terminal-set expansion + child_done emission
          // above, an aggregator sibling-count model now works: all N children
          // reach terminal → completeJob on a sibling (or the LAST terminal
          // transition here) flips parent → waiting once no non-terminal kids
          // remain. Run the resolve check here so the last child transitioning
          // via THIS code path still unblocks the parent.
          await tx.executeRaw(
            `UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
             WHERE id = $1 AND status = 'waiting-children'
               AND NOT EXISTS (
                 SELECT 1 FROM minion_jobs
                 WHERE parent_job_id = $1
                   AND status NOT IN ('completed', 'failed', 'dead', 'cancelled')
               )`,
            [failed.parent_job_id]
          );
        }
      }

      // remove_on_fail cleanup AFTER parent hook.
      if (terminal && failed.remove_on_fail) {
        await tx.executeRaw(
          `DELETE FROM minion_jobs WHERE id = $1`,
          [failed.id]
        );
      }

      return failed;
    }));
  }

  /**
   * v0.41 Bug 2 — release a job back to `delayed` after a
   * `RateLeaseUnavailableError` bounce, WITHOUT incrementing `attempts_made`.
   *
   * The field-report bug: pre-v0.41, lease-full bounces routed through
   * `failJob` which bumps `attempts_made`. After 3 bounces the job hit
   * `max_attempts` (default 3) and dead-lettered with message
   * `rate lease "anthropic:messages" full (8/8)`. Operators saw a dead
   * job and assumed a real failure.
   *
   * This method is the workhorse fix: status → `delayed`, jittered backoff
   * via `delay_until`, `attempts_made` UNCHANGED. The handler comment at
   * `src/core/minions/handlers/subagent.ts:425` ("treat as renewable
   * error so the worker re-claims") is now actually true.
   *
   * Audit row write to `minion_lease_pressure_log` is the caller's
   * responsibility (the worker has the model/queue context); this method
   * stays focused on the state-machine flip. Same `lock_token + status='active'`
   * idempotency guard as `failJob` so a racing stall sweep / cancel still
   * wins. Returns `null` on lock_token mismatch.
   *
   * Returns the updated `MinionJob` row on success so the caller can stamp
   * the audit row with provenance from the SAME row that just flipped.
   */
  async releaseLeaseFullJob(
    id: number,
    lockToken: string,
    errorText: string,
    backoffMs: number,
  ): Promise<MinionJob | null> {
    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `UPDATE minion_jobs SET
        status = 'delayed',
        error_text = $1,
        stacktrace = COALESCE(stacktrace, '[]'::jsonb) || to_jsonb($1::text),
        delay_until = now() + ($2::double precision * interval '1 millisecond'),
        started_at = NULL,
        lock_token = NULL, lock_until = NULL, updated_at = now()
       WHERE id = $3 AND status = 'active' AND lock_token = $4
       RETURNING *`,
      [errorText, backoffMs, id, lockToken],
    );
    if (rows.length === 0) return null;
    return rowToMinionJob(rows[0]);
  }

  /**
   * Return a claimed job to `delayed` without counting an attempt or growing
   * its stacktrace (a deferral is a state, not a failure; a job that waits for
   * a key for weeks must not accumulate one stack entry per retry).
   */
  async deferJob(id: number, lockToken: string, note: string, delayMs: number): Promise<MinionJob | null> {
    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `UPDATE minion_jobs SET
        status = 'delayed', error_text = $1,
        delay_until = now() + ($2::double precision * interval '1 millisecond'),
        started_at = NULL, timeout_at = NULL,
        lock_token = NULL, lock_until = NULL, updated_at = now()
       WHERE id = $3 AND status = 'active' AND lock_token = $4
       RETURNING *`,
      [note, Math.max(0, delayMs), id, lockToken],
    );
    return rows.length === 0 ? null : rowToMinionJob(rows[0]);
  }

  async releaseConfigurationJob(
    id: number,
    lockToken: string,
    signal: AbortSignal,
  ): Promise<'released' | 'no_op' | 'unconfirmed'> {
    if (signal.aborted) return 'unconfirmed';
    let onAbort: () => void = () => {};
    try {
      const cancelled = new Promise<'unconfirmed'>(resolve => {
        onAbort = () => resolve('unconfirmed');
        signal.addEventListener('abort', onAbort, { once: true });
      });
      const release = this.engine.executeRaw<{ id: number }>(
        `UPDATE minion_jobs SET
          status = 'delayed',
          error_text = 'worker_configuration_blocked',
          delay_until = now() + interval '30 seconds',
          started_at = NULL, timeout_at = NULL,
          lock_token = NULL, lock_until = NULL, updated_at = now()
         WHERE id = $1 AND status = 'active' AND lock_token = $2
         RETURNING id`,
        [id, lockToken],
        { signal },
      ).then(rows => signal.aborted ? 'unconfirmed' as const : rows.length > 0 ? 'released' as const : 'no_op' as const);
      return await Promise.race([release, cancelled]);
    } catch {
      return 'unconfirmed';
    } finally {
      signal.removeEventListener('abort', onAbort);
    }
  }

  /** Update job progress (token-fenced). */
  async updateProgress(id: number, lockToken: string, progress: unknown): Promise<boolean> {
    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `UPDATE minion_jobs SET progress = $1::jsonb, updated_at = now()
       WHERE id = $2 AND status = 'active' AND lock_token = $3
       RETURNING id`,
      [progress, id, lockToken]
    );
    return rows.length > 0;
  }

  /**
   * Renew lock (token-fenced). Returns false if token mismatch (job was reclaimed).
   *
   * `opts.signal` cancels the in-flight UPDATE (postgres.js `.cancel()`) when the
   * caller's timeout race gives up on it — otherwise the abandoned query holds a
   * checked-out pool slot for its full server-side duration (issue #6).
   * Cancellation is BEST-EFFORT (#4145 CDX-2/R2-2): pool acquisition and PG
   * protocol cancel are asynchronous, and PGLite ignores the signal — so
   * correctness never rests on it. A late-landing renewal UPDATE is fenced on
   * OUR token, meaning it can only extend a lock nobody else has claimed;
   * worst case is a stall-requeue delayed by ≤ one lease.
   */
  async renewLock(
    id: number,
    lockToken: string,
    lockDurationMs: number,
    opts?: { signal?: AbortSignal },
  ): Promise<boolean> {
    // Direct (session-mode) pool — see claim(). The heartbeat that keeps a job
    // alive for minutes cannot run on the transaction pooler without periodic
    // CONNECTION_ENDED drops that look like lock-expiry and orphan the job.
    const rows = await this.engine.executeRawDirect<Record<string, unknown>>(
      `UPDATE minion_jobs SET lock_until = now() + ($1::double precision * interval '1 millisecond'), updated_at = now()
       WHERE id = $2 AND lock_token = $3 AND status = 'active'
       RETURNING id`,
      [lockDurationMs, id, lockToken],
      opts
    );
    return rows.length > 0;
  }

  /**
   * issue #1678 — self-healing retry for the Minion hot-path lock SQL.
   * ONLY promoteDelayed routes through this: it's idempotent (re-running the
   * same UPDATE on already-promoted rows is a no-op), so a retry after a
   * reaped pooler socket can't cause double-work. `claim` and `renewLock`
   * deliberately do NOT use this — see their call sites for why (Codex #1/#2):
   * blind-retrying claim can double-claim a job, and retrying renewLock races
   * the renewal-tick's own timeout. The reconnect callback rebuilds the
   * instance pool between attempts when the engine supports it (Postgres);
   * PGLite has no pooler reaping so reconnect is absent and the retry is a
   * cheap pass-through.
   */
  private async lockRetry<T>(fn: () => Promise<T>): Promise<T> {
    const reconnect = (this.engine as { reconnect?: () => Promise<void> }).reconnect;
    const opts = resolveBulkRetryOpts();
    let prevDelay = 0;
    try {
      return await withRetry(fn, {
        maxRetries: opts.maxRetries,
        delayMs: opts.delayMs,
        delayMaxMs: opts.delayMaxMs,
        jitter: BULK_RETRY_OPTS.jitter,
        auditSite: 'minion-lock',
        onRetry: (attempt, err) => {
          const delay = computeNextDelay(attempt - 1, prevDelay, opts.delayMs, opts.delayMaxMs, BULK_RETRY_OPTS.jitter);
          prevDelay = delay;
          auditLogBatchRetry('minion-lock', 1, attempt, delay, err);
        },
        reconnect: reconnect ? () => reconnect.call(this.engine) : undefined,
      });
    } catch (err) {
      if (err instanceof Error && err.name === 'RetryAbortError') throw err;
      if (isRetryableConnError(err)) auditLogBatchExhausted('minion-lock', 1, opts.maxRetries + 1, err);
      throw err;
    }
  }

  /** Promote delayed jobs whose delay_until has passed. Returns promoted jobs. */
  async promoteDelayed(): Promise<MinionJob[]> {
    await assertNoUnreviewedJobs(this.engine);
    const rows = await this.lockRetry(() => this.engine.executeRaw<Record<string, unknown>>(
      `UPDATE minion_jobs SET status = 'waiting', delay_until = NULL,
        started_at = NULL,
        lock_token = NULL, lock_until = NULL, updated_at = now()
       WHERE status = 'delayed' AND delay_until <= now()
       RETURNING *`
    ));
    return rows.map(rowToMinionJob);
  }

  /** Detect and handle stalled jobs. Single CTE, no off-by-one. Returns affected jobs. */
  async handleStalled(graceMsOverride?: number): Promise<{ requeued: MinionJob[]; dead: MinionJob[] }> {
    await assertNoUnreviewedJobs(this.engine);
    // W0 fix-wave (Tier-1 #4): the dead-letter branch previously emitted NO
    // child_done and never unblocked aggregator parents — a child that died
    // via max-stall stranded its parent in 'waiting-children' forever (the
    // exact hang the v0.15 comment says was fixed for timeouts; there was no
    // compensating sweep anywhere). Restructured into the parents-first
    // discover/lock/kill shape (D5.12) with the shared killJobs() tail.
    //
    // #4145 (CDX-7): the reclaim predicate carries a grace — see
    // resolveStallReclaimGraceMs. Callers (tests) may pass an explicit
    // override; the worker sweep resolves from env/default.
    const graceMs = graceMsOverride ?? resolveStallReclaimGraceMs();
    return this.engine.transaction(async (tx) => {
      // GBRA-133: a drain-lease holder record whose lease lock is gone or expired outlived a killed holder; nothing else removes it.
      await tx.executeRaw(`DELETE FROM minion_jobs b WHERE b.queue = '${LEASE_BEACON_QUEUE}' AND b.status = 'active' AND b.lock_until < now()
        AND NOT EXISTS (SELECT 1 FROM gbrain_cycle_locks l WHERE l.acquisition_token::text = b.lock_token AND l.ttl_expires_at > now())`);
      const candidates = await tx.executeRaw<{ id: number; parent_job_id: number | null; stalled_counter: number; max_stalled: number }>(
        `SELECT id, parent_job_id, stalled_counter, max_stalled
           FROM minion_jobs
          WHERE status = 'active' AND queue <> '${LEASE_BEACON_QUEUE}'
            AND lock_until < now() - ($1::double precision * interval '1 millisecond')`,
        [graceMs]
      );
      if (candidates.length === 0) return { requeued: [], dead: [] };
      // GBRA-130 A-30: the claims of a worker that held several (a kernel OOM kill) requeue marked, no stall counted.
      const coRunners = await requeueCoRunnerStalls(tx, candidates, graceMs);
      const ids = coRunners.rest.map(c => c.id);
      // Only the dead-letter branch touches parents; lock just those, sorted.
      await this.lockParentsOrdered(
        tx,
        coRunners.rest.filter(c => Number(c.stalled_counter) + 1 >= Number(c.max_stalled)),
      );

      const requeuedRows = await tx.executeRaw<Record<string, unknown>>(
        `UPDATE minion_jobs SET
          status = 'waiting', stalled_counter = stalled_counter + 1,
          started_at = NULL,
          lock_token = NULL, lock_until = NULL, updated_at = now()
         WHERE id IN (
           SELECT id FROM minion_jobs
            WHERE id = ANY($1::bigint[])
              AND status = 'active'
              AND lock_until < now() - ($2::double precision * interval '1 millisecond')
              AND stalled_counter + 1 < max_stalled
            FOR UPDATE SKIP LOCKED
         )
         RETURNING *`,
        [ids, graceMs]
      );
      const deadRows = await tx.executeRaw<Record<string, unknown>>(
        `UPDATE minion_jobs SET
          status = 'dead', stalled_counter = stalled_counter + 1,
          attempts_made = attempts_made + 1,
          error_text = 'max stalled count exceeded',
          lock_token = NULL, lock_until = NULL, finished_at = now(), updated_at = now()
         WHERE id IN (
           SELECT id FROM minion_jobs
            WHERE id = ANY($1::bigint[])
              AND status = 'active'
              AND lock_until < now() - ($2::double precision * interval '1 millisecond')
              AND stalled_counter + 1 >= max_stalled
            FOR UPDATE SKIP LOCKED
         )
         RETURNING *`,
        [ids, graceMs]
      );
      // THE FIX: stall-death now notifies + unblocks parents like every
      // other terminal kill. Outcome 'dead' (not 'timeout') so consumers can
      // distinguish "died via max-stall" from "timed out during run".
      await this.killJobs(tx, deadRows, 'dead', 'max stalled count exceeded');
      return { requeued: [...coRunners.requeued, ...requeuedRows].map(rowToMinionJob), dead: deadRows.map(rowToMinionJob) };
    }).then(async (result) => {
      // W0 ship-review (data-migration): the per-kill unblock above is
      // forward-only — parents stranded in 'waiting-children' by PRE-upgrade
      // stall-deaths (children already status='dead') are never revisited by
      // any per-event unblock site. This idempotent sweep self-heals ALL
      // stranding classes, retroactive included, once per stall tick: any
      // waiting-children parent with zero non-terminal children flips back to
      // 'waiting'. Cheap (single UPDATE, NOT EXISTS on an indexed FK) at the
      // 30s sweep cadence.
      try {
        await this.engine.executeRaw(
          `UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
            WHERE status = 'waiting-children'
              AND NOT EXISTS (
                SELECT 1 FROM minion_jobs c
                WHERE c.parent_job_id = minion_jobs.id
                  AND c.status NOT IN ('completed', 'failed', 'dead', 'cancelled')
              )`
        );
      } catch { /* best-effort backstop; the per-kill unblock is the primary path */ }
      return result;
    });
  }

  /**
   * Check if all children of a parent are in ANY terminal state. If so,
   * unblock parent (flip waiting-children → waiting).
   *
   * v0.15: terminal set includes 'failed' so a child failing with
   * on_child_fail='continue'/'ignore' doesn't strand the parent.
   */
  async resolveParent(parentId: number): Promise<MinionJob | null> {
    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `UPDATE minion_jobs SET status = 'waiting', started_at = NULL, updated_at = now()
       WHERE id = $1 AND status = 'waiting-children'
         AND NOT EXISTS (
           SELECT 1 FROM minion_jobs
           WHERE parent_job_id = $1
             AND status NOT IN ('completed', 'failed', 'dead', 'cancelled')
         )
       RETURNING *`,
      [parentId]
    );
    return rows.length > 0 ? rowToMinionJob(rows[0]) : null;
  }

  /** Fail the parent when a child fails with fail_parent policy. */
  async failParent(parentId: number, childId: number, errorText: string): Promise<MinionJob | null> {
    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `UPDATE minion_jobs SET status = 'failed', lock_token = NULL, lock_until = NULL,
        error_text = $1, finished_at = now(), updated_at = now()
       WHERE id = $2 AND status = 'waiting-children'
       RETURNING *`,
      [`child job ${childId} failed: ${errorText}`, parentId]
    );
    return rows.length > 0 ? rowToMinionJob(rows[0]) : null;
  }

  /** Pause a waiting or active job. For active jobs, clears the lock so the worker's
   *  AbortController fires and the handler stops gracefully. */
  async pauseJob(id: number): Promise<MinionJob | null> {
    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `UPDATE minion_jobs SET status = 'paused',
        lock_token = NULL, lock_until = NULL, updated_at = now()
       WHERE id = $1 AND status IN ('waiting', 'active', 'delayed')
       RETURNING *`,
      [id]
    );
    return rows.length > 0 ? rowToMinionJob(rows[0]) : null;
  }

  /** Resume a paused job back to waiting. */
  async resumeJob(id: number): Promise<MinionJob | null> {
    const prior = await this.getJob(id);
    if (!prior) return null;
    await authorizeJobExecution(this.engine, prior);
    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `UPDATE minion_jobs SET status = 'waiting',
        lock_token = NULL, lock_until = NULL, updated_at = now()
       WHERE id = $1 AND status = 'paused'
       RETURNING *`,
      [id]
    );
    return rows.length > 0 ? rowToMinionJob(rows[0]) : null;
  }

  /** Send a message to a job's inbox. Sender must be the parent job or 'admin'. */
  async sendMessage(jobId: number, payload: unknown, sender: string): Promise<InboxMessage | null> {
    // Validate job exists and is in a messageable state
    const job = await this.getJob(jobId);
    if (!job) return null;
    if (['completed', 'dead', 'cancelled', 'failed'].includes(job.status)) return null;

    // Sender validation: must be parent job ID or 'admin'
    if (sender !== 'admin' && sender !== String(job.parent_job_id)) {
      return null;
    }

    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `INSERT INTO minion_inbox (job_id, sender, payload)
       VALUES ($1, $2, $3)
       RETURNING *`,
      [jobId, sender, payload]
    );
    return rows.length > 0 ? rowToInboxMessage(rows[0]) : null;
  }

  /** Read unread inbox messages for a job. Token-fenced. Marks messages as read. */
  async readInbox(jobId: number, lockToken: string): Promise<InboxMessage[]> {
    // Verify lock ownership
    const lockCheck = await this.engine.executeRaw<{ id: number }>(
      `SELECT id FROM minion_jobs WHERE id = $1 AND lock_token = $2 AND status = 'active'`,
      [jobId, lockToken]
    );
    if (lockCheck.length === 0) return [];

    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `UPDATE minion_inbox SET read_at = now()
       WHERE job_id = $1 AND read_at IS NULL
       RETURNING *`,
      [jobId]
    );
    return rows.map(rowToInboxMessage);
  }

  /** Update token counts for a job. Accumulates (adds to existing). Token-fenced. */
  async updateTokens(id: number, lockToken: string, tokens: TokenUpdate): Promise<boolean> {
    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `UPDATE minion_jobs SET
        tokens_input = tokens_input + $1,
        tokens_output = tokens_output + $2,
        tokens_cache_read = tokens_cache_read + $3,
        updated_at = now()
       WHERE id = $4 AND status = 'active' AND lock_token = $5
       RETURNING id`,
      [tokens.input ?? 0, tokens.output ?? 0, tokens.cache_read ?? 0, id, lockToken]
    );
    return rows.length > 0;
  }

  /** Replay a completed/failed/dead job with optional data overrides. Creates a new job. */
  async replayJob(id: number, dataOverrides?: Record<string, unknown>): Promise<MinionJob | null> {
    const source = await this.getJob(id);
    if (!source) return null;
    if (!['completed', 'failed', 'dead'].includes(source.status)) return null;
    const authority = await authorizeJobExecution(this.engine, source);
    if (authority.kind !== 'application' && dataOverrides && Object.keys(dataOverrides).length) throw new Error('Remote job replay cannot override accepted data; submit a new job');

    const { data, clientId } = prepareDelegatedReplay(source.name, source.data, dataOverrides);

    return this.add(source.name, data, {
      queue: source.queue,
      priority: source.priority,
      max_attempts: source.max_attempts,
      backoff_type: source.backoff_type,
      backoff_delay: source.backoff_delay,
      backoff_jitter: source.backoff_jitter,
    }, { submissionAuthority: authority, delegatedClientId: clientId,
      allowProtectedSubmit: authority.kind === 'remote_agent' || (authority.kind === 'application' && isProtectedJobName(source.name)) });
  }

  /** Remove a child's dependency on its parent. */
  async removeChildDependency(childId: number): Promise<void> {
    await this.engine.executeRaw(
      `UPDATE minion_jobs SET parent_job_id = NULL, updated_at = now() WHERE id = $1`,
      [childId]
    );
  }

  /**
   * Read child_done messages from a parent's inbox. Token-fenced (the parent
   * job must currently hold lockToken — same fence as readInbox to prevent a
   * stale process polling completions for jobs it no longer owns).
   *
   * Does NOT mark messages read (parent may want to poll repeatedly with a
   * cursor). Use `since` to fetch only newer entries.
   */
  async readChildCompletions(
    parentId: number,
    lockToken: string,
    opts?: { since?: Date }
  ): Promise<ChildDoneMessage[]> {
    // Verify the caller holds the parent's lock.
    const lockCheck = await this.engine.executeRaw<{ id: number }>(
      `SELECT id FROM minion_jobs WHERE id = $1 AND lock_token = $2 AND status = 'active'`,
      [parentId, lockToken]
    );
    if (lockCheck.length === 0) return [];

    const params: unknown[] = [parentId];
    let sinceClause = '';
    if (opts?.since) {
      sinceClause = ` AND sent_at > $2::timestamptz`;
      params.push(opts.since.toISOString());
    }

    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `SELECT payload FROM minion_inbox
       WHERE job_id = $1 AND (payload->>'type') = 'child_done'${sinceClause}
       ORDER BY sent_at ASC`,
      params
    );

    return rows.map(r => {
      const p = typeof r.payload === 'string' ? JSON.parse(r.payload) : r.payload;
      return p as ChildDoneMessage;
    });
  }

  /**
   * Attach a file to a job. Validates size, base64, filename safety, and
   * duplicate filename. Returns the persisted attachment metadata (not the
   * bytes — use getAttachment to fetch).
   *
   * The DB UNIQUE (job_id, filename) constraint is the authoritative duplicate
   * fence; the in-memory check just gives a faster error.
   */
  async addAttachment(jobId: number, input: AttachmentInput): Promise<Attachment> {
    await this.ensureSchema();

    // Verify job exists (FK guarantees this on insert too, but explicit error is clearer)
    const exists = await this.engine.executeRaw<{ id: number }>(
      `SELECT id FROM minion_jobs WHERE id = $1`,
      [jobId]
    );
    if (exists.length === 0) {
      throw new Error(`job ${jobId} not found`);
    }

    const existingRows = await this.engine.executeRaw<{ filename: string }>(
      `SELECT filename FROM minion_attachments WHERE job_id = $1`,
      [jobId]
    );
    const existingFilenames = new Set(existingRows.map(r => r.filename));

    const result = validateAttachment(input, {
      maxBytes: this.maxAttachmentBytes,
      existingFilenames,
    });
    if (!result.ok) {
      throw new Error(`attachment validation failed: ${result.error}`);
    }
    const { filename, content_type, bytes, size_bytes, sha256 } = result.normalized;

    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `INSERT INTO minion_attachments (job_id, filename, content_type, content, size_bytes, sha256)
       VALUES ($1, $2, $3, $4, $5, $6)
       RETURNING id, job_id, filename, content_type, storage_uri, size_bytes, sha256, created_at`,
      [jobId, filename, content_type, bytes, size_bytes, sha256]
    );
    return rowToAttachment(rows[0]);
  }

  /** List attachments for a job (metadata only, no bytes). */
  async listAttachments(jobId: number): Promise<Attachment[]> {
    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `SELECT id, job_id, filename, content_type, storage_uri, size_bytes, sha256, created_at
       FROM minion_attachments
       WHERE job_id = $1
       ORDER BY created_at ASC, id ASC`,
      [jobId]
    );
    return rows.map(rowToAttachment);
  }

  /**
   * Fetch a single attachment with bytes. Returns null if not found.
   * The bytes are returned as a Buffer (Uint8Array under the hood).
   */
  async getAttachment(jobId: number, filename: string): Promise<{ meta: Attachment; bytes: Buffer } | null> {
    const rows = await this.engine.executeRaw<Record<string, unknown>>(
      `SELECT id, job_id, filename, content_type, storage_uri, size_bytes, sha256, created_at, content
       FROM minion_attachments
       WHERE job_id = $1 AND filename = $2`,
      [jobId, filename]
    );
    if (rows.length === 0) return null;
    const row = rows[0];
    const meta = rowToAttachment(row);
    const raw = row.content;
    let bytes: Buffer;
    if (raw == null) {
      bytes = Buffer.alloc(0);
    } else if (Buffer.isBuffer(raw)) {
      bytes = raw;
    } else if (raw instanceof Uint8Array) {
      bytes = Buffer.from(raw);
    } else {
      bytes = Buffer.from(raw as ArrayBuffer);
    }
    return { meta, bytes };
  }

  /** Delete an attachment by job + filename. Returns true if a row was removed. */
  async deleteAttachment(jobId: number, filename: string): Promise<boolean> {
    const rows = await this.engine.executeRaw<{ id: number }>(
      `DELETE FROM minion_attachments WHERE job_id = $1 AND filename = $2 RETURNING id`,
      [jobId, filename]
    );
    return rows.length > 0;
  }
}

/**
 * issue #1801 / CLI→MCP gap-closure wave (CEO-F7) — derive the wedged-queue
 * boolean from getStats().wedge, shared by `gbrain jobs stats` and the
 * get_job_stats op so the two surfaces can never disagree. A queue is wedged
 * when a worker is alive but claiming nothing while work waits:
 * zero live-lock active rows, waiting > 0, and the last completion is older
 * than the threshold (or absent). Threshold matches the doctor wedged_queue
 * check: GBRAIN_WEDGED_QUEUE_WARN_MINUTES (server-side env), default 15.
 *
 * Queue honesty (agent-first operator wave E5): when the caller knows no
 * worker is running for the queue (`workerAlive: false` — always on PGLite
 * unless a `gbrain jobs work` drain is live), waiting work is `no_worker`, not
 * wedged: nothing is stuck, nothing is running it. Omitted liveness keeps the
 * pre-E5 derivation (doctor's wedged_queue check).
 */
export function deriveWedgeSignal(wedge: {
  queue?: string;
  active_healthy: number;
  waiting: number;
  minutes_since_completion: number | null;
}, liveness: { workerAlive?: boolean } = {}): { wedged: boolean; no_worker: boolean; wedge_threshold_minutes: number; private_queue: boolean } {
  const raw = parseInt(process.env.GBRAIN_WEDGED_QUEUE_WARN_MINUTES ?? '', 10);
  const wedge_threshold_minutes = Number.isFinite(raw) && raw > 0 ? raw : 15;
  // A dream-inline private queue is parent-owned: no shared worker will ever
  // claim it, so "wedged — restart the worker" is impossible advice (the
  // incident bug class). Surface it as private_queue instead so every consumer
  // (jobs stats, get_job_stats op, doctor) points at reconciliation.
  const private_queue = wedge.queue !== undefined && isDreamInlinePrivateQueue(wedge.queue);
  const mins = wedge.minutes_since_completion;
  const no_worker = !private_queue && liveness.workerAlive === false && wedge.waiting > 0;
  const wedged = !private_queue && !no_worker && wedge.active_healthy === 0 && wedge.waiting > 0
    && (mins === null || mins > wedge_threshold_minutes);
  return { wedged, no_worker, wedge_threshold_minutes, private_queue };
}