Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
21.8 KB
Raw Blame
/**
 * Inline drain for the dream cycle's private per-run child queues (peeled from
 * synthesize.ts; #4194 adds bounded concurrency).
 *
 * Why inline on BOTH engines:
 *   - PGLite: no separate Minions worker can run at all (the embedded
 *     data-dir holds an exclusive file lock), so children would sit in
 *     'waiting' until waitForCompletion times out.
 *   - Postgres (#2050): the parent phase itself runs as a job inside a
 *     `jobs work` process. A worker whose slots are all occupied by such
 *     parents can never claim the child the parent is blocking on — a
 *     structural self-deadlock. Running children inline means a child never
 *     needs a worker slot.
 *
 * Concurrency model (#4194): N independent `drainLoop`s share the queue via
 * the same SKIP-LOCKED `queue.claim` fencing workers use at concurrency N.
 * Everything job-scoped (lockToken, abort, timeout timer, keepalive,
 * renewTimer, outcome recording) is loop-local. Pool hygiene:
 *   - Housekeeping sweeps (handleStalled/handleTimeouts/handleWallClock) are
 *     LEADER-ONLY (loop 0) — they're global, idempotent SQL and N copies per
 *     iteration would add pooler load on exactly the deployments #4194 is
 *     about. `promoteDelayed` runs in EVERY loop: it is the cheap sweep that
 *     picks up 0-delay retries, and any surviving loop must be able to do it
 *     (the leader may have exited on an empty queue while a sibling's child
 *     was still running and later failed to 'delayed').
 *   - A NON-RETRYABLE loop-level failure aborts sibling loops after their
 *     current child (shared poolAbort). A child failure is NOT a loop failure
 *     — one failing child never blocks siblings (#4194 acceptance).
 *   - Provider ceilings stay with the rate leases: the handler's lease
 *     acquisition (legacy per-turn, gateway per-turn hook, oneshot per-call)
 *     is the API limiter; a lease-full child requeues WITHOUT burning an
 *     attempt (worker parity), so inline concurrency above the lease cap
 *     degrades to waiting, never to dead-letters.
 */

import { randomUUID } from 'node:crypto';
import type { BrainEngine } from '../engine.ts';
import { MinionQueue } from '../minions/queue.ts';
import type { MinionHandler, MinionJob, MinionJobContext } from '../minions/types.ts';
import { UnrecoverableError } from '../minions/types.ts';
import { jobRefusal, refusedJobResult } from '../minions/refusal.ts';
import { makeSubagentHandler } from '../minions/handlers/subagent.ts';
import { RateLeaseUnavailableError, leaseFullBackoffMs } from '../minions/rate-leases.ts';
import { reconnectAfterConnectionError } from '../minions/reconnect.ts';
import { withChatPhase } from '../ai/chat-usage.ts';
import { isRetryableConnError } from '../retry-matcher.ts';
import { anySignal, throwIfAborted } from '../abort-check.ts';
import { CYCLE_DEADLINE_RESERVE_MS } from './base-phase.ts';

/**
 * #4168: minimum budget a freshly-claimed child is worth starting with.
 * Mirrors patterns.ts's MIN_PATTERNS_SUBAGENT_BUDGET_MS (importing it here
 * would close a module cycle: patterns → synthesize → inline-drain).
 */
const MIN_CHILD_BUDGET_MS = 2 * 60 * 1000;

export const INLINE_LOCK_MS = 30_000;

/**
 * One lock-renewal tick with a per-call timeout + cancellation. The renewal
 * query rides an AbortSignal that is aborted when the timeout wins the race
 * (the query is cancelled and its slot released), and callers guard
 * re-entrancy so at most one renewal is in flight.
 *
 * Returns after the renewal settles or times out; a `false` renewal invokes
 * `onLost` (token fence lost — caller aborts the handler). Errors and
 * timeouts are swallowed: best-effort, the next tick retries.
 */
export async function runDrainRenewalTick(
  renewLock: (
    id: number,
    lockToken: string,
    lockMs: number,
    opts?: { signal?: AbortSignal },
  ) => Promise<boolean>,
  jobId: number,
  lockToken: string,
  lockMs: number,
  onLost: () => void,
  callTimeoutMs: number,
): Promise<void> {
  const callAbort = new AbortController();
  let timer: ReturnType<typeof setTimeout> | null = null;
  try {
    const ok = await Promise.race([
      renewLock(jobId, lockToken, lockMs, { signal: callAbort.signal }),
      new Promise<never>((_, reject) => {
        timer = setTimeout(() => {
          callAbort.abort();
          reject(new Error(`renewLock timed out after ${callTimeoutMs}ms`));
        }, callTimeoutMs);
      }),
    ]);
    if (!ok) onLost();
  } catch {
    /* best-effort; next tick retries */
  } finally {
    if (timer != null) clearTimeout(timer);
  }
}

/**
 * Nearest-rank percentile over an unsorted sample. Returns null on an empty
 * sample (never NaN — the phase-details JSON must stay clean when a run had
 * zero or one child).
 */
export function percentile(values: number[], p: number): number | null {
  if (values.length === 0) return null;
  const sorted = [...values].sort((a, b) => a - b);
  const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
  return sorted[idx];
}

export async function runSubagentsInline(
  engine: BrainEngine,
  queue: MinionQueue,
  queueName: string,
  yieldDuringPhase?: () => Promise<void>,
  handler: MinionHandler = makeSubagentHandler({ engine }),
  lockMs: number = INLINE_LOCK_MS,
  concurrency = 1,
  /** #4168 adversarial: absolute parent-job deadline. When the remaining
   *  budget drops under the minimum child budget, loops stop CLAIMING —
   *  the caller cancels the still-waiting children and defers their
   *  transcripts (children submit fast, so submit-time clamps alone cannot
   *  bound a sequential multi-child drain). Residual, documented: each
   *  loop's LAST claimed child may still overrun the parent by up to its
   *  own clamped timeout. */
  deadlineAtMs: number | null = null,
  /** #4077: cooperative cancellation from the enclosing cycle/minion job.
   *  Checked before every claim and combined into ctx.signal so an in-flight
   *  handler stops too; the current child is cancelled (truthful 'cancelled',
   *  not dead or a burned delayed retry) before the drain unwinds. */
  signal: AbortSignal | null = null,
): Promise<void> {
  const n = Math.max(1, Math.floor(concurrency));
  if (n === 1) {
    // Serial fast path: identical semantics to the pre-#4194 drain.
    await drainLoop(engine, queue, queueName, yieldDuringPhase, handler, lockMs, 0, null, deadlineAtMs, signal);
    return;
  }
  const poolAbort = new AbortController();
  const settled = await Promise.allSettled(
    Array.from({ length: n }, (_, loopIdx) =>
      drainLoop(engine, queue, queueName, yieldDuringPhase, handler, lockMs, loopIdx, poolAbort, deadlineAtMs, signal)),
  );
  // allSettled (not all): a rejecting loop must not strand siblings'
  // in-flight children mid-record. The first non-retryable rejection is
  // rethrown after every loop has wound down.
  const firstRejection = settled.find((s): s is PromiseRejectedResult => s.status === 'rejected');
  if (firstRejection) throw firstRejection.reason;
}

async function drainLoop(
  engine: BrainEngine,
  queue: MinionQueue,
  queueName: string,
  yieldDuringPhase: (() => Promise<void>) | undefined,
  handler: MinionHandler,
  lockMs: number,
  loopIdx: number,
  poolAbort: AbortController | null,
  deadlineAtMs: number | null = null,
  signal: AbortSignal | null = null,
): Promise<void> {
  // #3555 interaction: the drain's queue ops used to be bare awaits, so a
  // transient pooler reap mid-drain threw out of the loop and stranded the
  // remaining children in this per-run private queue — which no worker will
  // ever claim. Mirror the worker's recovery: on a retryable connection
  // error, rebuild the pool (shared reconnectAfterConnectionError) and retry
  // the loop; non-retryable errors still propagate (real bug → phase fails).
  // Conn-error streak is PER-LOOP (one loop's reconnect window is its own).
  const MAX_CONN_ERROR_STREAK = 5;
  let connErrorStreak = 0;
  let idlePollMs = 1000;
  const recoverOrThrow = async (site: string, e: unknown): Promise<void> => {
    if (!isRetryableConnError(e) || ++connErrorStreak > MAX_CONN_ERROR_STREAK) throw e;
    const msg = e instanceof Error ? e.message : String(e);
    process.stderr.write(`[dream] inline drain ${site} hit a connection error; reconnecting and retrying: ${msg}\n`);
    await reconnectAfterConnectionError(engine, `inline-${site}`, e);
    // Small cooperative backoff (setTimeout keeps the cycle-lock keepalive
    // and any concurrent timers firing) before the loop retries.
    await new Promise((r) => setTimeout(r, Math.min(1000, Math.max(50, Math.floor(lockMs / 3)))));
  };

  try {
    while (true) {
      // Shared-abort check: a sibling loop hit a non-retryable failure. Stop
      // claiming new children; the sibling's rejection carries the cause.
      if (poolAbort?.signal.aborted) return;
      // #4077: external cancellation unwinds the drain — unclaimed children
      // stay 'waiting' for the phase finally's reconcilePrivateQueue.
      throwIfAborted(signal, '[dream] inline drain');
      // #4168 adversarial: stop claiming when the parent budget cannot fit
      // another child. Already-running work is unaffected; unclaimed
      // children stay 'waiting' for the caller's cancel-and-defer pass.
      if (
        deadlineAtMs != null &&
        deadlineAtMs - CYCLE_DEADLINE_RESERVE_MS - Date.now() < MIN_CHILD_BUDGET_MS
      ) {
        return;
      }
      const lockToken = randomUUID();
      let job: Awaited<ReturnType<MinionQueue['claim']>>;
      try {
        // Housekeeping a worker would normally perform, so child rows can
        // reach terminal states before the synth parent enters
        // waitForCompletion polling. Global sweeps are leader-only (pool
        // hygiene, see module docblock); promoteDelayed runs everywhere so
        // any surviving loop picks up 0-delay retries.
        await queue.promoteDelayed();
        if (loopIdx === 0) {
          await queue.handleStalled();
          await queue.handleTimeouts();
          await queue.handleWallClockTimeouts(lockMs);
        }
        job = await queue.claim(lockToken, lockMs, queueName, ['subagent']);
      } catch (e) {
        await recoverOrThrow('queue-ops', e);
        continue;
      }
      connErrorStreak = 0;
      if (!job) {
        // Exit ONLY when the queue is truly quiet. Three non-claimable-but-
        // live shapes must keep the loop polling instead of returning:
        //   - 'active' under a sibling loop or under a lock nobody renews
        //     after a connection-error window (handleStalled requeues it once
        //     the lock expires, ≤ lockMs);
        //   - 'delayed' with a future delay_until — the lease-full bounce
        //     requeues with a 1-3s backoff (worker parity, #4194), and
        //     exiting here would strand the child until the parent's
        //     waitForCompletion timed out;
        //   - 'waiting' that lost a claim race this iteration.
        let pending = 0;
        try {
          const rows = await engine.executeRaw<{ n: number }>(
            `SELECT count(*)::int AS n FROM minion_jobs
              WHERE queue = $1 AND status IN ('active', 'waiting', 'delayed')`,
            [queueName],
          );
          pending = rows[0]?.n ?? 0;
        } catch (e) {
          await recoverOrThrow('pending-check', e);
          continue;
        }
        if (pending === 0) return;
        // Renew the private-queue lease (and cycle lock) from the idle poll
        // too: with every child delayed/backing off no per-child keepalive is
        // armed, and an unrenewed lease reads as orphaned to spawn recovery —
        // which would cancel this LIVE queue. The wrapper self-throttles (30s).
        if (yieldDuringPhase) { try { await yieldDuringPhase(); } catch { /* best-effort */ } }
        // Idle poll with backoff (1s → 5s): under a pool, idle loops must not
        // hammer the queue while the last slow child drains.
        await new Promise((r) => setTimeout(r, idlePollMs));
        idlePollMs = Math.min(5000, idlePollMs * 2);
        continue;
      }
      idlePollMs = 1000;
      // Same lease-liveness rationale as the idle branch, for the fast-failing
      // child shape: a handler that dies in <60s never fires its first
      // keepalive tick, so a claim-fail-claim loop would run lease-blind.
      if (yieldDuringPhase) { try { await yieldDuringPhase(); } catch { /* best-effort */ } }

      const abort = new AbortController();
      const shutdown = new AbortController();
      const context: MinionJobContext = {
        id: job.id,
        name: job.name,
        data: job.data,
        attempts_made: job.attempts_made,
        idempotency_key: job.idempotency_key,
        // #4077: fold the phase's cancellation into the per-job signal so an
        // in-flight handler (mid-LLM-call) stops without waiting for its own
        // timeout; `abort` stays the loop-local source for timedOut below.
        signal: anySignal(abort.signal, signal),
        deadlineAtMs: job.timeout_at != null ? job.timeout_at.getTime() : null,
        shutdownSignal: shutdown.signal,
        updateProgress: async (progress: unknown) => {
          await queue.updateProgress(job.id, lockToken, progress);
        },
        updateTokens: async (tokens) => {
          await queue.updateTokens(job.id, lockToken, tokens);
        },
        log: async (message) => {
          const value = typeof message === 'string' ? message : JSON.stringify(message);
          await engine.executeRaw(
            `UPDATE minion_jobs SET stacktrace = COALESCE(stacktrace, '[]'::jsonb) || to_jsonb($1::text),
              updated_at = now()
             WHERE id = $2 AND status = 'active' AND lock_token = $3`,
            [value, job.id, lockToken],
          );
        },
        isActive: async () => {
          const rows = await engine.executeRaw<{ id: number }>(
            `SELECT id FROM minion_jobs WHERE id = $1 AND status = 'active' AND lock_token = $2`,
            [job.id, lockToken],
          );
          return rows.length > 0;
        },
        readInbox: async () => queue.readInbox(job.id, lockToken),
      };

      // Per-job deadline enforcement (worker.ts parity). While the drain loop
      // awaits the handler, the handleTimeouts sweep above can't run, so
      // nothing else can stop a child that blows past timeout_ms — the
      // handler only stops when ctx.signal fires. Derive the delay from the
      // claim-time timeout_at stamp so timer, DB sweeper, and deadlineAtMs
      // agree.
      let timeoutTimer: ReturnType<typeof setTimeout> | null = null;
      if (job.timeout_ms != null) {
        const delayMs = job.timeout_at != null
          ? Math.max(0, job.timeout_at.getTime() - Date.now())
          : job.timeout_ms;
        timeoutTimer = setTimeout(() => {
          if (!abort.signal.aborted) abort.abort(new Error('timeout'));
        }, delayMs);
      }

      // Cycle-lock keepalive while the child runs (best-effort, never throws).
      // Concurrent invocation from N loops is safe: the callback is a
      // timestamp-refresh UPDATE on the cycle-lock row (verified against
      // buildYieldDuringPhase; ENG-2).
      const keepalive = yieldDuringPhase
        ? setInterval(() => { yieldDuringPhase().catch(() => { /* best-effort */ }); }, 60_000)
        : null;
      // #2050: heartbeat the child's claim lock while the handler runs so a
      // concurrent Postgres worker's handleStalled() sweep (all queues, not
      // just its own) can't requeue a live child. A false return means the
      // row was cancelled or reclaimed — abort the handler. Errors are
      // swallowed (best-effort; the next tick retries), never an
      // unhandledRejection. Re-entrancy guard + per-call cancellation via
      // runDrainRenewalTick: a hung renewLock no longer stacks a fresh
      // checked-out pool slot per interval firing (issue #6 abandoned-racer
      // class).
      let drainTickInFlight = false;
      const renewTimer = setInterval(() => {
        if (drainTickInFlight) return;
        drainTickInFlight = true;
        void runDrainRenewalTick(
          (id, tok, ms, opts) => queue.renewLock(id, tok, ms, opts),
          job.id,
          lockToken,
          lockMs,
          () => {
            if (!abort.signal.aborted) abort.abort(new Error('lock-renewal-failed'));
          },
          Math.max(1000, Math.floor(lockMs / 3)),
        ).finally(() => {
          drainTickInFlight = false;
        });
      }, Math.max(50, Math.floor(lockMs / 3)));
      // Run, then record — separated so a completeJob connection error can't
      // masquerade as a handler failure, and a failJob connection error can't
      // escape the drain and strand the remaining children (worker.ts #1720
      // parity: reconnect + retry the recording once; if it still fails,
      // leave the row for the loop's own handleStalled to requeue after lock
      // expiry).
      let result: unknown;
      let handlerErr: unknown;
      let handlerRan = false;
      try {
        // #4218 parity with minions/worker.ts: attribute every gateway.chat()
        // the handler makes to THIS job, so chat_usage_log rows carry
        // `phase = 'job:<name>'`. Without it the inline drain inherits its
        // caller's AsyncLocalStorage phase — a cycle phase that wraps its own
        // work (e.g. dream synthesize) would silently absorb every child's
        // spend into the phase tag, breaking the one-ledger-per-surface rule
        // the phase telemetry depends on.
        result = await withChatPhase(`job:${job.name}`, () => handler(context));
        handlerRan = true;
      } catch (e) {
        handlerErr = e;
      }
      const record = async (): Promise<void> => {
        if (handlerRan) {
          await queue.completeJob(
            job.id,
            lockToken,
            result != null ? (typeof result === 'object' ? result as Record<string, unknown> : { value: result }) : undefined,
          );
          return;
        }
        // Worker-parity outcome routing (#4217 CDX-3 + #4194):
        //   - RateLeaseUnavailableError → delayed requeue WITHOUT burning an
        //     attempt (mirrors worker.ts's releaseLeaseFullJob path; matters
        //     with concurrent loops against a shared lease cap).
        //   - UnrecoverableError → dead immediately (retry is provably futile
        //     — e.g. all-writes-failed accounting; pre-fix the drain retried
        //     these to exhaustion, exactly the incident-churn #4217
        //     describes).
        //   - Timeout stays terminal (handleTimeouts parity), never delayed.
        if (handlerErr instanceof RateLeaseUnavailableError) {
          const leaseBackoffMs = leaseFullBackoffMs();
          const released = await queue.releaseLeaseFullJob(
            job.id, lockToken, handlerErr.message, leaseBackoffMs,
          );
          if (released) return;
          // Lock-token mismatch (stall sweep won) — fall through to failJob,
          // which no-ops under the same fence.
        }
        await failChild(queue, job, lockToken, handlerErr, abort.signal.aborted);
      };
      try {
        // #4077: external cancellation outranks outcome recording — a child
        // stopped by the phase's abort must read 'cancelled', never dead or a
        // burned delayed retry. Cancel is best-effort; the phase finally's
        // reconcilePrivateQueue is the backstop.
        if (signal?.aborted) {
          try { await queue.cancelJob(job.id); } catch { /* reconcile backstop */ }
          throwIfAborted(signal, '[dream] inline drain');
        }
        try {
          await record();
        } catch (recordErr) {
          if (!isRetryableConnError(recordErr)) throw recordErr;
          const msg = recordErr instanceof Error ? recordErr.message : String(recordErr);
          process.stderr.write(`[dream] inline drain: recording job ${job.id} outcome hit a connection error; reconnecting and retrying once: ${msg}\n`);
          await reconnectAfterConnectionError(engine, 'inline-record', recordErr);
          try {
            await record();
          } catch (retryErr) {
            // Leave the row to the loop's own handleStalled: the claim lock
            // stops renewing (finally clears renewTimer), expires within
            // lockMs, and the next iteration requeues it on a live pool.
            const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);
            process.stderr.write(`[dream] inline drain: outcome recording retry for job ${job.id} also failed (${retryMsg}); leaving the row for stall requeue\n`);
          }
        }
      } finally {
        if (timeoutTimer) clearTimeout(timeoutTimer);
        if (keepalive) clearInterval(keepalive);
        clearInterval(renewTimer);
      }
    }
  } catch (e) {
    // Loop-level fatal (non-retryable conn error, unrecoverable record
    // failure). Signal siblings to stop claiming NEW children after their
    // current one — the phase is going to fail with this cause anyway;
    // letting siblings churn through the rest of the queue first would just
    // delay the report (OV-6).
    poolAbort?.abort(e);
    throw e;
  }
}

/**
 * Record a failed child: a timeout or an UnrecoverableError is dead; a deterministic refusal (#7239) is dead with its
 * code and fix on the result; anything else is retried (delayed, no backoff) until its attempts run out.
 */
async function failChild(queue: MinionQueue, job: MinionJob, lockToken: string, handlerErr: unknown, timedOut: boolean): Promise<void> {
  const errorText = timedOut ? 'timeout exceeded' : (handlerErr instanceof Error ? handlerErr.message : String(handlerErr));
  const refusal = timedOut ? null : jobRefusal(handlerErr);
  const isUnrecoverable = refusal !== null || handlerErr instanceof UnrecoverableError;
  const attemptsExhausted = job.attempts_made + 1 >= job.max_attempts;
  await queue.failJob(job.id, lockToken, errorText, timedOut || isUnrecoverable || attemptsExhausted ? 'dead' : 'delayed', 0,
    refusal ? refusedJobResult(refusal) : undefined);
}