Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
30.1 KB
Raw Blame
/**
 * issue #1678 — bounded single-hold drain for extract_atoms.
 *
 * The operator/agent escape hatch for a backlog the routine cycle won't touch
 * (pack-gated off) or can't keep up with. Design per Codex #8/#9/#10:
 *
 *  - SINGLE continuous lock hold (no release/reacquire between batches). The
 *    caller wraps the loop in `withRefreshingLock(cycleLockIdFor(sourceId))` —
 *    the SAME lock id the routine cycle uses for that source — so the two
 *    genuinely contend (no source-vs-legacy lock mismatch) and there's no
 *    release-gap where autopilot/sync could mutate pages mid-drain (which would
 *    let the drain extract atoms from stale content).
 *  - REDISCOVER eligibility each batch (the injected `runBatch` re-runs the
 *    NOT-EXISTS-on-source_hash discovery), so stale content simply doesn't
 *    match — no cross-window cursor of page lists.
 *  - BOUNDED by a wallclock window; reports `remaining` so a cron/agent loop
 *    knows whether to run again.
 *  - Two stop signals (#5809/#5832). HARD = caller signal (Minion job) + the
 *    lock's lease-loss signal + the job deadline: reaches the chat call, the
 *    pacing pause and every backlog count, and stops before the next commit.
 *    SOFT = hard + window: checked only between batches and items, so a
 *    window stop never aborts an in-flight paid call.
 *
 * Pure over injected deps: no DB, no LLM, no lock primitive imported here, so
 * the loop logic is unit-testable. Its only static imports are pure text
 * sanitizers (#4730) and the leaf signal combiner (`abort-signals.ts`). The
 * wiring helper `runExtractAtomsDrainForSource` (below) builds the real deps;
 * it uses DYNAMIC imports so the pure-loop unit tests don't drag in db-lock /
 * cycle.
 */

import type { BrainEngine } from '../engine.ts';
import type { ExtractAtomsOpts } from './extract-atoms.ts';
import { redactConnectionInfo } from '../audit/redact-connection-info.ts';
import { redactFindings } from '../secret-scan.ts';
import { ensureWellFormed, truncateUtf8 } from '../text-safe.ts';
import { anyAbortSignal } from '../abort-signals.ts';

/** #4730: bounded operator-facing failure detail; totals stay exact above the cap. */
export const MAX_DRAIN_FAILURE_RECORDS = 25;
/** Matches the repo's audit/error-summary privacy cap. */
export const MAX_DRAIN_FAILURE_SOURCE_CHARS = 256;
export const MAX_DRAIN_FAILURE_REASON_CHARS = 200;

/**
 * #4730: one preserved per-item failure. `failure_count` remains the exact
 * total; `failures` holds up to MAX_DRAIN_FAILURE_RECORDS of these in batch
 * order, and `omitted_failure_count` reconciles the difference — the cap is
 * reported, never silently applied.
 */
export interface ExtractAtomsDrainFailure {
  /** One-based batch number within this bounded drain window. */
  batch: number;
  /** Stable page slug / transcript locator emitted by runPhaseExtractAtoms (sanitized). */
  source: string;
  /** Bounded, sanitized failure reason (secrets/connection info redacted). */
  reason: string;
  /** #7514: a write that did not commit: its request id (`gbrain write-request` explains it) and the refusal's own code. */
  request_id?: string;
  code?: string;
}

/**
 * Sanitize operator-facing failure text: secret + connection-string redaction,
 * well-formed UTF-8, collapsed whitespace, bounded length. Locators and
 * reasons both route through here so neither can carry an unbounded provider
 * payload or credentials into `--json` output / job results.
 */
function sanitizeFailureText(raw: string, maxChars: number): string {
  const secretRedacted = redactFindings(raw, { highEntropy: true }).text;
  const connectionRedacted = redactConnectionInfo(secretRedacted);
  return truncateUtf8(
    ensureWellFormed(connectionRedacted).replace(/\s+/g, ' ').trim(),
    maxChars,
  );
}

export interface ExtractAtomsDrainDeps {
  /**
   * Run the loop body while holding the cycle lock. Implemented by the caller
   * via `withRefreshingLock`. MUST throw when the lock is held by another
   * process (e.g. `LockUnavailableError`) — the drain lets that propagate so
   * the caller can report `cycle_already_running` and exit, matching the
   * routine cycle's skip contract. `work` receives the lock's signal, which
   * aborts when the lease is lost (#5832); the drain maps that loss, even
   * when the helper rethrows it after `work` settled, to `stopped: 'lock_lost'`.
   */
  withLock: <T>(work: (signal: AbortSignal) => Promise<T>) => Promise<T>;
  /**
   * Process one bounded batch (rediscovers eligibility). Returns counts, plus
   * `providerFailure` (issue #3218) when EVERY item the batch attempted threw
   * (zero items succeeded, at least one failure) — i.e. the batch's warning
   * result was actually a total provider outage, not a partial/no-op batch.
   * Omit/false for the ordinary partial-success or nothing-to-do cases.
   *
   * #4539: `failureCount` (per-item failures in this batch) and `firstError`
   * (a representative failure message) let the drain surface WHY a run
   * stopped/underperformed. #4730: `failures` preserves the phase's per-item
   * `{source, reason}` records so a mixed-failure batch is fully
   * reconcilable from `--json` — pre-#4730 everything but ONE representative
   * error was dropped and the operator had to re-run the work to see the
   * other reasons.
   *
   * `signal` is the hard stop and `stopSignal` the soft one (see the module
   * header); the batch hands both to `runPhaseExtractAtoms`.
   *
   * #6425: `budgetSkipped` counts items the batch skipped because the
   * attempt's per-run cap was spent, with `budget` the spend and the cap. The
   * attempt shares one tracker across batches, so the drain stops `budget`.
   */
  runBatch: (signals: { signal: AbortSignal; stopSignal: AbortSignal }) => Promise<{
    extracted: number;
    skipped: number;
    providerFailure?: boolean;
    failureCount?: number;
    firstError?: string;
    failures?: Array<{ source: string; reason: string; request_id?: string; code?: string }>;
    /** #7514: write failures that recorded a strike toward their hold this batch; a batch of them is progress, not a stall. */
    writeStrikes?: number;
    budgetSkipped?: number;
    budget?: { spentUsd: number; capUsd: number };
    /** #6503: items the batch cut at `cycle.extract_atoms.max_input_chars` (partial disposition, not complete). */
    inputTruncated?: number;
    /** #6817: with `providerFailure`, what the outage was; unset reads as `transient`. */
    providerFailureClass?: DrainProviderFailureClass;
  }>;
  /**
   * Count remaining eligible-but-unextracted pages, or null on query error.
   * `signal` fires on a hard stop or when the count outlives its bound; the
   * drain stops waiting at that point even if the query ignores it.
   */
  countRemaining: (signal: AbortSignal) => Promise<number | null>;
  /** Injectable clock. Production: Date.now. */
  now: () => number;
  /** Optional progress sink (one line per batch). */
  onBatch?: (info: { batch: number; extracted: number; remaining: number | null }) => void;
}

export interface ExtractAtomsDrainOpts {
  /**
   * Wallclock window in ms, a hard deadline for starting work: no batch or
   * item starts after it, while the item in flight finishes (soft signal).
   */
  windowMs: number;
  /** Hard cap on batches (belt-and-suspenders against a 0-progress loop). Default 1000. */
  maxBatches?: number;
  /** Caller's cancellation signal (the Minion job's timeout/cancel signal, #5809). */
  signal?: AbortSignal;
  /** Absolute job deadline (epoch ms, `deadlineAtMs` clock); a hard stop at that instant. */
  deadlineAtMs?: number | null;
  /** Upper bound on one backlog count; the remaining deadline caps it further. Default 60 s. */
  countTimeoutMs?: number;
}

/**
 * Why a drain stopped. `window`/`no_progress`/`max_batches`/`budget` are soft
 * stops after a final recount (`budget`: the attempt's per-run cap is spent); `deadline` (job deadline), `lock_lost` (cycle lock
 * lease lost) and `aborted` (job cancelled or shutting down) are hard stops
 * whose `remaining` is the last count taken before the stop.
 */
export type DrainStop =
  | 'drained' | 'window' | 'deadline' | 'lock_lost' | 'aborted'
  | 'no_progress' | 'max_batches' | 'provider_failure' | 'budget';

/**
 * #6817: why every item of a provider_failure batch failed. `auth`, `billing`,
 * `model_not_found` and `content_block` repeat on every retry until someone
 * changes the key, plan, model or content; `rate_limit` and `transient`
 * (5xx, timeout, network, an unclassified error) can clear on their own.
 */
export type DrainProviderFailureClass = 'auth' | 'billing' | 'model_not_found' | 'content_block' | 'rate_limit' | 'transient';

export const DRAIN_COUNT_TIMEOUT_MS = 60_000;

/**
 * #5856: dead-letter text of a drain the atom session preflight refused before any spend
 * (`owner_unavailable` / `permission_denied`). The auto-drain cap does not count such a job.
 */
export const STRUCTURAL_REFUSAL_PREFIX = 'structural_refusal:';

/** The non-retryable dead-letter text for a structural preflight refusal, or null for any other error. */
export function structuralAtomRefusal(error: unknown): string | null {
  if (!(error instanceof Error) || error.name !== 'OperationError') return null;
  const code = (error as Error & { code?: unknown }).code;
  return code === 'owner_unavailable' || code === 'permission_denied' ? `${STRUCTURAL_REFUSAL_PREFIX} ${code}: ${error.message}` : null;
}
/** setTimeout fires at once past 2^31-1 ms; a longer window or deadline waits the maximum. */
const MAX_TIMER_MS = 2 ** 31 - 1;

export interface ExtractAtomsDrainResult {
  phase: 'extract_atoms';
  /**
   * issue #3218: 'provider_failure' when any batch reported `providerFailure`
   * (every item it attempted errored). The Minion handler throws on this
   * status so the durable job retries instead of completing over a backlog
   * that made zero forward progress. Partial-success batches (>=1 item
   * succeeded) always report 'ok', unchanged from before.
   */
  status: 'ok' | 'provider_failure';
  extracted: number;
  skipped: number;
  /** Eligible pages still pending (see DrainStop for hard stops). null if no count succeeded. */
  remaining: number | null;
  /** Batches actually processed. */
  batches: number;
  stopped: DrainStop;
  /**
   * #4539: total per-item failures across every batch in this run. 0 for a
   * clean run. Included in `--json` verbatim; dream.ts prints a stderr line
   * when non-zero so the operator sees WHY the drain underperformed.
   */
  failure_count: number;
  /**
   * #4730: bounded per-item failure details, in batch order, capped at
   * MAX_DRAIN_FAILURE_RECORDS. Locators and reasons are sanitized (secret +
   * connection-info redaction, bounded length). Rides `--json` verbatim.
   */
  failures: ExtractAtomsDrainFailure[];
  /**
   * #4730: failures beyond the record cap (or reported by count only, with
   * no per-item detail). `failure_count === failures.length +
   * omitted_failure_count` always holds, so the cap is visible, never silent.
   */
  omitted_failure_count: number;
  /**
   * #4539: representative failure message from the most recent batch that
   * reported one (`source: reason`), or null for a clean run.
   */
  last_error: string | null;
  /** #6425: on a `budget` stop, the attempt's spend and its per-run cap (`cycle.extract_atoms.budget_usd`). */
  spent_usd?: number;
  budget_usd?: number;
  /** #6503: items cut at the input cap across every batch; the model did not read their tails and they are not marked complete. */
  input_truncated?: number;
  /** #6817: with `status: 'provider_failure'`, the failing batch's class (`transient` when the batch named none). */
  provider_failure_class?: DrainProviderFailureClass;
}

/**
 * #3813: the Minion handler's provider_failure throw IS the job's error_text
 * once it dead-letters, so it carries the representative `last_error` (already
 * secret-redacted + bounded above) — a missing provider key is a one-line
 * diagnosis instead of an opaque batches/remaining.
 */
export function formatDrainProviderFailure(
  result: Pick<ExtractAtomsDrainResult, 'batches' | 'remaining' | 'last_error'>,
  opts: { final?: boolean } = {},
): string {
  return (
    `extract-atoms-drain: all provider calls failed this batch ` +
    `(batches=${result.batches}, remaining=${result.remaining ?? '?'})` +
    `${result.last_error ? `; last error: ${result.last_error}` : ''}${opts.final ? '' : ' — retrying'}`
  );
}

export async function runExtractAtomsDrain(
  deps: ExtractAtomsDrainDeps,
  opts: ExtractAtomsDrainOpts,
): Promise<ExtractAtomsDrainResult> {
  const maxBatches = opts.maxBatches ?? 1000;
  const countTimeoutMs = opts.countTimeoutMs ?? DRAIN_COUNT_TIMEOUT_MS;
  // Counters live outside the locked work so a lease loss the lock helper
  // rethrows after `work` settled still reports the partial progress.
  let extracted = 0;
  let skipped = 0;
  let batches = 0;
  // issue #3218: latched once any batch reports providerFailure — drives
  // the returned `status`, independent of how `stopped` reads after the
  // final (possibly overriding) remaining-count check below.
  let providerFailure = false;
  let providerFailureClass: DrainProviderFailureClass = 'transient';
  // #4539: accumulate per-item failure visibility across batches.
  let failureCount = 0;
  // #4730: bounded typed per-item records (batch order, sanitized).
  const failures: ExtractAtomsDrainFailure[] = [];
  let lastError: string | null = null;
  let lastCount: number | null = null;
  let lockSignal: AbortSignal | null = null;
  let budget: { spentUsd: number; capUsd: number } | undefined;
  let inputTruncated = 0;
  const result = (stopped: DrainStop, remaining: number | null): ExtractAtomsDrainResult => ({
    phase: 'extract_atoms',
    status: providerFailure ? 'provider_failure' : 'ok',
    extracted,
    skipped,
    remaining,
    batches,
    stopped,
    failure_count: failureCount,
    failures,
    omitted_failure_count: failureCount - failures.length,
    last_error: lastError,
    ...(stopped === 'budget' && budget ? { spent_usd: budget.spentUsd, budget_usd: budget.capUsd } : {}),
    input_truncated: inputTruncated,
    ...(providerFailure ? { provider_failure_class: providerFailureClass } : {}),
  });
  try {
    return await deps.withLock(async (signal) => {
      lockSignal = signal;
      const windowEnd = deps.now() + opts.windowMs;
      const deadlineStop = new AbortController();
      const windowStop = new AbortController();
      const deadlineTimer = opts.deadlineAtMs == null ? undefined : setTimeout(
        () => deadlineStop.abort(new Error('extract-atoms-drain: job deadline reached')),
        Math.min(MAX_TIMER_MS, Math.max(0, opts.deadlineAtMs - deps.now())),
      );
      const windowTimer = setTimeout(
        () => windowStop.abort(new Error('extract-atoms-drain: window elapsed')),
        Math.min(MAX_TIMER_MS, opts.windowMs),
      );
      deadlineTimer?.unref?.();
      windowTimer.unref?.();
      const hard = anyAbortSignal([...(opts.signal ? [opts.signal] : []), signal, deadlineStop.signal]);
      const soft = anyAbortSignal([hard.signal, windowStop.signal]);
      const hardStop = (): DrainStop => signal.aborted ? 'lock_lost'
        : deadlineStop.signal.aborted || (opts.deadlineAtMs != null && deps.now() >= opts.deadlineAtMs) ? 'deadline'
        : 'aborted';
      // E15: a count that blocks must not keep the lock refreshing, so it is
      // bounded by the remaining deadline and abandoned on a hard stop.
      const count = async (): Promise<number | null> => {
        const bound = opts.deadlineAtMs == null ? countTimeoutMs
          : Math.min(countTimeoutMs, Math.max(0, opts.deadlineAtMs - deps.now()));
        const timeout = new AbortController();
        const timer = setTimeout(() => timeout.abort(new Error('extract-atoms-drain: backlog count timed out')), bound);
        const bounded = anyAbortSignal([hard.signal, timeout.signal]);
        try {
          if (bounded.signal.aborted) return null;
          return await new Promise<number | null>((resolve, reject) => {
            bounded.signal.addEventListener('abort', () => resolve(null), { once: true });
            deps.countRemaining(bounded.signal).then(resolve, reject);
          });
        } finally {
          clearTimeout(timer);
          bounded.dispose();
        }
      };
      try {
        let stopped: DrainStop = 'window';
        while (deps.now() < windowEnd && !soft.signal.aborted) {
          if (batches >= maxBatches) { stopped = 'max_batches'; break; }

          const before = await count();
          if (hard.signal.aborted) break;
          if (before !== null) lastCount = before;
          if (before === 0) { stopped = 'drained'; break; }

          const r = await deps.runBatch({ signal: hard.signal, stopSignal: soft.signal });
          extracted += r.extracted;
          skipped += r.skipped;
          inputTruncated += Math.max(0, Math.floor(Number(r.inputTruncated ?? 0)) || 0);
          batches++;
          // #4730: preserve typed per-item records (bounded, sanitized) while
          // keeping failure_count exact and reconcilable — count-only adapters
          // (the #4539 shape) still contribute to the total via failureCount.
          const batchFailures = Array.isArray(r.failures)
            ? r.failures.filter(
                (f): f is { source: string; reason: string; request_id?: string; code?: string } =>
                  f != null &&
                  typeof f === 'object' &&
                  typeof f.source === 'string' &&
                  typeof f.reason === 'string',
              )
            : [];
          const reportedFailureCount =
            typeof r.failureCount === 'number' && Number.isFinite(r.failureCount) && r.failureCount > 0
              ? Math.floor(r.failureCount)
              : 0;
          failureCount += Math.max(reportedFailureCount, batchFailures.length);
          for (const f of batchFailures) {
            if (failures.length >= MAX_DRAIN_FAILURE_RECORDS) break;
            failures.push({
              batch: batches,
              source: sanitizeFailureText(f.source, MAX_DRAIN_FAILURE_SOURCE_CHARS),
              reason: sanitizeFailureText(f.reason, MAX_DRAIN_FAILURE_REASON_CHARS),
              ...(typeof f.request_id === 'string' ? { request_id: sanitizeFailureText(f.request_id, 64) } : {}),
              ...(typeof f.code === 'string' ? { code: sanitizeFailureText(f.code, 64) } : {}),
            });
          }
          const representative = batchFailures[0];
          if (representative) {
            lastError =
              `${sanitizeFailureText(representative.source, MAX_DRAIN_FAILURE_SOURCE_CHARS)}: ` +
              `${sanitizeFailureText(representative.reason, MAX_DRAIN_FAILURE_REASON_CHARS)}`;
          } else if (typeof r.firstError === 'string' && r.firstError.trim()) {
            // #4539 compatibility: count-only adapters still surface their
            // representative error — through the SAME sanitizer as the typed
            // records (secret/DSN redaction, whitespace collapse, bounded), so a
            // provider payload cannot ride the fallback path into --json output.
            lastError = sanitizeFailureText(
              r.firstError,
              MAX_DRAIN_FAILURE_SOURCE_CHARS + 2 + MAX_DRAIN_FAILURE_REASON_CHARS,
            );
          }
          if (hard.signal.aborted) break;
          deps.onBatch?.({ batch: batches, extracted: r.extracted, remaining: before });

          // issue #3218: every item this batch attempted failed (0 succeeded, >=1
          // error) — a total provider outage, not ordinary no-op/partial progress.
          // Stop immediately (same hot-loop guard as no_progress below) and flag
          // it so the caller can retry via its own policy instead of treating the
          // drain as a clean completion.
          if (r.providerFailure) {
            providerFailure = true;
            providerFailureClass = r.providerFailureClass ?? 'transient';
            stopped = 'provider_failure';
            break;
          }

          // #6425: the attempt's one BudgetTracker is spent, so no later batch can progress.
          if ((r.budgetSkipped ?? 0) > 0) {
            budget = r.budget;
            stopped = 'budget';
            break;
          }

          // Stop if a batch made zero forward progress — extraction is failing or
          // everything left is ineligible (e.g. all skipped). Prevents a hot loop
          // that spends budget without draining.
          //
          // #2144: a zero-ATOM batch can still be progress — tombstoned
          // zero-yield pages shrink the backlog without producing atoms. Only
          // stop when the backlog count genuinely didn't move.
          //
          // #8445: a skip-only batch gets the same recount. An item the phase
          // skips but the backlog still counts re-skipped every batch, so the
          // drain ran to max_batches holding the cycle lock.
          if (r.extracted === 0) {
            // #7826: the window closed before this batch started its first item; that is a window stop, not a stall.
            if (soft.signal.aborted) { stopped = 'window'; break; }
            const after = await count();
            if (hard.signal.aborted) break;
            if (after !== null) lastCount = after;
            if (after === null || before === null || (after >= before && !((r.writeStrikes ?? 0) > 0))) { stopped = 'no_progress'; break; }
          }
        }

        // Hard stops write nothing and skip the recount (it would outlive the
        // deadline or run without the lock); `remaining` is the last count.
        if (hard.signal.aborted) return result(hardStop(), lastCount);
        const remaining = await count();
        if (hard.signal.aborted) return result(hardStop(), lastCount);
        lastCount = remaining;
        // issue #3218 (codex P2): don't let a final remaining===0 recount
        // overwrite 'provider_failure' back to 'drained' — that would report the
        // contradictory {status: 'provider_failure', stopped: 'drained'} and
        // mislead the CLI/JSON consumer (dream.ts prints both fields verbatim).
        // status already takes precedence for the Minion handler's retry
        // decision; keep `stopped` consistent with it once a failure latched.
        if (!providerFailure && remaining === 0) stopped = 'drained';
        return result(stopped, remaining);
      } finally {
        clearTimeout(deadlineTimer);
        clearTimeout(windowTimer);
        soft.dispose();
        hard.dispose();
      }
    });
  } catch (err) {
    // E8: `withRefreshingLock` rethrows the recorded lease loss from its
    // finally, replacing whatever `work` returned or threw.
    const lock = lockSignal as AbortSignal | null;
    if (lock?.aborted && err === lock.reason) return result('lock_lost', lastCount);
    throw err;
  }
}

/**
 * #6817: classify a batch whose every item failed. The phase's whole-run halt
 * class (`details.aborted_global_error`) wins; a batch whose every failure is
 * a provider content block is `content_block`; anything else is `transient`,
 * because nothing shows that retrying cannot succeed.
 */
export function batchProviderFailureClass(halted: unknown, failures: ReadonlyArray<{ reason: string }>): DrainProviderFailureClass {
  if (halted === 'auth' || halted === 'billing' || halted === 'model_not_found' || halted === 'rate_limit') return halted;
  if (failures.length > 0 && failures.every(f => f.reason.startsWith('provider blocked content'))) return 'content_block';
  return 'transient';
}

// ─── Shared wiring helper (v0.42.x #1685 DECISION 5A) ──────────────────────
//
// ONE drain path, three callers: `gbrain dream --phase extract_atoms --drain`
// (dream.ts), the `extract-atoms-drain` Minion handler (jobs.ts), and the
// autopilot auto-drain submission (which routes through the handler). Before
// this helper the lock/batch/count wiring lived inline in dream.ts:482; a second
// copy in the handler would let lock id / window default / defer-on-lock-busy
// drift. Keeping the wiring here means those three callers can't diverge.
//
// Imports are dynamic so the pure `runExtractAtomsDrain` above stays cheap to
// import in unit tests (no db-lock / cycle / extract-atoms in the static graph).
//
// `LockUnavailableError` is NOT caught here — the pure loop's `withLock`
// (withRefreshingLock) throws it and it propagates to the caller, because each
// caller reports the busy-lock case differently (dream → exit 3;
// handler → `{ deferred: true }`). That matches the contract documented on
// `ExtractAtomsDrainDeps.withLock`.

export interface DrainForSourceOpts {
  /**
   * The RESOLVED source id, or `undefined` for the legacy unscoped cycle.
   * `undefined` → `cycleLockIdFor(undefined)` = the bare `gbrain-cycle` lock the
   * unscoped routine cycle holds; a real id → `gbrain-cycle:<id>`. Either way the
   * drain and the routine cycle for THIS source genuinely contend (Codex #9).
   * The extraction/backlog source is `sourceId ?? 'default'`.
   */
  sourceId: string | undefined;
  /** Wallclock budget in seconds. */
  windowSeconds: number;
  /** Brain checkout dir, threaded to `runPhaseExtractAtoms` (optional — DB-only ok). */
  brainDir?: string;
  /** Hard batch cap (belt-and-suspenders). */
  maxBatches?: number;
  /** Optional per-batch progress sink (stderr line in dream; job progress in the handler). */
  onBatch?: ExtractAtomsDrainDeps['onBatch'];
  /** #5809: the Minion job's signal (see ExtractAtomsDrainOpts.signal). */
  signal?: AbortSignal;
  /** #5809: the Minion job's `deadlineAtMs`. */
  deadlineAtMs?: number | null;
}

export async function runExtractAtomsDrainForSource(
  engine: BrainEngine,
  opts: DrainForSourceOpts,
): Promise<ExtractAtomsDrainResult> {
  const { withRefreshingLock } = await import('../db-lock.ts');
  const { countExtractAtomsBacklog } = await import('./extract-atoms.ts');
  const { runPhaseExtractAtomsStamped } = await import('./extract-atoms-stamp.ts');
  const { cycleLockIdFor } = await import('../cycle.ts');

  const { MaintenanceWriteWait } = await import('../persistence/maintenance-wait.ts');

  const extractionSourceId = opts.sourceId ?? 'default';
  const lockId = cycleLockIdFor(opts.sourceId);
  // #5856/#5854: one call is one attempt; its batches share one BudgetTracker
  // (the per-run cap holds for the whole attempt) and one publish wait.
  const attempt: NonNullable<ExtractAtomsOpts['attempt']> = { writeWait: new MaintenanceWriteWait(opts.deadlineAtMs ?? null) };

  return runExtractAtomsDrain(
    {
      withLock: (work) => withRefreshingLock(engine, lockId, (signal) => work(signal), { ttlMinutes: 5 }),
      runBatch: async ({ signal, stopSignal }) => {
        const r = await runPhaseExtractAtomsStamped(engine, {
          sourceId: extractionSourceId,
          dryRun: false,
          brainDir: opts.brainDir,
          signal,
          stopSignal,
          attempt,
        });
        const d = (r.details ?? {}) as Record<string, unknown>;
        // issue #3218: `r.status` collapses to 'warn' whether ONE item failed
        // (partial success — leave the drain's existing ok/no_progress path
        // alone) or EVERY item failed (a total provider outage the drain
        // adapter was silently swallowing). Re-derive the total-failure case
        // from the per-item counts `runPhaseExtractAtoms` already returns:
        // >=1 failure AND zero items successfully processed (transcripts_processed
        // + pages_processed both 0 means every attempted `chat()` call threw —
        // items that succeed with 0 atoms still count as processed, so this
        // does not fire on "provider fine, nothing extractable").
        const reported: unknown[] = Array.isArray(d.failures) ? d.failures : [];
        // #7514: a write that did not commit (its record names the request id) never reached a provider, so
        // `failures` is the provider side only: a write failure neither makes nor classes a provider failure.
        const written = (f: unknown): f is { request_id: string; code?: unknown; strikes?: unknown } =>
          f != null && typeof f === 'object' && typeof (f as { request_id?: unknown }).request_id === 'string';
        const failures = reported.filter(f => !written(f));
        const itemsSucceeded =
          Number(d.transcripts_processed ?? 0) + Number(d.pages_processed ?? 0);
        // #4730: carry EVERY per-item failure up as a typed {source, reason}
        // record (the pure loop bounds + sanitizes them) instead of the
        // #4539 collapse to count + one representative error — a mixed
        // three-failure batch used to be unrecoverable from `--json`.
        const typed = reported.filter(
          (f): f is { source: string; error: string } =>
            f != null &&
            typeof f === 'object' &&
            typeof (f as { source?: unknown }).source === 'string' &&
            typeof (f as { error?: unknown }).error === 'string',
        );
        const typedFailures: Array<{ source: string; reason: string; request_id?: string; code?: string }> = typed
          .map(({ source, error }) => ({ source, reason: error }))
          .map((f, i) => { const w = typed[i]; return written(w) ? { ...f, request_id: w.request_id, ...(typeof w.code === 'string' ? { code: w.code } : {}) } : f; });
        return {
          extracted: Number(d.atoms_extracted ?? 0),
          skipped: Number(d.duplicates_skipped ?? 0),
          providerFailure: failures.length > 0 && itemsSucceeded === 0,
          providerFailureClass: batchProviderFailureClass(d.aborted_global_error, typedFailures.filter(f => f.request_id === undefined)),
          failureCount: reported.length,
          failures: typedFailures,
          writeStrikes: reported.filter(f => written(f) && typeof f.strikes === 'number').length,
          budgetSkipped: Number(d.pages_skipped_budget ?? 0) + Number(d.transcripts_skipped_budget ?? 0),
          budget: { spentUsd: Number(d.estimated_spend_usd ?? 0), capUsd: Number(d.budget_usd ?? 0) },
          inputTruncated: Number(d.input_truncated ?? 0),
        };
      },
      countRemaining: (signal) => countExtractAtomsBacklog(engine, extractionSourceId, { signal }),
      now: Date.now,
      onBatch: opts.onBatch,
    },
    {
      windowMs: opts.windowSeconds * 1000,
      maxBatches: opts.maxBatches,
      signal: opts.signal,
      deadlineAtMs: opts.deadlineAtMs,
    },
  );
}