Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* issue #1678 — bounded single-hold drain for extract_atoms.
*
* The operator/agent escape hatch for a backlog the routine cycle won't touch
* (pack-gated off) or can't keep up with. Design per Codex #8/#9/#10:
*
* - SINGLE continuous lock hold (no release/reacquire between batches). The
* caller wraps the loop in `withRefreshingLock(cycleLockIdFor(sourceId))` —
* the SAME lock id the routine cycle uses for that source — so the two
* genuinely contend (no source-vs-legacy lock mismatch) and there's no
* release-gap where autopilot/sync could mutate pages mid-drain (which would
* let the drain extract atoms from stale content).
* - REDISCOVER eligibility each batch (the injected `runBatch` re-runs the
* NOT-EXISTS-on-source_hash discovery), so stale content simply doesn't
* match — no cross-window cursor of page lists.
* - BOUNDED by a wallclock window; reports `remaining` so a cron/agent loop
* knows whether to run again.
* - Two stop signals (#5809/#5832). HARD = caller signal (Minion job) + the
* lock's lease-loss signal + the job deadline: reaches the chat call, the
* pacing pause and every backlog count, and stops before the next commit.
* SOFT = hard + window: checked only between batches and items, so a
* window stop never aborts an in-flight paid call.
*
* Pure over injected deps: no DB, no LLM, no lock primitive imported here, so
* the loop logic is unit-testable. Its only static imports are pure text
* sanitizers (#4730) and the leaf signal combiner (`abort-signals.ts`). The
* wiring helper `runExtractAtomsDrainForSource` (below) builds the real deps;
* it uses DYNAMIC imports so the pure-loop unit tests don't drag in db-lock /
* cycle.
*/
import type { BrainEngine } from '../engine.ts';
import type { ExtractAtomsOpts } from './extract-atoms.ts';
import { redactConnectionInfo } from '../audit/redact-connection-info.ts';
import { redactFindings } from '../secret-scan.ts';
import { ensureWellFormed, truncateUtf8 } from '../text-safe.ts';
import { anyAbortSignal } from '../abort-signals.ts';
/** #4730: bounded operator-facing failure detail; totals stay exact above the cap. */
export const MAX_DRAIN_FAILURE_RECORDS = 25;
/** Matches the repo's audit/error-summary privacy cap. */
export const MAX_DRAIN_FAILURE_SOURCE_CHARS = 256;
export const MAX_DRAIN_FAILURE_REASON_CHARS = 200;
/**
* #4730: one preserved per-item failure. `failure_count` remains the exact
* total; `failures` holds up to MAX_DRAIN_FAILURE_RECORDS of these in batch
* order, and `omitted_failure_count` reconciles the difference — the cap is
* reported, never silently applied.
*/
export interface ExtractAtomsDrainFailure {
/** One-based batch number within this bounded drain window. */
batch: number;
/** Stable page slug / transcript locator emitted by runPhaseExtractAtoms (sanitized). */
source: string;
/** Bounded, sanitized failure reason (secrets/connection info redacted). */
reason: string;
/** #7514: a write that did not commit: its request id (`gbrain write-request` explains it) and the refusal's own code. */
request_id?: string;
code?: string;
}
/**
* Sanitize operator-facing failure text: secret + connection-string redaction,
* well-formed UTF-8, collapsed whitespace, bounded length. Locators and
* reasons both route through here so neither can carry an unbounded provider
* payload or credentials into `--json` output / job results.
*/
function sanitizeFailureText(raw: string, maxChars: number): string {
const secretRedacted = redactFindings(raw, { highEntropy: true }).text;
const connectionRedacted = redactConnectionInfo(secretRedacted);
return truncateUtf8(
ensureWellFormed(connectionRedacted).replace(/\s+/g, ' ').trim(),
maxChars,
);
}
export interface ExtractAtomsDrainDeps {
/**
* Run the loop body while holding the cycle lock. Implemented by the caller
* via `withRefreshingLock`. MUST throw when the lock is held by another
* process (e.g. `LockUnavailableError`) — the drain lets that propagate so
* the caller can report `cycle_already_running` and exit, matching the
* routine cycle's skip contract. `work` receives the lock's signal, which
* aborts when the lease is lost (#5832); the drain maps that loss, even
* when the helper rethrows it after `work` settled, to `stopped: 'lock_lost'`.
*/
withLock: <T>(work: (signal: AbortSignal) => Promise<T>) => Promise<T>;
/**
* Process one bounded batch (rediscovers eligibility). Returns counts, plus
* `providerFailure` (issue #3218) when EVERY item the batch attempted threw
* (zero items succeeded, at least one failure) — i.e. the batch's warning
* result was actually a total provider outage, not a partial/no-op batch.
* Omit/false for the ordinary partial-success or nothing-to-do cases.
*
* #4539: `failureCount` (per-item failures in this batch) and `firstError`
* (a representative failure message) let the drain surface WHY a run
* stopped/underperformed. #4730: `failures` preserves the phase's per-item
* `{source, reason}` records so a mixed-failure batch is fully
* reconcilable from `--json` — pre-#4730 everything but ONE representative
* error was dropped and the operator had to re-run the work to see the
* other reasons.
*
* `signal` is the hard stop and `stopSignal` the soft one (see the module
* header); the batch hands both to `runPhaseExtractAtoms`.
*
* #6425: `budgetSkipped` counts items the batch skipped because the
* attempt's per-run cap was spent, with `budget` the spend and the cap. The
* attempt shares one tracker across batches, so the drain stops `budget`.
*/
runBatch: (signals: { signal: AbortSignal; stopSignal: AbortSignal }) => Promise<{
extracted: number;
skipped: number;
providerFailure?: boolean;
failureCount?: number;
firstError?: string;
failures?: Array<{ source: string; reason: string; request_id?: string; code?: string }>;
/** #7514: write failures that recorded a strike toward their hold this batch; a batch of them is progress, not a stall. */
writeStrikes?: number;
budgetSkipped?: number;
budget?: { spentUsd: number; capUsd: number };
/** #6503: items the batch cut at `cycle.extract_atoms.max_input_chars` (partial disposition, not complete). */
inputTruncated?: number;
/** #6817: with `providerFailure`, what the outage was; unset reads as `transient`. */
providerFailureClass?: DrainProviderFailureClass;
}>;
/**
* Count remaining eligible-but-unextracted pages, or null on query error.
* `signal` fires on a hard stop or when the count outlives its bound; the
* drain stops waiting at that point even if the query ignores it.
*/
countRemaining: (signal: AbortSignal) => Promise<number | null>;
/** Injectable clock. Production: Date.now. */
now: () => number;
/** Optional progress sink (one line per batch). */
onBatch?: (info: { batch: number; extracted: number; remaining: number | null }) => void;
}
export interface ExtractAtomsDrainOpts {
/**
* Wallclock window in ms, a hard deadline for starting work: no batch or
* item starts after it, while the item in flight finishes (soft signal).
*/
windowMs: number;
/** Hard cap on batches (belt-and-suspenders against a 0-progress loop). Default 1000. */
maxBatches?: number;
/** Caller's cancellation signal (the Minion job's timeout/cancel signal, #5809). */
signal?: AbortSignal;
/** Absolute job deadline (epoch ms, `deadlineAtMs` clock); a hard stop at that instant. */
deadlineAtMs?: number | null;
/** Upper bound on one backlog count; the remaining deadline caps it further. Default 60 s. */
countTimeoutMs?: number;
}
/**
* Why a drain stopped. `window`/`no_progress`/`max_batches`/`budget` are soft
* stops after a final recount (`budget`: the attempt's per-run cap is spent); `deadline` (job deadline), `lock_lost` (cycle lock
* lease lost) and `aborted` (job cancelled or shutting down) are hard stops
* whose `remaining` is the last count taken before the stop.
*/
export type DrainStop =
| 'drained' | 'window' | 'deadline' | 'lock_lost' | 'aborted'
| 'no_progress' | 'max_batches' | 'provider_failure' | 'budget';
/**
* #6817: why every item of a provider_failure batch failed. `auth`, `billing`,
* `model_not_found` and `content_block` repeat on every retry until someone
* changes the key, plan, model or content; `rate_limit` and `transient`
* (5xx, timeout, network, an unclassified error) can clear on their own.
*/
export type DrainProviderFailureClass = 'auth' | 'billing' | 'model_not_found' | 'content_block' | 'rate_limit' | 'transient';
export const DRAIN_COUNT_TIMEOUT_MS = 60_000;
/**
* #5856: dead-letter text of a drain the atom session preflight refused before any spend
* (`owner_unavailable` / `permission_denied`). The auto-drain cap does not count such a job.
*/
export const STRUCTURAL_REFUSAL_PREFIX = 'structural_refusal:';
/** The non-retryable dead-letter text for a structural preflight refusal, or null for any other error. */
export function structuralAtomRefusal(error: unknown): string | null {
if (!(error instanceof Error) || error.name !== 'OperationError') return null;
const code = (error as Error & { code?: unknown }).code;
return code === 'owner_unavailable' || code === 'permission_denied' ? `${STRUCTURAL_REFUSAL_PREFIX} ${code}: ${error.message}` : null;
}
/** setTimeout fires at once past 2^31-1 ms; a longer window or deadline waits the maximum. */
const MAX_TIMER_MS = 2 ** 31 - 1;
export interface ExtractAtomsDrainResult {
phase: 'extract_atoms';
/**
* issue #3218: 'provider_failure' when any batch reported `providerFailure`
* (every item it attempted errored). The Minion handler throws on this
* status so the durable job retries instead of completing over a backlog
* that made zero forward progress. Partial-success batches (>=1 item
* succeeded) always report 'ok', unchanged from before.
*/
status: 'ok' | 'provider_failure';
extracted: number;
skipped: number;
/** Eligible pages still pending (see DrainStop for hard stops). null if no count succeeded. */
remaining: number | null;
/** Batches actually processed. */
batches: number;
stopped: DrainStop;
/**
* #4539: total per-item failures across every batch in this run. 0 for a
* clean run. Included in `--json` verbatim; dream.ts prints a stderr line
* when non-zero so the operator sees WHY the drain underperformed.
*/
failure_count: number;
/**
* #4730: bounded per-item failure details, in batch order, capped at
* MAX_DRAIN_FAILURE_RECORDS. Locators and reasons are sanitized (secret +
* connection-info redaction, bounded length). Rides `--json` verbatim.
*/
failures: ExtractAtomsDrainFailure[];
/**
* #4730: failures beyond the record cap (or reported by count only, with
* no per-item detail). `failure_count === failures.length +
* omitted_failure_count` always holds, so the cap is visible, never silent.
*/
omitted_failure_count: number;
/**
* #4539: representative failure message from the most recent batch that
* reported one (`source: reason`), or null for a clean run.
*/
last_error: string | null;
/** #6425: on a `budget` stop, the attempt's spend and its per-run cap (`cycle.extract_atoms.budget_usd`). */
spent_usd?: number;
budget_usd?: number;
/** #6503: items cut at the input cap across every batch; the model did not read their tails and they are not marked complete. */
input_truncated?: number;
/** #6817: with `status: 'provider_failure'`, the failing batch's class (`transient` when the batch named none). */
provider_failure_class?: DrainProviderFailureClass;
}
/**
* #3813: the Minion handler's provider_failure throw IS the job's error_text
* once it dead-letters, so it carries the representative `last_error` (already
* secret-redacted + bounded above) — a missing provider key is a one-line
* diagnosis instead of an opaque batches/remaining.
*/
export function formatDrainProviderFailure(
result: Pick<ExtractAtomsDrainResult, 'batches' | 'remaining' | 'last_error'>,
opts: { final?: boolean } = {},
): string {
return (
`extract-atoms-drain: all provider calls failed this batch ` +
`(batches=${result.batches}, remaining=${result.remaining ?? '?'})` +
`${result.last_error ? `; last error: ${result.last_error}` : ''}${opts.final ? '' : ' — retrying'}`
);
}
export async function runExtractAtomsDrain(
deps: ExtractAtomsDrainDeps,
opts: ExtractAtomsDrainOpts,
): Promise<ExtractAtomsDrainResult> {
const maxBatches = opts.maxBatches ?? 1000;
const countTimeoutMs = opts.countTimeoutMs ?? DRAIN_COUNT_TIMEOUT_MS;
// Counters live outside the locked work so a lease loss the lock helper
// rethrows after `work` settled still reports the partial progress.
let extracted = 0;
let skipped = 0;
let batches = 0;
// issue #3218: latched once any batch reports providerFailure — drives
// the returned `status`, independent of how `stopped` reads after the
// final (possibly overriding) remaining-count check below.
let providerFailure = false;
let providerFailureClass: DrainProviderFailureClass = 'transient';
// #4539: accumulate per-item failure visibility across batches.
let failureCount = 0;
// #4730: bounded typed per-item records (batch order, sanitized).
const failures: ExtractAtomsDrainFailure[] = [];
let lastError: string | null = null;
let lastCount: number | null = null;
let lockSignal: AbortSignal | null = null;
let budget: { spentUsd: number; capUsd: number } | undefined;
let inputTruncated = 0;
const result = (stopped: DrainStop, remaining: number | null): ExtractAtomsDrainResult => ({
phase: 'extract_atoms',
status: providerFailure ? 'provider_failure' : 'ok',
extracted,
skipped,
remaining,
batches,
stopped,
failure_count: failureCount,
failures,
omitted_failure_count: failureCount - failures.length,
last_error: lastError,
...(stopped === 'budget' && budget ? { spent_usd: budget.spentUsd, budget_usd: budget.capUsd } : {}),
input_truncated: inputTruncated,
...(providerFailure ? { provider_failure_class: providerFailureClass } : {}),
});
try {
return await deps.withLock(async (signal) => {
lockSignal = signal;
const windowEnd = deps.now() + opts.windowMs;
const deadlineStop = new AbortController();
const windowStop = new AbortController();
const deadlineTimer = opts.deadlineAtMs == null ? undefined : setTimeout(
() => deadlineStop.abort(new Error('extract-atoms-drain: job deadline reached')),
Math.min(MAX_TIMER_MS, Math.max(0, opts.deadlineAtMs - deps.now())),
);
const windowTimer = setTimeout(
() => windowStop.abort(new Error('extract-atoms-drain: window elapsed')),
Math.min(MAX_TIMER_MS, opts.windowMs),
);
deadlineTimer?.unref?.();
windowTimer.unref?.();
const hard = anyAbortSignal([...(opts.signal ? [opts.signal] : []), signal, deadlineStop.signal]);
const soft = anyAbortSignal([hard.signal, windowStop.signal]);
const hardStop = (): DrainStop => signal.aborted ? 'lock_lost'
: deadlineStop.signal.aborted || (opts.deadlineAtMs != null && deps.now() >= opts.deadlineAtMs) ? 'deadline'
: 'aborted';
// E15: a count that blocks must not keep the lock refreshing, so it is
// bounded by the remaining deadline and abandoned on a hard stop.
const count = async (): Promise<number | null> => {
const bound = opts.deadlineAtMs == null ? countTimeoutMs
: Math.min(countTimeoutMs, Math.max(0, opts.deadlineAtMs - deps.now()));
const timeout = new AbortController();
const timer = setTimeout(() => timeout.abort(new Error('extract-atoms-drain: backlog count timed out')), bound);
const bounded = anyAbortSignal([hard.signal, timeout.signal]);
try {
if (bounded.signal.aborted) return null;
return await new Promise<number | null>((resolve, reject) => {
bounded.signal.addEventListener('abort', () => resolve(null), { once: true });
deps.countRemaining(bounded.signal).then(resolve, reject);
});
} finally {
clearTimeout(timer);
bounded.dispose();
}
};
try {
let stopped: DrainStop = 'window';
while (deps.now() < windowEnd && !soft.signal.aborted) {
if (batches >= maxBatches) { stopped = 'max_batches'; break; }
const before = await count();
if (hard.signal.aborted) break;
if (before !== null) lastCount = before;
if (before === 0) { stopped = 'drained'; break; }
const r = await deps.runBatch({ signal: hard.signal, stopSignal: soft.signal });
extracted += r.extracted;
skipped += r.skipped;
inputTruncated += Math.max(0, Math.floor(Number(r.inputTruncated ?? 0)) || 0);
batches++;
// #4730: preserve typed per-item records (bounded, sanitized) while
// keeping failure_count exact and reconcilable — count-only adapters
// (the #4539 shape) still contribute to the total via failureCount.
const batchFailures = Array.isArray(r.failures)
? r.failures.filter(
(f): f is { source: string; reason: string; request_id?: string; code?: string } =>
f != null &&
typeof f === 'object' &&
typeof f.source === 'string' &&
typeof f.reason === 'string',
)
: [];
const reportedFailureCount =
typeof r.failureCount === 'number' && Number.isFinite(r.failureCount) && r.failureCount > 0
? Math.floor(r.failureCount)
: 0;
failureCount += Math.max(reportedFailureCount, batchFailures.length);
for (const f of batchFailures) {
if (failures.length >= MAX_DRAIN_FAILURE_RECORDS) break;
failures.push({
batch: batches,
source: sanitizeFailureText(f.source, MAX_DRAIN_FAILURE_SOURCE_CHARS),
reason: sanitizeFailureText(f.reason, MAX_DRAIN_FAILURE_REASON_CHARS),
...(typeof f.request_id === 'string' ? { request_id: sanitizeFailureText(f.request_id, 64) } : {}),
...(typeof f.code === 'string' ? { code: sanitizeFailureText(f.code, 64) } : {}),
});
}
const representative = batchFailures[0];
if (representative) {
lastError =
`${sanitizeFailureText(representative.source, MAX_DRAIN_FAILURE_SOURCE_CHARS)}: ` +
`${sanitizeFailureText(representative.reason, MAX_DRAIN_FAILURE_REASON_CHARS)}`;
} else if (typeof r.firstError === 'string' && r.firstError.trim()) {
// #4539 compatibility: count-only adapters still surface their
// representative error — through the SAME sanitizer as the typed
// records (secret/DSN redaction, whitespace collapse, bounded), so a
// provider payload cannot ride the fallback path into --json output.
lastError = sanitizeFailureText(
r.firstError,
MAX_DRAIN_FAILURE_SOURCE_CHARS + 2 + MAX_DRAIN_FAILURE_REASON_CHARS,
);
}
if (hard.signal.aborted) break;
deps.onBatch?.({ batch: batches, extracted: r.extracted, remaining: before });
// issue #3218: every item this batch attempted failed (0 succeeded, >=1
// error) — a total provider outage, not ordinary no-op/partial progress.
// Stop immediately (same hot-loop guard as no_progress below) and flag
// it so the caller can retry via its own policy instead of treating the
// drain as a clean completion.
if (r.providerFailure) {
providerFailure = true;
providerFailureClass = r.providerFailureClass ?? 'transient';
stopped = 'provider_failure';
break;
}
// #6425: the attempt's one BudgetTracker is spent, so no later batch can progress.
if ((r.budgetSkipped ?? 0) > 0) {
budget = r.budget;
stopped = 'budget';
break;
}
// Stop if a batch made zero forward progress — extraction is failing or
// everything left is ineligible (e.g. all skipped). Prevents a hot loop
// that spends budget without draining.
//
// #2144: a zero-ATOM batch can still be progress — tombstoned
// zero-yield pages shrink the backlog without producing atoms. Only
// stop when the backlog count genuinely didn't move.
//
// #8445: a skip-only batch gets the same recount. An item the phase
// skips but the backlog still counts re-skipped every batch, so the
// drain ran to max_batches holding the cycle lock.
if (r.extracted === 0) {
// #7826: the window closed before this batch started its first item; that is a window stop, not a stall.
if (soft.signal.aborted) { stopped = 'window'; break; }
const after = await count();
if (hard.signal.aborted) break;
if (after !== null) lastCount = after;
if (after === null || before === null || (after >= before && !((r.writeStrikes ?? 0) > 0))) { stopped = 'no_progress'; break; }
}
}
// Hard stops write nothing and skip the recount (it would outlive the
// deadline or run without the lock); `remaining` is the last count.
if (hard.signal.aborted) return result(hardStop(), lastCount);
const remaining = await count();
if (hard.signal.aborted) return result(hardStop(), lastCount);
lastCount = remaining;
// issue #3218 (codex P2): don't let a final remaining===0 recount
// overwrite 'provider_failure' back to 'drained' — that would report the
// contradictory {status: 'provider_failure', stopped: 'drained'} and
// mislead the CLI/JSON consumer (dream.ts prints both fields verbatim).
// status already takes precedence for the Minion handler's retry
// decision; keep `stopped` consistent with it once a failure latched.
if (!providerFailure && remaining === 0) stopped = 'drained';
return result(stopped, remaining);
} finally {
clearTimeout(deadlineTimer);
clearTimeout(windowTimer);
soft.dispose();
hard.dispose();
}
});
} catch (err) {
// E8: `withRefreshingLock` rethrows the recorded lease loss from its
// finally, replacing whatever `work` returned or threw.
const lock = lockSignal as AbortSignal | null;
if (lock?.aborted && err === lock.reason) return result('lock_lost', lastCount);
throw err;
}
}
/**
* #6817: classify a batch whose every item failed. The phase's whole-run halt
* class (`details.aborted_global_error`) wins; a batch whose every failure is
* a provider content block is `content_block`; anything else is `transient`,
* because nothing shows that retrying cannot succeed.
*/
export function batchProviderFailureClass(halted: unknown, failures: ReadonlyArray<{ reason: string }>): DrainProviderFailureClass {
if (halted === 'auth' || halted === 'billing' || halted === 'model_not_found' || halted === 'rate_limit') return halted;
if (failures.length > 0 && failures.every(f => f.reason.startsWith('provider blocked content'))) return 'content_block';
return 'transient';
}
// ─── Shared wiring helper (v0.42.x #1685 DECISION 5A) ──────────────────────
//
// ONE drain path, three callers: `gbrain dream --phase extract_atoms --drain`
// (dream.ts), the `extract-atoms-drain` Minion handler (jobs.ts), and the
// autopilot auto-drain submission (which routes through the handler). Before
// this helper the lock/batch/count wiring lived inline in dream.ts:482; a second
// copy in the handler would let lock id / window default / defer-on-lock-busy
// drift. Keeping the wiring here means those three callers can't diverge.
//
// Imports are dynamic so the pure `runExtractAtomsDrain` above stays cheap to
// import in unit tests (no db-lock / cycle / extract-atoms in the static graph).
//
// `LockUnavailableError` is NOT caught here — the pure loop's `withLock`
// (withRefreshingLock) throws it and it propagates to the caller, because each
// caller reports the busy-lock case differently (dream → exit 3;
// handler → `{ deferred: true }`). That matches the contract documented on
// `ExtractAtomsDrainDeps.withLock`.
export interface DrainForSourceOpts {
/**
* The RESOLVED source id, or `undefined` for the legacy unscoped cycle.
* `undefined` → `cycleLockIdFor(undefined)` = the bare `gbrain-cycle` lock the
* unscoped routine cycle holds; a real id → `gbrain-cycle:<id>`. Either way the
* drain and the routine cycle for THIS source genuinely contend (Codex #9).
* The extraction/backlog source is `sourceId ?? 'default'`.
*/
sourceId: string | undefined;
/** Wallclock budget in seconds. */
windowSeconds: number;
/** Brain checkout dir, threaded to `runPhaseExtractAtoms` (optional — DB-only ok). */
brainDir?: string;
/** Hard batch cap (belt-and-suspenders). */
maxBatches?: number;
/** Optional per-batch progress sink (stderr line in dream; job progress in the handler). */
onBatch?: ExtractAtomsDrainDeps['onBatch'];
/** #5809: the Minion job's signal (see ExtractAtomsDrainOpts.signal). */
signal?: AbortSignal;
/** #5809: the Minion job's `deadlineAtMs`. */
deadlineAtMs?: number | null;
}
export async function runExtractAtomsDrainForSource(
engine: BrainEngine,
opts: DrainForSourceOpts,
): Promise<ExtractAtomsDrainResult> {
const { withRefreshingLock } = await import('../db-lock.ts');
const { countExtractAtomsBacklog } = await import('./extract-atoms.ts');
const { runPhaseExtractAtomsStamped } = await import('./extract-atoms-stamp.ts');
const { cycleLockIdFor } = await import('../cycle.ts');
const { MaintenanceWriteWait } = await import('../persistence/maintenance-wait.ts');
const extractionSourceId = opts.sourceId ?? 'default';
const lockId = cycleLockIdFor(opts.sourceId);
// #5856/#5854: one call is one attempt; its batches share one BudgetTracker
// (the per-run cap holds for the whole attempt) and one publish wait.
const attempt: NonNullable<ExtractAtomsOpts['attempt']> = { writeWait: new MaintenanceWriteWait(opts.deadlineAtMs ?? null) };
return runExtractAtomsDrain(
{
withLock: (work) => withRefreshingLock(engine, lockId, (signal) => work(signal), { ttlMinutes: 5 }),
runBatch: async ({ signal, stopSignal }) => {
const r = await runPhaseExtractAtomsStamped(engine, {
sourceId: extractionSourceId,
dryRun: false,
brainDir: opts.brainDir,
signal,
stopSignal,
attempt,
});
const d = (r.details ?? {}) as Record<string, unknown>;
// issue #3218: `r.status` collapses to 'warn' whether ONE item failed
// (partial success — leave the drain's existing ok/no_progress path
// alone) or EVERY item failed (a total provider outage the drain
// adapter was silently swallowing). Re-derive the total-failure case
// from the per-item counts `runPhaseExtractAtoms` already returns:
// >=1 failure AND zero items successfully processed (transcripts_processed
// + pages_processed both 0 means every attempted `chat()` call threw —
// items that succeed with 0 atoms still count as processed, so this
// does not fire on "provider fine, nothing extractable").
const reported: unknown[] = Array.isArray(d.failures) ? d.failures : [];
// #7514: a write that did not commit (its record names the request id) never reached a provider, so
// `failures` is the provider side only: a write failure neither makes nor classes a provider failure.
const written = (f: unknown): f is { request_id: string; code?: unknown; strikes?: unknown } =>
f != null && typeof f === 'object' && typeof (f as { request_id?: unknown }).request_id === 'string';
const failures = reported.filter(f => !written(f));
const itemsSucceeded =
Number(d.transcripts_processed ?? 0) + Number(d.pages_processed ?? 0);
// #4730: carry EVERY per-item failure up as a typed {source, reason}
// record (the pure loop bounds + sanitizes them) instead of the
// #4539 collapse to count + one representative error — a mixed
// three-failure batch used to be unrecoverable from `--json`.
const typed = reported.filter(
(f): f is { source: string; error: string } =>
f != null &&
typeof f === 'object' &&
typeof (f as { source?: unknown }).source === 'string' &&
typeof (f as { error?: unknown }).error === 'string',
);
const typedFailures: Array<{ source: string; reason: string; request_id?: string; code?: string }> = typed
.map(({ source, error }) => ({ source, reason: error }))
.map((f, i) => { const w = typed[i]; return written(w) ? { ...f, request_id: w.request_id, ...(typeof w.code === 'string' ? { code: w.code } : {}) } : f; });
return {
extracted: Number(d.atoms_extracted ?? 0),
skipped: Number(d.duplicates_skipped ?? 0),
providerFailure: failures.length > 0 && itemsSucceeded === 0,
providerFailureClass: batchProviderFailureClass(d.aborted_global_error, typedFailures.filter(f => f.request_id === undefined)),
failureCount: reported.length,
failures: typedFailures,
writeStrikes: reported.filter(f => written(f) && typeof f.strikes === 'number').length,
budgetSkipped: Number(d.pages_skipped_budget ?? 0) + Number(d.transcripts_skipped_budget ?? 0),
budget: { spentUsd: Number(d.estimated_spend_usd ?? 0), capUsd: Number(d.budget_usd ?? 0) },
inputTruncated: Number(d.input_truncated ?? 0),
};
},
countRemaining: (signal) => countExtractAtomsBacklog(engine, extractionSourceId, { signal }),
now: Date.now,
onBatch: opts.onBatch,
},
{
windowMs: opts.windowSeconds * 1000,
maxBatches: opts.maxBatches,
signal: opts.signal,
deadlineAtMs: opts.deadlineAtMs,
},
);
}