Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
47.8 KB
Raw Blame
/**
 * gbrain enrich — batch enrichment primitive (issue #1700).
 *
 * 93.6% of people/company pages are stubs. There was no first-class way to
 * develop them at scale — you drove the agent-only `enrich` SKILL one page at a
 * time, or hand-rolled SQL + a bash fan-out. This command closes that gap with
 * BRAIN-INTERNAL GROUNDED SYNTHESIS:
 *
 *   1. `engine.listEnrichCandidates` enumerates thin pages, ordered by inbound
 *      links (the headline signal — most-referenced stubs first), source-aware
 *      and memory-bounded (lightweight projection, no bodies).
 *   2. For each candidate, deterministically retrieve everything the brain
 *      ALREADY knows about the entity (hybrid search on its name, inbound-link
 *      context, facts, the existing stub) — no web, no external tools.
 *   3. One grounded LLM call consolidates that context into a real, cited page.
 *      If the brain knows too little, SKIP rather than fabricate.
 *
 * Why brain-internal: gbrain's own LLM tooling can only see brain tools
 * (search/get_page/facts). External research (web/LinkedIn/Perplexity) is a
 * host-agent capability and stays the agent-driven `enrich` SKILL's job.
 *
 * Resumable (op-checkpoint), budget-capped (best-effort under --workers; pin
 * --workers 1 for an exact ceiling), per-page advisory-locked (no double-spend
 * across parallel workers / processes), and parallel (--workers K).
 *
 * Architecture mirrors `extract-conversation-facts.ts` (the closest precedent):
 * strict per-source core, optional externally-managed BudgetTracker, string-
 * encoded op-checkpoint resume state, and a `--background` Minion path that
 * fans out one job per source when --source is omitted.
 */

import { randomUUID } from 'node:crypto';
import type { BrainEngine } from '../core/engine.ts';
import type { EnrichCandidate, PageType } from '../core/types.ts';
import { operations, OperationError } from '../core/operations.ts';
import { maintenancePreflight, publishMaintenancePage, type MaintenanceAuthority } from '../core/persistence/prepared-maintenance.ts';
import { preserveCanonicalFences } from '../core/cycle/concept-publication.ts';
import type { WriteReceipt } from '../core/persistence/types.ts';
import type { OperationContext } from '../core/operations.ts';
import { configureGatewayIfUninitialized, isAvailable, chat, getChatModel, withBudgetTracker, isThinkingModel, THINKING_MODEL_MAX_OUTPUT_TOKENS } from '../core/ai/gateway.ts';
import { BudgetTracker, BudgetExhausted, loadPricingOverrides, type BudgetReason, type NoPricingGuidance } from '../core/budget/budget-tracker.ts';
import { noPricingSteps } from '../core/budget/no-pricing.ts';
import { CapFlagError, mergeCapFlag, parseCapFlag, type CapFlag } from '../core/budget/cap-flag.ts';
import { ERROR_CATALOGUE } from '../core/error-catalogue.ts';
import { hybridSearch } from '../core/search/hybrid.ts';
import { INTERNAL_BREADTH_SEARCH_OPTS } from '../core/search/internal-breadth.ts';
import { serializeMarkdown } from '../core/markdown.ts';
import { listSources } from '../core/sources-ops.ts';
import {
  loadOpCheckpoint,
  recordCompleted,
  clearOpCheckpoint,
  fingerprint,
  type OpCheckpointKey,
} from '../core/op-checkpoint.ts';
import { createProgress } from '../core/progress.ts';
import { consentGateOrExit, engineConsentEnv, tokenmaxUncappedEnv } from '../core/consent-cli.ts';
import { derivedCapUsd } from '../core/consent.ts';
import { jobSpendAuthorization, spendSubmitSummary, type SpendAuthorization } from '../core/minions/spend-authorization.ts';
import { getCliOptions, cliOptsToProgressOptions, maybeBackground } from '../core/cli-options.ts';
import { loadConfig } from '../core/config.ts';
import { isMustAbortError, runSlidingPool } from '../core/worker-pool.ts';
import { parseWorkers, resolveWorkersWithClamp } from '../core/sync-concurrency.ts';
import { withRefreshingLock, LockUnavailableError } from '../core/db-lock.ts';
import {
  DEFAULT_THIN_THRESHOLD,
  MIN_CONTEXT_CHARS,
  inferEnrichKind,
  renderEvidence,
  assessGrounding,
  buildEnrichPrompt,
  parseSynthesis,
  type EnrichEvidence,
} from '../core/enrich/thin.ts';
import { restoreNeutralized } from '../core/think/sanitize.ts';

// ---------------------------------------------------------------------------
// Tunables (exported for tests).
// ---------------------------------------------------------------------------

export const DEFAULT_LIMIT = 50;
export const DEFAULT_TYPES: PageType[] = ['person', 'company'];
export const DEFAULT_MAX_COST_USD = 5.0;
/** Default re-enrich window: skip pages enriched within the last 30 days. */
export const DEFAULT_REENRICH_DAYS = 30;
/** Per-page advisory lock TTL. withRefreshingLock refreshes at 1/6 the TTL. */
export const PER_PAGE_LOCK_TTL_MINUTES = 2;
export const CHECKPOINT_OP = 'enrich';
/** Frontmatter provenance marker. Survives put_page write-through (which only
 *  overrides ingested_via / ingested_at / source_kind). */
export const ENRICHED_BY = 'cli:enrich';
/** Retrieval fan-out caps (keep evidence bounded). */
export const HYBRID_SEARCH_LIMIT = 8;
export const BACKLINK_LIMIT = 12;
export const FACT_LIMIT = 20;
/** #2085: per-chunk clamp for hybrid evidence. One long chunk must not eat
 *  the whole render window (MAX_CONTEXT_CHARS) and evict facts/backlinks. */
export const HYBRID_CHUNK_CLAMP_CHARS = 2000;
/** Flush the resume checkpoint every N completions during a long run. */
const CHECKPOINT_FLUSH_EVERY = 25;
/** Rough per-page cost estimate (USD) for the dry-run preview. */
const COST_ESTIMATE_PER_PAGE_USD = 0.01;

export const ENRICH_ORDERS = ['inbound-links', 'salience', 'updated'] as const;
export type EnrichOrder = (typeof ENRICH_ORDERS)[number];

// ---------------------------------------------------------------------------
// Public types.
// ---------------------------------------------------------------------------

/**
 * DI seam for hermetic tests. Returns the model's raw synthesis text.
 * Default implementation calls the gateway; tests inject a stub so the full
 * pipeline runs with no API key (and stays parallel-safe — no mock.module).
 */
export type SynthesizeFn = (input: {
  system: string;
  user: string;
  model: string;
  abortSignal?: AbortSignal;
}) => Promise<string>;

/** Strict per-source core opts. Multi-source iteration is the caller's job. */
export interface EnrichCoreOpts {
  /** REQUIRED. Strict per-source contract. */
  sourceId: string;
  types?: PageType[];
  order?: EnrichOrder;
  limit?: number;
  /** In-process parallel workers. Default 1; PGLite clamps to 1. */
  workers?: number;
  /** Chat model override (provider:model). Default = configured chat model. */
  model?: string;
  /** Body char-length below which a page is "thin". */
  thinThreshold?: number;
  /** Minimum retrieved-context chars to attempt synthesis (no LLM below it). */
  minContextChars?: number;
  /** Skip pages enriched within this many ms. Default DEFAULT_REENRICH_DAYS. */
  reenrichAfterMs?: number;
  /** Cost cap (USD) when budgetTracker is NOT passed. Default DEFAULT_MAX_COST_USD. */
  maxCostUsd?: number;
  /** Externally-managed tracker. If present, used as-is (no withBudgetTracker wrap). */
  budgetTracker?: BudgetTracker;
  /** Preview only: count candidates + grounding decisions; no LLM, no write. */
  dryRun?: boolean;
  /** Clear this source's resume checkpoint before processing. */
  force?: boolean;
  /** Test seam — inject synthesis so tests skip the real gateway. */
  synthesizeFn?: SynthesizeFn;
}

export interface EnrichResult {
  candidates_considered: number;
  pages_enriched: number;
  /** Skipped because the brain knew too little (pre-LLM gate OR model SKIP).
   *  Legacy compat: equals pre_llm + model_skip + empty_output (#2085). */
  pages_skipped_insufficient: number;
  /** #2085 split: the grounding gate refused BEFORE any LLM call (no spend). */
  pages_skipped_pre_llm?: number;
  /** #2085 split: the model explicitly answered SKIP (a grounding verdict). */
  pages_model_skip?: number;
  /** #2085 split: the model returned blank output (a synthesis failure shape). */
  pages_empty_output?: number;
  /** Skipped because another worker/process held the per-page lock. */
  pages_skipped_lock: number;
  /** Skipped because the page disappeared between enumeration and fetch. */
  pages_skipped_disappeared: number;
  /** Synthesis or write errors (best-effort; pool continued). */
  pages_failed: number;
  /** Dry-run only: candidates that WOULD be enriched (passed grounding). */
  would_enrich?: number;
  spent_usd?: number;
  budget_exhausted?: boolean;
  /** Why the run aborted when budget_exhausted (#4032): BudgetExhausted.reason.
   *  'no_pricing' means the cap was never the problem — raising --max-usd won't help. */
  budget_exhausted_reason?: BudgetReason;
  /** Model that triggered a no_pricing abort, when the tracker knew it (#4032). */
  budget_exhausted_model?: string;
  /** no_pricing abort: the lookup-and-register guidance (model, provider, kind, units, command, docs). */
  budget_exhausted_pricing?: NoPricingGuidance;
  /** #2504 — first pool failure ('slug: message'), so pages_failed > 0 always
   *  carries a WHY (pool.failures was previously write-only). */
  first_failure?: string;
  /** Accepted publication IDs remain inspectable after a pending or failed run. */
  write_requests?: WriteReceipt[];
}

// ---------------------------------------------------------------------------
// Fingerprint — dimensions that change the candidate set OR the synthesis.
// Local to this command (matches the extract-conversation-facts precedent;
// no op-checkpoint.ts coupling). Source + types + order + thinThreshold +
// model: a change in any of these is a genuinely different run.
// ---------------------------------------------------------------------------

export function enrichFingerprint(opts: {
  sourceId: string;
  types: PageType[];
  order: EnrichOrder;
  thinThreshold: number;
  model: string;
  /**
   * #3629 residual: the pre-LLM grounding gate decides against exactly this
   * knob, so a banked skip must not outlive a relaxed --min-context — a
   * non-default value is a genuinely different run. Folded only when it
   * differs from MIN_CONTEXT_CHARS so every existing default-config
   * checkpoint keeps its key (no one-time re-bill on upgrade).
   */
  minContextChars?: number;
}): string {
  return fingerprint({
    sourceId: opts.sourceId,
    types: [...opts.types].sort(),
    order: opts.order,
    thinThreshold: opts.thinThreshold,
    model: opts.model,
    ...(opts.minContextChars !== undefined && opts.minContextChars !== MIN_CONTEXT_CHARS
      ? { minContextChars: opts.minContextChars }
      : {}),
  });
}

function checkpointKey(fp: string): OpCheckpointKey {
  return { op: CHECKPOINT_OP, fingerprint: fp };
}

function completedKey(sourceId: string, slug: string): string {
  return `${sourceId}|${slug}`;
}

// ---------------------------------------------------------------------------
// Default synthesis via the gateway.
// ---------------------------------------------------------------------------

/** No prompt cache: the system prompt is below the smallest cacheable prefix, so a cache would only mark this page's evidence (#9541). */
export const defaultSynthesize: SynthesizeFn = async ({ system, user, model, abortSignal }) => {
  const res = await chat({
    model,
    system,
    messages: [{ role: 'user', content: user }],
    maxTokens: isThinkingModel(model) ? THINKING_MODEL_MAX_OUTPUT_TOKENS : 2048,
    abortSignal,
  });
  return res.text;
};

// ---------------------------------------------------------------------------
// Retrieval — deterministic, brain-internal. No LLM.
// ---------------------------------------------------------------------------

/**
 * #2085: assemble evidence in PRIORITY order — facts, then backlinks, then
 * hybrid chunks. renderEvidence packs whole blocks in order into a fixed
 * window (MAX_CONTEXT_CHARS); pre-fix, hybrid came first and a few long
 * chunks consumed the window, pushing short high-signal facts/backlinks out
 * of the prompt — fact-rich stub pages false-SKIPped. Hybrid chunks are
 * additionally clamped per item. Pure — exported as the test seam.
 */
export function assembleEvidence(parts: {
  facts: EnrichEvidence[];
  backlinks: EnrichEvidence[];
  hybrid: EnrichEvidence[];
}): EnrichEvidence[] {
  return [
    ...parts.facts,
    ...parts.backlinks,
    ...parts.hybrid.map((h) => ({ ...h, text: h.text.slice(0, HYBRID_CHUNK_CLAMP_CHARS) })),
  ];
}

async function retrieveEvidence(
  engine: BrainEngine,
  sourceId: string,
  slug: string,
  title: string,
): Promise<EnrichEvidence[]> {
  const facts: EnrichEvidence[] = [];
  const backlinks: EnrichEvidence[] = [];
  const hybrid: EnrichEvidence[] = [];
  const seen = new Set<string>();

  // 1. Facts the brain has extracted about this entity (highest signal/char).
  try {
    const rows = await engine.executeRaw<{ fact: string; context: string | null }>(
      `SELECT fact, context FROM facts
        WHERE source_id = $1 AND entity_slug = $2 AND expired_at IS NULL
          AND (valid_until IS NULL OR valid_until > now())
        ORDER BY confidence DESC, id DESC
        LIMIT $3`,
      [sourceId, slug, FACT_LIMIT],
    );
    for (const r of rows) {
      const text = r.context ? `${r.fact} (${r.context})` : r.fact;
      facts.push({ source_slug: slug, text });
    }
  } catch {
    // Pre-facts brains / column drift → no facts evidence.
  }

  // 2. Inbound-link context — how OTHER pages describe this entity.
  try {
    const rows = await engine.getBacklinks(slug, { sourceId });
    let n = 0;
    for (const l of rows) {
      if (n >= BACKLINK_LIMIT) break;
      const ctx = (l.context ?? '').trim();
      if (!ctx) continue;
      const dedup = `${l.from_slug}:${ctx.slice(0, 40)}`;
      if (seen.has(dedup)) continue;
      seen.add(dedup);
      backlinks.push({ source_slug: l.from_slug, text: ctx });
      n++;
    }
  } catch {
    // ignore
  }

  // 3. Hybrid search on the entity name — pages that mention it.
  try {
    const hits = await hybridSearch(engine, title || slug, {
      ...INTERNAL_BREADTH_SEARCH_OPTS,
      limit: HYBRID_SEARCH_LIMIT,
      sourceId,
    });
    for (const h of hits) {
      if (h.slug === slug) continue; // don't feed the stub its own body twice
      const dedup = `${h.slug}:${h.chunk_text.slice(0, 40)}`;
      if (seen.has(dedup)) continue;
      seen.add(dedup);
      if (h.chunk_text && h.chunk_text.trim()) {
        hybrid.push({ source_slug: h.slug, text: h.chunk_text });
      }
    }
  } catch {
    // Search unavailable (no embeddings) → fall through to other signals.
  }

  return assembleEvidence({ facts, backlinks, hybrid });
}

// ---------------------------------------------------------------------------
// Per-page enrich (runs inside the worker pool, under a per-page lock).
// ---------------------------------------------------------------------------

interface EnrichOneCtx {
  engine: BrainEngine;
  sourceId: string;
  model: string;
  minContextChars: number;
  dryRun: boolean;
  synthesizeFn: SynthesizeFn;
  result: EnrichResult;
  done: Set<string>;
  signal?: AbortSignal;
  config: ReturnType<typeof loadConfig>;
  /** Managed brains publish through the maintenance coordinator (#5280); null when unmanaged. */
  maintenance: MaintenanceAuthority | null;
}

async function enrichOne(ctx: EnrichOneCtx, candidate: EnrichCandidate): Promise<void> {
  const { engine, sourceId } = ctx;
  const slug = candidate.slug;
  const lockId = `enrich:${sourceId}:${slug}`;

  try {
    await withRefreshingLock(
      engine,
      lockId,
      () => enrichOneLocked(ctx, candidate),
      { ttlMinutes: PER_PAGE_LOCK_TTL_MINUTES },
    );
  } catch (err) {
    if (err instanceof LockUnavailableError) {
      ctx.result.pages_skipped_lock++;
      return; // page stays in backlog; next run retries
    }
    throw err; // BudgetExhausted (aborts pool) + real errors → pool failures[]
  }
}

async function enrichOneLocked(ctx: EnrichOneCtx, candidate: EnrichCandidate): Promise<void> {
  const { engine, sourceId } = ctx;
  const slug = candidate.slug;

  const snapshot = await engine.readPageSnapshot(slug, { sourceId });
  if (!snapshot) {
    ctx.result.pages_skipped_disappeared++;
    return;
  }
  const page = snapshot.page;
  const requestId = randomUUID();

  const kind = inferEnrichKind(page.type, slug);
  const evidence = await retrieveEvidence(engine, sourceId, slug, page.title || slug);
  const rendered = renderEvidence(evidence);
  const grounding = assessGrounding(rendered, ctx.minContextChars);

  if (!grounding.grounded) {
    ctx.result.pages_skipped_insufficient++;
    ctx.result.pages_skipped_pre_llm = (ctx.result.pages_skipped_pre_llm ?? 0) + 1;
    if (!ctx.dryRun) ctx.done.add(completedKey(sourceId, slug));
    return;
  }

  if (ctx.dryRun) {
    ctx.result.would_enrich = (ctx.result.would_enrich ?? 0) + 1;
    return; // no LLM, no write, no checkpoint advance
  }

  const { system, user } = buildEnrichPrompt({
    slug,
    title: page.title || slug,
    kind,
    currentBody: page.compiled_truth ?? '',
    evidence,
  });

  // `ctx.signal` is the CALLER's abort signal (shutdown / cancel). It is NOT the
  // sliding pool's internal budget-abort signal: runSlidingPool aborts its own
  // controller on BUDGET_EXHAUSTED but does not thread it into onItem, so an
  // already-running synth here is NOT cancelled when a sibling worker hits the
  // cap. That is the documented best-effort posture (overshoot ~1 call/worker
  // under --workers > 1; pin --workers 1 for a hard ceiling). A true in-flight
  // cancel would require a shared runSlidingPool API change (used by embed/eval).
  const raw = await ctx.synthesizeFn({ system, user, model: ctx.model, abortSignal: ctx.signal });
  const parsed = parseSynthesis(raw);
  if (parsed.skip || !parsed.body.trim()) {
    ctx.result.pages_skipped_insufficient++;
    // #2085 split: an explicit SKIP is a grounding verdict; blank output (or a
    // body that strips to nothing) is a synthesis failure shape. parseSynthesis
    // reports skip=true for blank raw too, so discriminate on the raw text.
    if (parsed.skip && (raw ?? '').trim().length > 0) {
      ctx.result.pages_model_skip = (ctx.result.pages_model_skip ?? 0) + 1;
      // A real grounding verdict — durable until the checkpoint TTL/--force.
      ctx.done.add(completedKey(sourceId, slug));
    } else {
      ctx.result.pages_empty_output = (ctx.result.pages_empty_output ?? 0) + 1;
      // Synthesis-failure shape (#2085), usually transient provider trouble —
      // banking it would suppress the page for the full checkpoint TTL, so it
      // stays out of `done` and is re-attempted on the next run.
    }
    return;
  }

  // Write via the put_page op handler (trusted local: remote=false) so
  // auto-link + disk write-through fire, exactly like `gbrain capture`. The
  // retrieved context was sanitized in buildEnrichPrompt; the synthesized body
  // is the model's grounded output, with any neutralized span it copied from
  // the prompt restored from the raw stub and evidence (#7125).
  const body = restoreNeutralized(parsed.body, [page.compiled_truth ?? '', ...evidence.map(e => e.text)].join('\n')).text;
  const tags = snapshot.tags;
  const newFrontmatter: Record<string, unknown> = {
    ...page.frontmatter,
    // Provenance survives write-through (it only overrides ingested_via /
    // ingested_at / source_kind). enriched_at also drives the recency guard.
    enriched_at: new Date().toISOString(),
    enriched_by: ENRICHED_BY,
  };
  // The model owns the prose only: the page's facts/takes fences are carried
  // over verbatim so publication never expires fence facts or deletes takes.
  const content = serializeMarkdown(newFrontmatter, preserveCanonicalFences(page, body), page.timeline ?? '', {
    type: page.type,
    title: page.title,
    tags,
  });

  if (ctx.maintenance) {
    await publishMaintenancePage(engine, ctx.maintenance, slug, content, { expectedRevision: snapshot.revision });
    ctx.result.pages_enriched++;
    ctx.done.add(completedKey(sourceId, slug));
    return;
  }
  const putPageOp = operations.find((o) => o.name === 'put_page');
  if (!putPageOp) throw new Error('put_page operation missing (gbrain build issue)');
  const opCtx: OperationContext = {
    engine,
    config: ctx.config ?? { engine: 'pglite' as const },
    logger: {
      info: () => {},
      warn: (msg: string) => process.stderr.write(`[enrich] WARN: ${msg}\n`),
      error: (msg: string) => process.stderr.write(`[enrich] ERROR: ${msg}\n`),
    },
    dryRun: false,
    remote: false,
    sourceId,
  };
  await putPageOp.handler(opCtx, { slug, content, expected_revision: snapshot.revision, request_id: requestId });

  ctx.result.pages_enriched++;
  ctx.done.add(completedKey(sourceId, slug));
}

// ---------------------------------------------------------------------------
// Core (single source).
// ---------------------------------------------------------------------------

export async function runEnrichCore(
  engine: BrainEngine,
  opts: EnrichCoreOpts,
  signal?: AbortSignal,
): Promise<EnrichResult> {
  if (!opts.sourceId) throw new Error('runEnrichCore: opts.sourceId is required');

  const result: EnrichResult = {
    candidates_considered: 0,
    pages_enriched: 0,
    pages_skipped_insufficient: 0,
    pages_skipped_pre_llm: 0,
    pages_model_skip: 0,
    pages_empty_output: 0,
    pages_skipped_lock: 0,
    pages_skipped_disappeared: 0,
    pages_failed: 0,
  };

  if (await (await import('../core/storage-manifest/door.ts')).isStorageManifestSource(engine, opts.sourceId)) return result;
  const sourceId = opts.sourceId;
  const types = opts.types && opts.types.length > 0 ? opts.types : DEFAULT_TYPES;
  const order: EnrichOrder = ENRICH_ORDERS.includes(opts.order as EnrichOrder)
    ? (opts.order as EnrichOrder)
    : 'inbound-links';
  const limit = opts.limit && opts.limit > 0 ? opts.limit : DEFAULT_LIMIT;
  const thinThreshold = opts.thinThreshold ?? DEFAULT_THIN_THRESHOLD;
  const minContextChars = opts.minContextChars ?? MIN_CONTEXT_CHARS;
  const reenrichAfterMs = opts.reenrichAfterMs ?? DEFAULT_REENRICH_DAYS * 86_400_000;
  const model = opts.model || getChatModel();
  const dryRun = !!opts.dryRun;
  const synthesizeFn = opts.synthesizeFn ?? defaultSynthesize;
  const config = loadConfig();

  const workersResolved = resolveWorkersWithClamp(engine, opts.workers, 'enrich', 0);
  const workers = workersResolved.workers;

  const fp = enrichFingerprint({ sourceId, types, order, thinThreshold, model, minContextChars });
  const cpKey = checkpointKey(fp);

  // #3629: load the checkpoint BEFORE enumerating candidates. SKIP'd pages
  // stay thin (nothing is written), so they re-enter the candidate list on
  // every run — the checkpoint is the ONLY thing that stops them re-billing.
  // #3629 residual: --dry-run promises "no checkpoint advance"; a dry-run
  // --force must not destroy the real checkpoint either.
  if (opts.force && !dryRun) await clearOpCheckpoint(engine, cpKey);
  const done = new Set<string>(opts.force ? [] : await loadOpCheckpoint(engine, cpKey));

  // Candidate enumeration — ONE source-aware, memory-bounded SQL query.
  // #3629: over-fetch by the number of checkpointed keys so already-done
  // pages sitting at the top of the ranking can't wedge the limit window
  // (limit=N with N done candidates used to yield pending=[] forever while
  // lower-ranked candidates never got a turn), then slice back to `limit`.
  const candidates = await engine.listEnrichCandidates({
    types,
    sourceId,
    thinThreshold,
    order,
    limit: limit + done.size,
    reenrichAfterMs,
  });
  result.candidates_considered = candidates.length;
  if (candidates.length === 0) return result;

  // Filter out already-completed candidates (resume), bounded to this run's
  // requested window.
  const pending = candidates
    .filter((c) => !done.has(completedKey(sourceId, c.slug)))
    .slice(0, limit);
  // #3629: nothing to do → return WITHOUT recordCompleted. Re-recording the
  // same done set on every no-op run would refresh the checkpoint's activity
  // clock and the 7-day purge TTL would never expire, making SKIP keys
  // permanent instead of decaying (the intended retry channel; --force is
  // the immediate one).
  if (pending.length === 0) return result;
  // #5280: a managed brain publishes through the maintenance coordinator; the
  // preflight refuses a missing canonical owner before any model spend.
  const maintenance = opts.dryRun ? null : await maintenancePreflight(engine, opts.sourceId);

  const body = async () => {
    const oneCtx: EnrichOneCtx = {
      engine,
      sourceId,
      model,
      minContextChars,
      dryRun,
      synthesizeFn,
      result,
      done,
      signal,
      config,
      maintenance,
    };

    let lastFlush = 0;
    let pool;
    try {
      pool = await runSlidingPool<EnrichCandidate>({
      items: pending,
      workers,
      signal,
      failureLabel: (c) => c.slug,
      onItem: async (c) => {
        await enrichOne(oneCtx, c);
        // Periodic checkpoint flush so a crash mid-run doesn't lose progress.
        if (!dryRun && done.size - lastFlush >= CHECKPOINT_FLUSH_EVERY) {
          lastFlush = done.size;
          await recordCompleted(engine, cpKey, [...done]);
        }
      },
      });
    } catch (err) {
      // P2#1 (codex): BudgetExhausted aborts the pool and propagates. Flush the
      // pages completed since the last 25-item flush BEFORE it bubbles to
      // runEnrichCore's catch, else resume re-charges them (and SKIP pages stay
      // thin). `done` is in scope here; it isn't in the outer catch.
      if ((err instanceof BudgetExhausted || isMustAbortError(err)) && !dryRun) {
        await recordCompleted(engine, cpKey, [...done]);
      }
      throw err;
    }

    result.pages_failed = pool.errored;
    const writeRequests = pool.failures.flatMap(f => f.error instanceof OperationError && f.error.writeRequest ? [f.error.writeRequest] : []);
    if (writeRequests.length) {
      result.write_requests = writeRequests;
      for (const receipt of writeRequests) process.stderr.write(`[enrich:${sourceId}] Write request ${receipt.request_id}: ${receipt.state}; inspect get_write_request before repeating enrichment.\n`);
    }

    // #2504 — pool.failures used to be write-only: an operator saw
    // pages_failed:N with zero reason anywhere (the pricing hard-fail looked
    // like a model/route problem). Log the first failure loud and carry it on
    // the result so cycle/JSON consumers see WHY.
    if (pool.failures.length > 0) {
      const f = pool.failures[0];
      const fMsg = f.error instanceof Error ? f.error.message : String(f.error);
      result.first_failure = `${f.label}: ${fMsg}`;
      process.stderr.write(
        `[enrich:${sourceId}] ${pool.errored} page(s) failed; first: ${f.label}: ${fMsg}\n`,
      );
    }

    if (!dryRun) {
      // #3629: the checkpoint is NOT cleared on a clean run any more. The old
      // clear made SKIP keys vanish the moment a run completed, so a page
      // whose synthesis said SKIP (still thin, still a candidate) was
      // re-billed on every subsequent run — the "unchanged page never
      // re-spends" contract only held mid-run. Enriched pages drop out of the
      // thin set anyway; SKIP keys decay via the cycle purge's 7-day TTL
      // (purgeStaleCheckpoints) and `--force` clears immediately.
      await recordCompleted(engine, cpKey, [...done]);
    }
  };

  // One tracker reference for both the run and the post-hoc overage check.
  // External tracker (cycle phase): used as-is, no withBudgetTracker wrap (that
  // would REPLACE not stack). Internal: capped at maxCostUsd ?? DEFAULT.
  // v0.42.42.0 (#2139): Infinity = explicit "uncapped" (off / tokenmax) → pass
  // `undefined` so BudgetTracker runs without a ceiling (NOT raw Infinity, which
  // would serialize to null in audit rows). undefined-when-unset still → DEFAULT.
  const resolvedCap =
    opts.maxCostUsd === Infinity ? undefined : (opts.maxCostUsd ?? DEFAULT_MAX_COST_USD);
  // #4312: operator price overrides (config `pricing.overrides`) reach the
  // tracker at construction, so proxy-routed models (litellm chat AND embed)
  // with a declared rate price normally instead of TX2 no_pricing-aborting.
  // Loaded only on the internal-tracker path (`??` short-circuits); an
  // externally-supplied tracker (cycle phase) carries its own overrides.
  const tracker = opts.budgetTracker ?? new BudgetTracker({
    maxCostUsd: resolvedCap,
    label: `enrich:${sourceId}`,
    pricingOverrides: await loadPricingOverrides(engine),
  });
  try {
    if (opts.budgetTracker) {
      await body();
    } else {
      await withBudgetTracker(tracker, body);
    }
  } catch (err) {
    if (err instanceof BudgetExhausted) {
      result.budget_exhausted = true;
      // #4032: carry the reason (and model, for no_pricing) so the CLI can
      // branch its advice instead of collapsing every abort into "raise the cap".
      result.budget_exhausted_reason = err.reason;
      if (err.modelId) result.budget_exhausted_model = err.modelId;
      if (err.pricing) result.budget_exhausted_pricing = err.pricing;
      return result; // partial run; caller surfaces it (NOT a thrown failure)
    }
    throw err;
  } finally {
    result.spent_usd = tracker.totalSpent;
  }

  // P1#3 (codex): gateway.chat swallows a BudgetExhausted thrown by the FINAL
  // call's tracker.record() ("surfaced via next reserve") — but there is no next
  // reserve, so body() returns normally with budget_exhausted unset despite the
  // overage. Detect it post-hoc so the result is honest. Enrich-local: reads the
  // tracker's read-only cap; no shared gateway.ts change.
  if (tracker.cap !== undefined && tracker.totalSpent > tracker.cap) {
    result.budget_exhausted = true;
    result.budget_exhausted_reason ??= 'cost'; // post-hoc overage is a cost abort (#4032)
  }

  return result;
}

// ---------------------------------------------------------------------------
// CLI parsing + handler.
// ---------------------------------------------------------------------------

interface ParsedArgs {
  sourceId?: string;
  types?: PageType[];
  order?: EnrichOrder;
  limit?: number;
  workers?: number;
  model?: string;
  maxCostUsd?: number;
  minContextChars?: number;
  thinThreshold?: number;
  reenrichAfterMs?: number;
  dryRun?: boolean;
  force?: boolean;
  yes?: boolean;
  json?: boolean;
  help?: boolean;
  error?: string;
}

function parseDurationDays(raw: string): number | undefined {
  // Accept "30", "30d", "12h". Returns ms.
  const m = raw.match(/^(\d+)\s*(d|h)?$/);
  if (!m) return undefined;
  const n = parseInt(m[1], 10);
  if (!Number.isFinite(n) || n < 0) return undefined;
  const unit = m[2] ?? 'd';
  return unit === 'h' ? n * 3_600_000 : n * 86_400_000;
}

export function parseArgs(args: string[]): ParsedArgs {
  const out: ParsedArgs = {};
  let cap: CapFlag | undefined;
  for (let i = 0; i < args.length; i++) {
    const a = args[i];
    if (a === '--help' || a === '-h') { out.help = true; continue; }
    // --background / --follow are handled by the dispatcher (maybeBackground /
    // fan-out); accept them here as no-ops so the inline-degrade path (PGLite)
    // and buildJobParams don't trip the unknown-flag guard.
    if (a === '--background' || a === '--follow') { continue; }
    if (a === '--thin') { continue; } // accepted; thin-filter is always applied
    if (a === '--dry-run') { out.dryRun = true; continue; }
    if (a === '--force' || a === '--resume') {
      // --resume is the documented flag; it's the DEFAULT behavior (checkpoint
      // auto-resumes). --force clears the checkpoint. Treat --resume as a no-op
      // affirmation and --force as the clear.
      if (a === '--force') out.force = true;
      continue;
    }
    if (a === '--yes' || a === '-y') { out.yes = true; continue; }
    if (a === '--json') { out.json = true; continue; }
    if (a === '--source' || a === '--source-id') { out.sourceId = args[++i]; continue; }
    if (a === '--model') { out.model = args[++i]; continue; }
    if (a === '--order') {
      const v = args[++i] as EnrichOrder;
      if (!ENRICH_ORDERS.includes(v)) {
        out.error = `Invalid --order: ${v}. Allowed: ${ENRICH_ORDERS.join(', ')}`;
        return out;
      }
      out.order = v;
      continue;
    }
    if (a === '--types') {
      const v = args[++i] ?? '';
      const parts = v.split(',').map((s) => s.trim()).filter(Boolean);
      if (parts.length === 0) { out.error = '--types requires a comma-separated list'; return out; }
      out.types = parts as PageType[];
      continue;
    }
    if (a === '--limit') {
      const n = parseInt(args[++i] ?? '', 10);
      if (Number.isFinite(n) && n > 0) out.limit = n;
      continue;
    }
    if (a === '--workers' || a === '--concurrency') {
      try { out.workers = parseWorkers(args[++i]); }
      catch (e) { out.error = (e as Error).message; return out; }
      continue;
    }
    if (a === '--max-usd' || a === '--max-cost-usd') {
      // D19 shared parser. v0.42.42.0 (#2139): off/unlimited/none → run
      // uncapped (Infinity sentinel; mapped to "no BudgetTracker ceiling" in
      // runEnrichCore). Spend still ledgered. A malformed, 0 or conflicting
      // value is refused before any paid call instead of silently ignored.
      try {
        cap = mergeCapFlag(cap, parseCapFlag(a, args[++i]));
      } catch (e) {
        if (!(e instanceof CapFlagError)) throw e;
        out.error = e.message;
        return out;
      }
      out.maxCostUsd = cap.usd ?? Infinity;
      continue;
    }
    if (a === '--min-context') {
      const n = parseInt(args[++i] ?? '', 10);
      if (Number.isFinite(n) && n >= 0) out.minContextChars = n;
      continue;
    }
    if (a === '--thin-threshold') {
      const n = parseInt(args[++i] ?? '', 10);
      if (Number.isFinite(n) && n > 0) out.thinThreshold = n;
      continue;
    }
    if (a === '--reenrich-after') {
      const ms = parseDurationDays(args[++i] ?? '');
      if (ms === undefined) { out.error = 'Invalid --reenrich-after (use e.g. 30d or 12h)'; return out; }
      out.reenrichAfterMs = ms;
      continue;
    }
    if (a.startsWith('--')) { out.error = `Unknown flag: ${a}`; return out; }
  }
  return out;
}

const HELP = `Usage: gbrain enrich [options]

Develop thin (stub) pages into real, cited pages by consolidating what the
brain ALREADY knows about each entity — scattered mentions, inbound-link
context, facts, and the existing stub — via one grounded LLM call per page.
No web/external lookup (that stays the agent-driven 'enrich' skill); this is
brain-internal synthesis only.

Options:
  --thin                 Select stub pages (always applied; accepted for clarity).
  --order <signal>       Candidate ordering: inbound-links (default) | salience | updated.
  --types <list>         Comma-separated page types. Default: person,company.
  --limit <N>            Max pages this run. Default ${DEFAULT_LIMIT}.
  --workers <K>          Parallel page workers. Default 1. PGLite clamps to 1.
  --model <provider:id>  Chat model. Default: configured chat model.
                         For cheap bulk: --model anthropic:claude-haiku-4-5.
  --max-usd <FLOAT>      Cost cap (USD). Default ${DEFAULT_MAX_COST_USD}.
                         BEST-EFFORT under --workers > 1: can overshoot by up to
                         ~one in-flight call per worker. Pin --workers 1 for an
                         exact ceiling.
  --min-context <N>      Min retrieved-context chars to attempt synthesis.
                         Below it the page is skipped (insufficient context),
                         never fabricated. Default ${MIN_CONTEXT_CHARS}.
  --thin-threshold <N>   Body char length below which a page counts as thin.
                         Default ${DEFAULT_THIN_THRESHOLD}.
  --reenrich-after <dur> Skip pages enriched within this window (e.g. 30d, 12h).
                         Default ${DEFAULT_REENRICH_DAYS}d.
  --source <id>          Source to enrich. When omitted, all sources are
                         enumerated (CLI loops; --background fans out one job
                         per source).
  --dry-run              List candidates + cost estimate; no LLM, no write.
  --resume               Resume from the prior checkpoint (default behavior).
  --force                Clear the checkpoint and re-process every candidate.
  --background           Submit as Minion job(s); print job_id(s); exit.
  --json                 Machine-readable summary.
  --yes, -y              Auto-confirm cost preview in non-TTY contexts.
  --help, -h             Show this help.

Provenance: enriched pages get frontmatter enriched_at + enriched_by=${ENRICHED_BY}
(survives put_page write-through). The recency guard reads enriched_at.
`;

function buildJobParams(args: string[]): Record<string, unknown> {
  const p = parseArgs(args);
  return {
    sourceId: p.sourceId,
    types: p.types,
    order: p.order,
    limit: p.limit,
    workers: p.workers,
    model: p.model,
    maxCostUsd: p.maxCostUsd,
    minContextChars: p.minContextChars,
    thinThreshold: p.thinThreshold,
    reenrichAfterMs: p.reenrichAfterMs,
    dryRun: p.dryRun,
    force: p.force,
  };
}

/**
 * P1#4 (codex): the multi-source `--background` fan-out must key each per-source
 * Minion job on the FULL run config, not just the source id. `MinionQueue.add()`
 * returns any existing row for a key (including completed ones, since
 * remove_on_complete defaults false), so a bare `enrich:${sid}` key silently
 * returned the OLD job when the user re-ran with a different --model / --limit /
 * --force / --dry-run. Content-hashing the full job params (the same scheme the
 * single-source `maybeBackground` path uses) means a different intent enqueues
 * new work. `fingerprint()` is canonical-JSON + hash, so key order is stable.
 */
export function backgroundIdempotencyKey(sourceId: string, args: string[]): string {
  return `enrich:${sourceId}:${fingerprint({ ...buildJobParams(args), sourceId })}`;
}

function emptyAgg(): EnrichResult {
  return {
    candidates_considered: 0,
    pages_enriched: 0,
    pages_skipped_insufficient: 0,
    pages_skipped_pre_llm: 0,
    pages_model_skip: 0,
    pages_empty_output: 0,
    pages_skipped_lock: 0,
    pages_skipped_disappeared: 0,
    pages_failed: 0,
    would_enrich: 0,
  };
}

function addInto(agg: EnrichResult, r: EnrichResult): void {
  agg.candidates_considered += r.candidates_considered;
  agg.pages_enriched += r.pages_enriched;
  agg.pages_skipped_insufficient += r.pages_skipped_insufficient;
  agg.pages_skipped_pre_llm = (agg.pages_skipped_pre_llm ?? 0) + (r.pages_skipped_pre_llm ?? 0);
  agg.pages_model_skip = (agg.pages_model_skip ?? 0) + (r.pages_model_skip ?? 0);
  agg.pages_empty_output = (agg.pages_empty_output ?? 0) + (r.pages_empty_output ?? 0);
  agg.pages_skipped_lock += r.pages_skipped_lock;
  agg.pages_skipped_disappeared += r.pages_skipped_disappeared;
  agg.pages_failed += r.pages_failed;
  agg.would_enrich = (agg.would_enrich ?? 0) + (r.would_enrich ?? 0);
  if (r.budget_exhausted) agg.budget_exhausted = true;
  // #4032: no_pricing is the actionable reason — it wins over cost when
  // per-source runs disagree; otherwise first reason seen sticks.
  if (
    r.budget_exhausted_reason &&
    (agg.budget_exhausted_reason === undefined || r.budget_exhausted_reason === 'no_pricing')
  ) {
    agg.budget_exhausted_reason = r.budget_exhausted_reason;
    agg.budget_exhausted_model = r.budget_exhausted_model;
    agg.budget_exhausted_pricing = r.budget_exhausted_pricing;
  }
  // #2504 — first failure seen across sources sticks (a sample, not a log).
  if (r.first_failure && agg.first_failure === undefined) {
    agg.first_failure = r.first_failure;
  }
  if (r.write_requests?.length) (agg.write_requests ??= []).push(...r.write_requests);
}

/**
 * #4032: the exhaustion advice must branch on WHY the tracker aborted.
 * A no_pricing TX2 hard-fail is not a cost overrun — "raise --max-usd"
 * sends the operator after the wrong knob. Exported for tests.
 */
export function budgetExhaustedMessage(reason?: BudgetReason, modelId?: string, pricing?: NoPricingGuidance): string {
  if (reason === 'no_pricing') {
    const m = modelId ? ` for ${modelId}` : '';
    const steps = pricing
      ? noPricingSteps(pricing)
      : `Look up the model's per-token price and register it with \`gbrain pricing set\` (see ${ERROR_CATALOGUE.no_pricing.docs}), then retry.`;
    return `  No pricing${m} — the cost cap cannot be enforced. ${steps} Or re-run uncapped (--max-usd off).`;
  }
  return '  Budget cap reached. Re-run with a higher --max-usd to continue.';
}

export async function runEnrich(engine: BrainEngine, args: string[]): Promise<void> {
  if (args.includes('--help') || args.includes('-h')) {
    console.log(HELP);
    return;
  }

  // --background: fan out one Minion job per source (D4). With --source, one job.
  // PGLite has no worker daemon → fall through to inline (note emitted below).
  const background = args.includes('--background') && engine.kind !== 'pglite';
  if (args.includes('--background') && !background) {
    process.stderr.write('[--background] PGLite has no worker daemon; running enrich inline.\n');
  }

  const parsed = parseArgs(args);
  if (parsed.error) {
    console.error(parsed.error);
    console.error(HELP);
    process.exit(1);
  }

  // Chat gateway is required for non-dry-run inline work (the worker needs it
  // for background jobs). Recover a cold singleton before reporting an
  // availability error (#2590).
  if (!background && !parsed.dryRun && !isAvailable('chat')) configureGatewayIfUninitialized();
  if (!background && !parsed.dryRun && !isAvailable('chat')) {
    console.error(
      'Chat gateway unavailable. Set a provider key (OPENAI_API_KEY or ANTHROPIC_API_KEY — ' +
      'chat routes to whichever is present), or configure a model explicitly ' +
      '(`gbrain config set models.chat <provider:model>`), or pass --dry-run to preview candidates.',
    );
    process.exit(1);
  }

  const listed: string[] = parsed.sourceId
    ? [parsed.sourceId]
    : (await listSources(engine)).map((s) => s.id);
  const sourceIds = background && listed.length === 0 ? ['default'] : listed;

  // A4 consent before spending, and before queueing paid jobs. `--max-usd
  // <usd>`, `--yes` (derived cap: the estimate x1.5), a per-run preapproval or
  // spend.posture=tokenmax authorize it; `--max-usd off` is the explicit
  // uncapped choice. tokenmax keeps its documented meaning here (D15A: the
  // ceiling is removed, spend is still ledgered), so unattended tokenmax runs
  // do not flip to a derived-cap stop. Without authorization: a TTY prompt,
  // else exit 3 with the consent payload and nothing queued.
  const explicitOff = parsed.maxCostUsd === Infinity;
  let maxCostUsd = parsed.maxCostUsd;
  const base = args.filter(a => a !== '--yes');
  let spend: SpendAuthorization | undefined;
  if (background && !parsed.dryRun && explicitOff) {
    spend = jobSpendAuthorization({ uncapped: true, via: 'max_usd' }, { command: 'enrich', of: sourceIds.length, argv: ['gbrain', 'enrich', ...base] });
  }
  if (!parsed.dryRun && !explicitOff) {
    const limit = parsed.limit ?? DEFAULT_LIMIT;
    const estUsd = Math.ceil(limit * sourceIds.length * COST_ESTIMATE_PER_PAGE_USD * 100) / 100;
    const auth = await consentGateOrExit({
      command: 'enrich', effects: ['paid'], actor: 'agent',
      what: `Enrich up to ${limit} page(s) per source across ${sourceIds.length} source(s)${background ? ' as background jobs' : ''}`,
      why: 'Fills thin person and company pages with model-written summaries from the brain\'s own evidence.',
      risk: `Spends about $${estUsd.toFixed(2)} with the chat model provider; pages gain model-written text (each write is attributed and can be reviewed).${background ? ' Without --max-usd, a model with no known price runs unmetered under the derived or default cap.' : ''}`,
      user_message: `Enrich up to ${limit} thin page(s) per source across ${sourceIds.length} source(s) for about $${estUsd.toFixed(2)}?`,
      argv: ['gbrain', 'enrich', ...base, ...(background && parsed.maxCostUsd === undefined ? ['--max-usd', derivedCapUsd(estUsd).toFixed(2)] : [])],
      preview_argv: ['gbrain', 'enrich', ...base.filter(a => a !== '--json' && a !== '--background' && a !== '--follow'), '--dry-run'],
      est_usd: estUsd,
      args,
    }, { json: parsed.json === true, env: engineConsentEnv(engine, await tokenmaxUncappedEnv(engine, parsed.maxCostUsd !== undefined)) });
    if (maxCostUsd === undefined && auth.cap_usd !== null) maxCostUsd = auth.cap_usd;
    if (background) spend = jobSpendAuthorization(auth, { command: 'enrich', est_usd: estUsd, of: sourceIds.length, argv: ['gbrain', 'enrich', ...base] });
  }

  if (background) {
    if (sourceIds.length === 1) {
      await maybeBackground({
        engine,
        args: parsed.sourceId ? args : [...args, '--source', sourceIds[0]],
        jobName: 'enrich',
        paramBuilder: buildJobParams,
        spendAuthorization: spend,
      });
      return;
    }
    const { MinionQueue } = await import('../core/minions/queue.ts');
    const queue = new MinionQueue(engine);
    const jobs = [];
    for (const sid of sourceIds) {
      jobs.push(await queue.add(
        'enrich',
        { ...buildJobParams(args), sourceId: sid },
        { idempotency_key: backgroundIdempotencyKey(sid, args) },
        spend ? { spendAuthorization: spend } : undefined,
      ));
    }
    console.log(`Submitted ${jobs.length} enrich job(s) (one per source): ${jobs.map((j) => `job_id=${j.id}`).join(' ')}`);
    console.log('Follow with: gbrain jobs follow <id>');
    if (spend) for (const line of spendSubmitSummary(spend, jobs, spend.argv!).lines) console.error(line);
    return;
  }

  const uncapped = !parsed.dryRun && maxCostUsd === Infinity;
  if (uncapped) {
    console.error(`${explicitOff ? '--max-usd off' : 'spend.posture=tokenmax'}: running uncapped, spend ledgered. docs: docs/operations/spend-controls.md`);
  }

  const aggregate = emptyAgg();
  let totalSpent = 0;
  let anyBudgetExhausted = false;
  const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
  progress.start('enrich', sourceIds.length);

  try {
    for (const sourceId of sourceIds) {
      const r = await runEnrichCore(engine, {
        sourceId,
        types: parsed.types,
        order: parsed.order,
        limit: parsed.limit,
        workers: parsed.workers,
        model: parsed.model,
        // uncapped (off / tokenmax) → Infinity sentinel; runEnrichCore maps it
        // to "no BudgetTracker ceiling".
        maxCostUsd,
        minContextChars: parsed.minContextChars,
        thinThreshold: parsed.thinThreshold,
        reenrichAfterMs: parsed.reenrichAfterMs,
        dryRun: parsed.dryRun,
        force: parsed.force,
      });
      addInto(aggregate, r);
      if (r.spent_usd) totalSpent += r.spent_usd;
      if (r.budget_exhausted) anyBudgetExhausted = true;
      progress.tick(1, `${sourceId}: ${r.pages_enriched} enriched`);
    }
  } finally {
    progress.finish();
  }

  if (parsed.json) {
    console.log(JSON.stringify({
      schema_version: 1,
      ...aggregate,
      spent_usd: totalSpent,
      budget_exhausted: anyBudgetExhausted,
      sources: sourceIds.length,
      dry_run: !!parsed.dryRun,
    }, null, 2));
  } else if (parsed.dryRun) {
    console.log(
      `\n(dry run) ${aggregate.candidates_considered} thin candidate(s) across ${sourceIds.length} source(s); ` +
      `${aggregate.would_enrich ?? 0} have enough context to enrich, ` +
      `${aggregate.pages_skipped_insufficient} lack context. ` +
      `Est. ~$${(aggregate.candidates_considered * COST_ESTIMATE_PER_PAGE_USD).toFixed(2)} to run.`,
    );
  } else {
    console.log(
      `\nDone: enriched ${aggregate.pages_enriched} page(s) ` +
      `(${aggregate.pages_skipped_insufficient} skipped insufficient, ` +
      `${aggregate.pages_skipped_lock} lock-busy, ${aggregate.pages_failed} failed) ` +
      `across ${sourceIds.length} source(s). Spent ~$${totalSpent.toFixed(4)}.`,
    );
    if (anyBudgetExhausted) {
      console.log(
        budgetExhaustedMessage(aggregate.budget_exhausted_reason, aggregate.budget_exhausted_model, aggregate.budget_exhausted_pricing),
      );
    }
  }

  if (aggregate.pages_failed > 0 && aggregate.pages_enriched === 0 && !parsed.dryRun) {
    process.exit(1);
  }
}