Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
33.4 KB
Raw Blame
/**
 * v0.32.7 CJK wave — `gbrain reindex --markdown` sweep.
 *
 * Walks markdown pages whose `chunker_version` is below
 * MARKDOWN_CHUNKER_VERSION and re-imports each through the standard
 * `importFromFile` / `importFromContent` path. Bumps `chunker_version` on
 * success so re-runs are idempotent and a partial sweep can resume.
 *
 * Driven by:
 *   - `gbrain upgrade` post-upgrade hook (after the cost-estimate prompt).
 *   - Operators running `gbrain reindex --markdown` directly.
 *
 * Performance: batched 100 at a time so a 50K-page brain reindex doesn't
 * hold a single transaction open. `--limit` caps total work for triage
 * runs; `--dry-run` reports the count without writing.
 *
 * Codex outside-voice C2 — the original PR #599 `MARKDOWN_CHUNKER_VERSION`
 * fold into content_hash was a no-op because `performSync` only re-imports
 * files whose content actually changed, not files whose hash WOULD change
 * if recomputed. This sweep + the migration v54 column are how the bump
 * actually reaches existing markdown pages.
 */

import type { BrainEngine } from '../core/engine.ts';
import { setCliExitVerdict } from '../core/cli-force-exit.ts';
import { MARKDOWN_CHUNKER_VERSION } from '../core/chunkers/recursive.ts';
import { importFromContent, importFromFile } from '../core/import-file.ts';
import { serializeMarkdown } from '../core/markdown.ts';
import { createProgress } from '../core/progress.ts';
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
import { existsSync } from 'fs';
import { resolve } from 'path';
// v0.41.15.0 (T10, D9): per-batch parallel workers.
import { runSlidingPool } from '../core/worker-pool.ts';
import { resolveWorkersWithClamp } from '../core/sync-concurrency.ts';
import { refreshProjectionStatistics } from '../core/search/projection-statistics.ts';
import { managedPersistenceEnabled } from '../core/persistence/ownership.ts';
import { loadConfig } from '../core/config.ts';
import { BudgetExhausted, BudgetTracker } from '../core/budget/budget-tracker.ts';
import { withBudgetTracker } from '../core/ai/gateway.ts';
import type { SegmentedRechunkReceipt } from '../core/page-body/rechunk.ts';

interface ReindexOpts {
  /** Cap total pages reindexed. Useful for triage runs on huge brains. */
  limit?: number;
  /** Report would-do count; don't actually reindex. */
  dryRun?: boolean;
  /** Emit JSON envelope on stdout. */
  json?: boolean;
  /** Brain repo path (for reading source files). Falls back to sync.repo_path config or process.cwd(). */
  repoPath?: string;
  /**
   * Skip the embedding call during re-chunk. New chunks land with NULL
   * embedding and the next `gbrain embed --stale` pass fills them in.
   * Useful for offline / no-API-key brains and for tests.
   */
  noEmbed?: boolean;
  /** Optional open-world page type scope, e.g. atom. */
  type?: string;
  /**
   * v0.41.15.0 (T10, D9): in-process per-batch parallel workers.
   * Default 1. PGLite clamps to 1. Recommended 4-8 for large brains.
   * Each worker calls importFromFile / importFromContent independently;
   * the counters (reindexed/skipped/failed) are JS-single-thread atomic.
   */
  workers?: number;
}

export interface ReindexResult {
  pending: number;
  pendingAfter: number;
  reindexed: number;
  skipped: number;
  failed: number;
  dryRun: boolean;
  chunkerVersion: number;
  type: string | null;
  /** Parallel writers used for the run (#5181); absent when nothing ran. */
  workers?: number;
}

// #3686: real usage, reachable via `gbrain reindex --help` (the generic
// one-line CLI_ONLY stub used to shadow every target flag). The --multimodal
// flags are parsed by the cli.ts dispatcher (reindex-multimodal routing) but
// documented here so the whole surface lives in one block.
const REINDEX_HELP = `gbrain reindex — re-chunk / re-embed existing pages after a pipeline upgrade

USAGE
  gbrain reindex --markdown   [--type PAGE_TYPE] [--limit N] [--workers N] [--dry-run] [--no-embed] [--yes] [--max-usd USD] [--json] [--repo PATH]
  gbrain reindex --multimodal [--limit N] [--workers N] [--dry-run] [--cost-estimate] [--no-embed] [--yes] [--json]
  gbrain reindex --aliases    [--limit N] [--dry-run] [--json] [--source <id>]
  gbrain reindex --vectors    [--dry-run] [--json]
  gbrain reindex --oversized  [--limit N] [--dry-run] [--json] [--source <id>]
  gbrain reindex --segmented  [--source <id>] [--slug <slug>] [--apply] [--json]
  gbrain reindex --synopsis   [--source <id>] [--limit N] [--yes] [--max-usd USD] [--json]

TARGETS (exactly one required)
  --markdown        Re-chunk markdown pages whose chunker_version lags the
                    current chunker (or whose contextual-retrieval state is
                    unset when embedding is on). On a managed brain it
                    runs gbrain repair safe-chunks (and contextual-mode
                    unless --no-embed) instead: no page writes; --type is
                    refused there.
  --multimodal      Re-embed image/PDF chunks through the multimodal
                    embedding pipeline (Voyage batches).
  --aliases         Backfill the free-text alias layer (page_aliases) for
                    pages whose frontmatter aliases predate the projection.
  --vectors         Rebuild every HNSW vector index from the stored vectors
                    (no re-embedding, no cost). Run after a PGLite WAL repair.
  --oversized       Re-chunk pages an older gbrain stored without chunks
                    because they were over 500 KB (embed_skip: oversized).
                    Under content_sanity.bytes_block (default: the 24 MiB
                    page ceiling) the marker is removed; over it the marker
                    stays and the page gets no vectors. CPU only and free: it works
                    with embeddings off and makes them keyword-searchable
                    now. Pages over 1 MiB wait for approval before any paid
                    embedding (gbrain embed --held). Resumable.
  --segmented       Re-chunk pages over 1 MiB (segmented storage) whose chunks
                    an older segmented chunker made: long passages now cut at
                    sentence or line ends. Previews by default (pages, chunk
                    counts, new chunk texts; writes nothing). --apply re-chunks
                    only the segments that change, keeps the vectors of
                    unchanged chunk texts and leaves new texts to the embed
                    sweep (gbrain embed --stale). Never runs on upgrade.
  --synopsis        Put pages whose configured contextual-retrieval tier is
                    per_chunk_synopsis (search.mode tokenmax) on that tier:
                    import and sync embed them at the title tier and nothing
                    else schedules the synopsis sweep. Previews by default
                    (pages, chunks, tokens and dollars on the configured
                    synopsis model; spends nothing). --yes queues one
                    contextual_reindex_per_chunk job per page under one spend
                    cap (--max-usd, else derived from the estimate) and runs
                    them in this process. Resumable: run it again to continue.
                    Inline pages over 1 MiB stay on the title tier.

OPTIONS
  --type <t>        --markdown only: restrict to one page type
  --limit N         Cap pages/chunks processed this run (--synopsis: pages)
  --workers N       --markdown and --multimodal: parallel writers per batch
                    (--concurrency is an alias). --markdown on Postgres
                    defaults to 4 when more than 100 pages are pending;
                    PGLite always uses 1
  --dry-run         Report what would change; write nothing
  --cost-estimate   --multimodal only: print the embed cost estimate and stop
  --no-embed        Skip re-embedding (chunk-only reindex)
  --yes             Approve the paid run: --markdown re-embeds pages and asks
                    first (exit 3 without approval when not interactive;
                    --dry-run, --no-embed and a keyless brain never ask);
                    --multimodal skips its cost confirm
  --max-usd USD     --markdown: approve the paid run with this cost cap;
                    --synopsis: the cap shared by every queued job (with --yes)
  --source <id>     --aliases, --oversized, --segmented and --synopsis: restrict
                    to one source
  --slug <slug>     --segmented only: one page
  --apply           --segmented only: apply the previewed re-chunk
  --repo PATH       --markdown only: brain repo override
  --json            Machine-readable output
  --help, -h        Show this help

Related: \`gbrain reindex-code --help\`, \`gbrain reindex-frontmatter --help\`,
\`gbrain reindex-search-vector --help\`.
`;

export function printReindexHelp(): void {
  console.log(REINDEX_HELP);
}

const REINDEX_VALUE_FLAGS = new Set(['--type', '--limit', '--repo', '--workers', '--concurrency', '--max-usd']);

export function normalizeReindexArgs(args: string[]): string[] {
  return args.flatMap((arg) => {
    const equals = arg.indexOf('=');
    if (equals <= 0) return [arg];
    const flag = arg.slice(0, equals);
    return REINDEX_VALUE_FLAGS.has(flag) ? [flag, arg.slice(equals + 1)] : [arg];
  });
}

function parsePageType(raw: string | undefined): string | null {
  // PageType is open-world at runtime. Only reject an absent/empty value or a
  // following CLI flag; SQL receives the exact remaining value as a bind.
  if (raw == null || raw.trim().length === 0 || raw.trimStart().startsWith('--')) return null;
  return raw;
}

function pendingDriftPredicate(noEmbed: boolean): string {
  return noEmbed
    ? 'chunker_version < $1'
    : '(chunker_version < $1 OR contextual_retrieval_mode IS NULL)';
}

export function validateReindexModeScope(args: string[]): string | null {
  args = normalizeReindexArgs(args);
  // #5937: --limit is validated for every mode (--multimodal/--aliases parsed it with parseInt).
  if (invalidPositiveIntegerFlag(args, '--limit')) return 'invalid --limit: expected a positive integer';
  if (!args.includes('--type')) return null;
  if (args.includes('--multimodal')) return '--type is only supported with reindex --markdown, not --multimodal';
  if (args.includes('--aliases')) return '--type is only supported with reindex --markdown, not --aliases';
  if (args.includes('--oversized')) return '--type is only supported with reindex --markdown, not --oversized';
  if (args.includes('--segmented')) return '--type is only supported with reindex --markdown, not --segmented';
  if (args.includes('--synopsis')) return '--type is only supported with reindex --markdown, not --synopsis';
  return null;
}

function invalidPositiveIntegerFlag(args: string[], flag: string): boolean {
  return args.some((arg, index) => {
    if (arg !== flag) return false;
    const raw = args[index + 1];
    return raw == null || raw.startsWith('--') || !/^\d+$/.test(raw) ||
      !Number.isSafeInteger(Number(raw)) || Number(raw) <= 0;
  });
}

function invalidRequiredValueFlag(args: string[], flag: string): boolean {
  return args.some((arg, index) => {
    if (arg !== flag) return false;
    const raw = args[index + 1];
    return raw == null || raw.trim().length === 0 || raw.startsWith('--');
  });
}

function parseArgs(args: string[]): ReindexOpts {
  const out: ReindexOpts = {};
  for (let i = 0; i < args.length; i++) {
    const a = args[i];
    if (a === '--markdown') continue; // routing flag, no value
    if (a === '--dry-run') out.dryRun = true;
    else if (a === '--json') out.json = true;
    else if (a === '--no-embed') out.noEmbed = true;
    else if (a === '--limit') {
      const v = parseInt(args[++i] ?? '', 10);
      if (Number.isFinite(v) && v > 0) out.limit = v;
    } else if (a === '--repo') {
      out.repoPath = args[++i];
    } else if (a === '--type') {
      const type = parsePageType(args[++i]);
      if (type) out.type = type;
    } else if (a === '--workers' || a === '--concurrency') {
      // v0.41.15.0 (T10, D9): per-batch parallel workers.
      const v = parseInt(args[++i] ?? '', 10);
      if (Number.isFinite(v) && v >= 1) out.workers = v;
    }
  }
  return out;
}

/**
 * Count markdown pages that need re-embedding. v0.40.3.0: predicate
 * extends from chunker_version drift alone to ALSO catch contextual
 * retrieval state drift (D26 P0-1). A page enters the sweep if either:
 *
 *   (a) chunker_version is below the current value — pre-v40 pages that
 *       haven't been touched by the wrapper bump yet
 *   (b) contextual_retrieval_mode is NULL — pages that have never been
 *       evaluated against the CR ladder (pre-v40 brains)
 *
 * D26 P0-4 IS DISTINCT FROM is used where comparing against a value; for
 * NULL detection we use IS NULL directly. Page-frontmatter overrides
 * (D5) are handled at re-import time: importFromFile re-parses the
 * frontmatter and the resolver picks the right tier for that page.
 *
 * Global-mode-vs-stamped-mode drift (e.g. user upgraded balanced→tokenmax,
 * skipped the post-upgrade prompt, then later ran reindex) is caught by
 * the IS NULL clause for pre-v81 brains AND by a future T10 mode-switch
 * hook for post-v81 brains. The simple `chunker_version OR mode IS NULL`
 * predicate covers the headline upgrade case the wave is shipping.
 */
export async function countPending(engine: BrainEngine, type: string | null = null, noEmbed = false): Promise<number> {
  const driftPredicate = pendingDriftPredicate(noEmbed);
  if (type) {
    const rows = await engine.executeRaw<{ count: string | number }>(
      `SELECT COUNT(*)::bigint AS count
         FROM pages
        WHERE page_kind = 'markdown'
          AND ${driftPredicate}
          AND deleted_at IS NULL
          AND type = $2`,
      [MARKDOWN_CHUNKER_VERSION, type],
    );
    return Number(rows[0]?.count ?? 0);
  }

  const rows = await engine.executeRaw<{ count: string | number }>(
    `SELECT COUNT(*)::bigint AS count
       FROM pages
      WHERE page_kind = 'markdown'
        AND ${driftPredicate}
        AND deleted_at IS NULL`,
    [MARKDOWN_CHUNKER_VERSION],
  );
  return Number(rows[0]?.count ?? 0);
}

/**
 * Read a single batch of pending rows. Ordered by id so re-runs after
 * partial completion pick up where they left off without re-doing pages
 * whose chunker_version was already bumped.
 */
async function readBatch(
  engine: BrainEngine,
  batchSize: number,
  type: string | null = null,
  noEmbed = false,
  afterId: number | null = null,
): Promise<Array<{ id: number; slug: string; source_path: string | null; compiled_truth: string; source_id: string }>> {
  const driftPredicate = pendingDriftPredicate(noEmbed);
  if (type) {
    return engine.executeRaw(
      `SELECT id, slug, source_path, compiled_truth, source_id
         FROM pages
        WHERE page_kind = 'markdown'
          AND ${driftPredicate}
          AND deleted_at IS NULL
          AND type = $2
          AND ($4::integer IS NULL OR id > $4)
        ORDER BY id ASC
        LIMIT $3`,
      [MARKDOWN_CHUNKER_VERSION, type, batchSize, afterId],
    );
  }

  return engine.executeRaw(
    `SELECT id, slug, source_path, compiled_truth, source_id
       FROM pages
      WHERE page_kind = 'markdown'
        AND ${driftPredicate}
        AND deleted_at IS NULL
        AND ($3::integer IS NULL OR id > $3)
      ORDER BY id ASC
      LIMIT $2`,
    [MARKDOWN_CHUNKER_VERSION, batchSize, afterId],
  );
}

/** How a caller already authorized the paid run (W4.5): the job handler's stored spend record, or the upgrade prompt's TTY yes. */
export interface ReindexRunOpts {
  /** The paid run is already authorized; skip the consent gate. */
  authorized?: boolean;
  /** False for unattended callers (the job handler): the gate never prompts. */
  interactive?: boolean;
}

export async function runReindex(engine: BrainEngine, args: string[], runOpts: ReindexRunOpts = {}): Promise<ReindexResult> {
  args = normalizeReindexArgs(args);
  const invalidType = args.some((arg, index) =>
    arg === '--type' && !parsePageType(args[index + 1]));
  if (invalidType) {
    const payload = { error: 'invalid --type: expected a non-empty value, not another flag' };
    if (args.includes('--json')) {
      process.stdout.write(JSON.stringify(payload) + '\n');
    } else {
      process.stderr.write(`[reindex] ${payload.error}\n`);
    }
    setCliExitVerdict(2);
    return { pending: 0, pendingAfter: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: args.includes('--dry-run'), chunkerVersion: MARKDOWN_CHUNKER_VERSION, type: null };
  }
  if (invalidPositiveIntegerFlag(args, '--limit')) {
    const payload = { error: 'invalid --limit: expected a positive integer' };
    if (args.includes('--json')) {
      process.stdout.write(JSON.stringify(payload) + '\n');
    } else {
      process.stderr.write(`[reindex] ${payload.error}\n`);
    }
    setCliExitVerdict(2);
    return { pending: 0, pendingAfter: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: args.includes('--dry-run'), chunkerVersion: MARKDOWN_CHUNKER_VERSION, type: null };
  }
  const invalidWorkerFlag = ['--workers', '--concurrency'].find((flag) =>
    invalidPositiveIntegerFlag(args, flag));
  const invalidRepo = invalidRequiredValueFlag(args, '--repo');
  if (invalidWorkerFlag || invalidRepo) {
    const error = invalidWorkerFlag
      ? `invalid ${invalidWorkerFlag}: expected a positive integer`
      : 'invalid --repo: expected a path';
    const payload = { error };
    if (args.includes('--json')) {
      process.stdout.write(JSON.stringify(payload) + '\n');
    } else {
      process.stderr.write(`[reindex] ${error}\n`);
    }
    setCliExitVerdict(2);
    return { pending: 0, pendingAfter: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: args.includes('--dry-run'), chunkerVersion: MARKDOWN_CHUNKER_VERSION, type: null };
  }
  const opts = parseArgs(args);
  const type = opts.type ?? null;

  // Require `--markdown` explicitly. Future modes (e.g. --code) get their
  // own routing here.
  if (!args.includes('--markdown')) {
    if (opts.json) {
      process.stdout.write(JSON.stringify({ error: 'gbrain reindex requires a target flag, e.g. --markdown' }) + '\n');
    } else {
      process.stderr.write('Usage: gbrain reindex --markdown [--type PAGE_TYPE] [--limit N] [--dry-run] [--json] [--repo PATH]\n');
    }
    setCliExitVerdict(2);
    return { pending: 0, pendingAfter: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: !!opts.dryRun, chunkerVersion: MARKDOWN_CHUNKER_VERSION, type };
  }

  const pending = await countPending(engine, type, !!opts.noEmbed);

  if (opts.json && pending === 0) {
    process.stdout.write(JSON.stringify({ pending: 0, pending_after: 0, reindexed: 0, skipped: 0, failed: 0, chunker_version: MARKDOWN_CHUNKER_VERSION, type }) + '\n');
    return { pending: 0, pendingAfter: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: !!opts.dryRun, chunkerVersion: MARKDOWN_CHUNKER_VERSION, type };
  }

  if (pending === 0) {
    const scope = type ? ` type=${type}` : '';
    process.stderr.write(`[reindex] All markdown pages${scope} already at chunker_version ${MARKDOWN_CHUNKER_VERSION}. Nothing to do.\n`);
    return { pending: 0, pendingAfter: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: !!opts.dryRun, chunkerVersion: MARKDOWN_CHUNKER_VERSION, type };
  }

  const target = typeof opts.limit === 'number' ? Math.min(opts.limit, pending) : pending;

  if (opts.dryRun) {
    if (opts.json) {
      process.stdout.write(JSON.stringify({ pending, pending_after: pending, would_reindex: target, dry_run: true, chunker_version: MARKDOWN_CHUNKER_VERSION, type }) + '\n');
    } else {
      const scope = type ? ` (type=${type})` : '';
      process.stderr.write(`[reindex] DRY-RUN: would re-chunk ${target} of ${pending} pending markdown pages${scope}.\n`);
    }
    return { pending, pendingAfter: pending, reindexed: 0, skipped: 0, failed: 0, dryRun: true, chunkerVersion: MARKDOWN_CHUNKER_VERSION, type };
  }

  // W4.5: re-embedding is paid work; ask before it starts (throws the exit-3 consent refusal or cost_cap_exceeded).
  // W12 P1: the approved cap is enforced while the run spends (embedding and synopsis calls), not only checked up front.
  let cap: BudgetTracker | null = null;
  let capHit: BudgetExhausted | null = null;
  if (!opts.noEmbed && !runOpts.authorized) {
    const { requireReindexConsent } = await import('../core/reindex-consent.ts');
    const gate = await requireReindexConsent(engine, { args, type, target, ...(runOpts.interactive === false ? { interactive: false } : {}) });
    const capUsd = gate?.auth.cap_usd;
    if (typeof capUsd === 'number' && Number.isFinite(capUsd) && capUsd > 0) {
      cap = new BudgetTracker({ maxCostUsd: capUsd, label: 'reindex', capSource: gate!.auth.cap_source ?? undefined });
      // An import swallows a refused embedding (its chunks wait for `embed --stale`), so the run stops at the next page.
      const tracker = cap;
      tracker.onExhausted(() => { capHit = new BudgetExhausted(`reindex reached its $${capUsd.toFixed(2)} cost cap`, { reason: 'cost', spent: tracker.totalSpent, cap: capUsd }); });
    }
  }
  const metered = <T>(fn: () => Promise<T>): Promise<T> => (cap ? withBudgetTracker(cap, fn) : fn());

  if (await managedPersistenceEnabled(engine)) {
    try { return await metered(() => reindexManaged(engine, opts, type, pending)); } catch (e) {
      if (!(e instanceof BudgetExhausted)) throw e;
      return reportBudgetStop(engine, e, { pending, type, json: !!opts.json, reindexed: 0, skipped: 0, failed: 0 });
    }
  }

  const reporter = createProgress(cliOptsToProgressOptions(getCliOptions()));
  reporter.start('reindex.markdown', target);

  let reindexed = 0;
  let skipped = 0;
  let failed = 0;
  let afterId: number | null = null;
  const BATCH = 100;
  const repoPath = opts.repoPath ? resolve(opts.repoPath) : null;
  // #5181: size the pool from the whole run, not one batch. A full batch is
  // exactly the auto threshold (100), so batch length never enabled it.
  const { workers } = resolveWorkersWithClamp(engine, opts.workers, 'reindex', target);

  let stopped: BudgetExhausted | null = null;
  try { await metered(async () => { while (reindexed + skipped + failed < target) {
    const remaining = target - (reindexed + skipped + failed);
    const batchSize = Math.min(BATCH, remaining);
    const batch = await readBatch(engine, batchSize, type, !!opts.noEmbed, afterId);
    if (batch.length === 0) break;
    // Advance before processing so a failed row cannot be selected again in
    // this invocation and starve later IDs.
    afterId = batch[batch.length - 1]!.id;

    // v0.41.15.0 (T10, D9): per-batch sliding pool. Counters are JS-
    // single-thread atomic so reindexed++ / failed++ are race-free
    // across workers.
    await runSlidingPool({
      items: batch,
      workers,
      failureLabel: (row) => row.slug,
      onItem: async (row) => {
        if (capHit) throw capHit;
        reporter.tick();
        try {
          if (row.source_path && repoPath) {
            const absPath = resolve(repoPath, row.source_path);
            if (existsSync(absPath)) {
              const imported = await importFromFile(engine, absPath, row.source_path, {
                noEmbed: !!opts.noEmbed,
                sourceId: row.source_id,
                inferFrontmatter: false,
                forceRechunk: true,
              });
              if (imported.status === 'imported') {
                reindexed++;
              } else if (imported.status === 'error') {
                process.stderr.write(`[reindex] ${row.slug}: ${imported.error ?? 'import failed'}\n`);
                failed++;
              } else if (imported.error) {
                process.stderr.write(`[reindex] ${row.slug}: ${imported.error}\n`);
                failed++;
              } else {
                skipped++;
              }
              return;
            }
          }
          // No source file on disk (DB-only page, or repo not available) —
          // re-chunk from the stored page. v0.41.37.0 #1621: reconstruct the
          // FULL markdown (frontmatter + body + timeline) via serializeMarkdown
          // and re-import THAT, instead of passing body-only `compiled_truth`
          // to importFromContent. The body-only path re-parsed with empty
          // frontmatter and OVERWROTE the page's real frontmatter / title /
          // timeline (codex catch). The round-trip preserves everything while
          // still re-chunking + bumping chunker_version.
          const page = await engine.getPage(row.slug, { sourceId: row.source_id });
          if (!page) { skipped++; return; }
          const tags = await engine.getTags(row.slug, { sourceId: row.source_id });
          const fullMarkdown = serializeMarkdown(
            page.frontmatter ?? {},
            page.compiled_truth ?? row.compiled_truth,
            page.timeline ?? '',
            { type: page.type, title: page.title, tags },
          );
          const imported = await importFromContent(engine, row.slug, fullMarkdown, {
            sourceId: row.source_id, preserveGateMarkers: true,
            noEmbed: !!opts.noEmbed,
            forceRechunk: true,
          });
          if (imported.status === 'imported') {
            reindexed++;
          } else if (imported.status === 'error') {
            process.stderr.write(`[reindex] ${row.slug}: ${imported.error ?? 'import failed'}\n`);
            failed++;
          } else if (imported.error) {
            process.stderr.write(`[reindex] ${row.slug}: ${imported.error}\n`);
            failed++;
          } else {
            skipped++;
          }
        } catch (err) {
          if (err instanceof BudgetExhausted) throw err;
          const msg = err instanceof Error ? err.message : String(err);
          process.stderr.write(`[reindex] ${row.slug}: ${msg}\n`);
          failed++;
        }
      },
    });
  } }); } catch (e) {
    if (!(e instanceof BudgetExhausted)) throw e;
    stopped = e;
  }

  reporter.finish();
  if (stopped) return reportBudgetStop(engine, stopped, { pending, type, json: !!opts.json, reindexed, skipped, failed, workers });

  if (reindexed > 0) await refreshProjectionStatistics(engine, reindexed);
  const pendingAfter = await countPending(engine, type, !!opts.noEmbed);
  if (failed > 0) setCliExitVerdict(1);

  const result: ReindexResult = {
    pending,
    pendingAfter,
    reindexed,
    skipped,
    failed,
    dryRun: false,
    chunkerVersion: MARKDOWN_CHUNKER_VERSION,
    type,
    workers,
  };

  if (opts.json) {
    process.stdout.write(JSON.stringify({
      pending, pending_after: pendingAfter, reindexed, skipped, failed,
      chunker_version: MARKDOWN_CHUNKER_VERSION,
      type,
      workers,
    }) + '\n');
  } else {
    const scope = type ? ` type=${type}` : '';
    process.stderr.write(`[reindex] Done.${scope} reindexed=${reindexed} skipped=${skipped} failed=${failed} pending_before=${pending} pending_after=${pendingAfter} workers=${workers}\n`);
  }

  return result;
}

/**
 * W12 P1: the run stopped at its approved cost cap. Pages re-chunked so far are
 * kept (a rerun continues with the rest); exit 1 with the spend and the rerun.
 */
async function reportBudgetStop(engine: BrainEngine, e: BudgetExhausted, r: { pending: number; type: string | null; json: boolean; reindexed: number; skipped: number; failed: number; workers?: number }): Promise<ReindexResult> {
  if (r.reindexed > 0) await refreshProjectionStatistics(engine, r.reindexed);
  const pendingAfter = await countPending(engine, r.type, false);
  setCliExitVerdict(1);
  const message = `stopped at the $${e.cap.toFixed(2)} cost cap after spending $${e.spent.toFixed(4)} (${e.message}). ${pendingAfter} page(s) still pending, and pages re-chunked after the cap wait for gbrain embed --stale; ask the user before running again with a higher --max-usd.`;
  if (r.json) {
    process.stdout.write(JSON.stringify({ pending: r.pending, pending_after: pendingAfter, reindexed: r.reindexed, skipped: r.skipped, failed: r.failed,
      chunker_version: MARKDOWN_CHUNKER_VERSION, type: r.type, budget_exhausted: { cap_usd: e.cap, spent_usd: e.spent, reason: e.reason } }) + '\n');
  } else {
    process.stderr.write(`[reindex] ${message}\n`);
  }
  return { pending: r.pending, pendingAfter, reindexed: r.reindexed, skipped: r.skipped, failed: r.failed, dryRun: false, chunkerVersion: MARKDOWN_CHUNKER_VERSION, type: r.type,
    ...(r.workers !== undefined ? { workers: r.workers } : {}) };
}

/**
 * #5377: a managed brain cannot re-import through the legacy writer, so the
 * sweep delegates to the projection repairs: `safe-chunks` re-seals every page
 * chunked before the current chunker and `contextual-mode` stamps pages imported
 * without a contextual mode (skipped under --no-embed, which does not count
 * them as pending). Neither writes a page revision, journal request or
 * canonical file. `--type` cannot narrow those repairs, so it is refused with
 * the repair commands instead.
 */
async function reindexManaged(engine: BrainEngine, opts: ReindexOpts, type: string | null, pending: number): Promise<ReindexResult> {
  const base = { pending, dryRun: false, chunkerVersion: MARKDOWN_CHUNKER_VERSION, type };
  const repairs = opts.noEmbed ? ['safe-chunks'] : ['safe-chunks', 'contextual-mode'];
  const redirect = repairs.map(kind => `gbrain repair ${kind} --apply${opts.noEmbed ? ' --no-embed' : ''}`).join(' && ');
  if (type) {
    const error = `--type cannot narrow a managed reindex; re-chunk every drifted page with: ${redirect}`;
    if (opts.json) process.stdout.write(JSON.stringify({ error, delegated_to: repairs }) + '\n');
    else process.stderr.write(`[reindex] ${error}\n`);
    setCliExitVerdict(2);
    return { ...base, pendingAfter: pending, reindexed: 0, skipped: 0, failed: 0 };
  }
  const { contextualModeRepair } = await import('../core/repair/contextual-mode.ts');
  const { safeChunksRepair } = await import('../core/repair/safe-chunks.ts');
  const { resolveRepairScope, runRepair } = await import('../core/repair/core.ts');
  const ctx = { engine, config: loadConfig() ?? { engine: engine.kind }, remote: false, dryRun: false, sourceId: 'default',
    logger: { info: (msg: string) => process.stderr.write(`${msg}\n`), warn: (msg: string) => process.stderr.write(`${msg}\n`), error: (msg: string) => process.stderr.write(`${msg}\n`) } };
  const scope = await resolveRepairScope(engine);
  const stopped: string[] = [];
  const failures: string[] = [];
  let unsupported = 0;
  for (const handler of opts.noEmbed ? [safeChunksRepair] : [safeChunksRepair, contextualModeRepair]) {
    const run = await runRepair(ctx as Parameters<typeof runRepair>[0], handler, scope, { apply: true, limit: opts.limit, embed: !opts.noEmbed });
    if (run.stopped) stopped.push(run.stopped.message);
    unsupported += Object.values(run.residuals).reduce((sum, count) => sum + count, 0);
    // GBRA-130 A-26: pages the repair held (projection_unbuildable) or failed per item are counted, not thrown.
    failures.push(...run.failed.map(f => `${f.item} (${f.reason}): ${f.message}`),
      ...(run.outcome_items ?? []).filter(o => o.outcome === 'held').map(o => `${o.item} (${o.reason}): ${String(o.detail?.message ?? '')}`));
  }
  const failed = failures.length;
  if (pending > 0) await refreshProjectionStatistics(engine, pending);
  const pendingAfter = await countPending(engine, type, !!opts.noEmbed);
  const reindexed = Math.max(0, pending - pendingAfter);
  if (stopped.length || failed) setCliExitVerdict(1);
  if (opts.json) {
    process.stdout.write(JSON.stringify({ pending, pending_after: pendingAfter, reindexed, skipped: 0, failed, chunker_version: MARKDOWN_CHUNKER_VERSION, type,
      delegated_to: repairs, ...(unsupported ? { unsupported_pages: unsupported } : {}), ...(stopped.length ? { stopped } : {}), ...(failed ? { failures } : {}) }) + '\n');
  } else {
    process.stderr.write(`[reindex] Managed brain: re-chunked through ${repairs.map(kind => `gbrain repair ${kind}`).join(' and ')} (no page writes). `
      + `reindexed=${reindexed} pending_before=${pending} pending_after=${pendingAfter}\n`);
    if (unsupported) process.stderr.write(`[reindex] ${unsupported} page(s) the projection repairs cannot rebuild (code without a source path, or another page kind) are left for their importer.\n`);
    for (const message of stopped) process.stderr.write(`[reindex] ${message}\n`);
    for (const failure of failures) process.stderr.write(`[reindex] not re-chunked, its chunks stay withheld: ${failure}. Retry with: gbrain repair safe-chunks --apply\n`);
  }
  return { ...base, pendingAfter, reindexed, skipped: 0, failed };
}

/**
 * `gbrain reindex --segmented`: preview (default) or `--apply` the re-chunk of sealed segmented pages an older
 * segmented chunker made (core/page-body/rechunk.ts). Nothing on upgrade or migration runs it.
 */
export async function runReindexSegmented(engine: BrainEngine, args: string[]): Promise<SegmentedRechunkReceipt | null> {
  args = normalizeReindexArgs(args);
  const json = args.includes('--json');
  const invalid = ['--source', '--slug'].find(flag => invalidRequiredValueFlag(args, flag));
  if (invalid) {
    const error = `invalid ${invalid}: expected a value`;
    if (json) process.stdout.write(JSON.stringify({ error }) + '\n');
    else process.stderr.write(`[reindex] ${error}\n`);
    setCliExitVerdict(2);
    return null;
  }
  const value = (flag: string) => { const at = args.indexOf(flag); return at >= 0 ? args[at + 1] : undefined; };
  const { runSegmentedRechunk, formatSegmentedRechunkReceipt } = await import('../core/page-body/rechunk.ts');
  const receipt = await runSegmentedRechunk(engine, { apply: args.includes('--apply'), sourceId: value('--source'), slug: value('--slug') });
  if (json) process.stdout.write(JSON.stringify(receipt, null, 2) + '\n');
  else for (const line of formatSegmentedRechunkReceipt(receipt)) console.log(line);
  if (receipt.pages.some(page => page.state === 'failed')) setCliExitVerdict(1);
  return receipt;
}