Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.32.7 CJK wave — `gbrain reindex --markdown` sweep.
*
* Walks markdown pages whose `chunker_version` is below
* MARKDOWN_CHUNKER_VERSION and re-imports each through the standard
* `importFromFile` / `importFromContent` path. Bumps `chunker_version` on
* success so re-runs are idempotent and a partial sweep can resume.
*
* Driven by:
* - `gbrain upgrade` post-upgrade hook (after the cost-estimate prompt).
* - Operators running `gbrain reindex --markdown` directly.
*
* Performance: batched 100 at a time so a 50K-page brain reindex doesn't
* hold a single transaction open. `--limit` caps total work for triage
* runs; `--dry-run` reports the count without writing.
*
* Codex outside-voice C2 — the original PR #599 `MARKDOWN_CHUNKER_VERSION`
* fold into content_hash was a no-op because `performSync` only re-imports
* files whose content actually changed, not files whose hash WOULD change
* if recomputed. This sweep + the migration v54 column are how the bump
* actually reaches existing markdown pages.
*/
import type { BrainEngine } from '../core/engine.ts';
import { setCliExitVerdict } from '../core/cli-force-exit.ts';
import { MARKDOWN_CHUNKER_VERSION } from '../core/chunkers/recursive.ts';
import { importFromContent, importFromFile } from '../core/import-file.ts';
import { serializeMarkdown } from '../core/markdown.ts';
import { createProgress } from '../core/progress.ts';
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
import { existsSync } from 'fs';
import { resolve } from 'path';
// v0.41.15.0 (T10, D9): per-batch parallel workers.
import { runSlidingPool } from '../core/worker-pool.ts';
import { resolveWorkersWithClamp } from '../core/sync-concurrency.ts';
import { refreshProjectionStatistics } from '../core/search/projection-statistics.ts';
import { managedPersistenceEnabled } from '../core/persistence/ownership.ts';
import { loadConfig } from '../core/config.ts';
import { BudgetExhausted, BudgetTracker } from '../core/budget/budget-tracker.ts';
import { withBudgetTracker } from '../core/ai/gateway.ts';
import type { SegmentedRechunkReceipt } from '../core/page-body/rechunk.ts';
interface ReindexOpts {
/** Cap total pages reindexed. Useful for triage runs on huge brains. */
limit?: number;
/** Report would-do count; don't actually reindex. */
dryRun?: boolean;
/** Emit JSON envelope on stdout. */
json?: boolean;
/** Brain repo path (for reading source files). Falls back to sync.repo_path config or process.cwd(). */
repoPath?: string;
/**
* Skip the embedding call during re-chunk. New chunks land with NULL
* embedding and the next `gbrain embed --stale` pass fills them in.
* Useful for offline / no-API-key brains and for tests.
*/
noEmbed?: boolean;
/** Optional open-world page type scope, e.g. atom. */
type?: string;
/**
* v0.41.15.0 (T10, D9): in-process per-batch parallel workers.
* Default 1. PGLite clamps to 1. Recommended 4-8 for large brains.
* Each worker calls importFromFile / importFromContent independently;
* the counters (reindexed/skipped/failed) are JS-single-thread atomic.
*/
workers?: number;
}
export interface ReindexResult {
pending: number;
pendingAfter: number;
reindexed: number;
skipped: number;
failed: number;
dryRun: boolean;
chunkerVersion: number;
type: string | null;
/** Parallel writers used for the run (#5181); absent when nothing ran. */
workers?: number;
}
// #3686: real usage, reachable via `gbrain reindex --help` (the generic
// one-line CLI_ONLY stub used to shadow every target flag). The --multimodal
// flags are parsed by the cli.ts dispatcher (reindex-multimodal routing) but
// documented here so the whole surface lives in one block.
const REINDEX_HELP = `gbrain reindex — re-chunk / re-embed existing pages after a pipeline upgrade
USAGE
gbrain reindex --markdown [--type PAGE_TYPE] [--limit N] [--workers N] [--dry-run] [--no-embed] [--yes] [--max-usd USD] [--json] [--repo PATH]
gbrain reindex --multimodal [--limit N] [--workers N] [--dry-run] [--cost-estimate] [--no-embed] [--yes] [--json]
gbrain reindex --aliases [--limit N] [--dry-run] [--json] [--source <id>]
gbrain reindex --vectors [--dry-run] [--json]
gbrain reindex --oversized [--limit N] [--dry-run] [--json] [--source <id>]
gbrain reindex --segmented [--source <id>] [--slug <slug>] [--apply] [--json]
gbrain reindex --synopsis [--source <id>] [--limit N] [--yes] [--max-usd USD] [--json]
TARGETS (exactly one required)
--markdown Re-chunk markdown pages whose chunker_version lags the
current chunker (or whose contextual-retrieval state is
unset when embedding is on). On a managed brain it
runs gbrain repair safe-chunks (and contextual-mode
unless --no-embed) instead: no page writes; --type is
refused there.
--multimodal Re-embed image/PDF chunks through the multimodal
embedding pipeline (Voyage batches).
--aliases Backfill the free-text alias layer (page_aliases) for
pages whose frontmatter aliases predate the projection.
--vectors Rebuild every HNSW vector index from the stored vectors
(no re-embedding, no cost). Run after a PGLite WAL repair.
--oversized Re-chunk pages an older gbrain stored without chunks
because they were over 500 KB (embed_skip: oversized).
Under content_sanity.bytes_block (default: the 24 MiB
page ceiling) the marker is removed; over it the marker
stays and the page gets no vectors. CPU only and free: it works
with embeddings off and makes them keyword-searchable
now. Pages over 1 MiB wait for approval before any paid
embedding (gbrain embed --held). Resumable.
--segmented Re-chunk pages over 1 MiB (segmented storage) whose chunks
an older segmented chunker made: long passages now cut at
sentence or line ends. Previews by default (pages, chunk
counts, new chunk texts; writes nothing). --apply re-chunks
only the segments that change, keeps the vectors of
unchanged chunk texts and leaves new texts to the embed
sweep (gbrain embed --stale). Never runs on upgrade.
--synopsis Put pages whose configured contextual-retrieval tier is
per_chunk_synopsis (search.mode tokenmax) on that tier:
import and sync embed them at the title tier and nothing
else schedules the synopsis sweep. Previews by default
(pages, chunks, tokens and dollars on the configured
synopsis model; spends nothing). --yes queues one
contextual_reindex_per_chunk job per page under one spend
cap (--max-usd, else derived from the estimate) and runs
them in this process. Resumable: run it again to continue.
Inline pages over 1 MiB stay on the title tier.
OPTIONS
--type <t> --markdown only: restrict to one page type
--limit N Cap pages/chunks processed this run (--synopsis: pages)
--workers N --markdown and --multimodal: parallel writers per batch
(--concurrency is an alias). --markdown on Postgres
defaults to 4 when more than 100 pages are pending;
PGLite always uses 1
--dry-run Report what would change; write nothing
--cost-estimate --multimodal only: print the embed cost estimate and stop
--no-embed Skip re-embedding (chunk-only reindex)
--yes Approve the paid run: --markdown re-embeds pages and asks
first (exit 3 without approval when not interactive;
--dry-run, --no-embed and a keyless brain never ask);
--multimodal skips its cost confirm
--max-usd USD --markdown: approve the paid run with this cost cap;
--synopsis: the cap shared by every queued job (with --yes)
--source <id> --aliases, --oversized, --segmented and --synopsis: restrict
to one source
--slug <slug> --segmented only: one page
--apply --segmented only: apply the previewed re-chunk
--repo PATH --markdown only: brain repo override
--json Machine-readable output
--help, -h Show this help
Related: \`gbrain reindex-code --help\`, \`gbrain reindex-frontmatter --help\`,
\`gbrain reindex-search-vector --help\`.
`;
export function printReindexHelp(): void {
console.log(REINDEX_HELP);
}
const REINDEX_VALUE_FLAGS = new Set(['--type', '--limit', '--repo', '--workers', '--concurrency', '--max-usd']);
export function normalizeReindexArgs(args: string[]): string[] {
return args.flatMap((arg) => {
const equals = arg.indexOf('=');
if (equals <= 0) return [arg];
const flag = arg.slice(0, equals);
return REINDEX_VALUE_FLAGS.has(flag) ? [flag, arg.slice(equals + 1)] : [arg];
});
}
function parsePageType(raw: string | undefined): string | null {
// PageType is open-world at runtime. Only reject an absent/empty value or a
// following CLI flag; SQL receives the exact remaining value as a bind.
if (raw == null || raw.trim().length === 0 || raw.trimStart().startsWith('--')) return null;
return raw;
}
function pendingDriftPredicate(noEmbed: boolean): string {
return noEmbed
? 'chunker_version < $1'
: '(chunker_version < $1 OR contextual_retrieval_mode IS NULL)';
}
export function validateReindexModeScope(args: string[]): string | null {
args = normalizeReindexArgs(args);
// #5937: --limit is validated for every mode (--multimodal/--aliases parsed it with parseInt).
if (invalidPositiveIntegerFlag(args, '--limit')) return 'invalid --limit: expected a positive integer';
if (!args.includes('--type')) return null;
if (args.includes('--multimodal')) return '--type is only supported with reindex --markdown, not --multimodal';
if (args.includes('--aliases')) return '--type is only supported with reindex --markdown, not --aliases';
if (args.includes('--oversized')) return '--type is only supported with reindex --markdown, not --oversized';
if (args.includes('--segmented')) return '--type is only supported with reindex --markdown, not --segmented';
if (args.includes('--synopsis')) return '--type is only supported with reindex --markdown, not --synopsis';
return null;
}
function invalidPositiveIntegerFlag(args: string[], flag: string): boolean {
return args.some((arg, index) => {
if (arg !== flag) return false;
const raw = args[index + 1];
return raw == null || raw.startsWith('--') || !/^\d+$/.test(raw) ||
!Number.isSafeInteger(Number(raw)) || Number(raw) <= 0;
});
}
function invalidRequiredValueFlag(args: string[], flag: string): boolean {
return args.some((arg, index) => {
if (arg !== flag) return false;
const raw = args[index + 1];
return raw == null || raw.trim().length === 0 || raw.startsWith('--');
});
}
function parseArgs(args: string[]): ReindexOpts {
const out: ReindexOpts = {};
for (let i = 0; i < args.length; i++) {
const a = args[i];
if (a === '--markdown') continue; // routing flag, no value
if (a === '--dry-run') out.dryRun = true;
else if (a === '--json') out.json = true;
else if (a === '--no-embed') out.noEmbed = true;
else if (a === '--limit') {
const v = parseInt(args[++i] ?? '', 10);
if (Number.isFinite(v) && v > 0) out.limit = v;
} else if (a === '--repo') {
out.repoPath = args[++i];
} else if (a === '--type') {
const type = parsePageType(args[++i]);
if (type) out.type = type;
} else if (a === '--workers' || a === '--concurrency') {
// v0.41.15.0 (T10, D9): per-batch parallel workers.
const v = parseInt(args[++i] ?? '', 10);
if (Number.isFinite(v) && v >= 1) out.workers = v;
}
}
return out;
}
/**
* Count markdown pages that need re-embedding. v0.40.3.0: predicate
* extends from chunker_version drift alone to ALSO catch contextual
* retrieval state drift (D26 P0-1). A page enters the sweep if either:
*
* (a) chunker_version is below the current value — pre-v40 pages that
* haven't been touched by the wrapper bump yet
* (b) contextual_retrieval_mode is NULL — pages that have never been
* evaluated against the CR ladder (pre-v40 brains)
*
* D26 P0-4 IS DISTINCT FROM is used where comparing against a value; for
* NULL detection we use IS NULL directly. Page-frontmatter overrides
* (D5) are handled at re-import time: importFromFile re-parses the
* frontmatter and the resolver picks the right tier for that page.
*
* Global-mode-vs-stamped-mode drift (e.g. user upgraded balanced→tokenmax,
* skipped the post-upgrade prompt, then later ran reindex) is caught by
* the IS NULL clause for pre-v81 brains AND by a future T10 mode-switch
* hook for post-v81 brains. The simple `chunker_version OR mode IS NULL`
* predicate covers the headline upgrade case the wave is shipping.
*/
export async function countPending(engine: BrainEngine, type: string | null = null, noEmbed = false): Promise<number> {
const driftPredicate = pendingDriftPredicate(noEmbed);
if (type) {
const rows = await engine.executeRaw<{ count: string | number }>(
`SELECT COUNT(*)::bigint AS count
FROM pages
WHERE page_kind = 'markdown'
AND ${driftPredicate}
AND deleted_at IS NULL
AND type = $2`,
[MARKDOWN_CHUNKER_VERSION, type],
);
return Number(rows[0]?.count ?? 0);
}
const rows = await engine.executeRaw<{ count: string | number }>(
`SELECT COUNT(*)::bigint AS count
FROM pages
WHERE page_kind = 'markdown'
AND ${driftPredicate}
AND deleted_at IS NULL`,
[MARKDOWN_CHUNKER_VERSION],
);
return Number(rows[0]?.count ?? 0);
}
/**
* Read a single batch of pending rows. Ordered by id so re-runs after
* partial completion pick up where they left off without re-doing pages
* whose chunker_version was already bumped.
*/
async function readBatch(
engine: BrainEngine,
batchSize: number,
type: string | null = null,
noEmbed = false,
afterId: number | null = null,
): Promise<Array<{ id: number; slug: string; source_path: string | null; compiled_truth: string; source_id: string }>> {
const driftPredicate = pendingDriftPredicate(noEmbed);
if (type) {
return engine.executeRaw(
`SELECT id, slug, source_path, compiled_truth, source_id
FROM pages
WHERE page_kind = 'markdown'
AND ${driftPredicate}
AND deleted_at IS NULL
AND type = $2
AND ($4::integer IS NULL OR id > $4)
ORDER BY id ASC
LIMIT $3`,
[MARKDOWN_CHUNKER_VERSION, type, batchSize, afterId],
);
}
return engine.executeRaw(
`SELECT id, slug, source_path, compiled_truth, source_id
FROM pages
WHERE page_kind = 'markdown'
AND ${driftPredicate}
AND deleted_at IS NULL
AND ($3::integer IS NULL OR id > $3)
ORDER BY id ASC
LIMIT $2`,
[MARKDOWN_CHUNKER_VERSION, batchSize, afterId],
);
}
/** How a caller already authorized the paid run (W4.5): the job handler's stored spend record, or the upgrade prompt's TTY yes. */
export interface ReindexRunOpts {
/** The paid run is already authorized; skip the consent gate. */
authorized?: boolean;
/** False for unattended callers (the job handler): the gate never prompts. */
interactive?: boolean;
}
export async function runReindex(engine: BrainEngine, args: string[], runOpts: ReindexRunOpts = {}): Promise<ReindexResult> {
args = normalizeReindexArgs(args);
const invalidType = args.some((arg, index) =>
arg === '--type' && !parsePageType(args[index + 1]));
if (invalidType) {
const payload = { error: 'invalid --type: expected a non-empty value, not another flag' };
if (args.includes('--json')) {
process.stdout.write(JSON.stringify(payload) + '\n');
} else {
process.stderr.write(`[reindex] ${payload.error}\n`);
}
setCliExitVerdict(2);
return { pending: 0, pendingAfter: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: args.includes('--dry-run'), chunkerVersion: MARKDOWN_CHUNKER_VERSION, type: null };
}
if (invalidPositiveIntegerFlag(args, '--limit')) {
const payload = { error: 'invalid --limit: expected a positive integer' };
if (args.includes('--json')) {
process.stdout.write(JSON.stringify(payload) + '\n');
} else {
process.stderr.write(`[reindex] ${payload.error}\n`);
}
setCliExitVerdict(2);
return { pending: 0, pendingAfter: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: args.includes('--dry-run'), chunkerVersion: MARKDOWN_CHUNKER_VERSION, type: null };
}
const invalidWorkerFlag = ['--workers', '--concurrency'].find((flag) =>
invalidPositiveIntegerFlag(args, flag));
const invalidRepo = invalidRequiredValueFlag(args, '--repo');
if (invalidWorkerFlag || invalidRepo) {
const error = invalidWorkerFlag
? `invalid ${invalidWorkerFlag}: expected a positive integer`
: 'invalid --repo: expected a path';
const payload = { error };
if (args.includes('--json')) {
process.stdout.write(JSON.stringify(payload) + '\n');
} else {
process.stderr.write(`[reindex] ${error}\n`);
}
setCliExitVerdict(2);
return { pending: 0, pendingAfter: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: args.includes('--dry-run'), chunkerVersion: MARKDOWN_CHUNKER_VERSION, type: null };
}
const opts = parseArgs(args);
const type = opts.type ?? null;
// Require `--markdown` explicitly. Future modes (e.g. --code) get their
// own routing here.
if (!args.includes('--markdown')) {
if (opts.json) {
process.stdout.write(JSON.stringify({ error: 'gbrain reindex requires a target flag, e.g. --markdown' }) + '\n');
} else {
process.stderr.write('Usage: gbrain reindex --markdown [--type PAGE_TYPE] [--limit N] [--dry-run] [--json] [--repo PATH]\n');
}
setCliExitVerdict(2);
return { pending: 0, pendingAfter: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: !!opts.dryRun, chunkerVersion: MARKDOWN_CHUNKER_VERSION, type };
}
const pending = await countPending(engine, type, !!opts.noEmbed);
if (opts.json && pending === 0) {
process.stdout.write(JSON.stringify({ pending: 0, pending_after: 0, reindexed: 0, skipped: 0, failed: 0, chunker_version: MARKDOWN_CHUNKER_VERSION, type }) + '\n');
return { pending: 0, pendingAfter: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: !!opts.dryRun, chunkerVersion: MARKDOWN_CHUNKER_VERSION, type };
}
if (pending === 0) {
const scope = type ? ` type=${type}` : '';
process.stderr.write(`[reindex] All markdown pages${scope} already at chunker_version ${MARKDOWN_CHUNKER_VERSION}. Nothing to do.\n`);
return { pending: 0, pendingAfter: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: !!opts.dryRun, chunkerVersion: MARKDOWN_CHUNKER_VERSION, type };
}
const target = typeof opts.limit === 'number' ? Math.min(opts.limit, pending) : pending;
if (opts.dryRun) {
if (opts.json) {
process.stdout.write(JSON.stringify({ pending, pending_after: pending, would_reindex: target, dry_run: true, chunker_version: MARKDOWN_CHUNKER_VERSION, type }) + '\n');
} else {
const scope = type ? ` (type=${type})` : '';
process.stderr.write(`[reindex] DRY-RUN: would re-chunk ${target} of ${pending} pending markdown pages${scope}.\n`);
}
return { pending, pendingAfter: pending, reindexed: 0, skipped: 0, failed: 0, dryRun: true, chunkerVersion: MARKDOWN_CHUNKER_VERSION, type };
}
// W4.5: re-embedding is paid work; ask before it starts (throws the exit-3 consent refusal or cost_cap_exceeded).
// W12 P1: the approved cap is enforced while the run spends (embedding and synopsis calls), not only checked up front.
let cap: BudgetTracker | null = null;
let capHit: BudgetExhausted | null = null;
if (!opts.noEmbed && !runOpts.authorized) {
const { requireReindexConsent } = await import('../core/reindex-consent.ts');
const gate = await requireReindexConsent(engine, { args, type, target, ...(runOpts.interactive === false ? { interactive: false } : {}) });
const capUsd = gate?.auth.cap_usd;
if (typeof capUsd === 'number' && Number.isFinite(capUsd) && capUsd > 0) {
cap = new BudgetTracker({ maxCostUsd: capUsd, label: 'reindex', capSource: gate!.auth.cap_source ?? undefined });
// An import swallows a refused embedding (its chunks wait for `embed --stale`), so the run stops at the next page.
const tracker = cap;
tracker.onExhausted(() => { capHit = new BudgetExhausted(`reindex reached its $${capUsd.toFixed(2)} cost cap`, { reason: 'cost', spent: tracker.totalSpent, cap: capUsd }); });
}
}
const metered = <T>(fn: () => Promise<T>): Promise<T> => (cap ? withBudgetTracker(cap, fn) : fn());
if (await managedPersistenceEnabled(engine)) {
try { return await metered(() => reindexManaged(engine, opts, type, pending)); } catch (e) {
if (!(e instanceof BudgetExhausted)) throw e;
return reportBudgetStop(engine, e, { pending, type, json: !!opts.json, reindexed: 0, skipped: 0, failed: 0 });
}
}
const reporter = createProgress(cliOptsToProgressOptions(getCliOptions()));
reporter.start('reindex.markdown', target);
let reindexed = 0;
let skipped = 0;
let failed = 0;
let afterId: number | null = null;
const BATCH = 100;
const repoPath = opts.repoPath ? resolve(opts.repoPath) : null;
// #5181: size the pool from the whole run, not one batch. A full batch is
// exactly the auto threshold (100), so batch length never enabled it.
const { workers } = resolveWorkersWithClamp(engine, opts.workers, 'reindex', target);
let stopped: BudgetExhausted | null = null;
try { await metered(async () => { while (reindexed + skipped + failed < target) {
const remaining = target - (reindexed + skipped + failed);
const batchSize = Math.min(BATCH, remaining);
const batch = await readBatch(engine, batchSize, type, !!opts.noEmbed, afterId);
if (batch.length === 0) break;
// Advance before processing so a failed row cannot be selected again in
// this invocation and starve later IDs.
afterId = batch[batch.length - 1]!.id;
// v0.41.15.0 (T10, D9): per-batch sliding pool. Counters are JS-
// single-thread atomic so reindexed++ / failed++ are race-free
// across workers.
await runSlidingPool({
items: batch,
workers,
failureLabel: (row) => row.slug,
onItem: async (row) => {
if (capHit) throw capHit;
reporter.tick();
try {
if (row.source_path && repoPath) {
const absPath = resolve(repoPath, row.source_path);
if (existsSync(absPath)) {
const imported = await importFromFile(engine, absPath, row.source_path, {
noEmbed: !!opts.noEmbed,
sourceId: row.source_id,
inferFrontmatter: false,
forceRechunk: true,
});
if (imported.status === 'imported') {
reindexed++;
} else if (imported.status === 'error') {
process.stderr.write(`[reindex] ${row.slug}: ${imported.error ?? 'import failed'}\n`);
failed++;
} else if (imported.error) {
process.stderr.write(`[reindex] ${row.slug}: ${imported.error}\n`);
failed++;
} else {
skipped++;
}
return;
}
}
// No source file on disk (DB-only page, or repo not available) —
// re-chunk from the stored page. v0.41.37.0 #1621: reconstruct the
// FULL markdown (frontmatter + body + timeline) via serializeMarkdown
// and re-import THAT, instead of passing body-only `compiled_truth`
// to importFromContent. The body-only path re-parsed with empty
// frontmatter and OVERWROTE the page's real frontmatter / title /
// timeline (codex catch). The round-trip preserves everything while
// still re-chunking + bumping chunker_version.
const page = await engine.getPage(row.slug, { sourceId: row.source_id });
if (!page) { skipped++; return; }
const tags = await engine.getTags(row.slug, { sourceId: row.source_id });
const fullMarkdown = serializeMarkdown(
page.frontmatter ?? {},
page.compiled_truth ?? row.compiled_truth,
page.timeline ?? '',
{ type: page.type, title: page.title, tags },
);
const imported = await importFromContent(engine, row.slug, fullMarkdown, {
sourceId: row.source_id, preserveGateMarkers: true,
noEmbed: !!opts.noEmbed,
forceRechunk: true,
});
if (imported.status === 'imported') {
reindexed++;
} else if (imported.status === 'error') {
process.stderr.write(`[reindex] ${row.slug}: ${imported.error ?? 'import failed'}\n`);
failed++;
} else if (imported.error) {
process.stderr.write(`[reindex] ${row.slug}: ${imported.error}\n`);
failed++;
} else {
skipped++;
}
} catch (err) {
if (err instanceof BudgetExhausted) throw err;
const msg = err instanceof Error ? err.message : String(err);
process.stderr.write(`[reindex] ${row.slug}: ${msg}\n`);
failed++;
}
},
});
} }); } catch (e) {
if (!(e instanceof BudgetExhausted)) throw e;
stopped = e;
}
reporter.finish();
if (stopped) return reportBudgetStop(engine, stopped, { pending, type, json: !!opts.json, reindexed, skipped, failed, workers });
if (reindexed > 0) await refreshProjectionStatistics(engine, reindexed);
const pendingAfter = await countPending(engine, type, !!opts.noEmbed);
if (failed > 0) setCliExitVerdict(1);
const result: ReindexResult = {
pending,
pendingAfter,
reindexed,
skipped,
failed,
dryRun: false,
chunkerVersion: MARKDOWN_CHUNKER_VERSION,
type,
workers,
};
if (opts.json) {
process.stdout.write(JSON.stringify({
pending, pending_after: pendingAfter, reindexed, skipped, failed,
chunker_version: MARKDOWN_CHUNKER_VERSION,
type,
workers,
}) + '\n');
} else {
const scope = type ? ` type=${type}` : '';
process.stderr.write(`[reindex] Done.${scope} reindexed=${reindexed} skipped=${skipped} failed=${failed} pending_before=${pending} pending_after=${pendingAfter} workers=${workers}\n`);
}
return result;
}
/**
* W12 P1: the run stopped at its approved cost cap. Pages re-chunked so far are
* kept (a rerun continues with the rest); exit 1 with the spend and the rerun.
*/
async function reportBudgetStop(engine: BrainEngine, e: BudgetExhausted, r: { pending: number; type: string | null; json: boolean; reindexed: number; skipped: number; failed: number; workers?: number }): Promise<ReindexResult> {
if (r.reindexed > 0) await refreshProjectionStatistics(engine, r.reindexed);
const pendingAfter = await countPending(engine, r.type, false);
setCliExitVerdict(1);
const message = `stopped at the $${e.cap.toFixed(2)} cost cap after spending $${e.spent.toFixed(4)} (${e.message}). ${pendingAfter} page(s) still pending, and pages re-chunked after the cap wait for gbrain embed --stale; ask the user before running again with a higher --max-usd.`;
if (r.json) {
process.stdout.write(JSON.stringify({ pending: r.pending, pending_after: pendingAfter, reindexed: r.reindexed, skipped: r.skipped, failed: r.failed,
chunker_version: MARKDOWN_CHUNKER_VERSION, type: r.type, budget_exhausted: { cap_usd: e.cap, spent_usd: e.spent, reason: e.reason } }) + '\n');
} else {
process.stderr.write(`[reindex] ${message}\n`);
}
return { pending: r.pending, pendingAfter, reindexed: r.reindexed, skipped: r.skipped, failed: r.failed, dryRun: false, chunkerVersion: MARKDOWN_CHUNKER_VERSION, type: r.type,
...(r.workers !== undefined ? { workers: r.workers } : {}) };
}
/**
* #5377: a managed brain cannot re-import through the legacy writer, so the
* sweep delegates to the projection repairs: `safe-chunks` re-seals every page
* chunked before the current chunker and `contextual-mode` stamps pages imported
* without a contextual mode (skipped under --no-embed, which does not count
* them as pending). Neither writes a page revision, journal request or
* canonical file. `--type` cannot narrow those repairs, so it is refused with
* the repair commands instead.
*/
async function reindexManaged(engine: BrainEngine, opts: ReindexOpts, type: string | null, pending: number): Promise<ReindexResult> {
const base = { pending, dryRun: false, chunkerVersion: MARKDOWN_CHUNKER_VERSION, type };
const repairs = opts.noEmbed ? ['safe-chunks'] : ['safe-chunks', 'contextual-mode'];
const redirect = repairs.map(kind => `gbrain repair ${kind} --apply${opts.noEmbed ? ' --no-embed' : ''}`).join(' && ');
if (type) {
const error = `--type cannot narrow a managed reindex; re-chunk every drifted page with: ${redirect}`;
if (opts.json) process.stdout.write(JSON.stringify({ error, delegated_to: repairs }) + '\n');
else process.stderr.write(`[reindex] ${error}\n`);
setCliExitVerdict(2);
return { ...base, pendingAfter: pending, reindexed: 0, skipped: 0, failed: 0 };
}
const { contextualModeRepair } = await import('../core/repair/contextual-mode.ts');
const { safeChunksRepair } = await import('../core/repair/safe-chunks.ts');
const { resolveRepairScope, runRepair } = await import('../core/repair/core.ts');
const ctx = { engine, config: loadConfig() ?? { engine: engine.kind }, remote: false, dryRun: false, sourceId: 'default',
logger: { info: (msg: string) => process.stderr.write(`${msg}\n`), warn: (msg: string) => process.stderr.write(`${msg}\n`), error: (msg: string) => process.stderr.write(`${msg}\n`) } };
const scope = await resolveRepairScope(engine);
const stopped: string[] = [];
const failures: string[] = [];
let unsupported = 0;
for (const handler of opts.noEmbed ? [safeChunksRepair] : [safeChunksRepair, contextualModeRepair]) {
const run = await runRepair(ctx as Parameters<typeof runRepair>[0], handler, scope, { apply: true, limit: opts.limit, embed: !opts.noEmbed });
if (run.stopped) stopped.push(run.stopped.message);
unsupported += Object.values(run.residuals).reduce((sum, count) => sum + count, 0);
// GBRA-130 A-26: pages the repair held (projection_unbuildable) or failed per item are counted, not thrown.
failures.push(...run.failed.map(f => `${f.item} (${f.reason}): ${f.message}`),
...(run.outcome_items ?? []).filter(o => o.outcome === 'held').map(o => `${o.item} (${o.reason}): ${String(o.detail?.message ?? '')}`));
}
const failed = failures.length;
if (pending > 0) await refreshProjectionStatistics(engine, pending);
const pendingAfter = await countPending(engine, type, !!opts.noEmbed);
const reindexed = Math.max(0, pending - pendingAfter);
if (stopped.length || failed) setCliExitVerdict(1);
if (opts.json) {
process.stdout.write(JSON.stringify({ pending, pending_after: pendingAfter, reindexed, skipped: 0, failed, chunker_version: MARKDOWN_CHUNKER_VERSION, type,
delegated_to: repairs, ...(unsupported ? { unsupported_pages: unsupported } : {}), ...(stopped.length ? { stopped } : {}), ...(failed ? { failures } : {}) }) + '\n');
} else {
process.stderr.write(`[reindex] Managed brain: re-chunked through ${repairs.map(kind => `gbrain repair ${kind}`).join(' and ')} (no page writes). `
+ `reindexed=${reindexed} pending_before=${pending} pending_after=${pendingAfter}\n`);
if (unsupported) process.stderr.write(`[reindex] ${unsupported} page(s) the projection repairs cannot rebuild (code without a source path, or another page kind) are left for their importer.\n`);
for (const message of stopped) process.stderr.write(`[reindex] ${message}\n`);
for (const failure of failures) process.stderr.write(`[reindex] not re-chunked, its chunks stay withheld: ${failure}. Retry with: gbrain repair safe-chunks --apply\n`);
}
return { ...base, pendingAfter, reindexed, skipped: 0, failed };
}
/**
* `gbrain reindex --segmented`: preview (default) or `--apply` the re-chunk of sealed segmented pages an older
* segmented chunker made (core/page-body/rechunk.ts). Nothing on upgrade or migration runs it.
*/
export async function runReindexSegmented(engine: BrainEngine, args: string[]): Promise<SegmentedRechunkReceipt | null> {
args = normalizeReindexArgs(args);
const json = args.includes('--json');
const invalid = ['--source', '--slug'].find(flag => invalidRequiredValueFlag(args, flag));
if (invalid) {
const error = `invalid ${invalid}: expected a value`;
if (json) process.stdout.write(JSON.stringify({ error }) + '\n');
else process.stderr.write(`[reindex] ${error}\n`);
setCliExitVerdict(2);
return null;
}
const value = (flag: string) => { const at = args.indexOf(flag); return at >= 0 ? args[at + 1] : undefined; };
const { runSegmentedRechunk, formatSegmentedRechunkReceipt } = await import('../core/page-body/rechunk.ts');
const receipt = await runSegmentedRechunk(engine, { apply: args.includes('--apply'), sourceId: value('--source'), slug: value('--slug') });
if (json) process.stdout.write(JSON.stringify(receipt, null, 2) + '\n');
else for (const line of formatSegmentedRechunkReceipt(receipt)) console.log(line);
if (receipt.pages.some(page => page.state === 'failed')) setCliExitVerdict(1);
return receipt;
}