Local-first π§ memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly π΄ dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Index lifecycle orchestrator.
*
* `indexVault` is the incremental path: walk β diff against stored
* documents β upsert/replace/delete. `reindexVault` is the atomic
* rebuild: write to `brain.sqlite.new`, then a same-file rename swap
* with `.bak` retention.
*
* `indexStatus`, `indexCheck`, `indexRootCoverage` and
* `indexBehindCount` are the read-side diagnostics that power
* `o2b search status|check`, the MCP status enrichment and the recall
* gate's coverage receipt.
*
* Anchored in docs/plans/2026-05-16-brain-search-design.md Β§6, Β§8,
* Β§13, Β§15.
*/
import { accessSync, constants, existsSync, mkdirSync, readFileSync, statSync } from "node:fs";
import { basename, dirname } from "node:path";
import { Database } from "bun:sqlite";
import {
resolveSemanticCapability,
SEMANTIC_CAPABILITY_CODE,
SEMANTIC_CAPABILITY_TIER,
semanticCapabilityIsBlocked,
semanticCapabilityLabel,
SEMANTIC_VECTOR_CODE,
} from "./capability-tier.ts";
import { sha256Hex } from "../integrity/digest.ts";
import { parseAuthoredAtSeconds } from "./authored-at.ts";
import { CHUNKER_VERSION, chunkMarkdown } from "./chunker.ts";
import { expandTextForCjkFts } from "./cjk-tokenizer.ts";
import { effectiveInputWindowTokens, passagePrefixSentByProvider } from "./embeddings/presets.ts";
import {
activeEmbeddingModel,
activeSpendQuote,
chunkHoldExclusion,
COST_GATE_KEY,
EMBEDDING_GATE_REASON,
type EmbeddingGateReason,
type EmbeddingSpendPlan,
FORCE_COST_FLAG,
formatEstimatedUsd,
planEmbeddingSpend,
priceSpendPlan,
unpricedRefusalMessage,
} from "./embedding-spend.ts";
import { embeddingPriceRecommendations } from "./price-recommendations.ts";
import type { EmbeddingPriceSource } from "./embeddings/pricing.ts";
import { makeProvider } from "./embeddings/provider.ts";
import { resolveProviderConsent, providerConsentIsDeclined } from "./provider-consent.ts";
import {
embeddingSignature,
isStaleSignature,
MODEL_NATIVE_DIMENSION,
textExtent,
tokenHoldFloor,
type EmbeddingIdentity,
} from "./embeddings/signature.ts";
import { LOCAL_DEFAULT_DIMENSION } from "./embeddings/local-provider.ts";
import {
describeCredentialSources,
type CredentialSourceContext,
} from "./embeddings/credential-report.ts";
import { resolveEventAnchor } from "./event-anchor.ts";
import { extractLinks } from "./links.ts";
import { resolveDocumentEventTimeWindow } from "./pipeline/event-time.ts";
import {
probeProvider,
PROVIDER_PROBE,
PROVIDER_PROBE_NOT_CONFIGURED,
PROVIDER_PROBE_SKIPPED,
type ProviderProbeState,
} from "./provider-probe.ts";
import { extractFrontmatterRelations } from "../graph/frontmatter-relations.ts";
import { loadSchemaPack, type SchemaPack } from "../brain/schema-pack.ts";
import { tieredFieldsForKind } from "../brain/frontmatter-tiers.ts";
import { parsePinned } from "../brain/preference.ts";
import { normalizeSchemaToken } from "../brain/schema-vocab.ts";
import type { DegradationNotice } from "../integrity/degradation.ts";
import { parseFrontmatterTextWithNotices } from "../vault.ts";
import { appendMetric } from "../brain/metrics.ts";
import type { MaintenanceSpendReceipt } from "../brain/maintenance/journal.ts";
import {
OPERATION,
progressCounter,
withProgressAsync,
type ProgressCounter,
} from "../brain/progress.ts";
import { throwIfAborted } from "../brain/safeguard.ts";
import { extractEntities } from "./entities.ts";
import { compareStamps, formatStampMismatch, type StampMismatch } from "../integrity/stamp.ts";
import { pathCovers } from "../vault-scope/defaults.ts";
import {
acquireWriterLease,
keepWriterLeasesFresh,
WRITER_LEASE_BLOCKING_BUDGET_MS,
writerLeaseCheckIn,
} from "./store/writer-lock.ts";
import {
describeUnreadableIndex,
embeddingAbiFixCommand,
formatEmbedderRecordContradiction,
formatEmbeddingAbiDrift,
peekPendingVectorsSync,
peekVisibilityColumnCensus,
readEmbedderRecordCensusSync,
readEmbeddingAbiSync,
runtimeEmbeddingAbi,
Store,
CHUNKER_VERSION_STATE_KEY,
EMBEDDING_PREFIX_QUERY_STATE_KEY,
EMBEDDING_PREFIX_PASSAGE_STATE_KEY,
LAST_FULL_INDEX_AT_STATE_KEY,
LAST_INDEXED_AT_STATE_KEY,
LINK_RESOLUTION_PENDING_STATE_KEY,
} from "./store.ts";
import { LATEST_SCHEMA_VERSION, documentAttributesFromFrontmatter } from "./schema.ts";
import { chunkWindowDiagnosticCode, CHUNK_HOLD_REASON, SearchError } from "./types.ts";
import { walkVault } from "./walker.ts";
import { forgetCatchUpState, ingestStoreInto } from "./brain-store/catch-up.ts";
import { purgeStoreRecords } from "./brain-store/ingest.ts";
import { STORE_RECORD_PATH_PREFIX } from "./brain-store/constants.ts";
import {
callableVisibilitySurfaces,
excludedCallableVisibilitySurfaces,
} from "./visibility-surface-registry.ts";
import type { ChunkInput, LinkInput } from "./store.ts";
import type { ChunkHoldExclusion, PendingVectorScope } from "./store/chunks.ts";
import type {
ChunkWindowCensus,
EmbedderRecordCensus,
HeldChunkRecord,
HeldChunkReport,
HeldChunkReportItem,
IndexCheckReport,
IndexStats,
IndexStatusSnapshot,
PendingVectorCensus,
ResolvedEmbeddingConfig,
ResolvedSearchConfig,
VisibilityHonestyFinding,
} from "./types.ts";
import type { ValidityWindow } from "./validity.ts";
import { REMOTE_DENY_VISIBILITY_TOKEN } from "../graph/visibility.ts";
import { remoteDenyVisibilityTokens } from "../graph/privacy-markers.ts";
import { closeDatabase } from "../sqlite-close.ts";
import { renameWithRetry, unlinkWithRetry } from "../fs-atomic.ts";
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
export interface IndexProgressEvent {
readonly path: string;
readonly kind: "added" | "updated" | "unchanged" | "deleted" | "error";
readonly message?: string;
}
export interface IndexVaultOptions {
readonly embeddings?: boolean;
/** When true, every file is reindexed even if hash + mtime match. */
readonly force?: boolean;
/** When true, bypass the embedding cost gate for this run. */
readonly forceCost?: boolean;
readonly onFile?: (event: IndexProgressEvent) => void;
/**
* Cooperative deadline (t_06784b8d): checkpointed once per walked
* file, so a tripped guard aborts between files - never mid-write.
*/
readonly safeguard?: import("../brain/safeguard.ts").Safeguard;
/**
* On-demand cancellation (Indexer Durability suite). Checked at the
* same boundaries the deadline uses - between files and between embed
* batches, never mid-write. An aborted signal throws
* `SafeguardAbortError`; the deletion sweep runs only on full
* completion, so an aborted run leaves a consistent partial index.
*/
readonly signal?: AbortSignal;
/**
* Live progress observer (nothing-runs-unwatched, U1). Distinct from
* {@link IndexVaultOptions.onFile}, which reports WHAT happened to each
* file for the `--verbose` human stream; this reports HOW FAR the run
* has got, in the one shape every long operation here uses.
*
* The walk phase carries no denominator and cannot: `walkVault` is a
* generator, and counting first costs a second full traversal. The
* embed phase carries one, because its pending list is an array.
*/
readonly onProgress?: import("../brain/progress.ts").ProgressSink;
}
/** Site recorded on the frontmatter notices an index run collects. */
const INDEX_FRONTMATTER_SITE = "search.indexVault";
interface MutableStats {
added: number;
updated: number;
unchanged: number;
deleted: number;
chunksTotal: number;
embeddingsComputed: number;
embeddingsReused: number;
embeddingsRetries: number;
/** Held-back-over-window chunks (F2); frozen out of the stats when zero. */
embeddingsHeld: number;
heldChunks: HeldChunkRecord[];
errors: Array<{ readonly path: string; readonly message: string }>;
frontmatterNotices: DegradationNotice[];
relationViolations: IndexStats["relationViolations"];
tierDrift: IndexStats["tierDrift"];
aliasResolved: number;
linkResolutionSkipped: boolean;
eventAnchorsPending: number;
backend: IndexStats["backend"];
deferredReason: IndexStats["deferredReason"];
/** Null until the census runs, and null whenever it has nothing to report. */
chunkWindow: ChunkWindowCensus | null;
/**
* The embedding phase's own cost-gate result, set when that phase ran
* its census (t_9d155d0e). Undefined when nothing was pending or the
* phase never engaged; a phase that throws mid-spend leaves the run
* throwing, so a receipt can only ever be read off a completed pass.
*/
spend: MaintenanceSpendReceipt | undefined;
}
function newStats(): MutableStats {
return {
added: 0,
updated: 0,
unchanged: 0,
deleted: 0,
chunksTotal: 0,
embeddingsComputed: 0,
embeddingsReused: 0,
embeddingsRetries: 0,
// Filled by the embedding phase only when it holds a chunk back;
// zero means the frozen stats carry neither hold field at all.
embeddingsHeld: 0,
heldChunks: [],
errors: [],
frontmatterNotices: [],
relationViolations: [],
tierDrift: [],
aliasResolved: 0,
linkResolutionSkipped: false,
// Counted at the end of the run, once every changed document has
// been upserted: what is left is what this run did not reach.
eventAnchorsPending: 0,
// Resolved lazily at the end of the run, after content detection.
// Defaults to the deterministic offline backend.
backend: "offline",
deferredReason: null,
// Taken at the end of the run, over the index the run left behind.
chunkWindow: null,
// Set by the embedding phase when it ran its census; absent otherwise.
spend: undefined,
};
}
/**
* The spend receipt an index run's embedding phase recorded, if any.
* Present only when a phase ran over a non-empty pending census and the
* run completed - a run that threw has no stats to read, and the lane
* journaling the receipt must not price a pass it cannot vouch for.
* The field's declared home is `IndexStats.spend`; this accessor keeps
* the "completed run only" reading in one named place.
*/
export function embeddingSpendOf(stats: IndexStats): MaintenanceSpendReceipt | undefined {
return stats.spend;
}
function freezeStats(s: MutableStats, durationMs: number): IndexStats {
return Object.freeze({
added: s.added,
updated: s.updated,
unchanged: s.unchanged,
deleted: s.deleted,
chunksTotal: s.chunksTotal,
embeddingsComputed: s.embeddingsComputed,
embeddingsReused: s.embeddingsReused,
embeddingsRetries: s.embeddingsRetries,
// Absent together, never one without the other: a run that held
// nothing emits the statistics it emitted before the hold existed.
...(s.embeddingsHeld === 0
? {}
: { embeddingsHeld: s.embeddingsHeld, heldChunks: Object.freeze([...s.heldChunks]) }),
errors: Object.freeze([...s.errors]),
frontmatterNotices: Object.freeze([...s.frontmatterNotices]),
relationViolations: Object.freeze([...s.relationViolations]),
tierDrift: Object.freeze([...s.tierDrift]),
aliasResolved: s.aliasResolved,
linkResolutionSkipped: s.linkResolutionSkipped,
eventAnchorsPending: s.eventAnchorsPending,
backend: s.backend,
deferredReason: s.deferredReason,
// Absent, not null, when the census has nothing to report: a run
// with no oversize chunks and a declared window produces the exact
// statistics object it produced before this field existed.
...(s.chunkWindow === null ? {} : { chunkWindow: s.chunkWindow }),
// Same additive-absent convention: a run whose embedding phase did
// not engage carries no receipt.
...(s.spend === undefined ? {} : { spend: s.spend }),
durationMs,
});
}
/**
* Explain why the semantic backend was not engaged, inspecting only the
* already-resolved config (never `process.env`).
*
* The sentence is the diagnostics registry's, resolved from a REGISTERED
* code through the advisory rail (provenance-at-the-boundary, F1). It
* used to be assembled here from four hand-written branches, which is the
* one thing the v1.40.0 rail exists to stop.
*
* Two of those branches are now the shared capability tier. The third -
* a fully configured vault that simply was not asked for embeddings this
* run - is an INDEX state rather than a capability fault and carries its
* own code, whose registered exit is the vector backfill verb. The
* fourth was unreachable: the only call site passed a constant `false`
* for `embeddingsRequested`, because the caller reaches this function
* exclusively on the else-arm of `if (opts?.embeddings)`. The parameter
* and the branch are gone rather than kept as a shape that could never
* be observed.
*/
async function offlineDeferredReason(config: ResolvedSearchConfig): Promise<string> {
const capability = resolveSemanticCapability(config.semantic);
return semanticCapabilityLabel(
semanticCapabilityIsBlocked(capability) ? capability.code : SEMANTIC_VECTOR_CODE.deferred,
);
}
/** Deep-ish equality for frontmatter scalar/array values. */
function tierValueEquals(left: unknown, right: unknown): boolean {
if (Array.isArray(left) && Array.isArray(right)) {
return left.length === right.length && left.every((v, i) => tierValueEquals(v, right[i]));
}
return left === right;
}
/**
* The page's declared frontmatter `type` as a normalized token, or
* null when undeclared or not a usable scalar. Tolerant by design:
* a malformed `type` value never fails indexing, it just leaves the
* page untyped for constraint purposes.
*/
function pageTypeFromFrontmatter(frontmatter: Record<string, unknown>): string | null {
const raw = frontmatter["type"];
if (typeof raw !== "string") return null;
const normalized = normalizeSchemaToken(raw);
return normalized.length > 0 ? normalized : null;
}
/**
* The page's declared frontmatter `aliases` as raw strings (the store
* normalises). Tolerant by design: a non-array value or non-string
* entries never fail indexing, they are simply skipped.
*/
function aliasesFromFrontmatter(frontmatter: Record<string, unknown>): string[] {
const raw = frontmatter["aliases"];
if (!Array.isArray(raw)) return [];
return raw.filter((v): v is string => typeof v === "string");
}
/**
* The persisted half of the shared event-time resolution (v13,
* t_9e1a4b3f): the two column bounds for the window
* `resolveDocumentEventTimeWindow` resolved under the rung order the
* query side applies.
*
* A null window (nothing declared anywhere) and an INVALID declared
* window (a `valid_from`/`valid_until` that failed to parse) both persist
* null/null - the unmeasured state, because that is exactly how
* `eventTimeInRange` judges both: by storage mtime. Persisting anything
* else for a broken declaration would make the census disagree with what
* queries actually apply. A null SIDE on a valid window is a declared
* open one, and is persisted as the null it is.
*/
function persistedEventTimeBounds(window: ValidityWindow | null): {
eventTimeMinMs: number | null;
eventTimeMaxMs: number | null;
} {
if (window === null || window.invalid) {
return { eventTimeMinMs: null, eventTimeMaxMs: null };
}
return { eventTimeMinMs: window.validFromMs, eventTimeMaxMs: window.validUntilMs };
}
/**
* The note's `authored_at` frontmatter instant as unix seconds, or null
* when undeclared, unparseable, or not usable as an authoring instant
* (conversation chronology, S1). Tolerant by design: a malformed value never
* fails indexing, it just leaves the document with no turn instant so it
* ranks byte-identically.
*
* This key is read off EVERY indexed markdown file - not only the artifacts
* session import and the inbox backfill write - so the rule that decides
* what counts as an authoring instant lives in `authored-at.ts` and is
* enforced HERE, at the boundary that fills the column, rather than being
* re-derived by each consumer of it. See {@link usableAuthoredAtSeconds} for
* the two refusals (pre-epoch sentinel, and an instant later than the moment
* the indexer read the file).
*/
function authoredAtFromFrontmatter(
frontmatter: Record<string, unknown>,
nowMs: number,
): number | null {
return parseAuthoredAtSeconds(frontmatter["authored_at"], nowMs);
}
const UTF8_FATAL = new TextDecoder("utf-8", { fatal: true });
function readUtf8(absPath: string): string {
const buf = readFileSync(absPath);
try {
return UTF8_FATAL.decode(buf);
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
throw new SearchError("INVALID_INPUT", `file is not valid UTF-8: ${absPath} (${msg})`);
}
}
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
// indexVault β incremental
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
/**
* Refuse a vault root that is not an existing directory.
*
* Both builds create the index directory under the vault (`mkdir -p` on
* `dirname(dbPath)`) and write metrics rows into `Brain/metrics`. Without
* this check a build pointed at a vault that is not there - a mistyped
* `--vault`, or a vault moved or deleted while a detached self-heal
* reindex was starting up - recreated the vault directory, filled it with
* an empty index and reported "added: 0 files" with exit 0.
*/
function assertVaultDirectory(vault: string | null): void {
// No vault: the index holds the brain store only, and there is nothing to check.
if (vault === null) return;
let isDir = false;
try {
isDir = statSync(vault).isDirectory();
} catch {
throw new SearchError("INVALID_INPUT", `vault not found: ${vault}`);
}
if (!isDir) {
throw new SearchError("INVALID_INPUT", `vault path is not a directory: ${vault}`);
}
}
export async function indexVault(
config: ResolvedSearchConfig,
opts?: IndexVaultOptions,
): Promise<IndexStats> {
return indexInto(config, opts);
}
/**
* @param outerProgress The RUN's counter when the caller owns it.
* `reindexVault` does, because its run is not over when this build is:
* the rebuild only becomes real at the swap. A build reporting into a
* borrowed counter must not terminate it - the owner does that, on the
* far side of the work this build does not know about.
*/
async function indexInto(
config: ResolvedSearchConfig,
opts?: IndexVaultOptions,
storeOverride?: Store,
outerProgress?: ProgressCounter,
): Promise<IndexStats> {
if (outerProgress !== undefined) {
outerProgress.start(INDEX_STAGE.walk);
return await indexIntoRun(config, outerProgress, opts, storeOverride);
}
const progress = progressCounter(OPERATION.reindex, opts?.onProgress);
progress.start(INDEX_STAGE.walk);
return await withProgressAsync(progress, () =>
indexIntoRun(config, progress, opts, storeOverride),
);
}
async function indexIntoRun(
config: ResolvedSearchConfig,
progress: ProgressCounter,
opts?: IndexVaultOptions,
storeOverride?: Store,
): Promise<IndexStats> {
assertVaultDirectory(config.vault);
const t0 = Date.now();
const ownsStore = !storeOverride;
const store = storeOverride ?? (await Store.open(config, { mode: "write" }));
const stats = newStats();
let linkResolutionMarked = false;
const markLinkResolutionPending = (): void => {
if (linkResolutionMarked) return;
linkResolutionMarked = true;
store.setState(LINK_RESOLUTION_PENDING_STATE_KEY, "1");
};
try {
const existing = store.listDocuments();
// An empty index holds nothing cut by older chunking rules, and every
// chunk this run writes is cut by the current ones: stamp it now, so
// an interrupted build that is resumed later is not mistaken for a
// stale one. See CHUNKER_VERSION_STATE_KEY for when else it is written.
if (existing.size === 0) {
store.setState(CHUNKER_VERSION_STATE_KEY, String(CHUNKER_VERSION));
}
const seen = new Set<string>();
// Changed docs declaring a frontmatter `kind`, for the tier-guard
// post-pass (write-time-integrity-governance).
const tieredDocs: Array<{
docId: number;
relPath: string;
frontmatter: Record<string, unknown>;
}> = [];
for (const file of walkVault(config)) {
// Cooperative deadline: abort between files, never mid-write.
// Per-file document upserts are transactional, so a tripped
// guard leaves a consistent (partially refreshed) index.
opts?.safeguard?.checkpoint();
// The walk is synchronous between embed batches, so the lock's
// event-loop heartbeat cannot fire during it; this beats instead.
writerLeaseCheckIn();
progress.advance(INDEX_STAGE.walk);
// On-demand cancellation, same boundary as the deadline.
throwIfAborted(opts?.signal, "index");
// Mark seen FIRST. If anything downstream throws (read fault,
// chunker bug, transient FS error), the file must not look
// "missing" to the deletion sweep below β that would wipe a
// present file from the index just because a single read failed.
seen.add(file.relPath);
try {
const mtimeSec = Math.floor(file.stat.mtimeMs / 1000);
const size = file.stat.size;
const prev = existing.get(file.relPath);
// Fastpath: if mtime and size both match, skip the full
// content read + SHA256 hash. This is the same trade-off
// as `make` β mtime can theoretically be fooled but the
// probability is negligible for vault editing workflows.
if (!opts?.force && prev && prev.mtime === mtimeSec && prev.size === size) {
stats.unchanged++;
opts?.onFile?.({ path: file.relPath, kind: "unchanged" });
continue;
}
const content = readUtf8(file.absPath);
// Reading (and below, chunking) one large note can take a while:
// the lease is checked again before this file's first write, so a
// write store that lost its lock meanwhile stops here.
writerLeaseCheckIn();
const contentHash = sha256Hex(content);
// Fallback: fastpath missed (mtime or size changed). If
// the content hash still matches, the file is logically
// unchanged β update stored mtime/size to re-arm the
// fastpath on the next run, but skip chunk/link work.
if (!opts?.force && prev && prev.contentHash === contentHash) {
stats.unchanged++;
// Re-arm the fastpath: store the current stat so next
// run can skip the read entirely.
store.touchDocument(file.relPath, mtimeSec, size);
opts?.onFile?.({ path: file.relPath, kind: "unchanged" });
continue;
}
markLinkResolutionPending();
const filenameBase = basename(file.relPath, ".md");
const chunkResult = chunkMarkdown(content, filenameBase, {
maxTokens: config.chunkSize,
minTokens: config.chunkMinSize,
overlapTokens: config.chunkOverlap,
});
for (const w of chunkResult.warnings) {
stats.errors.push({ path: file.relPath, message: w });
}
// The document's declared frontmatter `type` is persisted so
// the link-constraint post-pass can join endpoint types
// without re-reading files (v6).
// Unit F: the scanner drops a line it cannot express instead of
// failing, so a note keeps indexing while a field silently
// vanishes. Collect what it dropped; the run reports it and
// nothing about the indexing changes.
const [frontmatter, body, fmNotices] = parseFrontmatterTextWithNotices(content, {
site: INDEX_FRONTMATTER_SITE,
path: file.relPath,
});
for (const n of fmNotices) stats.frontmatterNotices.push(n);
// Resolved once, consumed twice: the materialised anchor column
// and the resolved event-time window below must come from the
// same anchor answer.
const eventAnchor = resolveEventAnchor(frontmatter, body);
writerLeaseCheckIn();
const docId = store.upsertDocument({
path: file.relPath,
title: chunkResult.title,
contentHash,
mtime: mtimeSec,
size: file.stat.size,
pageType: pageTypeFromFrontmatter(frontmatter),
// `t0` - the instant this run started - is the reading clock the
// "not later than the read" refusal measures against. One clock
// for the whole run, so two files with the same declared instant
// resolve identically however long the run takes.
authoredAt: authoredAtFromFrontmatter(frontmatter, t0),
// The event anchor is materialised HERE, on the same pass that
// already holds the parsed frontmatter and the body, so the
// query side never re-scans a candidate's body. It is a pure
// function of this file's content, which is exactly what makes
// it safe to store behind the two content-identity fastpaths
// above: unchanged content can only ever resolve to the same
// anchor, so declining to recompute it cannot stale it.
eventAnchor,
// The document's RESOLVED event-time window (v13,
// t_9e1a4b3f), resolved from the already-parsed frontmatter
// and the materialised anchor above through the SHARED
// per-document resolver - the query side's own rung order, so
// the persisted bounds and the filter that consumes them
// cannot drift. Like the anchor, a pure function of this
// file's content, safe behind the same fastpaths; bounds
// refresh on each document's next content change, which is
// the lazy backfill the migration plans for.
...persistedEventTimeBounds(resolveDocumentEventTimeWindow(frontmatter, eventAnchor)),
// What this run MEASURED of the page's `pinned` frontmatter
// flag (t_f7bef96a), coerced by the preference parser's own
// rule so ranking and the preference flows read one
// spelling. Every upserted document is measured; the column's
// NULL state is reserved for rows the v13 migration carried
// over and no run has re-examined yet.
pinned: parsePinned(frontmatter),
// What this run MEASURED of the page's visibility declaration,
// from the frontmatter already parsed above (v12). The indexer
// marks; it never skips. A `continue` here would be a live
// defect: `seen.add` runs BEFORE the content is read, so the
// deletion sweep would not purge a skipped page's stale rows and
// the page would keep whatever chunks it already had.
//
// The measurement is the UNION of the declared `visibility:`
// tokens and any informal privacy marker's reservation
// (`graph/privacy-markers.ts`), because the column is what a read
// root consults for bytes already captured: stamping the
// declaration alone would let a page reserved by
// `access: private` serve its snippet remotely after the file
// changed.
visibility: remoteDenyVisibilityTokens(frontmatter),
});
// Framework-kind files feed the tier-guard post-pass: keep the
// parsed frontmatter of this run's changed docs that declare a
// `kind` (bounded - only Brain artifacts carry one).
if (typeof frontmatter["kind"] === "string" && frontmatter["kind"].length > 0) {
tieredDocs.push({ docId, relPath: file.relPath, frontmatter });
}
// Vault-wide alias resolution (v7): replace this document's
// declared aliases so the post-pass can materialize alias
// wikilink targets.
store.replaceDocAliases(docId, aliasesFromFrontmatter(frontmatter));
// Bounded frontmatter attributes (v14, t_82851eda): replace this
// document's side-table rows from the SAME parsed frontmatter
// above - the parse that already fed the degradation notices, so
// an unterminated block yields no attributes and records no new
// notice. The extractor owns the promoted-key skips and the
// bounds; this call is a pure writer. Only a content change
// reaches this line, so the write cost rides the same lazy
// refresh every other materialized column pays.
store.replaceDocumentAttributes(docId, documentAttributesFromFrontmatter(frontmatter));
const chunkInputs: ChunkInput[] = chunkResult.chunks.map((c) => ({
chunkIndex: c.chunkIndex,
content: c.content,
ftsContent: expandTextForCjkFts(c.content),
contentHash: sha256Hex(c.content),
startLine: c.startLine,
endLine: c.endLine,
tokenCount: c.tokenCount,
headingPath: c.headingPath,
}));
// An unchanged chunk is kept in place, and a moved one carries its
// stored vector, so the embedding phase below never re-pays for it.
const replaced = store.replaceDocumentChunks(docId, chunkInputs);
const chunkIds = replaced.chunkIds;
stats.embeddingsReused += replaced.embeddingsReused;
const links: LinkInput[] = [];
for (let i = 0; i < chunkResult.chunks.length; i++) {
const cid = chunkIds[i]!;
const content = chunkResult.chunks[i]!.content;
const extracted = extractLinks(content);
for (const l of extracted) {
links.push({
sourceChunkId: cid,
targetPath: l.targetPath,
linkText: l.linkText,
linkType: l.linkType,
});
}
// Entity-boosted retrieval (v0.13.0): persist the chunk's
// deterministic entity set alongside its links. A chunk kept in
// place has the entities of the content it still holds - so a
// release that changes `extractEntities` must bump CHUNKER_VERSION,
// which rebuilds every chunk, or kept rows keep the old entities.
if (!replaced.keptChunkIds.has(cid)) store.replaceEntities(cid, extractEntities(content));
}
// Typed graph semantics (v3): frontmatter relation fields
// (related / extends / contradicts / superseded_by) become typed
// edges. They belong to the document, not a chunk, so anchor them
// on the first chunk (or null when the doc produced no chunks).
const relationChunkId = chunkIds[0] ?? null;
for (const edge of extractFrontmatterRelations(frontmatter)) {
links.push({
sourceChunkId: relationChunkId,
targetPath: edge.target,
linkText: null,
linkType: "wikilink",
relation: edge.relation,
});
}
store.replaceLinks(docId, links);
stats.chunksTotal += chunkInputs.length;
if (!prev) {
stats.added++;
opts?.onFile?.({ path: file.relPath, kind: "added" });
} else {
stats.updated++;
opts?.onFile?.({ path: file.relPath, kind: "updated" });
}
} catch (e) {
// A lost writer lock is the run's end, not this file's error.
if (e instanceof SearchError && e.code === "INDEX_LOCKED") throw e;
const msg = e instanceof Error ? e.message : String(e);
stats.errors.push({ path: file.relPath, message: msg });
opts?.onFile?.({ path: file.relPath, kind: "error", message: msg });
}
}
for (const [path] of existing) {
writerLeaseCheckIn();
if (!seen.has(path)) {
markLinkResolutionPending();
store.deleteDocument(path);
stats.deleted++;
opts?.onFile?.({ path, kind: "deleted" });
}
}
// Store records are left out of the sweep above; an index whose vault
// reads no store any more drops them here (design D5). A run that
// could not tell whether a store is in use keeps them: a store.json a
// sync tool is rewriting would otherwise cost a re-read of every shard.
if (!config.brainStore && config.brainStoreUnknown !== true) {
const purged = purgeStoreRecords(store);
if (purged !== null) {
forgetCatchUpState(config.dbPath);
if (purged > 0) markLinkResolutionPending();
stats.deleted += purged;
}
}
// Link and alias resolution are a pure function of the documents and
// their links. A run that changed no document (and was not forced)
// would only rewrite every link row to the value it already holds -
// the alias pass resets and re-sets each alias-resolved link - which
// is what made a no-change run write to disk. Skipped then; the
// constraint and tier passes below still run.
// A run killed between its document writes and this point leaves the
// pending marker set, and the next run resolves even if it changes
// nothing itself.
if (
stats.added + stats.updated + stats.deleted === 0 &&
opts?.force !== true &&
store.getState(LINK_RESOLUTION_PENDING_STATE_KEY) !== "1"
) {
stats.linkResolutionSkipped = true;
} else {
// Whole-table statements that block for as long as the index is
// large: the writer lock is kept fresh off this thread meanwhile.
stats.aliasResolved = keepWriterLeasesFresh(WRITER_LEASE_BLOCKING_BUDGET_MS, () => {
store.resolveLinkTargets();
// Alias post-pass (v7): exact path matches above always win; this
// only fills still-unresolved slash-free targets from doc_aliases.
return store.resolveAliasTargets();
});
store.setState(LINK_RESOLUTION_PENDING_STATE_KEY, "0");
}
// Link-constraint materialization post-pass
// (write-time-integrity-governance): recompute every typed edge's
// blocked flag from the current schema pack. Runs every pass - with
// no constraints declared it only resets stale flags, so removing a
// constraint restores edges without touching files. Fail-soft: an
// unparseable pack indexes as if no constraints were declared.
let pack: SchemaPack | null = null;
try {
pack = loadSchemaPack(config.vault);
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
stats.errors.push({ path: "Brain/_brain.yaml", message: `schema pack unreadable: ${msg}` });
}
stats.relationViolations = Object.freeze(
keepWriterLeasesFresh(WRITER_LEASE_BLOCKING_BUDGET_MS, () =>
store.recomputeRelationConstraintFlags(pack?.link_constraints ?? {}),
),
);
// Tier-guard post-pass: for each changed framework-kind doc,
// compare the new identity-field values against the stored
// snapshot. A changed identity value stages a tier_drift finding
// (ask_user via `o2b brain tiers check|restore|accept`) and the
// snapshot keeps the expected value so repeated reindexes never
// absorb the hand-edit; system/business fields update silently -
// framework writers mutate them legitimately on every pass.
if (pack !== null) {
const detectedAt = new Date().toISOString();
for (const doc of tieredDocs) {
const kind = doc.frontmatter["kind"];
if (typeof kind !== "string") continue;
const fields = tieredFieldsForKind(pack, kind);
if (Object.keys(fields).length === 0) continue;
const prev = store.getTierSnapshot(doc.docId);
const next: Record<string, unknown> = {};
for (const [field, tier] of Object.entries(fields)) {
const current = doc.frontmatter[field];
if (prev === null || !(field in prev)) {
if (current !== undefined) next[field] = current;
continue;
}
const expected = prev[field];
// Deleting an identity field is as much a hand-edit as
// changing it: `current === undefined` stages drift with a
// null actual instead of silently dropping the snapshot.
if (tier === "identity" && !tierValueEquals(expected, current)) {
const actual = current === undefined ? null : current;
store.upsertTierDrift({
documentId: doc.docId,
field,
expected,
actual,
detectedAt,
});
stats.tierDrift = [...stats.tierDrift, { path: doc.relPath, field, expected, actual }];
next[field] = expected; // the snapshot keeps the truth
continue;
}
store.clearTierDrift(doc.docId, field);
if (current !== undefined) next[field] = current;
}
store.setTierSnapshot(doc.docId, next);
}
}
// Lazy backend resolution (offline code-only extraction, t_85252236).
// The credential-gated embedding path runs only when explicitly
// requested; reaching past populateEmbeddings without throwing means a
// credential was resolved (or the local provider needs none), so the
// semantic backend is genuinely active. Otherwise the run is offline
// and we record why the semantic backend was deferred.
if (opts?.embeddings) {
await runEmbeddingPhase(store, config, stats, {
forceCost: opts?.forceCost === true,
...(opts?.safeguard !== undefined ? { safeguard: opts.safeguard } : {}),
...(opts?.signal !== undefined ? { signal: opts.signal } : {}),
progress,
});
stats.backend = "semantic";
stats.deferredReason = null;
} else {
stats.backend = "offline";
stats.deferredReason = await offlineDeferredReason(config);
}
// Documents this run left with no event anchor RESOLVED - not
// documents that declare no date. A schema bump migrates in place
// and reindexes nothing, and both fastpaths above correctly decline
// to recompute an anchor for content that did not change, so a
// document carried over from a pre-anchor binary is never examined
// by any number of index runs. Reported here because this run is the
// surface an upgrading operator reaches first, and because a hard
// `since` / `until` filter now depends on the answer.
stats.eventAnchorsPending = store.countUnexaminedEventAnchors();
// Oversize-chunk census. Taken here, over the index this run just
// wrote, because the chunk boundaries the run produced are exactly
// what the configured model's declared window has to hold - and the
// configured chunk size is expressed in the chunker's units (words,
// or characters for unspaced scripts such as Han) while a
// window is expressed in tokens, so the two settings can only be
// compared through what they actually produced. The stored model
// fills in a config that leaves `embedding_model` unset.
stats.chunkWindow = censusChunkWindow(store, config, store.getState("embedding_model"));
const now = new Date().toISOString();
store.setState(LAST_INDEXED_AT_STATE_KEY, now);
// The SAME instant on a forced run: their equality is the
// "this index resolved the whole vault" predicate the broken-link
// ratchet verifies before it trusts a count (unit G).
if (opts?.force) store.setState(LAST_FULL_INDEX_AT_STATE_KEY, now);
// A completed forced run re-chunked every document it kept.
if (opts?.force) store.setState(CHUNKER_VERSION_STATE_KEY, String(CHUNKER_VERSION));
// Bump the corpus-generation revision whenever the index actually
// changed, so the persistent query cache (v0.20.0) is invalidated
// after a content reindex even though the embedding model/dimension
// and schema are unchanged.
if (stats.added + stats.updated + stats.deleted > 0) {
store.bumpIndexRevision();
// Dashboard contract (link-recall-intelligence): one run-level
// record per non-empty index run. Fail-soft - a metrics-layer
// problem never fails the index.
try {
appendMetric(config.vault, {
surface: "index",
runAt: now,
payload: {
added: stats.added,
updated: stats.updated,
deleted: stats.deleted,
alias_resolved: stats.aliasResolved,
relation_violations: stats.relationViolations.length,
tier_drift: stats.tierDrift.length,
},
});
} catch {
// Metrics are observability, not correctness.
}
}
return freezeStats(stats, Date.now() - t0);
} finally {
if (ownsStore) await store.close();
}
}
/**
* Canonical signature of the ACTIVE embedding configuration for status
* reporting. Null when semantic search is disabled. Stored model and
* dimension fill in fields the config leaves unset (e.g. an
* auto-detected dimension or the local provider's implicit model).
*/
function activeEmbeddingSignature(
config: ResolvedSearchConfig,
storedModel: string | null = null,
storedDim: number | null = null,
): string | null {
if (!config.semantic.enabled) return null;
return embeddingSignature({
provider: config.semantic.provider,
model: activeEmbeddingModel(config, storedModel),
dimension: config.semantic.dimension ?? storedDim,
});
}
/**
* The config key whose value produced the chunk boundaries under census.
* Named once so the warning and any future surface cannot spell it
* differently from `resolveSearchConfig`.
*/
const CHUNK_SIZE_CONFIG_KEY = "search_chunk_size";
/**
* Take the oversize-chunk census over an open index.
*
* Returns `null` when there is nothing to report, which covers three
* genuinely different silences, all of them states in which no chunk can
* overflow a window:
*
* - semantic search is off, so nothing is ever embedded;
* - the provider is the offline local embedder, which hashes n-gram
* features over the whole text - no encoder, no positional limit,
* nothing to truncate. A provably absent window, not an unknown one;
* - the census ran against a declared window and every chunk fits.
*
* The silences it does NOT produce are the two states the estimate
* cannot back: a model with no declared window (`window-undeclared`),
* and chunks the census cannot place inside a declared one
* (`estimate-undecided`). A check that could not run and a check that
* passed are not the same statement.
*/
function censusChunkWindow(
store: Store,
config: ResolvedSearchConfig,
storedModel: string | null,
): ChunkWindowCensus | null {
if (!config.semantic.enabled) return null;
if (config.semantic.provider === "local") return null;
const chunksMeasured = store.chunkCount();
if (chunksMeasured === 0) return null;
const model = activeEmbeddingModel(config, storedModel);
// The one resolver the query fit reads too: an operator-declared window
// enables the census for an uncurated model.
const windowTokens = effectiveInputWindowTokens(config.semantic, model);
if (model === null || windowTokens === null) {
return Object.freeze({ verdict: "window-undeclared" as const, model, chunksMeasured });
}
// The prefix the CONFIGURED backend prepends, not the one the config
// asks for: only `openai-compat` implements the instruction-prefix
// contract, and subtracting a prefix a backend never sends would
// report an overflow nobody can hit.
const tally = store.chunkWindowTally(
windowTokens,
passagePrefixSentByProvider(config.semantic.provider, config.semantic.passagePrefix),
);
if (tally.overWindow > 0) {
return Object.freeze({
verdict: "over-window" as const,
model,
windowTokens,
chunksOverWindow: tally.overWindow,
// Absent, never zero: an all-ASCII index has no undecided band and
// carries the record it carried before this field existed.
...(tally.undecided > 0 ? { chunksUndecided: tally.undecided } : {}),
chunksMeasured,
});
}
if (tally.undecided > 0) {
return Object.freeze({
verdict: "estimate-undecided" as const,
model,
windowTokens,
chunksUndecided: tally.undecided,
chunksMeasured,
});
}
return null;
}
/**
* The operator-facing warning line for a census that MEASURED something
* - chunks proved over the window, or chunks its estimates cannot place
* inside one - ending in the command its REGISTERED diagnostic names.
* The command is resolved from the registry rather than written here, so
* this sentence cannot name an exit the registry does not hold; an
* unregistered code throws at the point the sentence is built rather
* than shipping a dead end.
*
* Typed to the two measured members, so the undeclared verdict cannot
* reach the warning channel by accident - it is a snapshot field, and
* the type is what enforces that rather than a branch nobody calls.
*
* The undecided line says in words that it is not a pass, because that
* is the whole content of the state: the operator is being told the
* check ran and could not clear these chunks, not that they are broken.
*/
async function formatChunkWindowMeasured(
census: Extract<ChunkWindowCensus, { verdict: "over-window" | "estimate-undecided" }>,
): Promise<string> {
const { requireNextStep } = await import("../brain/next-step.ts");
const step = requireNextStep(chunkWindowDiagnosticCode(census));
const levers =
`Two levers change the outcome: split the source by lowering ` +
`${CHUNK_SIZE_CONFIG_KEY}, or exclude the page from embedding. Then run: ${step.nextCommand}`;
if (census.verdict === "estimate-undecided") {
return (
`${census.chunksUndecided} of ${census.chunksMeasured} indexed chunk(s) cannot be ` +
`placed inside the ${census.windowTokens}-token input window declared for ` +
`${census.model}: the census cannot bound a token window with character counts - the ` +
`average-rate estimate reads high on ordinary prose, and non-Latin scripts and dense ` +
`symbols cost far more than four characters per token - so this is not a pass. ` +
`Undecided chunks are still sent, and whatever does not fit is truncated by the ` +
`provider, silently - ${levers}`
);
}
const undecided =
census.chunksUndecided === undefined
? ""
: ` A further ${census.chunksUndecided} chunk(s) the census cannot place either way.`;
return (
`${census.chunksOverWindow} of ${census.chunksMeasured} indexed chunk(s) cannot fit the ` +
`${census.windowTokens}-token input window declared for ${census.model} even at a ` +
`conservative estimate; chunks proved over the window are held back from embedding ` +
`rather than sent and truncated, and \`o2b search status\` names the held ones in its ` +
`held-chunks report. ${levers}${undecided}`
);
}
/**
* The two counters the vector phase writes back. Declared as its own
* shape so the phase can serve the index run's `MutableStats` and the
* standalone vector backfill without either knowing about the other;
* `MutableStats` satisfies it structurally.
*/
/**
* The spans an index run has. `walk` cannot carry a denominator -
* `walkVault` is a generator and counting first costs a second full
* traversal - while `embed` always can, because its pending list is an
* array before the loop starts. One event shape carries all of them.
*
* `lock` and `swap` belong to a REBUILD only, and both exist because a
* rebuild is not over when its build is. `lock` covers the wait on
* `acquireWriterLock`, which can be the length of a competing reindex and
* which used to emit nothing at all - the silence this release calls
* indistinguishable from a hang. `swap` covers the two renames that make
* the staging build real; before it, the stream reported `finished` while
* the live index was still the old one.
*/
const INDEX_STAGE = Object.freeze({
lock: "lock",
walk: "walk",
embed: "embed",
swap: "swap",
} as const);
export interface EmbeddingPhaseTally {
embeddingsComputed: number;
embeddingsRetries: number;
/**
* Chunks this phase held back as provably over the model's declared
* input window (chunk skip-and-hold, F2), and one record per chunk.
* Absent until the phase holds something: a tally a hold-free run
* owns never carries either, which is what lets the stats freeze drop
* both together.
*/
embeddingsHeld?: number;
heldChunks?: HeldChunkRecord[];
/**
* The cost gate's own verdict for this phase's census, set just before
* the first provider call (t_9d155d0e). `forced` is true only when a
* force bypass overrode a gate that would have refused the spend, so a
* receipt can name its own bypass. Optional and undefined until then:
* a tally an embedding-free caller owns never carries one.
*/
spend?: MaintenanceSpendReceipt;
}
export interface EmbeddingPhaseOptions {
/** Bypass the configured spend ceiling for this run. */
readonly forceCost?: boolean;
/**
* Path scope of the pending census. It reaches the census only through
* the spend plan, so the gate, the refusal and the receipt read the
* same chunks the phase embeds. Vault-wide when absent.
*/
readonly scope?: PendingVectorScope;
/**
* A spend plan the caller already computed on this store and over the
* same `scope`; the phase embeds its pending census instead of reading
* it a second time, so a caller that reported the plan embeds exactly
* what it priced. Its price and gate verdict are recomputed here from
* the current config, so a forced or stale verdict is never trusted.
* Recomputed in full when absent.
*/
readonly plan?: EmbeddingSpendPlan;
readonly safeguard?: import("../brain/safeguard.ts").Safeguard;
readonly signal?: AbortSignal;
/**
* The RUN's progress counter, not a sink of its own.
*
* This phase is the half of an index run that can report a fraction -
* the pending list is an array before the loop starts - but it is a
* PHASE, not a run: it is called both from `indexInto` and standalone
* by the vector backfill. Given a sink it would build a second counter,
* and a stream carrying two terminators cannot say which one ended the
* run. So the caller that owns the run owns the counter, and this phase
* borrows it. That is why `progress-census.test.ts` carries a written
* exemption for this interface rather than a sink.
*/
readonly progress?: import("../brain/progress.ts").ProgressCounter;
}
/** One entry of the embedding phase's pending census. */
type PendingCensusEntry = EmbeddingSpendPlan["pending"][number];
/**
* Split a pending census into what the embedding phase may send and what
* it must HOLD back (chunk skip-and-hold, F2).
*
* The hold acts on proof only. A chunk is held when the conservative
* bound - code points charged at no more than one token per
* `MAX_CODE_POINTS_PER_TOKEN` (embeddings/signature.ts), through the
* prefix the configured backend actually prepends - already exceeds the
* model's effective declared window: real tokenizers absorb several code
* points per token, so a text over even that generous charge cannot fit.
* The chars-over-four
* average is deliberately NOT the trigger: it is an average, not a bound,
* and holding on it read whole English chunks that fit the window as over
* it - recall lost on an arithmetic guess.
*
* Everything between that bound and the average-rate estimate is the
* undecided band: those chunks stay embedded exactly as before this
* feature, because acting there would hold work on a guess, and the window
* census beside this phase reports the band rather than passing it in
* silence. A model with no declared window (or the local embedder, which
* has no window by construction) proves nothing at all - for them nothing
* is held and every chunk stays pending.
*
* The hold bound is the same measurement the oversize-chunk census makes
* its `over-window` arm with, so the phase and the report can never
* disagree about which chunks are provably over.
*/
function partitionHeldChunks(
pending: ReadonlyArray<PendingCensusEntry>,
semantic: ResolvedEmbeddingConfig,
model: string | null,
): { readonly embeddable: Array<PendingCensusEntry>; readonly held: HeldChunkRecord[] } {
const windowTokens = effectiveInputWindowTokens(semantic, model);
if (windowTokens === null) return { embeddable: [...pending], held: [] };
// Charge the text the provider will actually receive: only
// `openai-compat` prepends the configured passage prefix, and a
// prefix nobody sends cannot push a chunk over a window.
const prefix = passagePrefixSentByProvider(semantic.provider, semantic.passagePrefix);
const embeddable: Array<PendingCensusEntry> = [];
const held: HeldChunkRecord[] = [];
for (const chunk of pending) {
const heldTokens = tokenHoldFloor(textExtent(prefix + chunk.content));
if (heldTokens > windowTokens) {
held.push(
Object.freeze({
chunkId: chunk.chunkId,
estimatedTokens: heldTokens,
windowTokens,
reason: CHUNK_HOLD_REASON.overDeclaredWindow,
}),
);
} else {
embeddable.push(chunk);
}
}
return { embeddable, held };
}
/**
* Compute and store a vector for every indexed chunk that has none.
*
* Chunks whose token estimate PROVABLY exceeds the model's effective
* declared input window are held back instead (see
* {@link partitionHeldChunks}): they are never sent, never stored and
* never priced, and the tally records each one.
*
* Exported (provenance-at-the-boundary, F) because it is the whole of
* `o2b search vector-backfill --apply`: the population step is already
* resumable by construction - the anti-join finds exactly the chunks
* still missing a row and `vecUpsert` commits per chunk - so the backfill
* verb is this phase run on its own, not a second implementation of it.
*
* Every refusal here is a SearchError, because reaching this function is
* an attempt: the capability tier is what a caller reports when it
* declines to attempt.
*/
export async function runEmbeddingPhase(
store: Store,
config: ResolvedSearchConfig,
stats: EmbeddingPhaseTally,
opts: EmbeddingPhaseOptions = {},
): Promise<void> {
const forceCost = opts.forceCost === true;
const safeguard = opts.safeguard;
const signal = opts.signal;
const progress = opts.progress;
if (!config.semantic.enabled) {
throw new SearchError(
"EMBEDDING_DISABLED",
"set search_semantic_enabled=true and embedding_* keys to compute embeddings",
);
}
// The offline local provider needs no key; every remote provider does.
if (config.semantic.provider !== "local" && !config.semantic.apiKey) {
throw new SearchError(
"EMBEDDING_KEY_MISSING",
"embedding_api_key is required when computing embeddings",
);
}
if (!store.vecLoaded()) {
throw new SearchError(
"VEC_EXTENSION_UNAVAILABLE",
"sqlite-vec did not load; cannot store embeddings",
);
}
// A handed-in plan keeps its census but is priced and gated again here,
// unforced: the verdict this phase acts on is always its own.
const initialPlan =
opts.plan !== undefined
? priceSpendPlan(opts.plan.pending, config)
: planEmbeddingSpend(store, config, opts.scope ? { scope: opts.scope } : {});
const pending = initialPlan.pending;
if (pending.length === 0) return;
const consent = resolveProviderConsent({
configPath: config.configPath ?? null,
env: process.env,
semantic: config.semantic,
});
const provider = makeProvider(config.semantic, consent);
const model = config.semantic.model ?? provider.model;
// Skip-and-hold (F2): chunks provably over the declared window leave
// the census BEFORE it is priced, batched or gated, so held work can
// neither trip the cost gate nor appear on the receipt. Filtering ahead
// of the batch loop is also what keeps the 1:1 invariant below intact:
// the provider is only ever asked about chunks it will answer for.
const { embeddable, held } = partitionHeldChunks(pending, config.semantic, model);
if (held.length > 0) {
stats.embeddingsHeld = (stats.embeddingsHeld ?? 0) + held.length;
(stats.heldChunks ??= []).push(...held);
// The record is what makes the hold outlive the run: the pending
// anti-join excludes these chunks under this key, so the status
// estimate, the preview and the next census stop pricing work this
// config proved it will not spend. A hold the config-side resolver
// cannot key is not persisted - there is no stable identity to
// release it under.
const exclusion = chunkHoldExclusion(config);
if (exclusion !== null) {
store.recordChunkHolds(
held.map((h) => ({
chunkId: h.chunkId,
estimatedTokens: h.estimatedTokens,
windowTokens: h.windowTokens,
})),
{ model: exclusion.model, embeddingSignature: exclusion.embeddingSignature },
);
}
}
// A run that holds everything pending spends nothing: no provider
// request, no gate verdict, and no receipt on a census it will not
// embed.
if (embeddable.length === 0) return;
const plan = held.length > 0 ? priceSpendPlan(embeddable, config) : initialPlan;
// Cost gate: the shared spend plan estimates the whole pending set up
// front and the run is refused when it exceeds the configured ceiling,
// unless forced.
//
// The plan is evaluated UNFORCED so the refusal and the receipt read
// one verdict: `blocked && !forceCost` is exactly the blocked flag a
// forced plan would have returned, and `blocked && forceCost` is the
// honest meaning of the receipt's `forced` - this run overrode a gate
// that would have refused it.
if (plan.gate.blocked && !forceCost) {
const gateUsd = config.semantic.costGateUsd;
if (plan.gate.reason === EMBEDDING_GATE_REASON.unpriced) {
throw new SearchError("EMBEDDING_COST_UNPRICED", unpricedRefusalMessage(plan, gateUsd));
}
throw new SearchError(
"EMBEDDING_COST_GATE",
`estimated embedding cost ${formatEstimatedUsd(plan.estimatedUsd)} for ${plan.pending.length} chunk(s) ` +
`exceeds ${COST_GATE_KEY} ${formatEstimatedUsd(gateUsd)}. ` +
`Re-run with ${FORCE_COST_FLAG} to proceed or raise the gate.`,
);
}
stats.spend = {
model: plan.model,
tokens: plan.tokens,
estimatedUsd: plan.estimatedUsd,
forced: plan.gate.blocked && forceCost,
priceSource: plan.quote.source,
};
const batchSize = Math.max(1, config.semantic.batchSize);
// Hand the provider a super-batch large enough to keep its internal
// `embedding_concurrency` semaphore busy. Without this multiplier the
// indexer's outer loop would serialise provider.embed() calls and the
// configured concurrency would never kick in. It is an upper bound, not
// an exact fill: when `embedding_batch_tokens` is set the provider closes
// a batch on the token estimate as well as on the count, so a super-batch
// may split into more - and therefore smaller - requests than
// `embedding_concurrency`.
const superBatch = batchSize * Math.max(1, config.semantic.concurrency);
progress?.start(INDEX_STAGE.embed, embeddable.length);
for (let i = 0; i < embeddable.length; i += superBatch) {
// Cooperative deadline: embedding batches are the other long
// phase of an index run - abort between batches, never mid-batch.
safeguard?.checkpoint();
throwIfAborted(signal, "index");
const batch = embeddable.slice(i, i + superBatch);
progress?.advance(INDEX_STAGE.embed, batch.length);
const texts = batch.map((p) => p.content);
const vectors = await provider.embed(texts, "passage");
stats.embeddingsRetries += provider.consumeRetryCount?.() ?? 0;
if (vectors.length !== batch.length) {
throw new SearchError(
"EMBEDDING_PROVIDER_HTTP",
`provider returned ${vectors.length} vectors for ${batch.length} inputs`,
);
}
// Lock in the auto-detected dimension on the very first batch.
const dim = provider.dimension ?? vectors[0]?.length ?? 0;
if (dim <= 0) {
throw new SearchError(
"EMBEDDING_DIMENSION_MISMATCH",
"provider returned vectors of zero length",
);
}
store.ensureEmbeddingModel(model, dim, {
query: config.semantic.queryPrefix ?? "",
passage: config.semantic.passagePrefix ?? "",
});
for (let j = 0; j < batch.length; j++) {
const chunkId = batch[j]!.chunkId;
const vec = vectors[j]!;
const embHash = sha256Hex(vec.map((x) => x.toFixed(8)).join(","));
store.vecUpsert(chunkId, vec, model, dim, embHash);
stats.embeddingsComputed++;
}
}
}
/** What an embedding pass about to run on this config is predicted to spend. */
export interface EmbeddingSpendPreview {
/** The model the pass would name; null when the config leaves it unset. */
readonly model: string | null;
/** Chunks the phase's own anti-join still finds vectorless. */
readonly pendingChunks: number;
readonly tokens: number;
/** Null when nobody stated the model's price. */
readonly estimatedUsd: number | null;
/** Who stated the price the estimate used. */
readonly priceSource: EmbeddingPriceSource;
/**
* True when the configured gate would refuse this spend unforced - the
* fact a `--force-cost` receipt is honest about.
*/
readonly blocked: boolean;
/** Why the gate would refuse; null when it would not. */
readonly reason: EmbeddingGateReason | null;
}
/**
* The spend preview the maintenance lane announces before its reindex
* task and records on its journal row after it (t_9d155d0e).
*
* Deliberately a READ over the same spend plan {@link runEmbeddingPhase}
* gates on - the phase's guards, in its order, and `planEmbeddingSpend`
* (`embedding-spend.ts`) - so what a run ANNOUNCES and what it later RECORDS
* cannot drift from what would actually be refused or embedded. This
* function never spends: the phase remains the only spender, and a
* refusal it would throw stays its to throw.
*
* Null means "this run cannot spend", per guard: semantic not configured
* or its credential missing; the index missing entirely (the next pass
* builds it, so the pending census does not exist yet); sqlite-vec not
* loadable; or nothing pending. Every one of those is a verdict the
* caller may print or omit - never an error - and for that reason the
* banner it feeds is printed ONLY when this returns a preview.
*/
export async function estimatePendingEmbeddingSpend(
config: ResolvedSearchConfig,
): Promise<EmbeddingSpendPreview | null> {
if (semanticCapabilityIsBlocked(resolveSemanticCapability(config.semantic))) return null;
let store: Store;
try {
store = await Store.open(config, { mode: "read" });
} catch (e) {
if (e instanceof SearchError && e.code === "INDEX_MISSING") return null;
throw e;
}
try {
if (!store.vecLoaded()) return null;
const plan = planEmbeddingSpend(store, config);
if (plan.pending.length === 0) return null;
return {
model: plan.model,
pendingChunks: plan.pending.length,
tokens: plan.tokens,
estimatedUsd: plan.estimatedUsd,
priceSource: plan.quote.source,
blocked: plan.gate.blocked,
reason: plan.gate.reason,
};
} finally {
await store.close();
}
}
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
// reindexVault β atomic full rebuild
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
export async function reindexVault(
config: ResolvedSearchConfig,
opts?: IndexVaultOptions,
): Promise<IndexStats> {
const stats = await reindexVaultUnless(config, null, opts);
// No predicate, so the rebuild always runs.
return stats!;
}
/**
* {@link reindexVault} that re-checks, once it HOLDS the writer lock,
* whether the rebuild is still wanted: `stillCurrent` returning true
* abandons it before anything is built or swapped, and the call returns
* `null`. The self-heal paths pass their inspection here, so a second
* process that queued behind a rebuild another process just finished does
* not rebuild again over it (and over its `.bak`).
*/
export async function reindexVaultUnless(
config: ResolvedSearchConfig,
stillCurrent: (() => boolean) | null,
opts?: IndexVaultOptions,
): Promise<IndexStats | null> {
// The rebuild owns the run's counter from before the lock wait to after
// the swap, and hands it to the staging build rather than letting that
// build open one of its own. Two reasons, and they are the same reason
// twice: a rebuild is not finished when its build is. `indexVault`'s
// own counter terminated at the end of the build, so a caller tailing
// the stream read `finished` while the live index was still the old one
// and the rename that makes the rebuild real had not been attempted -
// and if that rename then failed (ENOSPC, EPERM, a cross-device
// `dbPath`), the command exited non-zero after a terminator that said
// the run had finished.
const progress = progressCounter(OPERATION.reindex, opts?.onProgress);
return await withProgressAsync(progress, () => reindexInto(config, progress, stillCurrent, opts));
}
async function reindexInto(
config: ResolvedSearchConfig,
progress: ProgressCounter,
stillCurrent: (() => boolean) | null,
opts?: IndexVaultOptions,
): Promise<IndexStats | null> {
const newPath = config.dbPath + ".new";
const bakPath = config.dbPath + ".bak";
assertVaultDirectory(config.vault);
mkdirSync(dirname(config.dbPath), { recursive: true });
// Opened BEFORE the wait, not after it. `acquireWriterLock` can block
// for the whole duration of a competing reindex, and a stream that
// emits nothing while it does is the state this release names as
// indistinguishable from a hang.
progress.start(INDEX_STAGE.lock, 1);
// Hold the writer lock on the LIVE index path for the whole rebuild +
// swap. Keyed on `config.dbPath` (not the `.new` staging path), so a
// second concurrent reindex β the double-reindex a schema-bump upgrade
// triggers when CLI and the long-lived MCP server both self-heal at
// once β waits-or-bails on the SAME lock instead of unlinking this run's
// in-progress staging DB and then having its empty seed swapped over the
// live index (INDEX_UNREADABLE, silent data loss). The staging-DB opens
// below lock a different path (`.new`), so there is no self-deadlock.
const lease = await acquireWriterLease(config.dbPath);
progress.advance(INDEX_STAGE.lock);
try {
if (stillCurrent !== null && stillCurrent()) return null;
// Build into the temp file with an override config.
const tempConfig: ResolvedSearchConfig = Object.freeze({
...config,
dbPath: newPath,
});
// Resumable staging (opt-in). A compatible in-progress `.new` from an
// interrupted run is resumed via the incremental fastpath instead of
// rebuilt from scratch; an incompatible or unreadable one is
// discarded. With the flag OFF, the temp file is always rebuilt fresh
// and no staging marker is written or read - byte-identical to the
// pre-suite path (no extra staging-DB open/close cycles).
let resume = false;
if (config.resumeReindex) {
const signature = reindexStagingSignature(config, opts?.embeddings === true);
// An unknown-identity embedding component (an unnamed model) is
// compatible with nothing, including a staging build stamped under
// the same unknown identity: such a build always rebuilds.
if (existsSync(newPath)) {
resume = await stagingSignatureMatches(tempConfig, signature);
}
if (!resume) tryUnlink(newPath);
// Stamp the signature so an interruption leaves a build a later run
// can recognise and resume. An unknown-identity signature is stamped
// too: it documents the identity, and the comparison above never
// lets it match.
await withStagingStore(tempConfig, (store) =>
store.setState(REINDEX_SIGNATURE_KEY, JSON.stringify(signature)),
);
} else {
tryUnlink(newPath);
}
// `force: false` on resume lets the fastpath skip the files the
// partial build already committed; a fresh build forces every file.
// `onProgress` is dropped from the forwarded options: the build
// reports into THIS run's counter, passed explicitly, rather than
// opening a second one over the same sink.
let stats = await indexInto(
tempConfig,
{ ...opts, force: !resume, onProgress: undefined },
undefined,
progress,
);
// The brain store goes into the staging index too (design D5), every
// shard from offset 0, so the swapped-in index answers with the store
// from its first read instead of catching up on it afterwards. It is
// best effort: a store that cannot be read must not throw the note
// rebuild away. A shard whose watermark was not written is read from
// offset 0 by the next catch-up, so nothing is lost; the failure is
// reported among the run's errors.
if (config.brainStore) {
try {
await withStagingStore(tempConfig, (store) => ingestStoreInto(store, tempConfig));
} catch (e) {
stats = {
...stats,
errors: [
...stats.errors,
{
path: STORE_RECORD_PATH_PREFIX,
message: `brain store not ingested, the next catch-up reads it: ${e instanceof Error ? e.message : String(e)}`,
},
],
};
}
}
// Clear the marker so the swapped-in live index carries no staging
// state. Only present when resume was enabled.
if (config.resumeReindex) {
await withStagingStore(tempConfig, (store) => store.deleteState(REINDEX_SIGNATURE_KEY));
}
// Same-directory rename swap. The two renames are each atomic on
// POSIX; the gap between them is the only crash window. A read that
// opens in that window is held off by this same lock (see
// restoreFromBakIfMissing in store.ts), so it cannot restore the stale
// `.bak` over the freshly built index; a genuine crash leaves the lock
// stale and the .bak restore on the next Store.open recovers.
//
// A `.bak` holding more stored embeddings than the live index is kept:
// it is the only copy of vectors a keyword-only rebuild cannot
// recompute, and replacing it with a vector-less live index (two
// self-heal rebuilds in a row) would discard them while the rebuild
// notice still points at `.bak`. The live index is parked beside it
// for the swap and removed once the swap succeeds; a crash between the
// renames leaves it at `.old`, which the next Store.open restores in
// preference to the older `.bak` (restoreFromBakIfMissing).
// Windows: a bun:sqlite connection with unfinalized statements survives
// close() as a zombie holding the staging DB handle until GC, so the
// rename swap below fails EBUSY against the process's OWN handle (POSIX
// is unaffected - rename(2) ignores open handles). Force collection to
// release finalizer-pending handles before the renames.
if (typeof Bun !== "undefined") Bun.gc(true);
else (globalThis as { gc?: () => void }).gc?.();
progress.start(INDEX_STAGE.swap, 1);
// Counting the stored embeddings reads both indexes in full, which on
// a large one blocks for a while; the lock is kept fresh meanwhile.
const keepBak = keepWriterLeasesFresh(
WRITER_LEASE_BLOCKING_BUDGET_MS,
() => countStoredEmbeddings(bakPath) > countStoredEmbeddings(config.dbPath),
);
// The last moment the rebuild can still be thrown away harmlessly,
// right before the first destructive step. A lease another writer took
// over (this run stalled past the stale window) no longer excludes that
// writer, and swapping now would replace whatever it committed with
// this run's staging build.
lease.assertHeld();
const parkPath = keepBak ? config.dbPath + ".old" : bakPath;
tryUnlink(parkPath);
tryRename(config.dbPath, parkPath); // no-op (ENOENT) on fresh reindex
try {
renameWithRetry(newPath, config.dbPath); // must succeed β `newPath` was just built
} catch (swapError) {
restoreLiveFromBak(parkPath, config.dbPath);
throw swapError;
}
if (keepBak) tryUnlink(parkPath);
progress.advance(INDEX_STAGE.swap);
return stats;
} finally {
lease.release();
}
}
/** index_state key carrying the staging-build compatibility signature. */
const REINDEX_SIGNATURE_KEY = "reindex_signature";
/** Embedding component of the staging signature when no embeddings are computed. */
const STAGING_EMBEDDING_OFF = "off";
/**
* Embedding component when the active signature cannot be resolved at
* all (semantic search off while embeddings were requested). Carried
* over from the pre-suite fallback rendering.
*/
const STAGING_EMBEDDING_UNRESOLVED = "active";
/** Embedding components that are markers, not embedding signatures. */
const STAGING_EMBEDDING_MARKERS: ReadonlySet<string> = new Set([
STAGING_EMBEDDING_OFF,
STAGING_EMBEDDING_UNRESOLVED,
]);
/** The parts of a staging signature, compared field by field on resume. */
interface StagingSignature {
readonly schema: number;
readonly chunker: number;
readonly chunkSize: number;
readonly chunkOverlap: number;
readonly chunkMinSize: number;
readonly embedding: string;
}
/**
* The dimension the staging identity is judged by: the configured one;
* the local embedder's fixed default; for a named remote model with none
* configured, the model's own native width ({@link MODEL_NATIVE_DIMENSION}).
* Null only for an unnamed model, whose identity stays unknown.
*/
function stagingEmbeddingDimension(
config: ResolvedSearchConfig,
model: string | null,
): EmbeddingIdentity["dimension"] {
if (config.semantic.dimension !== null) return config.semantic.dimension;
if (config.semantic.provider === "local") return LOCAL_DEFAULT_DIMENSION;
return model === null ? null : MODEL_NATIVE_DIMENSION;
}
/**
* The embedding component of the staging compatibility signature: the
* active embedding signature under the effective dimension, or one of
* the named markers above.
*/
function stagingEmbeddingComponent(config: ResolvedSearchConfig, embeddings: boolean): string {
if (!embeddings) return STAGING_EMBEDDING_OFF;
if (!config.semantic.enabled) return STAGING_EMBEDDING_UNRESOLVED;
const model = activeEmbeddingModel(config);
return embeddingSignature({
provider: config.semantic.provider,
model,
dimension: stagingEmbeddingDimension(config, model),
});
}
/**
* Compatibility signature for a staging rebuild: a resume is safe only
* when the schema version, chunker rules, chunk parameters, and (when embeddings are
* computed) the active embedding signature all match the partial build.
* Any drift invalidates the staging DB and forces a fresh rebuild.
*/
function reindexStagingSignature(
config: ResolvedSearchConfig,
embeddings: boolean,
): StagingSignature {
return {
schema: LATEST_SCHEMA_VERSION,
chunker: CHUNKER_VERSION,
chunkSize: config.chunkSize,
chunkOverlap: config.chunkOverlap,
chunkMinSize: config.chunkMinSize,
embedding: stagingEmbeddingComponent(config, embeddings),
};
}
/**
* Whether a stamped staging marker is compatible with the active
* signature. The embedding component goes through
* {@link isStaleSignature}, so an unknown identity (an unnamed model)
* never compares equal to anything, not even its own stamp; the named
* markers and every other field compare by value.
*/
function stagingSignatureCompatible(active: StagingSignature, storedRaw: string): boolean {
let stored: Partial<StagingSignature>;
try {
stored = JSON.parse(storedRaw) as Partial<StagingSignature>;
} catch {
return false;
}
if (typeof stored.embedding !== "string") return false;
const { embedding: activeEmbedding, ...activeRest } = active;
const { embedding: storedEmbedding, ...storedRest } = stored;
if (JSON.stringify(activeRest) !== JSON.stringify(storedRest)) return false;
if (STAGING_EMBEDDING_MARKERS.has(activeEmbedding)) return activeEmbedding === storedEmbedding;
return !isStaleSignature(activeEmbedding, storedEmbedding);
}
/** Read the staging signature from an existing `.new`; false on any
* read error (a corrupt or partial staging DB is rebuilt, not trusted). */
async function stagingSignatureMatches(
tempConfig: ResolvedSearchConfig,
signature: StagingSignature,
): Promise<boolean> {
try {
const store = await Store.open(tempConfig, { mode: "write", loadVec: false });
try {
const stored = store.getState(REINDEX_SIGNATURE_KEY);
return stored !== null && stagingSignatureCompatible(signature, stored);
} finally {
await store.close();
}
} catch {
return false;
}
}
/** Run a short write against the staging DB (set/clear the marker). */
async function withStagingStore(
tempConfig: ResolvedSearchConfig,
fn: (store: Store) => void,
): Promise<void> {
const store = await Store.open(tempConfig, { mode: "write", loadVec: false });
try {
fn(store);
} finally {
await store.close();
}
}
/**
* Stored vectors in the index at `dbPath`; 0 when it is absent or cannot
* be read. A plain read-only open with no extension loaded: the count
* reads the embeddings table's rows, not the vectors.
*/
export function countStoredEmbeddings(dbPath: string): number {
if (!existsSync(dbPath)) return 0;
let db: Database;
try {
db = new Database(dbPath, { readonly: true });
} catch {
return 0;
}
try {
return db.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM embeddings").get()?.n ?? 0;
} catch {
return 0;
} finally {
closeDatabase(db);
}
}
/** `unlinkWithRetry` that tolerates ENOENT (file already absent). */
function tryUnlink(p: string): void {
try {
// A stale `.bak` or `.new` held by an antivirus scan or a sync client
// is a Windows sharing violation that clears; ride it out like the
// renames beside it instead of failing the whole reindex.
unlinkWithRetry(p);
} catch (e) {
if (!isEnoent(e)) throw e;
}
}
/** `renameSync` that tolerates ENOENT on the source. */
function tryRename(from: string, to: string): void {
try {
renameWithRetry(from, to);
} catch (e) {
if (!isEnoent(e)) throw e;
}
}
/**
* Undo the first half of a reindex swap whose second rename failed: put the
* previous index (parked at `.bak`) back at the live path, so the failure
* reads as "reindex refused, old index intact" rather than "no search
* index" until the next `Store.open` happens to run its crash-restore
* (#187). ENOENT on `.bak` is a fresh reindex with nothing to restore.
*
* Best effort by design: the caller rethrows the swap's own error, and a
* rollback failure must not replace it. If the rollback fails too, `.bak`
* stays where it is and the crash-restore in `Store.open` still recovers it.
*
* `.new` is left in place. It holds a complete build, and in the Windows
* case that motivates this path it is the very file a sharing violation
* refuses to move, so deleting it would only spend a second retry budget;
* the next reindex discards it (or resumes from it with `resumeReindex`).
*/
function restoreLiveFromBak(bakPath: string, dbPath: string): void {
try {
tryRename(bakPath, dbPath);
} catch {
// Deliberately swallowed: see above.
}
}
function isEnoent(e: unknown): boolean {
return typeof e === "object" && e !== null && (e as { code?: string }).code === "ENOENT";
}
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
// indexStatus
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
/**
* How many held chunks the status snapshot names. A vault can hold
* thousands of chunks after a model swap, and the status JSON is read by
* humans and by agents on a token budget; the report's `total` carries
* the whole so the cap is a disclosure, not a concealment.
*/
export const HELD_CHUNK_REPORT_CAP = 20;
export async function indexStatus(config: ResolvedSearchConfig): Promise<IndexStatusSnapshot> {
let store: Store;
try {
store = await Store.open(config, { mode: "read" });
} catch (e) {
if (e instanceof SearchError && e.code === "INDEX_MISSING") {
return Object.freeze({
indexPath: config.dbPath,
exists: false,
schemaVersion: null,
documents: 0,
chunks: 0,
embeddings: 0,
staleEmbeddings: 0,
embeddingModel: null,
embeddingDimension: null,
embeddingSignature: activeEmbeddingSignature(config),
estimatedRefreshCostUsd: 0,
refreshPriceSource: null,
vecExtension: "unknown" as const,
semanticEnabled: config.semantic.enabled,
embeddingKeyPresent: !!config.semantic.apiKey,
lastIndexedAt: null,
lastFullIndexAt: null,
embeddingAbi: Object.freeze([]),
warnings: Object.freeze([]),
});
}
throw e;
}
try {
const counts = store.counts();
const model = store.getState("embedding_model");
const dimRaw = store.getState("embedding_dimension");
const dim = dimRaw ? Number(dimRaw) : null;
const last = store.getState(LAST_INDEXED_AT_STATE_KEY);
const full = store.getState(LAST_FULL_INDEX_AT_STATE_KEY);
const warnings: string[] = [];
if (config.semantic.enabled && !store.vecLoaded()) {
warnings.push("sqlite-vec unavailable; semantic search disabled this session");
}
// A FIFTH site that derived the credential fact for itself, in a
// fourth spelling, and which the unit-F brief did not name. Routed
// through the same resolver while the file is open: two of these
// copies disagreed about whether the local provider needs a key.
if (
resolveSemanticCapability(config.semantic).tier === SEMANTIC_CAPABILITY_TIER.credentialMissing
) {
warnings.push(await semanticCapabilityLabel(SEMANTIC_CAPABILITY_CODE.credentialMissing));
}
// Instruction-prefix drift (memory-write-path-integrity B2). Stored
// vectors were embedded under the prefix pair recorded at index time; if
// the configured pair now differs they are not comparable to prefixed
// queries, so surface the reindex-required warning. A stored null is a
// pre-feature (unprefixed) index, compared as the empty pair.
if (config.semantic.enabled && counts.embeddings > 0) {
const storedQuery = store.getState(EMBEDDING_PREFIX_QUERY_STATE_KEY) ?? "";
const storedPassage = store.getState(EMBEDDING_PREFIX_PASSAGE_STATE_KEY) ?? "";
const configuredQuery = config.semantic.queryPrefix ?? "";
const configuredPassage = config.semantic.passagePrefix ?? "";
if (storedQuery !== configuredQuery || storedPassage !== configuredPassage) {
warnings.push(
"embedding instruction prefixes changed since the last index run " +
`(stored [${storedQuery}|${storedPassage}], configured ` +
`[${configuredQuery}|${configuredPassage}]); run: o2b search reindex --embeddings`,
);
}
}
// Embedding-ABI drift (context-integrity-gates, Unit E). The read
// open already ran the gated comparison, so this only surfaces what
// it found: empty when the stamps agree AND whenever the operator
// has `integrity.embedding_abi` off. Gated on stored vectors for the
// same reason the prefix warning above is - an index with no
// embeddings has nothing that could be incomparable.
//
// Reported twice, deliberately: as one operator-facing line here and
// as the structured `embeddingAbi` field below. The MCP status block
// carries both, so an agent can branch on the field instead of
// matching on message text.
const abiDrift =
config.semantic.enabled && counts.embeddings > 0
? store.embeddingAbiMismatches()
: Object.freeze([] as ReadonlyArray<StampMismatch>);
if (abiDrift.length > 0) {
warnings.push(formatEmbeddingAbiDrift(abiDrift));
}
// Oversize-chunk census (what-the-index-already-knew, task H). The
// two reportable states leave by different doors, following the
// convention the ABI drift beside them already sets: the warning
// channel is for what an operator can act on, the structured field
// for what is simply true.
//
// - chunks past a DECLARED window is a measured fault with a
// command behind it, so it is a warning. So is the undecided
// band beside it: the census cannot rule the truncation out, the
// remedy is the same lever, and it fires only for chunks that
// actually carry non-ASCII text near the window - never on the
// all-Latin index, which is what keeps it from becoming noise;
// - a model with no declared window is a check that could not run.
// It has no command - nobody can declare a window for someone
// else's model - and it is the COMMON case, so a warning here
// would repeat forever on the majority of configurations and
// train operators to skip the channel. It rides the snapshot as
// its own field instead, which still distinguishes it from a
// pass.
//
// A census that cleared BOTH bounds is neither, so a status that had
// no warning before this feature still has none.
const chunkWindow = censusChunkWindow(store, config, model);
const chunkWindowUndeclared = chunkWindow?.verdict === "window-undeclared" ? chunkWindow : null;
if (
chunkWindow !== null &&
(chunkWindow.verdict === "over-window" || chunkWindow.verdict === "estimate-undecided")
) {
warnings.push(await formatChunkWindowMeasured(chunkWindow));
}
// What the index holds back right now (chunk skip-and-hold, F2), read
// from the durable records under the ACTIVE configuration's key. A
// row whose key a config change released is a decision the next run
// will not honour, so it is not a hold this status would report, and
// a configuration that can hold nothing (no model, no declared
// window) reads nothing. Absent unless something matches the key -
// the same byte discipline as the census field beside it.
let heldChunks: HeldChunkReport | undefined;
const holdKey = config.semantic.enabled ? chunkHoldExclusion(config) : null;
if (holdKey !== null) {
const heldCensus = store.heldChunkCensus(holdKey, HELD_CHUNK_REPORT_CAP);
if (heldCensus.total > 0) {
const items: HeldChunkReportItem[] = heldCensus.rows.map((row) =>
Object.freeze({
path: row.path,
chunkIndex: row.chunkIndex,
estimatedTokens: row.estimatedTokens,
windowTokens: row.windowTokens,
// The only decision the hold table records.
reason: CHUNK_HOLD_REASON.overDeclaredWindow,
}),
);
heldChunks = Object.freeze({ total: heldCensus.total, items: Object.freeze(items) });
}
}
// Best-effort spend estimate to bring stale/missing embeddings current,
// from the shared spend plan. Chunk content is scanned only when the
// price is a positive known rate: a known-free model costs 0. An
// unknown price is unknown only while chunks are pending - a fully
// embedded index has nothing to pay for, so it reads 0, counted
// without loading a chunk body.
let estimatedRefreshCostUsd: number | null = 0;
let refreshPriceSource: EmbeddingPriceSource | null = null;
if (config.semantic.enabled) {
const { quote } = activeSpendQuote(config);
refreshPriceSource = quote.source;
if (quote.usdPerMtok !== null && quote.usdPerMtok > 0) {
estimatedRefreshCostUsd = planEmbeddingSpend(store, config).estimatedUsd;
} else if (
quote.usdPerMtok === null &&
store.countChunksWithoutEmbeddings(undefined, chunkHoldExclusion(config) ?? undefined) > 0
) {
estimatedRefreshCostUsd = null;
} else {
estimatedRefreshCostUsd = 0;
}
}
return Object.freeze({
indexPath: config.dbPath,
exists: true,
schemaVersion: store.schemaVersion(),
documents: counts.documents,
chunks: counts.chunks,
embeddings: counts.embeddings,
staleEmbeddings: counts.staleEmbeddings,
embeddingModel: model,
embeddingDimension: dim,
embeddingSignature: activeEmbeddingSignature(config, model, dim),
estimatedRefreshCostUsd,
refreshPriceSource,
vecExtension: store.vecLoaded() ? ("loaded" as const) : ("unavailable" as const),
semanticEnabled: config.semantic.enabled,
embeddingKeyPresent: !!config.semantic.apiKey,
lastIndexedAt: last,
lastFullIndexAt: full,
embeddingAbi: abiDrift,
// Absent, not null, whenever the check could run - so a status
// over a curated model serializes the bytes it always has.
...(chunkWindowUndeclared === null ? {} : { chunkWindowUndeclared }),
...(heldChunks === undefined ? {} : { heldChunks }),
warnings: Object.freeze(warnings),
});
} finally {
await store.close();
}
}
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
// indexBehindCount
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
/**
* What the next incremental run would do, measured live (t_e4438152).
*/
export interface IndexBehind {
/** `unindexed + changed + deleted`: the run's whole to-do list. */
readonly behind: number;
/** Walked files with no stored row: the run would add them. */
readonly unindexed: number;
/** Stored mtime or size differs: the run would re-read and hash them. */
readonly changed: number;
/** Stored paths the walk never saw: the run would delete them. */
readonly deleted: number;
}
/**
* The live index-behind census: how many documents the next incremental
* run would add, update or delete, measured now rather than read back
* from the last completed run's count.
*
* Exactly the incremental fastpath's comparison, without the reads:
* `walkVault` yields the in-scope `.md` tree stat-only (no content read,
* no hash - the cost is one `readdir` plus one `stat` per entry), and
* every walked file is classified against the stored row:
*
* - `unindexed`: walked, no stored row;
* - `changed`: stored mtime (seconds) or size differs - the fastpath
* miss the next run must read and hash;
* - `deleted`: a stored path the walk never saw.
*
* Two deliberate semantics, shared with the fastpath this replicates:
*
* 1. Upper bound. A touch-only file (identical bytes, bumped mtime)
* counts as changed: the walk never hashes, so "behind" means
* "what the next run would re-read", not "notes missing from
* answers".
* 2. Second granularity. Stored mtimes are seconds, so an edit that
* lands twice within its indexed second at the same size reads as
* unchanged. The run after it catches it; do not "fix" that here.
*
* Read-only and self-contained: one read-mode store open, no writes, no
* freshen claim, no state file - nothing registers anywhere. On ANY
* failure - a missing or unreadable index, a walk that throws - the
* answer is `null`: a failed measurement is not a measurement, and
* `null` is never a zero.
*/
export async function indexBehindCount(config: ResolvedSearchConfig): Promise<IndexBehind | null> {
let store: Store;
try {
store = await Store.open(config, { mode: "read" });
} catch {
return null; // No index, or one that would not open: not measured.
}
try {
const existing = store.listDocuments();
const seen = new Set<string>();
let unindexed = 0;
let changed = 0;
for (const file of walkVault(config)) {
seen.add(file.relPath);
const prev = existing.get(file.relPath);
if (prev === undefined) {
unindexed++;
} else if (
prev.mtime !== Math.floor(file.stat.mtimeMs / 1000) ||
prev.size !== file.stat.size
) {
changed++;
}
}
let deleted = 0;
for (const path of existing.keys()) {
if (!seen.has(path)) deleted++;
}
return Object.freeze({
behind: unindexed + changed + deleted,
unindexed,
changed,
deleted,
});
} catch {
return null;
} finally {
try {
await store.close();
} catch {
// A close failure must not unseat the measurement (or the null).
}
}
}
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
// indexRootCoverage
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
/**
* Which authorized note roots the index REACHED, and how far.
*
* The field names say presence rather than coverage on purpose. "Covered"
* would claim the root was read out; what the index can show is that at
* least one document under it was indexed, which is a strictly weaker
* statement - see {@link indexRootCoverage}.
*/
export interface IndexRootCoverage {
/** Roots carrying at least one indexed (and, when filtered, admitted) document. */
readonly rootsWithDocuments: ReadonlyArray<string>;
/** Roots the index holds no (admitted) document under at all. */
readonly rootsWithoutDocuments: ReadonlyArray<string>;
}
/** The visibility an unmeasured indexed document answers `admit` with. */
const NO_INDEXED_TAGS: ReadonlyArray<string> = Object.freeze([]);
/**
* Partition note roots by whether the index carries any document under
* them (silence-is-not-an-answer, U2).
*
* `indexStatus` says how much was indexed in total; this says WHICH of
* the folders the operator authorized were reached at all. A negative
* recall needs both: the totals make the receipt, and the partition names
* the roots that were never touched.
*
* The guarantee is deliberately named as presence, not coverage, and the
* field names say so: a root lands in `rootsWithDocuments` on the
* strength of ONE indexed document, so one note out of a thousand puts it
* there. The index knows what it was given and never what it was not, so
* it cannot tell that case from a folder that holds exactly one note;
* deciding between them needs a filesystem census of the root, which is a
* different measurement with a different cost. Until something needs that
* census, the honest move is to promise only what this scan sees - the
* document TOTAL rides on the receipt beside the root list, so a reader
* weighing a negative is never left with the root list alone.
*
* Throws `SearchError` `INDEX_MISSING` when there is no index, rather
* than reporting every root absent. The caller must establish that the
* index exists first, and an all-absent answer over a missing index would
* read like a measurement.
*
* `admit`, when given, decides which indexed documents count: a root is
* reached only through a document `admit` accepts. It receives the path
* and the visibility tokens the index measured for it (empty when none
* were measured), so a caller below local reach can count only the pages
* it may read and a root holding nothing else answers exactly like an
* empty one. It is asked lazily, root by root, and stops at the first
* admitted document. Omitted, every indexed document counts and nothing
* beyond the path scan is read.
*
* Cost: one read-mode open and one full document-path scan, plus one
* visibility lookup per candidate asked of `admit`. It runs only on the
* recall gate's zero-result path, where a search has already failed to
* produce anything, and the alternative - a per-root prefix query - would
* buy microseconds for a new store surface.
*/
export async function indexRootCoverage(
config: ResolvedSearchConfig,
roots: ReadonlyArray<string>,
admit?: (path: string, indexedTags: ReadonlyArray<string>) => boolean,
): Promise<IndexRootCoverage> {
const store = await Store.open(config, { mode: "read" });
try {
const paths = [...store.listDocuments().keys()];
const withDocuments: string[] = [];
const withoutDocuments: string[] = [];
for (const root of roots) {
// Segment-wise: a root of `Notes` is not reached by a document in
// `Notes-archive/`. Roots arrive from `resolveNoteRoots`, which
// normalises them, and indexed paths are what the walker wrote.
const reached = paths.some(
(path) =>
pathCovers(root, path) &&
(admit === undefined ||
admit(path, store.indexedVisibilityByPaths([path]).get(path) ?? NO_INDEXED_TAGS)),
);
(reached ? withDocuments : withoutDocuments).push(root);
}
return Object.freeze({
rootsWithDocuments: Object.freeze(withDocuments),
rootsWithoutDocuments: Object.freeze(withoutDocuments),
});
} finally {
await store.close();
}
}
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
// indexCheck
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
function isDirectoryWritable(dir: string): boolean {
try {
// `recursive: true` is idempotent β no separate existsSync check.
mkdirSync(dir, { recursive: true });
accessSync(dir, constants.W_OK);
return true;
} catch {
return false;
}
}
/** What one `indexCheck` run may do beyond reading this machine. */
export interface IndexCheckOptions {
/**
* Whether to make the one outbound call this report can make: a single
* `ping` at the configured embedding provider. Defaults to `true`,
* which is what every release before it has done, so a caller that
* passes nothing gets the report it has always got. `false` produces
* the `skipped` probe state - never a pass and never a fault, because
* a call nobody made proves neither.
*/
readonly probeProvider?: boolean;
/**
* The env and provider registry the credential-source report reads
* names from. Passed explicitly so the report never reads the process
* on its own; absent, the report is not built and the result carries no
* `credentialSources` field.
*/
readonly credentialContext?: CredentialSourceContext;
}
export async function indexCheck(
config: ResolvedSearchConfig,
opts?: IndexCheckOptions,
): Promise<IndexCheckReport> {
const warnings: string[] = [];
const fatal: string[] = [];
// One resolver for what the operator configured, shared with the index
// run, the semantic lane and the readiness probe
// (provenance-at-the-boundary, F1).
const capability = resolveSemanticCapability(config.semantic);
let vaultReadable = false;
try {
// No vault (design D14): the index holds the brain store only, which
// the store checks report on; there is no vault to read.
if (config.vault === null) {
vaultReadable = true;
} else if (statSync(config.vault).isDirectory()) {
vaultReadable = true;
} else {
fatal.push(`vault path exists but is not a directory: ${config.vault}`);
}
} catch {
fatal.push(`vault not readable: ${config.vault}`);
}
const dir = dirname(config.dbPath);
const indexDirWritable = isDirectoryWritable(dir);
if (!indexDirWritable) fatal.push(`index directory not writable: ${dir}`);
let sqliteOk = false;
let fts5Ok = false;
let vecExtension: "loaded" | "unavailable" | "not-attempted" = "not-attempted";
/** Runtime ABI token; captured from the probe rather than re-queried. */
let vecVersion: string | null = null;
try {
// Use an in-memory DB so the check never touches the real index.
// bun:sqlite is imported statically; the open below is the probe.
const db = new Database(":memory:");
sqliteOk = true;
try {
db.exec(
"CREATE VIRTUAL TABLE probe USING fts5(content, tokenize='unicode61 remove_diacritics 2')",
);
fts5Ok = true;
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
fatal.push(`FTS5 not available: ${msg}`);
}
// Probed whenever semantic search is switched on at all - including
// while the credential is still missing, because the extension is a
// machine fact the operator can fix independently of the key.
if (capability.tier !== SEMANTIC_CAPABILITY_TIER.disabled) {
try {
// eslint-disable-next-line @typescript-eslint/no-require-imports
const vec = require("sqlite-vec") as { getLoadablePath(): string };
db.loadExtension(vec.getLoadablePath());
const row = db.query<{ v: string }, []>("SELECT vec_version() AS v").get();
vecVersion = typeof row?.v === "string" ? row.v : null;
vecExtension = "loaded";
} catch (e) {
vecExtension = "unavailable";
warnings.push(`sqlite-vec unavailable: ${e instanceof Error ? e.message : String(e)}`);
}
}
closeDatabase(db);
} catch (e) {
fatal.push(`bun:sqlite open failed: ${e instanceof Error ? e.message : String(e)}`);
}
// "The credential side of the configuration is satisfied" - which for
// the offline local provider means it needed none. This used to read
// `enabled && apiKey`, which reported a fully working local install as
// key-less and then recommended setting a key it does not use.
const embeddingKeyResolved = !semanticCapabilityIsBlocked(capability);
if (capability.tier === SEMANTIC_CAPABILITY_TIER.credentialMissing) {
warnings.push(await semanticCapabilityLabel(capability.code));
}
// The live probe (wiring-what-exists, E1). Four decisions, each one
// the difference between two states a single verdict used to blur:
//
// - A recorded DECLINE is `skipped`, with the consent's own sentence
// as the reason - decided, before the tiers are consulted. The two
// decline keys refuse independently, and the operator who wrote one
// made the same decision either way; reporting that as
// `not-configured` merged a decision with an absence and read a
// broken configuration as one that was never finished.
// - Nothing configured is `not-configured`, not a failed probe. The
// absence of a provider is not a broken provider, and it costs no
// network call: `probeProvider` is never reached on that arm.
// - An unconfigured setup takes that arm even when the caller asked
// for no probe, because "there is nothing to ask" is established
// without asking and is the more specific true statement.
// - Only a provider that ANSWERED with a refusal is a fatal finding.
// A probe that did not complete goes to `warnings`, because it
// proved nothing, and the CLI gives it an exit code of its own so
// it is not read as either verdict.
const consent = resolveProviderConsent({
configPath: config.configPath ?? null,
env: process.env,
semantic: config.semantic,
});
const consentDeclined = providerConsentIsDeclined(consent);
const probe = consentDeclined
? Object.freeze({ state: PROVIDER_PROBE.skipped, reason: consent.detail })
: !embeddingKeyResolved
? PROVIDER_PROBE_NOT_CONFIGURED
: opts?.probeProvider === false
? PROVIDER_PROBE_SKIPPED
: await probeProvider(() => makeProvider(config.semantic, consent));
const providerProbe = probe.state;
const providerReason = probe.reason;
if (providerProbe === PROVIDER_PROBE.unreachable) {
fatal.push(`embedding provider unreachable: ${providerReason}`);
} else if (providerProbe === PROVIDER_PROBE.timedOut) {
warnings.push(`embedding provider probe did not complete: ${providerReason}`);
}
// Stored-versus-runtime embedding ABI (context-integrity-gates, Unit
// E). The probe above deliberately runs in memory and never touches
// the real index, so the stored side is read explicitly here - without
// it there is nothing to compare against.
//
// This is the DIAGNOSTIC surface, so it compares ungated: an operator
// who ran `search check` asked to be told. The gate governs the read
// path, which is where a comparison can refuse; a report cannot.
const recordedAbi = config.semantic.enabled ? readEmbeddingAbiSync(config.dbPath) : null;
const embeddingAbi =
recordedAbi === null
? Object.freeze([] as ReadonlyArray<StampMismatch>)
: compareStamps(recordedAbi, runtimeEmbeddingAbi(config, vecVersion));
if (embeddingAbi.length > 0) warnings.push(formatEmbeddingAbiDrift(embeddingAbi));
// The measured pending-vector count (nothing-writes-silently, unit A),
// read through the same explicit peek the ABI tokens use and for the
// same reason. Two `COUNT(*)`s on a read-only handle; the report is a
// diagnostic an operator ran, never the query path. The config's own
// chunk-hold exclusion rides along: a held chunk is not work the index
// owes a vector for, and counting it here would recommend a backfill
// that finds nothing to do and never clears.
const pendingVectors = readPendingVectorCensus(
config.dbPath,
config.semantic.enabled ? (chunkHoldExclusion(config) ?? undefined) : undefined,
);
// The record-vs-data embedder audit (nothing-writes-silently, unit G).
// Ungated by the semantic switch on purpose: it compares the index's
// record against the index's own data, so what this build is
// configured to do next cannot change the answer.
const embedderRecord = readEmbedderRecordCensusSync(config.dbPath);
const contradiction = formatEmbedderRecordContradiction(embedderRecord);
if (contradiction !== null) warnings.push(contradiction);
// The visibility honesty finding (nothing-writes-silently, unit H,
// form B). Ungated by the semantic switch, like embedderRecord beside
// it: this reads chunk content the indexer already wrote, regardless
// of whether embeddings are configured at all.
const visibilityHonesty = readVisibilityHonestyFinding(config.dbPath);
// Β§E.2 β Actionable hints derived from the check state.
// Rules match the design doc table; agents and operators read the
// list to know what command to run next without learning the
// internals of OSB.
// Names only, and only for a credential-missing tier: every other tier
// yields null and the result stays byte-identical.
const credentialSources =
opts?.credentialContext === undefined
? null
: describeCredentialSources({ semantic: config.semantic, ...opts.credentialContext });
const recommendations = buildRecommendations({
config,
embeddingKeyResolved,
vecExtension,
providerProbe,
embeddingAbi,
pendingVectors,
embedderRecord,
});
return Object.freeze({
vaultReadable,
indexDirWritable,
sqliteOk,
fts5Ok,
embeddingAbi,
pendingVectors,
embedderRecord,
// Absent, not null: a vault that has never used the field gets no
// key at all, so its JSON stays byte-identical to before this unit.
...(visibilityHonesty === null ? {} : { visibilityHonesty }),
vecExtension,
embeddingKeyResolved,
providerProbe,
providerReason,
warnings: Object.freeze(warnings),
fatal: Object.freeze(fatal),
recommendations: Object.freeze(recommendations),
...(credentialSources === null ? {} : { credentialSources }),
});
}
/**
* The pending-vector census for one index path, with the peek's three
* outcomes folded into the report's two. Both non-`read` outcomes are
* UNRECORDED - they differ in their reason, not in what they prove -
* and neither is a count of zero.
*
* `hold` is the configuration's chunk-hold exclusion key; passing it
* keeps held chunks out of the pending count, the same way the spend
* plan and the backfill exclude them. Callers without a configuration
* in hand omit it and read the raw anti-join.
*
* Exported for the brain doctor's embeddings-health check, which
* consolidates this census beside the index counts instead of deriving
* a private copy whose unrecorded contract could drift from this one.
*/
export function readPendingVectorCensus(
dbPath: string,
hold?: ChunkHoldExclusion,
): PendingVectorCensus {
const peek = peekPendingVectorsSync(dbPath, hold);
if (peek.kind === "read") {
return Object.freeze({
verdict: "measured" as const,
pending: peek.value.pending,
chunks: peek.value.chunks,
});
}
return Object.freeze({
verdict: "unrecorded" as const,
reason:
peek.kind === "absent"
? `no search index at ${dbPath}`
: describeUnreadableIndex(dbPath, peek.detail),
});
}
/**
* The visibility honesty finding for an index path, or `null` when there
* is nothing to be honest ABOUT: the index does not exist, would not
* open, or - the common case - has never carried a `visibility:`-tagged
* page AND measured every document it holds. The surface counts are read off the registry's own exported list
* at call time and the document counts off the index's own column, never
* hand-written, so the finding cannot drift from what backs it.
*
* CALLABLE rows only. The registry also carries one `index_store` row -
* the fact that `chunks` stores a private page's text whatever a
* read-side filter hides - and counting a storage fact as a surface an
* operator could call would inflate a number reported to operators.
*/
function readVisibilityHonestyFinding(dbPath: string): VisibilityHonestyFinding | null {
const peek = peekVisibilityColumnCensus(dbPath, REMOTE_DENY_VISIBILITY_TOKEN);
if (peek.kind !== "read") return null;
// Reported when the index HAS something to be honest about, which is
// either a tagged page or a population it could not measure. Gating on
// `tagged` alone suppressed the unmeasured count in exactly the state
// it exists to surface: an index migrated to v12 whose chunk-zero rows
// are gone reads as untagged for every row, so the operator was told
// nothing at all about the documents the index measured nothing for.
if (!peek.value.tagged && peek.value.unmeasured === 0) return null;
return Object.freeze({
excludedSurfaceCount: excludedCallableVisibilitySurfaces().length,
totalSurfaceCount: callableVisibilitySurfaces().length,
// Both from the store, never hand-written, for the same reason the
// two surface counts are read off the registry: a number an operator
// acts on must be the measurement rather than a copy of one.
reservedDocumentCount: peek.value.reserved,
unmeasuredDocumentCount: peek.value.unmeasured,
});
}
interface BuildRecommendationsInput {
readonly config: ResolvedSearchConfig;
readonly embeddingKeyResolved: boolean;
readonly vecExtension: "loaded" | "unavailable" | "not-attempted";
readonly providerProbe: ProviderProbeState;
readonly embeddingAbi: ReadonlyArray<StampMismatch>;
readonly pendingVectors: PendingVectorCensus;
readonly embedderRecord: EmbedderRecordCensus;
}
/**
* The first-vectors recipe, unchanged from the release that wrote it -
* what changed is WHEN it fires. It is now reached only by an index
* that holds chunks and no vectors for them, which is the state the
* sentence has always described.
*/
const FIRST_VECTORS_RECOMMENDATION =
"Run `o2b search reindex --embeddings` to compute the first vectors, then optionally " +
"`o2b search reindex --cron-template` for periodic refresh.";
/**
* The verb that prices a partial backfill. Named here rather than
* priced here: `planVectorBackfill`'s dry run already counts the tokens
* and applies the model's rate, and a second estimator on this surface
* could quote a different number for the same work.
*/
const VECTOR_BACKFILL_COMMAND = "o2b search vector-backfill";
function buildRecommendations(input: BuildRecommendationsInput): string[] {
const recs: string[] = [];
// A record the data itself disproves comes first: every other hint
// below reasons from what the index claims about itself, and this is
// the one finding that says those claims cannot be trusted. Same
// sentence as the warning - one finding, one wording.
const contradiction = formatEmbedderRecordContradiction(input.embedderRecord);
if (contradiction !== null) recs.push(contradiction);
// Embedding-ABI drift is the one condition here whose remediation is a
// single command, so it is named verbatim and copy-pasteable.
if (input.embeddingAbi.length > 0) {
recs.push(
`Stored vectors were written under a different embedding ABI ` +
`(${input.embeddingAbi.map(formatStampMismatch).join("; ")}). Run: ` +
`${embeddingAbiFixCommand(input.embeddingAbi)}`,
);
}
if (input.config.semantic.enabled && !input.embeddingKeyResolved) {
recs.push(
"Set OPEN_SECOND_BRAIN_EMBEDDING_KEY in ~/.hermes/.env (or the configured env file).",
);
recs.push(
"Provider: OpenAI `text-embedding-3-small` is the default; any OpenAI-compatible endpoint works via OPEN_SECOND_BRAIN_EMBEDDING_BASE_URL.",
);
}
// The price the cost gate reasons from (Honest Embedding Spend): an
// unpriced active model and a declaration that prices another model.
// Silent for priced and local models, so their output is unchanged.
if (input.config.semantic.enabled) {
recs.push(...embeddingPriceRecommendations(input.config));
}
if (input.vecExtension === "unavailable") {
if (process.platform === "darwin") {
recs.push(
"Install Homebrew SQLite: `brew install sqlite`. The o2b wrapper picks it up automatically on the next invocation via DYLD_LIBRARY_PATH.",
);
} else {
recs.push(
"sqlite-vec did not load. Confirm the optional dependency with `bun pm ls`, or rebuild with `bun install --force`.",
);
}
}
// Everything wired: what to do next is decided by the MEASURED
// pending-vector count (nothing-writes-silently, unit A).
//
// The gate on a reachable provider and a loaded extension is the same
// one this branch has always carried - a probe that was skipped or
// timed out proves nothing, and advising a command that cannot
// succeed is worse than saying nothing. What it no longer does is
// stand IN for the count: those two facts say the machine is ready to
// embed, never that anything is waiting to be embedded, and a fully
// embedded vault was told to compute its first vectors for as long as
// they were the whole test. A vault with every chunk vectorised now
// gets no recommendation at all.
if (input.providerProbe === PROVIDER_PROBE.reachable && input.vecExtension === "loaded") {
const census = input.pendingVectors;
if (census.verdict === "unrecorded") {
recs.push(
`The pending-vector count is unrecorded, not zero (${census.reason}). ` +
FIRST_VECTORS_RECOMMENDATION,
);
} else if (census.pending === census.chunks) {
// Every chunk there is - including none at all, on an index that
// holds no chunks yet - is waiting for its first vector.
recs.push(FIRST_VECTORS_RECOMMENDATION);
} else if (census.pending > 0) {
recs.push(
`${census.pending} of ${census.chunks} indexed chunk(s) have no vector. ` +
`Run \`${VECTOR_BACKFILL_COMMAND}\` to see what embedding them would cost - it is a ` +
`dry run unless \`--apply\` is passed - and \`${VECTOR_BACKFILL_COMMAND} --apply\` to compute them.`,
);
}
}
return recs;
}