Local-first π§ memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly π΄ dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Single SQL boundary for `src/core/search/*`. Every read or write of
* the index passes through this module. Other modules use the typed
* surface defined here so that:
*
* - the SQLite backend can be swapped without touching callers;
* - the explicit two-step deletion of `chunk_vec` rows (which the
* SQLite FK cascade does NOT reach, see design Β§5) is centralised
* in one place;
* - the embedding-model fingerprint check runs in one place at open
* time.
*
* The statements themselves live one module per table or lane under
* `store/`; this file is the connection's lifetime, the vec/lock state
* those statements need, and the typed surface callers hold.
*
* Anchored in docs/plans/2026-05-16-brain-search-design.md Β§5, Β§15.
*/
import { Database } from "bun:sqlite";
import { statSync } from "node:fs";
import { join } from "node:path";
import { registerWriterDb, unregisterWriterDb } from "./store-exit.ts";
import { loadIntegrityConfigSafe } from "../brain/policy.ts";
import {
checkStamp,
compareStamps,
STAMP_VERDICT,
type StampMismatch,
} from "../integrity/stamp.ts";
import { SearchError } from "./types.ts";
import type { ResolvedSearchConfig } from "./types.ts";
import type { EventAnchor } from "./event-anchor.ts";
import { readSchemaVersion, type DocumentAttribute } from "./schema.ts";
import { withoutAttributeNarrowing, type PropertyPushdown } from "./property-pushdown.ts";
import { pathIsInside } from "../path-safety.ts";
import * as aliases from "./store/aliases.ts";
import * as chunkEntities from "./store/chunk-entities.ts";
import * as chunks from "./store/chunks.ts";
import * as census from "./store/counts.ts";
import * as documents from "./store/documents.ts";
import * as abi from "./store/embedding-abi.ts";
import * as entityBridges from "./store/entity-bridges.ts";
import * as keyword from "./store/keyword.ts";
import * as lifecycle from "./store/lifecycle.ts";
import * as links from "./store/links.ts";
import * as queryCache from "./store/query-cache-table.ts";
import * as relations from "./store/relations.ts";
import * as state from "./store/state.ts";
import * as tiers from "./store/tiers.ts";
import * as trigram from "./store/trigram.ts";
import * as vectors from "./store/vectors.ts";
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
// Public surface re-exported from the per-concern modules
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
export {
CHUNKER_VERSION_STATE_KEY,
describeUnreadableIndex,
EMBEDDING_DIMENSION_STATE_KEY,
EMBEDDING_MODEL_STATE_KEY,
EMBEDDING_PREFIX_PASSAGE_STATE_KEY,
EMBEDDING_PREFIX_QUERY_STATE_KEY,
EMBEDDING_VEC_VERSION_STATE_KEY,
INDEX_REVISION_STATE_KEY,
LAST_FULL_INDEX_AT_STATE_KEY,
LAST_INDEXED_AT_STATE_KEY,
LINK_RESOLUTION_PENDING_STATE_KEY,
peekCorpusGenerationSync,
readCorpusGenerationSync,
} from "./store/state.ts";
export type { IndexPeek } from "./store/state.ts";
export {
contradictedAbiFields,
EMBEDDING_ABI_FIX_COMMAND,
EMBEDDING_VEC_VERSION_FIX_COMMAND,
embeddingAbiFixCommand,
formatEmbeddingAbiDrift,
peekEmbeddingAbiSync,
readEmbeddingAbiSync,
runtimeEmbeddingAbi,
} from "./store/embedding-abi.ts";
export {
acquireWriterLock,
WRITER_LOCK_HEARTBEAT_MS,
WRITER_LOCK_STALE_MS,
} from "./store/writer-lock.ts";
export { peekPendingVectorsSync } from "./store/counts.ts";
export { peekVisibilityColumnCensus, type VisibilityColumnCensus } from "./store/visibility-tag.ts";
export {
auditEmbedderRecord,
formatEmbedderRecordContradiction,
readEmbedderRecordCensusSync,
} from "./store/embedder-audit.ts";
import { closeDatabase } from "../sqlite-close.ts";
export { normalizeAlias } from "./store/aliases.ts";
export type { DocumentInput, DocumentSummary } from "./store/documents.ts";
export type { EventAnchor } from "./event-anchor.ts";
export type { ChunkInput, ChunkRow, HydratedChunk } from "./store/chunks.ts";
export type { KeywordHit } from "./store/keyword.ts";
export type { DanglingLinkTarget, LinkInput, LinkResolutionCounts } from "./store/links.ts";
export type { EmbeddingPrefixPair, ModelChangeOutcome, SemanticHit } from "./store/vectors.ts";
export type { PendingVectorTally, StoreCounts } from "./store/counts.ts";
export interface StoreOpenOptions {
/** "read" never locks; "write" acquires an exclusive proper-lockfile. */
readonly mode: "read" | "write";
/** When false, the vec extension is not auto-loaded (used by tests). */
readonly loadVec?: boolean;
}
/**
* The freshness verdict the attribute-narrowing gate asks of every
* candidate it would otherwise exclude (answer-trust repair, M1): does the
* live file's stat fingerprint still match the one the index recorded?
*
* `documents.mtime` (unix seconds - the indexer's own floor of the file's
* mtime) and `documents.size` are the exact pair the indexer's fastpath
* already treats as "content unchanged" (`make`'s trade-off), so the gate
* reuses that definition instead of inventing a second one. The signature
* is the no-file-read pin: the whole liveness judgement is a comparison of
* two stat fingerprints, never a content read - the one file read a
* narrowed search pays stays the live post-rank filter's own, over the
* surfaced rows, exactly as before the gate existed. A candidate whose
* fingerprint matches is proven fresh and its SQL exclusion stands; any
* mismatch is STALE and the candidate falls through to the live filter,
* which owns the "judged live" contract (see {@link Store.gateAttributeNarrowing}).
*/
export function indexedSnapshotStale(
indexed: { readonly mtime: number; readonly size: number },
live: { readonly mtimeMs: number; readonly size: number },
): boolean {
return Math.floor(live.mtimeMs / 1000) !== indexed.mtime || live.size !== indexed.size;
}
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
// Store
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
export class Store {
private db: Database;
private readonly config: ResolvedSearchConfig;
private readonly loadedVecVersion: string | null;
private readonly vecExtensionLoaded: boolean;
private readonly release: (() => Promise<void>) | null;
private closed = false;
/** Empty unless a gated read-mode open found ABI drift; see {@link embeddingAbiMismatches}. */
private abiDrift: ReadonlyArray<StampMismatch> = Object.freeze([]);
private constructor(
db: Database,
config: ResolvedSearchConfig,
vecVersion: string | null,
release: (() => Promise<void>) | null,
) {
this.db = db;
this.config = config;
this.loadedVecVersion = vecVersion;
// One source of truth: the version is present exactly when the
// extension loaded, so "loaded" is derived rather than tracked twice.
this.vecExtensionLoaded = vecVersion !== null;
this.release = release;
}
static async open(config: ResolvedSearchConfig, opts: StoreOpenOptions): Promise<Store> {
const loadVec = opts.loadVec !== false;
// Crash recovery: if the main index is missing but `.bak` is present
// from a failed `reindex` rename window, restore it. Lock-guarded so a
// read opening during a live reindex's swap window cannot clobber the
// freshly built index with the stale `.bak`. Stderr notice (not a thrown
// error) so existing tooling keeps working.
lifecycle.restoreFromBakIfMissing(config.dbPath);
if (opts.mode === "read") {
const opened = lifecycle.openReadDatabase(config, loadVec);
try {
const store = new Store(opened.db, config, opened.vecVersion, null);
// The read path is where the silent-garbage window actually is:
// `ensureEmbeddingModel` runs only on a write open, so every MCP
// query reached `semanticTopK` - and a `chunk_vec` of unknown
// declared width - with nothing having compared anything.
store.resolveEmbeddingAbi();
return store;
} catch (e) {
closeDatabase(opened.db);
throw e;
}
}
const opened = await lifecycle.openWriteDatabase(config, loadVec);
try {
const store = new Store(opened.db, config, opened.vecVersion, opened.release);
store.ensureEmbeddingModel(config.semantic.model, config.semantic.dimension, {
query: config.semantic.queryPrefix ?? "",
passage: config.semantic.passagePrefix ?? "",
});
// Belt-and-suspenders: consolidate this writer's WAL on a
// bypassed close (process.exit / signal-driven exit).
registerWriterDb(opened.db);
return store;
} catch (e) {
await lifecycle.abandonWriteDatabase(opened.db, opened.release);
throw e;
}
}
vecLoaded(): boolean {
return this.vecExtensionLoaded;
}
/** sqlite-vec version this connection loaded, or null when unavailable. */
vecVersion(): string | null {
return this.loadedVecVersion;
}
/**
* Embedding-ABI fields whose recorded token no longer matches this
* build (context-integrity-gates, Unit E).
*
* Empty when the stamps agree, when semantic search is disabled, and -
* structurally - whenever `integrity.embedding_abi` is `off`, so a
* consumer wired to this accessor cannot report anything while the
* operator has the gate off. A mismatch whose recorded side is `null`
* is an UNRECORDED field (a store predating the stamp), which is
* reported but never refused: an absent marker cannot tell an old
* store from a wrong one.
*/
embeddingAbiMismatches(): ReadonlyArray<StampMismatch> {
return this.abiDrift;
}
/**
* Compare the recorded embedding ABI against this build's, under the
* operator's gate. Called on the READ open, which is where nothing
* checked before.
*
* Runs only when semantic search is enabled, because that is the only
* configuration that can reach `semanticTopK`; with it disabled the
* configured model and dimension are null and every comparison would
* be against nothing.
*
* The pure comparison runs FIRST and short-circuits, so the
* overwhelmingly common no-drift open never pays for a config read.
* When there is drift, {@link checkStamp} re-runs it under the
* resolved mode and remains the single authority on what `off`, `warn`
* and `fail` mean.
*
* It reports and refuses; it never rebuilds. `vec_version()` is not
* stable across environments, so two peers on different sqlite-vec
* builds each see a mismatch - an automatic rebuild here would loop
* across a synced vault, which is why the default is `warn` and why
* the remediation is a command the operator runs.
*/
private resolveEmbeddingAbi(): void {
if (!this.config.semantic.enabled) return;
const recorded = abi.recordedEmbeddingAbi((key) => this.getState(key));
const runtime = abi.runtimeEmbeddingAbi(this.config, this.loadedVecVersion);
if (compareStamps(recorded, runtime).length === 0) return;
const mode = loadIntegrityConfigSafe(this.config.vault).embedding_abi;
const check = checkStamp(recorded, runtime, mode);
this.abiDrift = check.mismatches;
if (check.verdict !== STAMP_VERDICT.fail) return;
const drifted = abi.contradictedAbiFields(check.mismatches);
if (drifted.length === 0) return;
throw new SearchError("EMBEDDING_DIMENSION_MISMATCH", abi.formatEmbeddingAbiDrift(drifted));
}
schemaVersion(): number {
return readSchemaVersion(this.db);
}
async close(): Promise<void> {
if (this.closed) return;
this.closed = true;
// Orderly close owns consolidation below; drop the exit-hook entry.
unregisterWriterDb(this.db);
try {
if (this.release) lifecycle.consolidateWal(this.db);
closeDatabase(this.db);
} finally {
if (this.release) await this.release();
}
}
// ββ index_state KV βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
getState(key: string): string | null {
return state.getState(this.db, key);
}
setState(key: string, value: string): void {
state.setState(this.db, key, value);
}
deleteState(key: string): void {
state.deleteState(this.db, key);
}
// ββ index revision + corpus generation (v0.20.0) βββββββββββββββββββββββββββββ
/** Monotonic counter bumped on every index mutation; 0 if never set. */
indexRevision(): number {
return state.indexRevision(this.db);
}
/** Increment the index revision. Called after a mutating index run. */
bumpIndexRevision(): void {
state.bumpIndexRevision(this.db);
}
/**
* Current corpus-generation fingerprint: embedding model + dimension +
* schema version + index revision. The query cache gates on this so any
* embedding change or content reindex invalidates cached results.
*/
corpusGeneration(): string {
return state.corpusGeneration(this.db);
}
// ββ query cache (v0.20.0) ββββββββββββββββββββββββββββββββββββββββββββββββββββ
queryCacheGet(key: string): { generation: string; payload: string; createdAt: number } | null {
return queryCache.queryCacheGet(this.db, key);
}
queryCachePut(key: string, generation: string, payload: string, createdAtMs: number): void {
queryCache.queryCachePut(this.db, key, generation, payload, createdAtMs);
}
/** Insert-only put: false when the key already held a row. */
queryCachePutIfAbsent(
key: string,
generation: string,
payload: string,
createdAtMs: number,
): boolean {
return queryCache.queryCachePutIfAbsent(this.db, key, generation, payload, createdAtMs);
}
/** Delete one query-cache row by key; idempotent. */
queryCacheDelete(key: string): void {
queryCache.queryCacheDelete(this.db, key);
}
/** Delete rows from a stale generation or created before the cutoff. */
queryCacheSweep(currentGeneration: string, expiredBeforeMs: number): void {
queryCache.queryCacheSweep(this.db, currentGeneration, expiredBeforeMs);
}
// ββ documents ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
listDocuments(): Map<string, documents.DocumentSummary> {
return documents.listDocuments(this.db);
}
getDocumentIdByPath(path: string): number | null {
return documents.getDocumentIdByPath(this.db, path);
}
/** Measured `visibility:` tokens per indexed path; see `documents.indexedVisibilityByPaths`. */
indexedVisibilityByPaths(paths: ReadonlyArray<string>): Map<string, ReadonlyArray<string>> {
return documents.indexedVisibilityByPaths(this.db, paths);
}
/**
* The candidate documents the index measured as pinned (v13); see
* `documents.pinnedDocumentIds`. This facade is the reader the ranking
* path consumes: the query-side signal collector hands the set to the
* ranker's pinned boost layer (t_f7bef96a).
*/
pinnedDocumentIds(documentIds: ReadonlyArray<number>): Set<number> {
return documents.pinnedDocumentIds(this.db, documentIds);
}
/**
* The stored promoted-column verdicts of a candidate pool; see
* `documents.promotedColumnStates`. The post-rank promoted-key gate
* consumes this to arbitrate the rows the lanes' SQL admitted through a
* NULL column: judged live, never trusted from SQL.
*/
promotedColumnStates(
documentIds: ReadonlyArray<number>,
): Map<number, documents.PromotedColumnState> {
return documents.promotedColumnStates(this.db, documentIds);
}
/**
* The materialised event anchor of one path (v11), or null when the
* document is absent or declared no readable date. The query-side
* event-time resolver consults this instead of re-scanning the note's
* body on every query.
*/
eventAnchorForPath(path: string): EventAnchor | null {
return documents.eventAnchorForPath(this.db, path);
}
/**
* Vault-relative paths of documents no anchor-aware binary has ever
* examined (v11) - the rows a pre-anchor index carried across the
* migration, which are NOT the same as documents that declare no date.
*/
unexaminedEventAnchorPaths(): string[] {
return documents.unexaminedEventAnchorPaths(this.db);
}
/** How many documents no anchor-aware binary has ever examined (v11). */
countUnexaminedEventAnchors(): number {
return documents.countUnexaminedEventAnchors(this.db);
}
/**
* Record one already-indexed document's event anchor and mark it
* examined, touching nothing else on the row. The backfill's only write.
*/
setEventAnchor(path: string, anchor: EventAnchor | null): void {
documents.setEventAnchor(this.db, path, anchor);
}
upsertDocument(doc: documents.DocumentInput): number {
return documents.upsertDocument(this.db, doc);
}
/**
* Delete a document and everything that hangs off it. The vec rows
* are removed first because the FK cascade does not reach the
* `chunk_vec` virtual table.
*/
deleteDocument(path: string): void {
documents.deleteDocument(this.db, this.vecExtensionLoaded, path);
}
/**
* Touch a document's mtime and size without changing its title,
* hash, or triggering chunk replacement. Used by the indexer's
* mtime-fastpath fallback to re-arm the stat cache after a
* same-content touch.
*/
touchDocument(path: string, mtime: number, size: number): void {
documents.touchDocument(this.db, path, mtime, size);
}
/**
* Document id -> { path, title } for every indexed document, reading
* ONLY the `documents` table (never chunk bodies). The graph query
* pre-pass uses this for its index-only short-circuit so it can rank and
* answer from index metadata with zero note bodies hydrated.
*/
documentTitles(): Map<number, { readonly path: string; readonly title: string | null }> {
return documents.documentTitles(this.db);
}
// ββ chunks βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
getChunksByDocument(documentId: number): chunks.ChunkRow[] {
return chunks.getChunksByDocument(this.db, documentId);
}
/**
* Atomically replace every chunk for a document, carrying the stored
* vector of every unchanged chunk (same content hash, recorded model
* and dimension) and purging only the rest. FTS5 stays in sync via the
* chunks_ai/ad/au triggers. Returns the new ids and the carried count.
*/
replaceDocumentChunks(
documentId: number,
input: ReadonlyArray<chunks.ChunkInput>,
): chunks.ChunkReplacement {
return chunks.replaceDocumentChunks(this.db, this.vecExtensionLoaded, documentId, input);
}
/**
* {@link Store.replaceDocumentChunks}, returning only the new chunk
* ids in `chunkIndex` order.
*/
replaceChunks(documentId: number, input: ReadonlyArray<chunks.ChunkInput>): number[] {
return chunks.replaceChunks(this.db, this.vecExtensionLoaded, documentId, input);
}
/**
* Delete a set of chunks by id. Vec rows removed first.
*/
deleteChunks(chunkIds: ReadonlyArray<number>): void {
chunks.deleteChunks(this.db, this.vecExtensionLoaded, chunkIds);
}
/** Ordered chunk ids for one document (surprisal, t_fddfe64a). */
chunksForDocument(documentId: number): ReadonlyArray<{ id: number; chunkIndex: number }> {
return chunks.chunksForDocument(this.db, documentId);
}
/** Total indexed chunk count - used to judge trigram-prefilter selectivity. */
chunkCount(): number {
return chunks.countChunks(this.db);
}
hydrateChunks(chunkIds: ReadonlyArray<number>): Map<number, chunks.HydratedChunk> {
return chunks.hydrateChunks(this.db, chunkIds);
}
/**
* Metadata-only pool hydration (recall-quality, C5): the pool's rows
* without `c.content` - content is hydrated for the ranked rows only,
* by {@link hydrateChunks}, inside the assembly. See
* `chunks.hydrateChunkRefs`.
*/
hydrateChunkRefs(chunkIds: ReadonlyArray<number>): Map<number, chunks.HydratedChunk> {
return chunks.hydrateChunkRefs(this.db, chunkIds);
}
/**
* One representative chunk per document - the lowest `chunk_index`,
* which for markdown is the document head (title / opening section).
* The traversal layer surfaces this when a linked document is not
* already a relevance hit.
*/
representativeChunks(documentIds: ReadonlyArray<number>): Map<number, chunks.HydratedChunk> {
return chunks.representativeChunks(this.db, documentIds);
}
/**
* Chunks that have no row in `embeddings`. Used by the indexer to
* populate vectors after a fresh index or after the model-change drop,
* optionally scoped to path prefixes and optionally excluding chunks
* held under `hold`'s key (chunk skip-and-hold, F2).
*/
findChunksWithoutEmbeddings(
scope?: chunks.PendingVectorScope,
hold?: chunks.ChunkHoldExclusion,
): Array<{ chunkId: number; content: string }> {
return chunks.findChunksWithoutEmbeddings(this.db, scope, hold);
}
/** How many chunks the scoped census above would return, without their bodies. */
countChunksWithoutEmbeddings(
scope?: chunks.PendingVectorScope,
hold?: chunks.ChunkHoldExclusion,
): number {
return chunks.countChunksWithoutEmbeddings(this.db, scope, hold);
}
/** Persist embedding-phase hold decisions under one (model, signature) stamp. */
recordChunkHolds(
entries: ReadonlyArray<chunks.ChunkHoldEntry>,
stamp: chunks.ChunkHoldStamp,
): void {
chunks.recordChunkHolds(this.db, entries, stamp);
}
/** The hold rows recorded for one chunk, whatever key made them. */
chunkHoldRowsFor(chunkId: number): chunks.ChunkHoldRow[] {
return chunks.chunkHoldRowsFor(this.db, chunkId);
}
/** Held chunks under one key: capped position-ordered rows plus the total. */
heldChunkCensus(
hold: chunks.ChunkHoldExclusion,
limit: number,
): { total: number; rows: chunks.HeldChunkCensusRow[] } {
return chunks.heldChunkCensus(this.db, hold, limit);
}
/** How many documents sit under one path prefix of a pending-vector scope. */
countDocumentsUnderPrefix(prefix: string): number {
return documents.countDocumentsUnderPrefix(this.db, prefix);
}
// ββ embeddings βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
/**
* Insert or replace a single embedding. The vec table receives the
* raw float32 bytes; the metadata row in `embeddings` tracks model /
* dimension / hash for stale detection.
*
* Throws VEC_EXTENSION_UNAVAILABLE if sqlite-vec didn't load. The
* caller decides whether to surface this (explicit semantic) or warn
* and skip (implicit semantic).
*/
vecUpsert(
chunkId: number,
vector: ReadonlyArray<number> | Float32Array,
model: string,
dimension: number,
embeddingHash: string,
): void {
vectors.vecUpsert(
this.db,
this.vecExtensionLoaded,
chunkId,
vector,
model,
dimension,
embeddingHash,
);
}
/**
* The stored embedding for one chunk, or null when the vec layer is
* unavailable or the chunk was never embedded (surprisal,
* t_fddfe64a).
*/
embeddingForChunk(chunkId: number): Float32Array | null {
return vectors.embeddingForChunk(this.db, this.vecExtensionLoaded, chunkId);
}
/** Every stored vector of a document's chunks with its recorded model and dimension. */
storedEmbeddingsForDocument(documentId: number): vectors.StoredChunkEmbedding[] {
return vectors.storedEmbeddingsForDocument(this.db, this.vecExtensionLoaded, documentId);
}
getEmbeddingHash(chunkId: number): string | null {
return vectors.getEmbeddingHash(this.db, chunkId);
}
/** Number of chunks with a stored embedding row. */
countEmbeddings(): number {
return vectors.countEmbeddings(this.db);
}
/**
* Drop all embeddings + vec storage. Used when the configured model
* or dimension changes. `chunks` and `chunk_fts` are preserved.
*/
clearEmbeddings(): void {
vectors.clearEmbeddings(this.db, this.vecExtensionLoaded);
}
/**
* Compare the configured embedding model/dimension with what was
* recorded in `index_state` on the last index run. If they differ
* and both old + new are non-null, drop embeddings + vec table and
* log one line per design Β§13. First-time set just records state.
*/
ensureEmbeddingModel(
model: string | null,
dimension: number | null,
prefixes?: vectors.EmbeddingPrefixPair,
): vectors.ModelChangeOutcome {
return vectors.ensureEmbeddingModel(
this.db,
{ loaded: this.vecExtensionLoaded, version: this.loadedVecVersion },
model,
dimension,
prefixes,
{ semantic: this.config.semantic },
);
}
/**
* The freshness gate on attribute narrowing (answer-trust repair, M1).
*
* `attributeNarrowingFragment` excludes a candidate when the attributes
* side table holds rows for a key and none of them is accepted - but
* those rows are a snapshot taken at index time, so the exclusion is
* only provable while the live file still matches it. The advertised
* contract is "narrow on the attribute snapshot, judged live"
* (`brain_search`'s `properties` schema), so a narrowed lane read is
* paired with a probe: the same lane read through
* {@link withoutAttributeNarrowing} (promoted predicates unchanged,
* narrowing lifted, same window). Candidates the probe surfaces and the
* narrowed read dropped are exactly the narrowing's exclusions; each
* gets one liveness check ({@link indexedSnapshotStale}) against the
* indexed stat fingerprint, and the stale ones are appended so the live
* post-rank filter - the arbiter - judges them. A fingerprint match
* proves the snapshot current, the SQL exclusion stands, and nothing is
* appended: the fresh path stays byte-identical to the ungated read.
*
* The gate's filesystem budget is one `statSync` per would-be-excluded
* candidate inside the lane window, bounded by the window itself; it
* never reads file contents. The `path` column is the index's own
* record, and it is trusted only as far as the vault it serves: a row
* whose path does not resolve inside the vault is never statted at all -
* the same lexical containment the live filter applies - and is ruled
* STALE by the shared rule below, which is the safe arm (the candidate
* falls through to the live filter and is judged by its unreadable-file
* semantics). An unstatable file cannot be proven
* current either, so its verdict is STALE by the same rule - that is a
* named arbitration, not a swallowed error: the live filter's
* unreadable-file semantics (`CachedFrontmatter.unreadable`, an empty
* frontmatter map that matches no accepted value) decide the row.
*/
private gateAttributeNarrowing<
T extends { readonly documentId: number; readonly chunkId: number },
>(
hits: T[],
pushdown: PropertyPushdown | null | undefined,
probeLane: (probePushdown: PropertyPushdown | null) => T[],
): T[] {
if (pushdown?.attributeNarrowing === undefined) return hits;
const probeHits = probeLane(withoutAttributeNarrowing(pushdown));
const keptChunks = new Set(hits.map((h) => h.chunkId));
const excluded = probeHits.filter((h) => !keptChunks.has(h.chunkId));
if (excluded.length === 0) return hits;
const keptDocs = new Set(hits.map((h) => h.documentId));
const excludedDocs = [...new Set(excluded.map((h) => h.documentId))].filter(
(id) => !keptDocs.has(id),
);
if (excludedDocs.length === 0) return hits;
const staleDocs = this.staleSnapshotDocumentIds(excludedDocs);
if (staleDocs.size === 0) return hits;
return [...hits, ...excluded.filter((h) => staleDocs.has(h.documentId))];
}
/**
* The stale half of the given documents, judged by
* {@link indexedSnapshotStale}: one indexed-fingerprint row per document
* (never a chunk body) and one `statSync` per document - stat metadata
* only, never a content read. A document whose row is missing yields no
* verdict and no rescue, which is the narrowed read's own answer.
*/
private staleSnapshotDocumentIds(documentIds: ReadonlyArray<number>): Set<number> {
// No vault (design D14): the index holds store records, no file to stat.
const vault = this.config.vault;
if (vault === null) return new Set();
const placeholders = documentIds.map(() => "?").join(", ");
const rows = this.db
.query<{ id: number; path: string; mtime: number; size: number }, number[]>(
`SELECT id, path, mtime, size FROM documents WHERE id IN (${placeholders})`,
)
.all(...documentIds);
const stale = new Set<number>();
for (const row of rows) {
// The index's path column is not trusted past the vault it serves: a
// row whose path escapes the vault is not a file this gate may stat,
// so it is ruled stale without touching the filesystem - the same
// lexical containment the live filter applies, and the safe arm (the
// candidate falls through to the live filter, which judges it
// unreadable rather than reading outside the vault).
if (!pathIsInside(join(vault, row.path), vault)) {
stale.add(row.id);
continue;
}
let liveSize: number;
let liveMtimeMs: number;
try {
const live = statSync(join(vault, row.path));
liveSize = live.size;
liveMtimeMs = live.mtimeMs;
} catch {
// Unstatable: the snapshot cannot be proven current. Named
// arbitration (see {@link gateAttributeNarrowing}), never silent.
stale.add(row.id);
continue;
}
if (
indexedSnapshotStale(
{ mtime: row.mtime, size: row.size },
{ mtimeMs: liveMtimeMs, size: liveSize },
)
) {
stale.add(row.id);
}
}
return stale;
}
semanticTopK(
queryVector: ReadonlyArray<number> | Float32Array,
opts: vectors.SemanticTopKOptions,
): vectors.SemanticHit[] {
const hits = vectors.semanticTopK(this.db, this.vecExtensionLoaded, queryVector, opts);
return this.gateAttributeNarrowing(hits, opts.pushdown, (probe) =>
vectors.semanticTopK(this.db, this.vecExtensionLoaded, queryVector, {
...opts,
pushdown: probe,
}),
);
}
// ββ links ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
replaceLinks(sourceDocumentId: number, input: ReadonlyArray<links.LinkInput>): void {
links.replaceLinks(this.db, sourceDocumentId, input);
}
resolveLinkTargets(): void {
links.resolveLinkTargets(this.db);
}
/**
* Vault-wide link-resolution census (context-integrity-gates, unit G).
* `unresolved` counts the link rows that carry a target path and that
* the read-time resolution ladder cannot resolve to any document -
* broken links as a reader experiences them.
*/
linkResolutionCounts(): links.LinkResolutionCounts {
return links.linkResolutionCounts(this.db);
}
/**
* The unresolved link targets themselves, grouped by target and capped
* at `limit` targets. The same predicate {@link linkResolutionCounts}
* counts, so the list and the number cannot disagree about which links
* are broken. A caller must decide whether this index is resolved
* enough to believe - an empty list from a partially-resolved index is
* not the same answer as a vault with no dangling links.
*/
listDangling(limit: number): ReadonlyArray<links.DanglingLinkTarget> {
return links.listDangling(this.db, limit);
}
/**
* Every link edge resolved to a (source, target) document-id pair,
* deduplicated. Unresolvable edges are absent. Read-only; feeds bridge
* discovery and community detection (link-recall-intelligence).
*/
resolvedDocLinkPairs(): Array<{ readonly source: number; readonly target: number }> {
return links.resolvedDocLinkPairs(this.db);
}
/**
* For each chunk id, return the document ids that link TO that
* chunk's document via wikilink or markdown_link. Pure data; the
* ranker decides how to convert this into a boost.
*/
inboundLinkSources(candidateChunkIds: ReadonlyArray<number>): Map<number, Set<number>> {
return links.inboundLinkSources(this.db, candidateChunkIds);
}
/**
* For each source document id, the list of resolved outbound link
* target document ids (wikilink / markdown_link only; tags and
* unresolved targets excluded; self-links dropped). Used by the
* recall traversal layer to walk one or more hops out from a hit.
*/
outboundLinkTargets(sourceDocumentIds: ReadonlyArray<number>): Map<number, number[]> {
return links.outboundLinkTargets(this.db, sourceDocumentIds);
}
/**
* For each chunk id, the set of tag link_text values associated with
* its document. Two chunks "share a tag" iff their tag sets intersect.
*/
tagsByChunkDocument(candidateChunkIds: ReadonlyArray<number>): Map<number, Set<string>> {
return links.tagsByChunkDocument(this.db, candidateChunkIds);
}
// ββ doc aliases (v7, link-recall-intelligence) βββββββββββββββββββββββββββββ
/**
* Replace one document's frontmatter aliases. Values are normalised
* via {@link normalizeAlias}; empties and duplicates are dropped.
*/
replaceDocAliases(documentId: number, values: ReadonlyArray<string>): void {
aliases.replaceDocAliases(this.db, documentId, values);
}
/** Normalised aliases of one document, sorted. */
aliasesForDocument(documentId: number): ReadonlyArray<string> {
return aliases.aliasesForDocument(this.db, documentId);
}
// ββ document attributes (v14, recall-quality) βββββββββββββββββββββββββββββββ
/**
* Replace one document's stored frontmatter attributes wholesale. The
* caller derives the bounded, promoted-key-free rows with
* `documentAttributesFromFrontmatter` (schema.ts) from the same parsed
* frontmatter that fed the upsert's other columns.
*/
replaceDocumentAttributes(documentId: number, attrs: ReadonlyArray<DocumentAttribute>): void {
documents.replaceDocumentAttributes(this.db, documentId, attrs);
}
/**
* Materialize `target_document_id` for unresolved, slash-free link
* targets that match a declared alias. Runs AFTER
* {@link resolveLinkTargets} so exact path matches always win.
*
* Returns the number of link rows resolved.
*/
resolveAliasTargets(): number {
return aliases.resolveAliasTargets(this.db);
}
// ββ typed relations ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
/**
* For each document id, the typed relation edges it declares
* (v3 / typed graph semantics): rows whose `relation` is set, in
* insertion order. The target is the edge's `target_path` as written.
* Documents with no typed edges are absent from the returned map.
*/
typedRelationsForDocuments(
documentIds: ReadonlyArray<number>,
): Map<number, Array<{ relation: string; target: string }>> {
return relations.typedRelationsForDocuments(this.db, documentIds);
}
/**
* Typed relation edges declared by the given documents, with the
* target resolved to a document id when possible (recall-trust-suite,
* relation polarity).
*/
typedRelationEdgesForDocuments(documentIds: ReadonlyArray<number>): Array<{
readonly sourceDocumentId: number;
readonly relation: string;
readonly target: string;
readonly targetDocumentId: number | null;
}> {
return relations.typedRelationEdgesForDocuments(this.db, documentIds);
}
/**
* Recompute every typed edge's `relation_blocked` flag from the
* current schema-pack constraints (write-time-integrity-governance).
* Runs after `resolveLinkTargets` on every index pass, so removing a
* constraint restores blocked edges on the next run without touching
* files. Returns the rows that ended up blocked. With an empty
* constraint map the pass only resets stale flags.
*/
recomputeRelationConstraintFlags(
constraints: Readonly<Record<string, ReadonlyArray<string>>>,
): Array<{
readonly relation: string;
readonly sourcePath: string;
readonly targetPath: string;
readonly sourceType: string;
readonly targetType: string;
readonly declared: ReadonlyArray<string>;
}> {
return relations.recomputeRelationConstraintFlags(this.db, constraints);
}
/**
* The typed edges currently blocked by link constraints, for lint
* surfacing. Read-only over the flags the last index pass computed.
*/
blockedRelationRows(): Array<{
readonly relation: string;
readonly sourcePath: string;
readonly targetPath: string;
readonly sourceType: string | null;
readonly targetType: string | null;
}> {
return relations.blockedRelationRows(this.db);
}
// ββ tiered frontmatter (write-time-integrity-governance) βββββββββββββββββββ
/**
* The tiered-frontmatter snapshot the last index pass recorded for a
* document, or null when none exists.
*/
getTierSnapshot(documentId: number): Record<string, unknown> | null {
return tiers.getTierSnapshot(this.db, documentId);
}
setTierSnapshot(documentId: number, snapshot: Readonly<Record<string, unknown>>): void {
tiers.setTierSnapshot(this.db, documentId, snapshot);
}
/** Stage one identity-field hand-edit; (document, field) upserts. */
upsertTierDrift(input: {
readonly documentId: number;
readonly field: string;
readonly expected: unknown;
readonly actual: unknown;
readonly detectedAt: string;
}): void {
tiers.upsertTierDrift(this.db, input);
}
clearTierDrift(documentId: number, field?: string): void {
tiers.clearTierDrift(this.db, documentId, field);
}
/** Open identity-drift findings, oldest first, with document paths. */
listTierDrift(): Array<{
readonly documentId: number;
readonly path: string;
readonly field: string;
readonly expected: unknown;
readonly actual: unknown;
readonly detectedAt: string;
}> {
return tiers.listTierDrift(this.db);
}
// ββ search βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
/**
* Corpus document frequency per term (recall-trust-suite, coverage
* engine): how many distinct documents match each term in FTS.
*/
documentFrequencies(terms: ReadonlyArray<string>): Map<string, number> {
return keyword.documentFrequencies(this.db, terms);
}
ftsIntegrityCounts(): { readonly chunks: number; readonly ftsRows: number } {
return keyword.ftsIntegrityCounts(this.db);
}
rebuildFtsIndex(): void {
keyword.rebuildFtsIndex(this.db);
}
rebuildFtsIndexWithWriterLock(): void {
keyword.rebuildFtsIndexWithWriterLock(this.db, this.config.dbPath, this.release !== null);
}
/**
* Top-K BM25 keyword hits. `fts5Query` is an already-escaped FTS5
* MATCH expression; building it is fts.ts's job. A narrowed pushdown
* rides the freshness gate ({@link gateAttributeNarrowing}).
*/
keywordTopK(fts5Query: string, opts: keyword.KeywordTopKOptions): keyword.KeywordHit[] {
const hits = keyword.keywordTopK(this.db, fts5Query, opts);
return this.gateAttributeNarrowing(hits, opts.pushdown, (probe) =>
keyword.keywordTopK(this.db, fts5Query, { ...opts, pushdown: probe }),
);
}
/**
* Trigram candidate lookup over the `chunk_trigram` FTS5 shadow (v9).
* An opt-in candidate source that broadens large-vault keyword recall
* with the substring / partial-token matches the word tokenizer misses.
* Degrades to no candidates - silently when the optional table is
* absent, with a warning for every other fault.
*/
trigramCandidates(
trigramQuery: string,
opts: {
readonly limit: number;
readonly pathPrefix?: string | null;
readonly kind?: "page" | "section" | null;
readonly pushdown?: PropertyPushdown | null;
},
): trigram.TrigramCandidateOutcome {
const outcome = trigram.trigramCandidates(this.db, trigramQuery, opts);
const hits = this.gateAttributeNarrowing(
outcome.hits,
opts.pushdown ?? null,
(probe) =>
trigram.trigramCandidates(this.db, trigramQuery, { ...opts, pushdown: probe }).hits,
);
if (hits === outcome.hits) return outcome;
return { ...outcome, hits };
}
// ββ entities βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
/**
* Replace a chunk's entity set (v0.13.0). Deletes any prior entries
* for the chunk, then inserts the deduped list. Entities are expected
* pre-normalised (lowercased) by the extractor.
*/
replaceEntities(chunkId: number, entities: ReadonlyArray<string>): void {
chunkEntities.replaceEntities(this.db, chunkId, entities);
}
/**
* Distinct entities across one document's chunks, sorted
* (link-recall-intelligence: shared-entity digest in cluster notes).
*/
entitiesForDocument(documentId: number): ReadonlyArray<string> {
return chunkEntities.entitiesForDocument(this.db, documentId);
}
/**
* For each candidate chunk, the count of distinct query entities it
* also carries. Empty `queryEntities` yields an empty map (no work).
* Pure read; used by the ranker to add a capped entity boost.
*/
chunkEntityMatches(
candidateChunkIds: ReadonlyArray<number>,
queryEntities: ReadonlyArray<string>,
): Map<number, number> {
return chunkEntities.chunkEntityMatches(this.db, candidateChunkIds, queryEntities);
}
/**
* The entity bridges of the given documents (deepened traversal,
* truth-correctable-time-aware): the deduplicated `chunk_entities`
* co-occurrence pairs, ordered by source then target id. Living on the
* typed surface is the wiring the relational arm's store view looks
* for, so the bridge flag has a reader to enable - a store without this
* method walks typed edges only.
*/
entityBridgesForDocuments(
documentIds: ReadonlyArray<number>,
): Array<entityBridges.EntityBridgePair> {
return entityBridges.entityBridgesForDocuments(this.db, documentIds);
}
// ββ counts βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
counts(): census.StoreCounts {
const baseline = census.staleBaseline(this.db, this.config.semantic);
return census.counts(this.db, baseline.model, baseline.dimension);
}
/**
* How many documents carry a persisted event-time window, the instants
* those windows span, and how many intersect `[sinceMs, untilMs]` (a
* null edge is open). One aggregate; see `store/counts.ts`.
*/
eventTimeWindowCensus(
sinceMs: number | null,
untilMs: number | null,
): census.EventTimeWindowCensus {
return census.eventTimeWindowCensus(this.db, sinceMs, untilMs);
}
/**
* Indexed chunks split by what the oversize-chunk census can decide
* about them against a model's declared input window: provably over,
* and undecided by the estimate. One aggregate pass, no chunk bodies
* read; see `store/counts.ts` for the unit and both bounds.
*
* `requestPrefix` is the instruction prefix the configured backend
* prepends to every passage, because the provider counts it too.
*/
chunkWindowTally(windowTokens: number, requestPrefix: string): census.ChunkWindowTally {
return census.chunkWindowTally(this.db, windowTokens, requestPrefix);
}
/**
* The provable half of {@link chunkWindowTally} for a backend that
* prepends nothing.
*/
chunksOverTokenWindow(windowTokens: number): number {
return census.chunksOverTokenWindow(this.db, windowTokens);
}
// ββ brain store records (v17, design D5) βββββββββββββββββββββββββββββββββββ
/**
* The connection the brain store tables are read and written through
* (`store/brain-records.ts`). Store-record documents and chunks still go
* through the typed document and chunk methods above; only the two v17
* tables, which no other caller touches, are handed the connection.
*/
brainRecordsConnection(): Database {
return this.db;
}
// ββ direct accessors used by indexer/CLI/status ββββββββββββββββββββββββββββ
/** Escape hatch for status queries that don't fit the typed API. */
rawQuery<T>(sql: string, params: ReadonlyArray<string | number | null> = []): T[] {
return this.db
.query<T, (string | number | null)[]>(sql)
.all(...(params as (string | number | null)[]));
}
}