Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* The semantic lane: the sqlite-vec extension, the `chunk_vec` virtual
* table and its `chunk_vec_map` rowid bridge, the `embeddings` metadata
* rows, and the k-nearest-neighbour query over them.
*
* The two-step deletion of `chunk_vec` rows that the SQLite FK cascade
* does NOT reach (design §5) is centralised here, which is why the
* document and chunk modules delete through this one.
*/
import { Database, type SQLQueryBindings } from "bun:sqlite";
import {
BLOCKED_TIER_ERROR_CODE,
isBlockedCapability,
resolveSemanticCapability,
} from "../capability-tier.ts";
import { LOCAL_EMBEDDING_MODEL } from "../embeddings/signature.ts";
import { dropVecTable, ensureVecTable } from "../schema.ts";
import type { PropertyPushdown, SearchKind } from "../property-pushdown.ts";
import { pushdownSqlFragment } from "../property-pushdown.ts";
import { SearchError, type ResolvedEmbeddingConfig, type SearchErrorCode } from "../types.ts";
import { assertValidVector } from "../vector-guard.ts";
import {
deleteState,
EMBEDDING_DIMENSION_STATE_KEY,
EMBEDDING_MODEL_STATE_KEY,
EMBEDDING_PREFIX_PASSAGE_STATE_KEY,
EMBEDDING_PREFIX_QUERY_STATE_KEY,
EMBEDDING_VEC_VERSION_STATE_KEY,
getState,
setState,
} from "./state.ts";
import { pathPrefixRange } from "./path-prefix.ts";
import { nowIso, sqlPlaceholders } from "./sql.ts";
/** The query/passage instruction prefixes active for an index run. */
export interface EmbeddingPrefixPair {
readonly query: string;
readonly passage: string;
}
export interface SemanticHit {
readonly chunkId: number;
readonly documentId: number;
/** L2 distance on unit-normalised vectors. */
readonly distance: number;
}
export interface ModelChangeOutcome {
readonly wasChanged: boolean;
readonly previousModel: string | null;
readonly previousDimension: number | null;
readonly currentModel: string | null;
readonly currentDimension: number | null;
/**
* Present when the verify-before-replace gate refused a clear: the
* stored vectors and the recorded model were kept, and the store opened
* anyway so keyword indexing goes on. The same sentence is logged.
*/
readonly refusal?: EmbeddingRebuildRefusal;
}
/** A refused clear, named by the blocked tier's error code. */
export interface EmbeddingRebuildRefusal {
readonly code: SearchErrorCode;
readonly message: string;
}
/** What an open connection knows about sqlite-vec. */
export interface VecRuntime {
readonly loaded: boolean;
/** The version the extension reported, or null when it never loaded. */
readonly version: string | null;
}
/**
* Load sqlite-vec and return the version it reports, or `null` when the
* extension is unavailable.
*
* The `vec_version()` probe was always executed and its row always
* discarded; returning it costs nothing and is what lets the ABI the
* vectors were written against be stamped rather than assumed
* (context-integrity-gates, Unit E). `null` doubles as the
* "not loaded" signal, so there is exactly one source of truth for both
* facts.
*/
export function loadVecExtension(db: Database): string | null {
try {
// sqlite-vec is an optional dependency. Wrap the import + load so
// a missing platform package degrades to "extension unavailable"
// instead of crashing the process.
// eslint-disable-next-line @typescript-eslint/no-require-imports
const vec = require("sqlite-vec") as { getLoadablePath(): string };
db.loadExtension(vec.getLoadablePath());
// Confirm by calling vec_version() — guards against partial loads.
const row = db.query<{ v: string }, []>("SELECT vec_version() AS v").get();
return typeof row?.v === "string" ? row.v : null;
} catch {
return null;
}
}
function vecToBuffer(values: ReadonlyArray<number> | Float32Array): Buffer {
const arr = values instanceof Float32Array ? values : Float32Array.from(values);
return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength);
}
export function purgeVecRowsForDocument(
db: Database,
vecLoaded: boolean,
documentId: number,
): void {
if (!vecLoaded) return;
const vecRows = db
.query<{ vec_rowid: number }, [number]>(
"SELECT vec_rowid FROM chunk_vec_map WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)",
)
.all(documentId);
if (vecRows.length === 0) return;
purgeVecRowidsRaw(
db,
vecLoaded,
vecRows.map((r) => r.vec_rowid),
);
}
export function purgeVecRowsByChunkIds(
db: Database,
vecLoaded: boolean,
chunkIds: ReadonlyArray<number>,
): void {
if (!vecLoaded || chunkIds.length === 0) return;
const placeholders = sqlPlaceholders(chunkIds);
const vecRows = db
.query<{ vec_rowid: number }, number[]>(
`SELECT vec_rowid FROM chunk_vec_map WHERE chunk_id IN (${placeholders})`,
)
.all(...(chunkIds as number[]));
if (vecRows.length === 0) return;
purgeVecRowidsRaw(
db,
vecLoaded,
vecRows.map((r) => r.vec_rowid),
);
}
function purgeVecRowidsRaw(db: Database, vecLoaded: boolean, vecRowids: number[]): void {
if (!vecLoaded || vecRowids.length === 0) return;
const placeholders = sqlPlaceholders(vecRowids);
db.run(`DELETE FROM chunk_vec WHERE rowid IN (${placeholders})`, vecRowids);
}
/**
* Insert or replace a single embedding. The vec table receives the
* raw float32 bytes; the metadata row in `embeddings` tracks model /
* dimension / hash for stale detection.
*
* Throws VEC_EXTENSION_UNAVAILABLE if sqlite-vec didn't load. The
* caller decides whether to surface this (explicit semantic) or warn
* and skip (implicit semantic).
*/
export function vecUpsert(
db: Database,
vecLoaded: boolean,
chunkId: number,
vector: ReadonlyArray<number> | Float32Array,
model: string,
dimension: number,
embeddingHash: string,
): void {
if (!vecLoaded) {
throw new SearchError(
"VEC_EXTENSION_UNAVAILABLE",
"sqlite-vec extension not loaded; cannot store embeddings",
);
}
if (vector.length !== dimension) {
throw new SearchError(
"EMBEDDING_DIMENSION_MISMATCH",
`vector dimension ${vector.length} != configured dimension ${dimension}`,
);
}
assertValidVector(vector, "vecUpsert");
db.exec("BEGIN");
try {
const existing = db
.query<{ vec_rowid: number }, [number]>(
"SELECT vec_rowid FROM chunk_vec_map WHERE chunk_id = ?",
)
.get(chunkId);
const buf = vecToBuffer(vector);
let vecRowid: number;
if (existing) {
db.run("UPDATE chunk_vec SET embedding = ? WHERE rowid = ?", [buf, existing.vec_rowid]);
vecRowid = existing.vec_rowid;
} else {
db.run("INSERT INTO chunk_vec(embedding) VALUES (?)", [buf]);
const row = db.query<{ id: number }, []>("SELECT last_insert_rowid() AS id").get();
if (!row) throw new SearchError("INDEX_UNREADABLE", "chunk_vec insert returned no rowid");
vecRowid = row.id;
db.run("INSERT INTO chunk_vec_map(chunk_id, vec_rowid) VALUES (?, ?)", [chunkId, vecRowid]);
}
const now = nowIso();
db.run(
"INSERT INTO embeddings(chunk_id, model, dimension, embedding_hash, created_at, updated_at) " +
"VALUES (?, ?, ?, ?, ?, ?) " +
"ON CONFLICT(chunk_id) DO UPDATE SET " +
" model = excluded.model, dimension = excluded.dimension, " +
" embedding_hash = excluded.embedding_hash, updated_at = excluded.updated_at",
[chunkId, model, dimension, embeddingHash, now, now],
);
db.exec("COMMIT");
} catch (e) {
db.exec("ROLLBACK");
throw e;
}
}
/**
* The stored embedding for one chunk, or null when the vec layer is
* unavailable or the chunk was never embedded (surprisal,
* t_fddfe64a).
*/
export function embeddingForChunk(
db: Database,
vecLoaded: boolean,
chunkId: number,
): Float32Array | null {
if (!vecLoaded) return null;
const row = db
.query<{ embedding: Uint8Array }, [number]>(
"SELECT v.embedding AS embedding FROM chunk_vec v " +
"JOIN chunk_vec_map m ON m.vec_rowid = v.rowid WHERE m.chunk_id = ?",
)
.get(chunkId);
if (!row) return null;
return vectorFromBlob(row.embedding);
}
/** Copy a stored vec0 blob into a `Float32Array`. */
function vectorFromBlob(bytes: Uint8Array): Float32Array {
// Copy instead of viewing: a pooled buffer with a non-4-byte-aligned
// byteOffset would make the Float32Array constructor throw.
return new Float32Array(
bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength),
);
}
/** One stored chunk vector with the identity its `embeddings` row recorded. */
export interface StoredChunkEmbedding {
readonly chunkId: number;
readonly vector: Float32Array;
readonly model: string;
readonly dimension: number;
}
/**
* Every stored vector of one document's chunks, in `chunk_index` order,
* each with the model and dimension recorded when it was written. A
* reader that scores against a fresh query vector needs the identity per
* row: a vector left by an older model is not comparable and must be
* told apart from a current one, which {@link embeddingForChunk} cannot.
*/
export function storedEmbeddingsForDocument(
db: Database,
vecLoaded: boolean,
documentId: number,
): StoredChunkEmbedding[] {
if (!vecLoaded) {
throw new SearchError(
"VEC_EXTENSION_UNAVAILABLE",
"sqlite-vec extension not loaded; cannot read stored embeddings",
);
}
return db
.query<{ chunk_id: number; embedding: Uint8Array; model: string; dimension: number }, [number]>(
"SELECT c.id AS chunk_id, v.embedding AS embedding, e.model AS model, " +
"e.dimension AS dimension FROM chunks c " +
"JOIN chunk_vec_map m ON m.chunk_id = c.id " +
"JOIN chunk_vec v ON v.rowid = m.vec_rowid " +
"JOIN embeddings e ON e.chunk_id = c.id " +
"WHERE c.document_id = ? ORDER BY c.chunk_index",
)
.all(documentId)
.map((r) => ({
chunkId: r.chunk_id,
vector: vectorFromBlob(r.embedding),
model: r.model,
dimension: r.dimension,
}));
}
export function getEmbeddingHash(db: Database, chunkId: number): string | null {
const row = db
.query<{ embedding_hash: string }, [number]>(
"SELECT embedding_hash FROM embeddings WHERE chunk_id = ?",
)
.get(chunkId);
return row?.embedding_hash ?? null;
}
/** Number of chunks with a stored embedding row. */
export function countEmbeddings(db: Database): number {
return db.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM embeddings").get()?.n ?? 0;
}
/**
* Embeddings whose `model`/`dimension` no longer match the current config.
*
* With no model there is no baseline, and the count is 0 by convention -
* a caller that must not read that as healthy checks the model first.
* With a model but no dimension (none configured, none stored), the
* vectors are compared by model alone: a model switch on a default
* configuration is exactly the case a dimension-gated zero used to hide.
*/
export function staleEmbeddings(
db: Database,
model: string | null,
dimension: number | null,
): number {
if (!model) return 0;
if (!dimension) {
return (
db
.query<{ c: number }, [string]>("SELECT count(*) AS c FROM embeddings WHERE model != ?")
.get(model)?.c ?? 0
);
}
const row = db
.query<{ c: number }, [string, number]>(
"SELECT count(*) AS c FROM embeddings WHERE model != ? OR dimension != ?",
)
.get(model, dimension);
return row?.c ?? 0;
}
/**
* Drop all embeddings + vec storage. Used when the configured model
* or dimension changes. `chunks` and `chunk_fts` are preserved.
*/
export function clearEmbeddings(db: Database, vecLoaded: boolean): void {
db.exec("BEGIN");
try {
db.run("DELETE FROM embeddings");
db.run("DELETE FROM chunk_vec_map");
if (vecLoaded) dropVecTable(db);
db.exec("COMMIT");
} catch (e) {
db.exec("ROLLBACK");
throw e;
}
}
/**
* Evidence a caller must hand the model-change gate: the resolved
* embedding configuration of THIS process, from which the gate derives
* the one capability verdict (`resolveSemanticCapability`) that decides
* whether a cleared corpus could actually be rebuilt.
*
* Callers that open write connections (the `Store` wrapper every
* production path funnels through) must pass `{ semantic }` from their
* resolved config. A caller that passes no gate supplies no capability
* evidence, and the legacy clear contract applies unchanged - the gate
* is evidence-driven, so absence of evidence is never read as evidence
* of rebuildability.
*/
export interface EmbeddingRebuildGate {
readonly semantic: ResolvedEmbeddingConfig;
}
/**
* Whether any `chunks` rows exist - the source-material half of the
* verify-before-replace gate.
*
* Deliberately a local query rather than an import of `countChunks`
* from `./chunks.ts`: that module already imports this one for the vec
* purge helpers, and a back-import would close a module cycle for one
* SQL line. The census spelling stays `countChunks`; this predicate
* only answers presence.
*/
function hasChunkRows(db: Database): boolean {
const row = db.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM chunks").get();
return (row?.n ?? 0) > 0;
}
/**
* The verify-before-replace gate (t_2fbdaf70): a clear that destroys
* stored vectors must not run unless the corpus can be rebuilt.
*
* Loss-bearing trigger: the stored embeddings row count. A store without
* embeddings loses nothing to a clear. In an FK-enforced store
* (`PRAGMA foreign_keys = ON`, `embeddings.chunk_id REFERENCES chunks`)
* stored embeddings imply chunk rows, so the design's chunk-material
* assertion cannot flip the verdict in a healthy store - it surfaces in
* the refusal detail instead, naming the one corruption state
* (embeddings without chunks) where the rebuild would have no source.
*
* The verdict is the capability tier: a `disabled` or
* `credential-missing` configuration cannot recompute the vectors, so
* the clear would be destruction without a recovery path. The refusal
* is returned, named by the tier's error code, BEFORE any mutation: the
* old vectors and the old recorded model both survive, the store still
* opens (keyword indexing does not depend on vectors), and every later
* open names it again until the operator restores a rebuilding
* configuration or reverts the model.
*/
function rebuildRefusalBeforeClear(
db: Database,
gate: EmbeddingRebuildGate | undefined,
context: {
readonly previousModel: string | null;
readonly previousDimension: number | null;
readonly model: string | null;
readonly dimension: number | null;
/**
* Set when the model and dimension are unchanged and only the prefix
* pair differs, so the refusal names what actually changed instead of
* a "model change from X to X" (#255).
*/
readonly prefixChange?: {
readonly from: EmbeddingPrefixPair;
readonly to: EmbeddingPrefixPair;
};
},
): EmbeddingRebuildRefusal | null {
if (gate === undefined) return null;
const stored = countEmbeddings(db);
if (stored === 0) return null;
const capability = resolveSemanticCapability(gate.semantic);
if (!isBlockedCapability(capability)) return null;
const material = hasChunkRows(db) ? "" : "; no chunk material remains to rebuild from";
const change =
context.prefixChange === undefined
? `embedding model change from ${context.previousModel}/${context.previousDimension} to ` +
`${context.model}/${context.dimension}`
: `embedding prefix change from ` +
`[${context.prefixChange.from.query}|${context.prefixChange.from.passage}] to ` +
`[${context.prefixChange.to.query}|${context.prefixChange.to.passage}]`;
const restore =
context.prefixChange === undefined
? "restore the previous model or the provider credential"
: "restore the previous prefixes or enable a provider that can re-embed";
return {
code: BLOCKED_TIER_ERROR_CODE[capability.tier],
message:
`${change} refused: ${stored} stored embedding(s) cannot be ` +
`rebuilt while the semantic capability is ${capability.tier}${material}; the stored ` +
`embeddings and the recorded model are kept; ${restore}, then reindex`,
};
}
/**
* Compare the configured embedding model/dimension with what was
* recorded in `index_state` on the last index run. If they differ
* and both old + new are non-null, drop embeddings + vec table and
* log one line per design §13. First-time set just records state.
*
* Two guarded edges (t_2fbdaf70):
* - The clear is gated on rebuildability when the caller passes an
* {@link EmbeddingRebuildGate}: with stored vectors present and a
* capability tier that cannot recompute them, the clear is refused:
* nothing is mutated, the refusal is logged and returned on the
* outcome under the tier's error code, and the caller's open goes on.
* - A named-model -> null-model transition is not a detected change
* (the ladder requires both sides non-null), but under the gate's own
* logic it must not clear either - rebuildability cannot be verified
* with no model configured. It warns one line naming the last recorded
* model and keeps the vectors; with nothing stored it stays silent.
*/
export function ensureEmbeddingModel(
db: Database,
vec: VecRuntime,
model: string | null,
dimension: number | null,
prefixes?: EmbeddingPrefixPair,
gate?: EmbeddingRebuildGate,
): ModelChangeOutcome {
const prevModel = getState(db, EMBEDDING_MODEL_STATE_KEY);
const prevDimRaw = getState(db, EMBEDDING_DIMENSION_STATE_KEY);
const prevDim = prevDimRaw === null ? null : Number(prevDimRaw);
const modelChanged = prevModel !== null && model !== null && prevModel !== model;
const dimChanged =
prevDim !== null && dimension !== null && Number.isFinite(prevDim) && prevDim !== dimension;
// A named-model -> null-model transition (t_2fbdaf70). The local
// embedder's model is implicit - the config never names it - so a
// recorded LOCAL_EMBEDDING_MODEL meeting a null incoming model is the
// steady state of every local-provider vault, not a removed model;
// warning there would be recurring noise on every write open.
const modelRemoved = prevModel !== null && model === null && prevModel !== LOCAL_EMBEDDING_MODEL;
// A prefix change invalidates stored vectors exactly as a model/dimension
// change does: vectors embedded under the old prefix are not comparable to
// queries embedded under the new one. Reuse the same clear-and-log path.
const prevQueryPrefix = getState(db, EMBEDDING_PREFIX_QUERY_STATE_KEY);
const prevPassagePrefix = getState(db, EMBEDDING_PREFIX_PASSAGE_STATE_KEY);
// Legacy stores predate prefix metadata: absent state (null) means the
// existing vectors were embedded with EMPTY prefixes. Treat missing state
// as the empty pair so a switch to non-empty prefixes (e.g. E5's
// `query:`/`passage:`) is detected as a change and the now-incompatible
// unprefixed vectors are cleared, rather than silently marked compatible.
const effectivePrevQueryPrefix = prevQueryPrefix ?? "";
const effectivePrevPassagePrefix = prevPassagePrefix ?? "";
const prefixChanged =
prefixes !== undefined &&
countEmbeddings(db) > 0 &&
(effectivePrevQueryPrefix !== prefixes.query ||
effectivePrevPassagePrefix !== prefixes.passage);
const refusalContext = {
previousModel: prevModel,
previousDimension: prevDim,
model,
dimension,
...(!modelChanged && !dimChanged && prefixChanged
? {
prefixChange: {
from: { query: effectivePrevQueryPrefix, passage: effectivePrevPassagePrefix },
to: prefixes,
},
}
: {}),
};
// Vectors kept on a named-model -> null-model open (#255): this writer
// has no model, so it never embedded anything and its prefix pair says
// nothing about the stored vectors. Recording it would re-stamp them with
// a foreign identity, and the next semantic writer would then see a
// spurious prefix change and clear compatible vectors.
const keptUnverified = modelRemoved && countEmbeddings(db) > 0;
const refusal =
modelChanged || dimChanged || (!modelRemoved && prefixChanged)
? rebuildRefusalBeforeClear(db, gate, refusalContext)
: null;
if (refusal !== null) {
// eslint-disable-next-line no-console
console.error(`${refusal.code}: ${refusal.message}`);
return Object.freeze({
wasChanged: false,
previousModel: prevModel,
previousDimension: prevDim,
currentModel: prevModel,
currentDimension: prevDim,
refusal,
});
}
if (modelChanged || dimChanged) {
clearEmbeddings(db, vec.loaded);
// eslint-disable-next-line no-console
console.error(
`embedding model changed from ${prevModel}/${prevDim} to ${model}/${dimension}, embeddings cleared`,
);
deleteState(db, EMBEDDING_MODEL_STATE_KEY);
deleteState(db, EMBEDDING_DIMENSION_STATE_KEY);
// The recorded sqlite-vec version described the vectors that were
// just cleared, so it must not outlive them: leaving it would
// claim an ABI for storage that no longer holds any.
deleteState(db, EMBEDDING_VEC_VERSION_STATE_KEY);
} else if (keptUnverified) {
// Named-model -> null-model: never a clear (see docblock), never silent.
// This branch also deliberately precedes the prefix branch: with no
// model configured there is nothing to verify a prefix clear against.
// eslint-disable-next-line no-console
console.error(
`embedding model removed from config (was ${prevModel}); ` +
`${countEmbeddings(db)} stored embedding(s) kept - rebuildability cannot be verified ` +
`with no model configured`,
);
} else if (prefixChanged) {
// Model/dimension unchanged: the prefix change alone triggers the clear.
clearEmbeddings(db, vec.loaded);
// eslint-disable-next-line no-console
console.error(
`embedding prefixes changed from [${effectivePrevQueryPrefix}|${effectivePrevPassagePrefix}] ` +
`to [${prefixes.query}|${prefixes.passage}], embeddings cleared`,
);
}
if (model !== null) setState(db, EMBEDDING_MODEL_STATE_KEY, model);
if (dimension !== null) setState(db, EMBEDDING_DIMENSION_STATE_KEY, String(dimension));
// Same shape as the model and dimension above: recorded only when
// this build actually has a value. With the extension unavailable
// there is nothing to record, and writing a placeholder would be a
// claim about storage this process never touched.
if (vec.version !== null) {
setState(db, EMBEDDING_VEC_VERSION_STATE_KEY, vec.version);
}
// Like the model and dimension: recorded only by a writer that owns the
// vectors' identity, never by one that kept them unverified.
if (prefixes !== undefined && !keptUnverified) {
setState(db, EMBEDDING_PREFIX_QUERY_STATE_KEY, prefixes.query);
setState(db, EMBEDDING_PREFIX_PASSAGE_STATE_KEY, prefixes.passage);
}
// (Re)create vec table when we know the dimension and vec is loaded.
if (vec.loaded && dimension !== null) {
ensureVecTable(db, dimension);
}
return Object.freeze({
wasChanged: modelChanged || dimChanged,
previousModel: prevModel,
previousDimension: prevDim,
currentModel: model,
currentDimension: dimension,
});
}
/**
* Total ordering for the rows the KNN returns. Distance ties are not
* exotic here - two chunks with identical text embed to the same vector
* and therefore to the same distance - and the caller truncates, so a
* partial order lets the scan decide which tied neighbour survives.
* `chunk_id` is `chunks.id`, an `INTEGER PRIMARY KEY`, so the extra sort
* key needs no schema change.
*/
const VEC_ORDER = "ORDER BY v.distance ASC, m.chunk_id ASC";
/**
* How much wider than `limit` the `k` of the KNN is asked for.
*
* `ORDER BY` alone cannot make the cut deterministic: sqlite-vec selects
* its `k` nearest rows FIRST, by its own internal order, and the sort
* only ever reaches the rows that selection already kept. Asking for a
* wider `k` and cutting here means a group of equidistant neighbours
* straddling the boundary is resolved by the unique sort key rather than
* by vec's traversal. The scan is a full one with a k-sized heap either
* way, so the wider k costs heap, not passes.
*
* The prefixed branch has always widened by this factor for the
* different reason that its path predicate drops rows after the KNN; the
* two branches now share one constant instead of one of them carrying a
* bare literal.
*/
const VEC_KNN_OVERFETCH = 4;
/**
* How often a starved FILTERED lane widens its window before it gives up
* (recall-quality review M2). The kind, pushdown and prefix predicates
* run AFTER vec0's k-selection, so a selective filter can drop most of
* the naive window - `kind=page` worst of all, one page chunk per
* document - and answer with little or nothing. Each escalation asks
* again with `k` multiplied by {@link VEC_KNN_OVERFETCH}, at most this
* many times, and {@link VEC_KNN_MAX} binds every window: it is
* sqlite-vec's own k ceiling, past which the extension refuses the query.
* An unfiltered lane never escalates - a short answer there is an
* exhausted corpus, not a starved filter.
*/
const VEC_KNN_ESCALATIONS = 2;
const VEC_KNN_MAX = 4096;
/** The named stderr notice a still-starved filtered lane emits. */
const VEC_FILTER_STARVED_NOTICE = "semantic-lane-filter-starved";
/**
* How many embedded chunks the KNN draws from. Read only on the starved
* path, and only to tell the two short answers apart: a window that
* already reaches the whole population judged every chunk in the store,
* so coming up short is the corpus's own shape and no widening can help;
* a window still below the population was truncated, and rows it never
* selected might have passed the filter - that, and only that, is
* starvation. `chunk_vec_map` is the plain rowid bridge, one row per
* stored vector, so counting it answers without scanning the vtable.
*/
function countVecPopulation(db: Database): number {
return db.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM chunk_vec_map").get()?.n ?? 0;
}
/**
* The composed semantic-lane query (recall-quality, C4). With no prefix,
* kind or pushed predicates this is byte-identical to the pre-C4 string;
* every active filter extends the WHERE clause beside the KNN, inside the
* same overfetch the prefixed branch has always shared. Exported so the
* default strings stay pinned by test.
*/
export interface SemanticTopKOptions {
readonly limit: number;
readonly pathPrefix?: string | null;
readonly kind?: SearchKind | null;
readonly pushdown?: PropertyPushdown | null;
}
/**
* The SQL builder reads everything but the overfetch row cap - the KNN
* `k = ?` placeholder is positional - so the default-argument form needs
* no limit, while the executor keeps it required.
*/
export type SemanticTopKSqlOptions = Omit<SemanticTopKOptions, "limit"> & {
readonly limit?: number;
};
export function semanticTopKSql(opts: SemanticTopKSqlOptions = {}): string {
const prefix = opts.pathPrefix && opts.pathPrefix.length > 0 ? opts.pathPrefix : null;
// The bound is null when the prefix's last code point cannot increment
// (path-prefix.ts): the read runs lower-bound-only, never bound to a
// stand-in upper value.
const range = prefix !== null ? pathPrefixRange(prefix) : null;
const pushFrag = pushdownSqlFragment(opts.pushdown ?? null);
const kind = opts.kind ?? null;
const needsDocuments = prefix !== null || kind !== null || pushFrag !== null;
const documentsJoin = needsDocuments ? "JOIN documents d ON d.id = c.document_id " : "";
const filters = [
"v.embedding MATCH ?",
"k = ?",
...(range !== null ? ["d.path >= ?"] : []),
...(range !== null && range.hi !== null ? ["d.path < ?"] : []),
...(pushFrag !== null ? [pushFrag.sql] : []),
...(kind !== null ? ["c.chunk_kind = ?"] : []),
];
return (
"SELECT m.chunk_id AS chunk_id, c.document_id AS document_id, v.distance AS distance " +
"FROM chunk_vec v " +
"JOIN chunk_vec_map m ON m.vec_rowid = v.rowid " +
"JOIN chunks c ON c.id = m.chunk_id " +
documentsJoin +
`WHERE ${filters.join(" AND ")} ` +
VEC_ORDER
);
}
export function semanticTopK(
db: Database,
vecLoaded: boolean,
queryVector: ReadonlyArray<number> | Float32Array,
opts: SemanticTopKOptions,
): SemanticHit[] {
if (!vecLoaded) {
throw new SearchError(
"VEC_EXTENSION_UNAVAILABLE",
"sqlite-vec extension not loaded; semantic search unavailable",
);
}
const limit = Math.max(1, opts.limit | 0);
const prefix = opts.pathPrefix && opts.pathPrefix.length > 0 ? opts.pathPrefix : null;
const pushFrag = pushdownSqlFragment(opts.pushdown ?? null);
const kind = opts.kind ?? null;
const filtered = prefix !== null || pushFrag !== null || kind !== null;
assertValidVector(queryVector, "semanticTopK");
const buf = vecToBuffer(queryVector);
const statement = db.query<
{ chunk_id: number; document_id: number; distance: number },
SQLQueryBindings[]
>(semanticTopKSql({ pathPrefix: prefix, kind, pushdown: opts.pushdown ?? null }));
const bindings = (): SQLQueryBindings[] => [
...(prefix !== null
? (() => {
// Sargable prefix range (C1); see path-prefix.ts. A null bound
// (last code point cannot increment) runs lower-bound-only.
const { lo, hi } = pathPrefixRange(prefix);
return hi !== null ? [lo, hi] : [lo];
})()
: []),
...(pushFrag !== null ? pushFrag.bindings : []),
...(kind !== null ? [kind] : []),
];
let knn = Math.min(limit * VEC_KNN_OVERFETCH, VEC_KNN_MAX);
let rows = statement.all(buf, knn, ...bindings());
// Filtered starvation (recall-quality review M2): the predicates above
// judge rows vec0 already selected, so a selective filter can empty the
// window. Widen it - bounded, see VEC_KNN_ESCALATIONS - before giving
// up; a window that already reaches the whole population needs no
// widening, and a widening that still came short names the starvation.
if (filtered && rows.length < limit) {
const population = countVecPopulation(db);
for (
let escalation = 0;
escalation < VEC_KNN_ESCALATIONS && rows.length < limit;
escalation++
) {
if (knn >= population) break;
const widened = Math.min(knn * VEC_KNN_OVERFETCH, VEC_KNN_MAX);
if (widened <= knn) break;
knn = widened;
rows = statement.all(buf, knn, ...bindings());
}
if (rows.length < limit && knn < population) {
// eslint-disable-next-line no-console
console.error(
`${VEC_FILTER_STARVED_NOTICE}: the filtered nearest-neighbour window widened to ` +
`${knn} of ${population} embedded chunks and still kept ${rows.length} of the ` +
`requested ${limit} row(s); results may be incomplete while this filter is so selective`,
);
}
}
return rows.slice(0, limit).map((r) => ({
chunkId: r.chunk_id,
documentId: r.document_id,
distance: r.distance,
}));
}