Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* The keyword lane: the `chunk_fts` FTS5 index - its availability probe,
* its BM25 top-K query, its corpus document frequencies, and the
* integrity counts and rebuild that repair it.
*/
import { Database, type SQLQueryBindings } from "bun:sqlite";
import type { PropertyPushdown } from "../property-pushdown.ts";
import type { SearchKind } from "../property-pushdown.ts";
import { pushdownSqlFragment } from "../property-pushdown.ts";
import { SearchError } from "../types.ts";
import { pathPrefixRange } from "./path-prefix.ts";
import {
acquireWriterLockSync,
keepWriterLeasesFresh,
WRITER_LEASE_BLOCKING_BUDGET_MS,
} from "./writer-lock.ts";
export interface KeywordHit {
readonly chunkId: number;
readonly documentId: number;
/** Lower is better (FTS5 returns negative bm25). */
readonly bm25: number;
}
/**
* BM25 column weights for `chunk_fts`: body first, heading breadcrumb
* discounted. Named once so the prefixed and unprefixed queries below
* cannot rank the same row two ways.
*/
const CHUNK_FTS_BM25 = "bm25(chunk_fts, 1.0, 0.3)";
/**
* Total ordering for the top-K cut. BM25 alone is not a total order -
* two chunks with the same term frequencies and the same length score
* identically - so `LIMIT` would otherwise take whichever tied rows the
* scan happened to reach first, which is an artefact of the b-tree and
* not a property of the query. `chunks.id` is `INTEGER PRIMARY KEY`, so
* it is unique and already present; ending the sort on it makes the cut
* reproducible without touching the schema.
*/
const CHUNK_ORDER = "ORDER BY bm25 ASC, chunk_id ASC";
export function ensureFts5(db: Database): void {
// Probe FTS5 by attempting a benign expression. bun:sqlite ships with
// FTS5 enabled in the embedded amalgamation; failing here means a
// custom build without FTS5 and the index is unusable.
try {
db.query("SELECT fts5_source('chunk_fts')").get();
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
// Older SQLite returns "no such function" or "no such table" depending
// on whether FTS5 is even compiled in. We treat any failure here as
// FTS5-unavailable rather than guessing the build flag.
if (!/fts5|chunk_fts/i.test(msg)) {
throw new SearchError("INDEX_UNREADABLE", `FTS5 probe failed: ${msg}`);
}
}
}
export function ftsIntegrityCounts(db: Database): {
readonly chunks: number;
readonly ftsRows: number;
} {
const chunks = db.query<{ c: number }, []>("SELECT count(*) AS c FROM chunks").get()?.c ?? 0;
const ftsRows =
db.query<{ c: number }, []>("SELECT count(*) AS c FROM chunk_fts_docsize").get()?.c ?? 0;
return Object.freeze({ chunks, ftsRows });
}
export function rebuildFtsIndex(db: Database): void {
db.run("INSERT INTO chunk_fts(chunk_fts) VALUES('rebuild')");
}
/**
* Rebuild under the writer lock when the caller does not already hold it
* (a read-mode store repairing the index in place). `heldWriterLock` is
* the caller's own ownership of the lock on `dbPath`: a write-mode store
* holds it for its whole session and would contend with itself here,
* since the lock is a file lock and re-entry from the holder is
* INDEX_LOCKED like any other contender.
*/
export function rebuildFtsIndexWithWriterLock(
db: Database,
dbPath: string,
heldWriterLock: boolean,
): void {
// One statement that can run for minutes on a large index, so the lock
// is kept fresh off the blocked thread for its duration.
const rebuild = (): void =>
keepWriterLeasesFresh(WRITER_LEASE_BLOCKING_BUDGET_MS, () => rebuildFtsIndex(db));
if (heldWriterLock) {
rebuild();
return;
}
const release = acquireWriterLockSync(dbPath);
try {
rebuild();
} finally {
release();
}
}
/**
* Top-K BM25 keyword hits. `fts5Query` is an already-escaped FTS5
* MATCH expression; building it is fts.ts's job.
*
* `kind` and `pushdown` (recall-quality, C4) extend the WHERE clause:
* the kind discriminator on the v14 generated column, and the promoted
* property predicates compiled by `pushdownSqlFragment`. Both are absent
* from the SQL unless asked for - the default and prefix-only strings are
* byte-identical to pre-C4, which `keywordTopKSql` pins.
*/
export interface KeywordTopKOptions {
readonly limit: number;
readonly pathPrefix?: string | null;
readonly kind?: SearchKind | null;
readonly pushdown?: PropertyPushdown | null;
}
/**
* The SQL builder reads everything but the row cap - the `LIMIT ?`
* placeholder is positional - so the default-argument form needs no
* limit, while the executor keeps it required.
*/
export type KeywordTopKSqlOptions = Omit<KeywordTopKOptions, "limit"> & {
readonly limit?: number;
};
export function keywordTopKSql(opts: KeywordTopKSqlOptions = {}): string {
const prefix = opts.pathPrefix && opts.pathPrefix.length > 0 ? opts.pathPrefix : null;
// The bound is null when the prefix's last code point cannot increment
// (path-prefix.ts): the read runs lower-bound-only, never bound to a
// stand-in upper value.
const range = prefix !== null ? pathPrefixRange(prefix) : null;
const pushFrag = pushdownSqlFragment(opts.pushdown ?? null);
const kind = opts.kind ?? null;
const select = `SELECT c.id AS chunk_id, c.document_id AS document_id, ${CHUNK_FTS_BM25} AS bm25 `;
if (prefix === null && kind === null && pushFrag === null) {
return (
select +
"FROM chunk_fts JOIN chunks c ON c.id = chunk_fts.rowid " +
`WHERE chunk_fts MATCH ? ${CHUNK_ORDER} LIMIT ?`
);
}
const filters = [
"chunk_fts MATCH ?",
...(range !== null ? ["d.path >= ?"] : []),
...(range !== null && range.hi !== null ? ["d.path < ?"] : []),
...(pushFrag !== null ? [pushFrag.sql] : []),
...(kind !== null ? ["c.chunk_kind = ?"] : []),
];
return (
select +
"FROM chunk_fts " +
"JOIN chunks c ON c.id = chunk_fts.rowid " +
"JOIN documents d ON d.id = c.document_id " +
`WHERE ${filters.join(" AND ")} ${CHUNK_ORDER} LIMIT ?`
);
}
export function keywordTopK(
db: Database,
fts5Query: string,
opts: KeywordTopKOptions,
): KeywordHit[] {
const limit = Math.max(1, opts.limit | 0);
const prefix = opts.pathPrefix && opts.pathPrefix.length > 0 ? opts.pathPrefix : null;
const pushFrag = pushdownSqlFragment(opts.pushdown ?? null);
const kind = opts.kind ?? null;
const rows = db
.query<{ chunk_id: number; document_id: number; bm25: number }, SQLQueryBindings[]>(
keywordTopKSql({ pathPrefix: prefix, kind, pushdown: opts.pushdown ?? null }),
)
.all(
fts5Query,
...(prefix !== null
? (() => {
const { lo, hi } = pathPrefixRange(prefix);
return hi !== null ? [lo, hi] : [lo];
})()
: []),
...(pushFrag !== null ? pushFrag.bindings : []),
...(kind !== null ? [kind] : []),
limit,
);
return rows.map((r) => ({
chunkId: r.chunk_id,
documentId: r.document_id,
bm25: r.bm25,
}));
}
/**
* Corpus document frequency per term (recall-trust-suite, coverage
* engine): how many distinct documents match each term in FTS. Each
* term is quoted so FTS5 metacharacters stay literal. A term that
* fails to query (e.g. tokenizer edge case) counts as 0 rather than
* failing the search.
*/
export function documentFrequencies(
db: Database,
terms: ReadonlyArray<string>,
): Map<string, number> {
const out = new Map<string, number>();
if (terms.length === 0) return out;
const q = db.query<{ n: number }, [string]>(
"SELECT COUNT(DISTINCT c.document_id) AS n " +
"FROM chunk_fts JOIN chunks c ON c.id = chunk_fts.rowid " +
"WHERE chunk_fts MATCH ?",
);
for (const term of terms) {
try {
out.set(term, q.get(`"${term.replace(/"/g, '""')}"`)?.n ?? 0);
} catch {
out.set(term, 0);
}
}
return out;
}