Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
7.7 KB
Raw Blame
/**
 * The keyword lane: the `chunk_fts` FTS5 index - its availability probe,
 * its BM25 top-K query, its corpus document frequencies, and the
 * integrity counts and rebuild that repair it.
 */

import { Database, type SQLQueryBindings } from "bun:sqlite";

import type { PropertyPushdown } from "../property-pushdown.ts";
import type { SearchKind } from "../property-pushdown.ts";
import { pushdownSqlFragment } from "../property-pushdown.ts";
import { SearchError } from "../types.ts";
import { pathPrefixRange } from "./path-prefix.ts";
import {
  acquireWriterLockSync,
  keepWriterLeasesFresh,
  WRITER_LEASE_BLOCKING_BUDGET_MS,
} from "./writer-lock.ts";

export interface KeywordHit {
  readonly chunkId: number;
  readonly documentId: number;
  /** Lower is better (FTS5 returns negative bm25). */
  readonly bm25: number;
}

/**
 * BM25 column weights for `chunk_fts`: body first, heading breadcrumb
 * discounted. Named once so the prefixed and unprefixed queries below
 * cannot rank the same row two ways.
 */
const CHUNK_FTS_BM25 = "bm25(chunk_fts, 1.0, 0.3)";

/**
 * Total ordering for the top-K cut. BM25 alone is not a total order -
 * two chunks with the same term frequencies and the same length score
 * identically - so `LIMIT` would otherwise take whichever tied rows the
 * scan happened to reach first, which is an artefact of the b-tree and
 * not a property of the query. `chunks.id` is `INTEGER PRIMARY KEY`, so
 * it is unique and already present; ending the sort on it makes the cut
 * reproducible without touching the schema.
 */
const CHUNK_ORDER = "ORDER BY bm25 ASC, chunk_id ASC";

export function ensureFts5(db: Database): void {
  // Probe FTS5 by attempting a benign expression. bun:sqlite ships with
  // FTS5 enabled in the embedded amalgamation; failing here means a
  // custom build without FTS5 and the index is unusable.
  try {
    db.query("SELECT fts5_source('chunk_fts')").get();
  } catch (e) {
    const msg = e instanceof Error ? e.message : String(e);
    // Older SQLite returns "no such function" or "no such table" depending
    // on whether FTS5 is even compiled in. We treat any failure here as
    // FTS5-unavailable rather than guessing the build flag.
    if (!/fts5|chunk_fts/i.test(msg)) {
      throw new SearchError("INDEX_UNREADABLE", `FTS5 probe failed: ${msg}`);
    }
  }
}

export function ftsIntegrityCounts(db: Database): {
  readonly chunks: number;
  readonly ftsRows: number;
} {
  const chunks = db.query<{ c: number }, []>("SELECT count(*) AS c FROM chunks").get()?.c ?? 0;
  const ftsRows =
    db.query<{ c: number }, []>("SELECT count(*) AS c FROM chunk_fts_docsize").get()?.c ?? 0;
  return Object.freeze({ chunks, ftsRows });
}

export function rebuildFtsIndex(db: Database): void {
  db.run("INSERT INTO chunk_fts(chunk_fts) VALUES('rebuild')");
}

/**
 * Rebuild under the writer lock when the caller does not already hold it
 * (a read-mode store repairing the index in place). `heldWriterLock` is
 * the caller's own ownership of the lock on `dbPath`: a write-mode store
 * holds it for its whole session and would contend with itself here,
 * since the lock is a file lock and re-entry from the holder is
 * INDEX_LOCKED like any other contender.
 */
export function rebuildFtsIndexWithWriterLock(
  db: Database,
  dbPath: string,
  heldWriterLock: boolean,
): void {
  // One statement that can run for minutes on a large index, so the lock
  // is kept fresh off the blocked thread for its duration.
  const rebuild = (): void =>
    keepWriterLeasesFresh(WRITER_LEASE_BLOCKING_BUDGET_MS, () => rebuildFtsIndex(db));
  if (heldWriterLock) {
    rebuild();
    return;
  }
  const release = acquireWriterLockSync(dbPath);
  try {
    rebuild();
  } finally {
    release();
  }
}

/**
 * Top-K BM25 keyword hits. `fts5Query` is an already-escaped FTS5
 * MATCH expression; building it is fts.ts's job.
 *
 * `kind` and `pushdown` (recall-quality, C4) extend the WHERE clause:
 * the kind discriminator on the v14 generated column, and the promoted
 * property predicates compiled by `pushdownSqlFragment`. Both are absent
 * from the SQL unless asked for - the default and prefix-only strings are
 * byte-identical to pre-C4, which `keywordTopKSql` pins.
 */
export interface KeywordTopKOptions {
  readonly limit: number;
  readonly pathPrefix?: string | null;
  readonly kind?: SearchKind | null;
  readonly pushdown?: PropertyPushdown | null;
}

/**
 * The SQL builder reads everything but the row cap - the `LIMIT ?`
 * placeholder is positional - so the default-argument form needs no
 * limit, while the executor keeps it required.
 */
export type KeywordTopKSqlOptions = Omit<KeywordTopKOptions, "limit"> & {
  readonly limit?: number;
};

export function keywordTopKSql(opts: KeywordTopKSqlOptions = {}): string {
  const prefix = opts.pathPrefix && opts.pathPrefix.length > 0 ? opts.pathPrefix : null;
  // The bound is null when the prefix's last code point cannot increment
  // (path-prefix.ts): the read runs lower-bound-only, never bound to a
  // stand-in upper value.
  const range = prefix !== null ? pathPrefixRange(prefix) : null;
  const pushFrag = pushdownSqlFragment(opts.pushdown ?? null);
  const kind = opts.kind ?? null;
  const select = `SELECT c.id AS chunk_id, c.document_id AS document_id, ${CHUNK_FTS_BM25} AS bm25 `;
  if (prefix === null && kind === null && pushFrag === null) {
    return (
      select +
      "FROM chunk_fts JOIN chunks c ON c.id = chunk_fts.rowid " +
      `WHERE chunk_fts MATCH ? ${CHUNK_ORDER} LIMIT ?`
    );
  }
  const filters = [
    "chunk_fts MATCH ?",
    ...(range !== null ? ["d.path >= ?"] : []),
    ...(range !== null && range.hi !== null ? ["d.path < ?"] : []),
    ...(pushFrag !== null ? [pushFrag.sql] : []),
    ...(kind !== null ? ["c.chunk_kind = ?"] : []),
  ];
  return (
    select +
    "FROM chunk_fts " +
    "JOIN chunks c ON c.id = chunk_fts.rowid " +
    "JOIN documents d ON d.id = c.document_id " +
    `WHERE ${filters.join(" AND ")} ${CHUNK_ORDER} LIMIT ?`
  );
}

export function keywordTopK(
  db: Database,
  fts5Query: string,
  opts: KeywordTopKOptions,
): KeywordHit[] {
  const limit = Math.max(1, opts.limit | 0);
  const prefix = opts.pathPrefix && opts.pathPrefix.length > 0 ? opts.pathPrefix : null;
  const pushFrag = pushdownSqlFragment(opts.pushdown ?? null);
  const kind = opts.kind ?? null;

  const rows = db
    .query<{ chunk_id: number; document_id: number; bm25: number }, SQLQueryBindings[]>(
      keywordTopKSql({ pathPrefix: prefix, kind, pushdown: opts.pushdown ?? null }),
    )
    .all(
      fts5Query,
      ...(prefix !== null
        ? (() => {
            const { lo, hi } = pathPrefixRange(prefix);
            return hi !== null ? [lo, hi] : [lo];
          })()
        : []),
      ...(pushFrag !== null ? pushFrag.bindings : []),
      ...(kind !== null ? [kind] : []),
      limit,
    );
  return rows.map((r) => ({
    chunkId: r.chunk_id,
    documentId: r.document_id,
    bm25: r.bm25,
  }));
}

/**
 * Corpus document frequency per term (recall-trust-suite, coverage
 * engine): how many distinct documents match each term in FTS. Each
 * term is quoted so FTS5 metacharacters stay literal. A term that
 * fails to query (e.g. tokenizer edge case) counts as 0 rather than
 * failing the search.
 */
export function documentFrequencies(
  db: Database,
  terms: ReadonlyArray<string>,
): Map<string, number> {
  const out = new Map<string, number>();
  if (terms.length === 0) return out;
  const q = db.query<{ n: number }, [string]>(
    "SELECT COUNT(DISTINCT c.document_id) AS n " +
      "FROM chunk_fts JOIN chunks c ON c.id = chunk_fts.rowid " +
      "WHERE chunk_fts MATCH ?",
  );
  for (const term of terms) {
    try {
      out.set(term, q.get(`"${term.replace(/"/g, '""')}"`)?.n ?? 0);
    } catch {
      out.set(term, 0);
    }
  }
  return out;
}