Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* The `chunk_trigram` FTS5 shadow (v9): an opt-in candidate source that
* broadens large-vault keyword recall with the substring / partial-token
* matches the word tokenizer misses.
*/
import { Database, type SQLQueryBindings } from "bun:sqlite";
import { RETRIEVAL_DEGRADATION } from "../retrieval-trail.ts";
import type { RetrievalDegradation } from "../retrieval-trail.ts";
import type { PropertyPushdown, SearchKind } from "../property-pushdown.ts";
import { pushdownSqlFragment } from "../property-pushdown.ts";
import { pathPrefixRange } from "./path-prefix.ts";
import type { KeywordHit } from "./keyword.ts";
/**
* What the lane returns: the candidates it gathered, the warnings its
* caller owes the query, and the same faults as machine codes. Same shape
* as the keyword lane's own outcome, because the two fill the same
* candidate pool.
*/
export interface TrigramCandidateOutcome {
readonly hits: KeywordHit[];
readonly warnings: ReadonlyArray<string>;
/**
* The typed half of {@link warnings}: the classification below survives
* to the caller instead of being destroyed by the sentence it is
* interpolated into.
*/
readonly degraded: ReadonlyArray<RetrievalDegradation>;
}
const NO_WARNINGS: ReadonlyArray<string> = Object.freeze([]);
const NO_DEGRADATIONS: ReadonlyArray<RetrievalDegradation> = Object.freeze([]);
/**
* Total ordering for the top-K cut, for the reason the keyword lane's
* own `CHUNK_ORDER` gives: BM25 ties are common in a trigram index -
* every chunk sharing the same trigram profile scores alike - and
* `LIMIT` over a partial order picks by scan artefact. `chunks.id` is
* `INTEGER PRIMARY KEY`, so no schema change is involved.
*/
const CHUNK_ORDER = "ORDER BY bm25 ASC, chunk_id ASC";
/**
* The one condition this lane stays SILENT for: `chunk_trigram` itself is
* absent, which is a store migrated to v9 but never rebuilt - the table
* is documented as optional. Anchored on the right, because a message
* naming `chunk_trigram_data` / `_idx` / `_content` / `_docsize` names an
* FTS5 shadow the index has LOST: the index HAS the trigram table and
* cannot read it, which is corruption and must be reported, not silently
* answered with an empty candidate list.
*/
export function isMissingTrigramTable(message: string): boolean {
return /no such table:?\s*(?:main\.)?chunk_trigram(?![A-Za-z0-9_])/i.test(message);
}
/**
* Why the lane could not answer, as a closed vocabulary. The SQLite
* message itself never reaches the warning: FTS5 reports a rejected MATCH
* by echoing the expression, and that expression is built from the
* caller's query, while other messages can name the index file - a
* warning travels into logs and MCP payloads, so it carries a cause and
* never content.
*/
type TrigramFault =
| "locked"
| "tokenizer_unavailable"
| "corrupt"
| "shadow_incomplete"
| "unknown";
function classifyTrigramFault(message: string): TrigramFault {
if (/\bis locked\b|SQLITE_BUSY/i.test(message)) return "locked";
if (/no such tokenizer/i.test(message)) return "tokenizer_unavailable";
if (/malformed|corrupt/i.test(message)) return "corrupt";
if (/no such (?:table|column|module)/i.test(message)) return "shadow_incomplete";
return "unknown";
}
/**
* The lane only ever ADDS to a pool the keyword lane already filled, so a
* fault degrades the lane instead of failing the query - but visibly:
* `trigram_degraded` is the greppable flag, alongside `hybrid_degraded`
* for the semantic lane. A silent empty list here would narrow recall
* while reading as a vault with nothing to match.
*/
function degradeWarning(fault: TrigramFault): string {
return `trigram_degraded [${fault}]: trigram candidate lane skipped; keyword candidates unaffected`;
}
/**
* The same fault as a machine code. The classification is what the caller
* actually needs, and it already existed here: until the retrieval trail
* it survived only as far as the sentence above, where the last typed
* step of this lane was interpolated away.
*/
function degradation(fault: TrigramFault): RetrievalDegradation {
return Object.freeze({
code: RETRIEVAL_DEGRADATION.keywordTrigramLaneFault,
detail: Object.freeze({ fault }),
});
}
/**
* Trigram candidate lookup over the `chunk_trigram` FTS5 shadow (v9).
* Returns bm25-ordered keyword hits whose content matches the trigram
* query - a strict superset of exact substring matches for the query's
* terms, used as an opt-in candidate source that broadens large-vault
* keyword recall (substring / partial-token matches the word tokenizer
* misses). Yields no candidates and no warning when the trigram table is
* absent; any other fault yields no candidates and one warning naming
* the cause.
*/
export function trigramCandidates(
db: Database,
trigramQuery: string,
opts: {
readonly limit: number;
readonly pathPrefix?: string | null;
/**
* The same pushdown the keyword lane ran (recall-quality, C4). The
* lane only ADDS to the keyword pool, so it must not re-admit rows
* the lane's own SQL excluded - a kind or promoted-property filter
* leaks here otherwise.
*/
readonly kind?: SearchKind | null;
readonly pushdown?: PropertyPushdown | null;
},
): TrigramCandidateOutcome {
const limit = Math.max(1, opts.limit | 0);
const prefix = opts.pathPrefix && opts.pathPrefix.length > 0 ? opts.pathPrefix : null;
const pushFrag = pushdownSqlFragment(opts.pushdown ?? null);
const kind = opts.kind ?? null;
const extraFilters = [
...(pushFrag !== null ? [pushFrag.sql] : []),
...(kind !== null ? ["c.chunk_kind = ?"] : []),
];
const extraBindings: SQLQueryBindings[] = [
...(pushFrag !== null ? pushFrag.bindings : []),
...(kind !== null ? [kind] : []),
];
try {
if (prefix !== null || extraFilters.length > 0) {
// The sargable prefix range every other scoped reader uses
// (path-prefix.ts); absent scope leaves the bounds unbound, and a
// boundless prefix (empty or all U+10FFFFs) runs lower-bound-only -
// honouring the module's contract to omit the `path < ?` clause
// exactly when `hi` is null, never bind a stand-in for it
// (`path < NULL` matches nothing).
const { lo, hi } = prefix !== null ? pathPrefixRange(prefix) : { lo: "", hi: "" };
const filters = [
"chunk_trigram MATCH ?",
...(prefix !== null ? ["d.path >= ?"] : []),
...(prefix !== null && hi !== null ? ["d.path < ?"] : []),
...extraFilters,
];
const rows = db
.query<{ chunk_id: number; document_id: number; bm25: number }, SQLQueryBindings[]>(
"SELECT c.id AS chunk_id, c.document_id AS document_id, bm25(chunk_trigram) AS bm25 " +
"FROM chunk_trigram " +
"JOIN chunks c ON c.id = chunk_trigram.rowid " +
"JOIN documents d ON d.id = c.document_id " +
`WHERE ${filters.join(" AND ")} ${CHUNK_ORDER} LIMIT ?`,
)
.all(
trigramQuery,
...(prefix !== null ? (hi !== null ? [lo, hi] : [lo]) : []),
...extraBindings,
limit,
);
return {
hits: rows.map((r) => ({ chunkId: r.chunk_id, documentId: r.document_id, bm25: r.bm25 })),
warnings: NO_WARNINGS,
degraded: NO_DEGRADATIONS,
};
}
const rows = db
.query<{ chunk_id: number; document_id: number; bm25: number }, [string, number]>(
"SELECT c.id AS chunk_id, c.document_id AS document_id, bm25(chunk_trigram) AS bm25 " +
"FROM chunk_trigram JOIN chunks c ON c.id = chunk_trigram.rowid " +
`WHERE chunk_trigram MATCH ? ${CHUNK_ORDER} LIMIT ?`,
)
.all(trigramQuery, limit);
return {
hits: rows.map((r) => ({ chunkId: r.chunk_id, documentId: r.document_id, bm25: r.bm25 })),
warnings: NO_WARNINGS,
degraded: NO_DEGRADATIONS,
};
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
if (isMissingTrigramTable(msg)) {
return { hits: [], warnings: NO_WARNINGS, degraded: NO_DEGRADATIONS };
}
const fault = classifyTrigramFault(msg);
return {
hits: [],
warnings: Object.freeze([degradeWarning(fault)]),
degraded: Object.freeze([degradation(fault)]),
};
}
}