Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* FTS5 query construction and keyword retrieval.
*
* The user-typed query is treated as a bag of phrase tokens joined by
* implicit AND. Natural-language connector words that are also uppercase
* FTS5 operators (`AND`, `OR`, `NOT`, `NEAR`) are dropped when other
* meaningful tokens remain. Each remaining token is wrapped in double
* quotes so FTS5 metacharacters (`*`, `(`, `)`, `:`, `^`) lose their
* special meaning. Internal `"` is escaped as `""` per the FTS5 grammar.
*
* Anchored in docs/plans/2026-05-16-brain-search-design.md §7.
*/
import type { KeywordHit, Store } from "./store.ts";
import { containsCjk, tokenizeCjkSearchText } from "./cjk-tokenizer.ts";
import { keywordTopKWithFtsSafety, type SafeKeywordOutcome } from "./fts-safety.ts";
import type { PropertyPushdown } from "./property-pushdown.ts";
import { RETRIEVAL_DEGRADATION, noteDegradation } from "./retrieval-trail.ts";
import type { RetrievalDegradationSink } from "./retrieval-trail.ts";
import { FTS_MATCH_MODE, type FtsMatchMode } from "./fts-match-mode.ts";
const FTS5_OPERATOR_TOKENS = new Set(["AND", "OR", "NOT", "NEAR"]);
export interface FtsMatchOptions {
readonly matchMode?: FtsMatchMode;
}
function quoteToken(t: string): string {
return `"${t.replace(/"/g, '""')}"`;
}
function dropStandaloneOperators(tokens: ReadonlyArray<string>): ReadonlyArray<string> {
const withoutOperators = tokens.filter((t) => !FTS5_OPERATOR_TOKENS.has(t));
return withoutOperators.length > 0 ? withoutOperators : tokens;
}
export function buildFtsMatch(rawQuery: string, opts: FtsMatchOptions = {}): string {
const tokens = containsCjk(rawQuery) ? tokenizeCjkSearchText(rawQuery) : rawQuery.split(/\s+/);
const cleaned = tokens.map((t) => t.trim()).filter((t) => t.length > 0);
if (cleaned.length === 0) return "";
const kept = dropStandaloneOperators(cleaned).map(quoteToken);
return opts.matchMode === FTS_MATCH_MODE.any ? kept.join(" OR ") : kept.join(" ");
}
/**
* Query-expansion match (v0.20.0): keep the original tokens as an
* implicit-AND group (the strong signal, still scored highest by bm25
* because it matches more terms) and OR in each expansion term as an
* optional broadening alternative:
*
* ("orig1" "orig2") OR "exp1" OR "exp2"
*
* With no expansion terms this is byte-identical to {@link buildFtsMatch},
* so expansion is a true no-op when disabled or empty. In `any` mode the
* base group is itself OR-joined - `("orig1" OR "orig2") OR "exp1"` - which
* is semantically the flat ANY superset through the same composition.
*/
export function buildExpandedFtsMatch(
rawQuery: string,
expandedTerms: ReadonlyArray<string>,
opts: FtsMatchOptions = {},
): string {
const base = buildFtsMatch(rawQuery, opts);
if (base === "" || expandedTerms.length === 0) return base;
const ors = expandedTerms.map(quoteToken).join(" OR ");
return `(${base}) OR ${ors}`;
}
export interface RunFtsOptions {
readonly limit: number;
readonly pathPrefix?: string | null;
/**
* Chunk-kind pushdown (recall-quality, C4): forwarded to the store's
* keyword top-K WHERE clause. Absent leaves the SQL byte-identical.
*/
readonly kind?: "page" | "section" | null;
/**
* Promoted property pushdown (recall-quality, C4): the compiled
* predicates from `resolvePropertyPushdown`, forwarded beside the kind
* filter. Absent leaves the SQL byte-identical.
*/
readonly pushdown?: PropertyPushdown | null;
/**
* Optional synonym-expansion terms (v0.20.0). When present and
* non-empty they are OR'd onto the original AND group to broaden
* recall; absent/empty leaves the query byte-identical.
*/
readonly expandedTerms?: ReadonlyArray<string>;
/**
* Caller-selectable match breadth (t_c5326ece). Default `all` keeps the
* implicit AND; `any` OR-joins the cleaned tokens. The derived-term and
* second-pass internal callers leave it unset.
*/
readonly matchMode?: FtsMatchMode;
/**
* Optional typed degradation sink (evidence-at-the-boundary, C2), the
* counterpart of the warnings the outcome already returns. An empty
* FTS match is the one condition here that produced neither a hit nor a
* word, so a caller that wants to distinguish "nothing to match on"
* from "nothing matched" hands one in. Absent leaves the call
* byte-identical, which is what the term-verification and second-pass
* callers want: they ask about a derived term, not about the query.
*/
readonly degraded?: RetrievalDegradationSink;
}
export function runFtsQueryDetailed(
store: Store,
rawQuery: string,
opts: RunFtsOptions,
): SafeKeywordOutcome {
const match =
opts.expandedTerms && opts.expandedTerms.length > 0
? buildExpandedFtsMatch(rawQuery, opts.expandedTerms, { matchMode: opts.matchMode })
: buildFtsMatch(rawQuery, { matchMode: opts.matchMode });
if (match === "") {
// The purest silent empty in the pipeline: every token was dropped,
// so no lookup ran at all and the caller was handed the same empty
// list a vault with no match hands back.
if (opts.degraded !== undefined) {
noteDegradation(opts.degraded, RETRIEVAL_DEGRADATION.keywordFtsMatchEmpty);
}
return Object.freeze({ hits: [], warnings: Object.freeze([]) });
}
return keywordTopKWithFtsSafety(store, match, opts);
}
export function runFtsQuery(store: Store, rawQuery: string, opts: RunFtsOptions): KeywordHit[] {
return runFtsQueryDetailed(store, rawQuery, opts).hits;
}