Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
5.4 KB
Raw Blame
/**
 * FTS5 query construction and keyword retrieval.
 *
 * The user-typed query is treated as a bag of phrase tokens joined by
 * implicit AND. Natural-language connector words that are also uppercase
 * FTS5 operators (`AND`, `OR`, `NOT`, `NEAR`) are dropped when other
 * meaningful tokens remain. Each remaining token is wrapped in double
 * quotes so FTS5 metacharacters (`*`, `(`, `)`, `:`, `^`) lose their
 * special meaning. Internal `"` is escaped as `""` per the FTS5 grammar.
 *
 * Anchored in docs/plans/2026-05-16-brain-search-design.md §7.
 */

import type { KeywordHit, Store } from "./store.ts";
import { containsCjk, tokenizeCjkSearchText } from "./cjk-tokenizer.ts";
import { keywordTopKWithFtsSafety, type SafeKeywordOutcome } from "./fts-safety.ts";
import type { PropertyPushdown } from "./property-pushdown.ts";
import { RETRIEVAL_DEGRADATION, noteDegradation } from "./retrieval-trail.ts";
import type { RetrievalDegradationSink } from "./retrieval-trail.ts";
import { FTS_MATCH_MODE, type FtsMatchMode } from "./fts-match-mode.ts";

const FTS5_OPERATOR_TOKENS = new Set(["AND", "OR", "NOT", "NEAR"]);

export interface FtsMatchOptions {
  readonly matchMode?: FtsMatchMode;
}

function quoteToken(t: string): string {
  return `"${t.replace(/"/g, '""')}"`;
}

function dropStandaloneOperators(tokens: ReadonlyArray<string>): ReadonlyArray<string> {
  const withoutOperators = tokens.filter((t) => !FTS5_OPERATOR_TOKENS.has(t));
  return withoutOperators.length > 0 ? withoutOperators : tokens;
}

export function buildFtsMatch(rawQuery: string, opts: FtsMatchOptions = {}): string {
  const tokens = containsCjk(rawQuery) ? tokenizeCjkSearchText(rawQuery) : rawQuery.split(/\s+/);
  const cleaned = tokens.map((t) => t.trim()).filter((t) => t.length > 0);
  if (cleaned.length === 0) return "";
  const kept = dropStandaloneOperators(cleaned).map(quoteToken);
  return opts.matchMode === FTS_MATCH_MODE.any ? kept.join(" OR ") : kept.join(" ");
}

/**
 * Query-expansion match (v0.20.0): keep the original tokens as an
 * implicit-AND group (the strong signal, still scored highest by bm25
 * because it matches more terms) and OR in each expansion term as an
 * optional broadening alternative:
 *
 *   ("orig1" "orig2") OR "exp1" OR "exp2"
 *
 * With no expansion terms this is byte-identical to {@link buildFtsMatch},
 * so expansion is a true no-op when disabled or empty. In `any` mode the
 * base group is itself OR-joined - `("orig1" OR "orig2") OR "exp1"` - which
 * is semantically the flat ANY superset through the same composition.
 */
export function buildExpandedFtsMatch(
  rawQuery: string,
  expandedTerms: ReadonlyArray<string>,
  opts: FtsMatchOptions = {},
): string {
  const base = buildFtsMatch(rawQuery, opts);
  if (base === "" || expandedTerms.length === 0) return base;
  const ors = expandedTerms.map(quoteToken).join(" OR ");
  return `(${base}) OR ${ors}`;
}

export interface RunFtsOptions {
  readonly limit: number;
  readonly pathPrefix?: string | null;
  /**
   * Chunk-kind pushdown (recall-quality, C4): forwarded to the store's
   * keyword top-K WHERE clause. Absent leaves the SQL byte-identical.
   */
  readonly kind?: "page" | "section" | null;
  /**
   * Promoted property pushdown (recall-quality, C4): the compiled
   * predicates from `resolvePropertyPushdown`, forwarded beside the kind
   * filter. Absent leaves the SQL byte-identical.
   */
  readonly pushdown?: PropertyPushdown | null;
  /**
   * Optional synonym-expansion terms (v0.20.0). When present and
   * non-empty they are OR'd onto the original AND group to broaden
   * recall; absent/empty leaves the query byte-identical.
   */
  readonly expandedTerms?: ReadonlyArray<string>;
  /**
   * Caller-selectable match breadth (t_c5326ece). Default `all` keeps the
   * implicit AND; `any` OR-joins the cleaned tokens. The derived-term and
   * second-pass internal callers leave it unset.
   */
  readonly matchMode?: FtsMatchMode;
  /**
   * Optional typed degradation sink (evidence-at-the-boundary, C2), the
   * counterpart of the warnings the outcome already returns. An empty
   * FTS match is the one condition here that produced neither a hit nor a
   * word, so a caller that wants to distinguish "nothing to match on"
   * from "nothing matched" hands one in. Absent leaves the call
   * byte-identical, which is what the term-verification and second-pass
   * callers want: they ask about a derived term, not about the query.
   */
  readonly degraded?: RetrievalDegradationSink;
}

export function runFtsQueryDetailed(
  store: Store,
  rawQuery: string,
  opts: RunFtsOptions,
): SafeKeywordOutcome {
  const match =
    opts.expandedTerms && opts.expandedTerms.length > 0
      ? buildExpandedFtsMatch(rawQuery, opts.expandedTerms, { matchMode: opts.matchMode })
      : buildFtsMatch(rawQuery, { matchMode: opts.matchMode });
  if (match === "") {
    // The purest silent empty in the pipeline: every token was dropped,
    // so no lookup ran at all and the caller was handed the same empty
    // list a vault with no match hands back.
    if (opts.degraded !== undefined) {
      noteDegradation(opts.degraded, RETRIEVAL_DEGRADATION.keywordFtsMatchEmpty);
    }
    return Object.freeze({ hits: [], warnings: Object.freeze([]) });
  }
  return keywordTopKWithFtsSafety(store, match, opts);
}

export function runFtsQuery(store: Store, rawQuery: string, opts: RunFtsOptions): KeywordHit[] {
  return runFtsQueryDetailed(store, rawQuery, opts).hits;
}