Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
3.3 KB
Raw Blame
/**
 * Pure, language-agnostic query expansion via local co-occurrence
 * (pseudo-relevance feedback), v0.20.0.
 *
 * Goal: broaden recall when the caller's wording differs from the stored
 * wording, WITHOUT any per-language synonym dictionary. The signal is
 * derived entirely from the vault's own content: given the top candidate
 * chunks for the original query, the terms that co-occur across several
 * of those chunks (but are not themselves in the query) are likely
 * related and worth adding as optional FTS alternatives.
 *
 * Tokenization is purely structural - maximal Unicode letter/digit runs,
 * lowercased. There is deliberately NO language-specific segmentation
 * (a continuous CJK run stays one token, exactly as FTS5 indexes it), so
 * the behaviour is identical across scripts and bit-stable on every
 * Syncthing peer.
 *
 * The module is pure and deterministic: no I/O, no clock, no randomness.
 * Ties break alphabetically so the output never depends on input order
 * beyond the documented frequency ranking.
 */

export interface ExpansionOptions {
  /** Maximum number of expansion terms to return. */
  readonly maxTerms: number;
  /** Minimum token length (in code points) to be eligible. */
  readonly minLength: number;
  /** Minimum number of distinct candidate docs a term must appear in. */
  readonly minDocFreq: number;
}

export const DEFAULT_EXPANSION: ExpansionOptions = Object.freeze({
  maxTerms: 3,
  minLength: 3,
  minDocFreq: 2,
});

const TOKEN_RE = /[\p{L}\p{N}]+/gu;

/** Maximal Unicode letter/digit runs, lowercased. No language rules. */
export function tokenizeForExpansion(text: string): string[] {
  const out: string[] = [];
  for (const m of text.matchAll(TOKEN_RE)) {
    out.push(m[0].toLowerCase());
  }
  return out;
}

/** Code-point length, so multi-byte scripts are measured fairly. */
function codePointLength(s: string): number {
  let n = 0;
  for (const _ of s) n++;
  return n;
}

/**
 * Derive expansion terms from the texts of the top candidate chunks for
 * a query. Returns up to `maxTerms` terms that (a) are not query tokens,
 * (b) are at least `minLength` code points, and (c) appear in at least
 * `minDocFreq` distinct candidate docs - ranked by document frequency
 * descending, then alphabetically. Empty input or no qualifying term
 * yields `[]` (a no-op for the caller).
 */
export function deriveExpansionTerms(
  queryTokens: ReadonlyArray<string>,
  candidateTexts: ReadonlyArray<string>,
  opts: ExpansionOptions,
): string[] {
  if (candidateTexts.length === 0 || opts.maxTerms <= 0) return [];

  const queryset = new Set(queryTokens.map((t) => t.toLowerCase()));
  const docFreq = new Map<string, number>();

  for (const text of candidateTexts) {
    // Count each term once per document (document frequency).
    const seen = new Set<string>();
    for (const tok of tokenizeForExpansion(text)) {
      if (seen.has(tok)) continue;
      seen.add(tok);
      if (queryset.has(tok)) continue;
      if (codePointLength(tok) < opts.minLength) continue;
      docFreq.set(tok, (docFreq.get(tok) ?? 0) + 1);
    }
  }

  return [...docFreq.entries()]
    .filter(([, df]) => df >= opts.minDocFreq)
    .toSorted((a, b) => (b[1] !== a[1] ? b[1] - a[1] : a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))
    .slice(0, opts.maxTerms)
    .map(([term]) => term);
}