Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
6.1 KB
Raw Blame
/**
 * Deterministic BM25-style lexical scorer over surface descriptors
 * (Agent Surface Suite). No LLM, no embeddings - tokenized term
 * matching with field weights, reproducible across processes.
 *
 * Field weighting follows the house convention from search ranking:
 * a name-token match is the strongest relevance signal, tags sit
 * between name and description. Weights are applied as term-frequency
 * multipliers so one BM25 core serves both turn-vs-tool and
 * turn-vs-skill scoring without divergent formulas.
 */

import type { SurfaceDescriptor } from "./descriptor.ts";

/**
 * One query token a descriptor matched, with the document frequency of that
 * token over the SCORED CORPUS (not the vault). Exposed because the score
 * alone cannot say whether a match rests on a term that separates this
 * descriptor from the others or on one every descriptor carries.
 */
export interface MatchedTerm {
  readonly term: string;
  /** Descriptors in the scored corpus containing the term. Always >= 1. */
  readonly df: number;
}

export interface ScoredDescriptor {
  readonly descriptor: SurfaceDescriptor;
  readonly score: number;
  /** Query tokens this descriptor matched, in query order. Never empty. */
  readonly matched: ReadonlyArray<MatchedTerm>;
}

export interface LexicalScoreOptions {
  /** Term-frequency multiplier for name tokens. Default 3. */
  readonly nameWeight?: number;
  /** Term-frequency multiplier for tag tokens. Default 2. */
  readonly tagWeight?: number;
  /** BM25 k1 (term-frequency saturation). Default 1.2. */
  readonly k1?: number;
  /** BM25 b (length normalisation). Default 0.75. */
  readonly b?: number;
}

const DEFAULTS = Object.freeze({ nameWeight: 3, tagWeight: 2, k1: 1.2, b: 0.75 });

/**
 * Lowercase, split on non-alphanumerics, drop 1-char tokens. Plain
 * toLowerCase keeps tokenisation identical across host locales.
 * Han text without spaces (e.g. "实现方式") additionally gets overlapping
 * bigrams so queries like "gbrain的实现方式" can match trigger/description
 * tokens like "实现方式". This bigram pass is intentionally global to the
 * tokenizer (not gated behind skills_attach_triggers): ASCII tokenisation
 * is provably unchanged, and the only consumer of scoreDescriptors today
 * is the skill-attach path. The range covers Han Unified Ideographs only;
 * kana / Hangul / CJK extensions are not bigram-split.
 *
 * Bigrams are extracted only from each maximal contiguous Han span within a
 * token, so a mixed token like "gbrain..." yields the inner Han bigram but
 * never a cross-script window (e.g. an ASCII+Han pair) that would match
 * nothing yet inflate term frequency / document length. Scope is thus the
 * Han run, not the total token length: an embedded 2-char Han run still
 * emits its bigram.
 */
const HAN_SPAN = /[一-鿿]+/gu;

export function tokenize(text: string): string[] {
  const out: string[] = [];
  for (const raw of text.toLowerCase().split(/[^\p{L}\p{N}]+/u)) {
    if (raw.length < 2) continue;
    // Han bigram pass: emit overlapping 2-char windows from each maximal Han
    // span so a spaceless run and a query prefixed run share their inner
    // bigrams, without crossing into adjacent ASCII.
    for (const span of raw.match(HAN_SPAN) ?? []) {
      for (let i = 0; i < span.length - 1; i++) {
        const bigram = span.slice(i, i + 2);
        // Skip a bigram equal to the whole token (a standalone 2-char Han
        // token) so it is not emitted twice.
        if (bigram !== raw) out.push(bigram);
      }
    }
    out.push(raw);
  }
  return out;
}

interface DescriptorDoc {
  readonly descriptor: SurfaceDescriptor;
  /** token -> weighted term frequency */
  readonly tf: ReadonlyMap<string, number>;
  readonly length: number;
}

function buildDoc(
  descriptor: SurfaceDescriptor,
  opts: Required<LexicalScoreOptions>,
): DescriptorDoc {
  const tf = new Map<string, number>();
  let length = 0;
  const add = (tokens: string[], weight: number): void => {
    for (const token of tokens) {
      tf.set(token, (tf.get(token) ?? 0) + weight);
      length += weight;
    }
  };
  add(tokenize(descriptor.name), opts.nameWeight);
  add(tokenize(descriptor.tags.join(" ")), opts.tagWeight);
  add(tokenize(descriptor.description), 1);
  return { descriptor, tf, length };
}

/**
 * Rank descriptors against a free-text query. Zero-score entries are
 * dropped; ties break by descriptor name ascending so output is fully
 * deterministic.
 */
export function scoreDescriptors(
  query: string,
  descriptors: ReadonlyArray<SurfaceDescriptor>,
  options: LexicalScoreOptions = {},
): ScoredDescriptor[] {
  const opts = { ...DEFAULTS, ...options };
  const queryTokens = [...new Set(tokenize(query))];
  if (queryTokens.length === 0 || descriptors.length === 0) return [];

  const docs = descriptors.map((d) => buildDoc(d, opts));
  const n = docs.length;
  const avgLength = docs.reduce((sum, doc) => sum + doc.length, 0) / n || 1;

  // Document frequency per query token (over weighted-tf presence).
  const df = new Map<string, number>();
  for (const token of queryTokens) {
    let count = 0;
    for (const doc of docs) if (doc.tf.has(token)) count++;
    df.set(token, count);
  }

  const scored: ScoredDescriptor[] = [];
  for (const doc of docs) {
    let score = 0;
    const matched: MatchedTerm[] = [];
    for (const token of queryTokens) {
      const tf = doc.tf.get(token);
      if (tf === undefined) continue;
      const docFreq = df.get(token) ?? 0;
      matched.push({ term: token, df: docFreq });
      // BM25+ style idf floor: keeps a term present in every document
      // from zeroing out, which matters in tiny descriptor corpora.
      const idf = Math.log(1 + (n - docFreq + 0.5) / (docFreq + 0.5));
      const norm = tf + opts.k1 * (1 - opts.b + opts.b * (doc.length / avgLength));
      score += idf * ((tf * (opts.k1 + 1)) / norm);
    }
    if (score > 0) {
      scored.push({ descriptor: doc.descriptor, score, matched: Object.freeze(matched) });
    }
  }

  scored.sort(
    (a, b) =>
      b.score - a.score ||
      (a.descriptor.name < b.descriptor.name ? -1 : a.descriptor.name > b.descriptor.name ? 1 : 0),
  );
  return scored;
}