Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Deterministic BM25-style lexical scorer over surface descriptors
* (Agent Surface Suite). No LLM, no embeddings - tokenized term
* matching with field weights, reproducible across processes.
*
* Field weighting follows the house convention from search ranking:
* a name-token match is the strongest relevance signal, tags sit
* between name and description. Weights are applied as term-frequency
* multipliers so one BM25 core serves both turn-vs-tool and
* turn-vs-skill scoring without divergent formulas.
*/
import type { SurfaceDescriptor } from "./descriptor.ts";
/**
* One query token a descriptor matched, with the document frequency of that
* token over the SCORED CORPUS (not the vault). Exposed because the score
* alone cannot say whether a match rests on a term that separates this
* descriptor from the others or on one every descriptor carries.
*/
export interface MatchedTerm {
readonly term: string;
/** Descriptors in the scored corpus containing the term. Always >= 1. */
readonly df: number;
}
export interface ScoredDescriptor {
readonly descriptor: SurfaceDescriptor;
readonly score: number;
/** Query tokens this descriptor matched, in query order. Never empty. */
readonly matched: ReadonlyArray<MatchedTerm>;
}
export interface LexicalScoreOptions {
/** Term-frequency multiplier for name tokens. Default 3. */
readonly nameWeight?: number;
/** Term-frequency multiplier for tag tokens. Default 2. */
readonly tagWeight?: number;
/** BM25 k1 (term-frequency saturation). Default 1.2. */
readonly k1?: number;
/** BM25 b (length normalisation). Default 0.75. */
readonly b?: number;
}
const DEFAULTS = Object.freeze({ nameWeight: 3, tagWeight: 2, k1: 1.2, b: 0.75 });
/**
* Lowercase, split on non-alphanumerics, drop 1-char tokens. Plain
* toLowerCase keeps tokenisation identical across host locales.
* Han text without spaces (e.g. "实现方式") additionally gets overlapping
* bigrams so queries like "gbrain的实现方式" can match trigger/description
* tokens like "实现方式". This bigram pass is intentionally global to the
* tokenizer (not gated behind skills_attach_triggers): ASCII tokenisation
* is provably unchanged, and the only consumer of scoreDescriptors today
* is the skill-attach path. The range covers Han Unified Ideographs only;
* kana / Hangul / CJK extensions are not bigram-split.
*
* Bigrams are extracted only from each maximal contiguous Han span within a
* token, so a mixed token like "gbrain..." yields the inner Han bigram but
* never a cross-script window (e.g. an ASCII+Han pair) that would match
* nothing yet inflate term frequency / document length. Scope is thus the
* Han run, not the total token length: an embedded 2-char Han run still
* emits its bigram.
*/
const HAN_SPAN = /[一-鿿]+/gu;
export function tokenize(text: string): string[] {
const out: string[] = [];
for (const raw of text.toLowerCase().split(/[^\p{L}\p{N}]+/u)) {
if (raw.length < 2) continue;
// Han bigram pass: emit overlapping 2-char windows from each maximal Han
// span so a spaceless run and a query prefixed run share their inner
// bigrams, without crossing into adjacent ASCII.
for (const span of raw.match(HAN_SPAN) ?? []) {
for (let i = 0; i < span.length - 1; i++) {
const bigram = span.slice(i, i + 2);
// Skip a bigram equal to the whole token (a standalone 2-char Han
// token) so it is not emitted twice.
if (bigram !== raw) out.push(bigram);
}
}
out.push(raw);
}
return out;
}
interface DescriptorDoc {
readonly descriptor: SurfaceDescriptor;
/** token -> weighted term frequency */
readonly tf: ReadonlyMap<string, number>;
readonly length: number;
}
function buildDoc(
descriptor: SurfaceDescriptor,
opts: Required<LexicalScoreOptions>,
): DescriptorDoc {
const tf = new Map<string, number>();
let length = 0;
const add = (tokens: string[], weight: number): void => {
for (const token of tokens) {
tf.set(token, (tf.get(token) ?? 0) + weight);
length += weight;
}
};
add(tokenize(descriptor.name), opts.nameWeight);
add(tokenize(descriptor.tags.join(" ")), opts.tagWeight);
add(tokenize(descriptor.description), 1);
return { descriptor, tf, length };
}
/**
* Rank descriptors against a free-text query. Zero-score entries are
* dropped; ties break by descriptor name ascending so output is fully
* deterministic.
*/
export function scoreDescriptors(
query: string,
descriptors: ReadonlyArray<SurfaceDescriptor>,
options: LexicalScoreOptions = {},
): ScoredDescriptor[] {
const opts = { ...DEFAULTS, ...options };
const queryTokens = [...new Set(tokenize(query))];
if (queryTokens.length === 0 || descriptors.length === 0) return [];
const docs = descriptors.map((d) => buildDoc(d, opts));
const n = docs.length;
const avgLength = docs.reduce((sum, doc) => sum + doc.length, 0) / n || 1;
// Document frequency per query token (over weighted-tf presence).
const df = new Map<string, number>();
for (const token of queryTokens) {
let count = 0;
for (const doc of docs) if (doc.tf.has(token)) count++;
df.set(token, count);
}
const scored: ScoredDescriptor[] = [];
for (const doc of docs) {
let score = 0;
const matched: MatchedTerm[] = [];
for (const token of queryTokens) {
const tf = doc.tf.get(token);
if (tf === undefined) continue;
const docFreq = df.get(token) ?? 0;
matched.push({ term: token, df: docFreq });
// BM25+ style idf floor: keeps a term present in every document
// from zeroing out, which matters in tiny descriptor corpora.
const idf = Math.log(1 + (n - docFreq + 0.5) / (docFreq + 0.5));
const norm = tf + opts.k1 * (1 - opts.b + opts.b * (doc.length / avgLength));
score += idf * ((tf * (opts.k1 + 1)) / norm);
}
if (score > 0) {
scored.push({ descriptor: doc.descriptor, score, matched: Object.freeze(matched) });
}
}
scored.sort(
(a, b) =>
b.score - a.score ||
(a.descriptor.name < b.descriptor.name ? -1 : a.descriptor.name > b.descriptor.name ? 1 : 0),
);
return scored;
}