Local-first π§ memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly π΄ dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Coverage engine (recall-trust-suite, Features C and E).
*
* The single source of truth for query-term verification: significant
* terms, per-term coverage postings, corpus document frequency turned
* into IDF weight, and the rare-term classification. Both the verified
* multi-record recall pass (Feature C: union fetch, IDF-weighted
* support, rare-term abstention) and the search-completeness guard
* (Feature E) read this one report, so the two can never disagree about
* what "covered" means.
*
* Pure module β callers (search.ts / evidence-pack.ts) gather document
* counts and per-term document frequencies from the store and hand them
* in. Deterministic, no LLM, no clock.
*/
import type { CompletenessReport, CompletenessVerdict } from "./types.ts";
/** Share of the corpus a term may appear in and still count as rare. */
export const RARE_TERM_CORPUS_SHARE = 0.02;
/**
* Significant query terms: every word-character run, deduplicated, in
* query order.
*
* Language-agnostic by construction: there is deliberately NO stopword
* list. A per-language stopword set (the old English-only one) would
* under-filter every other language while pretending to help. Instead,
* corpus-common terms are handled downstream by the IDF weighting in
* {@link buildCoverageReport} β a term that appears in most documents
* earns near-zero IDF and contributes almost nothing to the weighted
* coverage, in any language, without a vocabulary list.
*
* ## Why there is no length floor any more
*
* There used to be one: a token had to be three characters. It was a
* stopword list in disguise, written for a script whose function words
* are short and whose content words are not, and it did not survive
* contact with a script where the common word length IS one or two
* characters. `ζ€η΄’` ("search") is two characters and was dropped whole;
* so was every one- and two-character Chinese or Japanese query, which
* left the report with nothing to weigh and β before this changed β
* scoring a perfect 1.0 for a retrieval that matched nothing.
*
* A character count cannot be made script-neutral: the same count means
* "a function word" in one writing system and "a full noun" in another.
* So the count is gone rather than tuned, and the job it was pretending
* to do is left where it was always actually done, in the IDF weight. A
* one-character term that appears in most documents earns near-zero IDF
* in exactly the same way a three-character one does; a one-character
* term that appears in two documents is a discriminating term and is now
* treated as one.
*
* The visible cost is on the preview surfaces that share this splitter:
* `matchOffset` can now anchor a snippet on a one-character common word
* where it previously anchored on a three-character one. That cost is
* the same one its docblock already accepted, one character smaller.
*/
export function significantTerms(query: string): string[] {
const terms = new Set<string>();
for (const token of query.toLocaleLowerCase().split(/[^\p{L}\p{N}_-]+/u)) {
// At least one letter or digit. The splitter keeps `-` and `_` inside
// a token so `well-known` and `snake_case` survive whole, which means
// a run of pure punctuation (`--`) also survives the split; it is not
// a word in any language, and with the length floor gone nothing else
// would have stopped it from earning IDF mass.
if (HAS_WORD_CHARACTER.test(token)) terms.add(token);
}
return [...terms];
}
/** A token counts as a word once it carries one letter or digit. */
const HAS_WORD_CHARACTER = /[\p{L}\p{N}]/u;
/** Case-folded containment check shared by pack record building. */
export function termIncludedIn(haystack: string, term: string): boolean {
return haystack.toLocaleLowerCase().includes(term);
}
/**
* Smoothed inverse document frequency: `ln(1 + N / (1 + df))`. Always
* positive (even a term in every document keeps a small weight), higher
* for rarer terms, and stable for df = 0.
*/
export function idfForTerm(df: number, documentCount: number): number {
const n = Math.max(0, documentCount);
const d = Math.max(0, df);
return Math.log(1 + n / (1 + d));
}
/**
* A term is rare (high-signal) when it appears in at most `corpusShare` of
* the corpus, with a floor of one document so tiny corpora still classify
* their unique terms as rare.
*
* `corpusShare` defaults to {@link RARE_TERM_CORPUS_SHARE}, the vault-corpus
* value this function was written for. It is a parameter because the same
* classification - corpus document frequency against a share of the corpus,
* with no vocabulary list anywhere in it - is what separates a
* discriminating term from a corpus-common one in the far smaller skill
* descriptor corpus (t_ccb05134). The share differs by corpus; the shape
* must not, or the two would drift into disagreeing about what "common"
* means.
*/
export function isRareTerm(
df: number,
documentCount: number,
corpusShare: number = RARE_TERM_CORPUS_SHARE,
): boolean {
return df <= Math.max(1, Math.floor(corpusShare * documentCount));
}
export interface TermCoverage {
readonly term: string;
readonly df: number;
readonly idf: number;
readonly rare: boolean;
readonly covered: boolean;
}
export interface CoverageInputs {
readonly significantTerms: ReadonlyArray<string>;
/** Terms at least one returned result contains. */
readonly coveredTerms: ReadonlySet<string>;
readonly documentCount: number;
/** Corpus document frequency per significant term (absent β 0). */
readonly dfByTerm: ReadonlyMap<string, number>;
}
export interface CoverageReport {
readonly terms: ReadonlyArray<TermCoverage>;
/**
* Support coverage weighted by IDF: the share of the query's total
* IDF mass the covered terms carry. A result set matching only the
* common words scores low even when it matches most terms by count.
*
* `null` when the query carries no IDF mass to weigh at all β see
* {@link buildCoverageReport}. NOT a number, deliberately: the previous
* shape reported that case as `1`, so a query nothing could be measured
* about was indistinguishable from a complete retrieval, and every
* threshold reading it fired at maximum confidence. A nullable field
* makes the type system ask each consumer what it wants to do about an
* unmeasurable query, which is the same question the code was silently
* answering "perfect" to.
*/
readonly idfWeightedCoverage: number | null;
readonly rareTerms: ReadonlyArray<string>;
readonly uncoveredRareTerms: ReadonlyArray<string>;
}
/** IDF-weighted coverage at/above this is a complete retrieval. */
export const COMPLETENESS_COMPLETE_THRESHOLD = 0.8;
/** IDF-weighted coverage at/above this (below complete) is partial. */
export const COMPLETENESS_PARTIAL_THRESHOLD = 0.4;
/**
* The three-way verdict, plus the fourth state that is not a verdict.
*
* An unmeasurable coverage (`null`) becomes `unmeasurable` rather than
* being folded into `sparse`: "the retrieval missed" and "there was
* nothing here to weigh" call for different repairs, and reporting the
* second as the first would tell a reader the corpus is thin when the
* measurement never ran.
*/
export function buildCompletenessReport(coverage: CoverageReport): CompletenessReport {
const covered = coverage.terms.filter((t) => t.covered).map((t) => t.term);
const uncovered = coverage.terms.filter((t) => !t.covered);
const weighted = coverage.idfWeightedCoverage;
const verdict: CompletenessVerdict =
weighted === null
? "unmeasurable"
: weighted >= COMPLETENESS_COMPLETE_THRESHOLD
? "complete"
: weighted >= COMPLETENESS_PARTIAL_THRESHOLD
? "partial"
: "sparse";
return Object.freeze({
verdict,
idfWeightedCoverage: coverage.idfWeightedCoverage,
coveredTerms: Object.freeze(covered),
uncoveredTerms: Object.freeze(uncovered.map((t) => t.term)),
uncoveredButPresentInCorpus: Object.freeze(
uncovered.filter((t) => t.df > 0).map((t) => t.term),
),
});
}
/**
* Targeted self-correcting retry plan (t_8eb5ca32): the deterministic
* decision that connects coverage to a follow-up retrieval. A retry
* fires only when the IDF-weighted coverage is below the completeness
* threshold AND at least one RARE significant term is still uncovered;
* the follow-up is then aimed at exactly those uncovered rare terms β
* the specifically-missing high-signal facts β never a generic
* broadening of the whole query. The rare gate keeps the retry off when
* only corpus-common terms are missing (low IDF, low value) and bounds
* how often it can fire. `terms` is empty iff `fire` is false.
*
* Pure and deterministic: a verdict over an already-built report, no
* I/O. The caller decides how to turn the terms into a query (FTS OR,
* expansion, β¦) and how to cap the number of passes.
*/
export interface TargetedRetryPlan {
readonly fire: boolean;
readonly terms: ReadonlyArray<string>;
}
export function planTargetedRetry(coverage: CoverageReport): TargetedRetryPlan {
// An unmeasurable coverage does not fire the retry. Stated rather than
// left to the empty rare-term list that would also stop it: a retry
// aimed at "the specifically-missing high-signal facts" needs a
// measurement naming them, and there is none.
const belowThreshold =
coverage.idfWeightedCoverage !== null &&
coverage.idfWeightedCoverage < COMPLETENESS_COMPLETE_THRESHOLD;
const fire = belowThreshold && coverage.uncoveredRareTerms.length > 0;
return Object.freeze({
fire,
terms: fire ? coverage.uncoveredRareTerms : Object.freeze([] as string[]),
});
}
/**
* One follow-up sub-query: the terms to fetch in a single pull, and the
* group's significance (the summed IDF of its terms) β the ordering key
* that decides which gaps are worth a dedicated pull first.
*/
export interface SubQueryGroup {
readonly terms: ReadonlyArray<string>;
readonly significance: number;
}
/** The k-group generalized gap signal's verdict. `groups` is empty iff `fire` is false. */
export interface SubQueryGroupPlan {
readonly fire: boolean;
readonly groups: ReadonlyArray<SubQueryGroup>;
}
/** Most sub-query groups one gap-proposal round may emit. */
export const SUB_QUERY_GROUPS_MAX = 4;
/**
* The k-group generalization of {@link planTargetedRetry} (recall-quality
* wave): the same fire gate β measurable coverage below the completeness
* threshold with rare significant terms still uncovered β but the gaps
* become up to `maxGroups` follow-up sub-queries instead of one merged
* term list. The most significant uncovered term (highest IDF, query
* order breaking ties) earns a dedicated singleton group; any residual
* terms merge into one final group in query order, which is exactly the
* shape the round-two retry has always fetched. Groups are returned
* ordered by significance, descending, so a caller that can afford fewer
* pulls drops the tail, never the head.
*
* `maxGroups` below one is clamped to one (the single merged group), and
* an unmeasurable coverage yields zero groups for the same reason
* {@link planTargetedRetry} states above. A non-finite coverage fails
* closed the same way (fire false, no groups): a broken measurement
* proposes nothing. Pure and deterministic.
*/
export function planSubQueryGroups(coverage: CoverageReport, maxGroups: number): SubQueryGroupPlan {
// Refuse a non-finite report explicitly rather than let comparison
// semantics decide: NaN misses the `<` gate by accident, but -Infinity
// passes it and would fire on a measurement that never happened.
if (coverage.idfWeightedCoverage !== null && !Number.isFinite(coverage.idfWeightedCoverage)) {
return Object.freeze({ fire: false, groups: Object.freeze([]) });
}
const legacy = planTargetedRetry(coverage);
if (!legacy.fire) return Object.freeze({ fire: false, groups: Object.freeze([]) });
const idfByTerm = new Map(coverage.terms.map((t) => [t.term, t.idf]));
const uncovered = coverage.uncoveredRareTerms;
const bounded = Math.max(1, Math.floor(maxGroups));
// Query order preserved: `uncoveredRareTerms` follows the query's term
// order, and the stable sort below only re-orders groups, never members.
const ranked = uncovered
.map((term, index) => ({ term, idf: idfByTerm.get(term) ?? 0, index }))
.toSorted((a, b) => (a.idf !== b.idf ? b.idf - a.idf : a.index - b.index));
const groups: SubQueryGroup[] =
bounded === 1
? // The historical round-two shape byte-identically: ONE merged
// group carrying every uncovered rare term in query order.
[residualGroup(uncovered, uncovered, idfByTerm)]
: uncovered.length <= bounded
? ranked.map((entry) => ({
terms: Object.freeze([entry.term]),
significance: entry.idf,
}))
: [
...ranked.slice(0, bounded - 1).map((entry) => ({
terms: Object.freeze([entry.term]),
significance: entry.idf,
})),
residualGroup(
uncovered,
ranked.slice(bounded - 1).map((entry) => entry.term),
idfByTerm,
),
];
groups.sort(
(a, b) =>
b.significance - a.significance || rankOf(a.terms[0]!, ranked) - rankOf(b.terms[0]!, ranked),
);
return Object.freeze({ fire: true, groups: Object.freeze(groups) });
}
/** Summed-IDF group over the residual terms, kept in query order. */
function residualGroup(
queryOrder: ReadonlyArray<string>,
residual: ReadonlyArray<string>,
idfByTerm: ReadonlyMap<string, number>,
): SubQueryGroup {
const residualSet = new Set(residual);
const terms = queryOrder.filter((term) => residualSet.has(term));
let significance = 0;
for (const term of terms) significance += idfByTerm.get(term) ?? 0;
return { terms: Object.freeze(terms), significance };
}
function rankOf(term: string, ranked: ReadonlyArray<{ term: string }>): number {
return ranked.findIndex((entry) => entry.term === term);
}
/**
* The report, and the one condition under which there is no number.
*
* `idfWeightedCoverage` is a SHARE of the query's IDF mass, so it exists
* only when there is mass to take a share of. Two inputs leave none: a
* query with no word characters at all, and a corpus with no documents
* (every `idfForTerm` is then `ln(1) = 0`). Both are reported as `null`,
* because neither is a statement about how well the retrieval did.
*/
export function buildCoverageReport(inputs: CoverageInputs): CoverageReport {
const terms: TermCoverage[] = inputs.significantTerms.map((term) => {
const df = inputs.dfByTerm.get(term) ?? 0;
return Object.freeze({
term,
df,
idf: idfForTerm(df, inputs.documentCount),
rare: isRareTerm(df, inputs.documentCount),
covered: inputs.coveredTerms.has(term),
});
});
let totalIdf = 0;
let coveredIdf = 0;
for (const t of terms) {
totalIdf += t.idf;
if (t.covered) coveredIdf += t.idf;
}
const rareTerms = terms.filter((t) => t.rare).map((t) => t.term);
const uncoveredRareTerms = terms.filter((t) => t.rare && !t.covered).map((t) => t.term);
return Object.freeze({
terms: Object.freeze(terms),
// No IDF mass means there is nothing to take a share OF, so there is
// no coverage β not full coverage. `null`, and every consumer decides.
idfWeightedCoverage: totalIdf === 0 ? null : coveredIdf / totalIdf,
rareTerms: Object.freeze(rareTerms),
uncoveredRareTerms: Object.freeze(uncoveredRareTerms),
});
}