Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Per-store reranker evaluation gate (Retrieval & Ranking Quality,
* t_9f95ebb6).
*
* Before a store commits to reranking, this gate MEASURES whether the
* reranker actually improves recall/ranking quality on that store's own
* labelled queries. It runs the existing recall-benchmark twice - rerank
* OFF then ON - over the same dataset and compares the metric families
* (hit@k, MRR), pooled and per language. Reranking is recommended only
* when it lifts ranking without regressing hit@k in the pooled numbers
* OR in any single language (t_693fa22a review, M14), so a store where
* the reranker helps one language and hurts another never silently
* enables on the pooled delta.
*
* Reuses `runRecallBenchmark` for the metrics rather than a bespoke scorer,
* and is deterministic and offline when the store and reranker are (the
* bundled "local" reranker always is).
*/
import { runRecallBenchmark } from "./benchmark.ts";
import type { RecallBenchmarkDataset, RecallBenchmarkReport } from "./benchmark.ts";
import { MAINTENANCE_LANE_REACH } from "../graph/transport-reach.ts";
import type { ResolvedDecisionModelConfig } from "../decision-model/config.ts";
import { SearchError, type ResolvedRerankConfig, type ResolvedSearchConfig } from "./types.ts";
export interface RerankEvalGateOptions {
/** Rank depth for the benchmark (defaults to the benchmark default). */
readonly k?: number;
/** Reranker kind to evaluate (defaults to "local", the offline reranker). */
readonly kind?: ResolvedRerankConfig["kind"];
/** Minimum pooled MRR lift required to recommend enabling. Default 0.01. */
readonly minMrrDelta?: number;
/** Minimum pooled hit@k lift that alone justifies enabling. Default 0.01. */
readonly minHitDelta?: number;
/**
* The decision-model config for `kind: "decision-model"`, when the
* search config was not resolved with that kind. Must be active; the
* reranked arm always runs the `rerank` use in `enforce`, since a shadow
* arm would return the heuristic order and measure nothing.
*/
readonly decisionModel?: ResolvedDecisionModelConfig;
}
/** The rerank-arm minus baseline difference of one metric family. */
export interface RerankDelta {
readonly hitAtK: number;
readonly mrr: number;
}
export interface RerankEvalGateResult {
/** Whether reranking is recommended for this store. */
readonly improves: boolean;
readonly recommendation: "enable" | "keep-disabled";
readonly baseline: RecallBenchmarkReport;
readonly reranked: RecallBenchmarkReport;
/**
* Rerank-arm minus baseline, combined over the whole run and (t_693fa22a
* review, M14) per language: `byLang` carries one entry per `byLang`
* group of the two arms, so a reranker that helps one language and hurts
* another is visible per language instead of washing out in the pooled
* numbers. Same keys as `baseline.byLang`, sorted.
*/
readonly deltas: RerankEvalDeltas;
}
export interface RerankEvalDeltas {
/** The pooled delta over every query. */
readonly hitAtK: number;
readonly mrr: number;
/** The same difference folded per language group. */
readonly byLang: Readonly<Record<string, RerankDelta>>;
}
/**
* Slack for float noise between two folds of the same orders: identical
* rankings fold to bit-identical fractions, and a real demotion moves a
* fraction by far more than this, so only genuine dust lands inside it.
*/
const VERDICT_FP_NOISE = 1e-9;
/**
* The gate's verdict rule, pure so a constructed pair of reports pins it
* unit-wise (t_693fa22a review, M14). Enables reranking only when the
* POOLED numbers lift - hit@k does not regress and either MRR or hit@k
* gains at least the requested margin - AND no single language regresses
* on either family: a reranker that helps EN and hurts ZH must not be
* recommended on the pooled delta. The per-language check subsumes the
* pooled hit@k floor (a pooled regression is always some language's
* regression), and both stay so the rule reads as the contract it keeps.
*/
export function rerankEvalVerdict(
baseline: RecallBenchmarkReport,
reranked: RecallBenchmarkReport,
minMrrDelta: number,
minHitDelta: number,
): {
improves: boolean;
recommendation: "enable" | "keep-disabled";
deltas: RerankEvalDeltas;
} {
const hitDelta = reranked.hitAtK - baseline.hitAtK;
const mrrDelta = reranked.mrr - baseline.mrr;
// The per-language fold of the same difference (M14): one entry per
// language group of the baseline, so a verdict consumer sees which
// language moved which way. The keys are the baseline's, and both arms
// ran the same dataset, so the groups always pair up.
const byLang: Record<string, RerankDelta> = {};
let languageRegressed = false;
for (const [lang, before] of Object.entries(baseline.byLang)) {
const after = reranked.byLang[lang];
const hitAtK = (after?.hitAtK ?? before.hitAtK) - before.hitAtK;
const mrr = (after?.mrr ?? before.mrr) - before.mrr;
byLang[lang] = Object.freeze({ hitAtK, mrr });
if (hitAtK < -VERDICT_FP_NOISE || mrr < -VERDICT_FP_NOISE) languageRegressed = true;
}
// Never recommend a reranker that drops hit@k pooled or in any one
// language; among non-regressing candidates, require a real pooled MRR
// or hit@k lift.
const improves =
!languageRegressed && hitDelta >= 0 && (mrrDelta >= minMrrDelta || hitDelta >= minHitDelta);
return {
improves,
recommendation: improves ? "enable" : "keep-disabled",
deltas: Object.freeze({ hitAtK: hitDelta, mrr: mrrDelta, byLang: Object.freeze(byLang) }),
};
}
function withRerank(
config: ResolvedSearchConfig,
enabled: boolean,
kind: ResolvedRerankConfig["kind"],
decisionModel?: ResolvedDecisionModelConfig,
): ResolvedSearchConfig {
return Object.freeze({
...config,
rerank: Object.freeze({
...config.rerank,
enabled,
kind,
...(decisionModel !== undefined
? {
decisionModel: Object.freeze({
...decisionModel,
uses: Object.freeze({ ...decisionModel.uses, rerank: "enforce" as const }),
// Eval requests are real spend (the gate counts them) but not
// shadow data: the report keeps them out of the agreement.
recordOrigin: "eval" as const,
}),
}
: {}),
}),
});
}
/** The active decision config the reranked arm needs, or a typed refusal. */
function decisionModelFor(
config: ResolvedSearchConfig,
opts: RerankEvalGateOptions,
): ResolvedDecisionModelConfig {
const dm = opts.decisionModel ?? config.rerank.decisionModel;
if (dm === undefined || dm.status !== "active") {
throw new SearchError(
"INVALID_INPUT",
`the decision-model reranker is not active (${dm?.status ?? "not configured"}); ` +
"run `o2b decision-model check` for what is missing",
);
}
return dm;
}
/**
* Evaluate whether reranking helps this store. Runs the benchmark with
* rerank off and on and compares through {@link rerankEvalVerdict} - the
* pooled lift with the no-regression rule, pooled and per language.
*/
export async function runRerankEvalGate(
config: ResolvedSearchConfig,
dataset: RecallBenchmarkDataset,
opts: RerankEvalGateOptions = {},
): Promise<RerankEvalGateResult> {
const kind = opts.kind ?? "local";
const minMrrDelta = opts.minMrrDelta ?? 0.01;
const minHitDelta = opts.minHitDelta ?? 0.01;
const k = opts.k;
// An internal gate with no caller: it scores the reranker against the
// whole corpus, and a run that stopped seeing reserved pages would
// compare the two arms over different vaults. Named through the shared
// constant so the claim is the same claim every lane makes.
const reach = MAINTENANCE_LANE_REACH;
const decisionModel = kind === "decision-model" ? decisionModelFor(config, opts) : undefined;
const baseline = await runRecallBenchmark(withRerank(config, false, kind), dataset, {
k,
transportReach: reach,
});
const reranked = await runRecallBenchmark(
withRerank(config, true, kind, decisionModel),
dataset,
{
k,
transportReach: reach,
},
);
const { improves, recommendation, deltas } = rerankEvalVerdict(
baseline,
reranked,
minMrrDelta,
minHitDelta,
);
return Object.freeze({
improves,
recommendation,
baseline,
reranked,
deltas,
});
}