Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* `o2b search rerank-eval` - the operator entry point for the rerank eval
* gate (issue #213, Part 2).
*
* Runs the recall benchmark over a labelled dataset twice, rerank OFF and
* ON with the chosen kind, and reports hit@k and MRR for both plus the
* deltas and the gate's recommendation. `--compare-local` adds the bundled
* offline reranker as a third arm. For `--kind decision-model` the ON arm
* always runs the `rerank` use in enforce (a shadow arm would measure
* nothing), and it needs an active decision-model config: every query
* sends its top candidates to the configured endpoint.
*
* Read-only for the vault's notes; with the decision-model kind it appends
* one `decision_model_call` record per query, as any rerank would, tagged
* `origin: "eval"`. Beside the gate's recommendation it reports hit@1/3/5/10
* (up to k), per-query wins, losses and ties against off, the deltas per
* language (t_693fa22a review, M14: each `by_lang` group of a reranked arm
* carries `delta_hit_at_k` / `delta_mrr`, so a reranker that helps one
* language and hurts another does not hide in the pooled numbers), and for
* the decision-model kind the run's records by outcome, cost and latency, so
* an arm whose every call failed (and so equals off) is visible as such.
*/
import { readFileSync } from "node:fs";
import { discoverConfig } from "../../../core/config.ts";
import { resolveDecisionModelConfig } from "../../../core/decision-model/config.ts";
import {
buildDecisionModelReport,
type DecisionModelUseSummary,
} from "../../../core/decision-model/diagnostics.ts";
import type {
RecallBenchmarkQueryResult,
RecallBenchmarkReport,
} from "../../../core/search/benchmark.ts";
import { parseRecallBenchmarkDataset } from "../../../core/search/benchmark.ts";
import {
runRerankEvalGate,
type RerankDelta,
type RerankEvalGateResult,
} from "../../../core/search/rerank-eval-gate.ts";
import type { ResolvedRerankConfig } from "../../../core/search/types.ts";
import {
CliError,
flagBoolean,
flagString,
isIntegerWithin,
parseFlags,
resolveConfig,
resolveConfigPath,
VAULT_FLAGS,
requireSearchVault,
} from "../helpers.ts";
const KINDS: ReadonlyArray<ResolvedRerankConfig["kind"]> = [
"local",
"decision-model",
"openai-compat",
];
function isKind(value: string): value is ResolvedRerankConfig["kind"] {
return (KINDS as ReadonlyArray<string>).includes(value);
}
const HIT_DEPTHS = [1, 3, 5, 10] as const;
/** hit@n for each depth the run measured (n <= k). */
function hitsAtRows(
rows: ReadonlyArray<RecallBenchmarkQueryResult>,
k: number,
): Record<string, number> {
const out: Record<string, number> = {};
const total = Math.max(1, rows.length);
for (const n of HIT_DEPTHS) {
if (n > k) continue;
out[`hit_at_${n}`] = rows.filter((q) => q.rank !== null && q.rank <= n).length / total;
}
return out;
}
function hitsAt(report: RecallBenchmarkReport): Record<string, number> {
return hitsAtRows(report.perQuery, report.k);
}
/**
* The per-language split of one arm (t_693fa22a): the arm's `byLang`
* aggregates with the hit@n depths recomputed over the group's rows.
* When the caller passes the gate's per-language deltas (M14), each
* group also carries `delta_hit_at_k` / `delta_mrr` - that group's
* rerank-arm minus baseline difference - so a reranker that helps one
* language and hurts another is readable per language. Only a mixed
* dataset produces more than one key; a single-language run carries one
* group equal to the arm's own numbers.
*/
function byLangOf(
report: RecallBenchmarkReport,
deltas?: Readonly<Record<string, RerankDelta>>,
): Record<string, unknown> {
const out: Record<string, unknown> = {};
for (const [lang, agg] of Object.entries(report.byLang)) {
const rows = report.perQuery.filter((q) => q.lang === lang);
const delta = deltas?.[lang];
out[lang] = {
total: agg.total,
hit_at_k: agg.hitAtK,
mrr: agg.mrr,
...hitsAtRows(rows, report.k),
...(delta !== undefined ? { delta_hit_at_k: delta.hitAtK, delta_mrr: delta.mrr } : {}),
};
}
return out;
}
/** One line per language, sorted by tag, shown when a dataset is mixed. */
function renderByLang(
report: RecallBenchmarkReport,
deltas?: Readonly<Record<string, RerankDelta>>,
): string {
const langs = Object.keys(report.byLang);
if (langs.length <= 1) return "";
return (
"\n" +
langs
.map((lang) => {
const rows = report.perQuery.filter((q) => q.lang === lang);
const delta = deltas?.[lang];
return (
` ${lang}: ${report.byLang[lang]!.total} query(ies) ` +
`${renderHitsRows(rows, report.k)} MRR ${fmt(report.byLang[lang]!.mrr)}` +
(delta !== undefined
? ` (delta hit@k ${signed(delta.hitAtK)}, MRR ${signed(delta.mrr)})`
: "")
);
})
.join("\n")
);
}
/** Per-query comparison against off, by reciprocal rank. */
function versusOff(
baseline: RecallBenchmarkReport,
reranked: RecallBenchmarkReport,
): { wins: number; losses: number; ties: number } {
const base = new Map(baseline.perQuery.map((q) => [q.id, q.reciprocalRank]));
let wins = 0;
let losses = 0;
let ties = 0;
for (const q of reranked.perQuery) {
const before = base.get(q.id) ?? 0;
if (q.reciprocalRank > before) wins++;
else if (q.reciprocalRank < before) losses++;
else ties++;
}
return { wins, losses, ties };
}
function arm(result: RerankEvalGateResult): Record<string, unknown> {
return {
hit_at_k: result.reranked.hitAtK,
mrr: result.reranked.mrr,
...hitsAt(result.reranked),
delta_hit_at_k: result.deltas.hitAtK,
delta_mrr: result.deltas.mrr,
versus_off: versusOff(result.baseline, result.reranked),
recommendation: result.recommendation,
by_lang: byLangOf(result.reranked, result.deltas.byLang),
};
}
function callsOf(summary: DecisionModelUseSummary | undefined): Record<string, unknown> {
return {
total: summary?.calls ?? 0,
outcomes: summary?.outcomes ?? {},
cost_usd: summary?.cost_usd ?? 0,
unknown_cost_calls: summary?.unknown_cost_calls ?? 0,
latency_p50_ms: summary?.latency_p50_ms ?? null,
latency_p95_ms: summary?.latency_p95_ms ?? null,
};
}
function renderHits(report: RecallBenchmarkReport): string {
return renderHitsRows(report.perQuery, report.k);
}
function renderHitsRows(rows: ReadonlyArray<RecallBenchmarkQueryResult>, k: number): string {
return Object.entries(hitsAtRows(rows, k))
.map(([key, value]) => `${key.replace("hit_at_", "hit@")} ${fmt(value)}`)
.join(" ");
}
function fmt(x: number): string {
return x.toFixed(4);
}
function signed(x: number): string {
return `${x >= 0 ? "+" : ""}${x.toFixed(4)}`;
}
export async function cmdSearchRerankEval(argv: ReadonlyArray<string>): Promise<number> {
const { flags } = parseFlags(argv, {
...VAULT_FLAGS,
dataset: { type: "string" },
kind: { type: "string" },
k: { type: "string" },
"compare-local": { type: "boolean" },
json: { type: "boolean" },
});
const datasetPath = flagString(flags, "dataset");
if (datasetPath === undefined) throw new CliError("--dataset <path> is required");
const kindFlag = flagString(flags, "kind") ?? "local";
if (!isKind(kindFlag)) {
throw new CliError(`--kind must be one of ${KINDS.join(", ")}, got '${kindFlag}'`);
}
const kFlag = flagString(flags, "k");
const k = kFlag !== undefined ? Number(kFlag) : undefined;
if (k !== undefined && !isIntegerWithin(k, { min: 1 })) {
throw new CliError(`--k must be a positive integer, got '${kFlag}'`);
}
let raw: unknown;
try {
raw = JSON.parse(readFileSync(datasetPath, "utf8"));
} catch (e) {
throw new CliError(`--dataset could not be read as JSON: ${(e as Error).message}`);
}
const dataset = parseRecallBenchmarkDataset(raw);
const cfg = resolveConfig(flags);
// The decision-model arm reports from the vault's call records: refused
// before any query is sent to the model, not after.
const reportVault =
kindFlag === "decision-model" ? requireSearchVault(cfg, "search rerank-eval") : null;
const decisionModel =
kindFlag === "decision-model"
? (cfg.rerank.decisionModel ??
resolveDecisionModelConfig({
config: discoverConfig(resolveConfigPath(flags)).data,
vault: cfg.vault,
}))
: undefined;
const started = new Date().toISOString();
const gate = await runRerankEvalGate(cfg, dataset, {
kind: kindFlag,
...(k !== undefined ? { k } : {}),
...(decisionModel !== undefined ? { decisionModel } : {}),
});
const local =
flagBoolean(flags, "compare-local") && kindFlag !== "local"
? await runRerankEvalGate(cfg, dataset, { kind: "local", ...(k !== undefined ? { k } : {}) })
: null;
// The decision-model arm's own records: every call this run made.
const calls =
reportVault !== null
? callsOf(
buildDecisionModelReport(reportVault, {
since: started,
use: "rerank",
origin: "eval",
}).uses[0],
)
: null;
if (flagBoolean(flags, "json")) {
process.stdout.write(
JSON.stringify({
ok: true,
queries: gate.baseline.total,
k: gate.baseline.k,
baseline: {
hit_at_k: gate.baseline.hitAtK,
mrr: gate.baseline.mrr,
...hitsAt(gate.baseline),
by_lang: byLangOf(gate.baseline),
},
[kindFlag]: { ...arm(gate), ...(calls !== null ? { calls } : {}) },
...(local !== null ? { local: arm(local) } : {}),
}) + "\n",
);
return 0;
}
const line = (name: string, r: RerankEvalGateResult): string => {
const vs = versusOff(r.baseline, r.reranked);
return (
` ${name.padEnd(15)} ${renderHits(r.reranked)} MRR ${fmt(r.reranked.mrr)}` +
` (hit@k ${signed(r.deltas.hitAtK)}, MRR ${signed(r.deltas.mrr)}; ` +
`vs off ${vs.wins} win(s), ${vs.losses} loss(es), ${vs.ties} tie(s)) -> ${r.recommendation}` +
`${renderByLang(r.reranked, r.deltas.byLang)}\n`
);
};
let text =
`rerank eval over ${gate.baseline.total} query(ies), k=${gate.baseline.k}\n` +
` ${"off:".padEnd(15)} ${renderHits(gate.baseline)} MRR ${fmt(gate.baseline.mrr)}` +
`${renderByLang(gate.baseline)}\n` +
line(kindFlag, gate);
if (calls !== null) {
const outcomes = Object.entries(calls["outcomes"] as Record<string, number>)
.map(([outcome, n]) => `${outcome} ${n}`)
.join(", ");
const p50 = calls["latency_p50_ms"];
const p95 = calls["latency_p95_ms"];
text +=
` decision calls: ${String(calls["total"])} (${outcomes || "none"}); ` +
`cost $${(calls["cost_usd"] as number).toFixed(6)}; ` +
`latency p50 ${p50 === null ? "-" : String(p50)}ms, p95 ${p95 === null ? "-" : String(p95)}ms\n`;
}
if (local !== null) text += line("local", local);
process.stdout.write(text);
return 0;
}