Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Reproducible recall benchmark (link-recall-intelligence,
* t_e2215d49).
*
* A fixed query/expected-result dataset scored against the live
* hybrid pipeline (`search()`): hit@k and MRR per query and in
* aggregate. Three consumers share this runner:
*
* - the CI regression gate (`tests/core/search/recall-benchmark
* .test.ts`) pins thresholds over the committed fixture vault, so
* a ranking regression fails the suite deterministically;
* - `o2b brain benchmark run` scores an operator vault on demand and
* records the run in `Brain/metrics/recall_benchmark.jsonl`;
* - `tuneRecall` (t_ae973491) uses the report as the objective
* function when grid-evaluating retrieval parameters.
*
* The runner itself is read-only and emits nothing - metric emission
* belongs to the CLI/MCP callers, so library use stays side-effect
* free.
*/
import { Semaphore } from "./embeddings/http-util.ts";
import { search } from "./search.ts";
import { resolvedTransportReach, type TransportReach } from "../graph/transport-reach.ts";
import { BLOCKED_TIER_ERROR_CODE, type BlockedCapabilityTier } from "./capability-tier.ts";
import { RETRIEVAL_DEGRADATION, type RetrievalDegradationCode } from "./retrieval-trail.ts";
import { SearchError } from "./types.ts";
import type { ResolvedSearchConfig, SearchOutcome } from "./types.ts";
/** Default rank depth for hit@k / reciprocal rank. */
export const BENCHMARK_DEFAULT_K = 5;
/**
* Most dataset queries this process runs at once.
*
* The dataset is operator-supplied and unbounded in size, and `tuneRecall`
* multiplies it by the 24-point parameter grid, so the fan-out is the
* product of two numbers this module does not choose. Eight concurrent
* hybrid searches keep a benchmark run overlapping its SQLite reads
* without letting a large dataset - or the grid - open one connection and
* one query plan per query at the same instant.
*/
export const BENCHMARK_QUERY_CONCURRENCY = 8;
/**
* The ceiling itself, module-scoped so it spans the process: `tuneRecall`
* awaits 24 concurrent `runRecallBenchmark` calls, and a gate constructed
* per call would bound each one at 8 while the process ran 192.
*/
const queryGate = new Semaphore(BENCHMARK_QUERY_CONCURRENCY);
/**
* Test-only: the high-water mark of concurrent queries this process has
* reached. The leading underscore is this repo's marker for a test-only
* export, which is exactly what the rule below flags.
*/
// oxlint-disable-next-line no-underscore-dangle
export function _benchmarkQueryPeakForTests(): number {
return queryGate.peakInFlight;
}
export interface RecallBenchmarkQuery {
readonly id: string;
readonly query: string;
/** Vault-relative paths counted as a hit, any-of. */
readonly expected: ReadonlyArray<string>;
/** Per-query rank depth override. */
readonly k?: number;
/**
* Optional expected answer text. When present, the query also scores
* answer-containment@k: whether this substring appears (folded) in the
* retrieved content within the top k. Absent leaves the query measuring
* hit@k / MRR only, so existing datasets stay valid.
*/
readonly answer?: string;
/**
* Optional language tag for the per-language report split (t_693fa22a).
* Operator-supplied and free-form (an ISO-639-ish code such as `en` or
* `zh` works, but nothing parses it); validated like `answer` - a
* non-empty string when present - and defaulted to `"en"` when absent,
* so every existing dataset folds under the `en` key unchanged. It only
* partitions the report; no scoring or detection logic reads it.
*/
readonly lang?: string;
}
export interface RecallBenchmarkDataset {
readonly queries: ReadonlyArray<RecallBenchmarkQuery>;
}
export interface RecallBenchmarkOptions {
/** Rank depth, default {@link BENCHMARK_DEFAULT_K}. */
readonly k?: number;
/** Route every query through deterministic expansion (t_2fa95db1). */
readonly expand?: boolean;
/**
* The reach the party that supplied this dataset was established at.
*
* Load-bearing, not decorative. This is not an internal lane: the
* dataset is a required argument of `brain_benchmark` and `brain_tune`,
* both callable over any transport, and the report answers per query
* with `hit`, `rank`, `expectedFound` for caller-named paths and
* `answerContained` for a caller-supplied string. Run at a reach the
* caller did not earn, that is an existence oracle over the reserved
* corpus and a substring oracle over its bodies - reachable without
* ever calling a tool the registry classifies as covered.
*
* Absent resolves to the narrowest, on the same terms
* {@link SearchOptions.transportReach} does. The lanes that must score
* the whole corpus - the CLI benchmark, the tuning sweep's own
* measurement, the rerank eval gate - pass {@link TRANSPORT_REACH.local}
* explicitly, because a benchmark that stopped seeing reserved pages
* would score a smaller corpus than the one it is benchmarking.
*/
readonly transportReach?: TransportReach;
}
export interface RecallBenchmarkQueryResult {
readonly id: string;
readonly query: string;
/**
* The language group this query folds into: the query's `lang`, or
* `"en"` when the dataset entry declared none.
*/
readonly lang: string;
readonly hit: boolean;
/** 1-based rank of the first expected path, null on a miss. */
readonly rank: number | null;
readonly reciprocalRank: number;
/**
* Whether the declared answer appeared (folded) in the retrieved
* content within the top k. `null` when the query declares no answer.
*/
readonly answerContained: boolean | null;
/** Expected paths that surfaced in the top k for this query. */
readonly expectedFound: number;
/** Total expected paths declared for this query. */
readonly expectedTotal: number;
/**
* The retrieval trail's degradation codes for this query, present only
* when the search was degraded: a hit scored by a lane that did not run
* (the spend gate, a provider outage) is a keyword-only measurement.
*/
readonly degraded?: ReadonlyArray<RetrievalDegradationCode>;
}
export interface RecallBenchmarkReport {
readonly total: number;
readonly k: number;
readonly expand: boolean;
/** Fraction of queries with an expected path in the top k. */
readonly hitAtK: number;
/** Mean reciprocal rank over all queries (misses contribute 0). */
readonly mrr: number;
/**
* Number of queries that declared an answer (the answer-containment
* denominator). 0 when the dataset uses no answers.
*/
readonly answerQueries: number;
/**
* Fraction of answer-bearing queries whose answer appeared in the
* top-k content. Vacuously 1 when no query declares an answer, so a CI
* floor never fails a dataset that does not use the metric.
*/
readonly answerContainmentAtK: number;
/**
* Source utilization: fraction of all declared expected paths (summed
* across queries) that surfaced in their query's top k. Measures source
* coverage rather than per-query hit; equals hitAtK when every query
* declares exactly one expected path.
*/
readonly sourceUtilizationAtK: number;
/**
* Citation depth: mean 1-based rank of the first expected hit over the
* queries that hit (lower is better). 0 when no query hit.
*/
readonly citationDepth: number;
/**
* Source warnings: queries where NO expected path surfaced in the top
* k - an expected source the retrieval failed to use. The CI gate caps
* this with `source_warnings_max`.
*/
readonly sourceWarnings: number;
/**
* Every degradation code any query carried, sorted and de-duplicated.
* Empty when every query ran the pipeline it asked for; otherwise the
* scores above measure a degraded system, not the configured one.
*/
readonly degraded: ReadonlyArray<RetrievalDegradationCode>;
/**
* The capability rung behind `semantic-capability-blocked`, present only
* when {@link degraded} carries that code. The rung is configuration, so
* every query that carried the code carried the same one; it is what
* lets a refusal name the error the explicit lane throws for it.
*/
readonly capabilityTier?: BlockedCapabilityTier;
readonly perQuery: ReadonlyArray<RecallBenchmarkQueryResult>;
/**
* The per-language split (t_693fa22a): the SAME aggregate fold applied
* to each `lang` group of `perQuery`, keyed by the query-declared tag
* (`"en"` for entries that declared none). Additive to the return
* value - the benchmark emits no record, so no schema version moves -
* and a single-language dataset reports exactly one group whose numbers
* equal the top-level ones.
*/
readonly byLang: Readonly<Record<string, RecallBenchmarkLanguageReport>>;
}
/**
* The aggregate metrics for one language group of a benchmark run - the
* overall report's fold applied to the group's slice of `perQuery`.
*/
export interface RecallBenchmarkLanguageReport {
readonly total: number;
/** Fraction of the group's queries with an expected path in the top k. */
readonly hitAtK: number;
/** Mean reciprocal rank over the group's queries (misses contribute 0). */
readonly mrr: number;
/** The group's answer-containment denominator. */
readonly answerQueries: number;
readonly answerContainmentAtK: number;
readonly sourceUtilizationAtK: number;
readonly citationDepth: number;
readonly sourceWarnings: number;
}
/** Narrow a trail detail value to a blocked rung, without a cast. */
function isBlockedCapabilityTier(value: unknown): value is BlockedCapabilityTier {
return typeof value === "string" && Object.hasOwn(BLOCKED_TIER_ERROR_CODE, value);
}
/** Folded substring containment, language-agnostic (no word lists). */
function answerContainedIn(answer: string, contents: ReadonlyArray<string>): boolean {
const needle = answer.replace(/\s+/gu, " ").trim().toLowerCase();
if (needle.length === 0) return false;
return contents.some((c) => c.replace(/\s+/gu, " ").toLowerCase().includes(needle));
}
/**
* Validate a parsed dataset JSON value. Throws `SearchError`
* (INVALID_INPUT) naming the first offending entry.
*/
export function parseRecallBenchmarkDataset(raw: unknown): RecallBenchmarkDataset {
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
throw new SearchError("INVALID_INPUT", "benchmark dataset must be a JSON object");
}
const queries = (raw as { queries?: unknown }).queries;
if (!Array.isArray(queries) || queries.length === 0) {
throw new SearchError("INVALID_INPUT", "benchmark dataset needs a non-empty `queries` array");
}
const seen = new Set<string>();
const parsed = queries.map((entry, i) => {
if (entry === null || typeof entry !== "object" || Array.isArray(entry)) {
throw new SearchError("INVALID_INPUT", `benchmark query #${i} must be an object`);
}
const q = entry as {
id?: unknown;
query?: unknown;
expected?: unknown;
k?: unknown;
answer?: unknown;
lang?: unknown;
};
if (typeof q.id !== "string" || q.id.trim().length === 0) {
throw new SearchError("INVALID_INPUT", `benchmark query #${i} needs a non-empty string id`);
}
if (seen.has(q.id)) {
throw new SearchError("INVALID_INPUT", `benchmark query id '${q.id}' is duplicated`);
}
seen.add(q.id);
if (typeof q.query !== "string" || q.query.trim().length === 0) {
throw new SearchError("INVALID_INPUT", `benchmark query '${q.id}' needs a query string`);
}
if (
!Array.isArray(q.expected) ||
q.expected.length === 0 ||
q.expected.some((p) => typeof p !== "string" || p.length === 0)
) {
throw new SearchError(
"INVALID_INPUT",
`benchmark query '${q.id}' needs a non-empty expected path array`,
);
}
if (q.k !== undefined && (!Number.isInteger(q.k) || (q.k as number) < 1)) {
throw new SearchError("INVALID_INPUT", `benchmark query '${q.id}' k must be a positive int`);
}
if (q.answer !== undefined && (typeof q.answer !== "string" || q.answer.trim().length === 0)) {
throw new SearchError(
"INVALID_INPUT",
`benchmark query '${q.id}' answer must be a non-empty string when present`,
);
}
if (q.lang !== undefined && (typeof q.lang !== "string" || q.lang.trim().length === 0)) {
throw new SearchError(
"INVALID_INPUT",
`benchmark query '${q.id}' lang must be a non-empty string when present`,
);
}
// A language group is a spelling-insensitive fold: " ZH" and "zh" are
// one group, never two.
const lang = q.lang !== undefined ? (q.lang as string).trim().toLowerCase() : undefined;
return Object.freeze({
id: q.id,
query: q.query,
expected: Object.freeze([...(q.expected as string[])]) as ReadonlyArray<string>,
...(q.k !== undefined ? { k: q.k as number } : {}),
...(q.answer !== undefined ? { answer: q.answer as string } : {}),
// Present on every parsed entry: the default lives here so the
// report split never has to re-derive it. Hand-built datasets (the
// type allows omitting the field) fall back in the runner instead.
...(lang !== undefined ? { lang } : { lang: "en" }),
});
});
return Object.freeze({ queries: Object.freeze(parsed) });
}
/**
* The ONE aggregate fold, applied to the whole run and to each language
* group alike: a per-language number is never a second scorer, it is this
* reduce over the group's slice of `perQuery`.
*/
function foldQueryResults(
perQuery: ReadonlyArray<RecallBenchmarkQueryResult>,
): RecallBenchmarkLanguageReport {
const hits = perQuery.filter((r) => r.hit).length;
const answerScored = perQuery.filter((r) => r.answerContained !== null);
const answerHits = answerScored.filter((r) => r.answerContained === true).length;
const expectedFoundTotal = perQuery.reduce((sum, r) => sum + r.expectedFound, 0);
const expectedDeclaredTotal = perQuery.reduce((sum, r) => sum + r.expectedTotal, 0);
const hitRanks = perQuery.filter((r) => r.rank !== null).map((r) => r.rank!);
return {
total: perQuery.length,
hitAtK: hits / perQuery.length,
mrr: perQuery.reduce((sum, r) => sum + r.reciprocalRank, 0) / perQuery.length,
answerQueries: answerScored.length,
answerContainmentAtK: answerScored.length === 0 ? 1 : answerHits / answerScored.length,
sourceUtilizationAtK:
expectedDeclaredTotal === 0 ? 1 : expectedFoundTotal / expectedDeclaredTotal,
citationDepth:
hitRanks.length === 0 ? 0 : hitRanks.reduce((sum, r) => sum + r, 0) / hitRanks.length,
sourceWarnings: perQuery.length - hits,
};
}
/** Group `perQuery` by language, key order sorted, so the split is stable. */
function foldByLang(
perQuery: ReadonlyArray<RecallBenchmarkQueryResult>,
): Readonly<Record<string, RecallBenchmarkLanguageReport>> {
const groups = new Map<string, RecallBenchmarkQueryResult[]>();
for (const row of perQuery) {
const group = groups.get(row.lang);
if (group !== undefined) group.push(row);
else groups.set(row.lang, [row]);
}
const byLang: Record<string, RecallBenchmarkLanguageReport> = {};
for (const lang of [...groups.keys()].toSorted()) {
byLang[lang] = foldQueryResults(groups.get(lang)!);
}
return Object.freeze(byLang);
}
/**
* Score the dataset against the vault behind `config`. Queries run
* concurrently (read-only) up to {@link BENCHMARK_QUERY_CONCURRENCY}
* in flight process-wide; the report order follows the dataset.
*/
export async function runRecallBenchmark(
config: ResolvedSearchConfig,
dataset: RecallBenchmarkDataset,
opts: RecallBenchmarkOptions = {},
): Promise<RecallBenchmarkReport> {
if (dataset.queries.length === 0) {
throw new SearchError("INVALID_INPUT", "benchmark dataset needs a non-empty `queries` array");
}
if (opts.k !== undefined && (!Number.isInteger(opts.k) || opts.k < 1)) {
throw new SearchError("INVALID_INPUT", "benchmark k must be a positive integer");
}
const k = opts.k ?? BENCHMARK_DEFAULT_K;
const expand = opts.expand === true;
const reach = resolvedTransportReach(opts.transportReach);
let capabilityTier: BlockedCapabilityTier | undefined;
const perQuery = await Promise.all(
dataset.queries.map(async (q): Promise<RecallBenchmarkQueryResult> => {
const depth = q.k ?? k;
const permit = await queryGate.acquire();
let outcome: SearchOutcome;
try {
outcome = await search(config, {
query: q.query,
limit: depth,
...(expand ? { expand: true } : {}),
// The reach of whoever supplied the dataset - see
// `RecallBenchmarkOptions.transportReach` for why this is not
// the internal-lane constant it used to be.
transportReach: reach,
});
} finally {
permit();
}
const expected = new Set(q.expected);
const topK = outcome.results.slice(0, depth);
const topKPaths = new Set(topK.map((r) => r.path));
const expectedFound = [...expected].filter((p) => topKPaths.has(p)).length;
let rank: number | null = null;
for (let i = 0; i < topK.length; i++) {
if (expected.has(topK[i]!.path)) {
rank = i + 1;
break;
}
}
const answerContained =
q.answer === undefined
? null
: answerContainedIn(
q.answer,
topK.map((r) => r.content),
);
const degraded = (outcome.retrievalTrail?.degraded ?? []).map((d) => d.code);
for (const d of outcome.retrievalTrail?.degraded ?? []) {
const tier = d.detail?.tier;
if (
d.code === RETRIEVAL_DEGRADATION.semanticCapabilityBlocked &&
isBlockedCapabilityTier(tier)
) {
capabilityTier = tier;
}
}
return Object.freeze({
id: q.id,
query: q.query,
// A hand-built dataset (the gate unit tests build theirs inline)
// may omit the tag; the parsed form always carries it.
lang: q.lang ?? "en",
hit: rank !== null,
rank,
reciprocalRank: rank === null ? 0 : 1 / rank,
answerContained,
expectedFound,
expectedTotal: expected.size,
...(degraded.length > 0 ? { degraded: Object.freeze(degraded) } : {}),
});
}),
);
const degraded = [...new Set(perQuery.flatMap((r) => r.degraded ?? []))].toSorted();
return Object.freeze({
...foldQueryResults(perQuery),
k,
expand,
degraded: Object.freeze(degraded),
...(capabilityTier !== undefined ? { capabilityTier } : {}),
perQuery: Object.freeze(perQuery),
byLang: foldByLang(perQuery),
});
}