Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
18.4 KB
Raw Blame
/**
 * Reproducible recall benchmark (link-recall-intelligence,
 * t_e2215d49).
 *
 * A fixed query/expected-result dataset scored against the live
 * hybrid pipeline (`search()`): hit@k and MRR per query and in
 * aggregate. Three consumers share this runner:
 *
 *   - the CI regression gate (`tests/core/search/recall-benchmark
 *     .test.ts`) pins thresholds over the committed fixture vault, so
 *     a ranking regression fails the suite deterministically;
 *   - `o2b brain benchmark run` scores an operator vault on demand and
 *     records the run in `Brain/metrics/recall_benchmark.jsonl`;
 *   - `tuneRecall` (t_ae973491) uses the report as the objective
 *     function when grid-evaluating retrieval parameters.
 *
 * The runner itself is read-only and emits nothing - metric emission
 * belongs to the CLI/MCP callers, so library use stays side-effect
 * free.
 */

import { Semaphore } from "./embeddings/http-util.ts";
import { search } from "./search.ts";
import { resolvedTransportReach, type TransportReach } from "../graph/transport-reach.ts";
import { BLOCKED_TIER_ERROR_CODE, type BlockedCapabilityTier } from "./capability-tier.ts";
import { RETRIEVAL_DEGRADATION, type RetrievalDegradationCode } from "./retrieval-trail.ts";
import { SearchError } from "./types.ts";
import type { ResolvedSearchConfig, SearchOutcome } from "./types.ts";

/** Default rank depth for hit@k / reciprocal rank. */
export const BENCHMARK_DEFAULT_K = 5;

/**
 * Most dataset queries this process runs at once.
 *
 * The dataset is operator-supplied and unbounded in size, and `tuneRecall`
 * multiplies it by the 24-point parameter grid, so the fan-out is the
 * product of two numbers this module does not choose. Eight concurrent
 * hybrid searches keep a benchmark run overlapping its SQLite reads
 * without letting a large dataset - or the grid - open one connection and
 * one query plan per query at the same instant.
 */
export const BENCHMARK_QUERY_CONCURRENCY = 8;

/**
 * The ceiling itself, module-scoped so it spans the process: `tuneRecall`
 * awaits 24 concurrent `runRecallBenchmark` calls, and a gate constructed
 * per call would bound each one at 8 while the process ran 192.
 */
const queryGate = new Semaphore(BENCHMARK_QUERY_CONCURRENCY);

/**
 * Test-only: the high-water mark of concurrent queries this process has
 * reached. The leading underscore is this repo's marker for a test-only
 * export, which is exactly what the rule below flags.
 */
// oxlint-disable-next-line no-underscore-dangle
export function _benchmarkQueryPeakForTests(): number {
  return queryGate.peakInFlight;
}

export interface RecallBenchmarkQuery {
  readonly id: string;
  readonly query: string;
  /** Vault-relative paths counted as a hit, any-of. */
  readonly expected: ReadonlyArray<string>;
  /** Per-query rank depth override. */
  readonly k?: number;
  /**
   * Optional expected answer text. When present, the query also scores
   * answer-containment@k: whether this substring appears (folded) in the
   * retrieved content within the top k. Absent leaves the query measuring
   * hit@k / MRR only, so existing datasets stay valid.
   */
  readonly answer?: string;
  /**
   * Optional language tag for the per-language report split (t_693fa22a).
   * Operator-supplied and free-form (an ISO-639-ish code such as `en` or
   * `zh` works, but nothing parses it); validated like `answer` - a
   * non-empty string when present - and defaulted to `"en"` when absent,
   * so every existing dataset folds under the `en` key unchanged. It only
   * partitions the report; no scoring or detection logic reads it.
   */
  readonly lang?: string;
}

export interface RecallBenchmarkDataset {
  readonly queries: ReadonlyArray<RecallBenchmarkQuery>;
}

export interface RecallBenchmarkOptions {
  /** Rank depth, default {@link BENCHMARK_DEFAULT_K}. */
  readonly k?: number;
  /** Route every query through deterministic expansion (t_2fa95db1). */
  readonly expand?: boolean;
  /**
   * The reach the party that supplied this dataset was established at.
   *
   * Load-bearing, not decorative. This is not an internal lane: the
   * dataset is a required argument of `brain_benchmark` and `brain_tune`,
   * both callable over any transport, and the report answers per query
   * with `hit`, `rank`, `expectedFound` for caller-named paths and
   * `answerContained` for a caller-supplied string. Run at a reach the
   * caller did not earn, that is an existence oracle over the reserved
   * corpus and a substring oracle over its bodies - reachable without
   * ever calling a tool the registry classifies as covered.
   *
   * Absent resolves to the narrowest, on the same terms
   * {@link SearchOptions.transportReach} does. The lanes that must score
   * the whole corpus - the CLI benchmark, the tuning sweep's own
   * measurement, the rerank eval gate - pass {@link TRANSPORT_REACH.local}
   * explicitly, because a benchmark that stopped seeing reserved pages
   * would score a smaller corpus than the one it is benchmarking.
   */
  readonly transportReach?: TransportReach;
}

export interface RecallBenchmarkQueryResult {
  readonly id: string;
  readonly query: string;
  /**
   * The language group this query folds into: the query's `lang`, or
   * `"en"` when the dataset entry declared none.
   */
  readonly lang: string;
  readonly hit: boolean;
  /** 1-based rank of the first expected path, null on a miss. */
  readonly rank: number | null;
  readonly reciprocalRank: number;
  /**
   * Whether the declared answer appeared (folded) in the retrieved
   * content within the top k. `null` when the query declares no answer.
   */
  readonly answerContained: boolean | null;
  /** Expected paths that surfaced in the top k for this query. */
  readonly expectedFound: number;
  /** Total expected paths declared for this query. */
  readonly expectedTotal: number;
  /**
   * The retrieval trail's degradation codes for this query, present only
   * when the search was degraded: a hit scored by a lane that did not run
   * (the spend gate, a provider outage) is a keyword-only measurement.
   */
  readonly degraded?: ReadonlyArray<RetrievalDegradationCode>;
}

export interface RecallBenchmarkReport {
  readonly total: number;
  readonly k: number;
  readonly expand: boolean;
  /** Fraction of queries with an expected path in the top k. */
  readonly hitAtK: number;
  /** Mean reciprocal rank over all queries (misses contribute 0). */
  readonly mrr: number;
  /**
   * Number of queries that declared an answer (the answer-containment
   * denominator). 0 when the dataset uses no answers.
   */
  readonly answerQueries: number;
  /**
   * Fraction of answer-bearing queries whose answer appeared in the
   * top-k content. Vacuously 1 when no query declares an answer, so a CI
   * floor never fails a dataset that does not use the metric.
   */
  readonly answerContainmentAtK: number;
  /**
   * Source utilization: fraction of all declared expected paths (summed
   * across queries) that surfaced in their query's top k. Measures source
   * coverage rather than per-query hit; equals hitAtK when every query
   * declares exactly one expected path.
   */
  readonly sourceUtilizationAtK: number;
  /**
   * Citation depth: mean 1-based rank of the first expected hit over the
   * queries that hit (lower is better). 0 when no query hit.
   */
  readonly citationDepth: number;
  /**
   * Source warnings: queries where NO expected path surfaced in the top
   * k - an expected source the retrieval failed to use. The CI gate caps
   * this with `source_warnings_max`.
   */
  readonly sourceWarnings: number;
  /**
   * Every degradation code any query carried, sorted and de-duplicated.
   * Empty when every query ran the pipeline it asked for; otherwise the
   * scores above measure a degraded system, not the configured one.
   */
  readonly degraded: ReadonlyArray<RetrievalDegradationCode>;
  /**
   * The capability rung behind `semantic-capability-blocked`, present only
   * when {@link degraded} carries that code. The rung is configuration, so
   * every query that carried the code carried the same one; it is what
   * lets a refusal name the error the explicit lane throws for it.
   */
  readonly capabilityTier?: BlockedCapabilityTier;
  readonly perQuery: ReadonlyArray<RecallBenchmarkQueryResult>;
  /**
   * The per-language split (t_693fa22a): the SAME aggregate fold applied
   * to each `lang` group of `perQuery`, keyed by the query-declared tag
   * (`"en"` for entries that declared none). Additive to the return
   * value - the benchmark emits no record, so no schema version moves -
   * and a single-language dataset reports exactly one group whose numbers
   * equal the top-level ones.
   */
  readonly byLang: Readonly<Record<string, RecallBenchmarkLanguageReport>>;
}

/**
 * The aggregate metrics for one language group of a benchmark run - the
 * overall report's fold applied to the group's slice of `perQuery`.
 */
export interface RecallBenchmarkLanguageReport {
  readonly total: number;
  /** Fraction of the group's queries with an expected path in the top k. */
  readonly hitAtK: number;
  /** Mean reciprocal rank over the group's queries (misses contribute 0). */
  readonly mrr: number;
  /** The group's answer-containment denominator. */
  readonly answerQueries: number;
  readonly answerContainmentAtK: number;
  readonly sourceUtilizationAtK: number;
  readonly citationDepth: number;
  readonly sourceWarnings: number;
}

/** Narrow a trail detail value to a blocked rung, without a cast. */
function isBlockedCapabilityTier(value: unknown): value is BlockedCapabilityTier {
  return typeof value === "string" && Object.hasOwn(BLOCKED_TIER_ERROR_CODE, value);
}

/** Folded substring containment, language-agnostic (no word lists). */
function answerContainedIn(answer: string, contents: ReadonlyArray<string>): boolean {
  const needle = answer.replace(/\s+/gu, " ").trim().toLowerCase();
  if (needle.length === 0) return false;
  return contents.some((c) => c.replace(/\s+/gu, " ").toLowerCase().includes(needle));
}

/**
 * Validate a parsed dataset JSON value. Throws `SearchError`
 * (INVALID_INPUT) naming the first offending entry.
 */
export function parseRecallBenchmarkDataset(raw: unknown): RecallBenchmarkDataset {
  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
    throw new SearchError("INVALID_INPUT", "benchmark dataset must be a JSON object");
  }
  const queries = (raw as { queries?: unknown }).queries;
  if (!Array.isArray(queries) || queries.length === 0) {
    throw new SearchError("INVALID_INPUT", "benchmark dataset needs a non-empty `queries` array");
  }
  const seen = new Set<string>();
  const parsed = queries.map((entry, i) => {
    if (entry === null || typeof entry !== "object" || Array.isArray(entry)) {
      throw new SearchError("INVALID_INPUT", `benchmark query #${i} must be an object`);
    }
    const q = entry as {
      id?: unknown;
      query?: unknown;
      expected?: unknown;
      k?: unknown;
      answer?: unknown;
      lang?: unknown;
    };
    if (typeof q.id !== "string" || q.id.trim().length === 0) {
      throw new SearchError("INVALID_INPUT", `benchmark query #${i} needs a non-empty string id`);
    }
    if (seen.has(q.id)) {
      throw new SearchError("INVALID_INPUT", `benchmark query id '${q.id}' is duplicated`);
    }
    seen.add(q.id);
    if (typeof q.query !== "string" || q.query.trim().length === 0) {
      throw new SearchError("INVALID_INPUT", `benchmark query '${q.id}' needs a query string`);
    }
    if (
      !Array.isArray(q.expected) ||
      q.expected.length === 0 ||
      q.expected.some((p) => typeof p !== "string" || p.length === 0)
    ) {
      throw new SearchError(
        "INVALID_INPUT",
        `benchmark query '${q.id}' needs a non-empty expected path array`,
      );
    }
    if (q.k !== undefined && (!Number.isInteger(q.k) || (q.k as number) < 1)) {
      throw new SearchError("INVALID_INPUT", `benchmark query '${q.id}' k must be a positive int`);
    }
    if (q.answer !== undefined && (typeof q.answer !== "string" || q.answer.trim().length === 0)) {
      throw new SearchError(
        "INVALID_INPUT",
        `benchmark query '${q.id}' answer must be a non-empty string when present`,
      );
    }
    if (q.lang !== undefined && (typeof q.lang !== "string" || q.lang.trim().length === 0)) {
      throw new SearchError(
        "INVALID_INPUT",
        `benchmark query '${q.id}' lang must be a non-empty string when present`,
      );
    }
    // A language group is a spelling-insensitive fold: " ZH" and "zh" are
    // one group, never two.
    const lang = q.lang !== undefined ? (q.lang as string).trim().toLowerCase() : undefined;
    return Object.freeze({
      id: q.id,
      query: q.query,
      expected: Object.freeze([...(q.expected as string[])]) as ReadonlyArray<string>,
      ...(q.k !== undefined ? { k: q.k as number } : {}),
      ...(q.answer !== undefined ? { answer: q.answer as string } : {}),
      // Present on every parsed entry: the default lives here so the
      // report split never has to re-derive it. Hand-built datasets (the
      // type allows omitting the field) fall back in the runner instead.
      ...(lang !== undefined ? { lang } : { lang: "en" }),
    });
  });
  return Object.freeze({ queries: Object.freeze(parsed) });
}

/**
 * The ONE aggregate fold, applied to the whole run and to each language
 * group alike: a per-language number is never a second scorer, it is this
 * reduce over the group's slice of `perQuery`.
 */
function foldQueryResults(
  perQuery: ReadonlyArray<RecallBenchmarkQueryResult>,
): RecallBenchmarkLanguageReport {
  const hits = perQuery.filter((r) => r.hit).length;
  const answerScored = perQuery.filter((r) => r.answerContained !== null);
  const answerHits = answerScored.filter((r) => r.answerContained === true).length;
  const expectedFoundTotal = perQuery.reduce((sum, r) => sum + r.expectedFound, 0);
  const expectedDeclaredTotal = perQuery.reduce((sum, r) => sum + r.expectedTotal, 0);
  const hitRanks = perQuery.filter((r) => r.rank !== null).map((r) => r.rank!);
  return {
    total: perQuery.length,
    hitAtK: hits / perQuery.length,
    mrr: perQuery.reduce((sum, r) => sum + r.reciprocalRank, 0) / perQuery.length,
    answerQueries: answerScored.length,
    answerContainmentAtK: answerScored.length === 0 ? 1 : answerHits / answerScored.length,
    sourceUtilizationAtK:
      expectedDeclaredTotal === 0 ? 1 : expectedFoundTotal / expectedDeclaredTotal,
    citationDepth:
      hitRanks.length === 0 ? 0 : hitRanks.reduce((sum, r) => sum + r, 0) / hitRanks.length,
    sourceWarnings: perQuery.length - hits,
  };
}

/** Group `perQuery` by language, key order sorted, so the split is stable. */
function foldByLang(
  perQuery: ReadonlyArray<RecallBenchmarkQueryResult>,
): Readonly<Record<string, RecallBenchmarkLanguageReport>> {
  const groups = new Map<string, RecallBenchmarkQueryResult[]>();
  for (const row of perQuery) {
    const group = groups.get(row.lang);
    if (group !== undefined) group.push(row);
    else groups.set(row.lang, [row]);
  }
  const byLang: Record<string, RecallBenchmarkLanguageReport> = {};
  for (const lang of [...groups.keys()].toSorted()) {
    byLang[lang] = foldQueryResults(groups.get(lang)!);
  }
  return Object.freeze(byLang);
}

/**
 * Score the dataset against the vault behind `config`. Queries run
 * concurrently (read-only) up to {@link BENCHMARK_QUERY_CONCURRENCY}
 * in flight process-wide; the report order follows the dataset.
 */
export async function runRecallBenchmark(
  config: ResolvedSearchConfig,
  dataset: RecallBenchmarkDataset,
  opts: RecallBenchmarkOptions = {},
): Promise<RecallBenchmarkReport> {
  if (dataset.queries.length === 0) {
    throw new SearchError("INVALID_INPUT", "benchmark dataset needs a non-empty `queries` array");
  }
  if (opts.k !== undefined && (!Number.isInteger(opts.k) || opts.k < 1)) {
    throw new SearchError("INVALID_INPUT", "benchmark k must be a positive integer");
  }
  const k = opts.k ?? BENCHMARK_DEFAULT_K;
  const expand = opts.expand === true;
  const reach = resolvedTransportReach(opts.transportReach);

  let capabilityTier: BlockedCapabilityTier | undefined;
  const perQuery = await Promise.all(
    dataset.queries.map(async (q): Promise<RecallBenchmarkQueryResult> => {
      const depth = q.k ?? k;
      const permit = await queryGate.acquire();
      let outcome: SearchOutcome;
      try {
        outcome = await search(config, {
          query: q.query,
          limit: depth,
          ...(expand ? { expand: true } : {}),
          // The reach of whoever supplied the dataset - see
          // `RecallBenchmarkOptions.transportReach` for why this is not
          // the internal-lane constant it used to be.
          transportReach: reach,
        });
      } finally {
        permit();
      }
      const expected = new Set(q.expected);
      const topK = outcome.results.slice(0, depth);
      const topKPaths = new Set(topK.map((r) => r.path));
      const expectedFound = [...expected].filter((p) => topKPaths.has(p)).length;
      let rank: number | null = null;
      for (let i = 0; i < topK.length; i++) {
        if (expected.has(topK[i]!.path)) {
          rank = i + 1;
          break;
        }
      }
      const answerContained =
        q.answer === undefined
          ? null
          : answerContainedIn(
              q.answer,
              topK.map((r) => r.content),
            );
      const degraded = (outcome.retrievalTrail?.degraded ?? []).map((d) => d.code);
      for (const d of outcome.retrievalTrail?.degraded ?? []) {
        const tier = d.detail?.tier;
        if (
          d.code === RETRIEVAL_DEGRADATION.semanticCapabilityBlocked &&
          isBlockedCapabilityTier(tier)
        ) {
          capabilityTier = tier;
        }
      }
      return Object.freeze({
        id: q.id,
        query: q.query,
        // A hand-built dataset (the gate unit tests build theirs inline)
        // may omit the tag; the parsed form always carries it.
        lang: q.lang ?? "en",
        hit: rank !== null,
        rank,
        reciprocalRank: rank === null ? 0 : 1 / rank,
        answerContained,
        expectedFound,
        expectedTotal: expected.size,
        ...(degraded.length > 0 ? { degraded: Object.freeze(degraded) } : {}),
      });
    }),
  );

  const degraded = [...new Set(perQuery.flatMap((r) => r.degraded ?? []))].toSorted();
  return Object.freeze({
    ...foldQueryResults(perQuery),
    k,
    expand,
    degraded: Object.freeze(degraded),
    ...(capabilityTier !== undefined ? { capabilityTier } : {}),
    perQuery: Object.freeze(perQuery),
    byLang: foldByLang(perQuery),
  });
}