Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
42.9 KB
Raw Blame
/**
 * Bench phase pipeline (Memory Observability Suite, t_882c396a):
 * ingest -> index -> retrieve -> evaluate -> report.
 *
 * Each phase checkpoints on completion, so a resumed run (same run id,
 * same fixture hash) skips finished work - including the searches:
 * the retrieve phase persists per-question raw results to disk and the
 * evaluate phase works ONLY from those files, never from the live
 * vault. The pipeline drives the public `search` / `packContext` APIs
 * against a disposable vault inside the run directory and never
 * resolves the operator's configured vault. Deterministic and
 * network-free: keyword-only search, no embedding providers, judge
 * optional and external.
 */

import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";

import { evaluateAttentionFlow, listAttentionFlows } from "../brain/attention-flows.ts";
import { packContext } from "../brain/context-pack.ts";
import { TRANSPORT_REACH } from "../graph/transport-reach.ts";
import { loadNormalizedContinuityRecords } from "../brain/continuity/read-model.ts";
import type { ContinuityRecord } from "../brain/continuity/types.ts";
import { extractPreCompactRecords } from "../brain/pre-compact-extract.ts";
import {
  decideRecallInject,
  RECALL_INJECT_MAX_NOTES,
  type RecallInjectDecision,
} from "../brain/recall-inject.ts";
import { resolveSearchConfig, search } from "../search/index.ts";
import type { SearchLaneTiming } from "../search/index.ts";
import { evaluateSurfacingGate } from "../search/surfacing-gate.ts";
import {
  BENCH_EMBEDDING_RECEIPT_FILENAME,
  embedBenchVaultIndex,
  readBenchEmbeddingReceipt,
} from "./embedding-cache.ts";
import { countQuestionResults, preflightBenchRun, validateBenchTrace } from "./guards.ts";
import {
  benchResultsDir,
  benchRunDirPath,
  benchVaultDir,
  atomicWriteFileSync,
  completeBenchPhase,
  createBenchRun,
  loadBenchRun,
  nextBenchRunId,
  phaseDone,
  type BenchRunHandle,
} from "./run-store.ts";
import {
  benchRecallRetriever,
  classifyRecallDecision,
  injectedTokens,
  isolationViolations,
  scoreProactiveRecall,
  type RecallFailure,
} from "./failure-modes.ts";
import { fixtureHash, materializeBenchVault } from "./fixture.ts";
import { runJudge } from "./judge.ts";
import { LONGMEMEVAL_SUITE_NAME, scoreLongMemEvalRecall } from "./longmemeval.ts";
import {
  BENCH_REPORT_SCHEMA,
  DEFAULT_BENCH_KNOB_PIN,
  RETRIEVAL_CATEGORIES,
  type BenchFixture,
  type BenchEmbeddingReceipt,
  type BenchLongMemEvalFamily,
  type BenchPinnedKnobs,
  type BenchQuestion,
  type BenchQuestionResult,
  type BenchReport,
  type BenchSampleRequest,
  type BenchSuiteCapabilities,
  type BenchSuiteReceipt,
} from "./types.ts";

/**
 * The reach the benchmark harness runs its queries at.
 *
 * Local: a bench run measures this vault's own corpus and hands no page
 * to any caller, so a run that stopped seeing reserved pages would score
 * a smaller vault than the one under test.
 */
const BENCH_REACH = TRANSPORT_REACH.local;
/**
 * The standing-instruction facet category (recall-lane-latency). It is
 * the ONE category the headline quality family excludes: its rate stays
 * the named line of its own in `by_category`.
 */
const STANDING_INSTRUCTION_CATEGORY = "standing_instruction";

export interface BenchRunOptions {
  readonly fixture: BenchFixture;
  readonly runsDir: string;
  /** Registered suite name, when the run goes through a named suite. */
  readonly suite?: string;
  /**
   * What the run's suite may do, set by the CLI verb from the suite
   * registry. Inert on its own: a capability is exercised only when the
   * run's knob pin enables the feature too.
   */
  readonly capabilities?: BenchSuiteCapabilities;
  /**
   * Search-configuration knobs pinned for this run, applied last via
   * `resolveSearchConfig` and folded into the run identity (the
   * checkpoint hash). Absent means {@link DEFAULT_BENCH_KNOB_PIN}.
   */
  readonly pinnedKnobs?: BenchPinnedKnobs;
  /** Resume an existing run id; validates the fixture hash first. */
  readonly resume?: string;
  /**
   * Optional admission floor override for the proactive-recall decisions,
   * passed to the decision core's `confidenceFloor` option, which routes
   * through the shared surfacing calculator. Absent keeps the shipped
   * default (`RECALL_INJECT_CONFIDENCE_FLOOR`). The surfacing-replay
   * suite sweeps this to measure gate calibration.
   *
   * Part of the run identity: a resumed run whose floor changed fails
   * the identity guard instead of deciding the remaining questions under
   * a new floor while the persisted ones keep the old. A floor change
   * takes effect in a fresh run id.
   */
  readonly recallFloor?: number;
  /** Optional external judge command (config bench_judge_cmd). */
  readonly judgeCmd?: string;
  /**
   * The suite loader's conversion receipt, threaded through to the
   * report family - for LongMemEval, how many abstention questions the
   * conversion excluded from scoring.
   */
  readonly suiteReceipt?: BenchSuiteReceipt;
  /**
   * The seeded sampling request the run's fixture was loaded under.
   * The CLI samples at load time, so this folds into the run identity
   * (a resumed run whose sampling options changed fails the guard, even
   * when a cap at or above the population would make the sampled
   * fixtures identical) and is refused up front when malformed - it
   * drives no phase of its own.
   */
  readonly sample?: BenchSampleRequest;
  readonly now?: Date;
}

/** Raw per-question record persisted by the retrieve phase. */
interface RetrievedQuestion {
  readonly id: string;
  readonly latency_ms: number;
  /**
   * Overlapped lane milliseconds (recall-lane-latency), present only when
   * the semantic lane attempted the embed for this question.
   */
  readonly overlapped_ms?: number;
  /** Result paths in rank order (retrieval categories). */
  readonly paths?: ReadonlyArray<string>;
  /** Normalized turn texts (session_handoff). */
  readonly turn_texts?: ReadonlyArray<string>;
  /** Pack item ids (budget, source_isolation). */
  readonly item_ids?: ReadonlyArray<string>;
  /** Characters of context this question injected, when it injected any. */
  readonly context_chars?: number;
  /** `estimateTokens` over the same string `context_chars` measured. */
  readonly injected_tokens?: number;
  /** The decision core's verdict, verbatim (proactive_recall). */
  readonly recall_decision?: RecallInjectDecision;
  /** Normalized label tokens the extractor recovered (write_fidelity). */
  readonly labels?: ReadonlyArray<string>;
  /** Provenance keys present on every extracted record (write_fidelity). */
  readonly provenance_keys?: ReadonlyArray<string>;
  /** Whether a second identical extraction produced the same record ids. */
  readonly dedup_stable?: boolean;
  /**
   * The item strings the standing_query sections rendered, annotations
   * included (standing_instruction). The evaluator asserts the expected
   * earliest-cited coordinates against these.
   */
  readonly standing_items?: ReadonlyArray<string>;
  /**
   * Facet keys (`pref:<preference id>`) whose provenance join resolved
   * (standing_instruction). A preference the lane surfaced but could not
   * resolve is honestly absent - never an implied origin.
   */
  readonly facet_ids?: ReadonlyArray<string>;
}

/**
 * The search configuration every phase uses.
 *
 * `resolveSearchConfig` still reads `process.env`, and overrides are
 * applied last and win, so pinning the semantic lane off here is what
 * makes the harness's network-freedom an invariant instead of an
 * inherited accident. Pinning it OFF rather than onto the offline
 * `LocalProvider` is deliberate: the bench's scoring is keyword
 * containment, so an embedding lane would add a second ranking signal to
 * a number nobody asked to include, and cost time for it.
 *
 * Exported so the invariant is assertable directly. It has to be: a test
 * that sets the env vars and compares two reports passes whether or not
 * the override is there, because reaching the embedding lane also needs
 * an index built with embeddings, which this harness does not build. The
 * resolved config is where the difference is real, so that is what the
 * test reads.
 *
 * One residual, named rather than implied: `resolveSearchConfig` parses
 * the provider name BEFORE merging overrides, so an env var naming a
 * provider it does not recognise makes this throw. That is a loud, named
 * failure rather than a silently different bench.
 */
/**
 * The search configuration every phase uses, driven by the run's knob
 * pin.
 *
 * `resolveSearchConfig` still reads `process.env`, and overrides are
 * applied last and win, so the pin is what makes the harness's network-
 * freedom an invariant instead of an inherited accident. The default pin
 * ({@link DEFAULT_BENCH_KNOB_PIN}) is the hardcoded semantic-off override
 * this function always applied - a named value now, so the default run
 * identity is deterministic and locomo report numbers stay identical. A
 * suite that declares the embeddings capability may pin the semantic
 * lane on (the deterministic local provider keeps the run network-free).
 *
 * Exported so the invariant is assertable directly. It has to be: a test
 * that sets the env vars and compares two reports passes whether or not
 * the override is there, because reaching the embedding lane also needs
 * an index built with embeddings, which the default harness does not
 * build. The resolved config is where the difference is real, so that is
 * what the test reads.
 *
 * One residual, named rather than implied: `resolveSearchConfig` parses
 * the provider name BEFORE merging overrides, so an env var naming a
 * provider it does not recognise makes this throw. That is a loud, named
 * failure rather than a silently different bench.
 */
export function benchSearchConfig(
  vault: string,
  pinnedKnobs: BenchPinnedKnobs = DEFAULT_BENCH_KNOB_PIN,
): ReturnType<typeof resolveSearchConfig> {
  return resolveSearchConfig({ vault, overrides: pinnedKnobs });
}

/** Char budget per evidence excerpt a judged question carries. */
const JUDGE_EVIDENCE_EXCERPT_CHARS = 800;

export async function runMemoryBench(opts: BenchRunOptions): Promise<BenchReport> {
  // Refused BEFORE any run state exists (pr-250 review, L9): the floor
  // routes into the shared surfacing calculator, whose own contract
  // refuses out-of-range values per question - so an out-of-range floor
  // used to buy a full run whose every proactive decision was a
  // retrievalError, paid after ingest and index, instead of this one
  // named error. The bound is the calculator's own: a finite number in
  // [0, 1]. Zero stays legal - the replay suite's permissive sweep arm
  // uses it as the maximally permissive calibration point.
  if (
    opts.recallFloor !== undefined &&
    !(Number.isFinite(opts.recallFloor) && opts.recallFloor >= 0 && opts.recallFloor <= 1)
  ) {
    throw new Error(
      `bench run refused: recallFloor ${opts.recallFloor} is outside [0, 1]; the admission floor must be a finite number in the unit interval`,
    );
  }
  // Refused BEFORE any run state exists, on the same principle: the
  // sampling options are part of the run identity, so a malformed one
  // must not buy a full run under an identity that could never resume.
  if (opts.sample !== undefined) {
    if (!Number.isInteger(opts.sample.cap) || opts.sample.cap <= 0) {
      throw new Error(
        `bench run refused: sample cap ${opts.sample.cap} is not a positive integer; the sampling cap bounds the selected questions`,
      );
    }
    if (opts.sample.seed.trim() === "") {
      throw new Error("bench run refused: sample seed must be a non-empty string");
    }
  }
  const pinnedKnobs = opts.pinnedKnobs ?? DEFAULT_BENCH_KNOB_PIN;
  // The run's clock is resolved once so every consumer sees the same
  // second: the preflight's collision check must read the identity the
  // store is about to write.
  const now = opts.now ?? new Date();
  // The preflight catches every damaged state catchable BEFORE a run
  // exists (unusable runs dir, fresh-run collision, damaged or tampered
  // resume target, fixture cross-reference defects), so a refusal can
  // never leave a run directory behind. Guards add zero stdout and no
  // knobs: a healthy run is untouched.
  const freshRunId =
    opts.resume !== undefined
      ? undefined
      : nextBenchRunId(opts.fixture, {
          now,
          pinnedKnobs,
          ...(opts.recallFloor !== undefined ? { recallFloor: opts.recallFloor } : {}),
          ...(opts.sample !== undefined ? { sample: opts.sample } : {}),
        });
  preflightBenchRun({
    runsDir: opts.runsDir,
    fixture: opts.fixture,
    ...(opts.resume !== undefined ? { resume: opts.resume } : {}),
    ...(freshRunId !== undefined ? { freshRunId } : {}),
  });
  // Resolved BEFORE the run is created (t_d7ce8baf): `resolveSearchConfig`
  // parses the env BEFORE merging overrides, so an env var naming a
  // provider it does not recognise throws - and that refusal used to
  // leave an orphan run dir behind. The vault path is computed the way
  // the store resolves it, so the config here is the config the searches
  // will run under.
  const runDir = benchRunDirPath(opts.runsDir, opts.resume ?? freshRunId!);
  const vault = benchVaultDir(runDir);
  const config = benchSearchConfig(vault, pinnedKnobs);
  // The capability gate: the semantic lane is reachable only when the
  // run's suite declared the embeddings capability, so a pin can never
  // silently arm a lane the suite never asked for. Same ordering rule:
  // this refusal also used to orphan a run dir.
  if (config.semantic.enabled && opts.capabilities?.embeddings !== true) {
    throw new Error(
      "bench run pins the semantic lane on without the embeddings capability - declare { embeddings: true } for the suite or keep the semantic lane pinned off",
    );
  }
  const run: BenchRunHandle =
    opts.resume !== undefined
      ? loadBenchRun(opts.runsDir, opts.resume, {
          expectFixture: opts.fixture,
          pinnedKnobs,
          ...(opts.recallFloor !== undefined ? { recallFloor: opts.recallFloor } : {}),
          ...(opts.sample !== undefined ? { sample: opts.sample } : {}),
        })
      : createBenchRun(opts.runsDir, opts.fixture, {
          now,
          pinnedKnobs,
          ...(opts.recallFloor !== undefined ? { recallFloor: opts.recallFloor } : {}),
          ...(opts.sample !== undefined ? { sample: opts.sample } : {}),
        });
  let checkpoint = run.checkpoint;
  // Present when this invocation ran the embedding index phase; read
  // back from the run artifact on a resumed run.
  let embeddings: BenchEmbeddingReceipt | undefined;

  // Phase: ingest - materialize the disposable vault.
  if (!phaseDone(checkpoint, "ingest")) {
    materializeBenchVault(opts.fixture, vault);
    checkpoint = completeBenchPhase(run.runDir, checkpoint, "ingest");
  }

  // Phase: index - one warmup search triggers the store self-heal so
  // the FTS index exists before any timed retrieval. A suite that both
  // declares the embeddings capability and pins the semantic lane on
  // builds real vectors instead, through the apply-gated indexer path
  // and the shared runs-dir vector cache.
  if (!phaseDone(checkpoint, "index")) {
    if (config.semantic.enabled) {
      embeddings = await embedBenchVaultIndex({ config, runsDir: opts.runsDir });
      writeFileSync(
        join(run.runDir, BENCH_EMBEDDING_RECEIPT_FILENAME),
        `${JSON.stringify(embeddings, null, 2)}\n`,
        "utf8",
      );
    } else {
      await search(config, { query: "warmup", limit: 1, transportReach: BENCH_REACH });
    }
    checkpoint = completeBenchPhase(run.runDir, checkpoint, "index");
  }
  if (embeddings === undefined && config.semantic.enabled) {
    embeddings = readBenchEmbeddingReceipt(run.runDir);
  }

  // Phase: retrieve - run every question once, persist raw results.
  if (!phaseDone(checkpoint, "retrieve")) {
    mkdirSync(benchResultsDir(run.runDir), { recursive: true });
    // Deliberately sequential: per-question latency_ms must not include
    // contention from sibling searches running on the same store.
    // oxlint-disable-next-line no-await-in-loop
    for (const question of opts.fixture.questions) {
      // oxlint-disable-next-line no-await-in-loop
      const retrieved = await retrieveQuestion(
        vault,
        config,
        question,
        pinnedKnobs,
        opts.recallFloor,
      );
      atomicWriteFileSync(
        join(benchResultsDir(run.runDir), `${question.id}.json`),
        `${JSON.stringify(retrieved, null, 2)}\n`,
      );
    }
    checkpoint = completeBenchPhase(run.runDir, checkpoint, "retrieve");
  }

  // The trace guard, FIRST pass: every fixture question must have a
  // parseable, correctly-keyed record carrying its category's required
  // fields, no faulted decisions and no leftover files - refused by name
  // BEFORE the evaluator scores anything, so a tampered or truncated
  // trace can never be dressed up as an ordinary failing run. (The
  // second pass runs pre-write with the assembled report, cross-checking
  // the published counts against the same trace.)
  validateBenchTrace({ runDir: run.runDir, fixture: opts.fixture });

  // Phase: evaluate - pure over the persisted results; no vault access.
  // Each persisted record is read once and shared by the evaluator and
  // the suite metric families.
  const retrievedById = new Map(
    opts.fixture.questions.map((question) => [question.id, readRetrieved(run.runDir, question.id)]),
  );
  const results = opts.fixture.questions
    .map((question) => evaluateQuestion(question, retrievedById.get(question.id)!))
    .toSorted((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
  if (!phaseDone(checkpoint, "evaluate")) {
    checkpoint = completeBenchPhase(run.runDir, checkpoint, "evaluate");
  }

  // The LongMemEval suite's named metric family: strict recall_all@5
  // rolled off the same persisted results the evaluator read. The
  // exclusion count comes only from a real conversion receipt - without
  // one it reports null, never a manufactured 0. The sampled count
  // rides along only when the run sampled, so a whole-dataset report
  // serializes byte-identically to before.
  let longmemeval: BenchLongMemEvalFamily | undefined;
  if (opts.suite === LONGMEMEVAL_SUITE_NAME) {
    const scored = scoreLongMemEvalRecall(opts.fixture.questions, [...retrievedById.values()]);
    longmemeval = Object.freeze({
      recall_all_at_5: scored.scored > 0 ? round3(scored.passed / scored.scored) : 0,
      abstention_excluded: opts.suiteReceipt?.abstention_excluded ?? null,
      // The sampled bookkeeping rides the named family when the run
      // sampled - population, selected count and the per-stratum
      // breakdown - so a reader can see what the run's numbers cover
      // without opening the checkpoint. Spread-omitted otherwise: a
      // whole-dataset (or receipt-less) report serializes
      // byte-identically to before.
      ...(opts.suiteReceipt?.sample_population !== undefined
        ? { sample_population: opts.suiteReceipt.sample_population }
        : {}),
      ...(opts.suiteReceipt?.sample_selected !== undefined
        ? { sample_selected: opts.suiteReceipt.sample_selected }
        : {}),
      ...(opts.suiteReceipt?.sample_strata !== undefined
        ? { sample_strata: opts.suiteReceipt.sample_strata }
        : {}),
    });
  }

  // Optional judge (advisory, fail-open), then report. Questions whose
  // suite provides official-prompt judge input carry it on the stdin
  // payload; suites without it keep the payload unchanged.
  // Judged questions carry the retrieved evidence (path plus a bounded
  // excerpt, in rank order) beside the reference answer, so a judge can
  // evaluate against what the system actually showed.
  const bodiesByPath = new Map(opts.fixture.notes.map((n) => [n.path, n.body]));
  const excerpt = (body: string): string =>
    body.length > JUDGE_EVIDENCE_EXCERPT_CHARS
      ? `${body.slice(0, JUDGE_EVIDENCE_EXCERPT_CHARS)}...`
      : body;
  const judgeInputs = new Map(
    opts.fixture.questions.flatMap((question) => {
      if (question.judge_input === undefined) return [];
      const retrieved = retrievedById.get(question.id);
      const evidence = (retrieved?.paths ?? []).flatMap((path) => {
        const body = bodiesByPath.get(path);
        return body === undefined ? [] : [{ path, text: excerpt(body) }];
      });
      const input =
        evidence.length > 0 ? { ...question.judge_input, evidence } : question.judge_input;
      return [[question.id, input] as const];
    }),
  );
  const judge = runJudge(opts.judgeCmd, results, judgeInputs);
  const report = buildReport(run, opts.fixture, results, judge, embeddings, longmemeval);
  // The trace guard, SECOND pass: the assembled report must agree with
  // the trace it was built from (same aggregation, including the
  // standing_instruction headline exclusion) - refused before anything
  // is written, so a buildReport regression cannot publish silently.
  validateBenchTrace({ runDir: run.runDir, fixture: opts.fixture, results, report });
  atomicWriteFileSync(join(run.runDir, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
  if (!phaseDone(checkpoint, "report")) {
    completeBenchPhase(run.runDir, checkpoint, "report");
  }
  return report;
}

async function retrieveQuestion(
  vault: string,
  config: ReturnType<typeof resolveSearchConfig>,
  question: BenchQuestion,
  pinnedKnobs: BenchPinnedKnobs,
  recallFloor?: number,
): Promise<RetrievedQuestion> {
  const startedAt = Date.now();
  if (RETRIEVAL_CATEGORIES.has(question.category)) {
    // Overlapped lane accounting (recall-lane-latency): the per-call sink
    // rides this question's search and the figure is persisted only when
    // the semantic lane attempted the embed, so a keyword-only suite
    // keeps its report bytes.
    let laneTiming: SearchLaneTiming | undefined;
    const outcome = await search(config, {
      query: question.query ?? "",
      limit: question.top_k ?? 5,
      transportReach: BENCH_REACH,
      laneTiming: (timing) => {
        laneTiming = timing;
      },
    });
    return {
      id: question.id,
      latency_ms: Date.now() - startedAt,
      ...(laneTiming?.attempted === true ? { overlapped_ms: laneTiming.overlappedMs } : {}),
      paths: outcome.results.map((result) => result.path),
    };
  }
  if (question.category === "session_handoff") {
    const turns = loadNormalizedContinuityRecords(vault, {
      kind: "session_turn",
      ...(question.session_id !== undefined ? { sessionId: question.session_id } : {}),
    });
    return {
      id: question.id,
      latency_ms: Date.now() - startedAt,
      turn_texts: turns.map((turn) =>
        typeof turn.payload["text"] === "string" ? turn.payload["text"] : "",
      ),
    };
  }
  if (question.category === "proactive_recall") {
    // The structural pre-tier (pr-250 review, M9): the transports run
    // `evaluateSurfacingGate` ahead of the decision core, so a duplicate,
    // slash-command, shell-command or empty prompt never reaches
    // retrieval. The bench reproduces that composition exactly whenever
    // the fixture records the prompt this question follows: a structural
    // refusal is persisted as an honest abstention - the gate's own
    // reason, nothing measured, zero injected cost - and the decision
    // core never runs for it.
    const previousPrompt = question.previous_prompt;
    if (previousPrompt !== undefined && previousPrompt.length > 0) {
      const gate = evaluateSurfacingGate({ prompt: question.prompt ?? "", previousPrompt });
      if (!gate.retrieve) {
        return {
          id: question.id,
          latency_ms: Date.now() - startedAt,
          recall_decision: Object.freeze({
            kind: "abstain",
            reason: "structural",
            structuralReason: gate.reason,
            topScore: 0,
            matchQuality: null,
          }),
          context_chars: 0,
          injected_tokens: injectedTokens(""),
        };
      }
    }
    // The shipped decision core, driven directly over the fixture vault.
    // The retriever is a PARAMETER of that function, which is the only
    // reason this is measurable offline at all. The admission floor is a
    // parameter too (t_107a3249): the replay suite sweeps it to measure
    // gate calibration; absent keeps the shipped default.
    const decision = await decideRecallInject(
      question.prompt ?? "",
      benchRecallRetriever(config, question.top_k ?? RECALL_INJECT_MAX_NOTES),
      recallFloor === undefined ? {} : { confidenceFloor: recallFloor },
    );
    // An abstain injects nothing, and zero is the honest cost of nothing -
    // not an absent measurement.
    const brief = decision.kind === "inject" ? decision.brief : "";
    return {
      id: question.id,
      latency_ms: Date.now() - startedAt,
      recall_decision: decision,
      context_chars: brief.length,
      injected_tokens: injectedTokens(brief),
    };
  }
  if (question.category === STANDING_INSTRUCTION_CATEGORY) {
    return retrieveStandingInstruction(vault, question, startedAt, pinnedKnobs);
  }
  if (question.category === "write_fidelity") {
    return retrieveWriteFidelity(vault, question, startedAt);
  }
  // budget and source_isolation both deliver a context pack; the
  // isolation question additionally requests an owner scope, which is
  // only honoured when the fixture's `Brain/_brain.yaml` sets
  // `integrity.owner_scope_delivery: fail` - the shipped default is
  // `off`, under which no scope is evaluated and the question is vacuous.
  const pack = packContext(vault, {
    maxTokens: question.max_tokens ?? 1000,
    ...(question.max_total_chars !== undefined ? { maxTotalChars: question.max_total_chars } : {}),
    ...(question.agent_scope !== undefined ? { agentScope: question.agent_scope } : {}),
  });
  const body = pack.items.map((item) => item.body).join("\n");
  return {
    id: question.id,
    latency_ms: Date.now() - startedAt,
    item_ids: pack.items.map((item) => item.id),
    context_chars: body.length,
    injected_tokens: injectedTokens(body),
  };
}

/**
 * Run the shipped write path twice over the same intake and record what
 * survived.
 *
 * Twice on purpose: the second pass is the dedup measurement. Extraction
 * keys on (session, turn range, label, content hash), so a faithful write
 * path must return the SAME record ids the second time and append
 * nothing. A path that re-appends looks identical on a single run.
 *
 * No model is called, and none could be: the server never calls one on
 * any write path. What the fixture supplies is the structured intake that
 * stands in for an agent's distillation.
 */
function retrieveWriteFidelity(
  vault: string,
  question: BenchQuestion,
  startedAt: number,
): RetrievedQuestion {
  const input = {
    sessionId: `bench-${question.id}`,
    turnStart: "turn-1",
    turnEnd: "turn-2",
    text: question.intake_text ?? "",
    createdAt: BENCH_WRITE_FIDELITY_INSTANT,
  };
  const first = extractPreCompactRecords(vault, input);
  const second = extractPreCompactRecords(vault, input);
  const provenance = new Set<string>(first.records.flatMap(provenanceKeys));
  return {
    id: question.id,
    latency_ms: Date.now() - startedAt,
    labels: first.records.map((record) => String(record.payload["extract_type"] ?? "")),
    // Present on EVERY record, not on any: a provenance field that
    // survives only sometimes has not survived.
    provenance_keys: [...provenance].filter((key) =>
      first.records.every((record) => provenanceKeys(record).includes(key)),
    ),
    dedup_stable:
      first.records.length > 0 &&
      JSON.stringify(first.records.map((r) => r.id)) ===
        JSON.stringify(second.records.map((r) => r.id)),
  };
}

/**
 * Every provenance name one extracted record carries.
 *
 * Both levels count: the record envelope carries `id`, `kind`,
 * `createdAt` and `sourceRefs`, and the payload carries the
 * session/turn/hash keys. A fixture asks for either by name.
 */
function provenanceKeys(record: ContinuityRecord): ReadonlyArray<string> {
  return [...Object.keys(record), ...Object.keys(record.payload)];
}

/**
 * Run the shipped standing_query surfacing lane over the fixture's
 * attention flows (standing_instruction, recall-lane-latency).
 *
 * The lane's arm follows the run's knob pin: absent or true, the facet
 * option is on - the category exists to measure the provenance join.
 * Pinned false, the plain lane runs and emits the legacy strings, so the
 * category's facet assertions fail by name: the off arm of the A/B the
 * design promises, which is what makes the pin change what a run
 * measures instead of only naming it. Every flow that declares the
 * `standing_query` action is evaluated and its surfaced items and
 * resolved facet keys persisted; a preference the lane surfaces but
 * whose join does not resolve contributes its plain string and no facet,
 * exactly as the option contract renders it.
 */
function retrieveStandingInstruction(
  vault: string,
  question: BenchQuestion,
  startedAt: number,
  pinnedKnobs: BenchPinnedKnobs,
): RetrievedQuestion {
  // Absent keeps the lane on, so existing runs and the default pin
  // measure the facet lane exactly as before the pin was wired.
  const facetsOn = pinnedKnobs.standingInstructionFacets !== false;
  const items: string[] = [];
  const facetIds: string[] = [];
  for (const flow of listAttentionFlows(vault)) {
    if (!flow.actions.includes("standing_query")) continue;
    const evaluation = facetsOn
      ? evaluateAttentionFlow(vault, flow.id, { standingInstructionFacets: true })
      : evaluateAttentionFlow(vault, flow.id, { standingInstructionFacets: false });
    for (const section of evaluation.sections) {
      if (section.action !== "standing_query") continue;
      items.push(...section.items);
      for (const facet of section.facets ?? []) facetIds.push(facet.key);
    }
  }
  return {
    id: question.id,
    latency_ms: Date.now() - startedAt,
    standing_items: items,
    facet_ids: facetIds,
  };
}

/**
 * Fixed authoring instant for write-fidelity extraction. Extraction
 * defaults `createdAt` to wall clock, which would make the record ids -
 * and therefore the whole report - differ on every run.
 */
const BENCH_WRITE_FIDELITY_INSTANT = "2026-06-01T00:00:00.000Z";

function readRetrieved(runDir: string, questionId: string): RetrievedQuestion {
  const path = join(benchResultsDir(runDir), `${questionId}.json`);
  if (!existsSync(path)) {
    throw new Error(`bench run is missing retrieve results for question ${questionId}`);
  }
  return JSON.parse(readFileSync(path, "utf8")) as RetrievedQuestion;
}

function evaluateQuestion(
  question: BenchQuestion,
  retrieved: RetrievedQuestion,
): BenchQuestionResult {
  const base = {
    id: question.id,
    category: question.category,
    latency_ms: retrieved.latency_ms,
    ...(retrieved.overlapped_ms !== undefined ? { overlapped_ms: retrieved.overlapped_ms } : {}),
    ...(retrieved.context_chars !== undefined ? { context_chars: retrieved.context_chars } : {}),
    ...(retrieved.injected_tokens !== undefined
      ? { injected_tokens: retrieved.injected_tokens }
      : {}),
  };
  if (RETRIEVAL_CATEGORIES.has(question.category)) {
    const paths = retrieved.paths ?? [];
    const missing = (question.expected_paths ?? []).filter((path) => !paths.includes(path));
    if (missing.length > 0) {
      return Object.freeze({
        ...base,
        pass: false,
        failure: `missing expected: ${missing.join(", ")}`,
      });
    }
    const bestExpected = Math.min(
      ...(question.expected_paths ?? []).map((path) => paths.indexOf(path)),
    );
    for (const stale of question.not_expected_above ?? []) {
      const rank = paths.indexOf(stale);
      if (rank !== -1 && rank < bestExpected) {
        return Object.freeze({
          ...base,
          pass: false,
          failure: `stale path ranked above expected: ${stale}`,
        });
      }
    }
    return Object.freeze({ ...base, pass: true });
  }
  if (question.category === "proactive_recall") {
    return evaluateProactiveRecall(question, retrieved, base);
  }
  if (question.category === "write_fidelity") {
    return evaluateWriteFidelity(question, retrieved, base);
  }
  if (question.category === "source_isolation") {
    return evaluateSourceIsolation(question, retrieved, base);
  }
  if (question.category === "session_handoff") {
    const texts = retrieved.turn_texts ?? [];
    if (question.expected_turns !== undefined && texts.length < question.expected_turns) {
      return Object.freeze({
        ...base,
        pass: false,
        failure: `expected ${question.expected_turns} turns, found ${texts.length}`,
      });
    }
    if (
      question.expected_text !== undefined &&
      !texts.some((text) => text.includes(question.expected_text ?? ""))
    ) {
      return Object.freeze({
        ...base,
        pass: false,
        failure: `expected text not found in session turns: ${question.expected_text}`,
      });
    }
    return Object.freeze({ ...base, pass: true });
  }
  if (question.category === STANDING_INSTRUCTION_CATEGORY) {
    return evaluateStandingInstruction(question, retrieved, base);
  }
  // budget
  const ids = retrieved.item_ids ?? [];
  const missingIds = (question.expected_ids ?? []).filter((id) => !ids.includes(id));
  if (missingIds.length > 0) {
    return Object.freeze({
      ...base,
      pass: false,
      failure: `expected evidence missing from budgeted pack: ${missingIds.join(", ")}`,
    });
  }
  return Object.freeze({ ...base, pass: true });
}

/** Shared prefix of every evaluated question, built once by the caller. */
type ResultBase = Pick<
  BenchQuestionResult,
  "id" | "category" | "latency_ms" | "context_chars" | "injected_tokens"
>;

function evaluateProactiveRecall(
  question: BenchQuestion,
  retrieved: RetrievedQuestion,
  base: ResultBase,
): BenchQuestionResult {
  const decision = retrieved.recall_decision;
  if (decision === undefined) {
    return Object.freeze({
      ...base,
      pass: false,
      failure: "no recall decision was persisted for this question",
    });
  }
  const failure = classifyRecallDecision(question.expect_inject === true, decision);
  if (failure === null) return Object.freeze({ ...base, pass: true });
  return Object.freeze({
    ...base,
    pass: false,
    recall_failure: failure,
    failure: `${failure}: decision was ${describeDecision(decision)}`,
  });
}

/** One decision as a diffable one-liner. Classifications only, no prose. */
function describeDecision(decision: RecallInjectDecision): string {
  if (decision.kind === "abstain") {
    return decision.structuralReason !== undefined
      ? `abstain(${decision.reason}:${decision.structuralReason})`
      : `abstain(${decision.reason})`;
  }
  if (decision.kind === "error") return `error(${decision.fault})`;
  return `inject(${decision.noteCount} notes)`;
}

function evaluateWriteFidelity(
  question: BenchQuestion,
  retrieved: RetrievedQuestion,
  base: ResultBase,
): BenchQuestionResult {
  const labels = retrieved.labels ?? [];
  const expected = question.expected_labels ?? [];
  if (JSON.stringify(labels) !== JSON.stringify(expected)) {
    return Object.freeze({
      ...base,
      pass: false,
      failure: `labels [${labels.join(", ")}] do not match expected [${expected.join(", ")}]`,
    });
  }
  const present = retrieved.provenance_keys ?? [];
  const missing = (question.expected_provenance ?? []).filter((key) => !present.includes(key));
  if (missing.length > 0) {
    return Object.freeze({
      ...base,
      pass: false,
      failure: `provenance lost on the write path: ${missing.join(", ")}`,
    });
  }
  if (retrieved.dedup_stable !== true) {
    return Object.freeze({
      ...base,
      pass: false,
      failure: "a second identical intake did not resolve to the same records",
    });
  }
  return Object.freeze({ ...base, pass: true });
}

/**
 * Assert the standing-instruction facet lane delivered what the fixture
 * named (standing_instruction, recall-lane-latency).
 *
 * `expected_ids` are facet keys - `pref:<preference id>`, so the
 * preference id itself is what is asserted delivered. `expected_text`
 * is the full earliest-cited annotation that must CLOSE the first named
 * preference's item: the assertion finds the item for `expected_ids[0]`
 * and compares the annotation exactly, closing bracket included, so a
 * near-miss origin (`...#t1`) cannot satisfy an expectation about
 * another (`...#t10`) and an origin cannot pass by sitting on a
 * different preference's item. A fixture states the kind word ("earliest
 * cited turn ..."/"earliest cited session ...") so a session-annotated
 * origin cannot satisfy a turn expectation. A preference the lane
 * surfaced but could not resolve is a named failure, never an implied
 * provenance.
 */
function evaluateStandingInstruction(
  question: BenchQuestion,
  retrieved: RetrievedQuestion,
  base: ResultBase,
): BenchQuestionResult {
  const facetIds = retrieved.facet_ids ?? [];
  const missingFacets = (question.expected_ids ?? []).filter((id) => !facetIds.includes(id));
  if (missingFacets.length > 0) {
    return Object.freeze({
      ...base,
      pass: false,
      failure: `expected facet missing from the surfaced standing instructions: ${missingFacets.join(", ")}`,
    });
  }
  const expectedOrigin = question.expected_text;
  if (expectedOrigin !== undefined) {
    // Tie the annotation to the expected preference's own item: the
    // first expected facet key names the preference, and its item is the
    // `id (scope)` string the lane renders for it.
    const expectedKey = question.expected_ids?.[0];
    const expectedId =
      expectedKey !== undefined && expectedKey.startsWith("pref:")
        ? expectedKey.slice("pref:".length)
        : undefined;
    const item =
      expectedId === undefined
        ? undefined
        : (retrieved.standing_items ?? []).find((candidate) =>
            candidate.startsWith(`${expectedId} (`),
          );
    if (item === undefined || !item.endsWith(expectedOrigin)) {
      return Object.freeze({
        ...base,
        pass: false,
        failure: `expected origin annotation not found on the item for ${expectedKey ?? "(no facet named)"}: ${expectedOrigin}`,
      });
    }
  }
  return Object.freeze({ ...base, pass: true });
}

function evaluateSourceIsolation(
  question: BenchQuestion,
  retrieved: RetrievedQuestion,
  base: ResultBase,
): BenchQuestionResult {
  const ids = retrieved.item_ids ?? [];
  const leaked = isolationViolations(ids, question.forbidden_ids ?? []);
  if (leaked.length > 0) {
    return Object.freeze({
      ...base,
      pass: false,
      isolation_violations: leaked.length,
      failure: `delivered memory owned by another agent: ${leaked.join(", ")}`,
    });
  }
  // The converse invariant: a scope must isolate without narrowing what
  // the caller legitimately owns. Without this a pack that returned
  // NOTHING would score a perfect isolation result.
  const missing = (question.expected_ids ?? []).filter((id) => !ids.includes(id));
  if (missing.length > 0) {
    return Object.freeze({
      ...base,
      pass: false,
      isolation_violations: 0,
      failure: `scoping withheld the caller's own memory: ${missing.join(", ")}`,
    });
  }
  return Object.freeze({ ...base, pass: true, isolation_violations: 0 });
}

function buildReport(
  run: BenchRunHandle,
  fixture: BenchFixture,
  results: ReadonlyArray<BenchQuestionResult>,
  judge: ReturnType<typeof runJudge>,
  embeddings?: BenchEmbeddingReceipt,
  longmemeval?: BenchLongMemEvalFamily,
): BenchReport {
  // ONE copy of the counting rule - the guards' `countQuestionResults`,
  // which the pre-write trace cross-check (T7) runs against the same
  // results - so the published counts and their validation cannot drift.
  const { passed, total, byCategory } = countQuestionResults(results);
  const latencies = results.map((result) => result.latency_ms);
  // The latency family's overlap figure (recall-lane-latency): averaged
  // over the questions that attempted the semantic lane, present only
  // when at least one did - a keyword-only report keeps its bytes.
  const overlappedMsValues = results
    .map((result) => result.overlapped_ms)
    .filter((value): value is number => value !== undefined);
  const avgChars = mean(results.map((result) => result.context_chars));
  // The suite's ONE token number, from the ONE estimator. The inline
  // `ceil(avg_chars / 4)` that stood here was a sixth formula in a repo
  // that already had five, and it did not even import the shared helper -
  // so it disagreed with every token figure the Brain reports. Averaged
  // over the same population as `avg_chars` so the two are comparable.
  const avgInjectedTokens = mean(results.map((result) => result.injected_tokens));
  const recallFailures: ReadonlyArray<RecallFailure | null> = results
    .filter((result) => result.category === "proactive_recall")
    .map((result) => result.recall_failure ?? null);
  const proactive = scoreProactiveRecall(recallFailures);
  const isolationTotal = results.reduce(
    (sum, result) => sum + (result.isolation_violations ?? 0),
    0,
  );
  return Object.freeze({
    schema: BENCH_REPORT_SCHEMA,
    run_id: run.runId,
    fixture: fixture.name,
    fixture_hash: fixtureHash(fixture),
    created_at: run.checkpoint.created_at,
    quality: Object.freeze({
      passed,
      total,
      pass_rate: total > 0 ? round3(passed / total) : 0,
      by_category: Object.freeze(
        Object.fromEntries(Object.entries(byCategory).toSorted(([a], [b]) => (a < b ? -1 : 1))),
      ),
    }),
    latency_ms: Object.freeze({
      avg:
        latencies.length > 0
          ? Math.round(latencies.reduce((sum, value) => sum + value, 0) / latencies.length)
          : 0,
      max: latencies.length > 0 ? Math.max(...latencies) : 0,
      ...(overlappedMsValues.length > 0
        ? {
            overlapped_ms: Math.round(
              overlappedMsValues.reduce((sum, value) => sum + value, 0) / overlappedMsValues.length,
            ),
          }
        : {}),
    }),
    context_cost: Object.freeze({
      avg_chars: avgChars,
      avg_injected_tokens: avgInjectedTokens,
    }),
    failure_modes: Object.freeze({
      know_to_ask_failure_rate: proactive.know_to_ask_failure_rate,
      false_fire_rate: proactive.false_fire_rate,
      source_isolation_violations: isolationTotal,
    }),
    ...(embeddings !== undefined ? { embeddings: Object.freeze({ ...embeddings }) } : {}),
    ...(longmemeval !== undefined ? { longmemeval } : {}),
    judge: Object.freeze({
      status: judge.status,
      ...(judge.detail !== undefined ? { detail: judge.detail } : {}),
    }),
    questions: Object.freeze(
      results.map((result) =>
        Object.freeze({
          ...result,
          ...(judge.verdicts && result.id in judge.verdicts
            ? { judge_pass: judge.verdicts[result.id] }
            : {}),
        }),
      ),
    ),
  });
}

function round3(value: number): number {
  return Math.round(value * 1000) / 1000;
}

/** Mean over the values that are present, rounded; 0 when none are. */
function mean(values: ReadonlyArray<number | undefined>): number {
  const present = values.filter((value): value is number => typeof value === "number");
  if (present.length === 0) return 0;
  return Math.round(present.reduce((sum, value) => sum + value, 0) / present.length);
}