Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
31.9 KB
Raw Blame
/**
 * Bench phase pipeline (Memory Observability Suite, t_882c396a):
 * ingest -> index -> retrieve -> evaluate -> report over a disposable
 * fixture vault. Deterministic, no network; quality / latency /
 * context cost stay separate numbers; resume skips completed phases;
 * a stale-fact fixture catches superseded-recall regressions.
 */

import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

import {
  benchEmbeddingSignature,
  benchVectorCacheKey,
} from "../../../src/core/bench/embedding-cache.ts";
import { parseBenchFixture } from "../../../src/core/bench/fixture.ts";
import { countQuestionResults } from "../../../src/core/bench/guards.ts";
import { loadLongMemEvalFixture } from "../../../src/core/bench/longmemeval.ts";
import { FAKE_PROVIDER_KEY } from "../../helpers/fake-credentials.ts";
import { benchSearchConfig, runMemoryBench } from "../../../src/core/bench/phases.ts";
import { completeBenchPhase, createBenchRun } from "../../../src/core/bench/run-store.ts";
import {
  BENCH_REPORT_SCHEMA,
  DEFAULT_BENCH_KNOB_PIN,
  type BenchPinnedKnobs,
} from "../../../src/core/bench/types.ts";
import { cmdBrainBench } from "../../../src/cli/brain/verbs/bench.ts";
import { LOCAL_DEFAULT_DIMENSION } from "../../../src/core/search/embeddings/local-provider.ts";
import { LOCAL_EMBEDDING_MODEL } from "../../../src/core/search/embeddings/signature.ts";

let runsDir: string;

beforeEach(() => {
  runsDir = mkdtempSync(join(tmpdir(), "o2b-bench-phases-"));
});

afterEach(() => {
  rmSync(runsDir, { recursive: true, force: true });
});

const FIXTURE = parseBenchFixture({
  name: "phase-suite",
  notes: [
    {
      path: "Brain/notes/coffee.md",
      body: "# Coffee preference\n\nCoffee preference of the operator: flat white every morning. The coffee preference is stable.\n",
    },
    {
      path: "Brain/notes/coffee-old.md",
      body: "# Superseded note\n\nOld coffee preference: espresso. Replaced by the flat white note.\n",
    },
    {
      path: "Brain/notes/deploy-a.md",
      body: "# Deploy step A\n\nDeployment requires the staging gate to pass first.\n",
    },
    {
      path: "Brain/notes/deploy-b.md",
      body: "# Deploy step B\n\nDeployment finishes with the production smoke checklist.\n",
    },
    {
      path: "Brain/preferences/pref-tabs.md",
      body: "---\nid: pref-tabs\ntopic: tabs\nstatus: confirmed\nprinciple: Use tabs for indentation in Go files\ncreated_at: 2026-05-01T10:00:00\n---\n\nUse tabs for indentation in Go files\n",
    },
  ],
  continuity: [
    {
      kind: "session_turn",
      created_at: "2026-06-01T10:00:00.000Z",
      payload: { session_id: "s-1", turn_id: "t-1", role: "user", text: "handoff: flat white" },
    },
    {
      kind: "session_turn",
      created_at: "2026-06-01T10:01:00.000Z",
      payload: { session_id: "s-1", turn_id: "t-2", role: "assistant", text: "noted" },
    },
  ],
  questions: [
    {
      id: "q-single",
      category: "single_hop",
      query: "flat white",
      top_k: 5,
      expected_paths: ["Brain/notes/coffee.md"],
    },
    {
      id: "q-stale",
      category: "temporal",
      query: "coffee preference",
      top_k: 5,
      expected_paths: ["Brain/notes/coffee.md"],
      not_expected_above: ["Brain/notes/coffee-old.md"],
    },
    {
      id: "q-multi",
      category: "multi_evidence",
      query: "deployment",
      top_k: 5,
      expected_paths: ["Brain/notes/deploy-a.md", "Brain/notes/deploy-b.md"],
    },
    {
      id: "q-handoff",
      category: "session_handoff",
      session_id: "s-1",
      expected_turns: 2,
      expected_text: "flat white",
    },
    {
      id: "q-budget",
      category: "budget",
      expected_ids: ["pref-tabs"],
      max_tokens: 500,
    },
  ],
});

/**
 * One minimal LongMemEval dataset entry: a single-hop question over one
 * one-turn haystack session, shaped the way the suite parser reads it.
 */
function longMemEvalEntry(id: string) {
  return {
    q_id: id,
    question_type: "single-hop",
    question: `what does note ${id} say`,
    haystack_session_id: [`s-${id}`],
    haystack_session: [[{ role: "user", content: `note ${id} body` }]],
    haystack_dates: ["2023-05-20"],
    answer_session_ids: [`s-${id}`],
  };
}

describe("runMemoryBench", () => {
  test("the default bench config pins the semantic lane off", () => {
    // The resolved config is where network-freedom is real: the pin must
    // hold whatever the surrounding environment configures.
    const config = benchSearchConfig(runsDir);
    expect(config.semantic.enabled).toBe(false);
  });

  test("full pipeline produces a diffable report with separate metric families", async () => {
    const report = await runMemoryBench({ fixture: FIXTURE, runsDir });
    expect(report.schema).toBe(BENCH_REPORT_SCHEMA);
    expect(report.fixture).toBe("phase-suite");
    expect(report.quality.total).toBe(5);
    expect(report.quality.passed).toBe(5);
    expect(report.quality.pass_rate).toBe(1);
    expect(report.quality.by_category["single_hop"]).toEqual({ passed: 1, total: 1 });
    // Metric families stay separate - never one collapsed score.
    expect(report.latency_ms.avg).toBeGreaterThanOrEqual(0);
    expect(report.latency_ms.max).toBeGreaterThanOrEqual(report.latency_ms.avg);
    expect(report.context_cost.avg_chars).toBeGreaterThan(0);
    expect(report.context_cost.avg_injected_tokens).toBeGreaterThan(0);
    expect(report.judge.status).toBe("skipped");
    // No suite is named, so no suite report family may appear: the
    // default path stays byte-identical.
    expect(report.longmemeval).toBeUndefined();
    // Stable question order by id for diffability.
    expect(report.questions.map((q) => q.id)).toEqual(report.questions.map((q) => q.id).toSorted());
    // report.json lands in the run directory.
    const onDisk = JSON.parse(
      readFileSync(join(runsDir, report.run_id, "report.json"), "utf8"),
    ) as {
      schema: string;
    };
    expect(onDisk.schema).toBe(BENCH_REPORT_SCHEMA);
  }, 30_000);

  test("a stale-fact regression is caught: superseded note ranking above fails the question", async () => {
    const regression = parseBenchFixture({
      name: "stale-regression",
      notes: [
        {
          path: "Brain/notes/new.md",
          body: "# Current\n\nThe coffee preference today is flat white, replacing every older note about it.\n",
        },
        {
          path: "Brain/notes/old.md",
          body: "# Old\n\nCoffee preference coffee preference coffee preference: espresso coffee preference, morning coffee preference.\n",
        },
      ],
      questions: [
        {
          id: "q-stale",
          category: "temporal",
          query: "coffee preference",
          top_k: 5,
          expected_paths: ["Brain/notes/new.md"],
          not_expected_above: ["Brain/notes/old.md"],
        },
      ],
    });
    const report = await runMemoryBench({ fixture: regression, runsDir });
    expect(report.quality.passed).toBe(0);
    const q = report.questions.find((entry) => entry.id === "q-stale")!;
    expect(q.pass).toBe(false);
    expect(q.failure).toContain("Brain/notes/old.md");
  }, 30_000);

  test("resume by run id skips completed phases and reuses stored results", async () => {
    const first = await runMemoryBench({ fixture: FIXTURE, runsDir });
    // Remove the disposable vault: a resumed run must NOT need to
    // re-ingest or re-search - evaluation reuses stored results.
    rmSync(join(runsDir, first.run_id, "vault"), { recursive: true, force: true });
    const resumed = await runMemoryBench({
      fixture: FIXTURE,
      runsDir,
      resume: first.run_id,
    });
    expect(resumed.run_id).toBe(first.run_id);
    expect(resumed.quality).toEqual(first.quality);
    expect(existsSync(join(runsDir, first.run_id, "vault"))).toBe(false);
  }, 30_000);

  test("resume with a changed fixture is refused as a named fixture mismatch", async () => {
    const first = await runMemoryBench({ fixture: FIXTURE, runsDir });
    // Change the fixture CONTENT while keeping every referenced note path:
    // the preflight's cross-reference guard (P7) must stay out of the way
    // so the fixture mismatch is the refusal this pin is about.
    const raw = JSON.parse(JSON.stringify(FIXTURE)) as {
      notes: Array<{ path: string; body: string }>;
    };
    raw.notes[0]!.body = "# Coffee preference\n\nA different body under the same path.\n";
    const changed = parseBenchFixture(raw);
    await expect(
      runMemoryBench({ fixture: changed, runsDir, resume: first.run_id }),
    ).rejects.toThrow(/refused \(checkpoint_fixture_mismatch\).*same fixture, seed and cap/s);
  }, 30_000);

  test("a LongMemEval resume under a different sample seed is a named fixture mismatch", async () => {
    // The seeded sampler selects a different subset per seed, so the
    // fixture a different `--sample-seed` loads is a DIFFERENT fixture
    // under the same dataset. The resume must be refused by name with the
    // seed-aware repair - never misread as a tampered run directory and
    // never as the raw load error the verb does not render.
    const datasetPath = join(runsDir, "dataset.json");
    writeFileSync(
      datasetPath,
      JSON.stringify({
        name: "seeded",
        questions: [
          longMemEvalEntry("a"),
          longMemEvalEntry("b"),
          longMemEvalEntry("c"),
          longMemEvalEntry("d"),
        ],
      }),
    );
    const seedA = loadLongMemEvalFixture(datasetPath, { sample: { seed: "seed-a", cap: 2 } });
    const seedB = loadLongMemEvalFixture(datasetPath, { sample: { seed: "seed-b", cap: 2 } });
    // The seeds select different questions, so the resumed fixture's
    // CONTENT differs - not just the sampling options.
    expect(seedA.fixture.questions.map((q) => q.id)).not.toEqual(
      seedB.fixture.questions.map((q) => q.id),
    );
    const first = await runMemoryBench({ fixture: seedA.fixture, runsDir });
    await expect(
      runMemoryBench({ fixture: seedB.fixture, runsDir, resume: first.run_id }),
    ).rejects.toThrow(/refused \(checkpoint_fixture_mismatch\)/);
  }, 30_000);

  test("the default knob pin equals today's hardcoded semantic-off override", () => {
    expect(DEFAULT_BENCH_KNOB_PIN).toEqual({ semantic: { enabled: false } });
  });

  test("pinned knobs are applied last and win over env and config", () => {
    const pin: BenchPinnedKnobs = {
      keywordWeight: 0.6,
      semantic: { enabled: false, provider: "local" },
    };
    const config = benchSearchConfig(runsDir, pin);
    expect(config.keywordWeight).toBe(0.6);
    expect(config.semantic.enabled).toBe(false);
    expect(config.semantic.provider).toBe("local");
  });

  test("two runs with different knob pins get different run ids", async () => {
    const first = await runMemoryBench({
      fixture: FIXTURE,
      runsDir,
      pinnedKnobs: { semantic: { enabled: false }, keywordWeight: 0.6 },
    });
    const second = await runMemoryBench({
      fixture: FIXTURE,
      runsDir,
      pinnedKnobs: { semantic: { enabled: false }, keywordWeight: 0.5 },
    });
    expect(first.run_id).not.toBe(second.run_id);
  }, 30_000);

  test("resume with a changed knob pin fails fast on the identity guard", async () => {
    const first = await runMemoryBench({ fixture: FIXTURE, runsDir });
    await expect(
      runMemoryBench({
        fixture: FIXTURE,
        runsDir,
        resume: first.run_id,
        pinnedKnobs: { semantic: { enabled: false }, keywordWeight: 0.6 },
      }),
    ).rejects.toThrow(/run identity mismatch/);
  }, 30_000);

  test("resume with the same explicit pin reuses the run", async () => {
    const pin: BenchPinnedKnobs = { semantic: { enabled: false }, keywordWeight: 0.6 };
    const first = await runMemoryBench({ fixture: FIXTURE, runsDir, pinnedKnobs: pin });
    const resumed = await runMemoryBench({
      fixture: FIXTURE,
      runsDir,
      resume: first.run_id,
      pinnedKnobs: pin,
    });
    expect(resumed.run_id).toBe(first.run_id);
    expect(resumed.quality).toEqual(first.quality);
  }, 30_000);

  test("an out-of-range recallFloor is refused by name before any run starts", async () => {
    // Each of these used to finish a FULL run: the floor routes into the
    // shared surfacing calculator, whose own contract refuses out-of-range
    // values per question, so the report came back complete with a
    // retrievalError on every proactive decision - paid after ingest and
    // index, instead of one named error up front (pr-250 review, L9).
    for (const floor of [-1, 1.5, Number.POSITIVE_INFINITY, Number.NaN]) {
      await expect(
        runMemoryBench({ fixture: FIXTURE, runsDir, recallFloor: floor }),
      ).rejects.toThrow(/recallFloor .* is outside \[0, 1\]/);
    }
    // Refused before any run state exists.
    expect(readdirSync(runsDir)).toHaveLength(0);
    // The interval's ends are legal calibration points: zero is the
    // maximally permissive sweep arm, one admits only a perfect match.
    await expect(
      runMemoryBench({ fixture: FIXTURE, runsDir, recallFloor: 1 }),
    ).resolves.toBeDefined();
  }, 60_000);

  test("an invalid sampling request is refused by name before any run state exists", async () => {
    // The sampling options are part of the run identity, so a malformed
    // one is refused up front rather than after a full run pays for a
    // configuration that never existed.
    for (const sample of [
      { seed: "seed-1", cap: 0 },
      { seed: "seed-1", cap: -2 },
      { seed: "seed-1", cap: 1.5 },
      { seed: "", cap: 2 },
      { seed: "   ", cap: 2 },
    ]) {
      await expect(runMemoryBench({ fixture: FIXTURE, runsDir, sample })).rejects.toThrow(
        /bench run refused/,
      );
    }
    expect(readdirSync(runsDir)).toHaveLength(0);
  }, 30_000);

  test("sampling options fold into the run identity end to end", async () => {
    // Cap 5 sits above the five-question fixture's population, so both
    // runs would materialize the identical fixture - the OPTIONS alone
    // must split the run ids.
    const first = await runMemoryBench({
      fixture: FIXTURE,
      runsDir,
      sample: { seed: "seed-1", cap: 5 },
    });
    const second = await runMemoryBench({
      fixture: FIXTURE,
      runsDir,
      sample: { seed: "seed-2", cap: 5 },
    });
    expect(first.run_id).not.toBe(second.run_id);
    // Resuming with the same sampling options reuses the run.
    const resumed = await runMemoryBench({
      fixture: FIXTURE,
      runsDir,
      resume: first.run_id,
      sample: { seed: "seed-1", cap: 5 },
    });
    expect(resumed.run_id).toBe(first.run_id);
    expect(readdirSync(runsDir)).toHaveLength(2);
  }, 30_000);

  test("resume with changed sampling options fails fast on the identity guard", async () => {
    const first = await runMemoryBench({
      fixture: FIXTURE,
      runsDir,
      sample: { seed: "seed-1", cap: 5 },
    });
    await expect(
      runMemoryBench({
        fixture: FIXTURE,
        runsDir,
        resume: first.run_id,
        sample: { seed: "seed-1", cap: 3 },
      }),
    ).rejects.toThrow(/run identity mismatch/);
    // And so does resuming a sampled run with no sampling options.
    await expect(
      runMemoryBench({ fixture: FIXTURE, runsDir, resume: first.run_id }),
    ).rejects.toThrow(/run identity mismatch/);
  }, 30_000);

  test("a passing run publishes exactly the report shape it always had", async () => {
    const report = await runMemoryBench({ fixture: FIXTURE, runsDir });
    // The guards add no field: the serialized top-level key order IS the
    // published shape a baseline diffs.
    expect(Object.keys(report)).toEqual([
      "schema",
      "run_id",
      "fixture",
      "fixture_hash",
      "created_at",
      "quality",
      "latency_ms",
      "context_cost",
      "failure_modes",
      "judge",
      "questions",
    ]);
    // The quality family is the shared aggregation, by construction.
    const counts = countQuestionResults(report.questions);
    expect(report.quality.passed).toBe(counts.passed);
    expect(report.quality.total).toBe(counts.total);
    expect(report.quality.by_category).toEqual(counts.byCategory);
    // The published file equals the returned report, JSON for JSON.
    const onDisk = JSON.parse(
      readFileSync(join(runsDir, report.run_id, "report.json"), "utf8"),
    ) as unknown;
    expect(onDisk).toEqual(JSON.parse(JSON.stringify(report)));
  }, 30_000);

  test("a passing run writes zero stdout from the guards", async () => {
    const writes: string[] = [];
    const original = process.stdout.write;
    process.stdout.write = ((chunk?: unknown) => {
      writes.push(String(chunk));
      return true;
    }) as typeof process.stdout.write;
    try {
      await runMemoryBench({ fixture: FIXTURE, runsDir });
    } finally {
      process.stdout.write = original;
    }
    expect(writes).toEqual([]);
  }, 30_000);
});

/**
 * The integrity guards wired into the pipeline (t_d7ce8baf): refusals
 * fire BEFORE a run directory can exist (config/capability ordering),
 * a fresh-run collision preserves the prior run, a tampered run dir is
 * refused by name before any scoring, and the verb renders the refusal
 * contract `refused (<kind>): <detail>` on stderr with a nonzero exit.
 */
describe("the integrity guards wired in", () => {
  test("a traversal --resume id is refused before any run state exists", async () => {
    // Findings 30 + 31 end to end: the raw `--resume ..` value must be
    // refused by name by the preflight's run-id guard, never answered by
    // whatever the parent directory happens to hold, and no run state may
    // exist when it is.
    await expect(runMemoryBench({ fixture: FIXTURE, runsDir, resume: ".." })).rejects.toThrow(
      /all dots/,
    );
    expect(readdirSync(runsDir)).toEqual([]);
  });

  test("an env-garbage provider refusal leaves no run dir behind", async () => {
    const envKey = "OPEN_SECOND_BRAIN_EMBEDDING_PROVIDER";
    process.env[envKey] = "no-such-provider";
    try {
      await expect(runMemoryBench({ fixture: FIXTURE, runsDir })).rejects.toThrow(
        /embedding_provider must be/,
      );
    } finally {
      delete process.env[envKey];
    }
    expect(readdirSync(runsDir)).toEqual([]);
  }, 30_000);

  test("a capability-gate refusal leaves no run dir behind", async () => {
    await expect(
      runMemoryBench({
        fixture: FIXTURE,
        runsDir,
        pinnedKnobs: { semantic: { enabled: true, provider: "local" } },
      }),
    ).rejects.toThrow(/embeddings capability/);
    expect(readdirSync(runsDir)).toEqual([]);
  }, 30_000);

  test("a fresh-run collision is refused by name and the prior checkpoint survives", async () => {
    // The injected clock is the point: run ids are second-precision
    // timestamp + identity hex, so a wall-clock pair of back-to-back runs
    // collides only when both mint inside the same second - a race this
    // test lost about one full-suite run in fifty. One fixed instant for
    // both runs makes the collision a fact of the fixture, not of the
    // wall clock.
    const now = new Date("2026-06-03T12:00:00Z");
    const first = await runMemoryBench({ fixture: FIXTURE, runsDir, now });
    const checkpointBefore = readFileSync(join(runsDir, first.run_id, "checkpoint.json"), "utf8");
    await expect(runMemoryBench({ fixture: FIXTURE, runsDir, now })).rejects.toThrow(
      new RegExp(`refused \\(run_dir_exists\\).*--resume ${first.run_id}`),
    );
    expect(readFileSync(join(runsDir, first.run_id, "checkpoint.json"), "utf8")).toBe(
      checkpointBefore,
    );
    // The refused second run left no run dir of its own.
    expect(readdirSync(runsDir)).toEqual([first.run_id]);
    // And resuming the very same run keeps working.
    const resumed = await runMemoryBench({ fixture: FIXTURE, runsDir, resume: first.run_id, now });
    expect(resumed.run_id).toBe(first.run_id);
  }, 60_000);

  test("ingest-done with the vault deleted is refused before any scoring", async () => {
    // A partial run: only ingest is checkpointed, then the disposable
    // vault disappears. Retrieval against the emptied vault would score
    // an empty corpus and publish ordinary FAILs - refused by name.
    const run = createBenchRun(runsDir, FIXTURE, { now: new Date("2026-06-03T12:00:00Z") });
    completeBenchPhase(run.runDir, run.checkpoint, "ingest");
    rmSync(join(run.runDir, "vault"), { recursive: true, force: true });
    await expect(runMemoryBench({ fixture: FIXTURE, runsDir, resume: run.runId })).rejects.toThrow(
      /refused \(run_artifact_missing\)/,
    );
    // Refused before scoring: no results and no report exist.
    expect(existsSync(join(run.runDir, "results"))).toBe(false);
    expect(existsSync(join(run.runDir, "report.json"))).toBe(false);
  }, 30_000);

  test("retrieve-done with a result file deleted is refused naming the file", async () => {
    const first = await runMemoryBench({ fixture: FIXTURE, runsDir });
    rmSync(join(runsDir, first.run_id, "results", "q-multi.json"));
    await expect(
      runMemoryBench({ fixture: FIXTURE, runsDir, resume: first.run_id }),
    ).rejects.toThrow(/refused \(run_artifact_missing\).*q-multi\.json/s);
  }, 60_000);

  test("the bench verb renders the refusal contract and exits nonzero", async () => {
    // Deterministic refusal #1: the runs dir is a FILE, so the preflight's
    // usability probe refuses before anything is created.
    const fixturePath = join(runsDir, "fixture.json");
    writeFileSync(
      fixturePath,
      JSON.stringify({
        name: "refusal-render",
        notes: [{ path: "Brain/notes/a.md", body: "alpha note body" }],
        questions: [
          {
            id: "q1",
            category: "single_hop",
            query: "alpha",
            expected_paths: ["Brain/notes/a.md"],
          },
        ],
      }),
    );
    const blockedRunsDir = join(runsDir, "blocked-runs");
    writeFileSync(blockedRunsDir, "not a directory");
    const stderr: string[] = [];
    const stdout: string[] = [];
    const originalErr = process.stderr.write;
    const originalOut = process.stdout.write;
    process.stderr.write = ((chunk?: unknown) => {
      stderr.push(String(chunk));
      return true;
    }) as typeof process.stderr.write;
    process.stdout.write = ((chunk?: unknown) => {
      stdout.push(String(chunk));
      return true;
    }) as typeof process.stdout.write;
    try {
      const code = await cmdBrainBench([
        "memory",
        "--fixture",
        fixturePath,
        "--runs-dir",
        blockedRunsDir,
      ]);
      expect(code).not.toBe(0);
      // Scenario #1's render is asserted before the buffer is reused.
      expect(stderr.join("")).toContain("refused (runs_dir_unusable)");
      // Deterministic refusal #2: resuming a run whose checkpoint is corrupt.
      const brokenDir = join(runsDir, "broken-runs");
      const { createBenchRun: create } = await import("../../../src/core/bench/run-store.ts");
      const broken = create(
        brokenDir,
        parseBenchFixture(JSON.parse(readFileSync(fixturePath, "utf8"))),
        {
          now: new Date("2026-06-03T12:00:00Z"),
        },
      );
      writeFileSync(join(broken.runDir, "checkpoint.json"), "{not json");
      stderr.length = 0;
      const corruptCode = await cmdBrainBench([
        "memory",
        "--fixture",
        fixturePath,
        "--runs-dir",
        brokenDir,
        "--resume",
        broken.runId,
      ]);
      expect(corruptCode).not.toBe(0);
    } finally {
      process.stderr.write = originalErr;
      process.stdout.write = originalOut;
    }
    expect(stderr.join("")).toContain("refused (checkpoint_unparseable)");
    // Refusals are stderr-only: stdout stays byte-empty for both.
    expect(stdout.join("")).toBe("");
  }, 30_000);
});

describe("the embeddings capability path", () => {
  // Lexically disjoint notes, so the semantic lane cannot reorder a gold
  // path below an unrelated one and the assertions stay about the
  // capability machinery, not about fusion tuning.
  const SEMANTIC_FIXTURE = parseBenchFixture({
    name: "embedding-capability",
    notes: [
      {
        path: "Brain/notes/rust.md",
        body: "# Systems language\n\nThe systems team standardized on Rust for the new ingest service. Rust ownership keeps the parser safe.\n",
      },
      {
        path: "Brain/notes/piano.md",
        body: "# Evening hobby\n\nThe operator practices piano on weekends and prefers nocturnes. Piano practice happens after dinner.\n",
      },
    ],
    continuity: [],
    questions: [
      {
        id: "q-rust",
        category: "single_hop",
        query: "systems programming language for the ingest service",
        top_k: 5,
        expected_paths: ["Brain/notes/rust.md"],
      },
      {
        id: "q-piano",
        category: "single_hop",
        query: "weekend instrument hobby",
        top_k: 5,
        expected_paths: ["Brain/notes/piano.md"],
      },
    ],
  });
  const CAPABILITIES = { embeddings: true } as const;
  const SEMANTIC_PIN: BenchPinnedKnobs = { semantic: { enabled: true, provider: "local" } };

  test("a semantic pin without the embeddings capability is a named error", async () => {
    await expect(
      runMemoryBench({ fixture: SEMANTIC_FIXTURE, runsDir, pinnedKnobs: SEMANTIC_PIN }),
    ).rejects.toThrow(/embeddings capability/);
  }, 30_000);

  test("a blocked embedding spend gate is refused by name, never bypassed", async () => {
    const pin: BenchPinnedKnobs = {
      semantic: {
        enabled: true,
        provider: "openai-compat",
        baseUrl: "http://127.0.0.1:9",
        model: "bench-unpriced-model",
        apiKey: FAKE_PROVIDER_KEY,
        costGateUsd: 1,
      },
    };
    // The unpriced model under a positive gate is refused before any
    // provider contact: the run fails loudly instead of embedding anyway.
    await expect(
      runMemoryBench({
        fixture: SEMANTIC_FIXTURE,
        runsDir,
        capabilities: CAPABILITIES,
        pinnedKnobs: pin,
      }),
    ).rejects.toThrow(/spend gate/);
  }, 30_000);

  test("the capability builds the index through the apply gate and surfaces the spend plan", async () => {
    const report = await runMemoryBench({
      fixture: SEMANTIC_FIXTURE,
      runsDir,
      capabilities: CAPABILITIES,
      pinnedKnobs: SEMANTIC_PIN,
    });
    expect(report.quality.passed).toBe(report.quality.total);
    const receipt = report.embeddings;
    expect(receipt).toBeDefined();
    expect(receipt!.provider).toBe("local");
    expect(receipt!.model).toBe(LOCAL_EMBEDDING_MODEL);
    expect(receipt!.signature).toBe(`local:${LOCAL_EMBEDDING_MODEL}:${LOCAL_DEFAULT_DIMENSION}::`);
    expect(receipt!.planned).toBeGreaterThan(0);
    expect(receipt!.cache_hits).toBe(0);
    expect(receipt!.cache_misses).toBe(receipt!.planned);
    expect(receipt!.gate).toEqual({ blocked: false, reason: null });
    // The surfaced plan lands as a run artifact beside report.json.
    const receiptOnDisk = JSON.parse(
      readFileSync(join(runsDir, report.run_id, "embeddings.json"), "utf8"),
    ) as Record<string, unknown>;
    expect(receiptOnDisk).toEqual(receipt as unknown as Record<string, unknown>);
    // The shared cache lives under the runs dir, entries keyed by
    // provider, model, dimension and text hash.
    const cache = JSON.parse(readFileSync(join(runsDir, "embedding-cache.json"), "utf8")) as {
      entries: Record<string, ReadonlyArray<number>>;
    };
    const keys = Object.keys(cache.entries);
    expect(keys.length).toBeGreaterThan(0);
    const keyPattern = new RegExp(`^${receipt!.signature}\\|[0-9a-f]{64}$`);
    for (const key of keys) {
      expect(key).toMatch(keyPattern);
      expect(cache.entries[key]!.length).toBe(LOCAL_DEFAULT_DIMENSION);
    }
  }, 30_000);

  test("vector cache keys differ when only the prefix pins differ", () => {
    // The stored bytes depend on the prefixes the provider prepends, so
    // two arms that differ only in prefix pins must never share an entry.
    const base = benchEmbeddingSignature("local", "m", 384, "", "");
    const prefixed = benchEmbeddingSignature("local", "m", 384, "search: ", "passage: ");
    expect(base).not.toBe(prefixed);
    expect(benchVectorCacheKey(base, "same text")).not.toBe(
      benchVectorCacheKey(prefixed, "same text"),
    );
    // Case and surrounding whitespace canonicalize away, as everywhere
    // else the signature tokens are compared.
    expect(benchEmbeddingSignature("Local", "m", 384, "", "")).toBe(base);
  });

  test("a second arm reuses the shared cache and stores byte-identical vectors", async () => {
    const t0 = new Date("2026-06-01T00:00:00.000Z");
    const first = await runMemoryBench({
      fixture: SEMANTIC_FIXTURE,
      runsDir,
      capabilities: CAPABILITIES,
      pinnedKnobs: SEMANTIC_PIN,
      now: t0,
    });
    const cacheAfterFirst = readFileSync(join(runsDir, "embedding-cache.json"), "utf8");
    const second = await runMemoryBench({
      fixture: SEMANTIC_FIXTURE,
      runsDir,
      capabilities: CAPABILITIES,
      pinnedKnobs: SEMANTIC_PIN,
      now: new Date(t0.getTime() + 60_000),
    });
    expect(second.run_id).not.toBe(first.run_id);
    expect(second.embeddings!.cache_misses).toBe(0);
    expect(second.embeddings!.cache_hits).toBe(first.embeddings!.planned);
    // Arm 2 rewrote the cache from its own store read-back: identical
    // bytes prove the stored vectors equal arm 1's, element for element.
    expect(readFileSync(join(runsDir, "embedding-cache.json"), "utf8")).toBe(cacheAfterFirst);
  }, 30_000);

  test("a shared cache written under another schema is refused, never honored", async () => {
    const cachePath = join(runsDir, "embedding-cache.json");
    // Each attempt takes its own minute: run ids are second-precision, so
    // two fresh same-identity runs in one second are a collision refusal
    // now (P2) - the days of silently resetting an existing run dir.
    let attempt = 0;
    const run = () => {
      const now = new Date(Date.UTC(2026, 5, 1, 0, attempt++, 0));
      return runMemoryBench({
        fixture: SEMANTIC_FIXTURE,
        runsDir,
        capabilities: CAPABILITIES,
        pinnedKnobs: SEMANTIC_PIN,
        now,
      });
    };
    writeFileSync(
      cachePath,
      `${JSON.stringify({ schema: "some.other.vectors.v9", entries: {} }, null, 2)}\n`,
      "utf8",
    );
    await expect(run()).rejects.toThrow(
      /schema 'some\.other\.vectors\.v9', expected 'o2b\.bench\.vectors\.v1'/,
    );
    // Every writer stamps the schema, so a missing one is foreign state
    // too - not an older generation to silently adopt.
    writeFileSync(cachePath, `${JSON.stringify({ entries: {} }, null, 2)}\n`, "utf8");
    await expect(run()).rejects.toThrow(/schema 'undefined'/);
  }, 30_000);

  test("the latency family reports overlapped_ms when questions attempted the semantic lane", async () => {
    const report = await runMemoryBench({
      fixture: SEMANTIC_FIXTURE,
      runsDir,
      capabilities: CAPABILITIES,
      pinnedKnobs: SEMANTIC_PIN,
    });
    expect(report.quality.passed).toBe(report.quality.total);
    const semanticQuestions = report.questions.filter((q) => q.category === "single_hop");
    expect(semanticQuestions.length).toBeGreaterThan(0);
    for (const question of semanticQuestions) {
      expect(question.overlapped_ms).toBeGreaterThanOrEqual(0);
      expect(Number.isInteger(question.overlapped_ms)).toBe(true);
    }
    expect(report.latency_ms.overlapped_ms).toBeGreaterThanOrEqual(0);
  }, 30_000);

  test("a keyword-only run adds no overlapped_ms keys to the report", async () => {
    const report = await runMemoryBench({ fixture: FIXTURE, runsDir });
    // No question attempted the semantic lane, so no question carries the
    // figure - the latency block keeps exactly the keys it always had.
    expect(report.questions.every((q) => q.overlapped_ms === undefined)).toBe(true);
    expect(JSON.parse(JSON.stringify(report.latency_ms))).toEqual(
      expect.objectContaining({ avg: expect.any(Number), max: expect.any(Number) }),
    );
    expect(Object.keys(report.latency_ms).toSorted()).toEqual(["avg", "max"]);
  }, 30_000);
});