Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
14.0 KB
Raw Blame
/**
 * Bench run store (Memory Observability Suite, t_882c396a):
 * checkpointed runs under a runs directory, resume by run id, fixture
 * hash validation, and a path guard that refuses to operate outside
 * the runs directory.
 */

import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { parseBenchFixture } from "../../../src/core/bench/fixture.ts";
import {
  benchRunDirPath,
  benchRunIdentityHash,
  benchVaultDir,
  completeBenchPhase,
  createBenchRun,
  loadBenchRun,
  nextBenchRunId,
  readBenchCheckpoint,
} from "../../../src/core/bench/run-store.ts";
import { DEFAULT_BENCH_KNOB_PIN, type BenchPinnedKnobs } from "../../../src/core/bench/types.ts";

let runsDir: string;

beforeEach(() => {
  runsDir = mkdtempSync(join(tmpdir(), "o2b-bench-runs-"));
});

afterEach(() => {
  rmSync(runsDir, { recursive: true, force: true });
});

const FIXTURE = parseBenchFixture({
  name: "mini",
  notes: [{ path: "Brain/notes/a.md", body: "alpha" }],
  questions: [
    {
      id: "q1",
      category: "single_hop",
      query: "alpha",
      expected_paths: ["Brain/notes/a.md"],
    },
  ],
});

describe("bench run store", () => {
  test("createBenchRun writes a checkpoint and loadBenchRun round-trips it", () => {
    const run = createBenchRun(runsDir, FIXTURE, { now: new Date("2026-06-03T12:00:00Z") });
    expect(run.runId).toStartWith("run-20260603120000-");
    expect(existsSync(join(run.runDir, "checkpoint.json"))).toBe(true);
    expect(run.checkpoint.completed_phases).toEqual([]);

    const loaded = loadBenchRun(runsDir, run.runId);
    expect(loaded.checkpoint.run_id).toBe(run.runId);
    expect(loaded.checkpoint.fixture_name).toBe("mini");
    expect(benchVaultDir(loaded.runDir)).toBe(join(loaded.runDir, "vault"));
  });

  test("completeBenchPhase persists progress exactly once per phase", () => {
    const run = createBenchRun(runsDir, FIXTURE, { now: new Date("2026-06-03T12:00:00Z") });
    const afterIngest = completeBenchPhase(run.runDir, run.checkpoint, "ingest");
    expect(afterIngest.completed_phases).toEqual(["ingest"]);
    const again = completeBenchPhase(run.runDir, afterIngest, "ingest");
    expect(again.completed_phases).toEqual(["ingest"]);
    const reloaded = loadBenchRun(runsDir, run.runId);
    expect(reloaded.checkpoint.completed_phases).toEqual(["ingest"]);
  });

  test("a fixture-hash mismatch invalidates resume instead of mixing runs", () => {
    const run = createBenchRun(runsDir, FIXTURE, { now: new Date("2026-06-03T12:00:00Z") });
    const changed = parseBenchFixture({
      name: "mini",
      notes: [{ path: "Brain/notes/a.md", body: "CHANGED" }],
      questions: [
        {
          id: "q1",
          category: "single_hop",
          query: "alpha",
          expected_paths: ["Brain/notes/a.md"],
        },
      ],
    });
    expect(() => loadBenchRun(runsDir, run.runId, { expectFixture: changed })).toThrow("hash");
  });

  test("unknown run ids and path-escaping ids are rejected", () => {
    expect(() => loadBenchRun(runsDir, "run-nope")).toThrow();
    expect(() => loadBenchRun(runsDir, "../escape")).toThrow();
  });

  test("benchRunDirPath refuses traversal ids instead of resolving them", () => {
    // Finding 31: the preflight and the pipeline derive the run dir (and
    // the vault path for config resolution) through this helper BEFORE
    // loadBenchRun's guard runs, so the resolve must carry the same guard
    // the checkpoint reader applies - `--resume ..` used to stat and read
    // under the runs dir's parent before any refusal.
    expect(() => benchRunDirPath(runsDir, "..")).toThrow(/all dots/);
    expect(() => benchRunDirPath(runsDir, "../escape")).toThrow(/unsafe characters/);
    // A safe id resolves exactly as before.
    expect(benchRunDirPath(runsDir, "run-20260603120000-abc123")).toBe(
      join(runsDir, "run-20260603120000-abc123"),
    );
  });

  test("two runs with different knob pins get different run ids", () => {
    const pinA: BenchPinnedKnobs = { semantic: { enabled: false } };
    const pinB: BenchPinnedKnobs = { semantic: { enabled: false }, keywordWeight: 0.6 };
    const now = new Date("2026-06-03T12:00:00Z");
    const a = createBenchRun(runsDir, FIXTURE, { now, pinnedKnobs: pinA });
    const b = createBenchRun(runsDir, FIXTURE, { now, pinnedKnobs: pinB });
    // Same fixture, same second, different pin: the ids must not collide.
    expect(a.runId).not.toBe(b.runId);
  });

  test("the checkpoint carries the pinned knobs and the identity hash", () => {
    const pin: BenchPinnedKnobs = { semantic: { enabled: false }, keywordWeight: 0.6 };
    const run = createBenchRun(runsDir, FIXTURE, {
      now: new Date("2026-06-03T12:00:00Z"),
      pinnedKnobs: pin,
    });
    expect(run.checkpoint.pinned_knobs).toEqual(pin);
    expect(run.checkpoint.knobs_hash).toBeDefined();
    const reloaded = loadBenchRun(runsDir, run.runId);
    expect(reloaded.checkpoint.pinned_knobs).toEqual(pin);
    expect(reloaded.checkpoint.knobs_hash).toBe(run.checkpoint.knobs_hash);
  });

  test("resume with the same pin passes the guard, a changed pin fails it", () => {
    const pin: BenchPinnedKnobs = { semantic: { enabled: false } };
    const run = createBenchRun(runsDir, FIXTURE, {
      now: new Date("2026-06-03T12:00:00Z"),
      pinnedKnobs: pin,
    });
    const same = loadBenchRun(runsDir, run.runId, { expectFixture: FIXTURE, pinnedKnobs: pin });
    expect(same.runId).toBe(run.runId);
    expect(() =>
      loadBenchRun(runsDir, run.runId, {
        expectFixture: FIXTURE,
        pinnedKnobs: { semantic: { enabled: false }, keywordWeight: 0.5 },
      }),
    ).toThrow(/run identity mismatch/);
  });

  test("an all-dots run id is refused", () => {
    // "." passes the run-id charset and resolves inside the runs dir, so
    // a resume with it would write the checkpoint beside the shared cache.
    expect(() => loadBenchRun(runsDir, ".", {})).toThrow(/all dots/);
    expect(() => loadBenchRun(runsDir, "..", {})).toThrow(/all dots/);
  });

  test("a resume whose recall floor changed fails the identity guard", () => {
    const run = createBenchRun(runsDir, FIXTURE, {
      now: new Date("2026-06-03T12:00:00Z"),
      recallFloor: 0.4,
    });
    const same = loadBenchRun(runsDir, run.runId, {
      expectFixture: FIXTURE,
      recallFloor: 0.4,
    });
    expect(same.runId).toBe(run.runId);
    expect(() =>
      loadBenchRun(runsDir, run.runId, { expectFixture: FIXTURE, recallFloor: 0.5 }),
    ).toThrow(/run identity mismatch/);
  });

  test("a checkpoint that predates knob pinning fails resume loudly", () => {
    const run = createBenchRun(runsDir, FIXTURE, { now: new Date("2026-06-03T12:00:00Z") });
    // What an in-flight checkpoint written before knob pinning looks like.
    const path = join(run.runDir, "checkpoint.json");
    const legacy = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>;
    delete legacy["pinned_knobs"];
    delete legacy["knobs_hash"];
    writeFileSync(path, `${JSON.stringify(legacy, null, 2)}\n`, "utf8");
    expect(() => loadBenchRun(runsDir, run.runId, { expectFixture: FIXTURE })).toThrow(
      /predates knob pinning/,
    );
  });
});

/**
 * Checkpoint strictness and atomic persistence (t_d7ce8baf): a checkpoint
 * naming a phase this build does not know is a named refusal (never a
 * silent filter), the checkpoint reader is exposed for the preflight, and
 * every persist lands through temp+rename so a crash leaves the previous
 * generation intact.
 */
describe("checkpoint strictness and atomic persistence", () => {
  test("a checkpoint naming an unknown phase is refused, naming the build mismatch", () => {
    const run = createBenchRun(runsDir, FIXTURE, { now: new Date("2026-06-03T12:00:00Z") });
    const path = join(run.runDir, "checkpoint.json");
    const tampered = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>;
    tampered["completed_phases"] = ["ingest", "quantum_phase"];
    writeFileSync(path, `${JSON.stringify(tampered, null, 2)}\n`, "utf8");
    expect(() => loadBenchRun(runsDir, run.runId)).toThrow(/different build/);
    expect(() => loadBenchRun(runsDir, run.runId)).toThrow(/quantum_phase/);
  });

  test("readBenchCheckpoint round-trips and keeps the missing-run message", () => {
    const run = createBenchRun(runsDir, FIXTURE, { now: new Date("2026-06-03T12:00:00Z") });
    const checkpoint = readBenchCheckpoint(runsDir, run.runId);
    expect(checkpoint.run_id).toBe(run.runId);
    expect(checkpoint.completed_phases).toEqual([]);
    expect(() => readBenchCheckpoint(runsDir, "run-nope")).toThrow(/bench run not found: run-nope/);
  });

  test("readBenchCheckpoint refuses corrupt JSON naming the file", () => {
    const run = createBenchRun(runsDir, FIXTURE, { now: new Date("2026-06-03T12:00:00Z") });
    const path = join(run.runDir, "checkpoint.json");
    writeFileSync(path, "{not json", "utf8");
    expect(() => readBenchCheckpoint(runsDir, run.runId)).toThrow(
      new RegExp(`${path.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} is not valid JSON`),
    );
  });

  test("the next-run-id helper is the id createBenchRun persists", () => {
    const now = new Date("2026-06-03T12:00:00Z");
    const pin: BenchPinnedKnobs = { semantic: { enabled: false }, keywordWeight: 0.6 };
    const predicted = nextBenchRunId(FIXTURE, { now, pinnedKnobs: pin, recallFloor: 0.4 });
    const run = createBenchRun(runsDir, FIXTURE, { now, pinnedKnobs: pin, recallFloor: 0.4 });
    expect(predicted).toBe(run.runId);
    // Deterministic: the preflight can read the same identity the store writes.
    expect(nextBenchRunId(FIXTURE, { now, pinnedKnobs: pin, recallFloor: 0.4 })).toBe(predicted);
  });

  test("persist is atomic: no temp files survive a completed phase", () => {
    const run = createBenchRun(runsDir, FIXTURE, { now: new Date("2026-06-03T12:00:00Z") });
    completeBenchPhase(run.runDir, run.checkpoint, "ingest");
    const leftovers = readdirSync(run.runDir).filter((entry) => entry.includes(".tmp"));
    expect(leftovers).toEqual([]);
    // The store alone lays down the checkpoint; the vault is the ingest
    // phase's artifact (the pipeline materializes it before completing
    // the phase), so a bare create+complete leaves exactly one file.
    expect(readdirSync(run.runDir).toSorted()).toEqual(["checkpoint.json"]);
  });
});

/**
 * The sampling fold in the run identity (wiring-shipped-seams wave,
 * t_4501c662): the canonical record spreads `sample` in ONLY when
 * sampling is set - so a no-sampling run's identity equals the shipped
 * one byte for byte - and any sampling-option change is a different run
 * that the resume guard refuses to mix.
 */
describe("the sampling fold in the run identity", () => {
  /** The identity hash values computed before the fold existed. */
  const FROZEN_NO_SAMPLE_IDENTITY = "d3fbe6ad99439a0e";
  const FROZEN_FLOOR_IDENTITY = "7b9b430af41eb4a1";

  test("a no-sampling run's identity hash is byte-stable with the shipped fold", () => {
    expect(benchRunIdentityHash(FIXTURE, DEFAULT_BENCH_KNOB_PIN)).toBe(FROZEN_NO_SAMPLE_IDENTITY);
    expect(benchRunIdentityHash(FIXTURE, DEFAULT_BENCH_KNOB_PIN, 0.4)).toBe(FROZEN_FLOOR_IDENTITY);
  });

  test("a sampled run's identity hash differs, and differs by seed and by cap", () => {
    const base = { seed: "seed-1", cap: 1 };
    expect(benchRunIdentityHash(FIXTURE, DEFAULT_BENCH_KNOB_PIN, undefined, base)).not.toBe(
      FROZEN_NO_SAMPLE_IDENTITY,
    );
    expect(
      benchRunIdentityHash(FIXTURE, DEFAULT_BENCH_KNOB_PIN, undefined, {
        ...base,
        seed: "seed-2",
      }),
    ).not.toBe(benchRunIdentityHash(FIXTURE, DEFAULT_BENCH_KNOB_PIN, undefined, base));
    expect(
      benchRunIdentityHash(FIXTURE, DEFAULT_BENCH_KNOB_PIN, undefined, { ...base, cap: 2 }),
    ).not.toBe(benchRunIdentityHash(FIXTURE, DEFAULT_BENCH_KNOB_PIN, undefined, base));
  });

  test("two runs differing only in the seed, or only in the cap, get different run ids", () => {
    const now = new Date("2026-06-03T12:00:00Z");
    const base = { seed: "seed-1", cap: 1 };
    // Both caps sit at the one-question fixture's population, so the
    // materialized fixture is identical - the OPTIONS alone must split
    // the run ids.
    const a = createBenchRun(runsDir, FIXTURE, { now, sample: base });
    const b = createBenchRun(runsDir, FIXTURE, { now, sample: { ...base, seed: "seed-2" } });
    const c = createBenchRun(runsDir, FIXTURE, { now, sample: { ...base, cap: 2 } });
    expect(a.runId).not.toBe(b.runId);
    expect(a.runId).not.toBe(c.runId);
    // Same options, same id.
    expect(createBenchRun(runsDir, FIXTURE, { now, sample: base }).runId).toBe(a.runId);
  });

  test("the checkpoint carries the sampling request", () => {
    const run = createBenchRun(runsDir, FIXTURE, {
      now: new Date("2026-06-03T12:00:00Z"),
      sample: { seed: "seed-1", cap: 3 },
    });
    expect(run.checkpoint.sample).toEqual({ seed: "seed-1", cap: 3 });
    const reloaded = loadBenchRun(runsDir, run.runId);
    expect(reloaded.checkpoint.sample).toEqual({ seed: "seed-1", cap: 3 });
  });

  test("a resume whose sampling options changed fails the identity guard", () => {
    const run = createBenchRun(runsDir, FIXTURE, {
      now: new Date("2026-06-03T12:00:00Z"),
      sample: { seed: "seed-1", cap: 2 },
    });
    const same = loadBenchRun(runsDir, run.runId, {
      expectFixture: FIXTURE,
      sample: { seed: "seed-1", cap: 2 },
    });
    expect(same.runId).toBe(run.runId);
    expect(() =>
      loadBenchRun(runsDir, run.runId, {
        expectFixture: FIXTURE,
        sample: { seed: "seed-1", cap: 3 },
      }),
    ).toThrow(/run identity mismatch/);
    expect(() =>
      loadBenchRun(runsDir, run.runId, {
        expectFixture: FIXTURE,
        sample: { seed: "seed-2", cap: 2 },
      }),
    ).toThrow(/run identity mismatch/);
    // A no-sampling resume against a sampled run mismatches too.
    expect(() => loadBenchRun(runsDir, run.runId, { expectFixture: FIXTURE })).toThrow(
      /run identity mismatch/,
    );
  });
});