Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
14.7 KB
Raw Blame
/**
 * Bench run store (Memory Observability Suite, t_882c396a).
 *
 * Checkpointed runs under a caller-supplied runs directory:
 * `<runsDir>/<run-id>/checkpoint.json` plus per-question results and
 * the final report. Resume validates the fixture hash before skipping
 * phases - a changed fixture invalidates the checkpoint rather than
 * silently mixing two runs. The run identity covers the pinned search
 * knobs, the recall admission floor and the seeded sampling request
 * ({@link benchRunIdentityHash}), so a resumed run whose pin, floor or
 * sampling options changed fails the guard loudly; checkpoints written
 * before knob pinning existed are refused for the same reason. A path
 * guard refuses run ids that resolve outside the runs directory.
 */

import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
import { join, resolve, sep } from "node:path";

import { canonicalJson, sha256Hex } from "../integrity/digest.ts";
import { fixtureHash } from "./fixture.ts";
import {
  BENCH_PHASES,
  DEFAULT_BENCH_KNOB_PIN,
  type BenchCheckpoint,
  type BenchFixture,
  type BenchPhase,
  type BenchPinnedKnobs,
  type BenchSampleRequest,
} from "./types.ts";

export interface BenchRunHandle {
  readonly runId: string;
  readonly runDir: string;
  readonly checkpoint: BenchCheckpoint;
}

const CHECKPOINT_FILE = "checkpoint.json";

/** Hex characters kept from the run-identity digest (fixture + knob pin). */
const RUN_IDENTITY_HEX_CHARS = 16;

/**
 * The run-identity hash: the fixture hash AND the pinned knob record
 * AND the recall admission floor (null when unset) AND the seeded
 * sampling request (spread in only when set), canonically hashed
 * together. Two runs that differ only in their pin, floor or sampling
 * options therefore get different run ids - even when the sampled
 * fixture would be identical, because the options are what the run was
 * asked to do - and a resumed run whose pin, floor or sampling options
 * changed fails the identity guard instead of silently comparing two
 * pipelines - or worse, deciding the remaining questions under new
 * options while the persisted ones keep the old. The sampling half is
 * spread-omitted, so a no-sampling record is byte-identical to the
 * shipped one and every pre-change run id stays stable.
 */
export function benchRunIdentityHash(
  fixture: BenchFixture,
  pinnedKnobs: BenchPinnedKnobs,
  recallFloor?: number,
  sample?: BenchSampleRequest,
): string {
  return sha256Hex(
    canonicalJson({
      fixture: fixtureHash(fixture),
      knobs: pinnedKnobs,
      recall_floor: recallFloor ?? null,
      ...(sample !== undefined ? { sample } : {}),
    }),
  ).slice(0, RUN_IDENTITY_HEX_CHARS);
}

/** Disposable fixture vault location inside one run. */
export function benchVaultDir(runDir: string): string {
  return join(runDir, "vault");
}

/** Per-question raw results location inside one run. */
export function benchResultsDir(runDir: string): string {
  return join(runDir, "results");
}

/**
 * The run directory for a run id, resolved exactly the way the store
 * resolves it - through the {@link guardedRunDir} checks (finding 31):
 * the preflight and the pipeline consume this path BEFORE the store's
 * own guard runs (the search config is resolved against the run's future
 * vault so a refusal there can leave no run dir behind), so a run id the
 * guard would refuse is refused HERE, before any path is built or read.
 */
export function benchRunDirPath(runsDir: string, runId: string): string {
  return guardedRunDir(runsDir, runId);
}

/**
 * The run id a fresh run is about to take: second-precision timestamp
 * plus the identity hex. One fact, shared by `createBenchRun` and the
 * pipeline's preflight collision check, so the id the guard reads is the
 * id the store writes.
 */
export function nextBenchRunId(
  fixture: BenchFixture,
  opts: {
    now?: Date;
    pinnedKnobs?: BenchPinnedKnobs;
    recallFloor?: number;
    sample?: BenchSampleRequest;
  } = {},
): string {
  const now = opts.now ?? new Date();
  const knobs = opts.pinnedKnobs ?? DEFAULT_BENCH_KNOB_PIN;
  const identity = benchRunIdentityHash(fixture, knobs, opts.recallFloor, opts.sample);
  const stamp = now
    .toISOString()
    .replace(/[^0-9]/g, "")
    .slice(0, 14);
  return `run-${stamp}-${identity.slice(0, 6)}`;
}

/**
 * Write `data` atomically: a temp file renamed over the target, so a
 * crash mid-write leaves the previous generation intact instead of a
 * truncated published artifact. The bench vector cache established the
 * pattern (`embedding-cache.ts`); checkpoints, per-question results and
 * the report all go through it now.
 */
export function atomicWriteFileSync(filePath: string, data: string): void {
  const tmp = `${filePath}.${process.pid}.tmp`;
  try {
    writeFileSync(tmp, data, "utf8");
    renameSync(tmp, filePath);
  } catch (cause) {
    try {
      rmSync(tmp, { force: true });
    } catch {
      // The original failure is the one to report.
    }
    throw cause;
  }
}

export function createBenchRun(
  runsDir: string,
  fixture: BenchFixture,
  opts: {
    now?: Date;
    pinnedKnobs?: BenchPinnedKnobs;
    recallFloor?: number;
    sample?: BenchSampleRequest;
  } = {},
): BenchRunHandle {
  const now = opts.now ?? new Date();
  const knobs = opts.pinnedKnobs ?? DEFAULT_BENCH_KNOB_PIN;
  const hash = fixtureHash(fixture);
  // The identity is computed from the same inputs the run id was, so the
  // checkpoint's knobs_hash stays the exact value the id embeds a slice
  // of (`nextBenchRunId` shares this computation for the collision check).
  const identity = benchRunIdentityHash(fixture, knobs, opts.recallFloor, opts.sample);
  const runId = nextBenchRunId(fixture, {
    now,
    pinnedKnobs: knobs,
    ...(opts.recallFloor !== undefined ? { recallFloor: opts.recallFloor } : {}),
    ...(opts.sample !== undefined ? { sample: opts.sample } : {}),
  });
  const runDir = guardedRunDir(runsDir, runId);
  mkdirSync(runDir, { recursive: true });
  const checkpoint: BenchCheckpoint = Object.freeze({
    run_id: runId,
    fixture_name: fixture.name,
    fixture_hash: hash,
    pinned_knobs: knobs,
    knobs_hash: identity,
    ...(opts.sample !== undefined ? { sample: opts.sample } : {}),
    created_at: now.toISOString(),
    completed_phases: Object.freeze([]),
  });
  persist(runDir, checkpoint);
  return Object.freeze({ runId, runDir, checkpoint });
}

export function loadBenchRun(
  runsDir: string,
  runId: string,
  opts: {
    expectFixture?: BenchFixture;
    pinnedKnobs?: BenchPinnedKnobs;
    recallFloor?: number;
    sample?: BenchSampleRequest;
  } = {},
): BenchRunHandle {
  const checkpoint = readBenchCheckpoint(runsDir, runId);
  if (opts.expectFixture !== undefined) {
    const expected = fixtureHash(opts.expectFixture);
    if (expected !== checkpoint.fixture_hash) {
      throw new Error(
        `bench run ${runId}: fixture hash mismatch (checkpoint ${checkpoint.fixture_hash}, current ${expected}) - the fixture changed since this run started`,
      );
    }
    const knobs = opts.pinnedKnobs ?? DEFAULT_BENCH_KNOB_PIN;
    if (checkpoint.knobs_hash === undefined) {
      throw new Error(
        `bench run ${runId}: checkpoint predates knob pinning - the run identity cannot be verified, restart the run`,
      );
    }
    const expectedIdentity = benchRunIdentityHash(
      opts.expectFixture,
      knobs,
      opts.recallFloor,
      opts.sample,
    );
    if (expectedIdentity !== checkpoint.knobs_hash) {
      throw new Error(
        `bench run ${runId}: run identity mismatch (checkpoint ${checkpoint.knobs_hash}, current ${expectedIdentity}) - the pinned knobs, the recall floor or the sampling options changed since this run started`,
      );
    }
  }
  return Object.freeze({ runId, runDir: benchRunDirPath(runsDir, runId), checkpoint });
}

/** Mark a phase complete (idempotent) and persist the checkpoint. */
export function completeBenchPhase(
  runDir: string,
  checkpoint: BenchCheckpoint,
  phase: BenchPhase,
): BenchCheckpoint {
  if (checkpoint.completed_phases.includes(phase)) return checkpoint;
  const updated: BenchCheckpoint = Object.freeze({
    ...checkpoint,
    completed_phases: Object.freeze([...checkpoint.completed_phases, phase]),
  });
  persist(runDir, updated);
  return updated;
}

export function phaseDone(checkpoint: BenchCheckpoint, phase: BenchPhase): boolean {
  return checkpoint.completed_phases.includes(phase);
}

/**
 * Read and parse one run's checkpoint. Missing run: the long-standing
 * "bench run not found" error. Corrupt JSON: a NAMED error carrying the
 * file path - never a raw SyntaxError; the pipeline's preflight refuses
 * the same state with the rendering contract before this is reached.
 */
export function readBenchCheckpoint(runsDir: string, runId: string): BenchCheckpoint {
  const runDir = guardedRunDir(runsDir, runId);
  const path = join(runDir, CHECKPOINT_FILE);
  if (!existsSync(path)) {
    throw new Error(`bench run not found: ${runId}`);
  }
  let raw: unknown;
  try {
    raw = JSON.parse(readFileSync(path, "utf8"));
  } catch (cause) {
    throw new Error(
      `bench checkpoint at ${path} is not valid JSON: ${
        cause instanceof Error ? cause.message : String(cause)
      } - the run directory is damaged; start a fresh run`,
    );
  }
  return parseCheckpoint(raw);
}

function persist(runDir: string, checkpoint: BenchCheckpoint): void {
  atomicWriteFileSync(join(runDir, CHECKPOINT_FILE), `${JSON.stringify(checkpoint, null, 2)}\n`);
}

/**
 * Parse and validate one raw checkpoint document: the ONE definition of
 * the shape a resume may act on. Exported so the pipeline's resume
 * preflight runs this SAME gate before the store loads - a document that
 * parses as JSON but lacks the required fields is refused there by name
 * (the file and the missing fields) instead of surfacing here as a plain
 * Error the CLI does not render (M11).
 */
export function parseCheckpoint(raw: unknown): BenchCheckpoint {
  if (raw === null || typeof raw !== "object") {
    throw new Error("bench checkpoint: malformed JSON");
  }
  const record = raw as Record<string, unknown>;
  const runId = record["run_id"];
  const fixtureName = record["fixture_name"];
  const hash = record["fixture_hash"];
  const createdAt = record["created_at"];
  const phases = record["completed_phases"];
  const knobsHash = record["knobs_hash"];
  const pinnedKnobs = record["pinned_knobs"];
  const sample = record["sample"];
  // The missing fields are NAMED - the preflight renders this message as
  // the refusal detail, and an operator repairing a damaged run needs to
  // know which fields the checkpoint lost, not just that it lost some.
  // (The combined condition is what narrows the fields for the rest of
  // the parser, so the names are collected inside it.)
  if (
    typeof runId !== "string" ||
    typeof fixtureName !== "string" ||
    typeof hash !== "string" ||
    typeof createdAt !== "string" ||
    !Array.isArray(phases)
  ) {
    const missing = [
      ...(typeof runId !== "string" ? ["run_id"] : []),
      ...(typeof fixtureName !== "string" ? ["fixture_name"] : []),
      ...(typeof hash !== "string" ? ["fixture_hash"] : []),
      ...(typeof createdAt !== "string" ? ["created_at"] : []),
      ...(!Array.isArray(phases) ? ["completed_phases"] : []),
    ];
    throw new Error(`bench checkpoint: missing required field(s) ${missing.join(", ")}`);
  }
  if (knobsHash !== undefined && typeof knobsHash !== "string") {
    throw new Error("bench checkpoint: knobs_hash must be a string");
  }
  if (
    pinnedKnobs !== undefined &&
    (pinnedKnobs === null || typeof pinnedKnobs !== "object" || Array.isArray(pinnedKnobs))
  ) {
    throw new Error("bench checkpoint: pinned_knobs must be an object");
  }
  if (
    sample !== undefined &&
    (sample === null || typeof sample !== "object" || Array.isArray(sample))
  ) {
    throw new Error("bench checkpoint: sample must be an object");
  }
  // A phase name this build does not know means the checkpoint was
  // written by a different build. The old behavior silently FILTERED the
  // unknown name out and resumed as if the phase never happened - which
  // re-ran work under an identity that claimed it was done. Refuse by
  // name instead (the pipeline's preflight refuses the same state with
  // the `refused (<kind>)` render before this is reached).
  const completed: BenchPhase[] = [];
  for (const phase of phases) {
    if (typeof phase !== "string" || !(BENCH_PHASES as ReadonlyArray<string>).includes(phase)) {
      throw new Error(
        `bench checkpoint: unknown phase '${String(phase)}' in completed_phases - ` +
          "the checkpoint was written by a different build; restart the run",
      );
    }
    // The includes() guard above is the membership proof; the array's
    // element type does not carry it, so the narrow is asserted here.
    completed.push(phase as BenchPhase);
  }
  return Object.freeze({
    run_id: runId,
    fixture_name: fixtureName,
    fixture_hash: hash,
    ...(typeof knobsHash === "string" ? { knobs_hash: knobsHash } : {}),
    ...(pinnedKnobs !== undefined &&
    pinnedKnobs !== null &&
    typeof pinnedKnobs === "object" &&
    !Array.isArray(pinnedKnobs)
      ? { pinned_knobs: Object.freeze({ ...(pinnedKnobs as BenchPinnedKnobs) }) }
      : {}),
    ...(sample !== undefined &&
    sample !== null &&
    typeof sample === "object" &&
    !Array.isArray(sample)
      ? { sample: Object.freeze({ ...(sample as BenchSampleRequest) }) }
      : {}),
    created_at: createdAt,
    completed_phases: Object.freeze(completed),
  });
}

/**
 * Resolve a run directory and refuse anything outside the runs dir: the
 * charset rule (a run id is `[A-Za-z0-9._-]+`), the all-dots rule (`.` and
 * `..` pass the charset and resolve inside the root) and the containment
 * rule (the resolved dir must stay under the runs dir). Exported so every
 * site that derives a run-dir path from a CLI-typed id - the preflight's
 * probe reads, the pipeline's pre-creation path resolution - applies the
 * SAME guard BEFORE any read, not only the store's own checkpoint read.
 */
export function guardedRunDir(runsDir: string, runId: string): string {
  if (!/^[A-Za-z0-9._-]+$/.test(runId)) {
    throw new Error(`bench run id contains unsafe characters: ${runId}`);
  }
  // "." and ".." pass the charset and resolve inside the root: a run id
  // of dots would write the checkpoint, the report and the vault beside
  // the shared vector cache instead of into its own directory.
  if (runId.split(".").every((part) => part === "")) {
    throw new Error(`bench run id must not be all dots: ${runId}`);
  }
  const root = resolve(runsDir);
  const dir = resolve(root, runId);
  if (dir !== root && !dir.startsWith(`${root}${sep}`)) {
    throw new Error(`bench run directory escapes the runs dir: ${runId}`);
  }
  return dir;
}