Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
25.0 KB
Raw Blame
/**
 * Bench-run integrity guards (t_d7ce8baf): the named refusal vocabulary,
 * the pre-run preflight, and the pre-report trace validation.
 *
 * Two gaps motivated this module, both about a degraded run creating
 * state or publishing scores:
 *
 *  - Ordering/atomicity: search-config resolution and the capability
 *    gate used to run AFTER run creation (orphan run dirs on refusal),
 *    a fresh run silently reset an existing same-identity run dir, and
 *    a resume target with damaged or tampered artifacts failed late
 *    with generic errors. {@link preflightBenchRun} catches all of
 *    that BEFORE `createBenchRun`/`loadBenchRun` runs.
 *  - Trace trust: the evaluate phase re-reads the persisted per-question
 *    results and used to default absent fields into ordinary FAILs,
 *    silently ignore truncated or foreign records, and publish faulted
 *    decisions as if they were memory behavior. {@link
 *    validateBenchTrace} refuses those BEFORE the report writes.
 *
 * What is deliberately NOT a refusal: score content. An
 * all-questions-failed run is the harness's purpose - a real
 * catastrophic regression is exactly what it must publish. Empty-but-
 * present trace fields are legitimate misses. The judge staying
 * `error` is advisory by contract (`judge.ts`).
 *
 * Guards abort; scorers score. `failure-modes.ts` is unchanged and the
 * guards add no knob, no measured behavior and zero stdout on success:
 * existing run ids, checkpoints and reports stay byte-identical
 * (`fixtureHash` remains content-only).
 */

import { accessSync, existsSync, mkdirSync, readFileSync, readdirSync } from "node:fs";
import { join } from "node:path";
import { constants as fsConstants } from "node:fs";

import { benchResultsDir, benchVaultDir, guardedRunDir, parseCheckpoint } from "./run-store.ts";
import { fixtureHash } from "./fixture.ts";
import {
  BENCH_PHASES,
  type BenchCheckpoint,
  type BenchFixture,
  type BenchPhase,
  type BenchQuestionResult,
  type BenchReport,
} from "./types.ts";

/**
 * The closed refusal vocabulary: one member per catchable pre-run (P1-P7)
 * or pre-report (T1-T7) state. Frozen, like `RECALL_FAILURE` - a kind
 * that renders to an operator must not be renamable silently.
 */
export const BENCH_REFUSAL = Object.freeze({
  /** P1: the runs directory cannot be created or written. */
  runsDirUnusable: "runs_dir_unusable",
  /** P2: a fresh run's identity collides with an existing run dir. */
  runDirExists: "run_dir_exists",
  /** P3: a resume target's checkpoint.json is missing required shape. */
  checkpointUnparseable: "checkpoint_unparseable",
  /** P4a: the checkpoint names a phase this build does not know. */
  checkpointPhaseUnknown: "checkpoint_phase_unknown",
  /** P4b: the checkpoint's phases are out of order or duplicated. */
  checkpointPhaseOrder: "checkpoint_phase_order",
  /**
   * P4c: the resume was handed a fixture the run did not start with -
   * another `--fixture`, an edited copy, or (on a sampled suite) a
   * different `--sample-seed`/`--sample-cap`, which samples a different
   * fixture. Refused BEFORE the artifact check, which would read the new
   * fixture's absent artifacts as a tampered run.
   */
  checkpointFixtureMismatch: "checkpoint_fixture_mismatch",
  /** P5: a phase is marked done whose artifacts are gone or truncated. */
  runArtifactMissing: "run_artifact_missing",
  /** P7: the fixture cross-references notes or sessions it never defines. */
  fixtureCrossReference: "fixture_cross_reference",
  /** T1: a fixture question has no persisted result file. */
  traceResultMissing: "trace_result_missing",
  /** T2: a result file is not parseable JSON. */
  traceResultUnparseable: "trace_result_unparseable",
  /** T3: a result file's `id` differs from the question it is stored under. */
  traceIdMismatch: "trace_id_mismatch",
  /** T4: a category-required field is ABSENT from a persisted record. */
  traceFieldAbsent: "trace_field_absent",
  /** T5: a persisted proactive decision faulted. */
  traceFaultedDecisions: "trace_faulted_decisions",
  /** T6: `results/` holds files beyond the fixture's question ids. */
  traceExtraResults: "trace_extra_results",
  /** T7: the assembled report's counts disagree with its own trace. */
  reportCountsMismatch: "report_counts_mismatch",
} as const);

/** Closed union over {@link BENCH_REFUSAL}. */
export type BenchRefusalKind = (typeof BENCH_REFUSAL)[keyof typeof BENCH_REFUSAL];

/** Membership list, for listings and narrowings. */
export const BENCH_REFUSAL_KINDS: ReadonlyArray<BenchRefusalKind> = Object.freeze(
  Object.values(BENCH_REFUSAL),
);

/** Narrow a value read back across a boundary. */
export function isBenchRefusalKind(value: unknown): value is BenchRefusalKind {
  return (
    typeof value === "string" && (BENCH_REFUSAL_KINDS as ReadonlyArray<string>).includes(value)
  );
}

/**
 * The one error the bench guards raise. The message is the full CLI
 * render - `refused (<kind>): <detail>` - so a handler that prints the
 * message prints the contract; `kind` and `detail` stay structured for
 * callers that want to branch on the kind.
 */
export class BenchRefusalError extends Error {
  readonly kind: BenchRefusalKind;
  readonly detail: string;

  constructor(kind: BenchRefusalKind, detail: string) {
    super(`refused (${kind}): ${detail}`);
    this.name = "BenchRefusalError";
    this.kind = kind;
    this.detail = detail;
  }
}

function refuse(kind: BenchRefusalKind, detail: string): never {
  throw new BenchRefusalError(kind, detail);
}

/** The report file a completed report phase promises. Kept beside the pipeline's own writer. */
const REPORT_FILENAME = "report.json";

/** What the preflight needs to decide a run may start. */
export interface BenchPreflightOptions {
  /** The runs directory the run will live under (created if missing). */
  readonly runsDir: string;
  readonly fixture: BenchFixture;
  /** Resume target run id; preflight validates its checkpoint and artifacts. */
  readonly resume?: string;
  /**
   * The run id a FRESH run is about to create (the caller resolves it so
   * the collision check reads the same identity the store will write).
   * Absent on resume-only preflights.
   */
  readonly freshRunId?: string;
}

/**
 * Refuse every damaged state catchable BEFORE a run exists (P1-P7), so a
 * refusal can never leave a run directory behind. Adds zero stdout and,
 * on success, only the runs directory itself - which the store would
 * create anyway.
 */
export function preflightBenchRun(opts: BenchPreflightOptions): void {
  // P1: the runs dir must be creatable and writable. Probing here buys a
  // named refusal where a raw ENOENT/EACCES used to surface mid-store.
  try {
    mkdirSync(opts.runsDir, { recursive: true });
    accessSync(opts.runsDir, fsConstants.W_OK);
  } catch (cause) {
    refuse(
      BENCH_REFUSAL.runsDirUnusable,
      `runs directory ${opts.runsDir} is not usable: ${describeCause(cause)} - ` +
        "create it or pass a writable --runs-dir",
    );
  }
  if (opts.resume !== undefined) {
    preflightResume(opts.runsDir, opts.resume, opts.fixture);
  } else if (opts.freshRunId !== undefined) {
    // P2: run ids are second-precision timestamp + identity hex, so a
    // same-second rerun of the same fixture collides. The store would
    // silently RESET the prior run's checkpoint; the preflight refuses
    // instead and points at --resume.
    const existing = join(opts.runsDir, opts.freshRunId, "checkpoint.json");
    if (existsSync(existing)) {
      refuse(
        BENCH_REFUSAL.runDirExists,
        `a run with this identity already exists at ${join(opts.runsDir, opts.freshRunId)} - ` +
          `pass --resume ${opts.freshRunId} to continue it, or choose another --runs-dir`,
      );
    }
  }
  // P7: fixture cross-references, on every run. A dangling reference
  // scores as an ordinary missing-expected FAIL - an authoring error
  // indistinguishable from a real regression - so it is refused by name
  // instead.
  preflightFixtureCrossReferences(opts.fixture);
}

function describeCause(cause: unknown): string {
  return cause instanceof Error ? cause.message : String(cause);
}

/** P3, P4, the fixture-hash match, P5: the resume target's checkpoint and its promised artifacts. */
function preflightResume(runsDir: string, runId: string, fixture: BenchFixture): void {
  // The run-id guard runs BEFORE any read (finding 30): a raw `--resume`
  // value the guard would refuse is refused here, by name - never
  // answered by whatever exists at the path it resolves to (`--resume ..`
  // used to read the parent directory's checkpoint.json and surface that
  // file's parse error as the refusal).
  const runDir = guardedRunDir(runsDir, runId);
  const path = join(runDir, "checkpoint.json");
  // A missing checkpoint keeps the store's own "bench run not found"
  // message: nothing here to add.
  if (!existsSync(path)) return;
  let raw: unknown;
  try {
    raw = JSON.parse(readFileSync(path, "utf8"));
  } catch (cause) {
    refuse(
      BENCH_REFUSAL.checkpointUnparseable,
      `${path} is not valid JSON: ${describeCause(cause)} - ` +
        "the run directory is damaged; start a fresh run",
    );
  }
  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
    refuse(
      BENCH_REFUSAL.checkpointUnparseable,
      `${path} is not a checkpoint object - start a fresh run`,
    );
  }
  const record = raw as Record<string, unknown>;
  const phases = record["completed_phases"];
  if (
    !Array.isArray(phases) ||
    phases.some((phase) => typeof phase !== "string" || phase.length === 0)
  ) {
    refuse(
      BENCH_REFUSAL.checkpointUnparseable,
      `${path} has no usable completed_phases array - start a fresh run`,
    );
  }
  // P4a: unknown phase names mean a checkpoint from a different build;
  // the store's parser refuses these too, but naming the value here
  // renders the repair before any load work happens.
  const unknown = phases.filter(
    (phase) => !(BENCH_PHASES as ReadonlyArray<string>).includes(phase),
  );
  if (unknown.length > 0) {
    refuse(
      BENCH_REFUSAL.checkpointPhaseUnknown,
      `${path} lists unknown phase(s) ${unknown.map((phase) => `'${phase}'`).join(", ")} - ` +
        "the checkpoint was written by a different build; start a fresh run",
    );
  }
  // P4b: completed phases are appended in pipeline order and once each.
  let lastIndex = -1;
  for (const phase of phases as ReadonlyArray<BenchPhase>) {
    const index = BENCH_PHASES.indexOf(phase);
    if (index <= lastIndex) {
      refuse(
        BENCH_REFUSAL.checkpointPhaseOrder,
        `${path} lists '${phase}' out of order or duplicated (completed_phases: ${phases.join(", ")}) - ` +
          "the checkpoint is damaged or from a different build; start a fresh run",
      );
    }
    lastIndex = index;
  }
  // The store's parser is the ONE definition of a usable checkpoint, and
  // it runs HERE: a document that parses as JSON but lacks the required
  // fields passed the shape checks above (and P5, whenever the artifacts
  // on disk happened to exist) and then exploded in loadBenchRun as a
  // plain Error the verb does not catch - a source frame where the
  // contract promises `refused (checkpoint_unparseable)`. Mapped to that
  // refusal with the file and the missing fields named. P4a/P4b above
  // already refused the phase states under their own names, so any
  // failure left for the parser is a shape failure.
  let checkpoint: BenchCheckpoint;
  try {
    checkpoint = parseCheckpoint(raw);
  } catch (cause) {
    refuse(
      BENCH_REFUSAL.checkpointUnparseable,
      `${path} is not a usable checkpoint: ${describeCause(cause)} - start a fresh run`,
    );
  }
  // P4c: the fixture the resume was given is hashed the way loadBenchRun
  // hashes it and compared BEFORE P5. A different `--fixture`, an edited
  // fixture copy, or a different --sample-seed/--sample-cap (which sample
  // a different fixture) changes this hash - and P5, comparing the run
  // directory against the NEW fixture, would read its absent artifacts as
  // `run_artifact_missing` and advise discarding a run that is healthy
  // under its own fixture.
  const expectedHash = fixtureHash(fixture);
  if (checkpoint.fixture_hash !== expectedHash) {
    refuse(
      BENCH_REFUSAL.checkpointFixtureMismatch,
      `${path} records fixture hash '${checkpoint.fixture_hash}' but this resume was given ` +
        `a fixture hashing to '${expectedHash}' - ` +
        "resume with the same fixture, seed and cap, or start a fresh run",
    );
  }
  const completed = new Set<string>(phases as ReadonlyArray<string>);
  const done = (phase: BenchPhase): boolean => completed.has(phase);
  // P5: a phase marked done promises artifacts. Only phases the coming
  // invocation still READS are required: evaluate/report rerun from the
  // persisted results on every resume, but the disposable vault is
  // legitimately deletable once retrieval is done (it is never read
  // again), so the vault is required exactly when retrieval still has to
  // run against it.
  if (done("ingest") && !done("retrieve")) {
    const vault = benchVaultDir(runDir);
    if (!existsSync(vault)) {
      refuse(
        BENCH_REFUSAL.runArtifactMissing,
        `the run claims ingest done but the materialized vault is missing at ${vault} - ` +
          "the run directory was tampered with or truncated; start a fresh run",
      );
    }
    const missingNote = fixture.notes.find((note) => !existsSync(join(vault, note.path)));
    if (missingNote !== undefined) {
      refuse(
        BENCH_REFUSAL.runArtifactMissing,
        `the materialized vault is missing the fixture note ${missingNote.path} - ` +
          "the run directory was tampered with or truncated; start a fresh run",
      );
    }
  }
  if (done("retrieve")) {
    const results = benchResultsDir(runDir);
    if (!existsSync(results)) {
      refuse(
        BENCH_REFUSAL.runArtifactMissing,
        `the run claims retrieve done but the results directory is missing at ${results} - ` +
          "the run directory was tampered with or truncated; start a fresh run",
      );
    }
    const missingQuestion = fixture.questions.find(
      (question) => !existsSync(join(results, `${question.id}.json`)),
    );
    if (missingQuestion !== undefined) {
      refuse(
        BENCH_REFUSAL.runArtifactMissing,
        `the run claims retrieve done but ${join(results, `${missingQuestion.id}.json`)} is missing - ` +
          "the run directory was tampered with or truncated; start a fresh run",
      );
    }
  }
  if (done("report")) {
    const report = join(runDir, REPORT_FILENAME);
    if (!existsSync(report)) {
      refuse(
        BENCH_REFUSAL.runArtifactMissing,
        `the run claims report done but ${report} is missing - ` +
          "the run directory was tampered with or truncated; start a fresh run",
      );
    }
  }
}

/** P7: every referenced note path and session id must exist in the fixture. */
function preflightFixtureCrossReferences(fixture: BenchFixture): void {
  const notePaths = new Set(fixture.notes.map((note) => note.path));
  const sessionIds = new Set(
    fixture.continuity.flatMap((record) => {
      const sessionId = record.payload["session_id"];
      return typeof sessionId === "string" ? [sessionId] : [];
    }),
  );
  for (const question of fixture.questions) {
    for (const referenced of question.expected_paths ?? []) {
      if (!notePaths.has(referenced)) {
        refuse(
          BENCH_REFUSAL.fixtureCrossReference,
          `question ${question.id} expects note ${referenced}, which the fixture never defines - ` +
            "fix the fixture (the question cannot be scored as authored)",
        );
      }
    }
    for (const referenced of question.not_expected_above ?? []) {
      if (!notePaths.has(referenced)) {
        refuse(
          BENCH_REFUSAL.fixtureCrossReference,
          `question ${question.id} names ${referenced} in not_expected_above, ` +
            "which the fixture never defines - fix the fixture (the question cannot be scored as authored)",
        );
      }
    }
    if (question.session_id !== undefined && !sessionIds.has(question.session_id)) {
      refuse(
        BENCH_REFUSAL.fixtureCrossReference,
        `question ${question.id} (session_handoff) names session '${question.session_id}', ` +
          "which the fixture's continuity never records - fix the fixture (the question cannot be scored as authored)",
      );
    }
  }
}

/** The category-field contract the trace validators enforce (T4). */
const CATEGORY_REQUIRED_FIELDS: Readonly<Record<string, string>> = Object.freeze({
  retrieval: "paths",
  session_handoff: "turn_texts",
  budget: "item_ids",
  source_isolation: "item_ids",
  proactive_recall: "recall_decision",
  write_fidelity: "labels",
  standing_instruction: "standing_items",
});

/** The trace field a question's category requires, absent never allowed. */
export function requiredTraceField(category: string): string {
  // The six non-retrieval categories name their field explicitly; every
  // other category is a retrieval family (single_hop, temporal, ...),
  // whose required field is `paths`.
  return CATEGORY_REQUIRED_FIELDS[category] ?? "paths";
}

/** What the trace validator needs; `results`/`report` arm the counts check. */
export interface BenchTraceValidationOptions {
  readonly runDir: string;
  readonly fixture: BenchFixture;
  /** The evaluated results; required together with `report`. */
  readonly results?: ReadonlyArray<BenchQuestionResult>;
  /**
   * The assembled report. Provided on the pre-write validation call, it
   * arms T7: the report's published counts must equal the counts
   * recomputed from the results by the same aggregation buildReport used.
   */
  readonly report?: BenchReport;
}

/**
 * The shared atomic writer's own leftover shape (`<target>.<pid>.tmp`):
 * a process killed between the write and the rename leaves it beside its
 * target - the `rmSync` cleanup only runs on a thrown error - and T6
 * must not turn that crash artifact into an unresumable run (finding 29):
 * the resume exists precisely to recover a crashed one. Only this exact
 * shape is exempt; every other leftover still refuses.
 */
function isAtomicWriteTemp(entry: string): boolean {
  return /\.json\.\d+\.tmp$/.test(entry);
}

/**
 * Refuse every damaged trace state catchable BEFORE the report writes
 * (T1-T7). Called by the pipeline once before scoring (no report yet -
 * T1-T6) and once before the write (with the report - the full set), so
 * both fresh and resumed runs are covered.
 */
export function validateBenchTrace(opts: BenchTraceValidationOptions): void {
  const resultsDir = benchResultsDir(opts.runDir);
  const expectedIds = opts.fixture.questions.map((question) => question.id);
  const expectedFiles = new Set(expectedIds.map((id) => `${id}.json`));
  // T6: leftovers in a reused or tampered run dir silently skew nothing
  // today only because they are ignored - which is exactly the problem:
  // they would hide a stale trace from every check below. The atomic
  // writer's own crash-orphaned temp ({@link isAtomicWriteTemp}) is the
  // one exempt shape.
  if (existsSync(resultsDir)) {
    const extras = readdirSync(resultsDir).filter(
      (entry) => !expectedFiles.has(entry) && !isAtomicWriteTemp(entry),
    );
    if (extras.length > 0) {
      refuse(
        BENCH_REFUSAL.traceExtraResults,
        `${resultsDir} holds file(s) beyond the fixture's questions: ${extras.join(", ")} - ` +
          "delete the stale result files or start a fresh run",
      );
    }
  }
  const faulted: string[] = [];
  for (let index = 0; index < opts.fixture.questions.length; index++) {
    const question = opts.fixture.questions[index]!;
    const path = join(resultsDir, `${question.id}.json`);
    // T1: the result file every fixture question was promised.
    if (!existsSync(path)) {
      refuse(
        BENCH_REFUSAL.traceResultMissing,
        `no persisted result for question ${question.id} at ${path} - ` +
          "the retrieve phase did not complete for this run; start a fresh run",
      );
    }
    let raw: unknown;
    try {
      raw = JSON.parse(readFileSync(path, "utf8"));
    } catch (cause) {
      // T2: a crash mid-write (writes are atomic now, but the state must
      // stay refused by name, never a raw SyntaxError).
      refuse(
        BENCH_REFUSAL.traceResultUnparseable,
        `${path} is not valid JSON: ${describeCause(cause)} - ` +
          "the result file was truncated by a crash; start a fresh run",
      );
    }
    if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
      refuse(
        BENCH_REFUSAL.traceResultUnparseable,
        `${path} is not a result object - start a fresh run`,
      );
    }
    const record = raw as Record<string, unknown>;
    // T3: the id INSIDE the file must be the question it was stored under.
    if (record["id"] !== question.id) {
      refuse(
        BENCH_REFUSAL.traceIdMismatch,
        `${path} records id '${String(record["id"])}' but is stored under question ${question.id} - ` +
          "start a fresh run",
      );
    }
    // T4: category-required field ABSENT. The builders always set these
    // fields, so absence is structural (a truncated or foreign trace);
    // present-but-empty is a legitimate retrieval miss and passes.
    const field = requiredTraceField(question.category);
    if (record[field] === undefined || record[field] === null) {
      refuse(
        BENCH_REFUSAL.traceFieldAbsent,
        `question ${question.id} (${question.category}) has no '${field}' in ${path} - ` +
          "a truncated or foreign trace (present-but-empty would be a legitimate miss); start a fresh run",
      );
    }
    // T5: a faulted decision is the harness failing, and the scorer only
    // counts it - the report must never dress harness breakage up as
    // memory behavior. Resume would re-evaluate the same persisted
    // decision, so the repair is a fresh run.
    const decision = record["recall_decision"] as { kind?: string; fault?: string } | undefined;
    if (decision !== undefined && decision !== null && decision.kind === "error") {
      faulted.push(`${question.id} (${String(decision.fault ?? "unknown fault")})`);
    }
  }
  if (faulted.length > 0) {
    refuse(
      BENCH_REFUSAL.traceFaultedDecisions,
      `the decision core faulted on ${faulted.join(", ")} - the report must not publish harness ` +
        "breakage as memory behavior; start a fresh run (a resumed run re-evaluates the same persisted decisions)",
    );
  }
  // T7: the report must agree with the trace it was built from, by the
  // SAME aggregation buildReport uses (standing_instruction excluded from
  // the headline, never from by_category).
  if (opts.report !== undefined) {
    const counts = countQuestionResults(opts.results ?? []);
    const quality = opts.report.quality;
    const drifted: string[] = [];
    if (quality.passed !== counts.passed)
      drifted.push(`passed ${quality.passed} != ${counts.passed}`);
    if (quality.total !== counts.total) drifted.push(`total ${quality.total} != ${counts.total}`);
    for (const [category, bucket] of Object.entries(counts.byCategory)) {
      const published = quality.by_category[category];
      if (
        published === undefined ||
        published.passed !== bucket.passed ||
        published.total !== bucket.total
      ) {
        drifted.push(`by_category.${category} != trace`);
      }
    }
    for (const category of Object.keys(quality.by_category)) {
      if (counts.byCategory[category] === undefined) {
        drifted.push(`by_category.${category} not in trace`);
      }
    }
    if (drifted.length > 0) {
      refuse(
        BENCH_REFUSAL.reportCountsMismatch,
        `the report disagrees with its own trace: ${drifted.join("; ")} - start a fresh run`,
      );
    }
  }
}

/** The aggregate behind `buildReport`'s quality family, shared with T7. */
export interface BenchResultCounts {
  /** Headline passed count, excluding the standing_instruction category. */
  readonly passed: number;
  /** Headline total, same exclusion. */
  readonly total: number;
  /** Per-category passed/total, every category including the excluded one. */
  readonly byCategory: Record<string, { passed: number; total: number }>;
}

/**
 * Aggregate question results the way `buildReport` does. ONE copy of the
 * counting rule - including the standing_instruction headline exclusion -
 * so the report and the T7 cross-check cannot drift apart.
 */
export function countQuestionResults(
  results: ReadonlyArray<BenchQuestionResult>,
): BenchResultCounts {
  const byCategory: Record<string, { passed: number; total: number }> = {};
  let passed = 0;
  let total = 0;
  for (const result of results) {
    const bucket = (byCategory[result.category] ??= { passed: 0, total: 0 });
    bucket.total += 1;
    if (result.pass) {
      bucket.passed += 1;
    }
    // The standing_instruction category never folds into the headline
    // (recall-lane-latency): its rate stays the named line of its own in
    // `by_category`, so adding facet questions to a fixture cannot shift
    // the headline pass rate a baseline compares.
    if (result.category !== "standing_instruction") {
      total += 1;
      if (result.pass) passed += 1;
    }
  }
  return { passed, total, byCategory };
}