Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
12.7 KB
Raw Blame
/**
 * Bench fixture loading, validation, hashing, and materialization
 * (Memory Observability Suite, t_882c396a).
 *
 * Fixtures are repo-local JSON documents. Validation fails fast with
 * named errors (mutation posture); materialization writes ONLY into
 * the caller-supplied disposable directory - note paths are checked
 * against traversal and absolutes at parse time.
 */

import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, isAbsolute, join, normalize } from "node:path";

import { canonicalJson, sha256Hex } from "../integrity/digest.ts";
import { appendContinuityRecord } from "../brain/continuity/store.ts";
import {
  BENCH_CATEGORIES,
  type BenchCategory,
  type BenchFixture,
  type BenchFixtureContinuity,
  type BenchFixtureNote,
  type BenchJudgeInput,
  type BenchQuestion,
  RETRIEVAL_CATEGORIES,
} from "./types.ts";

/**
 * Named relaxations of the canonical fixture invariants. Each option is
 * an opt-in a caller must vouch for explicitly - never a looser default.
 */
export interface ParseBenchFixtureOptions {
  /**
   * Accept `notes: []`. The replay converter passes this: a recorded
   * session with no ingested notes has nothing to materialize, and its
   * prompts are judged against an empty vault. Committed fixture files
   * keep the non-empty requirement.
   */
  readonly allowEmptyNotes?: boolean;
}

/** Parse and validate a raw fixture document. Throws named errors. */
export function parseBenchFixture(raw: unknown, opts?: ParseBenchFixtureOptions): BenchFixture {
  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
    throw new Error("bench fixture: expected a JSON object");
  }
  const record = raw as Record<string, unknown>;
  const name = nonEmptyString(record["name"]);
  if (name === undefined) throw new Error("bench fixture: missing or empty name");
  const description = nonEmptyString(record["description"]);

  const notesRaw = record["notes"];
  if (!Array.isArray(notesRaw) || (notesRaw.length === 0 && opts?.allowEmptyNotes !== true)) {
    throw new Error("bench fixture: notes must be a non-empty array");
  }
  const notes = notesRaw.map(parseNote);

  const continuityRaw = record["continuity"];
  const continuity =
    continuityRaw === undefined
      ? []
      : Array.isArray(continuityRaw)
        ? continuityRaw.map(parseContinuity)
        : (() => {
            throw new Error("bench fixture: continuity must be an array");
          })();

  const questionsRaw = record["questions"];
  if (!Array.isArray(questionsRaw) || questionsRaw.length === 0) {
    throw new Error("bench fixture: at least one question is required");
  }
  const questions = questionsRaw.map(parseQuestion);
  const ids = new Set(questions.map((question) => question.id));
  if (ids.size !== questions.length) {
    throw new Error("bench fixture: question ids must be unique");
  }

  return Object.freeze({
    name,
    ...(description !== undefined ? { description } : {}),
    notes: Object.freeze(notes),
    continuity: Object.freeze(continuity),
    questions: Object.freeze(questions),
  });
}

/** Load a fixture from a JSON file. Fail-fast on unreadable or broken input. */
export function loadBenchFixture(path: string): BenchFixture {
  return parseBenchFixture(JSON.parse(readFileSync(path, "utf8")));
}

/** Hex characters kept from the fixture digest - enough to name a fixture in a
 * report without carrying a full 64-character hash through every line. */
const FIXTURE_HASH_HEX_CHARS = 16;

/** Canonical content hash - stable across key order, sensitive to content. */
export function fixtureHash(fixture: BenchFixture): string {
  return sha256Hex(canonicalJson(fixture)).slice(0, FIXTURE_HASH_HEX_CHARS);
}

/** Write the fixture's notes and continuity records into a disposable vault. */
export function materializeBenchVault(fixture: BenchFixture, vaultDir: string): void {
  mkdirSync(join(vaultDir, "Brain"), { recursive: true });
  for (const note of fixture.notes) {
    const target = join(vaultDir, note.path);
    mkdirSync(dirname(target), { recursive: true });
    writeFileSync(target, note.body, "utf8");
  }
  for (const record of fixture.continuity) {
    appendContinuityRecord(vaultDir, {
      kind: record.kind,
      createdAt: record.created_at,
      payload: record.payload,
    });
  }
}

function parseNote(raw: unknown, index: number): BenchFixtureNote {
  if (raw === null || typeof raw !== "object") {
    throw new Error(`bench fixture: note ${index} must be an object`);
  }
  const record = raw as Record<string, unknown>;
  const path = nonEmptyString(record["path"]);
  const body = typeof record["body"] === "string" ? record["body"] : undefined;
  if (path === undefined || body === undefined) {
    throw new Error(`bench fixture: note ${index} needs path and body`);
  }
  if (isAbsolute(path) || normalize(path).split(/[\\/]/).includes("..")) {
    throw new Error(`bench fixture: note path escapes the vault: ${path}`);
  }
  return Object.freeze({ path, body });
}

function parseContinuity(raw: unknown, index: number): BenchFixtureContinuity {
  if (raw === null || typeof raw !== "object") {
    throw new Error(`bench fixture: continuity ${index} must be an object`);
  }
  const record = raw as Record<string, unknown>;
  if (record["kind"] !== "session_turn") {
    throw new Error(`bench fixture: continuity ${index} kind must be session_turn`);
  }
  const createdAt = nonEmptyString(record["created_at"]);
  const payload = record["payload"];
  if (createdAt === undefined || payload === null || typeof payload !== "object") {
    throw new Error(`bench fixture: continuity ${index} needs created_at and payload`);
  }
  return Object.freeze({
    kind: "session_turn",
    created_at: createdAt,
    payload: Object.freeze({ ...(payload as Record<string, unknown>) }),
  });
}

function parseQuestion(raw: unknown, index: number): BenchQuestion {
  if (raw === null || typeof raw !== "object") {
    throw new Error(`bench fixture: question ${index} must be an object`);
  }
  const record = raw as Record<string, unknown>;
  const id = nonEmptyString(record["id"]);
  if (id === undefined) throw new Error(`bench fixture: question ${index} needs an id`);
  const category = nonEmptyString(record["category"]);
  if (category === undefined || !isBenchCategory(category)) {
    throw new Error(
      `bench fixture: question ${id} has an unknown category (expected one of ${BENCH_CATEGORIES.join(", ")})`,
    );
  }
  const judgeInput = parseJudgeInput(record["judge_input"]);
  const question: BenchQuestion = Object.freeze({
    id,
    category,
    ...(nonEmptyString(record["query"]) !== undefined
      ? { query: nonEmptyString(record["query"]) }
      : {}),
    ...(positiveInt(record["top_k"]) !== undefined ? { top_k: positiveInt(record["top_k"]) } : {}),
    ...(stringArray(record["expected_paths"]) !== undefined
      ? { expected_paths: stringArray(record["expected_paths"]) }
      : {}),
    ...(stringArray(record["not_expected_above"]) !== undefined
      ? { not_expected_above: stringArray(record["not_expected_above"]) }
      : {}),
    ...(nonEmptyString(record["session_id"]) !== undefined
      ? { session_id: nonEmptyString(record["session_id"]) }
      : {}),
    ...(positiveInt(record["expected_turns"]) !== undefined
      ? { expected_turns: positiveInt(record["expected_turns"]) }
      : {}),
    ...(nonEmptyString(record["expected_text"]) !== undefined
      ? { expected_text: nonEmptyString(record["expected_text"]) }
      : {}),
    ...(stringArray(record["expected_ids"]) !== undefined
      ? { expected_ids: stringArray(record["expected_ids"]) }
      : {}),
    ...(positiveInt(record["max_tokens"]) !== undefined
      ? { max_tokens: positiveInt(record["max_tokens"]) }
      : {}),
    ...(positiveInt(record["max_total_chars"]) !== undefined
      ? { max_total_chars: positiveInt(record["max_total_chars"]) }
      : {}),
    ...(nonEmptyString(record["prompt"]) !== undefined
      ? { prompt: nonEmptyString(record["prompt"]) }
      : {}),
    // Replay glue (surfacing-replay): the prompt this question follows,
    // consumed by the structural gate the phase pipeline reproduces.
    // Present only when the fixture carries it, so every fixture without
    // one parses byte-identically.
    ...(nonEmptyString(record["previous_prompt"]) !== undefined
      ? { previous_prompt: nonEmptyString(record["previous_prompt"]) }
      : {}),
    // `expect_inject` is the one boolean field, and `false` is its most
    // load-bearing value (the anti-gaming half of the fixture), so it is
    // read by type rather than by truthiness.
    ...(typeof record["expect_inject"] === "boolean"
      ? { expect_inject: record["expect_inject"] }
      : {}),
    ...(nonEmptyString(record["intake_text"]) !== undefined
      ? { intake_text: nonEmptyString(record["intake_text"]) }
      : {}),
    ...(stringArray(record["expected_labels"]) !== undefined
      ? { expected_labels: stringArray(record["expected_labels"]) }
      : {}),
    ...(stringArray(record["expected_provenance"]) !== undefined
      ? { expected_provenance: stringArray(record["expected_provenance"]) }
      : {}),
    ...(nonEmptyString(record["agent_scope"]) !== undefined
      ? { agent_scope: nonEmptyString(record["agent_scope"]) }
      : {}),
    ...(stringArray(record["forbidden_ids"]) !== undefined
      ? { forbidden_ids: stringArray(record["forbidden_ids"]) }
      : {}),
    // Judge glue (longmemeval): present only when the fixture carries
    // it, so every default-path fixture parses byte-identically.
    ...(judgeInput !== undefined ? { judge_input: judgeInput } : {}),
  });
  validateQuestionShape(question);
  return question;
}

/**
 * Parse the optional official-prompt judge input. Absent means absent;
 * present it is validated as a whole - a half-stated input is a named
 * error, never a silently partial payload.
 */
function parseJudgeInput(raw: unknown): BenchJudgeInput | undefined {
  if (raw === undefined) return undefined;
  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
    throw new Error("bench fixture: judge_input must be an object");
  }
  const record = raw as Record<string, unknown>;
  const question = nonEmptyString(record["question"]);
  if (question === undefined) throw new Error("bench fixture: judge_input needs a question");
  const answer = nonEmptyString(record["answer"]);
  const gold = stringArray(record["gold"]);
  if (gold === undefined || gold.length === 0) {
    throw new Error("bench fixture: judge_input needs a non-empty gold array");
  }
  return Object.freeze({
    question,
    ...(answer !== undefined ? { answer } : {}),
    gold: Object.freeze([...gold]),
  });
}

function validateQuestionShape(question: BenchQuestion): void {
  if (RETRIEVAL_CATEGORIES.has(question.category)) {
    if (question.query === undefined || (question.expected_paths ?? []).length === 0) {
      throw new Error(
        `bench fixture: question ${question.id} (${question.category}) needs query and expected_paths`,
      );
    }
  }
  if (question.category === "session_handoff" && question.session_id === undefined) {
    throw new Error(`bench fixture: question ${question.id} needs session_id`);
  }
  if (question.category === "budget" && (question.expected_ids ?? []).length === 0) {
    throw new Error(`bench fixture: question ${question.id} needs expected_ids`);
  }
  if (question.category === "proactive_recall") {
    if (question.prompt === undefined || question.expect_inject === undefined) {
      throw new Error(
        `bench fixture: question ${question.id} (proactive_recall) needs prompt and expect_inject`,
      );
    }
  }
  if (question.category === "write_fidelity") {
    if (question.intake_text === undefined || (question.expected_labels ?? []).length === 0) {
      throw new Error(
        `bench fixture: question ${question.id} (write_fidelity) needs intake_text and expected_labels`,
      );
    }
  }
  if (question.category === "source_isolation") {
    if (question.agent_scope === undefined || (question.forbidden_ids ?? []).length === 0) {
      throw new Error(
        `bench fixture: question ${question.id} (source_isolation) needs agent_scope and forbidden_ids`,
      );
    }
  }
}

function isBenchCategory(value: string): value is BenchCategory {
  return (BENCH_CATEGORIES as ReadonlyArray<string>).includes(value);
}

function nonEmptyString(value: unknown): string | undefined {
  return typeof value === "string" && value.trim().length > 0 ? value : undefined;
}

function positiveInt(value: unknown): number | undefined {
  return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : undefined;
}

function stringArray(value: unknown): ReadonlyArray<string> | undefined {
  if (!Array.isArray(value)) return undefined;
  if (!value.every((item) => typeof item === "string" && item.length > 0)) return undefined;
  return Object.freeze([...value]);
}