Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
11.0 KB
Raw Blame
/**
 * The preregistered decision-model eval specs (cycle 20261020, card
 * t_4e1681b3).
 *
 * "Preregistered" is a commitment with mechanical teeth: before any
 * grading happens, a spec freezes (a) the committed fixture and its
 * content hash, (b) the seed and the hash-order dev/counted split rule
 * with the derived membership as a frozen receipt, (c) the promotion
 * metric with its ceiling and minimum counted n, and (d) the exact
 * threshold values of the use being validated - so a profile can never
 * be marked tuned against thresholds that later moved. The whole spec is
 * content-hashed; every cached grade record and every machine-side
 * tuning mark stamps that hash, so a spec edited after grades were seen
 * invalidates them instead of quietly re-scoping the claim.
 *
 * This file, with the fixtures it references, landed in one reviewable
 * commit BEFORE the eval runner existed (task C1 precedes C2); the
 * runner verifies the fixture and take against the receipt at run time
 * (`eval/take.ts`) and refuses mismatches.
 *
 * Scope: the registry holds the uses with a committed, labeled take.
 * `extract_prefilter` is deliberately absent - its gate is the
 * zero-regret shadow-span rule already measured by its report
 * (`reports/extract-prefilter.ts`), and it keeps that loop. The other
 * advisory uses gain specs the same way when a take is committed for
 * them.
 *
 * Grading protocol, frozen with the specs: the deterministic scorer is
 * canonical, an external judge is advisory only, and graded requests are
 * real spend counted toward the daily cost gate (`eval/grade.ts`).
 */

import { canonicalJson, sha256Hex } from "../integrity/digest.ts";
import { isDecisionModelUse, type DecisionModelUse } from "./contract.ts";
import { RECALL_INJECT_ANY_MIN, RECALL_INJECT_NOTE_MIN } from "./questions.ts";

/** Bumped on any deliberate re-registration; part of the spec hash. */
export const DECISION_EVAL_SPEC_VERSION = 1;

/** The only split rule this build derives receipts with. */
export const DECISION_EVAL_SPLIT_RULE = "hash-order-30-70";

/** Dev share of the take; the counted remainder produces the metric. */
export const DECISION_EVAL_DEV_FRACTION = 0.3;

/** The only promotion metric this build writes marks for. */
export const DECISION_EVAL_METRIC = "counted_error_rate";

/**
 * The preregistered three-arm verdict, the same shape the
 * extract_prefilter report keeps: a use promotes only when the counted
 * error meets the ceiling, does not regress against the deterministic
 * baseline, and the counted n is met; too few counted items is its own
 * arm, not a failure.
 */
export type DecisionEvalVerdict = "promote" | "hold" | "not_enough_evidence";

export const DECISION_EVAL_VERDICTS: ReadonlyArray<DecisionEvalVerdict> = Object.freeze([
  "promote",
  "hold",
  "not_enough_evidence",
]);

/** sha-256 hex digest length. */
const HASH_HEX_LENGTH = 64;

export interface DecisionEvalSpec {
  readonly version: number;
  /** The use this spec validates. */
  readonly use: DecisionModelUse;
  /** The committed fixture file, repo-relative. */
  readonly fixture: string;
  /** The split seed; `sha256Hex(seed + ":" + id)` ranks the take ids. */
  readonly seed: string;
  readonly split: {
    readonly rule: typeof DECISION_EVAL_SPLIT_RULE;
    readonly dev_fraction: number;
  };
  /**
   * The frozen membership receipt: the take ids with their dev/counted
   * assignment, each list sorted lexicographically. Frozen at
   * preregistration so fixture growth cannot silently move the counted
   * boundary; the runner derives the split fresh and refuses mismatches.
   */
  readonly receipt: {
    readonly dev: ReadonlyArray<string>;
    readonly counted: ReadonlyArray<string>;
  };
  /** sha-256 of the canonical fixture document, frozen at preregistration. */
  readonly fixture_hash: string;
  readonly metric: typeof DECISION_EVAL_METRIC;
  /**
   * The counted error rate at or below which the use may be marked
   * tuned. Conservative starting constants: rerank 0.05 (one top-1 miss
   * in twenty), recall_inject 0 - a dropped useful note is regret, the
   * same zero-regret target its report family keeps.
   */
  readonly ceiling: number;
  /** Minimum counted n below which the verdict is "not enough evidence". */
  readonly min_counted: number;
  /**
   * The exact threshold values the use reads from `questions.ts` when it
   * enforces, pinned so a mark written against these values is refused
   * the day they move. Empty where the use reads no probability floor
   * from the table (rerank's reorder is threshold-free).
   */
  readonly thresholds: Readonly<Record<string, number>>;
}

/** Validate one spec. A missing or malformed element is a construction error. */
export function validateDecisionEvalSpec(spec: DecisionEvalSpec): void {
  const at = (what: string): string => `eval spec '${spec.use ?? "?"}': ${what}`;
  if (spec.version !== DECISION_EVAL_SPEC_VERSION) {
    throw new Error(
      at(`version must be ${DECISION_EVAL_SPEC_VERSION}, got ${String(spec.version)}`),
    );
  }
  if (!isDecisionModelUse(spec.use)) {
    throw new Error(at(`use ${JSON.stringify(spec.use)} is not a decision-model use`));
  }
  if (typeof spec.fixture !== "string" || spec.fixture.trim() === "") {
    throw new Error(at("fixture must name the committed fixture file"));
  }
  if (typeof spec.seed !== "string" || spec.seed.trim() === "") {
    throw new Error(at("seed must be a non-empty string"));
  }
  if (spec.split?.rule !== DECISION_EVAL_SPLIT_RULE) {
    throw new Error(at(`split rule must be '${DECISION_EVAL_SPLIT_RULE}'`));
  }
  if (spec.split?.dev_fraction !== DECISION_EVAL_DEV_FRACTION) {
    throw new Error(at(`dev fraction must be ${DECISION_EVAL_DEV_FRACTION}`));
  }
  const receiptOk =
    Array.isArray(spec.receipt?.dev) &&
    Array.isArray(spec.receipt?.counted) &&
    spec.receipt.dev.length > 0 &&
    spec.receipt.counted.length > 0 &&
    [...spec.receipt.dev, ...spec.receipt.counted].every(
      (id) => typeof id === "string" && id.trim() !== "",
    ) &&
    new Set([...spec.receipt.dev, ...spec.receipt.counted]).size ===
      spec.receipt.dev.length + spec.receipt.counted.length;
  if (!receiptOk) {
    throw new Error(at("receipt must freeze non-empty, disjoint dev and counted id lists"));
  }
  if (
    typeof spec.fixture_hash !== "string" ||
    !new RegExp(`^[0-9a-f]{${HASH_HEX_LENGTH}}$`).test(spec.fixture_hash)
  ) {
    throw new Error(at("fixture hash must be a 64-character sha-256 hex digest"));
  }
  if (spec.metric !== DECISION_EVAL_METRIC) {
    throw new Error(at(`metric must be '${DECISION_EVAL_METRIC}'`));
  }
  if (
    typeof spec.ceiling !== "number" ||
    !Number.isFinite(spec.ceiling) ||
    spec.ceiling < 0 ||
    spec.ceiling > 1
  ) {
    throw new Error(at("ceiling must be a number in [0, 1]"));
  }
  if (!Number.isInteger(spec.min_counted) || spec.min_counted < 1) {
    throw new Error(at("minimum counted n must be a positive integer"));
  }
  const thresholds = spec.thresholds ?? {};
  for (const [name, value] of Object.entries(thresholds)) {
    if (typeof value !== "number" || !Number.isFinite(value)) {
      throw new Error(at(`thresholds must hold finite numbers, got ${name}: ${String(value)}`));
    }
  }
}

/**
 * The spec hash every grade record and tuning mark stamps: sha-256 over
 * the canonical JSON of the whole spec, so any edit - receipt, seed,
 * ceiling, pinned thresholds, version - invalidates what came before.
 */
export function decisionEvalSpecHash(spec: DecisionEvalSpec): string {
  return sha256Hex(canonicalJson(spec));
}

/**
 * The preregistered specs, one per use with a committed labeled take.
 * `cycle-20261020` is the registration seed for this cycle's takes.
 */
export const DECISION_EVAL_SPECS: Readonly<Partial<Record<DecisionModelUse, DecisionEvalSpec>>> =
  Object.freeze({
    rerank: Object.freeze({
      version: DECISION_EVAL_SPEC_VERSION,
      use: "rerank",
      fixture: "tests/fixtures/decision-model/rerank-take.json",
      seed: "cycle-20261020",
      split: Object.freeze({
        rule: DECISION_EVAL_SPLIT_RULE,
        dev_fraction: DECISION_EVAL_DEV_FRACTION,
      }),
      receipt: Object.freeze({
        dev: Object.freeze([
          "rk-006",
          "rk-007",
          "rk-009",
          "rk-010",
          "rk-012",
          "rk-014",
          "rk-017",
          "rk-026",
          "rk-028",
          "rk-033",
          "rk-038",
          "rk-039",
          "rk-043",
        ]),
        counted: Object.freeze([
          "rk-001",
          "rk-002",
          "rk-003",
          "rk-004",
          "rk-005",
          "rk-008",
          "rk-011",
          "rk-013",
          "rk-015",
          "rk-016",
          "rk-018",
          "rk-019",
          "rk-020",
          "rk-021",
          "rk-022",
          "rk-023",
          "rk-024",
          "rk-025",
          "rk-027",
          "rk-029",
          "rk-030",
          "rk-031",
          "rk-032",
          "rk-034",
          "rk-035",
          "rk-036",
          "rk-037",
          "rk-040",
          "rk-041",
          "rk-042",
        ]),
      }),
      fixture_hash: "1c0553c47a2288b35bdd4c54a234ffa12642290d4a337d122020d96441615308",
      metric: DECISION_EVAL_METRIC,
      ceiling: 0.05,
      min_counted: 30,
      // The enforced reorder consults no probability floor in
      // questions.ts; the no-regression rule carries the quality burden.
      thresholds: Object.freeze({}),
    }),
    recall_inject: Object.freeze({
      version: DECISION_EVAL_SPEC_VERSION,
      use: "recall_inject",
      fixture: "tests/fixtures/decision-model/recall-inject-take.json",
      seed: "cycle-20261020",
      split: Object.freeze({
        rule: DECISION_EVAL_SPLIT_RULE,
        dev_fraction: DECISION_EVAL_DEV_FRACTION,
      }),
      receipt: Object.freeze({
        dev: Object.freeze([
          "ri-001",
          "ri-003",
          "ri-006",
          "ri-008",
          "ri-011",
          "ri-020",
          "ri-021",
          "ri-024",
          "ri-027",
        ]),
        counted: Object.freeze([
          "ri-002",
          "ri-004",
          "ri-005",
          "ri-007",
          "ri-009",
          "ri-010",
          "ri-012",
          "ri-013",
          "ri-014",
          "ri-015",
          "ri-016",
          "ri-017",
          "ri-018",
          "ri-019",
          "ri-022",
          "ri-023",
          "ri-025",
          "ri-026",
          "ri-028",
          "ri-029",
        ]),
      }),
      fixture_hash: "396e5e459d3a30ab5dd5c0df9d73187c42a6e9348b91f98868a195d66a850e93",
      metric: DECISION_EVAL_METRIC,
      // A dropped useful note is lost context: the target is zero regret,
      // exactly what the extract_prefilter report family demands.
      ceiling: 0,
      min_counted: 20,
      // Pinned to the enforce floors this use actually reads.
      thresholds: Object.freeze({
        any_min: RECALL_INJECT_ANY_MIN,
        note_min: RECALL_INJECT_NOTE_MIN,
      }),
    }),
  });

/** The spec registered for `use`, or null when none is (yet). */
export function decisionEvalSpecFor(use: DecisionModelUse): DecisionEvalSpec | null {
  return DECISION_EVAL_SPECS[use] ?? null;
}