Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
11.5 KB
Raw Blame
/**
 * The eval surfaces (cycle 20261020, card t_4e1681b3, task C4): the
 * measured tuning state in `o2b decision-model check`, the eval section
 * of `report`, and the prose/JSON rendering the CLI shows for both.
 *
 * The check is where an untuned use's shadow hold is explained; with a
 * valid machine-side mark the same line names the spec hash the mark was
 * accepted under, and a tampered or stale mark reads as untuned again -
 * the fail-closed direction, visible on the surface.
 */

import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

import type { DecisionRequest } from "../../../src/core/decision-model/contract.ts";
import {
  buildDecisionModelCheck,
  buildDecisionModelReport,
  renderDecisionModelCheck,
  renderDecisionModelReport,
} from "../../../src/core/decision-model/diagnostics.ts";
import {
  decisionEvalSpecFor,
  decisionEvalSpecHash,
} from "../../../src/core/decision-model/eval-spec.ts";
import { runDecisionModelEval } from "../../../src/core/decision-model/eval/grade.ts";
import {
  decisionModelTuningMarkPath,
  resetTuningMarkCache,
} from "../../../src/core/decision-model/eval/mark.ts";
import type { ResolvedDecisionModelConfig } from "../../../src/core/decision-model/config.ts";
import type { PlatformDirsEnv } from "../../../src/core/platform-dirs.ts";
import { appendContinuityRecord } from "../../../src/core/brain/continuity/store.ts";
import { resetDecisionSpendCache } from "../../../src/core/decision-model/record.ts";
import { FakeDecisionProvider } from "../../helpers/fake-decision-provider.ts";

let tmp: string;
let vault: string;
let stateEnv: PlatformDirsEnv;

beforeEach(() => {
  tmp = mkdtempSync(join(tmpdir(), "osb-dm-eval-surface-"));
  vault = join(tmp, "vault");
  mkdirSync(join(vault, "Brain"), { recursive: true });
  stateEnv = { platform: "linux", home: tmp, env: { XDG_STATE_HOME: join(tmp, "state") } };
  resetTuningMarkCache();
  resetDecisionSpendCache();
});
afterEach(() => {
  rmSync(tmp, { recursive: true, force: true });
  resetTuningMarkCache();
  resetDecisionSpendCache();
});

/** Answers by the fixtures' labeling rule: an on-topic passage helps. */
function byLabelRule(id: string, req: DecisionRequest): number {
  const state = req.state as {
    query?: string;
    prompt?: string;
    passages?: Record<string, string>;
    notes?: Record<string, string>;
  };
  const subject = /the (.+)\?/.exec(state.query ?? state.prompt ?? "")![1]!;
  const key = id.startsWith("rel_") ? `P${id.slice(4)}` : null;
  if (key === null) return 0.5;
  const text = state.passages?.[key] ?? "";
  return text.includes(subject) ? 0.99 : 0.01;
}

function evalConfig(): ResolvedDecisionModelConfig {
  const uses = {
    rerank: "shadow",
    answerable: "off",
    skills: "off",
    extract_prefilter: "off",
    dedup: "off",
    tension: "off",
    labels: "off",
    recall_inject: "off",
  } as ResolvedDecisionModelConfig["uses"];
  return {
    status: "active",
    errors: [],
    notes: [],
    enabled: true,
    provider: "laya",
    adapter: "systemone",
    baseUrl: "http://127.0.0.1:8000",
    model: "english",
    envKey: null,
    keyPresent: true,
    allowInsecureHttp: false,
    timeoutMs: 3000,
    hookBudgetMs: 700,
    maxStateTokens: 32_000,
    uses,
    configuredUses: uses,
    dailyCostGateUsd: 0.5,
    inputPriceUsdPerMtok: 0.042,
    allowUncalibrated: false,
    calibrated: true,
    thresholdProfile: "laya",
    shadowOnlyUses: ["rerank"],
    processor: null,
    vault,
  };
}

const CHECK_CONFIG = {
  decision_model_enabled: "true",
  decision_model_provider: "laya",
  decision_model_uses: "rerank:shadow",
} as const;

function checkWith(markSource: PlatformDirsEnv | null) {
  return buildDecisionModelCheck({
    config: { ...CHECK_CONFIG },
    vault,
    markSource,
    thresholdsPath: null,
  });
}

describe("the check's measured tuning state", () => {
  test("no mark reads as untuned, on the record and in the prose", async () => {
    const report = await checkWith(stateEnv);
    expect(report.tuning).toMatchObject({ profile: "laya", state: "untuned", marked_uses: [] });
    const text = renderDecisionModelCheck(report);
    expect(text).toContain("tuning: profile 'laya' is untuned");
    expect(text).toContain("eval run");
  });

  test("a valid mark reads as measured under its spec hash, naming the uses", async () => {
    await runDecisionModelEval("rerank", {
      config: evalConfig(),
      provider: new FakeDecisionProvider({ answer: byLabelRule }),
      markSource: stateEnv,
    });
    const report = await checkWith(stateEnv);
    const spec = decisionEvalSpecFor("rerank")!;
    expect(report.tuning.state).toBe("measured");
    expect(report.tuning.marked_uses).toEqual([
      { use: "rerank", spec_hash: decisionEvalSpecHash(spec) },
    ]);
    const text = renderDecisionModelCheck(report);
    expect(text).toContain(
      `tuning: profile 'laya' measured under spec hash ${decisionEvalSpecHash(spec).slice(0, 12)}… (rerank)`,
    );
  });

  test("a mark measured on model A reads as measured elsewhere, not measured, on model B", async () => {
    // The mark is bound to the endpoint that earned it (provider, model,
    // base-url hash). The gate reader refuses it for another endpoint and
    // holds the use shadow - so the check must not claim measured-enforce
    // here either: it names the mark and why it does not arm.
    await runDecisionModelEval("rerank", {
      config: evalConfig(),
      provider: new FakeDecisionProvider({ answer: byLabelRule }),
      markSource: stateEnv,
    });
    const spec = decisionEvalSpecFor("rerank")!;
    const elsewhere = await buildDecisionModelCheck({
      config: { ...CHECK_CONFIG, decision_model_id: "fake-model-2" },
      vault,
      markSource: stateEnv,
      thresholdsPath: null,
    });
    expect(elsewhere.tuning.state).toBe("measured_elsewhere");
    expect(elsewhere.tuning.marked_uses).toEqual([]);
    expect(elsewhere.tuning.elsewhere_uses).toEqual([
      { use: "rerank", spec_hash: decisionEvalSpecHash(spec) },
    ]);
    const text = renderDecisionModelCheck(elsewhere);
    expect(text).toContain("on another endpoint (provider, model or base-url mismatch)");
    expect(text).toContain("enforce holds as shadow");
    // The same holds when only the base URL moved and the model stayed.
    const movedUrl = await buildDecisionModelCheck({
      config: { ...CHECK_CONFIG, decision_model_base_url: "http://127.0.0.1:10" },
      vault,
      markSource: stateEnv,
      thresholdsPath: null,
    });
    expect(movedUrl.tuning.state).toBe("measured_elsewhere");
    expect(renderDecisionModelCheck(movedUrl)).toContain("on another endpoint");
  });

  test("a tampered mark fails closed to untuned on the surface", async () => {
    await runDecisionModelEval("rerank", {
      config: evalConfig(),
      provider: new FakeDecisionProvider({ answer: byLabelRule }),
      markSource: stateEnv,
    });
    const path = decisionModelTuningMarkPath(stateEnv, "laya", "rerank");
    const valid = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>;
    writeFileSync(path, JSON.stringify({ ...valid, fixture_hash: "0".repeat(64) }), "utf8");
    resetTuningMarkCache();
    const report = await checkWith(stateEnv);
    expect(report.tuning.state).toBe("untuned");
  });

  test("the baseline profile says baseline; no profile says none", async () => {
    // A hosted Jev-family route carries the baseline profile by preset.
    const baseline = await buildDecisionModelCheck({
      config: {
        decision_model_enabled: "true",
        decision_model_provider: "vercel",
        decision_model_uses: "rerank:shadow",
      },
      vault,
      markSource: stateEnv,
      env: { AI_GATEWAY_API_KEY: "unused" } as unknown as NodeJS.ProcessEnv,
    });
    expect(baseline.tuning).toMatchObject({ profile: "jev-1.13", state: "baseline" });
    expect(renderDecisionModelCheck(baseline)).toContain("is the baseline");

    const noProfile = await buildDecisionModelCheck({
      config: {
        decision_model_enabled: "true",
        decision_model_provider: "compatible",
        decision_model_base_url: "http://127.0.0.1:9",
        decision_model_id: "fake-model-1",
        decision_model_uses: "rerank:shadow",
      },
      vault,
      markSource: stateEnv,
    });
    expect(noProfile.tuning).toMatchObject({ profile: null, state: "none", marked_uses: [] });
    expect(renderDecisionModelCheck(noProfile)).toContain("no threshold profile resolves");
  });
});

describe("the check's cached eval grades", () => {
  test("lists cached grade records per use and skips malformed ones by count", async () => {
    await runDecisionModelEval("rerank", {
      config: evalConfig(),
      provider: new FakeDecisionProvider({ answer: byLabelRule }),
      markSource: stateEnv,
    });
    appendContinuityRecord(vault, {
      kind: "decision_model_eval_grade",
      createdAt: new Date().toISOString(),
      sourceRefs: [],
      payload: { use: "rerank", verdict: "promote" },
    });
    const report = await checkWith(stateEnv);
    expect(report.eval_grades.total).toBe(1);
    expect(report.eval_grades.skipped).toHaveLength(1);
    expect(report.eval_grades.uses[0]).toMatchObject({
      use: "rerank",
      runs: 1,
      last_verdict: "promote",
    });
    const text = renderDecisionModelCheck(report);
    expect(text).toContain("eval grades: 1 record(s)");
    expect(text).toContain("rerank: 1 run(s), last promote");
    expect(text).toContain("1 skipped");
  });

  test("no grades at all reads as none cached", async () => {
    const report = await checkWith(stateEnv);
    expect(report.eval_grades.total).toBe(0);
    expect(renderDecisionModelCheck(report)).toContain("eval grades: none cached");
  });
});

describe("the report's eval section", () => {
  test("carries the cached grades, oldest first, and skips malformed by note", async () => {
    await runDecisionModelEval("rerank", {
      config: evalConfig(),
      provider: new FakeDecisionProvider({ answer: byLabelRule }),
      markSource: stateEnv,
    });
    await runDecisionModelEval("rerank", {
      config: evalConfig(),
      provider: new FakeDecisionProvider({ answer: () => 0.01 }),
      markSource: stateEnv,
    });
    appendContinuityRecord(vault, {
      kind: "decision_model_eval_grade",
      createdAt: new Date().toISOString(),
      sourceRefs: [],
      payload: { verdict: "promote" },
    });
    const report = buildDecisionModelReport(vault);
    expect(report.eval).toBeDefined();
    expect(report.eval!.grades).toHaveLength(2);
    expect(report.eval!.grades[0]!.verdict).toBe("promote");
    expect(report.eval!.grades[1]!.verdict).toBe("hold");
    expect(report.eval!.skipped).toHaveLength(1);
    expect(report.eval!.skipped[0]).toMatch(/decision_model_eval_grade record .+ skipped:/);

    const text = renderDecisionModelReport(report);
    expect(text).toContain("eval grades: 2 record(s)");
    expect(text).toContain("rerank (laya) promote: counted error 0.000 over n=30");
    // A constant answer leaves the input order in place: the counted
    // error is exactly the input-order baseline's, and the verdict holds.
    expect(text).toContain("rerank (laya) hold: counted error 0.733 over n=30");
    expect(text).toContain("skipped: decision_model_eval_grade record");
  });

  test("a vault with no grades renders no eval section", () => {
    const report = buildDecisionModelReport(vault);
    expect(report.eval).toBeUndefined();
    expect(renderDecisionModelReport(report)).not.toContain("eval grades:");
  });
});