Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
10.9 KB
Raw Blame
/**
 * `o2b decision-model eval run|list` and the doctor probe's measured
 * state (cycle 20261020, card t_4e1681b3, task C4).
 *
 * `eval run` grades the preregistered take over the real adapter - the
 * loopback fake here, never a hosted route - and a promote verdict
 * writes the machine-side tuning mark under the state root, which the
 * config resolver and the doctor then honor. `eval list` round-trips
 * the cached grade records and skips malformed ones with a named note.
 */

import {
  afterAll,
  afterEach,
  beforeAll,
  beforeEach,
  describe,
  expect,
  setDefaultTimeout,
  test,
} from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { probeDecisionModel } from "../../src/core/doctor-readiness.ts";
import {
  decisionEvalSpecFor,
  decisionEvalSpecHash,
} from "../../src/core/decision-model/eval-spec.ts";
import { FAKE_DECISION_KEY } from "../helpers/fake-credentials.ts";
import { startFakeSystemOne, type FakeSystemOne } from "../helpers/fake-decision-provider.ts";
import { runCli } from "../helpers/run-cli.ts";
import { CLI_SPAWN_BUDGET_MS } from "../helpers/cli-timeout.ts";

// Tests here spawn processes; see tests/helpers/cli-timeout.ts.
setDefaultTimeout(CLI_SPAWN_BUDGET_MS);

const KEY_VAR = "O2B_TEST_DECISION_EVAL_KEY";

let server: FakeSystemOne;
let tmp: string;
let vault: string;
let configPath: string;
let stateDir: string;

beforeAll(async () => {
  server = await startFakeSystemOne();
});
afterAll(async () => {
  await server.close();
});
beforeEach(() => {
  tmp = mkdtempSync(join(tmpdir(), "osb-dm-eval-cli-"));
  vault = join(tmp, "vault");
  mkdirSync(join(vault, "Brain"), { recursive: true });
  configPath = join(tmp, "config.yaml");
  stateDir = join(tmp, "state");
  server.requests.length = 0;
});
afterEach(() => {
  rmSync(tmp, { recursive: true, force: true });
});

function writeConfig(entries: Record<string, string>): void {
  const lines = [`vault: "${vault}"`, ...Object.entries(entries).map(([k, v]) => `${k}: "${v}"`)];
  writeFileSync(configPath, lines.join("\n") + "\n");
}

/** Env for a CLI child: the loopback key, this vault's config, this state root. */
function childEnv(extra: Record<string, string> = {}): Record<string, string> {
  return {
    OPEN_SECOND_BRAIN_CONFIG: configPath,
    XDG_STATE_HOME: stateDir,
    [KEY_VAR]: FAKE_DECISION_KEY,
    ...extra,
  };
}

/** The active eval config; the loopback base URL is added at call time. */
const ENABLED = {
  decision_model_enabled: "true",
  decision_model_provider: "laya",
  decision_model_uses: "rerank:shadow",
};

function writeEnabledConfig(): void {
  writeConfig({ ...ENABLED, decision_model_base_url: server.url, decision_model_id: "english" });
}

/**
 * Answer every relevance question by the fixtures' labeling rule, read
 * off the request body the adapter actually sent: an on-topic passage
 * helps. This is the same rule the committed gold labels follow.
 */
function answerByLabel(req: { body: Record<string, unknown> }): {
  json: unknown;
} {
  const state = req.body["state"] as {
    query?: string;
    passages?: Record<string, string>;
  };
  const subject = /the (.+)\?/.exec(state.query ?? "")![1] ?? "\u0000";
  const questions = req.body["questions"] as Record<string, { type: string }>;
  const answers: Record<string, unknown> = {};
  for (const [id, q] of Object.entries(questions)) {
    const key = id.startsWith("rel_") ? `P${id.slice(4)}` : null;
    const text = key === null ? "" : (state.passages?.[key] ?? "");
    answers[id] = { type: q.type, noul: text.includes(subject) ? 0.99 : 0.01 };
  }
  return {
    json: {
      model: "fake-model-1.0",
      answers,
      usage: { input_tokens: 321, output_tokens: 7 },
    },
  };
}

describe("o2b decision-model eval", () => {
  test("an unknown subcommand prints usage and exits 2", async () => {
    const res = await runCli(["decision-model", "eval", "--config", configPath], {
      env: { OPEN_SECOND_BRAIN_CONFIG: configPath },
    });
    expect(res.returncode).toBe(2);
    expect(res.stderr).toContain("usage: o2b decision-model");
    expect(res.stderr).toContain("eval run");
    expect(res.stderr).toContain("eval list");
  });

  test("eval run refuses an unknown use, a specless use and an inactive config", async () => {
    writeEnabledConfig();
    const env = childEnv();
    const bad = await runCli(
      ["decision-model", "eval", "run", "--use", "nonsense", "--config", configPath],
      { env },
    );
    expect(bad.returncode).toBe(2);
    expect(bad.stderr).toContain("decision-model use");

    const specless = await runCli(
      ["decision-model", "eval", "run", "--use", "dedup", "--config", configPath],
      { env },
    );
    expect(specless.returncode).toBe(2);
    expect(specless.stderr).toContain("no preregistered spec");

    // An inactive config is a refusal: a keyed preset without its key is
    // no_key (the keyless laya loopback above cannot produce one).
    writeConfig({
      decision_model_enabled: "true",
      decision_model_provider: "compatible",
      decision_model_base_url: server.url,
      decision_model_id: "fake-model-1",
      decision_model_env_key: KEY_VAR,
      decision_model_uses: "rerank:shadow",
    });
    const noKey = await runCli(
      ["decision-model", "eval", "run", "--use", "rerank", "--config", configPath],
      { env: { OPEN_SECOND_BRAIN_CONFIG: configPath, XDG_STATE_HOME: stateDir } },
    );
    expect(noKey.returncode).toBe(2);
    expect(noKey.stderr).toContain("o2b decision-model check");
  });

  test("eval run refuses a use that is off before sending or caching anything", async () => {
    // decision_model_uses lists only rerank: recall_inject defaults to
    // off, and the runner's mode override would send nothing, score
    // every item degraded, cache that junk grade and exit 0 - a named
    // refusal is owed instead, before any of that happens.
    writeEnabledConfig();
    server.setReply(answerByLabel);
    const res = await runCli(
      ["decision-model", "eval", "run", "--use", "recall_inject", "--config", configPath],
      { env: childEnv() },
    );
    expect(res.returncode).toBe(2);
    expect(res.stderr).toContain("recall_inject");
    expect(res.stderr).toContain("decision_model_uses");
    expect(server.requests).toHaveLength(0);
    const listJson = await runCli(
      ["decision-model", "eval", "list", "--config", configPath, "--json"],
      { env: childEnv() },
    );
    expect(listJson.returncode).toBe(0);
    const listed = JSON.parse(listJson.stdout) as { grades: unknown[]; skipped: unknown[] };
    expect(listed.grades).toHaveLength(0);
    expect(listed.skipped).toHaveLength(0);
  }, 30000);

  test("a promote run writes the mark, caches the grade and the doctor names it", async () => {
    writeEnabledConfig();
    server.setReply(answerByLabel);
    const res = await runCli(
      ["decision-model", "eval", "run", "--use", "rerank", "--config", configPath, "--json"],
      { env: childEnv() },
    );
    expect(res.returncode).toBe(0);
    const run = JSON.parse(res.stdout) as {
      verdict: string;
      mark_written: boolean;
      grade_record_id: string | null;
      counted: { n: number; error: number | null };
    };
    expect(run.verdict).toBe("promote");
    expect(run.mark_written).toBe(true);
    expect(run.counted).toMatchObject({ n: 30, error: 0 });

    // The machine-side mark exists under this run's state root, for the
    // resolved profile and use.
    const markPath = join(
      stateDir,
      "open-second-brain",
      "decision-model",
      "tuning-marks",
      "laya",
      "rerank.json",
    );
    expect(existsSync(markPath)).toBe(true);
    const mark = JSON.parse(readFileSync(markPath, "utf8")) as { verdict: string; use: string };
    expect(mark).toMatchObject({ verdict: "promote", use: "rerank" });

    // The grade record is cached in the vault and round-trips through
    // eval list, in prose and as JSON.
    expect(run.grade_record_id).not.toBeNull();
    const listJson = await runCli(
      ["decision-model", "eval", "list", "--use", "rerank", "--config", configPath, "--json"],
      { env: childEnv() },
    );
    expect(listJson.returncode).toBe(0);
    const listed = JSON.parse(listJson.stdout) as {
      grades: Array<{ use: string; verdict: string; spec_hash: string }>;
      skipped: string[];
    };
    expect(listed.grades).toHaveLength(1);
    expect(listed.grades[0]).toMatchObject({ use: "rerank", verdict: "promote" });
    expect(listed.skipped).toEqual([]);

    const listProse = await runCli(["decision-model", "eval", "list", "--config", configPath], {
      env: childEnv(),
    });
    const spec12 = decisionEvalSpecHash(decisionEvalSpecFor("rerank")!).slice(0, 12);
    expect(listProse.stdout).toContain("eval grades: 1 record(s)");
    expect(listProse.stdout).toContain("rerank (laya) promote");
    expect(listProse.stdout).toContain(`spec ${spec12}`);

    // The doctor probe names the measured state for the active config.
    const verdict = await probeDecisionModel({
      vault,
      config: configPath,
      env: childEnv() as unknown as NodeJS.ProcessEnv,
    });
    expect(verdict.status).toBe("pass");
    expect(verdict.detail).toContain("measured under spec hash");
    expect(verdict.detail).toContain("(rerank)");
  }, 30000);

  test("eval list skips a malformed record with a named note, in prose and JSON", async () => {
    writeEnabledConfig();
    server.setReply(answerByLabel);
    await runCli(["decision-model", "eval", "run", "--use", "rerank", "--config", configPath], {
      env: childEnv(),
    });
    // A hand-written record that claims a measurement without the fields.
    const { appendContinuityRecord } = await import("../../src/core/brain/continuity/store.ts");
    appendContinuityRecord(vault, {
      kind: "decision_model_eval_grade",
      createdAt: new Date().toISOString(),
      sourceRefs: [],
      payload: { use: "rerank", verdict: "promote" },
    });

    const listJson = await runCli(
      ["decision-model", "eval", "list", "--config", configPath, "--json"],
      {
        env: childEnv(),
      },
    );
    const listed = JSON.parse(listJson.stdout) as { grades: unknown[]; skipped: string[] };
    expect(listed.grades).toHaveLength(1);
    expect(listed.skipped).toHaveLength(1);
    expect(listed.skipped[0]).toMatch(/decision_model_eval_grade record .+ skipped: spec_hash/);

    const listProse = await runCli(["decision-model", "eval", "list", "--config", configPath], {
      env: childEnv(),
    });
    expect(listProse.stdout).toContain("skipped: decision_model_eval_grade record");
  }, 30000);

  test("eval list with no records says so and exits 0", async () => {
    writeEnabledConfig();
    const res = await runCli(["decision-model", "eval", "list", "--config", configPath], {
      env: childEnv(),
    });
    expect(res.returncode).toBe(0);
    expect(res.stdout).toContain("eval grades: none cached");
  });
});