Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
12.5 KB
Raw Blame
/**
 * `o2b decision-model check|report` and `o2b search rerank-eval`
 * (issue #213, Parts 1 and 2). The check is the one surface that reports a
 * missing key; it names the variable and never prints its value.
 */

import {
  afterAll,
  afterEach,
  beforeAll,
  beforeEach,
  describe,
  expect,
  setDefaultTimeout,
  test,
} from "bun:test";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { probeDecisionModel } from "../../src/core/doctor-readiness.ts";
import { listDecisionModelCalls } from "../../src/core/decision-model/record.ts";
import { FAKE_DECISION_KEY } from "../helpers/fake-credentials.ts";
import {
  answerAll,
  startFakeSystemOne,
  type FakeSystemOne,
} from "../helpers/fake-decision-provider.ts";
import { runCli } from "../helpers/run-cli.ts";
import { CLI_SPAWN_BUDGET_MS } from "../helpers/cli-timeout.ts";

// Tests here spawn processes; see tests/helpers/cli-timeout.ts.
setDefaultTimeout(CLI_SPAWN_BUDGET_MS);

const KEY_VAR = "O2B_TEST_DECISION_CLI_KEY";

let server: FakeSystemOne;
let tmp: string;
let vault: string;
let configPath: string;

beforeAll(async () => {
  server = await startFakeSystemOne();
});
afterAll(async () => {
  await server.close();
});
beforeEach(() => {
  tmp = mkdtempSync(join(tmpdir(), "osb-dm-cli-"));
  vault = join(tmp, "vault");
  mkdirSync(join(vault, "Brain"), { recursive: true });
  configPath = join(tmp, "config.yaml");
  server.requests.length = 0;
});
afterEach(() => {
  rmSync(tmp, { recursive: true, force: true });
});

function writeConfig(entries: Record<string, string>): void {
  const lines = [`vault: "${vault}"`, ...Object.entries(entries).map(([k, v]) => `${k}: "${v}"`)];
  writeFileSync(configPath, lines.join("\n") + "\n");
}

const ENABLED = {
  decision_model_enabled: "true",
  decision_model_provider: "compatible",
  decision_model_id: "fake-model-1",
  decision_model_env_key: KEY_VAR,
  decision_model_uses: "rerank:shadow",
};

function check(args: string[] = [], env: Record<string, string> = {}) {
  return runCli(["decision-model", "check", "--config", configPath, ...args], {
    env: { OPEN_SECOND_BRAIN_CONFIG: configPath, ...env },
  });
}

describe("o2b decision-model check", () => {
  test("without config it reports off and exits 0", async () => {
    writeConfig({});
    const res = await check();
    expect(res.returncode).toBe(0);
    expect(res.stdout).toContain("decision model: disabled");
    expect(res.stdout).toContain("key variable: (not named)");
    expect(res.stdout).toContain("hint:");
  });

  test("configured but not enabled: shows the configured uses, hints only what is missing", async () => {
    const { decision_model_enabled: _, ...notEnabled } = ENABLED;
    writeConfig({ ...notEnabled, decision_model_base_url: server.url });
    const res = await check();
    expect(res.stdout).toContain("uses: rerank:shadow");
    expect(res.stdout).toContain("every use listed above runs off");
    expect(res.stdout).toContain(
      'hint: the feature is off; to try it, set decision_model_enabled: "true" in',
    );
    expect(res.stdout).not.toContain("a decision_model_provider");
  });

  test("a vault that tries to enable the feature is warned about in check and doctor", async () => {
    writeFileSync(
      join(vault, "Brain", "_brain.yaml"),
      "schema_version: 1\ndecision_model:\n  enabled: true\n",
    );
    writeConfig({ ...ENABLED, decision_model_base_url: server.url });
    const res = await check();
    expect(res.stdout).toContain("warning: vault: decision_model.enabled: true ignored");
    const verdict = await probeDecisionModel({ vault, config: configPath, env: {} });
    expect(verdict.detail).toContain("decision_model.enabled: true ignored");
  });

  test("enabled without the key: names the variable, says how to set it, exits 0", async () => {
    writeConfig({ ...ENABLED, decision_model_base_url: server.url });
    const res = await check();
    expect(res.returncode).toBe(0);
    expect(res.stdout).toContain("decision model: no_key");
    expect(res.stdout).toContain(`key variable: ${KEY_VAR} (not set)`);
    expect(res.stdout).toContain(`set the environment variable ${KEY_VAR}`);
    expect(server.requests).toHaveLength(0);
  });

  test("active: the key is reported as set and never printed, even with --ping", async () => {
    writeConfig({ ...ENABLED, decision_model_base_url: server.url });
    server.setReply((req) => ({ json: answerAll(req, () => 0.99, "fake-model-1.0") }));
    const res = await check(["--ping"], { [KEY_VAR]: FAKE_DECISION_KEY });
    expect(res.returncode).toBe(0);
    expect(res.stdout).toContain("decision model: active");
    expect(res.stdout).toContain(`key variable: ${KEY_VAR} (set)`);
    expect(res.stdout).toContain("ping: ok, answered by fake-model-1.0");
    expect(res.stdout).toContain("shadow and enforce both send data");
    expect(res.stdout + res.stderr).not.toContain(FAKE_DECISION_KEY);
    expect(server.requests).toHaveLength(1);
    expect(server.requests[0]!.bodyText).not.toContain(vault);
    // The ping is real spend: it is recorded like any request.
    const records = listDecisionModelCalls(vault);
    expect(records).toHaveLength(1);
    expect(records[0]!.payload).toMatchObject({ use: "ping", origin: "ping", outcome: "ok" });
  });

  test("--json carries the same facts without the key value", async () => {
    writeConfig({ ...ENABLED, decision_model_base_url: server.url });
    const res = await check(["--json"], { [KEY_VAR]: FAKE_DECISION_KEY });
    const report = JSON.parse(res.stdout) as Record<string, unknown>;
    expect(report["status"]).toBe("active");
    expect(report["env_key"]).toBe(KEY_VAR);
    expect(report["key_set"]).toBe(true);
    expect(res.stdout).not.toContain(FAKE_DECISION_KEY);
  });

  test("an invalid config exits 1 and names the key", async () => {
    writeConfig({
      ...ENABLED,
      decision_model_base_url: server.url,
      decision_model_uses: "rerank:loud",
    });
    const res = await check([], { [KEY_VAR]: FAKE_DECISION_KEY });
    expect(res.returncode).toBe(1);
    expect(res.stdout).toContain("error: decision_model_uses");
  });

  test("a failed ping of an active provider exits 1", async () => {
    writeConfig({ ...ENABLED, decision_model_base_url: server.url });
    server.setReply(() => ({ status: 401 }));
    const res = await check(["--ping"], { [KEY_VAR]: FAKE_DECISION_KEY });
    expect(res.returncode).toBe(1);
    expect(res.stdout).toContain("ping: failed (http_401)");
  });

  test("rerank endpoint keys are reported as ignored for the decision-model kind", async () => {
    writeConfig({
      ...ENABLED,
      decision_model_base_url: server.url,
      search_rerank_kind: "decision-model",
      search_rerank_base_url: "https://rerank.example.com",
    });
    const res = await check();
    expect(res.stdout).toContain("ignored for this kind: search_rerank_base_url");
  });
});

describe("o2b decision-model report", () => {
  test("summarises records per use with shadow agreement", async () => {
    writeConfig({
      ...ENABLED,
      decision_model_base_url: server.url,
      search_rerank_enabled: "true",
      search_rerank_kind: "decision-model",
    });
    writeFileSync(join(vault, "a.md"), "# A\n\nfox fox fox\n");
    writeFileSync(join(vault, "b.md"), "# B\n\nfox and hound\n");
    server.setReply((req) => ({ json: answerAll(req, () => 0.5) }));
    const env = { [KEY_VAR]: FAKE_DECISION_KEY };
    await runCli(["search", "index", "--config", configPath], {
      env: { OPEN_SECOND_BRAIN_CONFIG: configPath, ...env },
    });
    await runCli(["search", "query", "fox", "--config", configPath], {
      env: { OPEN_SECOND_BRAIN_CONFIG: configPath, ...env },
    });
    const res = await runCli(["decision-model", "report", "--config", configPath, "--json"], {
      env: { OPEN_SECOND_BRAIN_CONFIG: configPath },
    });
    const report = JSON.parse(res.stdout) as {
      total: number;
      uses: Array<{ use: string; outcomes: Record<string, number>; agreement?: unknown }>;
    };
    expect(report.total).toBe(1);
    expect(report.uses[0]).toMatchObject({ use: "rerank", outcomes: { ok: 1 } });
    expect(report.uses[0]!.agreement).toBeDefined();
  });
});

describe("doctor readiness line", () => {
  test("off, no key, active and invalid", async () => {
    writeConfig({});
    expect((await probeDecisionModel({ vault, config: configPath, env: {} })).status).toBe(
      "skipped",
    );

    writeConfig({ ...ENABLED, decision_model_base_url: server.url });
    const noKey = await probeDecisionModel({ vault, config: configPath, env: {} });
    expect(noKey.status).toBe("skipped");
    expect(noKey.detail).toContain(KEY_VAR);

    const active = await probeDecisionModel({
      vault,
      config: configPath,
      env: { [KEY_VAR]: FAKE_DECISION_KEY },
    });
    expect(active.status).toBe("pass");
    expect(active.detail).not.toContain(FAKE_DECISION_KEY);

    writeConfig({ ...ENABLED, decision_model_base_url: "http://decisions.example.com" });
    const invalid = await probeDecisionModel({
      vault,
      config: configPath,
      env: { [KEY_VAR]: FAKE_DECISION_KEY },
    });
    expect(invalid.status).toBe("fail");
    expect(invalid.detail).toContain("decision_model_base_url");
  });
});

describe("o2b search rerank-eval", () => {
  test("reports hit@k and MRR for off and the local reranker", async () => {
    writeConfig({});
    writeFileSync(
      join(vault, "ml.md"),
      "# Machine learning\n\nGradient descent optimizes the loss.\n",
    );
    writeFileSync(join(vault, "garden.md"), "# Gardening\n\nTomatoes and compost in spring.\n");
    const dataset = join(tmp, "dataset.json");
    writeFileSync(
      dataset,
      JSON.stringify({ queries: [{ id: "q1", query: "gradient descent", expected: ["ml.md"] }] }),
    );
    const env = { OPEN_SECOND_BRAIN_CONFIG: configPath };
    await runCli(["search", "index", "--config", configPath], { env });
    const res = await runCli(
      ["search", "rerank-eval", "--config", configPath, "--dataset", dataset, "--json"],
      { env },
    );
    expect(res.returncode).toBe(0);
    const out = JSON.parse(res.stdout) as Record<string, Record<string, number>>;
    expect(out["baseline"]).toMatchObject({ hit_at_k: 1, mrr: 1, hit_at_1: 1, hit_at_5: 1 });
    expect(out["baseline"]!["hit_at_10"]).toBeUndefined();
    expect(out["local"]).toMatchObject({
      delta_hit_at_k: 0,
      versus_off: { wins: 0, losses: 0, ties: 1 },
    });
  });

  test("--kind decision-model reports the run's calls, so a failing arm shows as failing", async () => {
    writeConfig({ ...ENABLED, decision_model_base_url: server.url });
    writeFileSync(join(vault, "ml.md"), "# Machine learning\n\nGradient descent optimizes.\n");
    writeFileSync(join(vault, "gd.md"), "# Descent\n\nGradient descent notes.\n");
    const dataset = join(tmp, "dataset.json");
    writeFileSync(
      dataset,
      JSON.stringify({ queries: [{ id: "q1", query: "gradient descent", expected: ["ml.md"] }] }),
    );
    const env = { OPEN_SECOND_BRAIN_CONFIG: configPath, [KEY_VAR]: FAKE_DECISION_KEY };
    await runCli(["search", "index", "--config", configPath], { env });
    server.setReply(() => ({ status: 400 }));
    const args = ["search", "rerank-eval", "--config", configPath, "--dataset", dataset];
    const res = await runCli([...args, "--kind", "decision-model", "--json"], { env });
    expect(res.returncode).toBe(0);
    const out = JSON.parse(res.stdout) as Record<string, Record<string, unknown>>;
    expect(out["decision-model"]!["calls"]).toMatchObject({
      total: 1,
      outcomes: { http_400: 1 },
    });
    const text = await runCli([...args, "--kind", "decision-model"], { env });
    expect(text.stdout).toContain("decision calls: 1 (http_400 1)");
    expect(text.stdout).toMatch(/vs off \d+ win\(s\)/);
  });

  test("--kind decision-model without an active config is refused with a pointer", async () => {
    writeConfig({ ...ENABLED, decision_model_base_url: server.url });
    const dataset = join(tmp, "dataset.json");
    writeFileSync(
      dataset,
      JSON.stringify({ queries: [{ id: "q", query: "x", expected: ["x.md"] }] }),
    );
    const env = { OPEN_SECOND_BRAIN_CONFIG: configPath };
    await runCli(["search", "index", "--config", configPath], { env });
    const res = await runCli(
      [
        "search",
        "rerank-eval",
        "--config",
        configPath,
        "--dataset",
        dataset,
        "--kind",
        "decision-model",
      ],
      { env },
    );
    expect(res.returncode).toBe(2);
    expect(res.stderr).toContain("o2b decision-model check");
    expect(server.requests).toHaveLength(0);
  });
});