Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
5.0 KB
Raw Blame
/**
 * `o2b brain benchmark` + `o2b brain tune` CLI surfaces
 * (t_e2215d49 / t_ae973491): benchmark scores a dataset and records
 * the recall_benchmark metric; tune runs the grid, persists, reports
 * status, and resets; search --expand reaches the expansion path.
 */

import { afterEach, beforeEach, expect, setDefaultTimeout, test } from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { indexVault } from "../../src/core/search/indexer.ts";
import { listMetrics } from "../../src/core/brain/metrics.ts";
import { makeConfig } from "../helpers/search-fixtures.ts";
import { runCli } from "../helpers/run-cli.ts";
import { CLI_SPAWN_BUDGET_MS } from "../helpers/cli-timeout.ts";

// Tests here spawn processes; see tests/helpers/cli-timeout.ts.
setDefaultTimeout(CLI_SPAWN_BUDGET_MS);

let tmp: string;
let vault: string;
let datasetPath: string;

beforeEach(async () => {
  tmp = mkdtempSync(join(tmpdir(), "o2b-cli-bench-"));
  vault = join(tmp, "vault");
  mkdirSync(join(vault, "Brain"), { recursive: true });
  writeFileSync(
    join(vault, "canary.md"),
    "# Canary rollout\n\nShip one instance first, observe, expand gradually.\n",
  );
  // "the" is seeded across >= 2 of the other notes (majority of the
  // corpus) but NOT canary.md, so the DF-driven, language-agnostic
  // common-token filter drops it from the implicit-AND lex lane.
  writeFileSync(join(vault, "other.md"), "# Other\n\nNothing in the list is relevant here.\n");
  writeFileSync(join(vault, "notes.md"), "# Notes\n\nThe team reviewed the plan in the meeting.\n");
  datasetPath = join(tmp, "dataset.json");
  writeFileSync(
    datasetPath,
    JSON.stringify({
      queries: [{ id: "canary", query: "canary rollout", expected: ["canary.md"] }],
    }),
  );
  await indexVault(
    makeConfig({ vault, dbPath: join(vault, ".open-second-brain", "brain.sqlite") }),
  );
});

afterEach(() => {
  rmSync(tmp, { recursive: true, force: true });
});

test("benchmark run scores the dataset and records the metric", async () => {
  const r = await runCli([
    "brain",
    "benchmark",
    "run",
    "--dataset",
    datasetPath,
    "--vault",
    vault,
    "--json",
  ]);
  expect(r.returncode).toBe(0);
  const parsed = JSON.parse(r.stdout) as { total: number; hit_at_k: number; mrr: number };
  expect(parsed.total).toBe(1);
  expect(parsed.hit_at_k).toBe(1);
  expect(parsed.mrr).toBe(1);

  const metrics = listMetrics(vault, { surface: "recall_benchmark" });
  expect(metrics).toHaveLength(1);
  expect(metrics[0]!.payload).toMatchObject({ total: 1, hit_at_k: 1 });
});

test("benchmark usage and dataset validation exit 2", async () => {
  const noDataset = await runCli(["brain", "benchmark", "run", "--vault", vault]);
  expect(noDataset.returncode).toBe(2);

  const badPath = join(tmp, "bad.json");
  writeFileSync(badPath, JSON.stringify({ queries: [] }));
  const bad = await runCli(["brain", "benchmark", "run", "--dataset", badPath, "--vault", vault]);
  expect(bad.returncode).toBe(2);
});

test("tune run -> status -> reset lifecycle with the self_tuning metric", async () => {
  const run = await runCli([
    "brain",
    "tune",
    "run",
    "--dataset",
    datasetPath,
    "--vault",
    vault,
    "--json",
  ]);
  expect(run.returncode).toBe(0);
  const ran = JSON.parse(run.stdout) as {
    chosen: { poolMultiplier: number };
    evaluated: unknown[];
  };
  expect(ran.evaluated).toHaveLength(24);
  expect([3, 4, 5]).toContain(ran.chosen.poolMultiplier);
  expect(existsSync(join(vault, "Brain", "search", "tuning.json"))).toBe(true);
  expect(listMetrics(vault, { surface: "self_tuning" })).toHaveLength(1);

  const status = await runCli(["brain", "tune", "status", "--vault", vault, "--json"]);
  const state = JSON.parse(status.stdout) as { enabled: boolean; tuned: unknown };
  expect(state.enabled).toBe(false);
  expect(state.tuned).not.toBeNull();

  const reset = await runCli(["brain", "tune", "reset", "--vault", vault, "--json"]);
  expect(JSON.parse(reset.stdout)).toMatchObject({ removed: true });
  const statusAfter = await runCli(["brain", "tune", "status", "--vault", vault, "--json"]);
  expect((JSON.parse(statusAfter.stdout) as { tuned: unknown }).tuned).toBeNull();
});

test("tune run without a dataset exits 2", async () => {
  const r = await runCli(["brain", "tune", "run", "--vault", vault]);
  expect(r.returncode).toBe(2);
});

test("search --expand recovers the stopword-blocked hit", async () => {
  // "the" appears nowhere in canary.md, so the implicit-AND lane misses.
  const plain = await runCli(["search", "the canary rollout", "--vault", vault, "--json"]);
  expect(JSON.parse(plain.stdout).results).toHaveLength(0);
  const expanded = await runCli([
    "search",
    "the canary rollout",
    "--expand",
    "--vault",
    vault,
    "--json",
  ]);
  const results = JSON.parse(expanded.stdout).results as Array<{ path: string }>;
  expect(results.some((r) => r.path === "canary.md")).toBe(true);
});