Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* `o2b brain benchmark run --dataset <path>` (t_e2215d49): score the
* vault's live hybrid recall against a fixed query/expected-result
* dataset - hit@k and MRR per query and aggregate - and record one
* `recall_benchmark` metric so the dashboard can chart recall quality
* over time.
*
* Exit codes: 0 on success, 1 on an operational failure, 2 on usage
* errors.
*/
import { readFileSync } from "node:fs";
import { parseRecallBenchmarkDataset, runRecallBenchmark } from "../../../core/search/benchmark.ts";
import { appendMetric } from "../../../core/brain/metrics.ts";
import { isoSecond } from "../../../core/brain/time.ts";
import { resolveSearchConfig } from "../../../core/search/index.ts";
import { SearchError } from "../../../core/search/types.ts";
import { brainVerbContext, fail, ok, okJson, parse } from "../helpers.ts";
import { CLI_TRANSPORT_REACH } from "../../transport-reach.ts";
const USAGE =
"usage: o2b brain benchmark run --dataset <path> [--k N] [--expand] [--vault <path>] [--json]";
export async function cmdBrainBenchmark(argv: string[]): Promise<number> {
const { flags, positional } = parse(argv, {
vault: { type: "string" },
dataset: { type: "string" },
k: { type: "string" },
expand: { type: "boolean" },
json: { type: "boolean" },
});
const asJson = flags["json"] === true;
const datasetPath = flags["dataset"] as string | undefined;
if (positional[0] !== "run" || positional.length !== 1 || datasetPath === undefined) {
process.stderr.write(`${USAGE}\n`);
return 2;
}
const k = flags["k"] !== undefined ? Number(flags["k"]) : undefined;
if (k !== undefined && (!Number.isInteger(k) || k < 1)) {
process.stderr.write("brain benchmark run: --k must be a positive integer\n");
return 2;
}
const { config, vault } = brainVerbContext(flags);
try {
let raw: unknown;
try {
raw = JSON.parse(readFileSync(datasetPath, "utf8"));
} catch (exc) {
process.stderr.write(
`brain benchmark run: cannot read dataset ${datasetPath}: ${(exc as Error).message}\n`,
);
return 2;
}
const dataset = parseRecallBenchmarkDataset(raw);
const searchConfig = resolveSearchConfig({ vault, configPath: config ?? undefined });
const now = new Date();
const report = await runRecallBenchmark(searchConfig, dataset, {
...(k !== undefined ? { k } : {}),
expand: flags["expand"] === true,
transportReach: CLI_TRANSPORT_REACH,
});
try {
appendMetric(vault, {
surface: "recall_benchmark",
runAt: isoSecond(now),
payload: {
total: report.total,
k: report.k,
expand: report.expand,
hit_at_k: report.hitAtK,
mrr: report.mrr,
answer_queries: report.answerQueries,
answer_containment_at_k: report.answerContainmentAtK,
source_utilization_at_k: report.sourceUtilizationAtK,
citation_depth: report.citationDepth,
source_warnings: report.sourceWarnings,
misses: report.perQuery.filter((q) => !q.hit).map((q) => q.id),
},
});
} catch {
// Metrics are observability, not correctness.
}
if (asJson) {
okJson({
total: report.total,
k: report.k,
expand: report.expand,
hit_at_k: report.hitAtK,
mrr: report.mrr,
answer_queries: report.answerQueries,
answer_containment_at_k: report.answerContainmentAtK,
source_utilization_at_k: report.sourceUtilizationAtK,
citation_depth: report.citationDepth,
source_warnings: report.sourceWarnings,
per_query: report.perQuery,
});
} else {
ok(
`benchmark: ${report.total} queries, hit@${report.k} ${report.hitAtK.toFixed(3)}, ` +
`MRR ${report.mrr.toFixed(3)}, ` +
`answer@${report.k} ${report.answerContainmentAtK.toFixed(3)} (${report.answerQueries}), ` +
`src-util ${report.sourceUtilizationAtK.toFixed(3)}, warnings ${report.sourceWarnings}` +
`${report.expand ? " (expanded)" : ""}`,
);
for (const q of report.perQuery) {
ok(` ${q.hit ? "hit " : "MISS"} ${q.id} rank=${q.rank ?? "-"}`);
}
}
return 0;
} catch (exc) {
if (exc instanceof SearchError && exc.code === "INVALID_INPUT") {
process.stderr.write(`brain benchmark run: ${exc.message}\n`);
return 2;
}
const message = `benchmark failed: ${(exc as Error).message ?? exc}`;
if (asJson) {
okJson({ ok: false, message });
return 1;
}
return fail(message);
}
}