Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* `o2b brain bench memory` (Memory Observability Suite, t_882c396a):
* the CLI surface over the bench pipeline. The harness materializes a
* disposable vault inside the runs directory and never resolves the
* operator's configured vault.
*/
import { afterEach, beforeEach, describe, expect, setDefaultTimeout, test } from "bun:test";
import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { VERB_HELP } from "../../src/cli/brain/help-text.ts";
import {
benchSuite,
benchSuiteNames,
parseBenchPinnedKnobs,
parseRecallFloorFlag,
parseSampleCapFlag,
parseSampleSeedFlag,
registerBenchSuite,
} from "../../src/cli/brain/verbs/bench.ts";
import { parseBenchFixture } from "../../src/core/bench/fixture.ts";
import type { BenchSuiteLoadOptions } from "../../src/core/bench/types.ts";
import { runCli } from "../helpers/run-cli.ts";
import { CLI_SPAWN_BUDGET_MS } from "../helpers/cli-timeout.ts";
// Tests here spawn processes; see tests/helpers/cli-timeout.ts.
setDefaultTimeout(CLI_SPAWN_BUDGET_MS);
let tmp: string;
let runsDir: string;
beforeEach(() => {
tmp = mkdtempSync(join(tmpdir(), "o2b-bench-cli-"));
runsDir = join(tmp, "bench-runs");
});
afterEach(() => {
rmSync(tmp, { recursive: true, force: true });
});
describe("o2b brain bench memory", () => {
test("runs the repo core-recall fixture green and reports separate metric families", async () => {
const out = await runCli(
["brain", "bench", "memory", "--fixture", "core-recall", "--runs-dir", runsDir, "--json"],
{},
);
expect(out.returncode).toBe(0);
const report = JSON.parse(out.stdout) as {
schema: string;
run_id: string;
quality: { passed: number; total: number };
latency_ms: { avg: number };
context_cost: { avg_chars: number };
judge: { status: string };
};
expect(report.schema).toBe("o2b.bench.v2");
expect(report.quality.total).toBe(6);
expect(report.quality.passed).toBe(6);
expect(report.judge.status).toBe("skipped");
// The disposable vault lives inside the run directory.
expect(existsSync(join(runsDir, report.run_id, "vault", "Brain"))).toBe(true);
expect(existsSync(join(runsDir, report.run_id, "report.json"))).toBe(true);
}, 60_000);
test("resume reuses the checkpoint instead of creating a second run", async () => {
const first = await runCli(
["brain", "bench", "memory", "--fixture", "stale-recall", "--runs-dir", runsDir, "--json"],
{},
);
expect(first.returncode).toBe(0);
const report = JSON.parse(first.stdout) as { run_id: string };
const resumed = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"stale-recall",
"--runs-dir",
runsDir,
"--resume",
report.run_id,
"--json",
],
{},
);
expect(resumed.returncode).toBe(0);
expect((JSON.parse(resumed.stdout) as { run_id: string }).run_id).toBe(report.run_id);
expect(readdirSync(runsDir)).toHaveLength(1);
}, 60_000);
test("an unknown fixture fails with a clear error", async () => {
const out = await runCli(
["brain", "bench", "memory", "--fixture", "no-such-fixture", "--runs-dir", runsDir],
{},
);
expect(out.returncode).not.toBe(0);
expect(out.stderr).toContain("fixture not found");
});
});
/**
* The suite registry (recall-quality wave, t_607d429d): a --suite name
* maps to `{ loader, capabilities }`, the registry is the appendable
* registration point for suite modules, and the no-flag default path is
* byte-identical to the pre-registry dispatch.
*/
describe("bench suite registry", () => {
const STUB_SUITE = "registry-stub";
const STUB_FIXTURE = parseBenchFixture({
name: "registry-stub",
notes: [{ path: "Brain/notes/a.md", body: "alpha note" }],
questions: [
{ id: "q1", category: "single_hop", query: "alpha", expected_paths: ["Brain/notes/a.md"] },
],
});
const stubLoaderCalls: string[] = [];
registerBenchSuite(STUB_SUITE, {
loader: (fixturePath: string) => {
stubLoaderCalls.push(fixturePath);
return { fixture: STUB_FIXTURE };
},
capabilities: { embeddings: false },
});
beforeEach(() => {
stubLoaderCalls.length = 0;
});
test("the registry maps suite names to loader and capabilities", () => {
const locomo = benchSuite("locomo");
expect(locomo).toBeDefined();
expect(locomo!.capabilities).toEqual({ embeddings: false });
expect(typeof locomo!.loader).toBe("function");
expect(benchSuiteNames()).toContain("locomo");
// Sorted for stable listings (order-dependent assertions use sorted listings).
expect(benchSuiteNames()).toEqual(benchSuiteNames().toSorted());
});
test("a registered suite reaches its loader through --suite", async () => {
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"core-recall",
"--suite",
STUB_SUITE,
"--runs-dir",
runsDir,
"--json",
],
{},
);
expect(out.returncode).toBe(0);
// The stub loader received the RESOLVED fixture path.
expect(stubLoaderCalls).toEqual([join("tests", "fixtures", "bench", "core-recall.json")]);
const report = JSON.parse(out.stdout) as { fixture: string };
expect(report.fixture).toBe("registry-stub");
}, 60_000);
test("an unknown suite fails with the current message shape", async () => {
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"core-recall",
"--suite",
"bogus",
"--runs-dir",
runsDir,
],
{},
);
expect(out.returncode).not.toBe(0);
expect(out.stderr).toContain("unknown --suite 'bogus' (supported:");
});
test("omitting --suite keeps the native loader and the report shape unchanged", async () => {
const out = await runCli(
["brain", "bench", "memory", "--fixture", "core-recall", "--runs-dir", runsDir, "--json"],
{},
);
expect(out.returncode).toBe(0);
const report = JSON.parse(out.stdout) as { fixture: string };
expect(report.fixture).toBe("core-recall");
// The native default path is not a named suite and adds no suite fields.
expect("suite" in report).toBe(false);
expect(stubLoaderCalls).toEqual([]);
}, 60_000);
test("the longmemeval suite prints its strict recall_all@5 family", async () => {
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"longmemeval-sample",
"--suite",
"longmemeval",
"--runs-dir",
runsDir,
],
{},
);
expect(out.returncode).toBe(0);
// The named metric family rides the CLI output beside the existing
// families - never folded into the pass rate.
expect(out.stdout).toContain("longmemeval: recall_all@5 1 abstention_excluded 1");
}, 60_000);
test("the longmemeval report family rides the --json report", async () => {
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"longmemeval-sample",
"--suite",
"longmemeval",
"--runs-dir",
runsDir,
"--json",
],
{},
);
expect(out.returncode).toBe(0);
const report = JSON.parse(out.stdout) as {
longmemeval?: { recall_all_at_5: number; abstention_excluded: number };
};
expect(report.longmemeval).toEqual({ recall_all_at_5: 1, abstention_excluded: 1 });
}, 60_000);
test("a duplicate suite registration is refused by name", () => {
expect(() =>
registerBenchSuite("locomo", {
loader: () => ({ fixture: STUB_FIXTURE }),
capabilities: { embeddings: false },
}),
).toThrow(/already registered/);
});
});
/**
* The knob pin and the recall floor reach `runMemoryBench` from the CLI
* (pr-250 review, M7): the pin is the run's search configuration and the
* floor is the admission bound the surfacing-replay suite sweeps, and
* both are part of the run identity - asserted here through the
* checkpoint each run persists.
*/
describe("bench knob pin and recall floor", () => {
/** A one-question fixture, so each e2e run stays cheap. */
function writePinFixture(): string {
const path = join(tmp, "pin-fixture.json");
writeFileSync(
path,
JSON.stringify({
name: "pin-check",
notes: [{ path: "Brain/notes/a.md", body: "alpha note" }],
questions: [
{
id: "q1",
category: "single_hop",
query: "alpha",
expected_paths: ["Brain/notes/a.md"],
},
],
}),
);
return path;
}
function checkpointOf(
runsDir: string,
runId: string,
): {
pinned_knobs?: unknown;
knobs_hash?: string;
} {
return JSON.parse(readFileSync(join(runsDir, runId, "checkpoint.json"), "utf8")) as {
pinned_knobs?: unknown;
knobs_hash?: string;
};
}
test("--pin reaches the run identity verbatim, replacing the default pin", async () => {
const fixturePath = writePinFixture();
// A pin the resolver accepts: the semantic lane stays off and one
// chunking knob moves. Value semantics are the resolver's, not this
// flag's - a pin like `keywordWeight: 0.7` alone dies there with its
// named sum-to-one error, which is the loud path by design.
const pin = { semantic: { enabled: false }, chunkSize: 512 };
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
fixturePath,
"--runs-dir",
runsDir,
"--pin",
JSON.stringify(pin),
"--json",
],
{},
);
expect(out.returncode).toBe(0);
const report = JSON.parse(out.stdout) as { run_id: string };
const checkpoint = checkpointOf(runsDir, report.run_id);
// The pin rides the checkpoint byte for byte - nothing merged in, so
// the run the report describes is exactly the configuration asked for.
expect(checkpoint.pinned_knobs).toEqual(pin);
}, 60_000);
test("--recall-floor reaches the run identity: a floor change is a different run", async () => {
const fixturePath = writePinFixture();
const plain = await runCli(
["brain", "bench", "memory", "--fixture", fixturePath, "--runs-dir", runsDir, "--json"],
{},
);
expect(plain.returncode).toBe(0);
const plainId = (JSON.parse(plain.stdout) as { run_id: string }).run_id;
const floored = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
fixturePath,
"--runs-dir",
runsDir,
"--recall-floor",
"0.5",
"--json",
],
{},
);
expect(floored.returncode).toBe(0);
const flooredId = (JSON.parse(floored.stdout) as { run_id: string }).run_id;
expect(flooredId).not.toBe(plainId);
// And resuming with the SAME floor reuses the run; the identity
// guard, not the CLI, owns what a floor change on resume means.
const resumed = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
fixturePath,
"--runs-dir",
runsDir,
"--recall-floor",
"0.5",
"--resume",
flooredId,
"--json",
],
{},
);
expect(resumed.returncode).toBe(0);
expect((JSON.parse(resumed.stdout) as { run_id: string }).run_id).toBe(flooredId);
expect(readdirSync(runsDir)).toHaveLength(2);
}, 60_000);
test("garbage is refused by name", async () => {
const fixturePath = writePinFixture();
const badJson = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
fixturePath,
"--runs-dir",
runsDir,
"--pin",
"{not json",
],
{},
);
expect(badJson.returncode).not.toBe(0);
expect(badJson.stderr).toContain("--pin is not valid JSON");
const notAnObject = await runCli(
["brain", "bench", "memory", "--fixture", fixturePath, "--pin", "[1,2]"],
{},
);
expect(notAnObject.stderr).toContain("--pin must be a JSON object");
const unknownKnob = await runCli(
["brain", "bench", "memory", "--fixture", fixturePath, "--pin", '{"frobNobs":1}'],
{},
);
expect(unknownKnob.stderr).toContain("unknown knob 'frobNobs'");
const unknownMember = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
fixturePath,
"--pin",
'{"semantic":{"enabeled":true}}',
],
{},
);
expect(unknownMember.stderr).toContain("--pin 'semantic.enabeled' is not a known knob");
const harnessOwned = await runCli(
["brain", "bench", "memory", "--fixture", fixturePath, "--pin", '{"vault":"/elsewhere"}'],
{},
);
expect(harnessOwned.stderr).toContain("--pin may not set 'vault'");
const badFloor = await runCli(
["brain", "bench", "memory", "--fixture", fixturePath, "--recall-floor", "abc"],
{},
);
expect(badFloor.returncode).not.toBe(0);
expect(badFloor.stderr).toContain("--recall-floor must be a positive number, got 'abc'");
const zeroFloor = await runCli(
["brain", "bench", "memory", "--fixture", fixturePath, "--recall-floor", "0"],
{},
);
expect(zeroFloor.stderr).toContain("--recall-floor must be a positive number, got '0'");
}, 60_000);
test("the pin parser refuses shape garbage and leaves value semantics to the resolver", () => {
expect(parseBenchPinnedKnobs('{"semantic":{"enabled":false}}')).toEqual({
semantic: { enabled: false },
});
// A scalar knob's VALUE is not shape-checked here: an out-of-range
// number is resolveSearchConfig's named error, not this parser's.
expect(parseBenchPinnedKnobs('{"keywordWeight":99}')).toEqual({ keywordWeight: 99 });
// The standing-instruction facet lane's arm is pinnable, so the
// lane-off arm of the category is reachable from the CLI too
// (recall-lane-latency).
expect(parseBenchPinnedKnobs('{"standingInstructionFacets":false}')).toEqual({
standingInstructionFacets: false,
});
expect(() => parseBenchPinnedKnobs('{"semantic":false}')).toThrow(
/'semantic' must be a JSON object/,
);
expect(() => parseRecallFloorFlag(undefined)).not.toThrow();
expect(parseRecallFloorFlag("0.35")).toBe(0.35);
expect(() => parseRecallFloorFlag("")).toThrow(/got ''/);
expect(() => parseRecallFloorFlag("-1")).toThrow(/got '-1'/);
});
});
/**
* The seeded sampling flags (wiring-shipped-seams wave, t_4501c662):
* `--sample-seed` and `--sample-cap` are a PAIR - refused singly and on
* any suite that does not declare the sampling capability - and
* together they reach the suite loader as
* `BenchSuiteLoadOptions { sample }`. Omitting both leaves today's CLI
* behavior byte-identical.
*/
describe("bench sampling flags", () => {
const SAMPLING_SUITE = "sampling-stub";
const SAMPLING_FIXTURE = parseBenchFixture({
name: "sampling-stub",
notes: [{ path: "Brain/notes/a.md", body: "alpha note" }],
questions: [
{ id: "q1", category: "single_hop", query: "alpha", expected_paths: ["Brain/notes/a.md"] },
],
});
/** Every loader invocation with the options object it received. */
const samplingLoaderCalls: Array<{ path: string; options: BenchSuiteLoadOptions }> = [];
registerBenchSuite(SAMPLING_SUITE, {
// The second parameter is OPTIONAL by contract: a suite registered
// before an option existed keeps its one-parameter loader and never
// sees the request. This stub declares it to record what arrives.
loader: (fixturePath: string, options?: BenchSuiteLoadOptions) => {
samplingLoaderCalls.push({ path: fixturePath, options: options ?? {} });
return { fixture: SAMPLING_FIXTURE };
},
capabilities: { embeddings: false, sampling: true },
});
beforeEach(() => {
samplingLoaderCalls.length = 0;
});
test("the sampling flag validators refuse garbage by name", () => {
expect(parseSampleSeedFlag(undefined)).toBeUndefined();
expect(parseSampleSeedFlag("run-42")).toBe("run-42");
// pr-251 L2: the parser validates but does not transform - the
// sampler trims the seed in one place before hashing, so the CLI and
// the API path hash the same value for the same input, and a padded
// seed reaches the loader verbatim.
expect(parseSampleSeedFlag(" seed-1 ")).toBe(" seed-1 ");
expect(() => parseSampleSeedFlag(" ")).toThrow(/--sample-seed/);
expect(parseSampleCapFlag(undefined)).toBeUndefined();
expect(parseSampleCapFlag("12")).toBe(12);
expect(() => parseSampleCapFlag("0")).toThrow(/got '0'/);
expect(() => parseSampleCapFlag("-2")).toThrow(/got '-2'/);
expect(() => parseSampleCapFlag("1.5")).toThrow(/got '1.5'/);
expect(() => parseSampleCapFlag("abc")).toThrow(/got 'abc'/);
});
test("--sample-seed alone is refused by name", async () => {
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"core-recall",
"--runs-dir",
runsDir,
"--sample-seed",
"seed-1",
],
{},
);
expect(out.returncode).not.toBe(0);
expect(out.stderr).toContain("--sample-seed and --sample-cap must be given together");
});
test("--sample-cap alone is refused by name", async () => {
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"core-recall",
"--runs-dir",
runsDir,
"--sample-cap",
"4",
],
{},
);
expect(out.returncode).not.toBe(0);
expect(out.stderr).toContain("--sample-seed and --sample-cap must be given together");
});
test("a non-positive or non-integer cap is refused naming the value", async () => {
for (const cap of ["0", "-2", "1.5", "abc"]) {
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"core-recall",
"--sample-seed",
"seed-1",
"--sample-cap",
cap,
],
{},
);
expect(out.returncode).not.toBe(0);
expect(out.stderr).toContain(`--sample-cap must be a positive integer, got '${cap}'`);
}
});
test("an empty seed is refused naming the flag", async () => {
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"core-recall",
"--sample-seed",
" ",
"--sample-cap",
"4",
],
{},
);
expect(out.returncode).not.toBe(0);
expect(out.stderr).toContain("--sample-seed must be a non-empty string");
});
test("both flags without a sampling-capable suite are refused naming the suite", async () => {
for (const suite of ["locomo", "surfacing-replay"]) {
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"core-recall",
"--suite",
suite,
"--sample-seed",
"seed-1",
"--sample-cap",
"4",
],
{},
);
expect(out.returncode).not.toBe(0);
expect(out.stderr).toContain(`suite '${suite}' does not declare it`);
}
// The no-suite default path has no sampling capability either.
const noSuite = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"core-recall",
"--sample-seed",
"seed-1",
"--sample-cap",
"4",
],
{},
);
expect(noSuite.returncode).not.toBe(0);
expect(noSuite.stderr).toContain("no --suite was given");
});
test("both flags with a sampling-capable suite reach the loader as a sample request", async () => {
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"core-recall",
"--suite",
SAMPLING_SUITE,
"--sample-seed",
"seed-1",
"--sample-cap",
"3",
"--runs-dir",
runsDir,
"--json",
],
{},
);
expect(out.returncode).toBe(0);
expect(samplingLoaderCalls).toEqual([
{
path: join("tests", "fixtures", "bench", "core-recall.json"),
options: { sample: { seed: "seed-1", cap: 3 } },
},
]);
const report = JSON.parse(out.stdout) as { fixture: string };
expect(report.fixture).toBe("sampling-stub");
}, 60_000);
test("omitting the flags passes empty load options - the off path is unchanged", async () => {
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"core-recall",
"--suite",
SAMPLING_SUITE,
"--runs-dir",
runsDir,
"--json",
],
{},
);
expect(out.returncode).toBe(0);
expect(samplingLoaderCalls).toEqual([
{ path: join("tests", "fixtures", "bench", "core-recall.json"), options: {} },
]);
}, 60_000);
test("the longmemeval suite samples through the flags and pins the run identity", async () => {
// The committed sample carries 3 eligible questions across 3 types;
// a cap of 2 selects 2, the CLI line reports the sampled count, and
// the run checkpoint carries the sampling request - so a resumed run
// under different sampling options fails the identity guard.
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"longmemeval-sample",
"--suite",
"longmemeval",
"--sample-seed",
"seed-1",
"--sample-cap",
"2",
"--runs-dir",
runsDir,
],
{},
);
expect(out.returncode).toBe(0);
expect(out.stdout).toContain("longmemeval: recall_all@5");
// The sampled text line shows the covered share: the population the
// sampler chose from beside the selected count (pr-251 L3). The
// whole-dataset line carries neither field.
expect(out.stdout).toContain("sample_population 3 sample_selected 2");
const runId = /^bench \S+ run (\S+)$/m.exec(out.stdout)?.[1];
expect(runId).toBeDefined();
const checkpoint = JSON.parse(
readFileSync(join(runsDir, runId!, "checkpoint.json"), "utf8"),
) as { sample?: unknown };
expect(checkpoint.sample).toEqual({ seed: "seed-1", cap: 2 });
}, 60_000);
});
/**
* The help prose's usage block (CodeRabbit, pr-251): it mirrors the
* verb's USAGE constant - the usage line plus its 7-space continuation
* lines - so a flag the verb accepts is listed there too, and the line
* cannot drift back to a subset of the accepted flags.
*/
describe("bench help text", () => {
test("the usage block lists every flag the verb accepts", () => {
const usageBlock = (VERB_HELP["bench"] ?? "")
.split("\n")
.filter((line) => line.startsWith("usage:") || line.startsWith(" "))
.join("\n");
for (const flag of [
"--fixture",
"--suite",
"--pin",
"--recall-floor",
"--sample-seed",
"--sample-cap",
"--resume",
"--runs-dir",
"--json",
]) {
expect(usageBlock).toContain(flag);
}
// The sample flags sit in the usage block itself, not only in the
// prose below it; the block wraps, so adjacency is checked over the
// continuation lines flattened back into one line.
const flattened = usageBlock.replaceAll("\n ", " ");
expect(flattened).toContain("[--sample-seed <value>] [--sample-cap <n>]");
});
});
/**
* The standing_instruction human line (recall-lane-latency): printed only
* when the report's quality buckets carry the category - a run without
* standing_instruction questions keeps its stdout byte-identical, a
* lane-on run prints the category beside its rates, and the exit code
* stays all-pass-or-nonzero with the category in play.
*/
describe("bench standing_instruction human line", () => {
/** The surfacing lane's fixture vault files, in the live writers' shapes. */
const STANDING_FLOW_MD = [
"---",
"kind: brain-attention-flow",
"id: open-loops",
"title: Open loops",
"actions:",
" - standing_query",
"standing_queries:",
" - testing",
"status: active",
"---",
"",
"# Open loops",
"",
].join("\n");
const STANDING_PREF_MD = [
"---",
"kind: brain-preference",
"id: pref-tabs",
"created_at: 2026-06-01T00:00:00Z",
"_confirmed_at: 2026-06-02T00:00:00Z",
"unconfirmed_until: 2026-07-01T00:00:00Z",
"tags:",
" - bench",
"topic: tabs",
"_status: confirmed",
"principle: Use tabs for indentation",
"_evidenced_by:",
' - "[[sig-bench-turn]]"',
"_applied_count: 0",
"_violated_count: 0",
"_last_evidence_at: null",
"_confidence: low",
"_confidence_value: null",
"pinned: false",
"scope: testing",
"---",
"",
"Use tabs for indentation",
"",
].join("\n");
const STANDING_SIGNAL_MD = [
"---",
"kind: brain-signal",
"id: sig-bench-turn",
"created_at: 2026-06-01T00:00:00Z",
"tags:",
" - bench",
"topic: tabs",
"signal: positive",
"agent: bench",
"principle: use tabs for indentation",
"scope: testing",
"session_ref: bench-session.json#turn-2",
"---",
"",
"recorded preference signal",
"",
].join("\n");
const STANDING_SUITE = "standing-stub";
registerBenchSuite(STANDING_SUITE, {
loader: () => ({
fixture: parseBenchFixture({
name: "standing-cli",
notes: [
{ path: "Brain/attention/flows/open-loops.md", body: STANDING_FLOW_MD },
{ path: "Brain/preferences/pref-tabs.md", body: STANDING_PREF_MD },
{ path: "Brain/inbox/sig-bench-turn.md", body: STANDING_SIGNAL_MD },
],
continuity: [],
questions: [
{
id: "q-standing",
category: "standing_instruction",
expected_ids: ["pref:pref-tabs"],
expected_text: "earliest cited turn bench-session.json#turn-2]",
},
],
}),
}),
capabilities: { embeddings: false },
});
/** The same fixture with the expectation unmet: the question fails. */
const FAILING_SUITE = "standing-failing-stub";
registerBenchSuite(FAILING_SUITE, {
loader: () => ({
fixture: parseBenchFixture({
name: "standing-cli-failing",
notes: [
{ path: "Brain/attention/flows/open-loops.md", body: STANDING_FLOW_MD },
{ path: "Brain/preferences/pref-tabs.md", body: STANDING_PREF_MD },
{ path: "Brain/inbox/sig-bench-turn.md", body: STANDING_SIGNAL_MD },
],
continuity: [],
questions: [
{
id: "q-standing",
category: "standing_instruction",
expected_ids: ["pref:pref-never-resolves"],
},
],
}),
}),
capabilities: { embeddings: false },
});
test("a run without the category keeps stdout byte-identical", async () => {
const out = await runCli(
["brain", "bench", "memory", "--fixture", "core-recall", "--runs-dir", runsDir],
{},
);
expect(out.returncode).toBe(0);
expect(out.stdout).not.toContain("standing_instruction");
}, 60_000);
test("a lane-on run prints the new category and its rates", async () => {
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"core-recall",
"--suite",
STANDING_SUITE,
"--runs-dir",
join(runsDir, "lane-on"),
],
{},
);
expect(out.returncode).toBe(0);
expect(out.stdout).toContain("standing_instruction: 1/1 (pass rate 1)");
}, 60_000);
test("the exit code stays all-pass-or-nonzero with the new category in play", async () => {
const out = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"core-recall",
"--suite",
FAILING_SUITE,
"--runs-dir",
join(runsDir, "failing"),
],
{},
);
expect(out.returncode).not.toBe(0);
// The failing question names the category and the missing facet.
expect(out.stdout).toContain("FAIL q-standing (standing_instruction)");
expect(out.stdout).toContain("pref:pref-never-resolves");
}, 60_000);
});
/**
* The overlapped_ms human line (recall-lane-latency): printed only when
* the aggregate latency family carries the figure - a keyword-only run
* (the default pin) keeps its stdout byte-identical, and a lane-on run
* through an embeddings-capable suite prints it.
*/
describe("bench overlapped_ms human line", () => {
const OVERLAP_SUITE = "overlap-stub";
const OVERLAP_FIXTURE = parseBenchFixture({
name: "overlap-stub",
notes: [
{
path: "Brain/notes/rust.md",
body: "# Rust\n\nThe systems team standardized on Rust for the new ingest service.\n",
},
],
questions: [
{
id: "q-rust",
category: "single_hop",
query: "systems language for the ingest service",
expected_paths: ["Brain/notes/rust.md"],
},
],
});
registerBenchSuite(OVERLAP_SUITE, {
loader: () => ({ fixture: OVERLAP_FIXTURE }),
capabilities: { embeddings: true },
});
test("the overlapped_ms line prints only when the aggregate carries it", async () => {
// Keyword-only default run: no line, byte-identical stdout.
const plain = await runCli(
["brain", "bench", "memory", "--fixture", "core-recall", "--runs-dir", runsDir],
{},
);
expect(plain.returncode).toBe(0);
expect(plain.stdout).not.toContain("overlapped_ms");
// Lane-on run through the embeddings-capable suite: the line prints.
const laneOn = await runCli(
[
"brain",
"bench",
"memory",
"--fixture",
"core-recall",
"--suite",
OVERLAP_SUITE,
"--runs-dir",
join(runsDir, "lane-on"),
"--pin",
JSON.stringify({ semantic: { enabled: true, provider: "local" } }),
],
{},
);
expect(laneOn.returncode).toBe(0);
expect(laneOn.stdout).toMatch(/overlapped_ms: avg \d+/);
}, 60_000);
});