Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* The four-failure-mode conformance suite (t_72d6eb23).
*
* Each test drives shipped code over a controlled vault. What ships by
* DEFAULT and what is measured here are not the same thing, and the
* difference is asserted rather than assumed: `recall_inject_enabled` is
* off by default (the bench calls the decision core directly), and
* `integrity.owner_scope_delivery` is off by default (the fixture turns it
* to `fail`, and a negative control below proves the gate is what does the
* isolating).
*/
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
loadBenchFixture,
materializeBenchVault,
parseBenchFixture,
} from "../../../src/core/bench/fixture.ts";
import {
benchRecallRetriever,
classifyRecallDecision,
isolationViolations,
RECALL_FAILURE,
scoreProactiveRecall,
type RecallFailure,
} from "../../../src/core/bench/failure-modes.ts";
import { benchSearchConfig, runMemoryBench } from "../../../src/core/bench/phases.ts";
import type { BenchFixture } from "../../../src/core/bench/types.ts";
import {
decideRecallInject,
RECALL_INJECT_CONFIDENCE_FLOOR,
type RecallResultSet,
} from "../../../src/core/brain/recall-inject.ts";
import { resolveSearchConfig, search } from "../../../src/core/search/index.ts";
import { lexSource } from "../../helpers/source-lexer.ts";
const FIXTURE_PATH = join("tests", "fixtures", "bench", "failure-modes.json");
let runsDir: string;
beforeEach(() => {
runsDir = mkdtempSync(join(tmpdir(), "o2b-bench-failure-modes-"));
});
afterEach(() => {
rmSync(runsDir, { recursive: true, force: true });
});
function fixture(): BenchFixture {
return loadBenchFixture(FIXTURE_PATH);
}
/**
* The degenerate retriever a gaming implementation ships: hand back your
* best note whatever was asked. Paired with a floor of 0 it is the
* "always inject" arm - not a stub, but the real shape of the strategy
* the scorer has to be able to rank below a selective one.
*/
async function alwaysConfident(): Promise<RecallResultSet> {
return Object.freeze({
candidates: Object.freeze([
Object.freeze({
path: "Brain/notes/pgbouncer.md",
title: "Connection pooling",
score: 1,
searchType: "keyword",
startLine: 1,
endLine: 3,
}),
]),
total: 1,
// The gaming strategy claims a perfect match for every prompt - that
// unconditional confidence is exactly what the floor has to see through.
idfWeightedCoverage: 1,
});
}
/** Both rates share one denominator, so their sum is what ranks strategies. */
function totalFailureRate(score: {
know_to_ask_failure_rate: number;
false_fire_rate: number;
}): number {
return score.know_to_ask_failure_rate + score.false_fire_rate;
}
describe("the committed failure-mode fixture", () => {
test("runs green and reports all four metrics", async () => {
const report = await runMemoryBench({ fixture: fixture(), runsDir });
expect(report.quality.passed).toBe(report.quality.total);
expect(report.quality.by_category["proactive_recall"]).toEqual({ passed: 4, total: 4 });
expect(report.quality.by_category["write_fidelity"]).toEqual({ passed: 2, total: 2 });
expect(report.quality.by_category["source_isolation"]).toEqual({ passed: 1, total: 1 });
expect(report.failure_modes.know_to_ask_failure_rate).toBe(0);
expect(report.failure_modes.false_fire_rate).toBe(0);
expect(report.failure_modes.source_isolation_violations).toBe(0);
// The fourth metric is a COST, so it lives in the cost family.
expect(report.context_cost.avg_injected_tokens).toBeGreaterThan(0);
}, 60_000);
test("is deterministic: two independent runs agree on every metric", async () => {
// Separate run directories, so the second run genuinely redoes the
// work instead of resuming the first one's checkpoints.
const otherRunsDir = mkdtempSync(join(tmpdir(), "o2b-bench-failure-modes-b-"));
const first = await runMemoryBench({ fixture: fixture(), runsDir });
const second = await runMemoryBench({ fixture: fixture(), runsDir: otherRunsDir });
rmSync(otherRunsDir, { recursive: true, force: true });
expect(second.fixture_hash).toBe(first.fixture_hash);
expect(second.quality).toEqual(first.quality);
expect(second.failure_modes).toEqual(first.failure_modes);
expect(second.context_cost).toEqual(first.context_cost);
}, 60_000);
});
describe("the harness holds its own determinism", () => {
test("a shell configured for semantic search still resolves the CI lane", () => {
// `resolveSearchConfig` reads `process.env`, so before the override
// the harness's "deterministic and network-free" claim was a property
// of an unset developer shell: a configured semantic lane is a second
// ranking signal, and on a remote provider a network call.
const previous = {
semantic: process.env["OPEN_SECOND_BRAIN_SEARCH_SEMANTIC"],
provider: process.env["OPEN_SECOND_BRAIN_EMBEDDING_PROVIDER"],
};
try {
process.env["OPEN_SECOND_BRAIN_SEARCH_SEMANTIC"] = "1";
process.env["OPEN_SECOND_BRAIN_EMBEDDING_PROVIDER"] = "openai-compat";
expect(resolveSearchConfig({ vault: runsDir }).semantic.enabled).toBe(true);
// Same shell, same vault - the harness refuses the lane anyway.
expect(benchSearchConfig(runsDir).semantic.enabled).toBe(false);
} finally {
for (const [key, value] of [
["OPEN_SECOND_BRAIN_SEARCH_SEMANTIC", previous.semantic],
["OPEN_SECOND_BRAIN_EMBEDDING_PROVIDER", previous.provider],
] as const) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
}
});
});
describe("metric 1: proactive know-to-ask, with the anti-gaming term", () => {
test("an always-inject strategy scores worse than one that abstains correctly", async () => {
// Both arms run the SAME shipped decision core over the SAME prompts
// and are scored by the SAME scorer. Only the retriever and the
// confidence floor differ, which is exactly what separates a memory
// that knows when to speak from one that always speaks.
const vault = join(runsDir, "vault");
const loaded = fixture();
materializeBenchVault(loaded, vault);
const config = resolveSearchConfig({
vault,
overrides: { semantic: { enabled: false } },
});
await search(config, { query: "warmup", limit: 1 });
const prompts = loaded.questions.filter((q) => q.category === "proactive_recall");
expect(prompts.length).toBeGreaterThan(0);
const shipped: Array<RecallFailure | null> = [];
const always: Array<RecallFailure | null> = [];
for (const question of prompts) {
const expectInject = question.expect_inject === true;
// oxlint-disable-next-line no-await-in-loop
shipped.push(
classifyRecallDecision(
expectInject,
// oxlint-disable-next-line no-await-in-loop
await decideRecallInject(question.prompt ?? "", benchRecallRetriever(config, 4)),
),
);
always.push(
classifyRecallDecision(
expectInject,
// oxlint-disable-next-line no-await-in-loop
await decideRecallInject(question.prompt ?? "", alwaysConfident, {
confidenceFloor: 0,
}),
),
);
}
const shippedScore = scoreProactiveRecall(shipped);
const alwaysScore = scoreProactiveRecall(always);
// The shipped strategy is right on both halves of the fixture.
expect(shippedScore.know_to_ask_failure_rate).toBe(0);
expect(shippedScore.false_fire_rate).toBe(0);
// The gaming strategy buys a perfect know-to-ask rate and pays for it
// in false fires - which is the whole point of carrying that term.
expect(alwaysScore.know_to_ask_failure_rate).toBe(0);
expect(alwaysScore.false_fire_rate).toBeGreaterThan(0);
// Shared denominator, so the total failure rate is what ranks them.
expect(totalFailureRate(alwaysScore)).toBeGreaterThan(totalFailureRate(shippedScore));
}, 60_000);
test("the anti-gaming pressure is on the confidence floor, not on the gate", () => {
// `evaluateSurfacingGate` fails open by design, so it can carry no
// pressure; the floor is the only discriminating bound the decision
// core exposes, and the scorer is denominated so that using it wrong
// costs something.
expect(RECALL_INJECT_CONFIDENCE_FLOOR).toBeGreaterThan(0);
const allWrong = scoreProactiveRecall([RECALL_FAILURE.falseFire, RECALL_FAILURE.falseFire]);
expect(allWrong.false_fire_rate).toBe(1);
// A fault is charged to neither rate: a broken retriever is not a
// cautious one, and folding it in would report a number about the
// harness under a name that promises a number about memory.
const faulted = scoreProactiveRecall([RECALL_FAILURE.faulted, null]);
expect(faulted.know_to_ask_failure_rate).toBe(0);
expect(faulted.false_fire_rate).toBe(0);
expect(faulted.faults).toBe(1);
});
});
describe("metric 2: write-back fidelity on the shipped path", () => {
test("grades provenance with no model call, and a lost field fails", async () => {
const loaded = fixture();
const broken = parseBenchFixture({
...(JSON.parse(JSON.stringify(loaded)) as Record<string, unknown>),
name: "write-fidelity-missing-provenance",
questions: [
{
id: "q-fidelity-demands-a-field-the-path-does-not-write",
category: "write_fidelity",
intake_text: "Decision: Cap the pool at ninety connections.",
expected_labels: ["decision"],
expected_provenance: ["dedupe_key", "confidence_score"],
},
],
});
const report = await runMemoryBench({ fixture: broken, runsDir });
expect(report.quality.passed).toBe(0);
expect(report.questions[0]!.failure).toContain("confidence_score");
}, 60_000);
test("a non-Latin fixture scores on structure, not on a word list", async () => {
const report = await runMemoryBench({ fixture: fixture(), runsDir });
const nonLatin = report.questions.find((q) => q.id === "q-fidelity-non-latin")!;
expect(nonLatin.pass).toBe(true);
// And it really did extract - a pass over zero records would be a lie.
const raw = JSON.parse(
readFileSync(join(runsDir, report.run_id, "results", "q-fidelity-non-latin.json"), "utf8"),
) as { labels: string[]; dedup_stable: boolean };
expect(raw.labels).toEqual(["решение", "открытый_вопрос"]);
expect(raw.dedup_stable).toBe(true);
}, 60_000);
});
describe("metric 3: cross-source isolation gates at zero, non-vacuously", () => {
test("the strict gate is what isolates: without it the same fixture leaks", async () => {
const loaded = JSON.parse(JSON.stringify(fixture())) as {
notes: Array<{ path: string }>;
questions: Array<{ category: string }>;
};
const defaultGate = parseBenchFixture({
...loaded,
name: "failure-modes-shipped-default-gate",
notes: loaded.notes.filter((note) => note.path !== "Brain/_brain.yaml"),
questions: loaded.questions.filter((q) => q.category === "source_isolation"),
});
const report = await runMemoryBench({ fixture: defaultGate, runsDir });
// Under the shipped `off` default no scope is evaluated at all, so the
// other agent's memory is delivered. A gate asserted under that
// default would pass for a reason that has nothing to do with
// isolation, which is why the fixture sets `fail`.
expect(report.failure_modes.source_isolation_violations).toBe(1);
expect(report.questions[0]!.failure).toContain("pref-other");
}, 60_000);
test("the assertion reads delivered contents, never the hidden counter", () => {
// `hiddenByOwnerScope` counts DELIVERED items under `warn` and is
// withheld entirely under `fail`, so it cannot serve as a violation
// numerator. The scorer takes ids that came back.
expect(isolationViolations(["a", "b"], ["b", "c"])).toEqual(["b"]);
expect(isolationViolations(["a"], ["b"])).toEqual([]);
});
test("isolation that withholds the caller's own memory is also a failure", async () => {
const loaded = JSON.parse(JSON.stringify(fixture())) as Record<string, unknown>;
const overreaching = parseBenchFixture({
...loaded,
name: "failure-modes-overreaching-scope",
questions: [
{
id: "q-isolation-must-not-narrow",
category: "source_isolation",
// Nobody owns this scope, so `pref-own` is correctly withheld -
// and demanding it back proves the converse invariant is armed.
agent_scope: "agent-z",
expected_ids: ["pref-own"],
forbidden_ids: ["pref-other"],
max_tokens: 500,
},
],
});
const report = await runMemoryBench({ fixture: overreaching, runsDir });
expect(report.quality.passed).toBe(0);
expect(report.questions[0]!.failure).toContain("pref-own");
expect(report.failure_modes.source_isolation_violations).toBe(0);
}, 60_000);
});
describe("metric 4: exactly one token estimator is reachable from the bench", () => {
const BENCH_DIR = join("src", "core", "bench");
function benchSources(): ReadonlyArray<{ file: string; code: string }> {
return readdirSync(BENCH_DIR)
.filter((file) => file.endsWith(".ts"))
.map((file) => ({
file,
// The shared census lexer rather than two regexes, and the
// `withoutComments` view specifically: every assertion below
// reads an import specifier, which IS a string literal.
code: lexSource(readFileSync(join(BENCH_DIR, file), "utf8")).withoutComments,
}));
}
test("the one estimator is imported once, from the shared tokenizer", () => {
const importers = benchSources().filter(({ code }) => code.includes("estimateTokens"));
expect(importers.map(({ file }) => file)).toEqual(["failure-modes.ts"]);
expect(importers[0]!.code).toContain('from "../brain/text/tokenizer.ts"');
});
test("no second estimator survives anywhere in the bench", () => {
// This repo carries five disagreeing estimators, two of which export
// the SAME name from different modules - so the check is on the import
// path and on the arithmetic, never on the identifier alone.
for (const { file, code } of benchSources()) {
expect(`${file}: ${code}`).not.toContain("embeddings/signature.ts");
expect(`${file}: ${code}`).not.toContain("CHARS_PER_ESTIMATED_TOKEN");
expect(`${file}: ${code}`).not.toContain("countTokens");
// The inline `ceil(chars / 4)` the report used to carry.
expect(code).not.toMatch(/\/\s*4\s*[);]/);
}
});
});