Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* LongMemEval bench suite adapter (recall-quality wave, t_607d429d):
* parser with named errors, converter with abstention exclusion and the
* direct gold-session-id join, no expected_text reliance for retrieval
* scoring, and an end-to-end run over the committed sample fixture.
*/
import { afterEach, describe, expect, test } from "bun:test";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { benchSuite } from "../../../src/cli/brain/verbs/bench.ts";
import { embedBenchVaultIndex } from "../../../src/core/bench/embedding-cache.ts";
import { parseBenchFixture, materializeBenchVault } from "../../../src/core/bench/fixture.ts";
import {
LONGMEMEVAL_JUDGE_QUESTION_TEMPLATE,
LONGMEMEVAL_JUDGE_SYSTEM_PROMPT,
LONGMEMEVAL_PROMPT_PROVENANCE,
LONGMEMEVAL_RECALL_CUTOFF,
loadLongMemEvalFixture,
longMemEvalToBenchFixture,
parseLongMemEvalDataset,
scoreLongMemEvalRecall,
} from "../../../src/core/bench/longmemeval.ts";
import { sampleLongMemEvalEntries } from "../../../src/core/bench/longmemeval-sample.ts";
import { benchSearchConfig, runMemoryBench } from "../../../src/core/bench/phases.ts";
import { BENCH_REPORT_SCHEMA, type BenchPinnedKnobs } from "../../../src/core/bench/types.ts";
const SAMPLE = join("tests", "fixtures", "bench", "longmemeval-sample.json");
const OFFICIAL_SAMPLE = join("tests", "fixtures", "bench", "longmemeval-official-sample.json");
const dirs: string[] = [];
afterEach(() => {
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true });
});
/** A minimal valid entry, overridden per test. */
function entry(overrides: Record<string, unknown>): Record<string, unknown> {
const sessions = [
{
session_id: "s1",
date: "2026-03-02T10:00:00Z",
turns: [
{ role: "user", content: "We picked Blue Heron as the launch codename." },
{ role: "assistant", content: "Noted: the launch codename is Blue Heron." },
],
},
];
return {
q_id: "q1",
question_type: "single-hop",
question: "launch codename",
haystack_session_id: sessions.map((s) => s.session_id),
haystack_dates: sessions.map((s) => s.date),
haystack_session: sessions.map((s) => s.turns),
answer_session_ids: ["s1"],
...overrides,
};
}
test("an all-abstention dataset names the abstention filter, not a missing dataset", () => {
const dataset = parseLongMemEvalDataset([
entry({ q_id: "abstain-1", question_type: "abstention" }),
]);
expect(() => longMemEvalToBenchFixture(dataset)).toThrow(/abstention/);
});
test("session ids that sanitize to one note path are a named parse error", () => {
const dataset = parseLongMemEvalDataset([
entry({
q_id: "collide",
haystack_session_id: ["a/b", "a-b"],
haystack_dates: ["2026-01-01", "2026-01-02"],
haystack_session: [
[{ role: "user", content: "alpha note" }],
[{ role: "user", content: "beta note" }],
],
answer_session_ids: ["a/b"],
}),
]);
expect(() => longMemEvalToBenchFixture(dataset)).toThrow(/same note path/);
});
describe("the official dataset shape", () => {
test("the committed official-shape sample parses (official keys, types, _abs, question_date)", () => {
const raw: unknown = JSON.parse(readFileSync(OFFICIAL_SAMPLE, "utf8"));
const dataset = parseLongMemEvalDataset(raw);
expect(dataset.entries.map((e) => e.q_id)).toEqual([
"single-session-user_1",
"multi-session_2",
"single-session-user_3_abs",
]);
expect(dataset.entries[0]!.question_type).toBe("single-session-user");
expect(dataset.entries[0]!.question_date).toBe("2023/05/20 (Sat) 02:21");
});
test("official key spellings are read; the legacy sample shape keeps working", () => {
const official = parseLongMemEvalDataset([
{
question_id: "q-official",
question_type: "single-session-user",
question: "launch codename",
question_date: "2023/05/20 (Sat) 02:21",
haystack_session_ids: ["s1"],
haystack_sessions: [
[{ role: "user", content: "We picked Blue Heron as the launch codename." }],
],
haystack_dates: ["2023/05/01 (Mon) 09:15"],
answer_session_ids: ["s1"],
},
]);
expect(official.entries[0]!.q_id).toBe("q-official");
expect(official.entries[0]!.haystack_session_id).toEqual(["s1"]);
expect(official.entries[0]!.haystack_session).toHaveLength(1);
const legacy = parseLongMemEvalDataset([entry({})]);
expect(legacy.entries[0]!.q_id).toBe("q1");
expect(legacy.entries[0]!.haystack_session_id).toEqual(["s1"]);
});
test("every official question type maps to a bench category", () => {
const types = [
["single-session-user", "single_hop"],
["single-session-assistant", "single_hop"],
["single-session-preference", "single_hop"],
["multi-session", "multi_evidence"],
["temporal-reasoning", "temporal"],
["knowledge-update", "contradiction"],
] as const;
const dataset = parseLongMemEvalDataset(
types.map(([type], i) =>
entry({
q_id: `q-type-${i}`,
question_type: type,
haystack_session_id: [`s-type-${i}`],
haystack_dates: ["2026-03-02T10:00:00Z"],
haystack_session: [[{ role: "user", content: `type marker ${i} zebra.` }]],
answer_session_ids: [`s-type-${i}`],
}),
),
);
const { fixture } = longMemEvalToBenchFixture(dataset);
expect(fixture.questions.map((q) => q.category)).toEqual(types.map(([, category]) => category));
});
test("abstention is the _abs question-id suffix; the explicit type keeps working", () => {
const dataset = parseLongMemEvalDataset([
entry({
q_id: "single-session-user_9_abs",
question_type: "single-session-user",
answer_session_ids: [],
}),
entry({ q_id: "abstain-legacy", question_type: "abstention", answer_session_ids: [] }),
entry({}),
]);
const { fixture, receipt } = longMemEvalToBenchFixture(dataset);
expect(fixture.questions.map((q) => q.id)).toEqual(["q1"]);
expect(receipt.abstention_excluded).toBe(2);
});
test("question_date is read, falling back to the last haystack date", () => {
const dataset = parseLongMemEvalDataset([
entry({ question_date: "2023/05/20 (Sat) 02:21" }),
entry({ q_id: "q2" }),
]);
expect(dataset.entries[0]!.question_date).toBe("2023/05/20 (Sat) 02:21");
expect(dataset.entries[1]!.question_date).toBe("2026-03-02T10:00:00Z");
});
test("a non-date question_date is a named error", () => {
expect(() => parseLongMemEvalDataset([entry({ question_date: "eventually" })])).toThrow(
/question_date/,
);
});
test("the official-shape sample converts with the _abs exclusion", () => {
const raw: unknown = JSON.parse(readFileSync(OFFICIAL_SAMPLE, "utf8"));
const { fixture, receipt } = longMemEvalToBenchFixture(parseLongMemEvalDataset(raw));
expect(receipt.abstention_excluded).toBe(1);
expect(fixture.questions.map((q) => q.id)).toEqual([
"single-session-user_1",
"multi-session_2",
]);
expect(fixture.questions.map((q) => q.category)).toEqual(["single_hop", "multi_evidence"]);
});
});
describe("parseLongMemEvalDataset", () => {
test("accepts the official bare-array shape and a named wrapper", () => {
const bare = parseLongMemEvalDataset([entry({})]);
expect(bare.entries).toHaveLength(1);
const wrapped = parseLongMemEvalDataset({ name: "sample", questions: [entry({})] });
expect(wrapped.name).toBe("sample");
expect(wrapped.entries).toHaveLength(1);
});
test("a missing q_id is a named error", () => {
expect(() => parseLongMemEvalDataset([entry({ q_id: "" })])).toThrow(/q_id/);
});
test("an unknown question type is a named error listing the known types", () => {
expect(() => parseLongMemEvalDataset([entry({ question_type: "quantum" })])).toThrow(
/question_type/,
);
});
test("a non-abstention question with no gold sessions is a named error", () => {
expect(() => parseLongMemEvalDataset([entry({ answer_session_ids: [] })])).toThrow(
/answer_session_ids/,
);
});
test("a gold session id that joins no haystack session is rejected by name", () => {
expect(() => parseLongMemEvalDataset([entry({ answer_session_ids: ["s-missing"] })])).toThrow(
/unknown session 's-missing'/,
);
});
test("misaligned haystack arrays are a named error", () => {
expect(() =>
parseLongMemEvalDataset([entry({ haystack_dates: ["2026-03-02T10:00:00Z", "2026-03-03"] })]),
).toThrow(/aligned/);
});
test("an unparseable session date is a named error", () => {
expect(() => parseLongMemEvalDataset([entry({ haystack_dates: ["not-a-date"] })])).toThrow(
/valid date/,
);
});
test("a session with the same id but different content is a named error", () => {
const other = entry({
q_id: "q2",
haystack_session: [[{ role: "user", content: "Something entirely different about coffee." }]],
});
expect(() => parseLongMemEvalDataset([entry({}), other])).toThrow(/session 's1'/);
});
test("a shared session id with per-instance dates parses (first date wins)", () => {
const dataset = parseLongMemEvalDataset([
entry({}),
entry({
q_id: "q2",
question_type: "multi-hop",
haystack_dates: ["2026-09-09T10:00:00Z"],
}),
]);
// Upstream assigns haystack dates per instance: the same session id
// carrying a different date in each entry is data, not corruption.
expect(dataset.entries).toHaveLength(2);
expect(dataset.entries[1]!.haystack_dates).toEqual(["2026-09-09T10:00:00Z"]);
});
});
describe("longMemEvalToBenchFixture", () => {
test("abstention questions are excluded with the count recorded", () => {
const dataset = parseLongMemEvalDataset([
entry({}),
entry({ q_id: "q-abs", question_type: "abstention", answer_session_ids: [] }),
]);
const { fixture, receipt } = longMemEvalToBenchFixture(dataset);
expect(fixture.questions.map((q) => q.id)).toEqual(["q1"]);
expect(receipt.abstention_excluded).toBe(1);
});
test("gold session ids join directly to the session note paths", () => {
const dataset = parseLongMemEvalDataset([entry({})]);
const { fixture } = longMemEvalToBenchFixture(dataset);
expect(fixture.questions[0]!.expected_paths).toEqual(["sessions/q1/s1.md"]);
expect(fixture.questions[0]!.top_k).toBe(LONGMEMEVAL_RECALL_CUTOFF);
});
test("converted questions never set expected_text (retrieval scoring ignores it)", () => {
const dataset = parseLongMemEvalDataset([entry({ answer: "Blue Heron" })]);
const { fixture } = longMemEvalToBenchFixture(dataset);
for (const question of fixture.questions) {
expect(question.expected_text).toBeUndefined();
}
});
test("the converted fixture round-trips parseBenchFixture", () => {
const { fixture } = longMemEvalToBenchFixture(
parseLongMemEvalDataset([
entry({}),
entry({ q_id: "q2", question_type: "multi-hop", answer_session_ids: ["s1"] }),
]),
);
expect(() => parseBenchFixture(fixture)).not.toThrow();
});
test("sessions become notes and session_turn continuity records", () => {
const { fixture } = longMemEvalToBenchFixture(parseLongMemEvalDataset([entry({})]));
expect(fixture.notes.map((n) => n.path)).toEqual(["sessions/q1/s1.md"]);
expect(fixture.continuity).toHaveLength(2);
for (const record of fixture.continuity) {
expect(record.kind).toBe("session_turn");
expect(record.payload["question_id"]).toBe("q1");
expect(record.payload["session_id"]).toBe("s1");
expect(typeof record.payload["text"]).toBe("string");
}
});
test("each question's evidence is namespaced under its own subtree", () => {
// LongMemEval scores every question against ITS OWN haystack, so
// pooling all entries into one corpus would have each query search
// the union of all haystacks. The converted notes, continuity and
// expected_paths of one entry all live under sessions/<q_id>/.
const shared = [{ role: "user", content: "shared zebra session about quilting." }];
const dataset = parseLongMemEvalDataset([
entry({
q_id: "qA",
haystack_session_id: ["s1", "shared"],
haystack_dates: ["2026-03-02T10:00:00Z", "2026-03-02T11:00:00Z"],
haystack_session: [[{ role: "user", content: "quokka alpha note." }], shared],
answer_session_ids: ["s1"],
}),
entry({
q_id: "qB",
question_type: "multi-hop",
haystack_session_id: ["shared", "s3"],
haystack_dates: ["2026-03-02T11:00:00Z", "2026-03-03T10:00:00Z"],
haystack_session: [shared, [{ role: "user", content: "wombat beta note." }]],
answer_session_ids: ["s3"],
}),
]);
const { fixture } = longMemEvalToBenchFixture(dataset);
for (const question of fixture.questions) {
for (const path of question.expected_paths!) {
expect(path.startsWith(`sessions/${question.id}/`)).toBe(true);
}
}
const notePaths = fixture.notes.map((n) => n.path);
expect(notePaths).toContain("sessions/qA/s1.md");
expect(notePaths).toContain("sessions/qA/shared.md");
expect(notePaths).toContain("sessions/qB/s3.md");
expect(notePaths).toContain("sessions/qB/shared.md");
// A session shared by two entries exists once per scope, and its
// continuity records name the question that produced them.
const sharedRecords = fixture.continuity.filter((r) => r.payload["session_id"] === "shared");
expect(sharedRecords.map((r) => r.payload["question_id"]).sort()).toEqual(["qA", "qB"]);
});
test("distinct question ids that sanitize to one scope are a named error", () => {
const dataset = parseLongMemEvalDataset([
entry({ q_id: "q/1" }),
entry({ q_id: "q-1", question_type: "multi-hop" }),
]);
expect(() => longMemEvalToBenchFixture(dataset)).toThrow(/same path scope/);
});
test("a duplicate question id is a named error", () => {
const dataset = parseLongMemEvalDataset([entry({}), entry({})]);
expect(() => longMemEvalToBenchFixture(dataset)).toThrow(/appears twice/);
});
test("converted questions carry the suite judge input", () => {
const { fixture } = longMemEvalToBenchFixture(
parseLongMemEvalDataset([entry({ answer: "Blue Heron" })]),
);
expect(fixture.questions[0]!.judge_input).toEqual({
question: "launch codename",
answer: "Blue Heron",
gold: ["s1"],
});
});
test("a question without a stated reference answer omits the judge answer field", () => {
const { fixture } = longMemEvalToBenchFixture(parseLongMemEvalDataset([entry({})]));
expect(fixture.questions[0]!.judge_input).toBeDefined();
expect(fixture.questions[0]!.judge_input!.answer).toBeUndefined();
expect(fixture.questions[0]!.judge_input!.gold).toEqual(["s1"]);
});
});
describe("the judge prompts", () => {
test("the prompt constants state their self-written provenance", () => {
expect(LONGMEMEVAL_PROMPT_PROVENANCE).toContain("self-written");
expect(LONGMEMEVAL_PROMPT_PROVENANCE).toContain("github.com/xiaowu0162/LongMemEval");
expect(LONGMEMEVAL_PROMPT_PROVENANCE).toContain("evaluate_qa.py");
// No pretense of an upstream transcription or a revision pin.
expect(LONGMEMEVAL_PROMPT_PROVENANCE.toLowerCase()).not.toContain("pinned");
});
test("the system prompt and question template cover the payload fields", () => {
expect(LONGMEMEVAL_JUDGE_SYSTEM_PROMPT).toContain("evidence");
expect(LONGMEMEVAL_JUDGE_QUESTION_TEMPLATE).toContain("{question}");
expect(LONGMEMEVAL_JUDGE_QUESTION_TEMPLATE).toContain("{answer}");
expect(LONGMEMEVAL_JUDGE_QUESTION_TEMPLATE).toContain("{gold}");
});
});
describe("date parsing", () => {
test("official slash dates derive UTC turn timestamps, not host-local ones", () => {
const dataset = parseLongMemEvalDataset([
entry({
haystack_dates: ["2023/05/20 (Sat) 02:21"],
haystack_session: [[{ role: "user", content: "zebra anchor." }]],
}),
]);
const { fixture } = longMemEvalToBenchFixture(dataset);
// Host-local parsing shifts this by the zone offset; UTC pins it.
expect(fixture.continuity[0]!.created_at).toBe("2023-05-20T02:21:00.000Z");
});
test("the committed official sample derives UTC turn timestamps", () => {
const raw: unknown = JSON.parse(readFileSync(OFFICIAL_SAMPLE, "utf8"));
const { fixture } = longMemEvalToBenchFixture(parseLongMemEvalDataset(raw));
expect(fixture.continuity[0]!.created_at).toBe("2023-05-01T09:15:00.000Z");
});
test("a naive ISO date-time is read as UTC, not host-local", () => {
const dataset = parseLongMemEvalDataset([entry({ haystack_dates: ["2026-03-02T10:00:00"] })]);
const { fixture } = longMemEvalToBenchFixture(dataset);
expect(fixture.continuity[0]!.created_at).toBe("2026-03-02T10:00:00.000Z");
});
test("a date form only the host zone could interpret is a named error", () => {
expect(() => parseLongMemEvalDataset([entry({ haystack_dates: ["May 20, 2023"] })])).toThrow(
/valid date/,
);
});
});
describe("answer coercion", () => {
test("numeric answers are coerced to strings for the judge payload", () => {
const dataset = parseLongMemEvalDataset([
entry({ answer: 42 }),
entry({ q_id: "q2", question_type: "multi-hop", answer: 0 }),
]);
const { fixture } = longMemEvalToBenchFixture(dataset);
expect(fixture.questions[0]!.judge_input!.answer).toBe("42");
expect(fixture.questions[1]!.judge_input!.answer).toBe("0");
});
test("other non-string answer types stay refused", () => {
const dataset = parseLongMemEvalDataset([entry({ answer: true })]);
const { fixture } = longMemEvalToBenchFixture(dataset);
expect(fixture.questions[0]!.judge_input!.answer).toBeUndefined();
});
});
describe("scoreLongMemEvalRecall", () => {
const QUESTIONS = [
{
id: "q-both",
category: "multi_evidence" as const,
query: "q",
top_k: 5,
expected_paths: ["sessions/a.md", "sessions/b.md"],
},
{
id: "q-missing",
category: "single_hop" as const,
query: "q",
top_k: 5,
expected_paths: ["sessions/a.md", "sessions/z.md"],
},
{
id: "q-outside",
category: "single_hop" as const,
query: "q",
top_k: 5,
expected_paths: ["sessions/far.md"],
},
];
test("passes a question only when every gold note ranks inside the cutoff", () => {
const retrieved = [
{ id: "q-both", paths: ["sessions/b.md", "sessions/a.md", "sessions/c.md"] },
{
id: "q-missing",
paths: [
"sessions/a.md",
"sessions/b.md",
"sessions/c.md",
"sessions/d.md",
"sessions/e.md",
],
},
{
id: "q-outside",
paths: [
"sessions/a.md",
"sessions/b.md",
"sessions/c.md",
"sessions/d.md",
"sessions/e.md",
"sessions/far.md",
],
},
];
// q-missing never surfaces sessions/z.md; q-outside's gold note sits
// one rank past the strict cutoff. Exactly one question passes.
expect(scoreLongMemEvalRecall(QUESTIONS, retrieved)).toEqual({ scored: 3, passed: 1 });
});
test("a question with no persisted result scores as a miss, never a pass", () => {
expect(scoreLongMemEvalRecall(QUESTIONS, [])).toEqual({ scored: 3, passed: 0 });
});
test("non-retrieval questions are not scored", () => {
const mixed = [
...QUESTIONS,
{ id: "q-handoff", category: "session_handoff" as const, session_id: "s1" },
];
const scored = scoreLongMemEvalRecall(mixed, []);
expect(scored.scored).toBe(3);
});
});
describe("the longmemeval suite", () => {
test("is registered with the embeddings and sampling capabilities", () => {
const registration = benchSuite("longmemeval");
expect(registration).toBeDefined();
expect(registration!.capabilities).toEqual({ embeddings: true, sampling: true });
expect(typeof registration!.loader).toBe("function");
});
test("the loader converts the committed sample with its receipt", () => {
const { fixture, receipt } = loadLongMemEvalFixture(SAMPLE);
expect(fixture.name).toBe("longmemeval-sample");
expect(fixture.questions).toHaveLength(3);
expect(receipt.abstention_excluded).toBe(1);
});
test("the loader converts the committed official-shape sample with its receipt", () => {
const { fixture, receipt } = loadLongMemEvalFixture(OFFICIAL_SAMPLE);
expect(fixture.name).toBe("longmemeval-suite");
expect(fixture.questions).toHaveLength(2);
expect(receipt.abstention_excluded).toBe(1);
for (const question of fixture.questions) {
expect(question.expected_paths!.every((p) => p.startsWith(`sessions/${question.id}/`))).toBe(
true,
);
}
});
test("the official-shape sample runs end to end with the strict family", async () => {
const { fixture, receipt } = loadLongMemEvalFixture(OFFICIAL_SAMPLE);
const runsDir = mkdtempSync(join(tmpdir(), "o2b-longmemeval-runs-"));
dirs.push(runsDir);
const report = await runMemoryBench({
fixture,
runsDir,
suite: "longmemeval",
suiteReceipt: receipt,
capabilities: { embeddings: true },
// The official questions are natural sentences; the deterministic
// local provider matches them to their haystack notes, which is
// the lane the suite exists for.
pinnedKnobs: { semantic: { enabled: true, provider: "local" } },
});
expect(report.embeddings).toBeDefined();
expect(report.quality.passed).toBe(report.quality.total);
expect(report.longmemeval).toEqual({ recall_all_at_5: 1, abstention_excluded: 1 });
}, 30_000);
test("the keyword-only default pin matches nothing for natural official questions", async () => {
const { fixture, receipt } = loadLongMemEvalFixture(OFFICIAL_SAMPLE);
const runsDir = mkdtempSync(join(tmpdir(), "o2b-longmemeval-runs-"));
dirs.push(runsDir);
const report = await runMemoryBench({
fixture,
runsDir,
suite: "longmemeval",
suiteReceipt: receipt,
});
// The keyword lane requires every query token to hit, so a natural
// question scores zero notes - this is why the suite declares the
// embeddings capability, and the report says so rather than
// manufacturing a passing number.
expect(report.longmemeval).toEqual({ recall_all_at_5: 0, abstention_excluded: 1 });
expect(report.quality.passed).toBe(0);
}, 30_000);
test("running the committed sample produces a green report with the judge skipped", async () => {
const { fixture } = loadLongMemEvalFixture(SAMPLE);
const runsDir = mkdtempSync(join(tmpdir(), "o2b-longmemeval-runs-"));
dirs.push(runsDir);
const report = await runMemoryBench({ fixture, runsDir });
expect(report.schema).toBe(BENCH_REPORT_SCHEMA);
expect(report.fixture).toBe("longmemeval-sample");
expect(report.judge.status).toBe("skipped");
// Keyword-distinctive queries over a small haystack: all retrieval
// questions find their gold sessions inside the recall cutoff.
expect(report.quality.passed).toBe(report.quality.total);
expect(report.quality.total).toBe(3);
}, 30_000);
test("the suite runs the embeddings capability path end to end", async () => {
const { fixture } = loadLongMemEvalFixture(SAMPLE);
const runsDir = mkdtempSync(join(tmpdir(), "o2b-longmemeval-runs-"));
dirs.push(runsDir);
const report = await runMemoryBench({
fixture,
runsDir,
capabilities: { embeddings: true },
pinnedKnobs: { semantic: { enabled: true, provider: "local" } },
});
expect(report.embeddings).toBeDefined();
expect(report.embeddings!.provider).toBe("local");
expect(report.embeddings!.gate).toEqual({ blocked: false, reason: null });
expect(report.judge.status).toBe("skipped");
expect(report.quality.passed).toBe(report.quality.total);
}, 30_000);
test("a second identical vault is served by the shared vector cache", async () => {
const { fixture } = loadLongMemEvalFixture(SAMPLE);
const runsDir = mkdtempSync(join(tmpdir(), "o2b-longmemeval-runs-"));
dirs.push(runsDir);
// Two disposable vaults, same content, one shared runs-dir cache:
// the first arm pays the full census through the misses path, the
// second is served whole - the cache-hit contract the pending
// lookup exists for.
const vaultA = mkdtempSync(join(tmpdir(), "o2b-longmemeval-vault-"));
const vaultB = mkdtempSync(join(tmpdir(), "o2b-longmemeval-vault-"));
dirs.push(vaultA, vaultB);
materializeBenchVault(fixture, vaultA);
materializeBenchVault(fixture, vaultB);
const pin: BenchPinnedKnobs = { semantic: { enabled: true, provider: "local" } };
const first = await embedBenchVaultIndex({
config: benchSearchConfig(vaultA, pin),
runsDir,
});
expect(first.cache_hits).toBe(0);
expect(first.cache_misses).toBeGreaterThan(0);
const second = await embedBenchVaultIndex({
config: benchSearchConfig(vaultB, pin),
runsDir,
});
expect(second.cache_hits).toBe(first.cache_misses);
expect(second.cache_misses).toBe(0);
}, 30_000);
test("a suite run surfaces the strict recall_all@5 family with the exclusion count", async () => {
const { fixture, receipt } = loadLongMemEvalFixture(SAMPLE);
const runsDir = mkdtempSync(join(tmpdir(), "o2b-longmemeval-runs-"));
dirs.push(runsDir);
const report = await runMemoryBench({
fixture,
runsDir,
suite: "longmemeval",
suiteReceipt: receipt,
});
// The sample's three retrieval questions all find their gold
// sessions inside the cutoff; one abstention question was excluded.
expect(report.longmemeval).toEqual({ recall_all_at_5: 1, abstention_excluded: 1 });
}, 30_000);
test("without a suite receipt the exclusion count is null, never a manufactured 0", async () => {
const { fixture } = loadLongMemEvalFixture(SAMPLE);
const runsDir = mkdtempSync(join(tmpdir(), "o2b-longmemeval-runs-"));
dirs.push(runsDir);
const report = await runMemoryBench({ fixture, runsDir, suite: "longmemeval" });
// A direct caller that skips the conversion receipt must not get a
// report asserting zero exclusions: unknown is null, and a reader
// can tell the two apart.
expect(report.longmemeval).toEqual({ recall_all_at_5: 1, abstention_excluded: null });
}, 30_000);
test("the report family stays absent without the suite name", async () => {
const { fixture } = loadLongMemEvalFixture(SAMPLE);
const runsDir = mkdtempSync(join(tmpdir(), "o2b-longmemeval-runs-"));
dirs.push(runsDir);
const report = await runMemoryBench({ fixture, runsDir });
expect(report.longmemeval).toBeUndefined();
}, 30_000);
// The judge command runs under the platform shell (sh on POSIX,
// cmd.exe on Windows), so the fake judge avoids shell operators
// entirely: the test writes a script and passes both paths as
// quoted, forward-slashed arguments.
function writeFakeJudgeScript(runsDir: string): string {
const script = join(runsDir, "fake-judge.mjs");
writeFileSync(
script,
[
'import { writeFileSync } from "node:fs";',
"const chunks = [];",
"for await (const chunk of Bun.stdin.stream()) chunks.push(chunk);",
'const text = Buffer.concat(chunks).toString("utf8");',
"writeFileSync(process.argv[2], text);",
"const parsed = JSON.parse(text);",
"const verdicts = {};",
"for (const q of parsed.questions) verdicts[q.id] = true;",
"console.log(JSON.stringify({ verdicts }));",
"",
].join("\n"),
);
return script;
}
const shellSafe = (path: string) => `"${path.replaceAll("\\", "/")}"`;
test("a fake judge receives question, answer and gold and its verdicts land", async () => {
const { fixture } = loadLongMemEvalFixture(SAMPLE);
const runsDir = mkdtempSync(join(tmpdir(), "o2b-longmemeval-runs-"));
dirs.push(runsDir);
const stdinCapture = join(runsDir, "judge-stdin.json");
// The external judge stays a dumb command: capture the payload, then
// echo a true verdict for every question id it received.
const judgeCmd = `bun ${shellSafe(writeFakeJudgeScript(runsDir))} ${shellSafe(stdinCapture)}`;
const report = await runMemoryBench({ fixture, runsDir, judgeCmd });
expect(report.judge.status).toBe("ran");
for (const question of report.questions) {
expect(question.judge_pass).toBe(true);
}
const payload = JSON.parse(readFileSync(stdinCapture, "utf8")) as {
questions: Array<Record<string, unknown>>;
};
expect(payload.questions).toHaveLength(3);
// Judged questions carry the evidence the run actually injected, so
// the judge can evaluate against what the system showed.
const withEvidence = payload.questions.find(
(q) => Array.isArray(q["evidence"]) && (q["evidence"] as unknown[]).length > 0,
);
expect(withEvidence).toBeDefined();
for (const entry of withEvidence!["evidence"] as Array<Record<string, unknown>>) {
expect(typeof entry["path"]).toBe("string");
expect(typeof entry["text"]).toBe("string");
}
for (const payloadEntry of payload.questions) {
expect(typeof payloadEntry["question"]).toBe("string");
expect(Array.isArray(payloadEntry["gold"])).toBe(true);
expect((payloadEntry["gold"] as ReadonlyArray<string>).length).toBeGreaterThan(0);
}
// The sample states a reference answer for at least its first
// question; the payload carries it verbatim.
const withAnswer = payload.questions.find((q) => typeof q["answer"] === "string");
expect(withAnswer).toBeDefined();
}, 20_000);
test("a suite without judge input keeps the judge payload unchanged", async () => {
const native = parseBenchFixture({
name: "judge-payload-default",
notes: [{ path: "Brain/notes/a.md", body: "alpha note" }],
continuity: [],
questions: [
{ id: "q1", category: "single_hop", query: "alpha", expected_paths: ["Brain/notes/a.md"] },
],
});
const runsDir = mkdtempSync(join(tmpdir(), "o2b-longmemeval-runs-"));
dirs.push(runsDir);
const stdinCapture = join(runsDir, "judge-stdin.json");
const judgeCmd = `bun ${shellSafe(writeFakeJudgeScript(runsDir))} ${shellSafe(stdinCapture)}`;
const report = await runMemoryBench({ fixture: native, runsDir, judgeCmd });
expect(report.judge.status).toBe("ran");
const payload = JSON.parse(readFileSync(stdinCapture, "utf8")) as {
questions: Array<Record<string, unknown>>;
};
expect(payload.questions).toHaveLength(1);
// No suite-provided judge input: no question, answer or gold fields.
const first = payload.questions[0]!;
expect("question" in first).toBe(false);
expect("answer" in first).toBe(false);
expect("gold" in first).toBe(false);
}, 20_000);
});
/**
* Seeded sampling through the loader (wiring-shipped-seams wave,
* t_4501c662): sampling happens PRE-conversion on the parsed entries,
* the receipt carries the sample beside the abstention count, and the
* no-options path stays byte-identical to the shipped loader.
*/
describe("seeded sampling", () => {
/**
* A synthetic dataset: six questions across six types plus one
* abstention, all sharing the minimal valid entry's haystack shape.
*/
function writeSyntheticDataset(): string {
const dir = mkdtempSync(join(tmpdir(), "o2b-longmemeval-dataset-"));
dirs.push(dir);
const path = join(dir, "synthetic.json");
const entries = [
entry({}),
entry({ q_id: "q2", question_type: "multi-hop" }),
entry({ q_id: "q3", question_type: "temporal-reasoning" }),
entry({ q_id: "q4", question_type: "knowledge-update" }),
entry({ q_id: "q5", question_type: "single-session-user" }),
entry({ q_id: "q6", question_type: "multi-session" }),
entry({ q_id: "abs-1", question_type: "abstention", answer_session_ids: [] }),
];
writeFileSync(path, `${JSON.stringify(entries)}\n`, "utf8");
return path;
}
test("the no-options loader path returns the shipped conversion", () => {
const plain = loadLongMemEvalFixture(SAMPLE);
expect(loadLongMemEvalFixture(SAMPLE, {})).toEqual(plain);
expect(loadLongMemEvalFixture(SAMPLE, undefined)).toEqual(plain);
// The receipt carries no sample fields on the off path.
expect(plain.receipt).toEqual({ abstention_excluded: 1 });
});
test("with sample options the sampler runs pre-conversion and the receipt carries the sample", () => {
const datasetPath = writeSyntheticDataset();
const options = { sample: { seed: "seed-1", cap: 3 } as const };
const conversion = loadLongMemEvalFixture(datasetPath, options);
// Sampling provably happened BEFORE conversion: the converted
// fixture's question count is the sampled count, not the dataset's.
const dataset = parseLongMemEvalDataset(JSON.parse(readFileSync(datasetPath, "utf8")));
const direct = sampleLongMemEvalEntries(dataset.entries, options.sample);
expect(conversion.fixture.questions).toHaveLength(direct.receipt.selected);
expect(conversion.fixture.questions.map((q) => q.id)).toEqual(
direct.entries.map((e) => e.q_id),
);
// The receipt mirrors the abstention precedent and adds the sample
// beside it. The abstention count is the SAMPLER's - the exclusion
// happened before selection, so the conversion's own count is 0.
expect(conversion.receipt).toEqual({
abstention_excluded: 1,
sample_population: direct.receipt.population,
sample_selected: direct.receipt.selected,
sample_strata: direct.receipt.strata,
});
expect(conversion.receipt.sample_population).toBe(6);
expect(conversion.receipt.sample_selected).toBe(3);
// Same options, same subset - the loader adds no nondeterminism.
expect(loadLongMemEvalFixture(datasetPath, options).fixture).toEqual(conversion.fixture);
});
test("a sampled run reports the sampled count through the longmemeval family", async () => {
const datasetPath = writeSyntheticDataset();
const { fixture, receipt } = loadLongMemEvalFixture(datasetPath, {
sample: { seed: "seed-1", cap: 3 },
});
const runsDir = mkdtempSync(join(tmpdir(), "o2b-longmemeval-runs-"));
dirs.push(runsDir);
const report = await runMemoryBench({
fixture,
runsDir,
suite: "longmemeval",
suiteReceipt: receipt,
});
expect(report.quality.total).toBe(3);
// The sampled bookkeeping - selected count, population and the
// per-stratum breakdown - rides the named family, so the report
// shows what the run's numbers cover. Frozen worked example: six
// one-per-type survivors at cap 3 give every stratum quota 0.5, and
// the three surplus seats go to the lexicographically first strata.
expect(report.longmemeval).toEqual({
recall_all_at_5: 1,
abstention_excluded: 1,
sample_population: 6,
sample_selected: 3,
sample_strata: [
{ question_type: "knowledge-update", population: 1, selected: 1 },
{ question_type: "multi-hop", population: 1, selected: 1 },
{ question_type: "multi-session", population: 1, selected: 1 },
{ question_type: "single-hop", population: 1, selected: 0 },
{ question_type: "single-session-user", population: 1, selected: 0 },
{ question_type: "temporal-reasoning", population: 1, selected: 0 },
],
});
// And the family in the persisted report.json carries the same
// fields - the receipt is visible where an operator actually reads it.
const persisted = JSON.parse(
readFileSync(join(runsDir, report.run_id, "report.json"), "utf8"),
) as { longmemeval: Record<string, unknown> };
const persistedFamily = persisted.longmemeval!;
expect(Object.keys(persistedFamily).toSorted()).toEqual([
"abstention_excluded",
"recall_all_at_5",
"sample_population",
"sample_selected",
"sample_strata",
]);
expect(persistedFamily["sample_population"]).toBe(6);
expect(persistedFamily["sample_selected"]).toBe(3);
expect(persistedFamily["sample_strata"]).toEqual(report.longmemeval!.sample_strata);
}, 20_000);
test("a whole-dataset run's family carries no sample fields", async () => {
const datasetPath = writeSyntheticDataset();
const { fixture, receipt } = loadLongMemEvalFixture(datasetPath);
const runsDir = mkdtempSync(join(tmpdir(), "o2b-longmemeval-runs-"));
dirs.push(runsDir);
const report = await runMemoryBench({
fixture,
runsDir,
suite: "longmemeval",
suiteReceipt: receipt,
});
// Spread-omission, not manufactured zeros: the unsampled report's
// family names exactly the strict fields and nothing else. The key
// set is the pin, not the score.
expect(Object.keys(report.longmemeval!)).toEqual(["recall_all_at_5", "abstention_excluded"]);
expect(report.longmemeval!.abstention_excluded).toBe(1);
}, 20_000);
});