Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Per-language rerank-eval split (recall-quality, t_693fa22a / task D1).
*
* The dataset schema grows an optional `lang` tag (validated like
* `answer`: a non-empty string when present, defaulting to "en"), and the
* benchmark report folds a `byLang` aggregate out of `perQuery` with the
* same reduce the overall report already uses - one fold, run per
* language group, never a second scorer. The change is additive to the
* RETURN VALUE: the benchmark emits no record and no schema version
* moves, so existing datasets without a `lang` field stay valid and the
* single-language report keeps its numbers.
*
* The offline ZH fixture gate that consumes the split lives in
* `recall-benchmark-zh.test.ts`.
*/
import { afterAll, beforeAll, describe, expect, setDefaultTimeout, test } from "bun:test";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
parseRecallBenchmarkDataset,
runRecallBenchmark,
type RecallBenchmarkLanguageReport,
type RecallBenchmarkQueryResult,
type RecallBenchmarkReport,
} from "../../../src/core/search/benchmark.ts";
import { rerankEvalVerdict, runRerankEvalGate } from "../../../src/core/search/rerank-eval-gate.ts";
import { indexVault } from "../../../src/core/search/indexer.ts";
import { SearchError, type ResolvedSearchConfig } from "../../../src/core/search/types.ts";
import { makeConfig, writeMd } from "../../helpers/search-fixtures.ts";
import { CLI_SPAWN_BUDGET_MS } from "../../helpers/cli-timeout.ts";
import { runCli } from "../../helpers/run-cli.ts";
// The CLI section below drives the real `o2b` command line.
setDefaultTimeout(CLI_SPAWN_BUDGET_MS);
describe("lang group folding", () => {
test("spelling variants of one language fold into one group", () => {
const dataset = {
queries: [
{ id: "q1", query: "a", expected: ["a.md"], lang: " ZH" },
{ id: "q2", query: "b", expected: ["b.md"], lang: "zh" },
{ id: "q3", query: "c", expected: ["c.md"], lang: "ZH" },
],
};
const parsed = parseRecallBenchmarkDataset(dataset);
const langs = new Set(parsed.queries.map((q) => q.lang));
expect([...langs]).toEqual(["zh"]);
});
});
describe("parseRecallBenchmarkDataset lang field", () => {
test("an absent lang defaults to en", () => {
const dataset = parseRecallBenchmarkDataset({
queries: [{ id: "q1", query: "canary rollout", expected: ["deploy-canary.md"] }],
});
expect(dataset.queries[0]!.lang).toBe("en");
});
test("an explicit lang is preserved verbatim", () => {
const dataset = parseRecallBenchmarkDataset({
queries: [{ id: "q1", query: "灰度发布", expected: ["zh-canary.md"], lang: "zh" }],
});
expect(dataset.queries[0]!.lang).toBe("zh");
});
test("a blank or non-string lang is refused naming the field, like answer", () => {
expect(() =>
parseRecallBenchmarkDataset({
queries: [{ id: "x", query: "q", expected: ["a.md"], lang: " " }],
}),
).toThrow(SearchError);
expect(() =>
parseRecallBenchmarkDataset({
queries: [{ id: "x", query: "q", expected: ["a.md"], lang: " " }],
}),
).toThrow(/lang must be a non-empty string when present/);
expect(() =>
parseRecallBenchmarkDataset({
queries: [{ id: "x", query: "q", expected: ["a.md"], lang: 42 }],
}),
).toThrow(/lang must be a non-empty string when present/);
});
});
// ----- the byLang fold -------------------------------------------------------
/** The same reduce the overall report applies, written out for comparison. */
function foldExpected(rows: ReadonlyArray<RecallBenchmarkQueryResult>) {
const hits = rows.filter((r) => r.hit).length;
const answerScored = rows.filter((r) => r.answerContained !== null);
const answerHits = answerScored.filter((r) => r.answerContained === true).length;
const foundTotal = rows.reduce((sum, r) => sum + r.expectedFound, 0);
const declaredTotal = rows.reduce((sum, r) => sum + r.expectedTotal, 0);
const hitRanks = rows.filter((r) => r.rank !== null).map((r) => r.rank!);
return {
total: rows.length,
hitAtK: hits / rows.length,
mrr: rows.reduce((sum, r) => sum + r.reciprocalRank, 0) / rows.length,
answerQueries: answerScored.length,
answerContainmentAtK: answerScored.length === 0 ? 1 : answerHits / answerScored.length,
sourceUtilizationAtK: declaredTotal === 0 ? 1 : foundTotal / declaredTotal,
citationDepth:
hitRanks.length === 0 ? 0 : hitRanks.reduce((sum, r) => sum + r, 0) / hitRanks.length,
sourceWarnings: rows.length - hits,
};
}
/** The mixed en/zh dataset every gate and fold test below scores. */
const mixedDataset = () =>
parseRecallBenchmarkDataset({
queries: [
{
id: "en-canary",
query: "canary rollout error budget",
expected: ["deploy-canary.md"],
answer: "expand the rollout gradually",
},
{ id: "en-garden", query: "compost tomato balcony", expected: ["garden.md"] },
{
id: "zh-canary",
query: "金丝雀发布 错误预算",
expected: ["zh-canary.md"],
answer: "错误预算",
lang: "zh",
},
{ id: "zh-garden", query: "阳台种植 番茄", expected: ["zh-garden.md"], lang: "zh" },
],
});
describe("runRecallBenchmark byLang fold", () => {
let vault: string;
let config: ResolvedSearchConfig;
beforeAll(async () => {
vault = mkdtempSync(join(tmpdir(), "o2b-bench-lang-"));
// Self-authored two-language mini corpus: the same note in each
// language is a different page, so each group has its own hit and
// its own miss candidate and the fold has something to separate.
writeMd(
vault,
"deploy-canary.md",
"# Canary rollouts\n\nExpand the rollout gradually and watch the error budget before widening.",
);
writeMd(
vault,
"garden.md",
"# Balcony garden\n\nTomatoes and basil move to the south planter in spring; turn the compost two weeks ahead.",
);
writeMd(
vault,
"zh-canary.md",
"# 灰度发布笔记\n\n金丝雀发布先把小部分流量切到新版本,再逐步扩大范围,扩量之前要盯住错误预算并演练一次回滚。",
);
writeMd(
vault,
"zh-garden.md",
"# 阳台种植笔记\n\n春季把番茄和罗勒移到朝南的花箱,堆肥提前两周翻一次,浇水选在清晨进行。",
);
config = makeConfig({
vault,
dbPath: join(vault, "index.sqlite"),
semantic: { enabled: true, provider: "local", dimension: 256 },
});
await indexVault(config, { embeddings: true });
});
afterAll(() => rmSync(vault, { recursive: true, force: true }));
test("perQuery rows carry their language and untagged rows read en", async () => {
const report = await runRecallBenchmark(config, mixedDataset(), { k: 5 });
expect(report.perQuery.map((q) => q.lang)).toEqual(["en", "en", "zh", "zh"]);
});
test("byLang partitions perQuery and folds each group with the overall reduce", async () => {
const report = await runRecallBenchmark(config, mixedDataset(), { k: 5 });
expect(Object.keys(report.byLang).toSorted()).toEqual(["en", "zh"]);
expect(report.byLang["en"]!.total).toBe(2);
expect(report.byLang["zh"]!.total).toBe(2);
// The language aggregate IS the overall fold applied to the group.
expect(report.byLang["en"]).toEqual(
foldExpected(report.perQuery.filter((q) => q.lang === "en")),
);
expect(report.byLang["zh"]).toEqual(
foldExpected(report.perQuery.filter((q) => q.lang === "zh")),
);
// Answer-bearing queries are counted per group, not pooled.
expect(report.byLang["en"]!.answerQueries).toBe(1);
expect(report.byLang["zh"]!.answerQueries).toBe(1);
// The overall numbers stay the pooled ones the report always had.
expect(report.total).toBe(4);
expect(report.answerQueries).toBe(2);
});
test("a dataset without any lang tag folds under en alone", async () => {
const report = await runRecallBenchmark(
config,
parseRecallBenchmarkDataset({
queries: [{ id: "plain", query: "canary rollout", expected: ["deploy-canary.md"] }],
}),
{ k: 5 },
);
expect(Object.keys(report.byLang)).toEqual(["en"]);
expect(report.byLang["en"]!.total).toBe(1);
});
test("the split is deterministic across runs", async () => {
const first = await runRecallBenchmark(config, mixedDataset(), { k: 5 });
const second = await runRecallBenchmark(config, mixedDataset(), { k: 5 });
expect(second).toEqual(first);
});
});
// ----- per-language rerank deltas (t_693fa22a review, M14 part 2) ------------
describe("runRerankEvalGate per-language deltas", () => {
let vault: string;
let config: ResolvedSearchConfig;
beforeAll(async () => {
vault = mkdtempSync(join(tmpdir(), "o2b-rerank-lang-"));
// The same two-language mini corpus the byLang fold above uses: one
// hit and one miss candidate per group, so each language's delta is
// its own number, never a pooled one.
writeMd(
vault,
"deploy-canary.md",
"# Canary rollouts\n\nExpand the rollout gradually and watch the error budget before widening.",
);
writeMd(
vault,
"garden.md",
"# Balcony garden\n\nTomatoes and basil move to the south planter in spring; turn the compost two weeks ahead.",
);
writeMd(
vault,
"zh-canary.md",
"# 灰度发布笔记\n\n金丝雀发布先把小部分流量切到新版本,再逐步扩大范围,扩量之前要盯住错误预算并演练一次回滚。",
);
writeMd(
vault,
"zh-garden.md",
"# 阳台种植笔记\n\n春季把番茄和罗勒移到朝南的花箱,堆肥提前两周翻一次,浇水选在清晨进行。",
);
config = makeConfig({
vault,
dbPath: join(vault, "index.sqlite"),
semantic: { enabled: true, provider: "local", dimension: 256 },
});
await indexVault(config, { embeddings: true });
});
afterAll(() => rmSync(vault, { recursive: true, force: true }));
test("deltas.byLang carries a hit and MRR delta per language, each equal to the arm difference", async () => {
const gate = await runRerankEvalGate(config, mixedDataset(), { kind: "local", k: 5 });
expect(Object.keys(gate.deltas.byLang).toSorted()).toEqual(["en", "zh"]);
for (const lang of ["en", "zh"]) {
const before = gate.baseline.byLang[lang]!;
const after = gate.reranked.byLang[lang]!;
expect(gate.deltas.byLang[lang]!.hitAtK).toBeCloseTo(after.hitAtK - before.hitAtK, 12);
expect(gate.deltas.byLang[lang]!.mrr).toBeCloseTo(after.mrr - before.mrr, 12);
}
// The combined deltas stay the pooled ones the report always had.
expect(gate.deltas.hitAtK).toBe(gate.reranked.hitAtK - gate.baseline.hitAtK);
expect(gate.deltas.mrr).toBe(gate.reranked.mrr - gate.baseline.mrr);
});
});
// ----- the verdict rule blocks on a one-language regression (M14 part 3) -----
describe("rerankEvalVerdict blocks on a one-language regression", () => {
// Hand-built reports: the rule is pure, so constructed runs pin it
// exactly where a full two-arm offline run would need a reranker that
// behaves differently per language. Each group has 2 queries, so the
// pooled numbers are the plain average of the two groups'.
const group = (hitAtK: number, mrr: number): RecallBenchmarkLanguageReport => ({
total: 2,
hitAtK,
mrr,
answerQueries: 0,
answerContainmentAtK: 1,
sourceUtilizationAtK: 1,
citationDepth: 0,
sourceWarnings: Math.round(2 * (1 - hitAtK)),
});
const report = (
hitAtK: number,
mrr: number,
byLang: Record<string, RecallBenchmarkLanguageReport>,
): RecallBenchmarkReport => ({
total: 4,
k: 5,
expand: false,
hitAtK,
mrr,
answerQueries: 0,
answerContainmentAtK: 1,
sourceUtilizationAtK: 1,
citationDepth: 0,
sourceWarnings: 0,
degraded: [],
perQuery: [],
byLang,
});
// Baseline: en weak (the reranker has room to help), zh already perfect.
const baseline = report(0.75, 0.75, { en: group(0.5, 0.5), zh: group(1, 1) });
test("a zh MRR regression blocks enable despite a pooled lift on both families", () => {
// en doubles its numbers; zh loses 0.2 MRR. The pooled delta is
// +0.25 hit@k and +0.15 MRR - the pooled-only rule read this as
// enable, which is exactly the M14 hole.
const reranked = report(1, 0.9, { en: group(1, 1), zh: group(1, 0.8) });
const v = rerankEvalVerdict(baseline, reranked, 0.01, 0.01);
expect(v.deltas.hitAtK).toBeGreaterThan(0.01);
expect(v.deltas.mrr).toBeGreaterThan(0.01);
expect(v.deltas.byLang["zh"]!.mrr).toBeCloseTo(-0.2, 12);
expect(v.improves).toBe(false);
expect(v.recommendation).toBe("keep-disabled");
});
test("a zh hit@k regression blocks enable despite a pooled MRR lift", () => {
const reranked = report(0.875, 0.9, { en: group(1, 1), zh: group(0.75, 0.8) });
const v = rerankEvalVerdict(baseline, reranked, 0.01, 0.01);
expect(v.deltas.mrr).toBeGreaterThan(0.01);
expect(v.deltas.byLang["zh"]!.hitAtK).toBeCloseTo(-0.25, 12);
expect(v.improves).toBe(false);
expect(v.recommendation).toBe("keep-disabled");
});
test("every language lifting enables", () => {
const reranked = report(1, 1, { en: group(1, 1), zh: group(1, 1) });
const v = rerankEvalVerdict(baseline, reranked, 0.01, 0.01);
expect(v.improves).toBe(true);
expect(v.recommendation).toBe("enable");
});
test("a flat language is not a regression", () => {
// zh stays exactly where it was while en lifts: no regression, so
// the pooled hit@k lift is enough to enable.
const reranked = report(0.875, 0.875, { en: group(1, 1), zh: group(1, 1) });
const v = rerankEvalVerdict(baseline, reranked, 0.01, 0.01);
expect(v.deltas.byLang["zh"]!.hitAtK).toBe(0);
expect(v.deltas.byLang["zh"]!.mrr).toBe(0);
expect(v.improves).toBe(true);
expect(v.recommendation).toBe("enable");
});
});
// ----- the CLI renders the split ---------------------------------------------
describe("o2b search rerank-eval language split", () => {
let tmp: string;
let vault: string;
let configPath: string;
beforeAll(async () => {
tmp = mkdtempSync(join(tmpdir(), "osb-rerank-eval-lang-"));
vault = join(tmp, "vault");
mkdirSync(vault, { recursive: true });
configPath = join(tmp, "config.yaml");
writeFileSync(configPath, `vault: "${vault}"\n`);
writeMd(vault, "ml.md", "# Machine learning\n\nGradient descent optimizes the loss surface.\n");
writeMd(vault, "zh-ml.md", "# 机器学习笔记\n\n梯度下降法沿着损失面往下走,逐步逼近极小值。\n");
writeFileSync(
join(tmp, "dataset.json"),
JSON.stringify({
queries: [
{ id: "en-q", query: "gradient descent", expected: ["ml.md"] },
{ id: "zh-q", query: "梯度下降 损失", expected: ["zh-ml.md"], lang: "zh" },
],
}),
);
const env = { OPEN_SECOND_BRAIN_CONFIG: configPath };
const indexed = await runCli(["search", "index", "--config", configPath], { env });
expect(indexed.returncode).toBe(0);
});
afterAll(() => rmSync(tmp, { recursive: true, force: true }));
const args = (extra: ReadonlyArray<string> = []) => [
"search",
"rerank-eval",
"--config",
configPath,
"--dataset",
join(tmp, "dataset.json"),
...extra,
];
test("--json carries a by_lang block on every arm", async () => {
const res = await runCli(args(["--json"]), { env: { OPEN_SECOND_BRAIN_CONFIG: configPath } });
expect(res.returncode).toBe(0);
const out = JSON.parse(res.stdout) as Record<string, Record<string, unknown>>;
expect(out["baseline"]!["by_lang"]).toMatchObject({
en: { total: 1 },
zh: { total: 1 },
});
expect(out["local"]!["by_lang"]).toMatchObject({
en: { total: 1 },
zh: { total: 1 },
});
});
test("--json carries per-language deltas on the reranked arms, none on the baseline", async () => {
const res = await runCli(args(["--json"]), { env: { OPEN_SECOND_BRAIN_CONFIG: configPath } });
expect(res.returncode).toBe(0);
const out = JSON.parse(res.stdout) as Record<string, Record<string, unknown>>;
const baselineByLang = out["baseline"]!["by_lang"] as Record<string, Record<string, number>>;
// The baseline arm is the reference: absolute numbers only.
expect(baselineByLang["zh"]).not.toHaveProperty("delta_mrr");
const armByLang = out["local"]!["by_lang"] as Record<string, Record<string, number>>;
for (const lang of ["en", "zh"]) {
const d = armByLang[lang]!;
expect(d).toHaveProperty("delta_hit_at_k");
expect(d).toHaveProperty("delta_mrr");
// Correctness: each language's delta is that group's arm difference,
// not the pooled one.
const base = baselineByLang[lang]! as Record<string, number>;
expect(d["delta_hit_at_k"]).toBeCloseTo(d["hit_at_k"]! - base["hit_at_k"]!, 12);
expect(d["delta_mrr"]).toBeCloseTo(d["mrr"]! - base["mrr"]!, 12);
}
});
test("text output renders the per-language delta on each reranked-arm language line", async () => {
const res = await runCli(args(), { env: { OPEN_SECOND_BRAIN_CONFIG: configPath } });
expect(res.returncode).toBe(0);
// The arm lines carry a signed per-language delta; the off line does not.
expect(res.stdout).toContain("(delta hit@k +0.0000, MRR +0.0000)");
expect(res.stdout.match(/\(delta hit@k/g)?.length).toBe(2);
});
test("text output renders one line per language per arm", async () => {
const res = await runCli(args(), { env: { OPEN_SECOND_BRAIN_CONFIG: configPath } });
expect(res.returncode).toBe(0);
expect(res.stdout).toContain("en: 1 query(ies)");
expect(res.stdout).toContain("zh: 1 query(ies)");
});
});