Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import {
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { queryDemandLogPath } from "../../../src/core/brain/paths.ts";
import {
DEMAND_LOG_MAX_BYTES,
DEMAND_MAX_TERMS_PER_RECORD,
DEMAND_TERM_MAX_LEN,
aggregateQueryDemand,
normalizeQueryTerms,
readQueryDemand,
recordQueryDemand,
} from "../../../src/core/brain/query-demand.ts";
import { fakeCredential } from "../../helpers/fake-credentials.ts";
import { withDeviceId } from "../../helpers/device-id.ts";
const SECRET_SHAPED_TERM = fakeCredential("sk-", "abcdef0123456789abcdef0123456789");
let vault: string;
beforeEach(() => {
vault = mkdtempSync(join(tmpdir(), "o2b-query-demand-"));
// A legacy Brain/ tree: an unswitched machine (a vault without one is store-only).
mkdirSync(join(vault, "Brain"), { recursive: true });
});
afterEach(() => {
rmSync(vault, { recursive: true, force: true });
});
describe("normalizeQueryTerms", () => {
test("keeps significant terms, sorts, dedupes, drops short tokens", () => {
expect(normalizeQueryTerms("How does the AUTH flow work flow")).toEqual([
"auth",
"does",
"flow",
"how",
"the",
"work",
]);
});
test("drops secret-shaped tokens the redactor rewrites", () => {
const terms = normalizeQueryTerms(`api key ${SECRET_SHAPED_TERM} rotation`);
expect(terms).toContain("api");
expect(terms).toContain("key");
expect(terms).toContain("rotation");
expect(terms.some((t) => t.includes("sk-abcdef"))).toBe(false);
expect(terms.some((t) => t.includes("REDACTED"))).toBe(false);
});
test("empty only when the query carries no word characters at all", () => {
// Short tokens are terms now. The three-character floor that dropped
// them was an English-shaped stopword rule: it also erased every
// one- and two-character CJK query, so a whole class of recall gap
// could never form a bucket and was invisible to the promotion loop.
expect(normalizeQueryTerms("a it")).toEqual(["a", "it"]);
expect(normalizeQueryTerms("検索")).toEqual(["検索"]);
expect(normalizeQueryTerms("?? -- ??")).toEqual([]);
});
});
describe("recordQueryDemand", () => {
test("appends a normalized record and never stores the raw query", () => {
const record = recordQueryDemand(vault, {
query: "deployment rollback procedure",
resultCount: 0,
coverage: 0.1,
at: "2026-07-01T10:00:00.000Z",
});
expect(record).not.toBeNull();
expect(record!.terms).toEqual(["deployment", "procedure", "rollback"]);
expect(record!.results).toBe(0);
expect(record!.coverage).toBe(0.1);
const raw = readFileSync(queryDemandLogPath(vault), "utf8");
expect(raw).not.toContain("deployment rollback procedure");
expect(raw.trim().split("\n")).toHaveLength(1);
});
test("omits coverage when not provided", () => {
const record = recordQueryDemand(vault, { query: "orphaned notes cleanup", resultCount: 3 });
expect(record!.coverage).toBeUndefined();
});
test("does not record a query with no significant terms", () => {
// Punctuation only: nothing to bucket in any language.
const record = recordQueryDemand(vault, { query: "?!? ...", resultCount: 5 });
expect(record).toBeNull();
expect(existsSync(queryDemandLogPath(vault))).toBe(false);
});
test("clamps out-of-range coverage and negative result counts", () => {
const r = recordQueryDemand(vault, {
terms: ["alpha"],
resultCount: -4,
coverage: 1.7,
at: "2026-07-01T10:00:00.000Z",
});
expect(r!.results).toBe(0);
expect(r!.coverage).toBe(1);
});
});
describe("readQueryDemand", () => {
test("filters by since/until and sorts by timestamp", () => {
recordQueryDemand(vault, { terms: ["aaa"], resultCount: 1, at: "2026-07-03T00:00:00.000Z" });
recordQueryDemand(vault, { terms: ["bbb"], resultCount: 1, at: "2026-07-01T00:00:00.000Z" });
recordQueryDemand(vault, { terms: ["ccc"], resultCount: 1, at: "2026-07-02T00:00:00.000Z" });
const all = readQueryDemand(vault);
expect(all.map((r) => r.ts)).toEqual([
"2026-07-01T00:00:00.000Z",
"2026-07-02T00:00:00.000Z",
"2026-07-03T00:00:00.000Z",
]);
const windowed = readQueryDemand(vault, {
since: "2026-07-02T00:00:00.000Z",
until: "2026-07-02T23:59:59.000Z",
});
expect(windowed.map((r) => r.terms[0])).toEqual(["ccc"]);
});
test("empty when the log does not exist", () => {
expect(readQueryDemand(vault)).toEqual([]);
});
test("second-precision since/until bounds compare consistently against ms records", () => {
// Records are stored at millisecond precision. A raw lexical compare
// against a second-precision bound misfires because `.` (0x2E) sorts
// before `Z` (0x5A): `since 00:00:00Z` wrongly DROPS the whole
// [.000, .999] window, and `until 00:00:00Z` wrongly KEEPS it.
recordQueryDemand(vault, { terms: ["before"], resultCount: 1, at: "2026-06-30T23:59:59.500Z" });
recordQueryDemand(vault, {
terms: ["lowedge"],
resultCount: 1,
at: "2026-07-01T00:00:00.000Z",
});
recordQueryDemand(vault, { terms: ["midlow"], resultCount: 1, at: "2026-07-01T00:00:00.500Z" });
recordQueryDemand(vault, {
terms: ["highedge"],
resultCount: 1,
at: "2026-07-01T00:00:00.999Z",
});
recordQueryDemand(vault, { terms: ["after"], resultCount: 1, at: "2026-07-01T00:00:01.000Z" });
// `since 00:00:00Z` (normalized to .000Z) keeps the whole second and the
// record after it; the pre-fix lexical compare dropped lowedge/midlow/
// highedge because `.NNNZ` < `Z`.
const fromSecond = readQueryDemand(vault, { since: "2026-07-01T00:00:00Z" });
expect(fromSecond.map((r) => r.terms[0])).toEqual(["lowedge", "midlow", "highedge", "after"]);
// `until 00:00:00Z` (normalized to .000Z) is inclusive of exactly .000Z
// and excludes the rest of the second; the pre-fix lexical compare
// wrongly kept midlow/highedge (`.NNNZ` < `Z` ⇒ not `> until`).
const untilSecond = readQueryDemand(vault, { until: "2026-07-01T00:00:00Z" });
expect(untilSecond.map((r) => r.terms[0])).toEqual(["before", "lowedge"]);
});
});
describe("aggregateQueryDemand", () => {
test("buckets recurring poorly-answered queries and ranks by demand", () => {
// "vault backups" asked 3x, always weak coverage -> high demand.
for (let i = 0; i < 3; i++) {
recordQueryDemand(vault, {
query: "vault backups schedule",
resultCount: 0,
coverage: 0.05,
at: `2026-07-0${i + 1}T00:00:00.000Z`,
});
}
// "release process" asked 2x, well covered -> excluded by default.
for (let i = 0; i < 2; i++) {
recordQueryDemand(vault, {
query: "release process steps",
resultCount: 5,
coverage: 0.95,
at: `2026-07-0${i + 1}T01:00:00.000Z`,
});
}
// "token budget" asked once -> below minOccurrences.
recordQueryDemand(vault, {
query: "token budget tuning",
resultCount: 1,
coverage: 0.2,
at: "2026-07-01T02:00:00.000Z",
});
const report = aggregateQueryDemand(vault);
expect(report.totalRecords).toBe(6);
expect(report.distinctQueries).toBe(3);
expect(report.gaps).toHaveLength(1);
const gap = report.gaps[0]!;
expect(gap.terms).toEqual(["backups", "schedule", "vault"]);
expect(gap.occurrences).toBe(3);
expect(gap.meanCoverage).toBeCloseTo(0.05, 5);
expect(gap.emptyCount).toBe(3);
expect(gap.emptyRate).toBe(1);
expect(gap.coverageKnown).toBe(true);
expect(gap.verdict).toBe("sparse");
expect(gap.demandScore).toBeCloseTo(3 * (1 - 0.05), 4);
expect(gap.firstSeen).toBe("2026-07-01T00:00:00.000Z");
expect(gap.lastSeen).toBe("2026-07-03T00:00:00.000Z");
});
test("falls back to result-presence when no coverage was recorded", () => {
recordQueryDemand(vault, { query: "cron secrets", resultCount: 0, at: "2026-07-01T00:00:00Z" });
recordQueryDemand(vault, { query: "cron secrets", resultCount: 0, at: "2026-07-02T00:00:00Z" });
const report = aggregateQueryDemand(vault);
expect(report.gaps).toHaveLength(1);
const gap = report.gaps[0]!;
expect(gap.coverageKnown).toBe(false);
expect(gap.meanCoverage).toBeNull();
expect(gap.satisfaction).toBe(0);
expect(gap.demandScore).toBe(2);
});
test("honors minOccurrences, maxSatisfaction, and limit", () => {
for (let i = 0; i < 4; i++) {
recordQueryDemand(vault, {
query: "graph traversal depth",
resultCount: 2,
coverage: 0.5,
at: `2026-07-0${i + 1}T00:00:00.000Z`,
});
}
// Excluded when maxSatisfaction is stricter than 0.5.
expect(aggregateQueryDemand(vault, { maxSatisfaction: 0.4 }).gaps).toHaveLength(0);
// Included at the default 0.8 ceiling.
expect(aggregateQueryDemand(vault, { maxSatisfaction: 0.8 }).gaps).toHaveLength(1);
// Excluded when minOccurrences exceeds the count.
expect(aggregateQueryDemand(vault, { minOccurrences: 5 }).gaps).toHaveLength(0);
for (let i = 0; i < 3; i++) {
recordQueryDemand(vault, {
query: "another weak query",
resultCount: 0,
coverage: 0.0,
at: `2026-07-0${i + 1}T03:00:00.000Z`,
});
}
const limited = aggregateQueryDemand(vault, { limit: 1 });
expect(limited.gaps).toHaveLength(1);
// The zero-coverage query outranks the 0.5-coverage one.
expect(limited.gaps[0]!.terms).toEqual(["another", "query", "weak"]);
});
});
describe("rolling cap", () => {
test("compacts the log once it exceeds the byte budget", () => {
const path = queryDemandLogPath(vault);
// Reach the 1MB cap with FAT records rather than many small ones: the
// branch under test is byte-budget-driven, so a record built out to the
// documented per-record maximum (DEMAND_MAX_TERMS_PER_RECORD terms of
// DEMAND_TERM_MAX_LEN chars) crosses the same threshold in ~1k appends
// instead of ~20k. Each append takes a lockfile, so append count — not
// byte count — was the runtime, and 20k of them ran ~4.7s against the
// 5s limit and flaked under parallel load.
const terms = Array.from({ length: DEMAND_MAX_TERMS_PER_RECORD }, (_, i) =>
// Pure-alpha so sanitizeTerms keeps it: a long letters+digits token is
// dropped as secret-shaped.
`compaction${String.fromCharCode(97 + i)}`.padEnd(DEMAND_TERM_MAX_LEN, "x"),
);
const bucketKey = [...terms].toSorted().join(" ");
const recordBytes = Buffer.byteLength(
JSON.stringify({ ts: "2026-07-01T00:00:00.000Z", terms, results: 0, coverage: 0.1 }) + "\n",
"utf8",
);
// Overshoot the cap so compaction is provably triggered, not borderline.
const appends = Math.ceil((DEMAND_LOG_MAX_BYTES / recordBytes) * 1.2);
for (let i = 0; i < appends; i++) {
recordQueryDemand(vault, {
terms,
resultCount: 0,
coverage: 0.1,
at: "2026-07-01T00:00:00.000Z",
});
}
const size = statSync(path).size;
expect(size).toBeLessThanOrEqual(DEMAND_LOG_MAX_BYTES);
// Compaction actually ran: fewer lines survive than were appended.
const records = readQueryDemand(vault);
expect(records.length).toBeGreaterThan(0);
expect(records.length).toBeLessThan(appends);
// Compaction keeps recent lines readable and parseable.
expect(records.every((r) => r.terms.join(" ") === bucketKey)).toBe(true);
});
});
describe("query-demand per-device shards (t_774dea61)", () => {
const INPUT = {
query: "how does the auth flow work",
at: "2026-07-01T00:00:00.000Z",
resultCount: 3,
};
test("two devices write two shard files and reads merge both", () => {
withDeviceId("a", () => recordQueryDemand(vault, INPUT));
withDeviceId("b", () => recordQueryDemand(vault, INPUT));
expect(existsSync(join(vault, "Brain", "log", "query-demand.a.jsonl"))).toBe(true);
expect(existsSync(join(vault, "Brain", "log", "query-demand.b.jsonl"))).toBe(true);
expect(existsSync(join(vault, "Brain", "log", "query-demand.jsonl"))).toBe(false);
const records = readQueryDemand(vault);
expect(records).toHaveLength(2);
});
test("the empty device id keeps the legacy un-sharded file", () => {
withDeviceId("", () => recordQueryDemand(vault, INPUT));
expect(existsSync(join(vault, "Brain", "log", "query-demand.jsonl"))).toBe(true);
expect(readQueryDemand(vault)).toHaveLength(1);
});
test("compaction after a write on one device caps only that device's shard", () => {
const logDir = join(vault, "Brain", "log");
mkdirSync(logDir, { recursive: true });
const line =
JSON.stringify({ ts: INPUT.at, terms: ["auth", "flow"], results: 3, coverage: 0.5 }) + "\n";
// Both shards already sit over the byte budget. Only the shard the
// write lands on may be rewritten; the peer's file is not this
// device's to touch, however large it is.
const overBudget = line.repeat(Math.ceil(DEMAND_LOG_MAX_BYTES / line.length) + 1);
const ownShard = join(logDir, "query-demand.a.jsonl");
const peerShard = join(logDir, "query-demand.b.jsonl");
writeFileSync(ownShard, overBudget, "utf8");
writeFileSync(peerShard, overBudget, "utf8");
withDeviceId("a", () => {
expect(queryDemandLogPath(vault)).toBe(ownShard);
recordQueryDemand(vault, INPUT);
});
expect(statSync(ownShard).size).toBeLessThanOrEqual(DEMAND_LOG_MAX_BYTES);
expect(readFileSync(peerShard, "utf8")).toBe(overBudget);
});
});