Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Document-dedup detector (storage lifecycle, Lane 4 task 4.1).
*
* Ingest summary pages (kind `brain-source`) that carry near-identical
* bodies are retirement candidates: one canonical page should survive and
* the stale copies should be retired with their records. The detector
* only nominates groups; retirement happens through the hygiene apply
* plan (action `retire`), which routes to the retirement core.
*
* Decisions pinned here:
* - canonical is the NEWEST `created_at` (keep-newest);
* - a group holding ANY member without `created_at` is ineligible -
* the mtime fallback is merge-unstable (GitHub #180), so such a
* group is reported for review, never for retirement;
* - merge-resolved members leave the pool (the shared predicate);
* - scope keys keep folding: identical text in two scopes never groups.
*/
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { EmbeddingProvider } from "../../../src/core/search/embeddings/contract.ts";
import { DEFAULT_SCAN_IDS, HYGIENE_DETECTOR_IDS } from "../../../src/core/brain/hygiene/types.ts";
import { applyHygienePlan } from "../../../src/core/brain/hygiene/apply.ts";
import { buildHygienePlan } from "../../../src/core/brain/hygiene/plan.ts";
import { runHygieneScan } from "../../../src/core/brain/hygiene/scan.ts";
import {
DOCUMENT_DEDUP_LEXICAL_THRESHOLD,
detectDocumentDedup,
detectSemanticDocumentDedup,
} from "../../../src/core/brain/hygiene/detectors/document-dedup.ts";
import { hygieneFindingId } from "../../../src/core/brain/hygiene/detectors/id.ts";
let vault: string;
beforeEach(() => {
vault = mkdtempSync(join(tmpdir(), "o2b-document-dedup-"));
mkdirSync(join(vault, "Brain", "sources"), { recursive: true });
});
afterEach(() => {
rmSync(vault, { recursive: true, force: true });
});
const NEAR_A =
"Collect the latency metrics before optimizing the render loop and archive the traces weekly";
const NEAR_B =
"Collect the latency metrics before optimizing the render loop and archive the traces daily";
const UNRELATED = "Use tabs inside makefiles and keep the tab width at two spaces";
interface SourcePageOptions {
readonly body: string;
readonly createdAt?: string;
readonly sourcePath?: string;
readonly mergedInto?: string;
readonly kind?: string;
readonly project?: string;
}
/** Write one ingest summary page; returns its vault-relative path. */
function writeSourcePage(slug: string, opts: SourcePageOptions): string {
const lines = [
"---",
`kind: ${opts.kind ?? "brain-source"}`,
`source_path: ${opts.sourcePath ?? `imports/${slug}.md`}`,
`source_hash: ${"a".repeat(64)}`,
];
if (opts.createdAt !== undefined) lines.push(`created_at: ${opts.createdAt}`);
if (opts.mergedInto !== undefined) lines.push(`merged_into: ${opts.mergedInto}`);
if (opts.project !== undefined) lines.push(`project: ${opts.project}`);
lines.push("tags: [brain, brain/source]", "---", "", opts.body, "");
const abs = join(vault, "Brain", "sources", `src-${slug}.md`);
writeFileSync(abs, lines.join("\n"), "utf8");
return `Brain/sources/src-${slug}.md`;
}
function detect(): ReturnType<typeof detectDocumentDedup> {
return detectDocumentDedup(vault, { now: new Date(0) });
}
describe("detectDocumentDedup", () => {
test("groups near-duplicate summary pages and keeps the newest created_at canonical", () => {
const older = writeSourcePage("older", { body: NEAR_A, createdAt: "2026-01-01T00:00:00Z" });
const newer = writeSourcePage("newer", { body: NEAR_B, createdAt: "2026-02-01T00:00:00Z" });
writeSourcePage("unrelated", { body: UNRELATED, createdAt: "2026-03-01T00:00:00Z" });
const findings = detect();
expect(findings).toHaveLength(1);
const finding = findings[0]!;
expect(finding.detector).toBe("document-dedup");
expect(finding.severity).toBe("warning");
expect(finding.proposed_action).toBe("retire");
expect(finding.targets).toEqual([older, newer].toSorted());
expect(finding.id).toBe(hygieneFindingId("document-dedup", [older, newer]));
expect(finding.evidence["method"]).toBe("lexical");
expect(finding.evidence["threshold"]).toBe(DOCUMENT_DEDUP_LEXICAL_THRESHOLD);
expect(finding.evidence["canonical"]).toBe(newer);
});
test("a created_at tie breaks on the id ascending", () => {
const alpha = writeSourcePage("alpha", { body: NEAR_A, createdAt: "2026-01-01T00:00:00Z" });
const beta = writeSourcePage("beta", { body: NEAR_B, createdAt: "2026-01-01T00:00:00Z" });
const findings = detect();
expect(findings).toHaveLength(1);
expect(findings[0]!.evidence["canonical"]).toBe([alpha, beta].toSorted()[0]);
});
test("a group holding any member without created_at is ineligible for retirement", () => {
const stamped = writeSourcePage("stamped", {
body: NEAR_A,
createdAt: "2026-01-01T00:00:00Z",
});
const unstamped = writeSourcePage("unstamped", { body: NEAR_B });
const findings = detect();
expect(findings).toHaveLength(1);
const finding = findings[0]!;
expect(finding.proposed_action).toBe("review");
expect(finding.evidence["ineligible"]).toBe(true);
expect(String(finding.evidence["reason"])).toContain("created_at");
expect(finding.evidence["members_without_created_at"]).toEqual([unstamped]);
expect(finding.evidence["canonical"]).toBeUndefined();
expect(finding.targets).toEqual([stamped, unstamped].toSorted());
});
test("a member a merge already resolved leaves the pool", () => {
writeSourcePage("live", { body: NEAR_A, createdAt: "2026-01-01T00:00:00Z" });
writeSourcePage("resolved", {
body: NEAR_B,
createdAt: "2026-02-01T00:00:00Z",
mergedInto: "pref-elsewhere",
});
expect(detect()).toHaveLength(0);
});
test("identical text in two scopes never groups", () => {
writeSourcePage("alpha", {
body: NEAR_A,
createdAt: "2026-01-01T00:00:00Z",
project: "alpha",
});
writeSourcePage("beta", {
body: NEAR_B,
createdAt: "2026-01-02T00:00:00Z",
project: "beta",
});
expect(detect()).toHaveLength(0);
});
test("a pair below the lexical threshold is not nominated", () => {
writeSourcePage("one", {
body: "Collect the latency metrics before optimizing the render loop",
createdAt: "2026-01-01T00:00:00Z",
});
writeSourcePage("two", {
body: "Collect the garbage after every render pass and keep the loop quiet",
createdAt: "2026-01-02T00:00:00Z",
});
expect(detect()).toHaveLength(0);
});
test("pages that are not ingest summaries are ignored", () => {
writeSourcePage("pref", {
body: NEAR_A,
createdAt: "2026-01-01T00:00:00Z",
kind: "brain-preference",
});
writeSourcePage("other", {
body: NEAR_B,
createdAt: "2026-01-02T00:00:00Z",
kind: "brain-preference",
});
expect(detect()).toHaveLength(0);
});
test("pages too short to compare safely are skipped", () => {
writeSourcePage("tiny-a", { body: "Use tabs", createdAt: "2026-01-01T00:00:00Z" });
writeSourcePage("tiny-b", { body: "Use tabs", createdAt: "2026-01-02T00:00:00Z" });
expect(detect()).toHaveLength(0);
});
test("a reached candidate cap emits the coverage notice with scanned and total", () => {
// The review M9 shape: a long-lived vault whose duplicates sort AFTER
// the cap. 200 filler pages keep the first comparison slots (each with
// a distinct token payload, so no filler pairs with another), and the
// near-duplicate pair sorts last, so the cap never reads it. The scan
// must say its coverage was bounded instead of answering clean.
for (let i = 0; i < 200; i++) {
const tag = `p${String(i).padStart(4, "0")}`;
writeSourcePage(`filler-${String(i).padStart(4, "0")}`, {
body: `${UNRELATED}\n${tag}a ${tag}b ${tag}c ${tag}d ${tag}e ${tag}f ${tag}g ${tag}h`,
createdAt: "2026-01-01T00:00:00Z",
});
}
writeSourcePage("zz-dup-a", { body: NEAR_A, createdAt: "2026-01-01T00:00:00Z" });
writeSourcePage("zz-dup-b", { body: NEAR_B, createdAt: "2026-01-02T00:00:00Z" });
const findings = detect();
// The pair behind the cap was never read, so no retirement is claimed
// for it - that is the bound working. The truncation itself is named.
expect(findings.some((f) => f.proposed_action === "retire")).toBe(false);
const notice = findings.find(
(f) => (f.evidence as Record<string, unknown>)["candidate_cap_reached"] === true,
);
expect(notice).toBeDefined();
expect(notice!.proposed_action).toBe("review");
const evidence = notice!.evidence as Record<string, unknown>;
expect(evidence["scanned"]).toBe(200);
expect(evidence["total"]).toBe(202);
expect(evidence["cap"]).toBe(200);
// `listed` counts every readable .md under the reach - non-summary
// kinds and unparseable frontmatter included - so the row's only
// sentence about coverage must not call the skipped total
// summary-page candidates.
expect(notice!.title).toContain(`of 202 listed pages`);
expect(notice!.title).not.toContain("summary-page candidates");
});
test("a merge-resolved page inside the capped read set does not hide the truncation", () => {
// The M9 corner: the cap bounds what was COLLECTED, and the
// merge-resolved filter thins the kept set afterwards. One
// merged_into stamp among exactly the capped 200 leaves 199 records,
// and a notice gated on the post-filter count reads 199 < cap and
// goes quiet - while pages 201+ were never read. A truncated scan
// must never answer clean, so the guard asks the pre-filter count.
for (let i = 0; i < 200; i++) {
const tag = `p${String(i).padStart(4, "0")}`;
writeSourcePage(`filler-${String(i).padStart(4, "0")}`, {
body: `${UNRELATED}\n${tag}a ${tag}b ${tag}c ${tag}d ${tag}e ${tag}f ${tag}g ${tag}h`,
createdAt: "2026-01-01T00:00:00Z",
// Exactly one page the read set collected was already resolved
// by a page merge - ordinary in a vault that uses the merge lane.
...(i === 100 ? { mergedInto: "pref-elsewhere" } : {}),
});
}
writeSourcePage("zz-dup-a", { body: NEAR_A, createdAt: "2026-01-01T00:00:00Z" });
writeSourcePage("zz-dup-b", { body: NEAR_B, createdAt: "2026-01-02T00:00:00Z" });
const findings = detect();
// The pair behind the cap was never read, so no retirement is
// claimed for it - and the truncation is named even though the
// merge-resolved filter thinned the kept set below the cap.
expect(findings.some((f) => f.proposed_action === "retire")).toBe(false);
const notice = findings.find(
(f) => (f.evidence as Record<string, unknown>)["candidate_cap_reached"] === true,
);
expect(notice).toBeDefined();
const evidence = notice!.evidence as Record<string, unknown>;
// `scanned` counts what the cap admitted - the pre-filter read set -
// not the thinner kept set the merge filter left behind.
expect(evidence["scanned"]).toBe(200);
expect(evidence["total"]).toBe(202);
});
test("no coverage notice while the cap does not bind", () => {
writeSourcePage("a", { body: NEAR_A, createdAt: "2026-01-01T00:00:00Z" });
writeSourcePage("b", { body: NEAR_B, createdAt: "2026-01-02T00:00:00Z" });
const findings = detect();
expect(
findings.some((f) => (f.evidence as Record<string, unknown>)["candidate_cap_reached"]),
).toBe(false);
});
test("a member the caller cannot read is treated as absent", () => {
writeSourcePage("visible", { body: NEAR_A, createdAt: "2026-01-01T00:00:00Z" });
const hidden = writeSourcePage("hidden", { body: NEAR_B, createdAt: "2026-01-02T00:00:00Z" });
const findings = detectDocumentDedup(vault, {
now: new Date(0),
readable: (rel) => rel !== hidden,
});
expect(findings).toHaveLength(0);
});
test("findings come out in a deterministic order across runs", () => {
writeSourcePage("r-one", { body: NEAR_A, createdAt: "2026-01-01T00:00:00Z" });
writeSourcePage("r-two", { body: NEAR_B, createdAt: "2026-01-02T00:00:00Z" });
writeSourcePage("r-three", {
body: "Watch the commit queue and rebase the patch series before review",
createdAt: "2026-01-03T00:00:00Z",
});
writeSourcePage("r-four", {
body: "Watch the commit queue and rebase the patch series before merge",
createdAt: "2026-01-04T00:00:00Z",
});
const first = detect().map((f) => f.id);
const second = detect().map((f) => f.id);
expect(first).toEqual(second);
expect(first).toHaveLength(2);
});
// Token sets of ten with a one-word slide: A~B and B~C clear the
// lexical threshold while A~C (two slides apart) sits under it.
const CHAIN_A = Array.from({ length: 10 }, (_, i) => `kw${String(i + 1).padStart(2, "0")}`).join(
" ",
);
const CHAIN_B = Array.from({ length: 10 }, (_, i) => `kw${String(i + 2).padStart(2, "0")}`).join(
" ",
);
const CHAIN_C = Array.from({ length: 10 }, (_, i) => `kw${String(i + 3).padStart(2, "0")}`).join(
" ",
);
/**
* The A-B-C chain: (A,B) and (B,C) pair above the threshold, (A,C)
* does not, so the union-find still forms ONE component. Keep-newest
* picks C - which is NOT directly similar to A - and the transitive
* gate decides what may retire.
*/
function seedChain(): { a: string; b: string; c: string } {
const a = writeSourcePage("chain-a", { body: CHAIN_A, createdAt: "2026-01-01T00:00:00Z" });
const b = writeSourcePage("chain-b", { body: CHAIN_B, createdAt: "2026-01-02T00:00:00Z" });
const c = writeSourcePage("chain-c", { body: CHAIN_C, createdAt: "2026-01-03T00:00:00Z" });
return { a, b, c };
}
test("a member with no direct pair to the canonical is never a retire target", () => {
seedChain();
const findings = detect();
expect(findings).toHaveLength(2);
const retire = findings.find((f) => f.proposed_action === "retire")!;
const review = findings.find((f) => f.proposed_action === "review")!;
// Only the copy DIRECTLY similar to the canonical C is retirable.
const { b, c } = { b: "Brain/sources/src-chain-b.md", c: "Brain/sources/src-chain-c.md" };
expect(retire.targets).toEqual([b, c].toSorted());
expect(retire.evidence["canonical"]).toBe(c);
// The transitive member lands in review with the reason in evidence.
expect(review.targets).toEqual(["Brain/sources/src-chain-a.md", c].toSorted());
expect(review.evidence["ineligible"]).toBe(true);
expect(String(review.evidence["reason"])).toContain("transitive");
expect(review.evidence["members_without_direct_pair"]).toEqual([
"Brain/sources/src-chain-a.md",
]);
});
test("every member with a direct pair still retires through the pipeline", () => {
const { a, b } = seedChain();
const report = runHygieneScan(vault, { detectors: ["document-dedup"], now: new Date(0) });
const retire = report.findings.find((f) => f.proposed_action === "retire")!;
const plan = buildHygienePlan(report, { ids: [retire.id] });
expect(plan.selected).toHaveLength(1);
expect(plan.excluded_review).toEqual([]);
// The finding is derelived by the applier from the same evidence
// shape a scan produced: applying it must remove ONLY the copy
// directly similar to the canonical (B) and leave the transitive
// member (A) on disk.
return applyHygienePlan(vault, plan, { agent: "tester", now: new Date(0) }).then((result) => {
expect(result.errors).toEqual([]);
expect(result.applied).toHaveLength(1);
expect(result.applied[0]!.detail).toContain("retired 1 near-duplicate cop");
expect(existsSync(join(vault, a))).toBe(true);
expect(existsSync(join(vault, b))).toBe(false);
});
});
});
describe("document-dedup registration", () => {
test("the detector is registered and opt-in", () => {
expect(HYGIENE_DETECTOR_IDS).toContain("document-dedup");
expect(DEFAULT_SCAN_IDS).not.toContain("document-dedup");
});
test("the default sweep does not run it; an explicit subset does", () => {
writeSourcePage("scan-a", { body: NEAR_A, createdAt: "2026-01-01T00:00:00Z" });
writeSourcePage("scan-b", { body: NEAR_B, createdAt: "2026-01-02T00:00:00Z" });
const defaultSweep = runHygieneScan(vault, { now: new Date(0) });
expect(defaultSweep.counts["document-dedup"]).toBeUndefined();
const explicit = runHygieneScan(vault, {
detectors: ["document-dedup"],
now: new Date(0),
});
expect(explicit.errors).toHaveLength(0);
expect(explicit.counts["document-dedup"]).toBe(1);
expect(explicit.findings[0]!.proposed_action).toBe("retire");
});
});
describe("detectSemanticDocumentDedup", () => {
function fixedVectorProvider(vectors: Readonly<Record<string, number[]>>): EmbeddingProvider {
return {
name: "fixed",
model: "fixed-test",
dimension: 3,
embed: (texts) => Promise.resolve(texts.map((text) => vectors[text] ?? [0, 0, 1])),
ping: () => Promise.resolve({ ok: true as const, dimension: 3 }),
};
}
test("groups by embedding cosine at the embedding threshold", async () => {
const older = writeSourcePage("emb-old", { body: NEAR_A, createdAt: "2026-01-01T00:00:00Z" });
const newer = writeSourcePage("emb-new", { body: NEAR_B, createdAt: "2026-02-01T00:00:00Z" });
const provider = fixedVectorProvider({
[NEAR_A]: [1, 0, 0],
[NEAR_B]: [0.999, 0.0447, 0],
[UNRELATED]: [0, 1, 0],
});
const result = await detectSemanticDocumentDedup(vault, { provider });
expect(result.method).toBe("embedding");
expect(result.findings).toHaveLength(1);
const finding = result.findings[0]!;
expect(finding.targets).toEqual([older, newer].toSorted());
expect(finding.proposed_action).toBe("retire");
expect(finding.evidence["method"]).toBe("embedding");
expect(finding.evidence["model"]).toBe("fixed-test");
expect(finding.evidence["canonical"]).toBe(newer);
});
test("falls back to the lexical layer, labeled, when the provider fails", async () => {
writeSourcePage("lex-a", { body: NEAR_A, createdAt: "2026-01-01T00:00:00Z" });
writeSourcePage("lex-b", { body: NEAR_B, createdAt: "2026-01-02T00:00:00Z" });
const broken: EmbeddingProvider = {
name: "broken",
model: "broken",
dimension: 3,
embed: () => Promise.reject(new Error("provider down")),
ping: () => Promise.resolve({ ok: false as const, reason: "down" }),
};
const result = await detectSemanticDocumentDedup(vault, { provider: broken });
expect(result.method).toBe("lexical");
expect(result.findings).toHaveLength(1);
expect(result.findings[0]!.evidence["method"]).toBe("lexical");
});
test("a lexical fallback over a capped scan names the cap once", async () => {
// The lexical layer already carries its coverage notice; wrapping its
// result again added a second row with the same id.
for (let i = 0; i < 201; i++) {
const tag = `p${String(i).padStart(4, "0")}`;
writeSourcePage(`filler-${String(i).padStart(4, "0")}`, {
body: `${UNRELATED}\n${tag}a ${tag}b ${tag}c ${tag}d ${tag}e ${tag}f ${tag}g ${tag}h`,
createdAt: "2026-01-01T00:00:00Z",
});
}
const broken: EmbeddingProvider = {
name: "broken",
model: "broken",
dimension: 3,
embed: () => Promise.reject(new Error("provider down")),
ping: () => Promise.resolve({ ok: false as const, reason: "down" }),
};
const result = await detectSemanticDocumentDedup(vault, { provider: broken });
expect(result.method).toBe("lexical");
const notices = result.findings.filter(
(f) => (f.evidence as Record<string, unknown>)["candidate_cap_reached"] === true,
);
expect(notices).toHaveLength(1);
});
test("no usable provider means the lexical layer", async () => {
const result = await detectSemanticDocumentDedup(vault, { provider: null });
expect(result.method).toBe("lexical");
expect(result.findings).toHaveLength(0);
});
test("a page the caller cannot read is scored and named by no semantic pair", async () => {
writeSourcePage("emb-vis-1", { body: NEAR_A, createdAt: "2026-01-01T00:00:00Z" });
writeSourcePage("emb-vis-2", { body: NEAR_B, createdAt: "2026-01-02T00:00:00Z" });
const hidden = writeSourcePage("emb-hidden", {
body: NEAR_A,
createdAt: "2026-01-03T00:00:00Z",
});
const provider = fixedVectorProvider({
[NEAR_A]: [1, 0, 0],
[NEAR_B]: [0.999, 0.0447, 0],
});
const result = await detectSemanticDocumentDedup(vault, {
provider,
readable: (rel) => rel !== hidden,
});
expect(result.method).toBe("embedding");
expect(result.findings).toHaveLength(1);
expect(JSON.stringify(result.findings)).not.toContain(hidden);
});
test("records across different scope keys never pair, semantically either", async () => {
writeSourcePage("scope-alpha", {
body: NEAR_A,
createdAt: "2026-01-01T00:00:00Z",
project: "alpha",
});
writeSourcePage("scope-beta", {
body: NEAR_B,
createdAt: "2026-01-02T00:00:00Z",
project: "beta",
});
const provider = fixedVectorProvider({
[NEAR_A]: [1, 0, 0],
[NEAR_B]: [1, 0, 0],
});
const result = await detectSemanticDocumentDedup(vault, { provider });
expect(result.method).toBe("embedding");
expect(result.findings).toHaveLength(0);
});
test("a provider returning inconsistent vector dimensions falls back, labeled", async () => {
writeSourcePage("rag-a", { body: NEAR_A, createdAt: "2026-01-01T00:00:00Z" });
writeSourcePage("rag-b", { body: NEAR_B, createdAt: "2026-01-02T00:00:00Z" });
// Dimension mismatch inside one embed call: the vectors-length check
// models this for a count mismatch; a shape mismatch is the same
// broken-provider answer.
const ragged: EmbeddingProvider = {
name: "ragged",
model: "ragged",
dimension: 3,
embed: (texts) => Promise.resolve(texts.map((_, i) => (i === 0 ? [1, 0] : [1, 0, 0]))),
ping: () => Promise.resolve({ ok: true as const, dimension: 3 }),
};
const result = await detectSemanticDocumentDedup(vault, { provider: ragged });
expect(result.method).toBe("lexical");
// The fallback still finds the lexical pair - it just never calls
// the fabricated cosine score semantic.
expect(result.findings).toHaveLength(1);
expect(result.findings[0]!.evidence["method"]).toBe("lexical");
});
});