Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
14.0 KB
Raw Blame
/**
 * A1 (t_11d51443): the store-learned entity lexicon.
 *
 * The lexicon is a derived artifact (Brain/entities/lexicon.json,
 * `o2b.entity-lexicon.v1`) folded from a corpus walk over the vault's
 * operator pages - Brain root excluded, so session transcripts and every
 * other Brain-resident page are excluded with it. Nothing is hardcoded:
 * the only vocabulary is what the store itself reads. The suite pins the
 * fold's scope and case buckets, the S2-helper artifact round trip with
 * dataset-hash fail-open (mirroring the tuning-store family), the top-K
 * cap, and the pinned `EntityLexiconHandle` resolver.
 */

import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";

import { bootstrapBrain } from "../../../src/core/brain/init.ts";
import { resetVaultIdentityPins } from "../../../src/core/brain/vault-identity.ts";
import { atomicWriteFileSync } from "../../../src/core/fs-atomic.ts";
import { validateEntityLabel } from "../../../src/core/brain/entities/canonical.ts";
import {
  ENTITY_LEXICON_ARTIFACT_REL,
  ENTITY_LEXICON_CORPUS_SCOPE,
  ENTITY_LEXICON_SCHEMA_VERSION,
  ENTITY_LEXICON_TOP_K,
  __clearEntityLexiconCache,
  deriveEntityLexicon,
  deriveEntityLexiconFromPaths,
  entityLexiconDatasetHash,
  loadEntityLexicon,
  resolveEntityLexicon,
  writeEntityLexicon,
} from "../../../src/core/brain/entities/lexicon.ts";

let vault: string;
let configHome: string;

const NOW = new Date("2026-10-11T12:00:00Z");

beforeEach(() => {
  // Resolve-memo and identity pins are keyed by resolved root; tests run
  // many vaults in one process, so every test starts unpinned and clear.
  resetVaultIdentityPins();
  __clearEntityLexiconCache();
  vault = mkdtempSync(join(tmpdir(), "o2b-entity-lexicon-"));
  configHome = mkdtempSync(join(tmpdir(), "o2b-entity-lexicon-cfg-"));
  const configPath = join(configHome, "config.yaml");
  atomicWriteFileSync(configPath, `vault: ${vault}\n`);
  bootstrapBrain(vault, { configPath });
});

afterEach(() => {
  rmSync(vault, { recursive: true, force: true });
  rmSync(configHome, { recursive: true, force: true });
  __clearEntityLexiconCache();
});

/** Write an operator corpus page (outside Brain) with the given body. */
function writeCorpusPage(rel: string, body: string): void {
  const path = join(vault, rel);
  mkdirSync(dirname(path), { recursive: true });
  atomicWriteFileSync(path, `${body}\n`);
}

/** Derive the lexicon from the current corpus and persist it. */
function rebuild(): ReturnType<typeof deriveEntityLexicon> {
  const doc = deriveEntityLexicon(vault, { now: NOW });
  writeEntityLexicon(vault, doc);
  return doc;
}

describe("deriveEntityLexicon corpus scope", () => {
  test("folds tokens from operator pages and never from Brain-resident pages", () => {
    writeCorpusPage("alpha.md", "the runner kept notes about the setup. the notes were quick.");
    // A session-transcript-shaped page: it lives under the Brain root,
    // which the walk excludes whole, so none of its tokens may surface.
    writeCorpusPage(
      join("Brain", "sessions", "transcript-2026-10-11.md"),
      "Zebra QUOKKA herelocation transcript words appear here only.",
    );
    const doc = deriveEntityLexicon(vault, { now: NOW });
    const tokens = Object.keys(doc.entries);
    expect(tokens).toContain("notes");
    expect(tokens).toContain("runner");
    expect(tokens).toContain("quick");
    expect(tokens).not.toContain("zebra");
    expect(tokens).not.toContain("quokka");
    expect(tokens).not.toContain("transcript");
  });

  test("the artifact header asserts the walked scope", () => {
    writeCorpusPage("alpha.md", "the runner kept notes.");
    const doc = deriveEntityLexicon(vault, { now: NOW });
    expect(doc.schema).toBe(ENTITY_LEXICON_SCHEMA_VERSION);
    expect(ENTITY_LEXICON_SCHEMA_VERSION).toBe("o2b.entity-lexicon.v1");
    expect(doc.corpus_scope).toBe(ENTITY_LEXICON_CORPUS_SCOPE);
    // The header names the exclusion the scope relies on.
    expect(doc.corpus_scope).toContain("Brain");
    expect(doc.generated_at).toBe("2026-10-11T12:00:00Z");
  });
});

describe("deriveEntityLexicon case buckets", () => {
  test("counts lower, capitalised, and upper occurrences per lowercased token", () => {
    writeCorpusPage("atlas.md", "Atlas ran. atlas fell. ATLAS rose.");
    const doc = deriveEntityLexicon(vault, { now: NOW });
    expect(doc.entries["atlas"]).toEqual({ lower: 1, capitalised: 1, upper: 1 });
    expect(doc.caseful).toBe(true);
  });

  test("a caseless-script corpus resolves a caseless handle", () => {
    writeCorpusPage(join("notes", "han.md"), "项目 笔记 设置 关于 项目");
    const doc = rebuild();
    expect(doc.caseful).toBe(false);
    const handle = resolveEntityLexicon(vault);
    expect(handle).not.toBeNull();
    expect(handle!.caseful).toBe(false);
    // A caseless handle suppresses BOTH case-based verdicts: the label
    // gate degrades to the structural gate alone.
    expect(validateEntityLabel("MEETING NOTES", { lexicon: handle })).toEqual({ valid: true });
  });
});

describe("writeEntityLexicon / loadEntityLexicon", () => {
  test("round-trips through the S2 helper with schema and dataset hash", () => {
    writeCorpusPage("alpha.md", "the runner kept notes about the setup.");
    const doc = rebuild();
    const path = join(vault, ENTITY_LEXICON_ARTIFACT_REL);
    expect(ENTITY_LEXICON_ARTIFACT_REL).toBe(join("Brain", "entities", "lexicon.json"));
    expect(existsSync(path)).toBe(true);
    const raw = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>;
    expect(raw["schema"]).toBe("o2b.entity-lexicon.v1");
    expect(raw["generated_at"]).toBe("2026-10-11T12:00:00Z");
    expect(raw["dataset_hash"]).toBe(doc.dataset_hash);
    expect(raw["corpus_scope"]).toBe(ENTITY_LEXICON_CORPUS_SCOPE);

    // The artifact lives under the Brain root, so writing it does not
    // move the dataset it was derived from: the current hash still matches.
    const reloaded = loadEntityLexicon(vault, entityLexiconDatasetHash(vault));
    expect(reloaded).not.toBeNull();
    expect(reloaded!.entries).toEqual(doc.entries);
    expect(reloaded!.caseful).toBe(doc.caseful);
  });

  test("torn JSON reads as null and resolves as null (fail-open)", () => {
    writeCorpusPage("alpha.md", "the runner kept notes.");
    rebuild();
    const path = join(vault, ENTITY_LEXICON_ARTIFACT_REL);
    atomicWriteFileSync(path, "{not json");
    __clearEntityLexiconCache();
    expect(loadEntityLexicon(vault, entityLexiconDatasetHash(vault))).toBeNull();
    expect(resolveEntityLexicon(vault)).toBeNull();
  });

  test("a stale artifact (the corpus moved on) reads as null (fail-open)", () => {
    writeCorpusPage("alpha.md", "the runner kept notes.");
    rebuild();
    // The store grows after the rebuild: the stored derivation no longer
    // fingerprints the current input set, so it reads as absent.
    writeCorpusPage("beta.md", "fresh pages about the new setup arrived.");
    expect(loadEntityLexicon(vault, entityLexiconDatasetHash(vault))).toBeNull();
    // The resolver re-walks when the artifact changed or the cache cleared;
    // a cleared cache is how a fresh process meets the stale artifact.
    __clearEntityLexiconCache();
    expect(resolveEntityLexicon(vault)).toBeNull();
  });

  test("a missing artifact resolves as null: the gate degrades to the structural gate", () => {
    writeCorpusPage("alpha.md", "the runner kept notes.");
    expect(resolveEntityLexicon(vault)).toBeNull();
    expect(validateEntityLabel("MEETING NOTES")).toEqual({ valid: true });
  });
});

describe("top-K cap", () => {
  test("keeps the highest-total entries, ties broken lexicographically", () => {
    writeCorpusPage("alpha.md", "zebra zebra zebra yak yak quail vastus unicorn");
    const doc = deriveEntityLexicon(vault, { now: NOW, topK: 3 });
    expect(Object.keys(doc.entries)).toEqual(["zebra", "yak", "quail"]);
    // The default cap is the exported constant; a small corpus keeps
    // everything under it.
    expect(deriveEntityLexicon(vault, { now: NOW }).entries["vastus"]).toBeDefined();
    expect(ENTITY_LEXICON_TOP_K).toBeGreaterThan(0);
  });
});

describe("resolveEntityLexicon handle", () => {
  test("returns the pinned EntityLexiconHandle with learned commonness", () => {
    writeCorpusPage(
      "alpha.md",
      "notes about the setup. the notes were quick. Quantum was here. Quantum again.",
    );
    rebuild();
    const handle = resolveEntityLexicon(vault);
    expect(handle).not.toBeNull();
    expect(handle!.caseful).toBe(true);
    // Lowercase-heavy tokens are learned common words.
    expect(handle!.isCommonWord("notes")).toBe(true);
    expect(handle!.isCommonWord("about")).toBe(true);
    // Name-evidence-dominant tokens are not: "quantum" only ever
    // appeared capitalised.
    expect(handle!.isCommonWord("quantum")).toBe(false);
    // Unknown tokens are not common words either.
    expect(handle!.isCommonWord("unseentoken")).toBe(false);
    // The handle contract: tokens arrive LOWERCASED and the lookup is
    // exact - no re-normalisation inside the handle.
    expect(handle!.isCommonWord("Notes")).toBe(false);
  });

  test("a caseful handle lets the case-based verdicts fire end to end", () => {
    writeCorpusPage("alpha.md", "Atlas kept notes about the setup.");
    rebuild();
    const handle = resolveEntityLexicon(vault);
    expect(validateEntityLabel("MEETING NOTES", { lexicon: handle })).toEqual({
      valid: false,
      reason: "shouted-heading",
    });
  });
});

/**
 * Recall-quality review M5: the freshness signature must be a COARSE
 * corpus fingerprint - the page set plus per-page token-count buckets,
 * with no mtimes - so an edited vault keeps the gate resolving instead
 * of silently switching it off until a manual rebuild. Real corpus
 * shifts (pages added or removed, a token-count bucket crossed) still
 * expire the artifact honestly.
 */
describe("the corpus signature survives small edits (recall-quality M5)", () => {
  const BODY = "the runner kept notes about the setup. ";

  test("editing one page's body keeps the gate resolving", () => {
    writeCorpusPage("alpha.md", BODY.repeat(30));
    rebuild();
    expect(resolveEntityLexicon(vault)).not.toBeNull();
    // A small in-place edit: reword every sentence (the content changes,
    // the learned entries lag) without moving the page's token-count
    // bucket. A fresh process - the cleared cache - must still resolve.
    writeCorpusPage("alpha.md", "a runner kept notes about the setup. ".repeat(30));
    __clearEntityLexiconCache();
    expect(resolveEntityLexicon(vault)).not.toBeNull();
  });

  test("a small append that stays in the token-count bucket keeps it too", () => {
    writeCorpusPage("alpha.md", BODY.repeat(30));
    rebuild();
    writeCorpusPage("alpha.md", BODY.repeat(32));
    __clearEntityLexiconCache();
    expect(resolveEntityLexicon(vault)).not.toBeNull();
  });

  test("an edit that crosses the token-count bucket expires the artifact", () => {
    writeCorpusPage("alpha.md", BODY.repeat(30));
    rebuild();
    expect(resolveEntityLexicon(vault)).not.toBeNull();
    // Well past the next bucket boundary: the corpus genuinely shifted.
    writeCorpusPage("alpha.md", BODY.repeat(95));
    __clearEntityLexiconCache();
    expect(resolveEntityLexicon(vault)).toBeNull();
  });

  test("removing a page expires the artifact", () => {
    writeCorpusPage("alpha.md", BODY.repeat(30));
    writeCorpusPage("beta.md", "another page about the runner.");
    rebuild();
    expect(resolveEntityLexicon(vault)).not.toBeNull();
    rmSync(join(vault, "beta.md"));
    __clearEntityLexiconCache();
    expect(resolveEntityLexicon(vault)).toBeNull();
  });

  test("the derive and freshness signatures agree without stat calls", () => {
    writeCorpusPage("alpha.md", BODY);
    const doc = deriveEntityLexicon(vault, { now: NOW });
    expect(doc.dataset_hash).toBe(entityLexiconDatasetHash(vault));
  });
});

describe("deriveEntityLexicon mid-derive races", () => {
  test("a corpus file removed between enumeration and fold does not poison the artifact", () => {
    writeCorpusPage("keep.md", "atlas kept notes about the setup.");
    writeCorpusPage("vanish.md", "orphanword vanishes mid derive.");
    // The race, reproduced deterministically: enumerate the corpus the way
    // the derive does, then lose a file BEFORE the fold reads it.
    const enumerated = [join(vault, "keep.md"), join(vault, "vanish.md")];
    rmSync(join(vault, "vanish.md"));

    const doc = deriveEntityLexiconFromPaths(vault, enumerated, { now: NOW });

    // The fold reflects what was enumerated and still readable: the
    // vanished file's tokens are gone by absence, the survivor's remain -
    // skipped by name, never folded as a silent empty page while the hash
    // vouches for its bytes.
    expect(doc.entries["atlas"]).toBeDefined();
    expect(doc.entries["orphanword"]).toBeUndefined();

    // The hash fingerprints the enumerated set with the vanished file
    // skipped, so it agrees with the fingerprint a fresh walk computes now:
    // the artifact does not fail freshness on every read until re-derived.
    expect(doc.dataset_hash).toBe(entityLexiconDatasetHash(vault));

    // End to end: the persisted artifact validates at resolve time.
    writeEntityLexicon(vault, doc);
    expect(loadEntityLexicon(vault, entityLexiconDatasetHash(vault))).not.toBeNull();
    expect(resolveEntityLexicon(vault)).not.toBeNull();
  });

  test("deriving over a fresh enumeration is the single-pass derive", () => {
    writeCorpusPage("alpha.md", "the runner kept notes about the setup.");
    const doc = deriveEntityLexicon(vault, { now: NOW });
    // The one-walk derive and the explicit-paths derive over the same
    // corpus agree on everything but are driven by the same enumeration.
    const paths = [join(vault, "alpha.md")];
    const explicit = deriveEntityLexiconFromPaths(vault, paths, { now: NOW });
    expect(explicit.entries).toEqual(doc.entries);
    expect(explicit.dataset_hash).toBe(doc.dataset_hash);
    expect(explicit.caseful).toBe(doc.caseful);
  });
});