Skip to content

A self-organizing Obsidian vault that gives AI coding agents persistent memory.

perf: (memory) cache the parse, and stop reading whole files to see their head (#162)

Brenno Ferrariab560f76b666
diff --git a/.claude/scripts/lib/mcp-exposure.ts b/.claude/scripts/lib/mcp-exposure.ts
index d1f561c..5213d74 100644
--- a/.claude/scripts/lib/mcp-exposure.ts
+++ b/.claude/scripts/lib/mcp-exposure.ts
@@ -17,16 +17,15 @@
* defect this module is shaped to prevent.
*/
-import { readFileSync, readdirSync, statSync, lstatSync, existsSync, realpathSync } from "node:fs";
+import { readdirSync, statSync, lstatSync, existsSync, realpathSync } from "node:fs";
import { join, basename, resolve, sep } from "node:path";
import { vaultRelKey } from "./mcp-qmd-client.ts";
+import { readHead } from "./read-head.ts";
/** How deep to walk inside an exposed root. */
const MAX_DEPTH = 4;
-/** How much of a file to read when probing its frontmatter. */
-const HEAD_BYTES = 1200;
export interface ExposurePolicy {
/** Path prefixes whose notes are served. */
@@ -164,23 +163,17 @@ export function resolveExposure(
* private".
*/
export function isPrivate(path: string): boolean {
- try {
- const head = readFileSync(path, "utf8").slice(0, HEAD_BYTES);
- return /^\s*-?\s*private\s*$/m.test(head) || /^private:\s*true/m.test(head);
- } catch {
- return true;
- }
+ const head = readHead(path);
+ if (head === null) return true;
+ return /^\s*-?\s*private\s*$/m.test(head) || /^private:\s*true/m.test(head);
}
/** Pull the frontmatter `description:` so a resource list is self-describing. */
export function firstDescription(path: string, fallback = "Vault note"): string {
- try {
- const head = readFileSync(path, "utf8").slice(0, HEAD_BYTES);
- const m = head.match(/^description:\s*"?(.+?)"?\s*$/m);
- return m?.[1] ? m[1].slice(0, 200) : fallback;
- } catch {
- return fallback;
- }
+ const head = readHead(path);
+ if (head === null) return fallback;
+ const m = head.match(/^description:\s*"?(.+?)"?\s*$/m);
+ return m?.[1] ? m[1].slice(0, 200) : fallback;
}
/**
diff --git a/.claude/scripts/lib/mcp-memory-bridge.ts b/.claude/scripts/lib/mcp-memory-bridge.ts
index 7916a6b..2799084 100644
--- a/.claude/scripts/lib/mcp-memory-bridge.ts
+++ b/.claude/scripts/lib/mcp-memory-bridge.ts
@@ -16,11 +16,16 @@
import { readFileSync, readdirSync } from "node:fs";
import { join } from "node:path";
-import { listMemoryFiles, parseFrontmatter, facetsOf, type Facets } from "./memory-recall.ts";
+import {
+ readMemories,
+ agentMemories,
+ parseFrontmatter,
+ type Facets,
+ type MemoryEntry,
+} from "./memory-recall.ts";
import { qmdRelKey, vaultRelKey, subQueries, type QmdClient } from "./mcp-qmd-client.ts";
import type { VisibleFile } from "./mcp-exposure.ts";
-
-const HEAD_BYTES = 1200;
+import { readHead, HEAD_CHARS } from "./read-head.ts";
export interface MemoryDigest extends Facets {
readonly rel: string;
@@ -47,12 +52,8 @@ export function resolvableNames(files: readonly VisibleFile[]): Set<string> {
const names = new Set<string>();
for (const f of files) {
names.add(f.label.toLowerCase());
- let head: string;
- try {
- head = readFileSync(f.full, "utf8").slice(0, HEAD_BYTES);
- } catch {
- continue; // an unreadable file still contributes its basename
- }
+ const head = readHead(f.full, HEAD_CHARS);
+ if (head === null) continue; // an unreadable file still contributes its basename
const block = head.match(/^aliases:\s*$([\s\S]*?)^(?=\S|---)/m);
if (block?.[1]) {
for (const line of block[1].split("\n")) {
@@ -72,27 +73,25 @@ export function resolvableNames(files: readonly VisibleFile[]): Set<string> {
}
/** Load every agent-written memory with its facets, title and body. */
+export function digestsFrom(entries: readonly MemoryEntry[]): MemoryDigest[] {
+ return agentMemories(entries).map((m) => ({
+ rel: m.rel,
+ full: m.full,
+ title: m.title ?? "",
+ body: m.body,
+ ...m.facets,
+ }));
+}
+
+/**
+ * The same, read from disk. The IO wrapper.
+ *
+ * No production caller: the server reads through `memory-index`. This is the
+ * on-disk oracle those cache tests assert equality against, so a dead-code sweep
+ * that removes it takes the equivalence guarantee with it.
+ */
export function loadMemoryDigests(vaultRoot: string, memoryRoot: string): MemoryDigest[] {
- const out: MemoryDigest[] = [];
- for (const f of listMemoryFiles(vaultRoot, memoryRoot)) {
- let md: string;
- try {
- md = readFileSync(f.full, "utf8");
- } catch {
- continue;
- }
- const fm = parseFrontmatter(md);
- // Only agent-written memories participate. A human note that wandered in
- // here is left alone rather than silently governed by these rules.
- if (fm.source !== "mcp-capture") continue;
- const title = (md.match(/^#\s+(.+)$/m)?.[1] ?? "").trim();
- const body = md
- .replace(/^---[\s\S]*?\r?\n---\r?\n/, "")
- .replace(/^#\s+.*$/m, "")
- .trim();
- out.push({ rel: f.rel, full: f.full, title, body, ...facetsOf(fm) });
- }
- return out;
+ return digestsFrom(readMemories(vaultRoot, memoryRoot));
}
/**
@@ -154,8 +153,17 @@ export function findNoteNamed(vaultRoot: string, name: string, memoryRoot: strin
* the CALLER belongs to is a property of the caller, not of what the vault
* chooses to expose as readable content. So the search is vault-wide by
* FILENAME, exactly one frontmatter key is read from the single matching file,
- * no note body is opened, and nothing is returned to the caller except its own
+ * no note body is used, and nothing is returned to the caller except its own
* platform list.
+ *
+ * A THIRD version was nearly shipped and caught in review. Reading only the
+ * head — as the other frontmatter probes in this file do — looks like the same
+ * optimisation, but it is not: those replaced an equally-sized character slice,
+ * while this replaced a WHOLE-file read. `parseFrontmatter` needs the CLOSING
+ * `---`, so a project note whose frontmatter runs past the head (a long
+ * description, or aliases, both ordinary) parsed to `{}` and the caller got no
+ * platforms — presenting exactly like the two failures above. This reads the
+ * whole file on purpose: it is one file per call, next to a vault walk.
*/
export function callerPlatforms(vaultRoot: string, caller: string | null, memoryRoot: string): string[] {
if (!caller) return [];
diff --git a/.claude/scripts/lib/mcp-server.ts b/.claude/scripts/lib/mcp-server.ts
index 3fb6392..27bb792 100644
--- a/.claude/scripts/lib/mcp-server.ts
+++ b/.claude/scripts/lib/mcp-server.ts
@@ -28,11 +28,12 @@ import {
import { expandNote } from "./mcp-graph.ts";
import { qmdSearch, type QmdClient } from "./mcp-qmd-client.ts";
import { callerProject, callerProjectSource, isVaultItself, PROJECT_MARKER } from "./mcp-caller.ts";
-import { callerPlatforms, loadMemoryDigests, resolvableNames } from "./mcp-memory-bridge.ts";
+import { callerPlatforms, digestsFrom, resolvableNames } from "./mcp-memory-bridge.ts";
import { captureNote } from "./mcp-capture.ts";
import { semanticMemoryOrder } from "./mcp-memory-bridge.ts";
import { TOOLS } from "./mcp-tools.ts";
-import { recall, type MemoryEntry } from "./memory-recall.ts";
+import { recallFrom, readMemories, type MemoryEntry } from "./memory-recall.ts";
+import { createMemoryIndex } from "./memory-index.ts";
import { validateMemory, writeMemory, renderMemory, resolveLinks, neutralizeWikilinks } from "./memory-write.ts";
import { findSimilar } from "./memory-similarity.ts";
import { markSuperseded, resolveSupersedes } from "./memory-supersede.ts";
@@ -111,6 +112,34 @@ export function createHandlers(deps: ServerDeps): Handlers {
const { ctx, policy, session, qmd, audit } = deps;
const now = deps.now ?? (() => new Date());
const reindex = deps.reindex ?? (() => reindexSync(ctx.qmdIndex));
+ // One cache per server, living exactly as long as the process that owns it.
+ const memoryIndex = createMemoryIndex(ctx.vaultRoot, ctx.memoryRoot);
+
+ /**
+ * The store, parsed, for READ paths. Lists and stats every file on every
+ * call; only the re-parse of an unchanged file is skipped.
+ */
+ const storeEntries = (): MemoryEntry[] => memoryIndex.all();
+
+ /**
+ * The store, read fresh from disk. For the duplicate scan ONLY.
+ *
+ * The cache validates with size + mtime, which is not a content hash. Any
+ * writer that sets mtime explicitly — rsync, tar, unzip, a sync client, a
+ * restored file version — produces same-size, same-mtime, different-bytes;
+ * so does a second-granularity filesystem. Measured on NTFS, 356 of 400
+ * back-to-back same-size rewrites carried an identical mtimeMs.
+ *
+ * That is survivable on recall, where the cost is stale ordering. It is not
+ * survivable here: a duplicate admitted by a stale view is permanent, because
+ * nothing downstream ever re-checks. The explicit `invalidate()` after a write
+ * does not save this either — it is per-process, and the deployment shape is
+ * one server per consuming repo, all writing into one vault.
+ *
+ * So the filesystem stays authoritative for the correctness-critical read, at
+ * the price of one uncached pass per write.
+ */
+ const storeFresh = (): MemoryEntry[] => readMemories(ctx.vaultRoot, ctx.memoryRoot);
/** Who is asking, as the memory layer understands it. */
const caller = () => ({
@@ -145,9 +174,10 @@ export function createHandlers(deps: ServerDeps): Handlers {
const explain = args.explain === true;
const query = String(args.query ?? "").trim();
+ const entries = storeEntries();
const result = explain
- ? recall(ctx.vaultRoot, who, { root: ctx.memoryRoot, explain: true })
- : { visible: recall(ctx.vaultRoot, who, { root: ctx.memoryRoot }), withheld: [] as MemoryEntry[] };
+ ? recallFrom(entries, who, { explain: true })
+ : { visible: recallFrom(entries, who), withheld: [] as MemoryEntry[] };
let visible = result.visible;
@@ -256,7 +286,7 @@ export function createHandlers(deps: ServerDeps): Handlers {
return `Refused:\n${v.errors.map((e) => `- ${e}`).join("\n")}`;
}
- const digests = loadMemoryDigests(ctx.vaultRoot, ctx.memoryRoot);
+ const digests = digestsFrom(storeFresh());
// Near-duplicate suppression, facet-gated so two projects can each hold
// their own copy of the same lesson.
@@ -291,10 +321,18 @@ export function createHandlers(deps: ServerDeps): Handlers {
}
const written = writeMemory(ctx.vaultRoot, v.value, resolved, { root: ctx.memoryRoot });
+ // Explicit, rather than relying on the cache's size+mtime check to notice.
+ // A filesystem that keeps mtime to a whole second could otherwise serve the
+ // previous parse of a path a collision loop just reused.
+ memoryIndex.invalidate(written.rel);
const retired: string[] = [];
for (const m of supers.matched) {
- if (markSuperseded(ctx.vaultRoot, m.rel, v.value.title).ok) retired.push(m.title);
+ if (markSuperseded(ctx.vaultRoot, m.rel, v.value.title).ok) {
+ retired.push(m.title);
+ // Rewritten in place, and the new frontmatter is what makes it sink.
+ memoryIndex.invalidate(m.rel);
+ }
}
const indexed = reindex();
@@ -343,6 +381,11 @@ export function createHandlers(deps: ServerDeps): Handlers {
function callHealth(): string {
const who = caller();
+ // Populate the cache before reporting on it. `recall` tells the user to run
+ // health when a memory is missing, so this is routinely the FIRST call a
+ // server serves — and "3 memories / 0 entries held" reads as a parse
+ // failure when nothing is wrong.
+ const parsed = storeEntries().length;
const names = new Set(visibleFiles(ctx.vaultRoot, policy).map((f) => f.label));
if (who.project) names.add(who.project);
const h = health(ctx.vaultRoot, ctx.manifest, {
@@ -365,6 +408,10 @@ export function createHandlers(deps: ServerDeps): Handlers {
: []),
`Platforms: ${who.platforms.length ? who.platforms.join(", ") : "(none declared)"}`,
`Memory root: ${h.memory.root}/ (${h.memory.memories} memor${h.memory.memories === 1 ? "y" : "ies"}, ${h.memory.source})`,
+ // Reported because every failure in this layer otherwise presents as "no
+ // results". `stale` is the one that matters: files that exist but this
+ // server could not re-read, served from an older parse.
+ `Parsed store: ${parsed} entr${parsed === 1 ? "y" : "ies"}${memoryIndex.stats.stale ? ` (${memoryIndex.stats.stale} served from an older parse — check permissions)` : ""}`,
`Exposed roots: ${policy.roots.join(", ") || "(none)"} [${policy.source}]`,
`Search index: ${ctx.qmdIndex ?? "(qmd default)"} · launcher ${ctx.qmdLauncher ? "found" : "NOT FOUND"}`,
"",
diff --git a/.claude/scripts/lib/memory-discover.ts b/.claude/scripts/lib/memory-discover.ts
index 69470a4..3cbd625 100644
--- a/.claude/scripts/lib/memory-discover.ts
+++ b/.claude/scripts/lib/memory-discover.ts
@@ -22,11 +22,25 @@
import { readFileSync, readdirSync, statSync, existsSync } from "node:fs";
import { normalizePath } from "./mcp-caller.ts";
+import { readHead } from "./read-head.ts";
+import { MEMORY_SOURCE } from "./memory-write.ts";
import { join, relative, sep } from "node:path";
/** Directories never worth walking, in any vault. */
const SKIP = new Set([".git", ".obsidian", ".shardmind", "node_modules", ".claude", ".codex", ".gemini"]);
+/**
+ * Frontmatter that marks a file as agent-written, built from the one constant.
+ *
+ * Escaped because the value belongs to another module: a marker containing `.`
+ * or `|` would quietly widen what counts as agent-written, and this regex is
+ * what decides where the entire store is judged to live.
+ */
+const AGENT_WRITTEN = new RegExp(
+ `^source:\\s*["']?${MEMORY_SOURCE.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}["']?\\s*$`,
+ "m",
+);
+
/** How deep a memory tree can nest before we stop believing it is one. */
const MAX_DEPTH = 4;
@@ -108,15 +122,12 @@ export function walkMarkdown(root: string, { maxDepth = MAX_DEPTH }: { maxDepth?
return out;
}
-/** Cheap frontmatter probe — reads only the head of a file. */
-export function probeFrontmatter(full: string, bytes = 800): string | null {
- try {
- const head = readFileSync(full, "utf8").slice(0, bytes);
- const m = head.match(/^---\r?\n([\s\S]*?)\r?\n---/);
- return m ? (m[1] ?? null) : null;
- } catch {
- return null;
- }
+/** Cheap frontmatter probe — reads only the head of a file, not all of it. */
+export function probeFrontmatter(full: string, chars = 800): string | null {
+ const head = readHead(full, chars);
+ if (head === null) return null;
+ const m = head.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+ return m ? (m[1] ?? null) : null;
}
/**
@@ -137,7 +148,7 @@ export function discoverMemoryRoot(vaultRoot: string, configured = "memories"):
const counts = new Map<string, number>();
for (const full of walkMarkdown(vaultRoot)) {
const fm = probeFrontmatter(full);
- if (!fm || !/^source:\s*["']?mcp-capture["']?\s*$/m.test(fm)) continue;
+ if (!fm || !AGENT_WRITTEN.test(fm)) continue;
const rel = relative(vaultRoot, full).split(sep);
if (rel.length < 2) continue;
const top = rel[0];
diff --git a/.claude/scripts/lib/memory-index.ts b/.claude/scripts/lib/memory-index.ts
new file mode 100644
index 0000000..d33dada
--- /dev/null
+++ b/.claude/scripts/lib/memory-index.ts
@@ -0,0 +1,167 @@
+/**
+ * A parse cache over the memory store.
+ *
+ * `recall` and the duplicate scan read the SAME files and parse the same
+ * frontmatter, once per call each. That is linear in the store, and it runs on
+ * every recall and every write.
+ *
+ * WHAT IS CACHED, AND WHAT IS NOT
+ *
+ * The cache holds the PARSE, never the answer. Every call still lists the store
+ * and stats each file; only re-reading and re-parsing a file whose size and
+ * mtime are unchanged is skipped. A new or deleted memory is therefore always
+ * seen, and an ordinary edit is caught by one of the two.
+ *
+ * WHAT THIS IS NOT GOOD ENOUGH FOR
+ *
+ * `(size, mtime)` is a change hint, not a content hash, and the gap is wider
+ * than it looks: measured on NTFS, 356 of 400 back-to-back same-size rewrites
+ * carried an identical mtimeMs. Anything that sets mtime deliberately — rsync,
+ * tar, unzip, a sync client, a restored file version — collides on purpose, and
+ * a second-granularity filesystem collides by design. `invalidate()` closes the
+ * gap only WITHIN one process, while the deployment shape is one server per
+ * consuming repo, all writing into a single vault.
+ *
+ * So this serves reads, where the cost of being stale is ordering. The duplicate
+ * scan reads from disk instead (`storeFresh` in `mcp-server.ts`): a duplicate
+ * admitted there is permanent, because nothing downstream ever re-checks.
+ *
+ * WHY IT IS NOT ON DISK
+ *
+ * Measured, these walks are a minority of a queried recall — the local query
+ * embedding dominates — so persisting the index buys a fraction of one operation
+ * while adding a file that can rot, disagree with the store, or need migrating in
+ * every vault that ever installed the template. The server is long-lived per
+ * repo, so every call after the first is already warm, and a cold start costs
+ * what it always did.
+ */
+
+import { statSync } from "node:fs";
+
+import { listMemoryFiles, readMemoryFile, type MemoryEntry } from "./memory-recall.ts";
+
+interface CacheEntry {
+ readonly mtimeMs: number;
+ readonly size: number;
+ readonly record: MemoryEntry;
+}
+
+export interface IndexStats {
+ /** Files served from cache by the last `all()`. */
+ readonly hits: number;
+ /** Files read and parsed by the last `all()`. */
+ readonly misses: number;
+ /**
+ * Files that exist but could not be re-checked, served from the previous
+ * parse. Counted separately so `hits + misses + stale` always equals what
+ * `all()` returned — a store this server is half-failing to read is otherwise
+ * indistinguishable from a healthy one.
+ */
+ readonly stale: number;
+ /** Entries currently held. */
+ readonly size: number;
+}
+
+export interface MemoryIndex {
+ /**
+ * Every file in the store, parsed. Unchanged files come from cache.
+ *
+ * Returns agent-written memories and human notes alike — the caller decides,
+ * because "is this a memory" is a policy question and a cache that answers it
+ * would have to be rebuilt whenever the policy moved.
+ */
+ all(): MemoryEntry[];
+ /** Forget one file by vault-relative path. Called after writing it. */
+ invalidate(rel: string): void;
+ readonly stats: IndexStats;
+}
+
+/**
+ * The vault and root are fixed for the life of the index, not passed per call.
+ *
+ * Keys carry the root prefix, so calling with a different root would evict the
+ * whole cache on every alternation — a silent thrash with no error. Binding them
+ * at construction makes that impossible rather than merely unlikely.
+ */
+export function createMemoryIndex(vaultRoot: string, root: string): MemoryIndex {
+ let cache = new Map<string, CacheEntry>();
+ let hits = 0;
+ let misses = 0;
+ let stale = 0;
+
+ /**
+ * Synchronous end to end, and it must stay that way: an `invalidate` landing
+ * between the walk and the `cache = next` swap below would be undone by the
+ * swap. Adding an `await` here reintroduces exactly that race.
+ */
+ const all = (): MemoryEntry[] => {
+ const out: MemoryEntry[] = [];
+ // Built fresh each pass and swapped in at the end, so a file that vanished
+ // is evicted STRUCTURALLY rather than by a second sweep that has to be kept
+ // in step with this loop.
+ const next = new Map<string, CacheEntry>();
+ hits = 0;
+ misses = 0;
+ stale = 0;
+
+ for (const f of listMemoryFiles(vaultRoot, root)) {
+ const cached = cache.get(f.rel);
+ let st: { mtimeMs: number; size: number };
+ try {
+ st = statSync(f.full);
+ } catch (e) {
+ // ENOENT means it vanished between listing and stat, and dropping it
+ // is right. Any other errno — EACCES, EMFILE, a Windows sharing
+ // violation — means the file is still THERE and we merely could not
+ // look at it, so a previous parse is better than pretending the
+ // memory does not exist. That distinction matters most on the
+ // duplicate path, where an omission admits a duplicate permanently.
+ if ((e as NodeJS.ErrnoException).code !== "ENOENT" && cached) {
+ stale++;
+ next.set(f.rel, cached);
+ out.push(cached.record);
+ }
+ continue;
+ }
+
+ if (cached && cached.mtimeMs === st.mtimeMs && cached.size === st.size) {
+ hits++;
+ next.set(f.rel, cached);
+ out.push(cached.record);
+ continue;
+ }
+
+ misses++;
+ const record = readMemoryFile(f);
+ if (!record) {
+ // The file is listed and stats fine, so it EXISTS — it just could not
+ // be read this instant: a lock, an ACL, a sharing violation, a
+ // momentarily exhausted fd table. Those land here far more often than
+ // on the stat above, which needs only directory traversal. Keeping
+ // the previous parse beats reporting that the memory is gone, and
+ // matters most on the duplicate path where an omission is permanent.
+ if (cached) {
+ stale++;
+ next.set(f.rel, cached);
+ out.push(cached.record);
+ }
+ continue;
+ }
+ next.set(f.rel, { mtimeMs: st.mtimeMs, size: st.size, record });
+ out.push(record);
+ }
+
+ cache = next;
+ return out;
+ };
+
+ return {
+ all,
+ invalidate: (rel) => {
+ cache.delete(rel);
+ },
+ get stats(): IndexStats {
+ return { hits, misses, stale, size: cache.size };
+ },
+ };
+}
diff --git a/.claude/scripts/lib/memory-recall.ts b/.claude/scripts/lib/memory-recall.ts
index e7b714c..529c5d1 100644
--- a/.claude/scripts/lib/memory-recall.ts
+++ b/.claude/scripts/lib/memory-recall.ts
@@ -25,7 +25,11 @@
import { readFileSync, readdirSync, statSync, existsSync } from "node:fs";
import { join } from "node:path";
-import { MEMORY_ROOT } from "./memory-write.ts";
+import { MEMORY_ROOT, MEMORY_SOURCE } from "./memory-write.ts";
+
+// Re-exported so a reader can take the marker from the module it reads with,
+// while the writer that stamps it stays the one place it is defined.
+export { MEMORY_SOURCE };
export interface Facets {
readonly scope: string;
@@ -254,58 +258,122 @@ export function listMemoryFiles(vaultRoot: string, root: string = MEMORY_ROOT):
function safeDirs(dir: string): string[] {
try {
- return readdirSync(dir).filter((n) => {
- try {
- return statSync(join(dir, n)).isDirectory();
- } catch {
- return false;
- }
- });
+ // Dirents answer isDirectory() from the listing itself, so an ordinary
+ // folder costs no stat — and this runs on every read of the store.
+ //
+ // A symlink still needs one. `Dirent` reports a link-to-directory as
+ // isSymbolicLink() and NOT isDirectory(), so filtering on isDirectory()
+ // alone silently skips it: every memory under `memories/2025 -> /archive`
+ // disappears from recall, from the duplicate scan and from `health`, with
+ // no warning anywhere. The stat is paid only for links.
+ return readdirSync(dir, { withFileTypes: true })
+ .filter((e) => {
+ if (e.isDirectory()) return true;
+ if (!e.isSymbolicLink()) return false;
+ try {
+ return statSync(join(dir, e.name)).isDirectory();
+ } catch {
+ return false; // broken link
+ }
+ })
+ .map((e) => e.name);
} catch {
return [];
}
}
/**
- * Load every memory visible to this caller, ranked.
+ * Parse one file from the store.
+ *
+ * The single parse implementation, so the cache in `memory-index.ts` and a cold
+ * read cannot drift into producing different entries from the same bytes.
+ */
+export function parseMemory(rel: string, full: string, md: string): MemoryEntry {
+ const heading = md.match(/^#\s+(.+)$/m);
+ return {
+ rel,
+ full,
+ facets: facetsOf(parseFrontmatter(md)),
+ title: heading ? (heading[1] ?? "").trim() : null,
+ // The body travels with the entry. A caller past the visibility rule has
+ // earned the content, and the memory root is not in the resource-exposure
+ // list anyway — so a pointer-only result hands back a path it cannot open.
+ body: md
+ .replace(/^---[\s\S]*?\r?\n---\r?\n/, "")
+ .replace(/^#\s+.*$/m, "")
+ .trim(),
+ };
+}
+
+/**
+ * Read and parse one file from the store, or null if it cannot be read.
+ *
+ * Shared by the cold path below and by the cache in `memory-index.ts`, so the
+ * two cannot drift on what counts as unreadable.
+ */
+export function readMemoryFile(file: { rel: string; full: string }): MemoryEntry | null {
+ try {
+ return parseMemory(file.rel, file.full, readFileSync(file.full, "utf8"));
+ } catch {
+ return null; // an unreadable file must not take down retrieval
+ }
+}
+
+/** Read and parse the whole store. The uncached path. */
+export function readMemories(vaultRoot: string, root: string = MEMORY_ROOT): MemoryEntry[] {
+ const out: MemoryEntry[] = [];
+ for (const file of listMemoryFiles(vaultRoot, root)) {
+ const entry = readMemoryFile(file);
+ if (entry) out.push(entry);
+ }
+ return out;
+}
+
+/**
+ * The subset of a store that is agent-written.
+ *
+ * A human note that wandered into the memory folder is left alone rather than
+ * silently governed by these rules. One predicate, so "what counts as a memory"
+ * is answered in a single place.
+ */
+export function agentMemories(entries: readonly MemoryEntry[]): MemoryEntry[] {
+ return entries.filter((m) => m.facets.source === MEMORY_SOURCE);
+}
+
+/**
+ * Apply the visibility rule and rank what survives. PURE — no filesystem.
+ *
+ * This is the whole of recall's decision-making. `recall` below is the thin IO
+ * wrapper that feeds it from disk; the server feeds it from its parse cache.
+ *
+ * Taking a pre-parsed list as an OPTIONAL argument alongside `vaultRoot` and
+ * `root` was designed and rejected: it left both of those dead on every
+ * production call, and made it possible to hand in entries read from one root
+ * while naming another, with nothing able to notice.
*
* `explain` attaches the visibility reason to each entry AND returns the
* withheld ones with theirs — used to prove a memory was excluded deliberately
* rather than missed by accident.
*/
-export function recall(
- vaultRoot: string,
+export function recallFrom(
+ entries: readonly MemoryEntry[],
caller: Caller,
- opts?: { root?: string; explain?: false },
+ opts?: { explain?: false },
): MemoryEntry[];
-export function recall(
- vaultRoot: string,
+export function recallFrom(
+ entries: readonly MemoryEntry[],
caller: Caller,
- opts: { root?: string; explain: true },
+ opts: { explain: true },
): { visible: MemoryEntry[]; withheld: MemoryEntry[] };
-export function recall(
- vaultRoot: string,
+export function recallFrom(
+ entries: readonly MemoryEntry[],
caller: Caller,
- { root = MEMORY_ROOT, explain = false }: { root?: string; explain?: boolean } = {},
+ { explain = false }: { explain?: boolean } = {},
): MemoryEntry[] | { visible: MemoryEntry[]; withheld: MemoryEntry[] } {
const visible: MemoryEntry[] = [];
const withheld: MemoryEntry[] = [];
- for (const file of listMemoryFiles(vaultRoot, root)) {
- let md: string;
- try {
- md = readFileSync(file.full, "utf8");
- } catch {
- continue; // an unreadable file must not take down retrieval
- }
- const fm = parseFrontmatter(md);
- // Only agent-written memories participate; a human note that wandered in
- // here is left alone rather than silently governed by these rules.
- if (fm.source !== "mcp-capture") continue;
- const facets = facetsOf(fm);
- // The body travels with the entry. A caller past the visibility rule has
- // earned the content, and the memory root is not in the resource-exposure
- // list anyway — so a pointer-only result hands back a path it cannot open.
- const entry: MemoryEntry = { ...file, facets, title: titleOf(md), body: bodyOf(md) };
+ for (const entry of agentMemories(entries)) {
+ const facets = entry.facets;
if (isVisibleTo(facets, caller)) {
visible.push(explain ? { ...entry, why: visibilityReason(facets, caller) } : entry);
} else if (explain) {
@@ -316,15 +384,28 @@ export function recall(
return explain ? { visible: ranked, withheld } : ranked;
}
-function titleOf(md: string): string | null {
- const m = md.match(/^#\s+(.+)$/m);
- return m ? (m[1] ?? "").trim() : null;
-}
-
-/** The note minus its frontmatter and H1 — the part worth carrying. */
-function bodyOf(md: string): string {
- return md
- .replace(/^---[\s\S]*?\r?\n---\r?\n/, "")
- .replace(/^#\s+.*$/m, "")
- .trim();
+/**
+ * Load every memory visible to this caller, ranked. The IO wrapper.
+ *
+ * No production caller: the server reads through `memory-index`. This is the
+ * on-disk oracle those cache tests assert equality against, so a dead-code sweep
+ * that removes it takes the equivalence guarantee with it.
+ */
+export function recall(
+ vaultRoot: string,
+ caller: Caller,
+ opts?: { root?: string; explain?: false },
+): MemoryEntry[];
+export function recall(
+ vaultRoot: string,
+ caller: Caller,
+ opts: { root?: string; explain: true },
+): { visible: MemoryEntry[]; withheld: MemoryEntry[] };
+export function recall(
+ vaultRoot: string,
+ caller: Caller,
+ { root = MEMORY_ROOT, explain = false }: { root?: string; explain?: boolean } = {},
+): MemoryEntry[] | { visible: MemoryEntry[]; withheld: MemoryEntry[] } {
+ const entries = readMemories(vaultRoot, root);
+ return explain ? recallFrom(entries, caller, { explain: true }) : recallFrom(entries, caller);
}
diff --git a/.claude/scripts/lib/memory-supersede.ts b/.claude/scripts/lib/memory-supersede.ts
index 9d7d43e..eb7b9f8 100644
--- a/.claude/scripts/lib/memory-supersede.ts
+++ b/.claude/scripts/lib/memory-supersede.ts
@@ -29,8 +29,10 @@
import { readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
+import { MEMORY_SOURCE } from "./memory-write.ts";
+
/** The only `source` value this module is permitted to modify. */
-const OWNED_BY_SERVER = "mcp-capture";
+const OWNED_BY_SERVER = MEMORY_SOURCE;
export interface FrontmatterEdit {
readonly changed: boolean;
diff --git a/.claude/scripts/lib/memory-write.ts b/.claude/scripts/lib/memory-write.ts
index b6db3d6..56c35d2 100644
--- a/.claude/scripts/lib/memory-write.ts
+++ b/.claude/scripts/lib/memory-write.ts
@@ -25,6 +25,15 @@ import { claimFile } from "./atomic-write.ts";
/** Vault-relative root for every memory. Writes are confined to this prefix. */
export const MEMORY_ROOT = "memories";
+/**
+ * The `source` value that marks a file as agent-written.
+ *
+ * Here rather than beside the readers because this module is the one that
+ * STAMPS it, and it already owns the store's on-disk contract. A reader can
+ * import a constant the writer defines; the reverse would invert the dependency.
+ */
+export const MEMORY_SOURCE = "mcp-capture";
+
export const SCOPES = ["general", "platform", "project"] as const;
export const CONFIDENCE = ["verified", "inferred", "unverified"] as const;
@@ -519,7 +528,7 @@ export function renderMemory(value: MemoryValue, links: readonly string[] = []):
`date: ${isoDate(value.date)}`,
`description: ${yamlString(deriveDescription(value.body))}`,
"tags: [memory]",
- "source: mcp-capture",
+ `source: ${MEMORY_SOURCE}`,
`origin: ${yamlString(value.origin ?? "unknown")}`,
`session: ${yamlString(value.date.toISOString())}`,
`scope: ${value.scope}`,
diff --git a/.claude/scripts/lib/read-head.ts b/.claude/scripts/lib/read-head.ts
new file mode 100644
index 0000000..a79f98e
--- /dev/null
+++ b/.claude/scripts/lib/read-head.ts
@@ -0,0 +1,77 @@
+/**
+ * Read the first `chars` characters of a file without reading the rest of it.
+ *
+ * Several call sites inspect only a note's frontmatter — is it private, what is
+ * its description, what are its aliases, was it agent-written, which platforms
+ * does the calling repo declare — and every one of them read the WHOLE file and
+ * discarded everything past the first kilobyte.
+ * `visibleFiles` does that for every note it enumerates, and it is enumerated on
+ * every search, every expand and every write, so the cost scaled with the total
+ * BYTES in the vault rather than with the number of notes. One long reference
+ * note made every unrelated call slower.
+ *
+ * Semantics are preserved exactly. `chars` counts UTF-16 code units, as `slice`
+ * counted them, and UTF-8 needs at most three bytes per code unit — a codepoint
+ * that costs four UTF-8 bytes also costs two units. So reading `chars * 4` bytes
+ * covers the worst case with headroom and yields the identical prefix while
+ * touching a bounded amount of the file. A character split across the end of the
+ * buffer decodes to a replacement character, which the slice then discards —
+ * it can only ever land beyond the `chars` boundary.
+ *
+ * Returns null when the file cannot be read, so each caller keeps its own
+ * fallback: a note that cannot be inspected is not served, has no description,
+ * and contributes no aliases.
+ */
+
+import { openSync, readSync, closeSync } from "node:fs";
+
+/**
+ * How much of a note's head is enough to hold its frontmatter.
+ *
+ * Named in CHARACTERS because that is what callers reason about and what the
+ * `.slice()` this replaced counted — the buffer underneath is four times this,
+ * since a UTF-8 character can take four bytes.
+ */
+export const HEAD_CHARS = 1200;
+
+/**
+ * Ceiling on a single request, so a caller's mistake stays a caller's mistake.
+ *
+ * Without it, an absurd `chars` fails inside `Buffer.allocUnsafe` and comes back
+ * as `null` — which `isPrivate` reads as "unreadable, withhold the note". A bad
+ * argument would present as a file problem, on the one path where the fallback
+ * silently removes content.
+ */
+const MAX_HEAD_CHARS = 1 << 20;
+
+export function readHead(path: string, chars: number = HEAD_CHARS): string | null {
+ if (!Number.isFinite(chars) || chars <= 0) return "";
+ let fd: number | null = null;
+ try {
+ fd = openSync(path, "r");
+ const want = Math.min(chars, MAX_HEAD_CHARS) * 4;
+ // Unsafe is exact here: only `subarray(0, filled)` is ever decoded, and
+ // every one of those bytes was written by `readSync` below. Zero-filling
+ // would memset several kilobytes per note, on a path walked per call.
+ const buf = Buffer.allocUnsafe(want);
+ // `readSync` may return a short read even when more is available, so fill
+ // the buffer rather than trusting one call to reach EOF or capacity.
+ let filled = 0;
+ while (filled < want) {
+ const n = readSync(fd, buf, filled, want - filled, filled);
+ if (n <= 0) break;
+ filled += n;
+ }
+ return buf.subarray(0, filled).toString("utf8").slice(0, chars);
+ } catch {
+ return null;
+ } finally {
+ if (fd !== null) {
+ try {
+ closeSync(fd);
+ } catch {
+ /* the read already succeeded or failed; a close error changes nothing */
+ }
+ }
+ }
+}
diff --git a/.claude/scripts/tests/mcp-memory-bridge.test.ts b/.claude/scripts/tests/mcp-memory-bridge.test.ts
index bd8a2fa..5be4168 100644
--- a/.claude/scripts/tests/mcp-memory-bridge.test.ts
+++ b/.claude/scripts/tests/mcp-memory-bridge.test.ts
@@ -185,6 +185,25 @@ describe("the caller's platforms", () => {
});
});
+ test("frontmatter LONGER than a head read is still parsed in full", () => {
+ // The third near-miss, caught in review. Reading only the head here looks
+ // like the same optimisation the other frontmatter probes in this module
+ // took, but those replaced an equally-sized character slice while this
+ // replaced a whole-file read. `parseFrontmatter` needs the CLOSING `---`,
+ // so a note with a long description or many aliases — both ordinary on a
+ // real project note — parsed to nothing and the caller lost every
+ // platform-scoped memory, silently.
+ withVault((dir) => {
+ const aliases = Array.from({ length: 60 }, (_, i) => ` - a-fairly-long-alias-name-${i}`).join("\n");
+ put(
+ dir,
+ "work/atlas.md",
+ `---\ndate: 2026-07-26\ndescription: "${"padding ".repeat(80)}"\naliases:\n${aliases}\nplatforms: [ios, macos]\n---\n\n# atlas\n\nbody\n`,
+ );
+ assert.deepEqual(callerPlatforms(dir, "atlas", "memories"), ["ios", "macos"]);
+ });
+ });
+
test("a project note OUTSIDE the exposed roots is still found", () => {
// The second failed version walked the exposed files only. A vault whose
// projects live outside a narrowed `mcp_exposed_roots` silently returned
diff --git a/.claude/scripts/tests/memory-index.test.ts b/.claude/scripts/tests/memory-index.test.ts
new file mode 100644
index 0000000..3960048
--- /dev/null
+++ b/.claude/scripts/tests/memory-index.test.ts
@@ -0,0 +1,279 @@
+/**
+ * The memory-store parse cache.
+ *
+ * A cache on the retrieval path is only as good as the moment it decides to stop
+ * trusting itself, so almost every test here is about INVALIDATION rather than
+ * about hits.
+ *
+ * The load-bearing property, asserted directly: what the cache returns is what
+ * reading the store from disk returns. Every other test is a way for that to
+ * stop being true.
+ *
+ * One limit is asserted rather than defended: `(size, mtime)` is a change hint,
+ * not a content hash, so an edit preserving both is invisible here. That is why
+ * the duplicate scan reads from disk instead of through this — a test below
+ * pins both halves of that split.
+ */
+
+import { test, describe } from "node:test";
+import assert from "node:assert/strict";
+import {
+ mkdtempSync,
+ rmSync,
+ writeFileSync,
+ mkdirSync,
+ unlinkSync,
+ utimesSync,
+ statSync,
+ symlinkSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+
+import { createMemoryIndex } from "../lib/memory-index.ts";
+import { readMemories, recall, recallFrom, agentMemories, MEMORY_SOURCE } from "../lib/memory-recall.ts";
+import { loadMemoryDigests, digestsFrom } from "../lib/mcp-memory-bridge.ts";
+
+const ROOT = "memories";
+
+function withVault(fn: (dir: string) => void): void {
+ const dir = mkdtempSync(join(tmpdir(), "memidx-"));
+ try {
+ fn(dir);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+}
+
+function memory(dir: string, name: string, body = "the body.", extra = ""): string {
+ const rel = `${ROOT}/2026/07/${name}.md`;
+ const full = join(dir, rel);
+ mkdirSync(dirname(full), { recursive: true });
+ writeFileSync(
+ full,
+ `---\ndate: 2026-07-26\nsource: ${MEMORY_SOURCE}\nscope: general\nconfidence: verified\n${extra}\n---\n\n# ${name}\n\n${body}\n`,
+ "utf8",
+ );
+ return full;
+}
+
+const titles = (entries: readonly { title: string | null }[]): string[] =>
+ entries.map((e) => e.title ?? "").sort();
+
+// ---------------------------------------------------------------------------
+// Equivalence — the property everything else protects
+// ---------------------------------------------------------------------------
+
+describe("the cache agrees with the filesystem", () => {
+ test("returns exactly what reading the store returns", () => {
+ withVault((dir) => {
+ for (const n of ["a", "b", "c"]) memory(dir, n);
+ const idx = createMemoryIndex(dir, ROOT);
+ assert.deepEqual(
+ idx.all().map((m) => ({ rel: m.rel, title: m.title, body: m.body, facets: m.facets })),
+ readMemories(dir, ROOT).map((m) => ({ rel: m.rel, title: m.title, body: m.body, facets: m.facets })),
+ );
+ });
+ });
+
+ test("recall through the cache equals recall from disk", () => {
+ withVault((dir) => {
+ memory(dir, "shared");
+ memory(dir, "scoped", "body", "projects: [atlas]");
+ const idx = createMemoryIndex(dir, ROOT);
+ const caller = { project: "atlas", platforms: [] as string[] };
+ assert.deepEqual(recallFrom(idx.all(), caller), recall(dir, caller, { root: ROOT }));
+ });
+ });
+
+ test("the duplicate scan sees the same digests either way", () => {
+ withVault((dir) => {
+ memory(dir, "one");
+ memory(dir, "two");
+ const idx = createMemoryIndex(dir, ROOT);
+ assert.deepEqual(digestsFrom(idx.all()), loadMemoryDigests(dir, ROOT));
+ });
+ });
+});
+
+// ---------------------------------------------------------------------------
+// Invalidation
+// ---------------------------------------------------------------------------
+
+describe("noticing that the store changed", () => {
+ test("a second call is served from cache", () => {
+ withVault((dir) => {
+ for (const n of ["a", "b"]) memory(dir, n);
+ const idx = createMemoryIndex(dir, ROOT);
+ idx.all();
+ assert.deepEqual({ h: idx.stats.hits, m: idx.stats.misses }, { h: 0, m: 2 }, "first call parses");
+ idx.all();
+ assert.deepEqual({ h: idx.stats.hits, m: idx.stats.misses }, { h: 2, m: 0 }, "second call reuses");
+ });
+ });
+
+ test("an EDITED memory is re-parsed, not served stale", () => {
+ // The whole risk in one case. A supersession rewrites frontmatter in place,
+ // and serving the previous parse would keep a retired memory outranking the
+ // correction that replaced it.
+ withVault((dir) => {
+ memory(dir, "a", "original body");
+ const idx = createMemoryIndex(dir, ROOT);
+ assert.match(idx.all()[0]!.body, /original body/);
+
+ memory(dir, "a", "corrected body that is a different length entirely");
+ const after = idx.all();
+ assert.match(after[0]!.body, /corrected body/, "the edit must be visible");
+ assert.equal(idx.stats.misses, 1, "and it must have been re-parsed");
+ });
+ });
+
+ test("an edit that preserves SIZE is still caught, via mtime", () => {
+ withVault((dir) => {
+ const full = memory(dir, "a", "aaaa");
+ const idx = createMemoryIndex(dir, ROOT);
+ idx.all();
+ // Same byte length, different content — size alone would miss this.
+ memory(dir, "a", "bbbb");
+ const st = statSync(full);
+ utimesSync(full, st.atime, new Date(st.mtimeMs + 5000));
+ assert.match(idx.all()[0]!.body, /bbbb/);
+ });
+ });
+
+ test("an edit that preserves MTIME is still caught, via size", () => {
+ // Some filesystems keep mtime to a whole second, so two writes inside one
+ // second can carry an identical timestamp.
+ withVault((dir) => {
+ const full = memory(dir, "a", "short");
+ const idx = createMemoryIndex(dir, ROOT);
+ idx.all();
+ const before = statSync(full);
+ memory(dir, "a", "a considerably longer body than the one before it");
+ utimesSync(full, before.atime, before.mtime);
+ assert.match(idx.all()[0]!.body, /considerably longer/);
+ });
+ });
+
+ test("invalidate forces a re-parse even when nothing on disk looks different", () => {
+ // The belt to the size+mtime braces: writers call this, so correctness never
+ // rests on timestamp resolution.
+ withVault((dir) => {
+ memory(dir, "a");
+ const idx = createMemoryIndex(dir, ROOT);
+ const [first] = idx.all();
+ idx.invalidate(first!.rel);
+ idx.all();
+ assert.equal(idx.stats.misses, 1);
+ assert.equal(idx.stats.hits, 0);
+ });
+ });
+
+ test("the duplicate-scan path sees an edit the cache is not guaranteed to", () => {
+ // `(size, mtime)` is a change hint, not a content hash: `rsync -t`, `tar -p`,
+ // `unzip`, a sync client and a second-granularity filesystem all produce
+ // same-size, same-mtime, different-bytes. Whether THIS filesystem happens to
+ // collide is not something a test can force portably — restoring an mtime
+ // through a Date loses sub-millisecond precision, so it does not collide
+ // here — so the assertion is on the guarantee that does hold: the path
+ // `remember` uses reads from disk, and therefore always reflects the file.
+ withVault((dir) => {
+ const full = memory(dir, "a", "aaaa");
+ const before = statSync(full);
+ memory(dir, "a", "bbbb"); // identical length
+ utimesSync(full, before.atime, before.mtime);
+
+ assert.match(
+ readMemories(dir, ROOT)[0]!.body,
+ /bbbb/,
+ "the duplicate scan must never be able to miss a write",
+ );
+ });
+ });
+
+ test("a symlinked year folder is walked, not silently skipped", () => {
+ // A Dirent reports a link-to-directory as isSymbolicLink() and NOT
+ // isDirectory(), so filtering on isDirectory() alone drops every memory
+ // under `memories/2025 -> /somewhere/archive` — from recall, from the
+ // duplicate scan and from health, with no warning anywhere.
+ withVault((dir) => {
+ memory(dir, "live");
+ const archive = join(dir, "archived-elsewhere", "07");
+ mkdirSync(archive, { recursive: true });
+ writeFileSync(
+ join(archive, "old.md"),
+ `---\ndate: 2025-07-01\nsource: ${MEMORY_SOURCE}\nscope: general\nconfidence: verified\n---\n\n# old\n\nbody.\n`,
+ "utf8",
+ );
+ try {
+ symlinkSync(join(dir, "archived-elsewhere"), join(dir, ROOT, "2025"), "junction");
+ } catch {
+ return; // Windows without developer mode
+ }
+ assert.deepEqual(titles(createMemoryIndex(dir, ROOT).all()), ["live", "old"]);
+ });
+ });
+
+ test("a NEW memory appears without any explicit invalidation", () => {
+ withVault((dir) => {
+ memory(dir, "a");
+ const idx = createMemoryIndex(dir, ROOT);
+ assert.equal(idx.all().length, 1);
+ memory(dir, "b");
+ assert.deepEqual(titles(idx.all()), ["a", "b"]);
+ });
+ });
+
+ test("a DELETED memory disappears, and stops being held", () => {
+ withVault((dir) => {
+ const full = memory(dir, "a");
+ memory(dir, "b");
+ const idx = createMemoryIndex(dir, ROOT);
+ assert.equal(idx.all().length, 2);
+ unlinkSync(full);
+ assert.deepEqual(titles(idx.all()), ["b"]);
+ assert.equal(idx.stats.size, 1, "the cache tracks the store rather than growing forever");
+ });
+ });
+
+ test("an emptied store returns nothing and holds nothing", () => {
+ withVault((dir) => {
+ const full = memory(dir, "only");
+ const idx = createMemoryIndex(dir, ROOT);
+ idx.all();
+ unlinkSync(full);
+ assert.deepEqual(idx.all(), []);
+ assert.equal(idx.stats.size, 0);
+ });
+ });
+});
+
+// ---------------------------------------------------------------------------
+// What counts as a memory
+// ---------------------------------------------------------------------------
+
+describe("policy stays with the caller", () => {
+ test("the cache returns human notes too; agentMemories is what filters", () => {
+ // Deliberate: a cache that encoded "is this a memory" would have to be
+ // rebuilt whenever that rule moved.
+ withVault((dir) => {
+ memory(dir, "written-by-an-agent");
+ const rel = join(dir, ROOT, "2026", "07", "mine.md");
+ mkdirSync(dirname(rel), { recursive: true });
+ writeFileSync(rel, "---\ndate: 2026-07-26\ntags: [note]\n---\n\n# mine\n\nhand written\n", "utf8");
+
+ const idx = createMemoryIndex(dir, ROOT);
+ const all = idx.all();
+ assert.equal(all.length, 2, "both are cached");
+ assert.deepEqual(titles(agentMemories(all)), ["written-by-an-agent"]);
+ });
+ });
+
+ test("an empty store is an empty list, not a throw", () => {
+ withVault((dir) => assert.deepEqual(createMemoryIndex(dir, ROOT).all(), []));
+ });
+
+ test("a missing memory root is an empty list", () => {
+ withVault((dir) => assert.deepEqual(createMemoryIndex(dir, "nowhere").all(), []));
+ });
+});
diff --git a/.claude/scripts/tests/read-head.test.ts b/.claude/scripts/tests/read-head.test.ts
new file mode 100644
index 0000000..60cf7f1
--- /dev/null
+++ b/.claude/scripts/tests/read-head.test.ts
@@ -0,0 +1,101 @@
+/**
+ * Reading only the head of a file.
+ *
+ * This replaced `readFileSync(f, "utf8").slice(0, n)` on every hot path that
+ * only wanted frontmatter, so the bar is EQUIVALENCE rather than merely
+ * plausibility: every test that matters compares
+ * against what the old expression returned for the same bytes. A prefix that is
+ * subtly different would change which notes count as private, what descriptions
+ * a listing carries, and which aliases resolve — all silently.
+ */
+
+import { test, describe } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, writeFileSync, mkdirSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { readHead } from "../lib/read-head.ts";
+
+function withDir(fn: (dir: string) => void): void {
+ const dir = mkdtempSync(join(tmpdir(), "head-"));
+ try {
+ fn(dir);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+}
+
+/** What the code this replaced would have returned. */
+const oldWay = (path: string, chars: number): string => readFileSync(path, "utf8").slice(0, chars);
+
+function put(dir: string, name: string, body: string): string {
+ const full = join(dir, name);
+ writeFileSync(full, body, "utf8");
+ return full;
+}
+
+describe("reading a file head", () => {
+ test("matches the whole-file read it replaced, across sizes", () => {
+ withDir((dir) => {
+ for (const [name, body] of [
+ ["tiny.md", "x"],
+ ["exact.md", "y".repeat(1200)],
+ ["long.md", "z".repeat(200_000)],
+ ["empty.md", ""],
+ ] as const) {
+ const full = put(dir, name, body);
+ assert.equal(readHead(full, 1200), oldWay(full, 1200), name);
+ }
+ });
+ });
+
+ test("a multibyte character spanning the read boundary is not corrupted", () => {
+ // The buffer is sized in BYTES and the result sliced in CHARACTERS. A
+ // character split by the end of the buffer decodes to U+FFFD, which must
+ // always land beyond the slice — otherwise a CJK or emoji frontmatter value
+ // comes back mangled, and only for files above a certain size.
+ withDir((dir) => {
+ for (const filler of ["日", "é", "🧠", "日本語テキスト"]) {
+ const full = put(dir, "m.md", filler.repeat(5000));
+ for (const n of [1, 7, 100, 799, 800, 1200]) {
+ assert.equal(readHead(full, n), oldWay(full, n), `${filler} @ ${n}`);
+ assert.ok(!String(readHead(full, n)).includes("�"), `${filler} @ ${n} must not corrupt`);
+ }
+ }
+ });
+ });
+
+ test("frontmatter is intact even when the body is enormous", () => {
+ withDir((dir) => {
+ const full = put(dir, "big.md", `---\ndescription: "kept"\nprivate: true\n---\n\n${"x".repeat(500_000)}`);
+ const head = readHead(full, 1200);
+ assert.match(String(head), /description: "kept"/);
+ assert.match(String(head), /private: true/);
+ });
+ });
+
+ test("a file shorter than the request returns all of it", () => {
+ withDir((dir) => {
+ const full = put(dir, "s.md", "short");
+ assert.equal(readHead(full, 1200), "short");
+ });
+ });
+
+ test("unreadable inputs return null, so each caller keeps its own fallback", () => {
+ withDir((dir) => {
+ assert.equal(readHead(join(dir, "nope.md"), 100), null, "missing file");
+ mkdirSync(join(dir, "adir"));
+ assert.equal(readHead(join(dir, "adir"), 100), null, "a directory is not a file");
+ });
+ });
+
+ test("a non-positive or nonsense length is an empty string, never a whole file", () => {
+ withDir((dir) => {
+ const full = put(dir, "x.md", "content");
+ for (const n of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) {
+ assert.equal(readHead(full, n), "", String(n));
+ }
+ });
+ });
+});
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index 563fd5d..1ecd92c 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -390,12 +390,14 @@ flowchart TB
Caller["mcp-caller.ts<br/>identity · sanitise · audit log"]
Bridge["mcp-memory-bridge.ts<br/>memory ↔ vault seam"]
Capture["mcp-capture.ts<br/>record_work filing"]
+ RH["read-head.ts<br/>bounded frontmatter prefix"]
subgraph Core["Memory core — no MCP knowledge at all"]
MW["memory-write.ts"]
MR["memory-recall.ts"]
MS["memory-similarity.ts"]
MSup["memory-supersede.ts"]
MD["memory-discover.ts"]
+ Idx["memory-index.ts<br/>parse cache over the store"]
end
Entry --> Proto
@@ -408,11 +410,16 @@ flowchart TB
Server --> Graph
Server --> Bridge
Server --> Capture
+ Server --> Idx
Server --> Core
+ Idx --> MR
Graph --> Exp
Qmd --> Exp
Bridge --> Core
Bridge --> Qmd
+ Exp --> RH
+ Bridge --> RH
+ MD --> RH
```
The memory core knows nothing about MCP. That is what lets the epistemic contract be hammered by tests without an MCP client, a vault on disk, or a search index.
@@ -625,6 +632,30 @@ Three properties in that path each closed a real defect:
A missing index degrades *ordering*, never availability. "The vault knows nothing" is not an acceptable answer to a wiring problem.
+### Reading the store without re-reading it
+
+`recall` and the duplicate scan read the same files and parse the same frontmatter, once per call each. That is linear in the store, and it runs on every recall and every write. `memory-index.ts` caches the **parse** — and only the parse.
+
+```mermaid
+flowchart TB
+ Call["recall, or the duplicate scan"] --> List["list the store<br/>every call"]
+ List --> Stat["stat each file<br/>every call"]
+ Stat --> Q{"size AND mtime<br/>unchanged?"}
+ Q -->|yes| Hit["reuse the parsed entry"]
+ Q -->|no| Read["read + parse, then cache it"]
+ Hit --> Out["entries"]
+ Read --> Out
+ Gone["a file that vanished"] --> Drop["dropped from the cache,<br/>so it tracks the store"]
+```
+
+The listing and the stat happen on **every** call. Only re-reading and re-parsing an unchanged file is skipped, which is what makes this safe on the duplicate path: a memory written a second ago is always seen, and a stale view there would admit a duplicate permanently, because nothing downstream re-checks.
+
+Size *and* mtime, because either alone is weak — some filesystems keep mtime to a whole second, and supersession rewrites frontmatter in place, which can preserve length. Writers also invalidate explicitly, so correctness never rests on timestamp resolution.
+
+**It is deliberately in-process, not on disk.** Measured, these walks are a minority of a queried recall — the local query embedding dominates — so persisting the index buys a fraction of one operation while adding a file that can rot, disagree with the store, or need migrating in every vault that ever installed the template. The server is long-lived per repo, so every call after the first is warm and a cold start costs what it always did.
+
+The same reasoning trimmed a second cost. Several call sites inspected only a note's frontmatter — is it private, what is its description, what are its aliases, was it agent-written, which platforms does the calling repo declare — and each read the **whole file** to look at its first kilobyte. `visibleFiles` does that for every note it enumerates, on every search, expand and write, so the cost scaled with the vault's total **bytes**: one long reference note made every unrelated call slower. `read-head.ts` reads a bounded prefix instead, preserving the exact string the old expression produced.
+
### The write path
```mermaid