Self-hosted MCP server + dashboard giving Claude Code, Cursor & Hermes agents shared long-term memory over a git-backed markdown vault. Agent-native, Obsidian-compatible second brain — markdown RAG, no vector DB.
TypeScript agent-memoryai-agentsanthropicclaude-codecursorgitknowledge-baseknowledge-graphllmmarkdownmcpmcp-servermodel-context-protocolnote-takingobsidianpkmragsecond-brainself-hostedzettelkasten
import fs from "node:fs";
import path from "node:path";
import MiniSearch from "minisearch";
import { watch, type FSWatcher } from "chokidar";
import { VAULT_IGNORE } from "@/lib/config";
import { activeVaultDir } from "@/lib/repos";
import { scanVault } from "./scan";
import { parseNote, stemOf } from "./parse";
import { authorityOf, authorityRules, isArchivedPath, isUnrecognizedStatus, overlayValidity, weightOf, type Authority } from "./authority";
import { tryResolveInVault } from "./paths";
import type { Graph, GraphEdge, GraphNode, Note, NoteMeta, TreeNode } from "./types";
interface IndexDoc {
id: string;
title: string;
aliases: string;
tags: string;
body: string;
folder: string;
type: string;
/** stored, not indexed — used for filtering + authority-weighted ranking */
status: string;
authority: Authority;
/** epoch ms deadline (or null). Read at query time so expiry is live without re-indexing. */
validUntil: number | null;
/** stem of the superseding note (or null). Marks this doc retired at query time. */
supersededBy: string | null;
mtimeMs: number;
}
interface IndexState {
dir: string;
notes: Map<string, NoteMeta>;
/** path -> wikilink target stems. Kept so link/graph rebuilds need no disk IO. */
linksBySource: Map<string, string[]>;
stemToPath: Map<string, string>;
/** stems claimed by more than one file — wikilinks to these resolve to the first, silently. */
duplicateStems: Map<string, string[]>;
outEdges: Map<string, Set<string>>;
inEdges: Map<string, Set<string>>;
search: MiniSearch<IndexDoc>;
builtAt: number;
}
let state: IndexState | null = null;
let watcher: FSWatcher | null = null;
let watchedDir = "";
function newIndex(): MiniSearch<IndexDoc> {
return new MiniSearch<IndexDoc>({
fields: ["title", "aliases", "tags", "body", "folder", "type"],
storeFields: ["title", "folder", "type", "status", "authority", "validUntil", "supersededBy", "mtimeMs"],
searchOptions: {
boost: { title: 3, aliases: 3, tags: 2 },
prefix: true,
fuzzy: 0.2,
// OR, deliberately. With AND, one query word the canonical note happens to lack
// ("tiers") drops it from the results entirely — and the superseded note that *does*
// use the word becomes the only answer. Authority weighting can only reorder what
// survives the combiner, so the combiner must not throw the truth away.
// Docs matching more terms still score higher; noise is handled by ranking, not exclusion.
combineWith: "OR",
},
});
}
function toDoc(meta: NoteMeta, body: string): IndexDoc {
return {
id: meta.path,
title: meta.title,
aliases: meta.aliases.join(" "),
tags: meta.tags.join(" "),
body,
folder: meta.folder,
type: meta.type ?? "",
status: meta.status ?? "",
authority: authorityOf(meta),
validUntil: meta.validUntil ?? null,
supersededBy: meta.supersededBy ?? null,
mtimeMs: meta.mtimeMs,
};
}
function readAndParse(
dir: string,
rel: string,
mtimeMs: number,
): { meta: NoteMeta; body: string; stems: string[] } | null {
let raw: string;
try {
raw = fs.readFileSync(path.join(dir, rel), "utf8");
} catch {
return null;
}
const { meta, body, rawLinks } = parseNote(rel, raw);
return { meta: { ...meta, mtimeMs }, body, stems: rawLinks.map((l) => stemOf(l.target)) };
}
/**
* Rebuild stem resolution + the link graph from in-memory maps only.
* Cheap (no disk IO), so it can run after every single-file change.
*/
function recomputeLinks(s: IndexState): void {
s.stemToPath = new Map();
s.duplicateStems = new Map();
for (const rel of [...s.notes.keys()].sort()) {
const slug = s.notes.get(rel)!.slug;
const first = s.stemToPath.get(slug);
if (first === undefined) {
s.stemToPath.set(slug, rel);
} else {
const dupes = s.duplicateStems.get(slug) ?? [first];
dupes.push(rel);
s.duplicateStems.set(slug, dupes);
}
}
// A stem collision is a property of the vault's layout, not an event, so it is reported by
// `vaultConventions().integrity` — queryable on demand via brain_schema and the dashboard —
// and deliberately not logged. Console output is the wrong channel for standing state: it
// cannot be dismissed or acted on where it appears, and printing from here (which runs on
// every write, flush and rebuild) is what buried the deploy log in the first place.
s.outEdges = new Map();
s.inEdges = new Map();
for (const [src, stems] of s.linksBySource) {
for (const stem of stems) {
const tgt = s.stemToPath.get(stem);
if (!tgt || tgt === src) continue;
(s.outEdges.get(src) ?? s.outEdges.set(src, new Set()).get(src)!).add(tgt);
(s.inEdges.get(tgt) ?? s.inEdges.set(tgt, new Set()).get(tgt)!).add(src);
}
}
}
function buildState(): IndexState {
const dir = activeVaultDir();
const s: IndexState = {
dir,
notes: new Map(),
linksBySource: new Map(),
stemToPath: new Map(),
duplicateStems: new Map(),
outEdges: new Map(),
inEdges: new Map(),
search: newIndex(),
builtAt: Date.now(),
};
const docs: IndexDoc[] = [];
for (const f of scanVault(dir)) {
const parsed = readAndParse(dir, f.rel, f.mtimeMs);
if (!parsed) continue;
s.notes.set(f.rel, parsed.meta);
s.linksBySource.set(f.rel, parsed.stems);
docs.push(toDoc(parsed.meta, parsed.body));
}
s.search.addAll(docs);
recomputeLinks(s);
return s;
}
/** Add or update one file in the live index. Returns false when the file is gone. */
function upsert(s: IndexState, rel: string): boolean {
let mtimeMs = 0;
try {
mtimeMs = fs.statSync(path.join(s.dir, rel)).mtimeMs;
} catch {
return false;
}
const parsed = readAndParse(s.dir, rel, mtimeMs);
if (!parsed) return false;
if (s.notes.has(rel)) {
try {
s.search.discard(rel);
} catch {
/* not in the index — fall through to add */
}
}
s.notes.set(rel, parsed.meta);
s.linksBySource.set(rel, parsed.stems);
s.search.add(toDoc(parsed.meta, parsed.body));
return true;
}
function remove(s: IndexState, rel: string): void {
if (!s.notes.has(rel)) return;
try {
s.search.discard(rel);
} catch {
/* already gone */
}
s.notes.delete(rel);
s.linksBySource.delete(rel);
}
function startWatcher(dir: string) {
if (watcher && watchedDir === dir) return;
if (watcher) {
watcher.close().catch(() => {});
watcher = null;
}
watchedDir = dir;
try {
watcher = watch(dir, {
ignoreInitial: true,
persistent: true,
depth: 12,
ignored: (p) => {
const base = path.basename(p);
if (base.startsWith(".")) return true;
return [...VAULT_IGNORE].some((ig) => base === ig || p.includes(`${path.sep}${ig}${path.sep}`));
},
});
const pending = new Set<string>();
let t: ReturnType<typeof setTimeout> | null = null;
let fullRebuild = false;
const flush = () => {
try {
if (fullRebuild || !state || state.dir !== dir) {
state = buildState();
} else {
for (const rel of pending) if (!upsert(state, rel)) remove(state, rel);
recomputeLinks(state);
}
} catch (e) {
console.error("[vault] index update failed", e);
}
pending.clear();
fullRebuild = false;
};
const schedule = () => {
if (t) clearTimeout(t);
t = setTimeout(flush, 250);
};
const onFile = (abs: string) => {
if (!abs.toLowerCase().endsWith(".md")) return;
pending.add(path.relative(dir, abs).split(path.sep).join("/"));
schedule();
};
const onDir = () => {
fullRebuild = true;
schedule();
};
watcher.on("add", onFile).on("change", onFile).on("unlink", onFile).on("addDir", onDir).on("unlinkDir", onDir);
} catch (e) {
console.error("[vault] watcher failed to start", e);
}
}
function ensure(): IndexState {
const dir = activeVaultDir();
if (!state || state.dir !== dir) state = buildState();
startWatcher(dir);
return state;
}
/** Force a synchronous full rebuild (workspace switch, or after a git pull changed many files). */
export function rebuildIndex(): void {
state = buildState();
startWatcher(state.dir);
}
/** Update just these paths (created, edited, moved, or deleted). No full re-scan. */
export function refreshPaths(relPaths: string[]): void {
const s = ensure();
for (const rel of relPaths) {
if (!rel.toLowerCase().endsWith(".md")) continue;
if (!upsert(s, rel)) remove(s, rel);
}
recomputeLinks(s);
}
export function listNotes(): NoteMeta[] {
return [...ensure().notes.values()];
}
/** Notes ordered by last modification, newest first. Powers "what changed lately". */
export function listRecent(sinceMs?: number, limit = 20): NoteMeta[] {
return [...ensure().notes.values()]
.filter((n) => (sinceMs ? n.mtimeMs >= sinceMs : true))
.sort((a, b) => b.mtimeMs - a.mtimeMs)
.slice(0, limit);
}
export function getNote(relPath: string): Note | null {
const s = ensure();
// Caller-controlled (brain_read, the dashboard note route) — must not escape the vault.
const abs = tryResolveInVault(s.dir, relPath);
if (!abs) return null;
let raw: string;
try {
raw = fs.readFileSync(abs, "utf8");
} catch {
return null;
}
const { meta, body } = parseNote(relPath, raw);
return { ...meta, mtimeMs: s.notes.get(relPath)?.mtimeMs ?? 0, body, raw };
}
export interface SearchHit {
path: string;
title: string;
folder: string;
type?: string;
status?: string;
/** Trust class, independent of relevance. See lib/vault/authority.ts. */
authority: Authority;
score: number;
snippet?: string;
/** Present only when a retired note is returned anyway (includeInvalid) — why it's stale. */
reason?: string;
}
/** A note that matched the query but was withheld — the "explainable rejection". */
export interface SearchExclusion {
path: string;
title: string;
folder: string;
authority: Authority;
/** Why it was withheld, e.g. "superseded by price-live" or "expired 2026-06-01". */
reason: string;
}
export interface SearchResult {
hits: SearchHit[];
/** Matches held back as archived/superseded/expired, so an agent can say what it ignored and why. */
excluded: SearchExclusion[];
}
export interface SearchOpts {
limit?: number;
/** Restrict to one top-level folder. */
folder?: string;
/** Include notes in archive folders (still ranked far below live ones). Default false. */
includeArchive?: boolean;
/** Include superseded/expired notes as hits (still demoted). Default false — they go to `excluded`. */
includeInvalid?: boolean;
/** Attach ~200 chars of matching context per hit. Default true. */
snippets?: boolean;
}
/** First body line containing a query term (or the first prose line), trimmed for display. */
function snippetFor(dir: string, rel: string, terms: string[]): string | undefined {
try {
const raw = fs.readFileSync(path.join(dir, rel), "utf8");
const { body } = parseNote(rel, raw);
const lines = body
.split("\n")
.map((l) => l.trim())
.filter((l) => l.length > 0 && !l.startsWith("#") && !l.startsWith("---"));
const hit = lines.find((l) => {
const low = l.toLowerCase();
return terms.some((t) => low.includes(t));
});
const chosen = hit ?? lines[0];
if (!chosen) return undefined;
return chosen.length > 220 ? `${chosen.slice(0, 217)}…` : chosen;
} catch {
return undefined;
}
}
interface StoredDoc {
id: string;
title: string;
folder: string;
type?: string;
status?: string;
authority: Authority;
validUntil?: number | null;
supersededBy?: string | null;
score: number;
}
/**
* Keyword search, reweighted by authority, then partitioned by validity.
*
* Three stages: MiniSearch says how well a note *matches*; authority says how far it can be
* *trusted*; and the validity overlay (superseded / expired) decides whether it's still true at
* all. A retired note is withheld from `hits` and reported in `excluded` with a reason — so an
* agent can state what it ignored and why, instead of quietly quoting a dead fact.
*/
export function searchNotes(q: string, opts: SearchOpts = {}): SearchResult {
const s = ensure();
if (!q.trim()) return { hits: [], excluded: [] };
const { limit = 20, folder, includeArchive = false, includeInvalid = false, snippets = true } = opts;
const now = Date.now();
const EXCLUDE_CAP = 10;
const raw = s.search.search(q, {
// Effective weight: overlay validity onto the stored authority so an expired/superseded note
// is demoted at query time — no re-index needed (the index-time authority is frozen).
boostDocument: (_id, _term, stored) => {
const d = stored as { authority?: Authority; validUntil?: number | null; supersededBy?: string | null } | undefined;
const eff = overlayValidity((d?.authority as Authority) ?? "current", d?.validUntil, d?.supersededBy, now);
return weightOf(eff.authority);
},
// Folder is the only hard filter here; archived/superseded/expired flow through so they can be
// partitioned into `excluded` with a reason rather than silently dropped.
filter: (r) => {
const stored = r as unknown as { folder: string };
return !folder || stored.folder === folder;
},
});
const hits: SearchHit[] = [];
const excluded: SearchExclusion[] = [];
for (const r of raw) {
if (hits.length >= limit && excluded.length >= EXCLUDE_CAP) break;
const d = r as unknown as StoredDoc;
const eff = overlayValidity(d.authority, d.validUntil, d.supersededBy, now);
const isArchived = eff.authority === "archived";
const withhold = isArchived ? !includeArchive : eff.retired && !includeInvalid;
if (withhold) {
if (excluded.length < EXCLUDE_CAP) {
excluded.push({ path: d.id, title: d.title, folder: d.folder, authority: eff.authority, reason: eff.reason ?? "retired" });
}
continue;
}
if (hits.length < limit) {
hits.push({
path: d.id,
title: d.title,
folder: d.folder,
type: d.type || undefined,
status: d.status || undefined,
authority: eff.authority,
score: d.score,
reason: eff.retired ? eff.reason : undefined,
});
}
}
if (snippets && hits.length > 0) {
const terms = q
.toLowerCase()
.split(/\s+/)
.map((t) => t.replace(/[^a-z0-9-]/g, ""))
.filter((t) => t.length >= 3);
for (const h of hits) h.snippet = snippetFor(s.dir, h.path, terms);
}
return { hits, excluded };
}
export function getBacklinks(relPath: string): NoteMeta[] {
const s = ensure();
return [...(s.inEdges.get(relPath) ?? [])].map((p) => s.notes.get(p)).filter(Boolean) as NoteMeta[];
}
export function getOutlinks(relPath: string): NoteMeta[] {
const s = ensure();
return [...(s.outEdges.get(relPath) ?? [])].map((p) => s.notes.get(p)).filter(Boolean) as NoteMeta[];
}
/**
* What this vault actually looks like + how search will treat it. Returned by brain_schema so
* an agent meeting an unfamiliar vault discovers its conventions instead of assuming ours.
*/
export function vaultConventions() {
const s = ensure();
const folders = new Set<string>();
const statuses = new Map<string, number>();
const types = new Set<string>();
const malformed: Array<{ path: string; error: string }> = [];
const unknownStatus: Array<{ path: string; status: string }> = [];
let archived = 0;
let superseded = 0;
let expired = 0;
const now = Date.now();
for (const n of s.notes.values()) {
folders.add(n.folder);
if (n.type) types.add(n.type);
if (n.status) statuses.set(n.status, (statuses.get(n.status) ?? 0) + 1);
if (isUnrecognizedStatus(n.status)) unknownStatus.push({ path: n.path, status: n.status! });
if (isArchivedPath(n.path)) archived++;
if (n.frontmatterError) malformed.push({ path: n.path, error: n.frontmatterError });
if (n.supersededBy) superseded++;
else if (n.validUntil != null && n.validUntil < now) expired++;
}
// Which ambiguous stems a wikilink actually reaches. Computed here, on demand, rather than in
// recomputeLinks: that runs on every write and this is read only when someone asks for the
// report. `linkedFrom` names one note holding such a link, so the finding points at a file to
// edit instead of only asserting that a collision exists somewhere.
const linkedFrom = new Map<string, string>();
for (const [src, stems] of s.linksBySource) {
for (const stem of stems) if (!linkedFrom.has(stem)) linkedFrom.set(stem, src);
}
const duplicateStems = [...s.duplicateStems.entries()].map(([stem, paths]) => {
const from = linkedFrom.get(stem);
return from === undefined ? { stem, paths } : { stem, paths, linkedFrom: from };
});
const misresolving = duplicateStems.filter(
(d): d is { stem: string; paths: string[]; linkedFrom: string } => "linkedFrom" in d,
);
return {
noteCount: s.notes.size,
archivedCount: archived,
supersededCount: superseded,
expiredCount: expired,
folders: [...folders].sort(),
types: [...types].sort(),
statusesInUse: [...statuses.entries()]
.sort((a, b) => b[1] - a[1])
.map(([status, count]) => ({ status, count })),
ranking: authorityRules(),
integrity: {
duplicateStems,
malformedFrontmatter: malformed,
unrecognizedStatus: unknownStatus,
// Only flag collisions a link actually hits. Two notes sharing a filename nothing
// references changes no resolution — every vault has a few READMEs — so warning about
// those buries the one collision that is silently sending a wikilink to the wrong note.
...(misresolving.length > 0
? {
duplicateStemWarning:
`Wikilinks to a duplicated stem resolve to the first path only. ` +
misresolving
.map((d) => `[[${d.stem}]] (linked from ${d.linkedFrom}) resolves to ${d.paths[0]}`)
.join("; ") +
`. Rename the colliding file — a folder-qualified link like [[folder/${misresolving[0].stem}]] ` +
`will not help, the path is only a hint and still collapses to the bare stem.`,
}
: {}),
...(malformed.length > 0
? {
malformedFrontmatterWarning:
"These notes open with `---` but their YAML is unparseable, so their status, tags and title are being ignored — they rank as ordinary notes regardless of what they claim. Usual cause: an unquoted `:` in a value. Fix by quoting it.",
}
: {}),
...(unknownStatus.length > 0
? {
unrecognizedStatusWarning:
"These notes carry a `status:` the ranking model does not recognise, so they rank as plain `current` no matter what they claim. Often a typo (`lokced`), sometimes a vault convention this instance has not been told about. See ranking.statusWords for the vocabulary.",
}
: {}),
},
};
}
export function getGraph(folder?: string): Graph {
const s = ensure();
const inScope = (p: string) => !folder || s.notes.get(p)?.folder === folder;
const degree = new Map<string, number>();
const bump = (p: string) => degree.set(p, (degree.get(p) ?? 0) + 1);
const edges: GraphEdge[] = [];
for (const [src, set] of s.outEdges) {
for (const tgt of set) {
if (!inScope(src) || !inScope(tgt)) continue;
edges.push({ source: src, target: tgt });
bump(src);
bump(tgt);
}
}
const nodes: GraphNode[] = [...s.notes.values()]
.filter((n) => inScope(n.path))
.map((n) => ({ id: n.path, label: n.title, folder: n.folder, type: n.type, degree: degree.get(n.path) ?? 0 }));
return { nodes, edges };
}
export function getTree(): TreeNode {
const s = ensure();
const root: TreeNode = { name: "", path: "", type: "dir", children: [] };
const dirs = new Map<string, TreeNode>([["", root]]);
function ensureDir(dirPath: string): TreeNode {
const existing = dirs.get(dirPath);
if (existing) return existing;
const parentPath = dirPath.split("/").slice(0, -1).join("/");
const parent = ensureDir(parentPath);
const node: TreeNode = { name: dirPath.split("/").pop()!, path: dirPath, type: "dir", children: [] };
parent.children!.push(node);
dirs.set(dirPath, node);
return node;
}
for (const n of [...s.notes.values()].sort((a, b) => a.path.localeCompare(b.path))) {
const dirPath = n.path.split("/").slice(0, -1).join("/");
ensureDir(dirPath).children!.push({ name: n.slug, path: n.path, type: "file", title: n.title });
}
(function sortRec(node: TreeNode) {
node.children?.sort((a, b) => (a.type !== b.type ? (a.type === "dir" ? -1 : 1) : a.name.localeCompare(b.name)));
node.children?.forEach(sortRec);
})(root);
return root;
}
/** Read a top-level meta file (SCHEMA.md / INDEX.md) from the active vault, or null. */
export function readVaultFile(name: string): string | null {
try {
const abs = tryResolveInVault(activeVaultDir(), name);
if (!abs) return null;
return fs.readFileSync(abs, "utf8");
} catch {
return null;
}
}