Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* §14 Brain explorer — data collector.
*
* Pure read. Walks `Brain/preferences/` and `Brain/retired/`, builds
* a node list plus an edge list derived from:
*
* 1. Frontmatter `supersedes` / `superseded_by` (kind `supersedes`).
* 2. Inline `[[wikilinks]]` anywhere in the file's frontmatter
* values and body, filtered to refs landing on another node
* (kind `wikilink`).
*
* Signal (`sig-*`) and log refs never appear in `edges` — the
* explorer view is about the rule graph, not the audit trail.
*
* Output is deterministic: nodes sorted by `(kind, id)`, edges by
* `(source, target, kind)`. The collector emits a frozen object so
* the live HTTP handler and the static export branch cannot
* accidentally mutate it.
*
* A file that cannot be parsed is REPORTED on the graph
* ({@link ExplorerParseFailure}), never dropped. This surface both
* displays the graph and exports it, and a graph silently missing a rule
* is indistinguishable from a vault that never had one.
*
* The privacy deny markers reserve a node without refusing it: the
* collector keeps every parsed node and
* {@link collectExplorerDenials} reports which ones a SHARED output
* must withhold - the live view is the operator's own, the export is
* not, so the narrowing lives with the caller that writes bytes outward.
*/
import { readFileSync } from "node:fs";
import { basename, dirname, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { buildBacklinkIndex, type BacklinkIndex } from "./backlinks.ts";
import { deniedReservationKey, PREFERENCE_DENY_REASON, type PreferenceDenial } from "./export.ts";
import { memoryFolderPages, type MemoryFolder } from "./memory-pages.ts";
import { parsePreferenceMeta, parseRetiredMeta } from "./preference.ts";
import {
BRAIN_PREFERENCE_STATUS,
type BrainConfidence,
type BrainPreference,
type BrainRetired,
} from "./types.ts";
import { collectExplorerConsole } from "./explorer-console.ts";
import { buildOperatorSnapshot } from "./operator-snapshot.ts";
import { redactRawOutput } from "../redactor.ts";
import { sortedDistinctLiterals } from "../secret-ref.ts";
export const EXPLORER_SCHEMA_VERSION = 1 as const;
export type ExplorerNodeKind = "preference" | "retired";
export type ExplorerNodeStatus = "unconfirmed" | "confirmed" | "quarantine" | "retired";
export type ExplorerEdgeKind = "supersedes" | "wikilink";
export interface ExplorerNode {
readonly id: string;
readonly kind: ExplorerNodeKind;
readonly topic: string;
readonly scope: string | null;
readonly principle: string;
readonly status: ExplorerNodeStatus;
readonly confidence: BrainConfidence | null;
readonly confidence_value: number | null;
readonly applied_count: number;
readonly violated_count: number;
readonly pinned: boolean;
readonly retired_reason: string | null;
readonly last_evidence_at: string | null;
readonly backlink_count: number;
readonly memory_layer?: BrainPreference["memory_layer"];
readonly memory_branch?: string;
}
export interface ExplorerEdge {
readonly source: string;
readonly target: string;
readonly kind: ExplorerEdgeKind;
readonly relation?: string;
}
/**
* A file under `Brain/preferences/` or `Brain/retired/` that could not be
* parsed into a node.
*
* The collector used to skip these silently, on the reasoning that the
* doctor reports parse-level corruption. That reasoning holds for a VIEW
* and fails for an EXPORT: a graph missing a rule reads exactly like a
* vault that never had it, and the file that carries the answer is the one
* the reader no longer has. So the collector carries the failure out and
* the caller decides - which for every path that writes bytes outward is
* to refuse and name the file.
*/
export interface ExplorerParseFailure {
/** Vault-relative path, so the message names a file the operator can open. */
readonly path: string;
/** The parser's own message. Never flattened to "failed". */
readonly reason: string;
}
export interface ExplorerGraph {
readonly generated_at: string;
readonly schema_version: typeof EXPLORER_SCHEMA_VERSION;
readonly vault_basename: string;
readonly nodes: ReadonlyArray<ExplorerNode>;
readonly edges: ReadonlyArray<ExplorerEdge>;
/** Empty on a healthy vault; never absent, so a reader cannot miss it. */
readonly parse_failures: ReadonlyArray<ExplorerParseFailure>;
}
export interface CollectExplorerDataOptions {
/** Wall clock for `generated_at`. Defaults to `new Date()`. */
readonly now?: Date;
}
/**
* The nodes the privacy deny markers reserve, as the shared lanes'
* denial report: one list over BOTH node kinds, preference and retired,
* each named with its id and the frontmatter key that denied it.
*
* The same judgment the preference exporters make, read from the
* frontmatter of every `pref-*.md` under `Brain/preferences/` and every
* `ret-*.md` under `Brain/retired/`. It is a report beside the
* collector, not a filter inside it: {@link collectExplorerData} keeps
* every parsed node, because the live server is the operator's own view
* and a silently smaller graph there is the exact lie the parse-failure
* refusal exists to prevent. A caller that writes bytes for another
* reader narrows at its own seam and names what it left out.
*
* A file the parsers would refuse appears in
* {@link ExplorerGraph.parse_failures}, and every export caller refuses
* on that before reading this list - so a file with no readable
* frontmatter id can never match a node and is skipped here rather than
* guessed at.
*/
export function collectExplorerDenials(vault: string): ReadonlyArray<PreferenceDenial> {
const denied: PreferenceDenial[] = [];
const scan = (folder: MemoryFolder, prefix: string): void => {
for (const entry of memoryFolderPages(vault, folder, { regularFilesOnly: true })) {
if (!entry.name.endsWith(".md")) continue;
if (!entry.name.startsWith(prefix)) continue;
// `parseFrontmatter` cannot throw; a file it cannot read yields an
// empty map, which denies nothing - and the parse-failure refusal
// has already stopped the export before this list is read.
const [meta] = entry.read();
// A record the store holds in `private` is denied whatever its
// fields say (`store-record-visibility.ts`): fail closed.
const key = deniedReservationKey(meta) ?? (entry.private ? "visibility" : null);
if (key === null) continue;
const id = meta["id"];
if (typeof id !== "string") continue;
denied.push({ id, reason: PREFERENCE_DENY_REASON, denied_by: key });
}
};
scan("preferences", "pref-");
scan("retired", "ret-");
// Sorted like the shared collectors' denial lists: the report rides
// byte-deterministic surfaces (the stderr notice), and the explorer's
// own output ordering is stable by id.
denied.sort((a, b) => a.id.localeCompare(b.id));
return denied;
}
export function collectExplorerData(
vault: string,
opts: CollectExplorerDataOptions = {},
): ExplorerGraph {
// Build the backlink index first so node factories can stamp the
// final count in one pass — avoids the prior write-then-overwrite
// shape that left a transient `backlink_count: 0` on every node.
const backlinkIndex = buildBacklinkIndex(vault);
const countFor = (id: string): number => (backlinkIndex.get(id) ?? []).length;
const nodes: ExplorerNode[] = [];
const knownIds = new Set<string>();
const parseFailures: ExplorerParseFailure[] = [];
const recordFailure = (path: string, err: unknown): void => {
parseFailures.push({
path: relative(vault, path),
reason: err instanceof Error ? err.message : String(err),
});
};
for (const entry of memoryFolderPages(vault, "preferences", { regularFilesOnly: true })) {
if (!entry.name.endsWith(".md")) continue;
if (!entry.name.startsWith("pref-")) continue;
const path = entry.path;
try {
const pref = parsePreferenceMeta(entry.read()[0], path);
nodes.push(nodeFromPreference(pref, countFor(pref.id)));
knownIds.add(pref.id);
} catch (err) {
recordFailure(path, err);
}
}
for (const entry of memoryFolderPages(vault, "retired", { regularFilesOnly: true })) {
if (!entry.name.endsWith(".md")) continue;
if (!entry.name.startsWith("ret-")) continue;
const path = entry.path;
try {
const ret = parseRetiredMeta(entry.read()[0], path);
nodes.push(nodeFromRetired(ret, countFor(ret.id)));
knownIds.add(ret.id);
} catch (err) {
recordFailure(path, err);
}
}
parseFailures.sort((a, b) => a.path.localeCompare(b.path));
// Edges derived from the backlink index. The index maps
// `target → ref[]` where each ref carries `source` + `field`.
// Field name `supersedes` / `superseded_by` → kind `supersedes`;
// anything else (`evidenced_by`, `body`, …) → kind `wikilink`.
// Self-references and refs whose source is not a node (log
// files, signals) are filtered out.
const edges = deriveEdges(backlinkIndex, knownIds);
// 5. Stable order.
nodes.sort((a, b) => {
if (a.kind !== b.kind) return a.kind.localeCompare(b.kind);
return a.id.localeCompare(b.id);
});
edges.sort((a, b) => {
const s = a.source.localeCompare(b.source);
if (s !== 0) return s;
const t = a.target.localeCompare(b.target);
if (t !== 0) return t;
return a.kind.localeCompare(b.kind);
});
const now = opts.now ?? new Date();
return Object.freeze({
generated_at: now.toISOString(),
schema_version: EXPLORER_SCHEMA_VERSION,
vault_basename: basename(vault),
nodes: Object.freeze(nodes),
edges: Object.freeze(edges),
parse_failures: Object.freeze(parseFailures),
});
}
function nodeFromPreference(pref: BrainPreference, backlinkCount: number): ExplorerNode {
return {
id: pref.id,
kind: "preference",
topic: pref.topic,
scope: pref.scope ?? null,
principle: pref.principle,
status: statusOf(pref.status),
confidence: pref.confidence,
confidence_value: pref.confidence_value,
applied_count: pref.applied_count,
violated_count: pref.violated_count,
pinned: pref.pinned,
retired_reason: null,
last_evidence_at: pref.last_evidence_at,
backlink_count: backlinkCount,
...(pref.memory_layer !== undefined ? { memory_layer: pref.memory_layer } : {}),
...(pref.memory_branch !== undefined ? { memory_branch: pref.memory_branch } : {}),
};
}
function nodeFromRetired(ret: BrainRetired, backlinkCount: number): ExplorerNode {
return {
id: ret.id,
kind: "retired",
topic: ret.topic,
scope: ret.scope ?? null,
principle: ret.principle,
status: "retired",
confidence: ret.confidence ?? null,
confidence_value: ret.confidence_value ?? null,
applied_count: ret.applied_count,
violated_count: ret.violated_count,
pinned: ret.pinned ?? false,
retired_reason: ret.retired_reason,
last_evidence_at: ret.last_evidence_at,
backlink_count: backlinkCount,
...(ret.memory_layer !== undefined ? { memory_layer: ret.memory_layer } : {}),
...(ret.memory_branch !== undefined ? { memory_branch: ret.memory_branch } : {}),
};
}
function statusOf(s: BrainPreference["status"]): ExplorerNodeStatus {
switch (s) {
case BRAIN_PREFERENCE_STATUS.unconfirmed:
return "unconfirmed";
case BRAIN_PREFERENCE_STATUS.confirmed:
return "confirmed";
case BRAIN_PREFERENCE_STATUS.quarantine:
return "quarantine";
default:
// Forward-compat: future statuses fall back to the closest
// visual category (unconfirmed) rather than crashing.
return "unconfirmed";
}
}
// ---- Template rendering -------------------------------------------------
const TEMPLATE_PATH = resolve(
dirname(fileURLToPath(import.meta.url)),
"..",
"..",
"..",
"templates",
"brain-explorer.html",
);
/**
* Unique sentinel string substituted with the graph JSON. Must appear
* exactly once in the template; the renderer fails loudly otherwise so
* a stale template (missing or accidentally double-substituted marker)
* surfaces immediately instead of producing a half-rendered HTML file.
*/
const PLACEHOLDER = "__GRAPH_JSON__";
const VAULT_PATH_PLACEHOLDER = "__VAULT_PATH__";
let templateCache: string | undefined;
function loadTemplate(): string {
if (templateCache !== undefined) return templateCache;
let raw: string;
try {
raw = readFileSync(TEMPLATE_PATH, "utf8");
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
throw new Error(`Failed to load brain-explorer template from ${TEMPLATE_PATH}: ${msg}`, {
cause: err,
});
}
const first = raw.indexOf(PLACEHOLDER);
const last = raw.lastIndexOf(PLACEHOLDER);
if (first === -1) {
throw new Error(`brain-explorer.html template is missing the ${PLACEHOLDER} marker`);
}
if (first !== last) {
throw new Error(
`brain-explorer.html template carries ${PLACEHOLDER} more than once;` +
` substitution must be unique to keep the output deterministic`,
);
}
templateCache = raw;
return templateCache;
}
/**
* Substitute the placeholder with a single-line JSON serialisation of
* `graph`. Single-line keeps the export size lean — the rendered file
* routinely lands in a Drive backup or PR comment.
*
* Uses the function-form of `String.prototype.replace` to avoid `$&`
* / `$1` injection through the JSON body (principle bodies are free
* text and may contain `$`).
*/
export function renderExportedHtml(graph: ExplorerGraph, vaultPath?: string): string {
const json = JSON.stringify(graph).replace(/[<>&]/g, (c) => {
switch (c) {
case "<":
return "\\u003c";
case ">":
return "\\u003e";
case "&":
return "\\u0026";
default:
return c;
}
});
let html = loadTemplate().replace(PLACEHOLDER, () => json);
if (vaultPath) {
html = html.replace(
VAULT_PATH_PLACEHOLDER,
JSON.stringify(vaultPath).slice(1, -1).replace(/\//g, "\\/"),
);
} else {
html = html.replace(VAULT_PATH_PLACEHOLDER, "");
}
return html;
}
/** Test-only escape hatch: forget the cached template body. */
export function __resetExplorerTemplateCacheForTests(): void {
templateCache = undefined;
}
// ---- Live server -------------------------------------------------------
/**
* The Host names this server answers to, compared case-insensitively and
* with any port (or none). Everything else - an attacker-chosen rebinding
* name above all - is refused before any route runs.
*/
const LOOPBACK_HOST_NAMES: ReadonlySet<string> = new Set(["127.0.0.1", "localhost", "[::1]"]);
/**
* The Host header's name part, lowercased, with any port suffix removed:
* `127.0.0.1:7777` → `127.0.0.1`, `[::1]:7777` → `[::1]`, `LOCALHOST` →
* `localhost`. Answers null for a missing header (required on HTTP/1.1
* and this server's contract regardless) or a value whose name part is
* not one plain host token - several unbracketed colons, a malformed
* bracket form - none of which a loopback client sends.
*/
function loopbackHostOf(header: string | null): string | null {
if (header === null) return null;
const value = header.trim().toLowerCase();
if (value === "") return null;
if (value.startsWith("[")) {
const end = value.indexOf("]");
if (end === -1) return null;
const name = value.slice(0, end + 1);
const rest = value.slice(end + 1);
return rest === "" || rest.startsWith(":") ? name : null;
}
const colon = value.indexOf(":");
return colon === -1 ? value : value.slice(0, colon);
}
export interface LiveServerHandle {
readonly url: string;
readonly port: number;
close(): Promise<void>;
}
/**
* Additive options for the live server. `configPath` reaches the
* operator snapshot's search-index and profile lookups; a caller that
* passes nothing behaves exactly as before.
*/
export interface LiveServerOptions {
readonly configPath?: string;
}
/**
* Boot a loopback-only HTTP server that serves the explorer template,
* the graph JSON, the console counts (`/counts.json`, from
* {@link collectExplorerConsole}) and the operator snapshot
* (`/status.json`, from `buildOperatorSnapshot`). Each GET re-reads the
* vault — no caches, no watchers, no write endpoints. That includes the
* snapshot route: every GET re-runs the dry-run dream inside
* `buildOperatorSnapshot`, the most expensive read it composes, and
* that bounded read-only cost is accepted rather than papered over with
* a cache that would go stale silently. Binding to `127.0.0.1` is
* non-optional, and the bind alone is not a browser boundary - a
* DNS-rebinding page can resolve an attacker-chosen hostname to
* `127.0.0.1` and reach the routes same-origin - so every request whose
* `Host` header is not a loopback name (127.0.0.1, localhost, [::1],
* any port) is refused with a named 403 before any route runs.
*
* The JSON routes surface a failed read by name: a 500 whose body
* carries the error message, never an empty payload that could read as
* "no records".
*
* Throws when the port is unavailable; the CLI catches `EADDRINUSE`
* and turns it into a friendly exit message.
*/
export function buildLiveServer(
vault: string,
port: number,
options: LiveServerOptions = {},
): LiveServerHandle {
const configPath = options.configPath;
const snapshotOptions = configPath === undefined ? {} : { configPath };
const server = Bun.serve({
hostname: "127.0.0.1",
port,
async fetch(req) {
// Host allowlist (security review, finding 2): the loopback bind
// does not stop a rebinding browser, whose request arrives with an
// attacker-chosen Host. Loopback names only, any port; the Host
// header is required.
const hostName = loopbackHostOf(req.headers.get("host"));
if (hostName === null || !LOOPBACK_HOST_NAMES.has(hostName)) {
return jsonResponse(
JSON.stringify({
error:
"explorer: refusing a request whose Host header is not a loopback name " +
"(127.0.0.1, localhost, [::1], any port)",
}),
403,
);
}
// A loopback Host does not make a request the operator's own: any
// web page can send a blind cross-origin GET to 127.0.0.1 and make
// the server run the operator snapshot (a dry-run dream) even though
// the page cannot read the answer. The browser names a cross-origin
// sender in Sec-Fetch-Site and Origin; only the console's own page
// (same-origin) and a typed URL (none) are served.
const fetchSite = req.headers.get("sec-fetch-site");
const origin = req.headers.get("origin");
const ownOrigin = `http://${req.headers.get("host")}`;
if (
(fetchSite !== null && fetchSite !== "same-origin" && fetchSite !== "none") ||
(origin !== null && origin !== ownOrigin)
) {
return jsonResponse(
JSON.stringify({
error: "explorer: refusing a cross-origin request; open the console URL directly",
}),
403,
);
}
const url = new URL(req.url);
if (url.pathname === "/" || url.pathname === "/index.html") {
const graph = collectExplorerData(vault);
const html = renderExportedHtml(graph, vault);
return new Response(html, {
headers: { "content-type": "text/html; charset=utf-8" },
});
}
if (url.pathname === "/data.json") {
const graph = collectExplorerData(vault);
return new Response(JSON.stringify(graph), {
headers: { "content-type": "application/json; charset=utf-8" },
});
}
if (url.pathname === "/counts.json") {
try {
const counts = collectExplorerConsole(vault, snapshotOptions);
return jsonResponse(JSON.stringify(counts));
} catch (err) {
return errorResponse("/counts.json", err);
}
}
if (url.pathname === "/status.json") {
try {
const snapshot = await buildOperatorSnapshot(vault, snapshotOptions);
// The explorer serves browsers, so the store group answers the
// way the MCP surface answers below local reach: the writer
// lock PATH is withheld - it is a resolved host path the index
// may keep outside the vault - while every state field stays,
// and `state_files.search_index` still answers presence.
const store = { ...snapshot.store, writerLockPath: null };
// The probe's failure reason can embed the same resolved host
// path this field withholds (proper-lockfile's fs errors carry
// it verbatim), so the path - in both its spellings - is
// scrubbed from the reason too: the browser surface answers
// below local reach for the whole store group, not for one
// field of it.
if (store.writerLockReason !== null && snapshot.store.writerLockPath !== null) {
const index = snapshot.store.writerLockPath;
store.writerLockReason = redactRawOutput(store.writerLockReason, {
literals: sortedDistinctLiterals([index, `${index}.lock`]),
});
}
return jsonResponse(JSON.stringify({ ...snapshot, store }));
} catch (err) {
return errorResponse("/status.json", err);
}
}
return new Response("not found", { status: 404 });
},
});
// Bun's `Server.port` is typed as `number | undefined` because the
// server might bind to a Unix socket. The HTTP path above always
// resolves to a TCP port, so a missing value is a runtime
// contradiction worth surfacing explicitly.
const actualPort = server.port;
if (typeof actualPort !== "number") {
server.stop();
throw new Error("buildLiveServer: server bound without a TCP port");
}
return {
url: `http://127.0.0.1:${actualPort}/`,
port: actualPort,
close: async () => {
server.stop();
},
};
}
// ---- Helpers ------------------------------------------------------------
function jsonResponse(body: string, status = 200): Response {
return new Response(body, {
status,
headers: { "content-type": "application/json; charset=utf-8" },
});
}
/**
* A route's failure surfaces by name: the failing route and the error
* message travel in the body, so a broken section can never read as a
* healthy one that simply has no records.
*/
function errorResponse(route: string, err: unknown): Response {
const message = err instanceof Error ? err.message : String(err);
return jsonResponse(JSON.stringify({ error: `${route}: ${message}` }), 500);
}
function deriveEdges(index: BacklinkIndex, knownIds: ReadonlySet<string>): ExplorerEdge[] {
const edges: ExplorerEdge[] = [];
const seen = new Set<string>();
for (const [target, refs] of index) {
if (!knownIds.has(target)) continue;
for (const ref of refs) {
if (!knownIds.has(ref.source)) continue; // logs / signals
if (ref.source === target) continue;
const kind: ExplorerEdgeKind =
ref.field === "supersedes" || ref.field === "superseded_by" ? "supersedes" : "wikilink";
const relation = ref.relation;
const key = `${ref.source}\x00${target}\x00${kind}\x00${relation ?? ""}`;
if (seen.has(key)) continue;
seen.add(key);
edges.push({
source: ref.source,
target,
kind,
...(relation ? { relation } : {}),
});
}
}
return edges;
}