Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
23.2 KB
Raw Blame
/**
 * §14 Brain explorer — data collector.
 *
 * Pure read. Walks `Brain/preferences/` and `Brain/retired/`, builds
 * a node list plus an edge list derived from:
 *
 *   1. Frontmatter `supersedes` / `superseded_by` (kind `supersedes`).
 *   2. Inline `[[wikilinks]]` anywhere in the file's frontmatter
 *      values and body, filtered to refs landing on another node
 *      (kind `wikilink`).
 *
 * Signal (`sig-*`) and log refs never appear in `edges` — the
 * explorer view is about the rule graph, not the audit trail.
 *
 * Output is deterministic: nodes sorted by `(kind, id)`, edges by
 * `(source, target, kind)`. The collector emits a frozen object so
 * the live HTTP handler and the static export branch cannot
 * accidentally mutate it.
 *
 * A file that cannot be parsed is REPORTED on the graph
 * ({@link ExplorerParseFailure}), never dropped. This surface both
 * displays the graph and exports it, and a graph silently missing a rule
 * is indistinguishable from a vault that never had one.
 *
 * The privacy deny markers reserve a node without refusing it: the
 * collector keeps every parsed node and
 * {@link collectExplorerDenials} reports which ones a SHARED output
 * must withhold - the live view is the operator's own, the export is
 * not, so the narrowing lives with the caller that writes bytes outward.
 */

import { readFileSync } from "node:fs";
import { basename, dirname, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";

import { buildBacklinkIndex, type BacklinkIndex } from "./backlinks.ts";
import { deniedReservationKey, PREFERENCE_DENY_REASON, type PreferenceDenial } from "./export.ts";
import { memoryFolderPages, type MemoryFolder } from "./memory-pages.ts";
import { parsePreferenceMeta, parseRetiredMeta } from "./preference.ts";
import {
  BRAIN_PREFERENCE_STATUS,
  type BrainConfidence,
  type BrainPreference,
  type BrainRetired,
} from "./types.ts";
import { collectExplorerConsole } from "./explorer-console.ts";
import { buildOperatorSnapshot } from "./operator-snapshot.ts";
import { redactRawOutput } from "../redactor.ts";
import { sortedDistinctLiterals } from "../secret-ref.ts";

export const EXPLORER_SCHEMA_VERSION = 1 as const;

export type ExplorerNodeKind = "preference" | "retired";
export type ExplorerNodeStatus = "unconfirmed" | "confirmed" | "quarantine" | "retired";
export type ExplorerEdgeKind = "supersedes" | "wikilink";

export interface ExplorerNode {
  readonly id: string;
  readonly kind: ExplorerNodeKind;
  readonly topic: string;
  readonly scope: string | null;
  readonly principle: string;
  readonly status: ExplorerNodeStatus;
  readonly confidence: BrainConfidence | null;
  readonly confidence_value: number | null;
  readonly applied_count: number;
  readonly violated_count: number;
  readonly pinned: boolean;
  readonly retired_reason: string | null;
  readonly last_evidence_at: string | null;
  readonly backlink_count: number;
  readonly memory_layer?: BrainPreference["memory_layer"];
  readonly memory_branch?: string;
}

export interface ExplorerEdge {
  readonly source: string;
  readonly target: string;
  readonly kind: ExplorerEdgeKind;
  readonly relation?: string;
}

/**
 * A file under `Brain/preferences/` or `Brain/retired/` that could not be
 * parsed into a node.
 *
 * The collector used to skip these silently, on the reasoning that the
 * doctor reports parse-level corruption. That reasoning holds for a VIEW
 * and fails for an EXPORT: a graph missing a rule reads exactly like a
 * vault that never had it, and the file that carries the answer is the one
 * the reader no longer has. So the collector carries the failure out and
 * the caller decides - which for every path that writes bytes outward is
 * to refuse and name the file.
 */
export interface ExplorerParseFailure {
  /** Vault-relative path, so the message names a file the operator can open. */
  readonly path: string;
  /** The parser's own message. Never flattened to "failed". */
  readonly reason: string;
}

export interface ExplorerGraph {
  readonly generated_at: string;
  readonly schema_version: typeof EXPLORER_SCHEMA_VERSION;
  readonly vault_basename: string;
  readonly nodes: ReadonlyArray<ExplorerNode>;
  readonly edges: ReadonlyArray<ExplorerEdge>;
  /** Empty on a healthy vault; never absent, so a reader cannot miss it. */
  readonly parse_failures: ReadonlyArray<ExplorerParseFailure>;
}

export interface CollectExplorerDataOptions {
  /** Wall clock for `generated_at`. Defaults to `new Date()`. */
  readonly now?: Date;
}

/**
 * The nodes the privacy deny markers reserve, as the shared lanes'
 * denial report: one list over BOTH node kinds, preference and retired,
 * each named with its id and the frontmatter key that denied it.
 *
 * The same judgment the preference exporters make, read from the
 * frontmatter of every `pref-*.md` under `Brain/preferences/` and every
 * `ret-*.md` under `Brain/retired/`. It is a report beside the
 * collector, not a filter inside it: {@link collectExplorerData} keeps
 * every parsed node, because the live server is the operator's own view
 * and a silently smaller graph there is the exact lie the parse-failure
 * refusal exists to prevent. A caller that writes bytes for another
 * reader narrows at its own seam and names what it left out.
 *
 * A file the parsers would refuse appears in
 * {@link ExplorerGraph.parse_failures}, and every export caller refuses
 * on that before reading this list - so a file with no readable
 * frontmatter id can never match a node and is skipped here rather than
 * guessed at.
 */
export function collectExplorerDenials(vault: string): ReadonlyArray<PreferenceDenial> {
  const denied: PreferenceDenial[] = [];
  const scan = (folder: MemoryFolder, prefix: string): void => {
    for (const entry of memoryFolderPages(vault, folder, { regularFilesOnly: true })) {
      if (!entry.name.endsWith(".md")) continue;
      if (!entry.name.startsWith(prefix)) continue;
      // `parseFrontmatter` cannot throw; a file it cannot read yields an
      // empty map, which denies nothing - and the parse-failure refusal
      // has already stopped the export before this list is read.
      const [meta] = entry.read();
      // A record the store holds in `private` is denied whatever its
      // fields say (`store-record-visibility.ts`): fail closed.
      const key = deniedReservationKey(meta) ?? (entry.private ? "visibility" : null);
      if (key === null) continue;
      const id = meta["id"];
      if (typeof id !== "string") continue;
      denied.push({ id, reason: PREFERENCE_DENY_REASON, denied_by: key });
    }
  };
  scan("preferences", "pref-");
  scan("retired", "ret-");
  // Sorted like the shared collectors' denial lists: the report rides
  // byte-deterministic surfaces (the stderr notice), and the explorer's
  // own output ordering is stable by id.
  denied.sort((a, b) => a.id.localeCompare(b.id));
  return denied;
}

export function collectExplorerData(
  vault: string,
  opts: CollectExplorerDataOptions = {},
): ExplorerGraph {
  // Build the backlink index first so node factories can stamp the
  // final count in one pass — avoids the prior write-then-overwrite
  // shape that left a transient `backlink_count: 0` on every node.
  const backlinkIndex = buildBacklinkIndex(vault);
  const countFor = (id: string): number => (backlinkIndex.get(id) ?? []).length;

  const nodes: ExplorerNode[] = [];
  const knownIds = new Set<string>();
  const parseFailures: ExplorerParseFailure[] = [];
  const recordFailure = (path: string, err: unknown): void => {
    parseFailures.push({
      path: relative(vault, path),
      reason: err instanceof Error ? err.message : String(err),
    });
  };

  for (const entry of memoryFolderPages(vault, "preferences", { regularFilesOnly: true })) {
    if (!entry.name.endsWith(".md")) continue;
    if (!entry.name.startsWith("pref-")) continue;
    const path = entry.path;
    try {
      const pref = parsePreferenceMeta(entry.read()[0], path);
      nodes.push(nodeFromPreference(pref, countFor(pref.id)));
      knownIds.add(pref.id);
    } catch (err) {
      recordFailure(path, err);
    }
  }

  for (const entry of memoryFolderPages(vault, "retired", { regularFilesOnly: true })) {
    if (!entry.name.endsWith(".md")) continue;
    if (!entry.name.startsWith("ret-")) continue;
    const path = entry.path;
    try {
      const ret = parseRetiredMeta(entry.read()[0], path);
      nodes.push(nodeFromRetired(ret, countFor(ret.id)));
      knownIds.add(ret.id);
    } catch (err) {
      recordFailure(path, err);
    }
  }
  parseFailures.sort((a, b) => a.path.localeCompare(b.path));

  // Edges derived from the backlink index. The index maps
  //    `target → ref[]` where each ref carries `source` + `field`.
  //    Field name `supersedes` / `superseded_by` → kind `supersedes`;
  //    anything else (`evidenced_by`, `body`, …) → kind `wikilink`.
  //    Self-references and refs whose source is not a node (log
  //    files, signals) are filtered out.
  const edges = deriveEdges(backlinkIndex, knownIds);

  // 5. Stable order.
  nodes.sort((a, b) => {
    if (a.kind !== b.kind) return a.kind.localeCompare(b.kind);
    return a.id.localeCompare(b.id);
  });
  edges.sort((a, b) => {
    const s = a.source.localeCompare(b.source);
    if (s !== 0) return s;
    const t = a.target.localeCompare(b.target);
    if (t !== 0) return t;
    return a.kind.localeCompare(b.kind);
  });

  const now = opts.now ?? new Date();
  return Object.freeze({
    generated_at: now.toISOString(),
    schema_version: EXPLORER_SCHEMA_VERSION,
    vault_basename: basename(vault),
    nodes: Object.freeze(nodes),
    edges: Object.freeze(edges),
    parse_failures: Object.freeze(parseFailures),
  });
}

function nodeFromPreference(pref: BrainPreference, backlinkCount: number): ExplorerNode {
  return {
    id: pref.id,
    kind: "preference",
    topic: pref.topic,
    scope: pref.scope ?? null,
    principle: pref.principle,
    status: statusOf(pref.status),
    confidence: pref.confidence,
    confidence_value: pref.confidence_value,
    applied_count: pref.applied_count,
    violated_count: pref.violated_count,
    pinned: pref.pinned,
    retired_reason: null,
    last_evidence_at: pref.last_evidence_at,
    backlink_count: backlinkCount,
    ...(pref.memory_layer !== undefined ? { memory_layer: pref.memory_layer } : {}),
    ...(pref.memory_branch !== undefined ? { memory_branch: pref.memory_branch } : {}),
  };
}

function nodeFromRetired(ret: BrainRetired, backlinkCount: number): ExplorerNode {
  return {
    id: ret.id,
    kind: "retired",
    topic: ret.topic,
    scope: ret.scope ?? null,
    principle: ret.principle,
    status: "retired",
    confidence: ret.confidence ?? null,
    confidence_value: ret.confidence_value ?? null,
    applied_count: ret.applied_count,
    violated_count: ret.violated_count,
    pinned: ret.pinned ?? false,
    retired_reason: ret.retired_reason,
    last_evidence_at: ret.last_evidence_at,
    backlink_count: backlinkCount,
    ...(ret.memory_layer !== undefined ? { memory_layer: ret.memory_layer } : {}),
    ...(ret.memory_branch !== undefined ? { memory_branch: ret.memory_branch } : {}),
  };
}

function statusOf(s: BrainPreference["status"]): ExplorerNodeStatus {
  switch (s) {
    case BRAIN_PREFERENCE_STATUS.unconfirmed:
      return "unconfirmed";
    case BRAIN_PREFERENCE_STATUS.confirmed:
      return "confirmed";
    case BRAIN_PREFERENCE_STATUS.quarantine:
      return "quarantine";
    default:
      // Forward-compat: future statuses fall back to the closest
      // visual category (unconfirmed) rather than crashing.
      return "unconfirmed";
  }
}

// ---- Template rendering -------------------------------------------------

const TEMPLATE_PATH = resolve(
  dirname(fileURLToPath(import.meta.url)),
  "..",
  "..",
  "..",
  "templates",
  "brain-explorer.html",
);

/**
 * Unique sentinel string substituted with the graph JSON. Must appear
 * exactly once in the template; the renderer fails loudly otherwise so
 * a stale template (missing or accidentally double-substituted marker)
 * surfaces immediately instead of producing a half-rendered HTML file.
 */
const PLACEHOLDER = "__GRAPH_JSON__";
const VAULT_PATH_PLACEHOLDER = "__VAULT_PATH__";

let templateCache: string | undefined;

function loadTemplate(): string {
  if (templateCache !== undefined) return templateCache;
  let raw: string;
  try {
    raw = readFileSync(TEMPLATE_PATH, "utf8");
  } catch (err) {
    const msg = err instanceof Error ? err.message : String(err);
    throw new Error(`Failed to load brain-explorer template from ${TEMPLATE_PATH}: ${msg}`, {
      cause: err,
    });
  }
  const first = raw.indexOf(PLACEHOLDER);
  const last = raw.lastIndexOf(PLACEHOLDER);
  if (first === -1) {
    throw new Error(`brain-explorer.html template is missing the ${PLACEHOLDER} marker`);
  }
  if (first !== last) {
    throw new Error(
      `brain-explorer.html template carries ${PLACEHOLDER} more than once;` +
        ` substitution must be unique to keep the output deterministic`,
    );
  }
  templateCache = raw;
  return templateCache;
}

/**
 * Substitute the placeholder with a single-line JSON serialisation of
 * `graph`. Single-line keeps the export size lean — the rendered file
 * routinely lands in a Drive backup or PR comment.
 *
 * Uses the function-form of `String.prototype.replace` to avoid `$&`
 * / `$1` injection through the JSON body (principle bodies are free
 * text and may contain `$`).
 */
export function renderExportedHtml(graph: ExplorerGraph, vaultPath?: string): string {
  const json = JSON.stringify(graph).replace(/[<>&]/g, (c) => {
    switch (c) {
      case "<":
        return "\\u003c";
      case ">":
        return "\\u003e";
      case "&":
        return "\\u0026";
      default:
        return c;
    }
  });
  let html = loadTemplate().replace(PLACEHOLDER, () => json);
  if (vaultPath) {
    html = html.replace(
      VAULT_PATH_PLACEHOLDER,
      JSON.stringify(vaultPath).slice(1, -1).replace(/\//g, "\\/"),
    );
  } else {
    html = html.replace(VAULT_PATH_PLACEHOLDER, "");
  }
  return html;
}

/** Test-only escape hatch: forget the cached template body. */
export function __resetExplorerTemplateCacheForTests(): void {
  templateCache = undefined;
}

// ---- Live server -------------------------------------------------------

/**
 * The Host names this server answers to, compared case-insensitively and
 * with any port (or none). Everything else - an attacker-chosen rebinding
 * name above all - is refused before any route runs.
 */
const LOOPBACK_HOST_NAMES: ReadonlySet<string> = new Set(["127.0.0.1", "localhost", "[::1]"]);

/**
 * The Host header's name part, lowercased, with any port suffix removed:
 * `127.0.0.1:7777` → `127.0.0.1`, `[::1]:7777` → `[::1]`, `LOCALHOST` →
 * `localhost`. Answers null for a missing header (required on HTTP/1.1
 * and this server's contract regardless) or a value whose name part is
 * not one plain host token - several unbracketed colons, a malformed
 * bracket form - none of which a loopback client sends.
 */
function loopbackHostOf(header: string | null): string | null {
  if (header === null) return null;
  const value = header.trim().toLowerCase();
  if (value === "") return null;
  if (value.startsWith("[")) {
    const end = value.indexOf("]");
    if (end === -1) return null;
    const name = value.slice(0, end + 1);
    const rest = value.slice(end + 1);
    return rest === "" || rest.startsWith(":") ? name : null;
  }
  const colon = value.indexOf(":");
  return colon === -1 ? value : value.slice(0, colon);
}

export interface LiveServerHandle {
  readonly url: string;
  readonly port: number;
  close(): Promise<void>;
}

/**
 * Additive options for the live server. `configPath` reaches the
 * operator snapshot's search-index and profile lookups; a caller that
 * passes nothing behaves exactly as before.
 */
export interface LiveServerOptions {
  readonly configPath?: string;
}

/**
 * Boot a loopback-only HTTP server that serves the explorer template,
 * the graph JSON, the console counts (`/counts.json`, from
 * {@link collectExplorerConsole}) and the operator snapshot
 * (`/status.json`, from `buildOperatorSnapshot`). Each GET re-reads the
 * vault — no caches, no watchers, no write endpoints. That includes the
 * snapshot route: every GET re-runs the dry-run dream inside
 * `buildOperatorSnapshot`, the most expensive read it composes, and
 * that bounded read-only cost is accepted rather than papered over with
 * a cache that would go stale silently. Binding to `127.0.0.1` is
 * non-optional, and the bind alone is not a browser boundary - a
 * DNS-rebinding page can resolve an attacker-chosen hostname to
 * `127.0.0.1` and reach the routes same-origin - so every request whose
 * `Host` header is not a loopback name (127.0.0.1, localhost, [::1],
 * any port) is refused with a named 403 before any route runs.
 *
 * The JSON routes surface a failed read by name: a 500 whose body
 * carries the error message, never an empty payload that could read as
 * "no records".
 *
 * Throws when the port is unavailable; the CLI catches `EADDRINUSE`
 * and turns it into a friendly exit message.
 */
export function buildLiveServer(
  vault: string,
  port: number,
  options: LiveServerOptions = {},
): LiveServerHandle {
  const configPath = options.configPath;
  const snapshotOptions = configPath === undefined ? {} : { configPath };
  const server = Bun.serve({
    hostname: "127.0.0.1",
    port,
    async fetch(req) {
      // Host allowlist (security review, finding 2): the loopback bind
      // does not stop a rebinding browser, whose request arrives with an
      // attacker-chosen Host. Loopback names only, any port; the Host
      // header is required.
      const hostName = loopbackHostOf(req.headers.get("host"));
      if (hostName === null || !LOOPBACK_HOST_NAMES.has(hostName)) {
        return jsonResponse(
          JSON.stringify({
            error:
              "explorer: refusing a request whose Host header is not a loopback name " +
              "(127.0.0.1, localhost, [::1], any port)",
          }),
          403,
        );
      }
      // A loopback Host does not make a request the operator's own: any
      // web page can send a blind cross-origin GET to 127.0.0.1 and make
      // the server run the operator snapshot (a dry-run dream) even though
      // the page cannot read the answer. The browser names a cross-origin
      // sender in Sec-Fetch-Site and Origin; only the console's own page
      // (same-origin) and a typed URL (none) are served.
      const fetchSite = req.headers.get("sec-fetch-site");
      const origin = req.headers.get("origin");
      const ownOrigin = `http://${req.headers.get("host")}`;
      if (
        (fetchSite !== null && fetchSite !== "same-origin" && fetchSite !== "none") ||
        (origin !== null && origin !== ownOrigin)
      ) {
        return jsonResponse(
          JSON.stringify({
            error: "explorer: refusing a cross-origin request; open the console URL directly",
          }),
          403,
        );
      }
      const url = new URL(req.url);
      if (url.pathname === "/" || url.pathname === "/index.html") {
        const graph = collectExplorerData(vault);
        const html = renderExportedHtml(graph, vault);
        return new Response(html, {
          headers: { "content-type": "text/html; charset=utf-8" },
        });
      }
      if (url.pathname === "/data.json") {
        const graph = collectExplorerData(vault);
        return new Response(JSON.stringify(graph), {
          headers: { "content-type": "application/json; charset=utf-8" },
        });
      }
      if (url.pathname === "/counts.json") {
        try {
          const counts = collectExplorerConsole(vault, snapshotOptions);
          return jsonResponse(JSON.stringify(counts));
        } catch (err) {
          return errorResponse("/counts.json", err);
        }
      }
      if (url.pathname === "/status.json") {
        try {
          const snapshot = await buildOperatorSnapshot(vault, snapshotOptions);
          // The explorer serves browsers, so the store group answers the
          // way the MCP surface answers below local reach: the writer
          // lock PATH is withheld - it is a resolved host path the index
          // may keep outside the vault - while every state field stays,
          // and `state_files.search_index` still answers presence.
          const store = { ...snapshot.store, writerLockPath: null };
          // The probe's failure reason can embed the same resolved host
          // path this field withholds (proper-lockfile's fs errors carry
          // it verbatim), so the path - in both its spellings - is
          // scrubbed from the reason too: the browser surface answers
          // below local reach for the whole store group, not for one
          // field of it.
          if (store.writerLockReason !== null && snapshot.store.writerLockPath !== null) {
            const index = snapshot.store.writerLockPath;
            store.writerLockReason = redactRawOutput(store.writerLockReason, {
              literals: sortedDistinctLiterals([index, `${index}.lock`]),
            });
          }
          return jsonResponse(JSON.stringify({ ...snapshot, store }));
        } catch (err) {
          return errorResponse("/status.json", err);
        }
      }
      return new Response("not found", { status: 404 });
    },
  });
  // Bun's `Server.port` is typed as `number | undefined` because the
  // server might bind to a Unix socket. The HTTP path above always
  // resolves to a TCP port, so a missing value is a runtime
  // contradiction worth surfacing explicitly.
  const actualPort = server.port;
  if (typeof actualPort !== "number") {
    server.stop();
    throw new Error("buildLiveServer: server bound without a TCP port");
  }
  return {
    url: `http://127.0.0.1:${actualPort}/`,
    port: actualPort,
    close: async () => {
      server.stop();
    },
  };
}

// ---- Helpers ------------------------------------------------------------

function jsonResponse(body: string, status = 200): Response {
  return new Response(body, {
    status,
    headers: { "content-type": "application/json; charset=utf-8" },
  });
}

/**
 * A route's failure surfaces by name: the failing route and the error
 * message travel in the body, so a broken section can never read as a
 * healthy one that simply has no records.
 */
function errorResponse(route: string, err: unknown): Response {
  const message = err instanceof Error ? err.message : String(err);
  return jsonResponse(JSON.stringify({ error: `${route}: ${message}` }), 500);
}

function deriveEdges(index: BacklinkIndex, knownIds: ReadonlySet<string>): ExplorerEdge[] {
  const edges: ExplorerEdge[] = [];
  const seen = new Set<string>();
  for (const [target, refs] of index) {
    if (!knownIds.has(target)) continue;
    for (const ref of refs) {
      if (!knownIds.has(ref.source)) continue; // logs / signals
      if (ref.source === target) continue;
      const kind: ExplorerEdgeKind =
        ref.field === "supersedes" || ref.field === "superseded_by" ? "supersedes" : "wikilink";
      const relation = ref.relation;
      const key = `${ref.source}\x00${target}\x00${kind}\x00${relation ?? ""}`;
      if (seen.has(key)) continue;
      seen.add(key);
      edges.push({
        source: ref.source,
        target,
        kind,
        ...(relation ? { relation } : {}),
      });
    }
  }
  return edges;
}