Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
18.2 KB
Raw Blame
/**
 * Backlink index over the Brain layer.
 *
 * A "backlink" is any reference from one Brain artifact to another's
 * id via the Obsidian wikilink form `[[<id>]]`. The index inverts this
 * graph: given a target id, return every source that points at it.
 *
 * Sources walked:
 *
 *   - `Brain/preferences/pref-*.md` — frontmatter (`supersedes`,
 *     `evidenced_by[]`) and the body prose (any embedded `[[...]]`).
 *   - `Brain/retired/ret-*.md` — same fields, plus `superseded_by` and
 *     `retired_by`.
 *   - `Brain/log/<YYYY-MM-DD>.md` — every event whose payload references
 *     a preference (`preference`, `signal`, `superseded_by`, `run_id`,
 *     etc.). The log is append-only so we don't lose history.
 *
 * The index is recomputed on demand (no on-disk cache). The cost is
 * O(N+L) parse work per build where N is preferences+retired+signals
 * and L is total log entries. For typical vaults this is a small
 * fraction of `dream`'s cost; a smarter cache can land later if a
 * profile shows it pays.
 *
 * Pure read. `brain_doctor` is the surface that flags malformed
 * artifacts, not this aggregator — but an artifact this collector could
 * not parse is REPORTED here rather than skipped in silence, because a
 * dropped source produces a confidently wrong `count: 0` and the caller
 * has no way to tell that from a genuine zero
 * (a-label-is-not-a-boundary, U3; recon defect D4).
 */

import { existsSync, readdirSync } from "node:fs";
import { join } from "node:path";

import { memoryFromStore, storePagesIn } from "./memory-pages.ts";
import { parseFrontmatter } from "../vault.ts";
import { ownerScopeView } from "./owner-scope-view.ts";
import { relationFromFrontmatterField } from "../graph/relation-vocab.ts";
import { normalizeRelationTarget } from "../graph/frontmatter-relations.ts";
import { buildAliasIndex } from "./link-graph/alias-index.ts";
import { extractWikilinkRichBodies, parseWikilinkRich } from "./link-graph/parse-wikilink.ts";
import { storeLogDays } from "./log-days.ts";
import { withoutPrivateEntries } from "./log-events-at-reach.ts";
import { listLogShardFiles, readLogDay } from "./log-jsonl.ts";
import { brainDirs } from "./paths.ts";
import { normalizeDerivedKeys } from "./preference.ts";
import { normaliseWikilinkTarget } from "./wikilink.ts";

// ----- Public types --------------------------------------------------------

export type BacklinkSourceKind = "preference" | "retired" | "signal" | `log-${string}`;

export interface BacklinkRef {
  /** Source id (basename without `.md`, e.g. `pref-foo`, `ret-bar`, `sig-2026-05-14-baz`). */
  readonly source: string;
  /** Where the reference lives. `log-<kind>` for log entries (e.g. `log-apply-evidence`). */
  readonly sourceKind: BacklinkSourceKind;
  /** Field name carrying the reference (`principle`, `supersedes`, body text, etc.). */
  readonly field: string;
  /** ISO-8601 timestamp for log entries; absent for preference/retired sources. */
  readonly timestamp?: string;
  /**
   * Heading-anchor text from `[[target#Heading]]` (v0.10.17+). Present
   * only when the wikilink carried a heading anchor; absent for plain
   * targets and for block references.
   */
  readonly targetAnchor?: string;
  /**
   * Block-id text from `[[target#^abc]]` (v0.10.17+). Present only
   * when the wikilink carried a block anchor; absent otherwise. The
   * `^` sigil is stripped; just the bare id is recorded.
   */
  readonly targetBlock?: string;
  /**
   * When the wikilink was written via an alias declared in the
   * target's frontmatter `aliases:` array (v0.10.17+), the alias
   * string the linker actually typed. `source` still keys against the
   * canonical id; this field surfaces the alias spelling for
   * downstream consumers (digest, doctor, concept synthesis) that
   * want to show how the link was phrased.
   */
  readonly aliasSource?: string;
  /**
   * Semantic relation type (v3 / typed graph semantics) when the
   * carrying frontmatter field is a known relation
   * (`related` / `extends` / `contradicts` / `superseded_by`). Absent
   * for body wikilinks and non-relation fields (`evidenced_by`,
   * `supersedes`, `retired_by`, …). Classified via the single
   * vocabulary boundary in src/core/graph/relation-vocab.ts.
   */
  readonly relation?: string;
}

/**
 * One artifact whose references could not be collected, and why.
 *
 * `source` is the artifact id, never a host path: this record crosses the
 * MCP boundary through `brain_backlinks` and the contract at
 * `src/mcp/tools.ts:94-119` keeps absolute paths out of model context.
 */
export interface BacklinkParseFailure {
  /** Source id (basename without `.md`), as {@link BacklinkRef.source} spells it. */
  readonly source: string;
  /**
   * Which collector was reading it, in the SAME vocabulary
   * {@link BacklinkRef.sourceKind} uses - a failure and a ref describe
   * the same artifact, so a second spelling of "which lane" would be a
   * parallel vocabulary over one population. A log shard that cannot be
   * read has no event type to name, so it reports {@link LOG_READ_KIND}.
   */
  readonly sourceKind: BacklinkSourceKind;
  /** The failure's own message, with no host path in it. */
  readonly reason: string;
}

/**
 * `sourceKind` for a log shard whose entries could not be read at all.
 *
 * The per-ref form is `log-<eventType>`, and an unreadable shard has no
 * events to take a type from; this names the read itself instead, inside
 * the same `log-` namespace so no consumer needs a second branch.
 */
const LOG_READ_KIND = "log-read" as const;

/**
 * Frozen target → refs map, plus the artifacts the walk could not read.
 *
 * The failures ride on the index rather than in a second return value so
 * that the nine existing call sites keep compiling unchanged; the ones
 * that must report them read {@link BacklinkIndex.unparsed} and the rest
 * carry on using the map.
 */
export interface BacklinkIndex extends ReadonlyMap<string, ReadonlyArray<BacklinkRef>> {
  /**
   * Artifacts skipped because they could not be parsed.
   *
   * A `count: 0` beside a NON-EMPTY list here is not a measurement, and
   * a surface reporting the count must say so — the same distinction
   * `listDanglingTargets` draws between "measured zero" and "could not
   * measure". Empty on a healthy vault, so a clean payload is unchanged.
   */
  readonly unparsed: ReadonlyArray<BacklinkParseFailure>;
}

// ----- Public API ----------------------------------------------------------

/**
 * Build the inverted reference index for the current Brain state.
 *
 * The returned map is frozen and each entry's array is frozen too —
 * callers cannot mutate the shared index. Recompute by calling this
 * function again; there is no incremental update path on purpose.
 *
 * `ownerScope` (optional, second) drops every ref whose SOURCE artifact
 * the scope may not see, and drops it from `unparsed` too — an artifact
 * whose owner is unknowable is somebody else's, so naming it in a failure
 * report would leak exactly what the filter withheld. Absent (every
 * existing call site) means no ownership filtering at all.
 */
export function buildBacklinkIndex(
  vault: string,
  ownerScope?: string | null,
  opts: { readonly withholdsPrivate?: boolean } = {},
): BacklinkIndex {
  const dirs = brainDirs(vault);
  const view = ownerScopeView(vault, ownerScope ?? null);
  const aliasIndex = buildAliasIndex(vault);
  const unparsed: BacklinkParseFailure[] = [];
  const map = new Map<string, BacklinkRef[]>();
  // Dedup key: `<source>\x00<canonical>\x00<anchor>\x00<block>`.
  // Anchor + block are part of the dedup key so two refs to
  // different anchors of the same target keep both entries (Unit 3
  // requirement).
  const seen = new Set<string>();

  const push = (target: string, ref: BacklinkRef): void => {
    // The SOURCE decides visibility: a ref names the artifact that wrote
    // it, so publishing one publishes that artifact's id.
    if (!view.visible(ref.source)) return;
    const parsed = parseWikilinkRich(target);
    const bare = parsed.target;
    if (!bare) return;

    // Resolve via the alias index. Lookup key is NFC + lower-case
    // (matches the key normalisation `buildAliasIndex` emits). The
    // alias resolves to a different canonical id only when the
    // alias entry actually maps somewhere else.
    const aliasKey = bare.normalize("NFC").toLowerCase();
    let canonicalId = normaliseWikilinkTarget(bare);
    let aliasSource: string | undefined;
    const aliasHit = aliasIndex.get(aliasKey);
    if (aliasHit && aliasHit !== canonicalId) {
      aliasSource = bare;
      canonicalId = aliasHit;
    }
    if (!canonicalId || canonicalId === ref.source) return;

    const key = `${ref.source}\x00${canonicalId}\x00${parsed.anchor ?? ""}\x00${parsed.block ?? ""}`;
    if (seen.has(key)) return;
    seen.add(key);

    const enriched: BacklinkRef = {
      ...ref,
      ...(parsed.anchor !== undefined ? { targetAnchor: parsed.anchor } : {}),
      ...(parsed.block !== undefined ? { targetBlock: parsed.block } : {}),
      ...(aliasSource !== undefined ? { aliasSource } : {}),
    };

    const arr = map.get(canonicalId);
    if (arr) arr.push(enriched);
    else map.set(canonicalId, [enriched]);
  };

  const fail = (row: BacklinkParseFailure): void => {
    if (!view.visible(row.source)) return;
    unparsed.push(Object.freeze(row));
  };

  // Store-only (task 2.10): the preferences and signals are the folds'
  // records, read as the pages they would be; the log is not one of them.
  const fromStore = memoryFromStore(vault);
  const pagesOf = (dir: string): ReadonlyArray<SourcePage> =>
    fromStore ? storeSourcePages(vault, dir) : legacySourcePages(dir);
  collectPreferences(pagesOf(dirs.preferences), "preference", push, fail);
  collectPreferences(pagesOf(dirs.retired), "retired", push, fail);
  collectSignals(pagesOf(dirs.inbox), push, fail);
  collectSignals(pagesOf(dirs.processed), push, fail);
  collectSignals(pagesOf(dirs.archived), push, fail);
  collectLog(vault, dirs.log, push, fail, opts.withholdsPrivate === true);

  // Freeze each entry's array so downstream callers can't mutate the
  // shared index.
  const frozen = new Map<string, ReadonlyArray<BacklinkRef>>();
  for (const [k, v] of map) frozen.set(k, Object.freeze(v));
  return Object.assign(frozen, { unparsed: Object.freeze(unparsed) });
}

/** The failure's own message, never the host path it happened at. */
function failureReason(err: unknown, path: string): string {
  const raw = err instanceof Error ? err.message : String(err);
  return raw.split(path).join("").replaceAll(" ()", "").trim();
}

/**
 * Convenience: count of inbound references for one target id. Equal to
 * `(index.get(target) ?? []).length`. Kept as a named helper so call
 * sites that only need the count don't fight `undefined`.
 */
export function backlinkCount(index: BacklinkIndex, target: string): number {
  const norm = normaliseWikilinkTarget(target);
  return index.get(norm)?.length ?? 0;
}

// ----- Collectors ----------------------------------------------------------

/** One page a collector reads: its file name, its path, and how to read it. */
interface SourcePage {
  readonly name: string;
  readonly full: string;
  readonly read: () => readonly [Record<string, unknown>, string];
}

/** The `.md` files of `dir`, as the collectors always listed them. */
function legacySourcePages(dir: string): ReadonlyArray<SourcePage> {
  if (!existsSync(dir)) return [];
  return readdirSync(dir)
    .filter((name) => name.endsWith(".md"))
    .map((name) => {
      const full = join(dir, name);
      return { name, full, read: () => parseFrontmatter(full) };
    });
}

/** The store's records whose page sat in `dir`, read as those pages. */
function storeSourcePages(vault: string, dir: string): ReadonlyArray<SourcePage> {
  return storePagesIn(vault, dir).map((page) => ({
    name: `${page.id}.md`,
    full: page.path,
    read: () => [page.meta, page.body] as const,
  }));
}

function collectPreferences(
  pages: ReadonlyArray<SourcePage>,
  kind: "preference" | "retired",
  push: (target: string, ref: BacklinkRef) => void,
  fail: (row: BacklinkParseFailure) => void,
): void {
  for (const { name, full, read } of pages) {
    const source = name.slice(0, -".md".length);
    // Single read: pull both frontmatter and body in one pass so we
    // don't pay 2× readFileSync per preference on every index build.
    // Field extraction is defensive — schema enforcement is the
    // doctor's job, not the backlink scanner's.
    let meta: Record<string, unknown>;
    let body: string;
    try {
      const [rawMeta, rawBody] = read();
      // Collapse legacy / `_`-prefixed Group C keys so this
      // collector reads the same shape regardless of which the file
      // was written in. Collision aborts the row (caller's job to
      // flag via `brain_doctor`), but the rest of the index keeps
      // building.
      meta = normalizeDerivedKeys(rawMeta);
      body = rawBody;
    } catch (err) {
      // The row still aborts - the fields cannot be read - but it aborts
      // LOUDLY. `normalizeDerivedKeys` throws on the un-prefixed Group C
      // shape a legacy vault is full of, and the silent `continue` this
      // replaced produced an index missing every preference-sourced ref
      // with no diagnostic anywhere.
      fail({ source, sourceKind: kind, reason: failureReason(err, full) });
      continue;
    }

    const evidenced = meta["evidenced_by"];
    if (Array.isArray(evidenced)) {
      for (const e of evidenced) {
        if (typeof e === "string") {
          push(e, { source, sourceKind: kind, field: "evidenced_by" });
        }
      }
    }
    const supersedes = meta["supersedes"];
    if (typeof supersedes === "string" && supersedes.length > 0) {
      push(supersedes, { source, sourceKind: kind, field: "supersedes" });
    }
    if (kind === "retired") {
      const retiredBy = meta["retired_by"];
      if (typeof retiredBy === "string" && retiredBy.length > 0) {
        push(retiredBy, { source, sourceKind: kind, field: "retired_by" });
      }
    }
    // Typed semantic relations (v3): map known relation frontmatter
    // fields (related / extends / contradicts / superseded_by) to a
    // relation type via the single vocabulary boundary. Covers any
    // preference or retired artifact, generalising the prior
    // retired-only `superseded_by` handling. Runs before the body
    // wikilink pass so the relation-tagged ref wins dedup over a bare
    // body reference to the same target.
    for (const [field, value] of Object.entries(meta)) {
      const relation = relationFromFrontmatterField(field);
      if (!relation) continue;
      const list = Array.isArray(value) ? value : typeof value === "string" && value ? [value] : [];
      for (const t of list) {
        if (typeof t !== "string" || t.length === 0) continue;
        // The lightweight frontmatter parser can mangle `[[id]]` into
        // `[id]`; recover the bare target before pushing.
        const target = normalizeRelationTarget(t);
        if (target) push(target, { source, sourceKind: kind, field, relation });
      }
    }
    for (const body0 of extractWikilinkRichBodies(body)) {
      push(body0, { source, sourceKind: kind, field: "body" });
    }
  }
}

function collectSignals(
  pages: ReadonlyArray<SourcePage>,
  push: (target: string, ref: BacklinkRef) => void,
  fail: (row: BacklinkParseFailure) => void,
): void {
  for (const { name, full, read } of pages) {
    if (!name.startsWith("sig-")) continue;
    const source = name.slice(0, -".md".length);
    try {
      const [meta, body] = read();
      const sources = meta["source"];
      const list = Array.isArray(sources) ? sources : sources ? [String(sources)] : [];
      for (const t of list) push(t, { source, sourceKind: "signal", field: "source" });
      for (const body0 of extractWikilinkRichBodies(body)) {
        push(body0, { source, sourceKind: "signal", field: "body" });
      }
    } catch (err) {
      fail({ source, sourceKind: "signal", reason: failureReason(err, full) });
      continue;
    }
  }
}

function collectLog(
  vault: string,
  dir: string,
  push: (target: string, ref: BacklinkRef) => void,
  fail: (row: BacklinkParseFailure) => void,
  withholdsPrivate: boolean,
): void {
  // Store-only (task 2.10): the log is the store's `log-event` records.
  const stored = storeLogDays(vault);
  if (stored === null && !existsSync(dir)) return;
  // Shard-aware (Memory Integrity Suite): merged per-day reads. List the
  // log directory once and share it across every date's readLogDay call
  // instead of one readdirSync+sort per date (this loop can span a
  // vault's entire log history).
  const shards = stored === null ? listLogShardFiles(vault) : [];
  const storedDays = new Map((stored ?? []).map((d) => [d.date, d.entries] as const));
  const dates =
    stored === null
      ? [...new Set(shards.map((f) => f.date))].toSorted()
      : stored.map((d) => d.date);
  for (const date of dates) {
    const source = `log-${date}`;
    let entries;
    try {
      entries = stored === null ? readLogDay(vault, date, shards).entries : storedDays.get(date)!;
    } catch (err) {
      fail({ source, sourceKind: LOG_READ_KIND, reason: failureReason(err, dir) });
      continue;
    }
    // Below local reach an event of the store's `private` partition
    // references nothing (`withholdsPrivate`).
    for (const e of withoutPrivateEntries(entries, withholdsPrivate)) {
      // Walk every payload value, pulling wikilink targets out of
      // scalar fields and string arrays alike. We don't whitelist
      // which fields can carry references — the payload key becomes
      // the `field` so callers can filter per-event-kind downstream.
      //
      // A payload value carries a wikilink either as the `[[...]]`
      // form or as bare text that happens to be a Brain id
      // (`pref-...`, `ret-...`, `sig-...`, `dream-...`). Both shapes
      // route through the same push.
      for (const [field, value] of Object.entries(e.body)) {
        const values = Array.isArray(value) ? value : [value];
        for (const v of values) {
          if (typeof v !== "string") continue;
          if (!v.startsWith("[[") && !/^(pref|ret|sig|dream)-/.test(v)) continue;
          push(v, {
            source,
            sourceKind: `log-${e.eventType}`,
            field,
            timestamp: e.timestamp,
          });
        }
      }
    }
  }
}