Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
9.3 KB
Raw Blame
/**
 * Unlinked-mentions scanner.
 *
 * Given a target Brain artifact id, walk every other preference /
 * retired note and emit each raw-text occurrence of the target's
 * title (or any frontmatter alias) that is NOT already inside a
 * `[[...]]` wikilink and NOT inside a fenced / inline code span.
 *
 * The matcher is purely structural:
 *
 *   - Word boundaries use Unicode codepoint classes (`\p{L}`,
 *     `\p{N}`); both edges of the match must sit against a
 *     non-letter, non-digit codepoint (or string edge).
 *   - Single-codepoint terms are rejected as too noisy.
 *   - No vocabulary list, no stopword set, no per-language data.
 *
 * The scanner is read-only and bounded by vault size; pure helper,
 * no I/O beyond the directory walk + per-file read.
 */

import { existsSync, readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";

import { memoryFromStore, storePageInPreferenceFolders, storePagesIn } from "../memory-pages.ts";
import { FRONTMATTER_RE, parseFrontmatter } from "../../vault.ts";
import { ownerScopeView } from "../owner-scope-view.ts";
import { UNFILTERED_ARTIFACT_REFS, type ArtifactRefView } from "../artifact-ref-view.ts";
import { brainDirs } from "../paths.ts";
import { maskCodeRegions } from "../wikilink.ts";
import { requireVaultAdapter, WIKILINK_AUDIT } from "../../vault-adapter/index.ts";

/** One unlinked-mention occurrence. */
export interface MentionRef {
  /** Source artifact id (basename without `.md`). */
  readonly source: string;
  /** 1-based line number in the source body. */
  readonly line: number;
  /** The literal term that matched (title or alias spelling). */
  readonly term: string;
  /** Line content with the match in situ (single line, untrimmed). */
  readonly contextSnippet: string;
}

export interface FindUnlinkedMentionsOptions {
  /**
   * Maximum number of mentions returned. Default `100`. Pagination is
   * not exposed; callers needing more should call with a larger cap.
   */
  readonly limit?: number;
  /**
   * Owner scope the mentions are filtered against
   * (a-label-is-not-a-boundary, U3). A mention carries its source
   * artifact's id AND the line of prose the term sits in, so an
   * unfiltered scan published an owner-private body verbatim. Absent /
   * `null` means no ownership filtering, which is what every existing
   * caller gets.
   */
  readonly ownerScope?: string | null;
  /**
   * The transport-reach view (`../reach-view.ts`). A mention carries its
   * source's id and a line of its body, and its `term` is the target's
   * title or alias, so both the target and every source are asked.
   * A withheld target answers as an absent one (no mentions). Absent
   * means no reach filtering.
   */
  readonly reachView?: ArtifactRefView;
}

const DEFAULT_LIMIT = 100;

/** Minimum codepoint count for a search term to count as a match. */
const MIN_TERM_CODEPOINTS = 2;

/** Replacement char used to mask brackets / code spans. */
const MASK_CHAR = " ";

const BRACKET_RE = /\[\[[^\]\n]+\]\]/g;
const WORD_CHAR_RE = /[\p{L}\p{N}]/u;

/**
 * Walk `Brain/preferences/` and `Brain/retired/` for unlinked
 * mentions of the target. Returns a frozen array capped at
 * `opts.limit`.
 */
export function findUnlinkedMentions(
  vaultOrNull: string | null,
  targetId: string,
  opts: FindUnlinkedMentionsOptions = {},
): ReadonlyArray<MentionRef> {
  const vault = requireVaultAdapter(vaultOrNull, WIKILINK_AUDIT.unlinkedMentions);
  const limit = opts.limit ?? DEFAULT_LIMIT;
  const dirs = brainDirs(vault);
  const reach = opts.reachView ?? UNFILTERED_ARTIFACT_REFS;
  if (!reach.visible(targetId)) return Object.freeze([]) as ReadonlyArray<MentionRef>;

  // Resolve the target's title + aliases from frontmatter.
  const terms = resolveSearchTerms(vault, targetId);
  if (terms.length === 0) return Object.freeze([]) as ReadonlyArray<MentionRef>;

  const owner = ownerScopeView(vault, opts.ownerScope ?? null);
  const view = (source: string): boolean => owner.visible(source) && reach.visible(source);
  const collected: MentionRef[] = [];
  // Store-only (task 2.10): the preference fold's records, as their pages.
  const fromStore = memoryFromStore(vault);
  const pagesOf = (dir: string): ReadonlyArray<MentionPage> =>
    fromStore
      ? storePagesIn(vault, dir).map((page) => ({ name: `${page.id}.md`, text: () => page.text }))
      : legacyPages(dir);
  scanDir(pagesOf(dirs.preferences), targetId, terms, collected, limit, view);
  if (collected.length < limit) {
    scanDir(pagesOf(dirs.retired), targetId, terms, collected, limit, view);
  }

  return Object.freeze(collected) as ReadonlyArray<MentionRef>;
}

function resolveSearchTerms(vault: string, targetId: string): ReadonlyArray<string> {
  const dirs = brainDirs(vault);
  // Store-only (task 2.10): the fold's record, as its page; no file is read.
  const fromStore = memoryFromStore(vault);
  const candidates = fromStore
    ? []
    : [join(dirs.preferences, `${targetId}.md`), join(dirs.retired, `${targetId}.md`)];
  let metaSource: Record<string, unknown> | null = null;
  if (fromStore) {
    // The same two folders the legacy lookup reads.
    const page = storePageInPreferenceFolders(vault, targetId);
    if (page !== null) metaSource = page.meta;
  }
  for (const c of candidates) {
    if (!existsSync(c)) continue;
    try {
      const [meta] = parseFrontmatter(c);
      metaSource = meta as Record<string, unknown>;
      break;
    } catch {
      continue;
    }
  }

  const terms: string[] = [];
  const seen = new Set<string>();
  const push = (raw: unknown): void => {
    if (typeof raw !== "string") return;
    const trimmed = raw.trim();
    if (trimmed.length === 0) return;
    if (Array.from(trimmed).length < MIN_TERM_CODEPOINTS) return;
    const key = trimmed.normalize("NFC").toLowerCase();
    if (seen.has(key)) return;
    seen.add(key);
    terms.push(trimmed);
  };

  if (metaSource) {
    push(metaSource["title"]);
    const aliases = metaSource["aliases"];
    if (Array.isArray(aliases)) for (const a of aliases) push(a);
  }
  // Fall back to the id itself when no title is declared. The id
  // already has at least two codepoints in every realistic case
  // (`pref-x`, `ret-y`).
  if (terms.length === 0) push(targetId);

  return terms;
}

/** One `.md` page of a folder: its file name and how to read its text. */
interface MentionPage {
  readonly name: string;
  readonly text: () => string;
}

function legacyPages(dir: string): ReadonlyArray<MentionPage> {
  if (!existsSync(dir)) return [];
  return readdirSync(dir).map((name) => ({
    name,
    text: () => readFileSync(join(dir, name), "utf8"),
  }));
}

function scanDir(
  pages: ReadonlyArray<MentionPage>,
  targetId: string,
  terms: ReadonlyArray<string>,
  out: MentionRef[],
  limit: number,
  visible: (source: string) => boolean,
): void {
  for (const { name, text: read } of pages) {
    if (out.length >= limit) return;
    if (!name.endsWith(".md")) continue;
    const source = name.slice(0, -".md".length);
    if (source === targetId) continue;
    // A mention discloses its source artifact's id and one line of its
    // body, so the source decides visibility before the file is read.
    if (!visible(source)) continue;
    let text: string;
    try {
      text = read();
    } catch {
      continue;
    }
    // Strip frontmatter before scanning the body.
    const body = stripFrontmatter(text);
    scanBody(body, source, terms, out, limit);
  }
}

function stripFrontmatter(text: string): string {
  const m = FRONTMATTER_RE.exec(text);
  if (!m) return text;
  return text.slice(m[0].length);
}

function scanBody(
  body: string,
  source: string,
  terms: ReadonlyArray<string>,
  out: MentionRef[],
  limit: number,
): void {
  // Mask wikilinks and code spans so matches inside them are
  // ignored. The mask preserves character offsets so line/column
  // bookkeeping stays exact - replace each masked region with a
  // run of MASK_CHAR of identical length.
  const masked = maskCodeRegions(maskRegions(body, [BRACKET_RE]), (s) =>
    MASK_CHAR.repeat(s.length),
  );
  const lines = masked.split("\n");
  const originalLines = body.split("\n");

  for (let i = 0; i < lines.length; i++) {
    if (out.length >= limit) return;
    const maskedLine = lines[i]!;
    const original = originalLines[i] ?? "";
    for (const term of terms) {
      const lowerLine = maskedLine.toLowerCase();
      const lowerTerm = term.toLowerCase();
      let from = 0;
      while (from <= lowerLine.length) {
        const idx = lowerLine.indexOf(lowerTerm, from);
        if (idx < 0) break;
        const before = idx > 0 ? maskedLine[idx - 1] : undefined;
        const after = maskedLine[idx + lowerTerm.length];
        if (isWordEdge(before) && isWordEdge(after)) {
          out.push(
            Object.freeze({
              source,
              line: i + 1,
              term,
              contextSnippet: original,
            }),
          );
          if (out.length >= limit) return;
        }
        from = idx + Math.max(1, lowerTerm.length);
      }
    }
  }
}

function isWordEdge(ch: string | undefined): boolean {
  if (ch === undefined) return true;
  return !WORD_CHAR_RE.test(ch);
}

function maskRegions(text: string, regexes: ReadonlyArray<RegExp>): string {
  let masked = text;
  for (const re of regexes) {
    masked = masked.replace(re, (s) => MASK_CHAR.repeat(s.length));
  }
  return masked;
}