Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
12.0 KB
Raw Blame
/**
 * Atomic-fact decomposition (t_cbd22536): turn session/turn prose into
 * discrete single-sentence assertions - deterministically, with zero
 * model calls. Markdown structure does the heavy lifting (headings
 * carry context, list items are natural assertion units) and a
 * conservative sentence splitter with an abbreviation guard handles
 * paragraphs. The same precision discipline as `fact-extract.ts`:
 * fenced/inline code, quoted lines, and frontmatter never produce
 * assertions, because a hallucinated fact pollutes memory while a
 * missed one costs nothing.
 */

import {
  MIN_ANCHOR_FORM_LENGTH,
  entityMatchForms,
  matchFormVariants,
  matchVariantsOccurIn,
  normalizeEntityName,
  type EntityLexiconHandle,
  type ValidateEntityLabelOptions,
} from "./entities/canonical.ts";
import { ENTITY_STATUS_SCOPE, entityStatusInScope } from "./entities/status-scope.ts";

/** Cap on one assertion's text length. */
export const MAX_ASSERTION_CHARS = 300;

/** Fragments shorter than this many characters are noise, not facts. */
const MIN_ASSERTION_CHARS = 15;

export type AtomicAssertionKind = "sentence" | "list_item";

/** The slice of a registry entity the anchorer needs. */
export interface AtomicEntityLike {
  readonly id: string;
  readonly name: string;
  readonly aliases: ReadonlyArray<string>;
  readonly status: string;
}

export interface AtomicAssertion {
  /** Whitespace-collapsed assertion text, length-capped. */
  readonly text: string;
  /** 1-based line number in the original input. */
  readonly line: number;
  /** Enclosing heading texts, outermost first. */
  readonly headingPath: ReadonlyArray<string>;
  readonly kind: AtomicAssertionKind;
  /** Canonical entity ids anchored in this assertion, sorted. */
  readonly entities: ReadonlyArray<string>;
}

export interface DecomposeOptions {
  /** Registry entities to anchor against (active only). */
  readonly entities?: ReadonlyArray<AtomicEntityLike>;
  /**
   * Label-gate options (denylist, store-learned lexicon) threaded into the
   * shared anchoring kernel (recall-quality A2). Vault-aware callers
   * resolve the handle through `resolveEntityLexicon` (recall-quality M6);
   * null or absent keeps every verdict byte-identical to the pre-lexicon
   * behavior.
   */
  readonly lexicon?: EntityLexiconHandle | null;
}

// Abbreviations a period may follow without ending the sentence.
// Lowercased, period-free. Single letters (initials) are handled
// separately by the splitter.
const ABBREVIATIONS = new Set([
  "e.g",
  "i.e",
  "etc",
  "vs",
  "cf",
  "ca",
  "approx",
  "dr",
  "mr",
  "mrs",
  "ms",
  "prof",
  "st",
  "no",
  "fig",
  "min",
  "max",
]);

const HEADING_RE = /^(#{1,6})\s+(.*)$/;
const LIST_ITEM_RE = /^\s*(?:[-*+]|\d{1,3}[.)])\s+(.+)$/;

function collapse(span: string): string {
  return span.replace(/\s+/g, " ").trim().slice(0, MAX_ASSERTION_CHARS);
}

/**
 * Split one paragraph into sentences. A boundary is `.`/`!`/`?`
 * followed by whitespace and an upper-case/digit/quote opener -
 * EXCEPT after a known abbreviation, a single-letter initial, or a
 * digit-dot-digit number (versions, decimals never contain
 * whitespace, so those survive by construction).
 */
export function splitSentences(text: string): string[] {
  const out: string[] = [];
  let start = 0;
  for (let i = 0; i < text.length; i++) {
    const ch = text[i]!;
    if (ch !== "." && ch !== "!" && ch !== "?") continue;
    const next = text[i + 1];
    if (next === undefined) break;
    if (!/\s/.test(next)) continue;
    const after = text.slice(i + 1).trimStart();
    if (after === "" || !/^[A-Z0-9"'([]/.test(after)) continue;
    if (ch === ".") {
      // Word immediately before the period, lowercased, period-free
      // prefix kept (so "e.g" matches as written in ABBREVIATIONS).
      const before = text.slice(start, i);
      const m = /(\S+)$/.exec(before);
      const word = (m?.[1] ?? "").toLowerCase();
      if (ABBREVIATIONS.has(word) || ABBREVIATIONS.has(word.replace(/\.$/, ""))) continue;
      if (/^[a-z]$/i.test(word)) continue; // single-letter initial
    }
    out.push(text.slice(start, i + 1));
    start = i + 1;
  }
  const tail = text.slice(start);
  if (tail.trim() !== "") out.push(tail);
  return out.map((s) => s.trim()).filter((s) => s !== "");
}

/**
 * The occurrence-anchoring kernel (truth-correctable-time-aware,
 * task 4): the quality-gated normalized match forms that occur in
 * `text`. Candidate labels pass through the quality gate
 * (sanitize decoration, then normalize, drop structurally-invalid
 * forms) before matching, and forms shorter than
 * {@link MIN_ANCHOR_FORM_LENGTH} never participate. Since F3
 * (note-change truth maintenance) the occurrence test itself is
 * separator-tolerant: both the form AND the haystack fold through
 * {@link matchFormVariants} to their canonical (space) joins - runs of
 * space, hyphen and underscore are interchangeable, a folder-prefixed
 * form also matches at a path-segment boundary - and every occurrence
 * must have a clean leading edge (a no-space-script neighbour suspends
 * the requirement), so the fold never plants a mid-word hit nothing
 * anchored before (`canada` never anchors `ada`). The returned forms are
 * still the gate's normalized forms, never the expanded variants. Pure:
 * no I/O, no registry - so this layer never resolves the lexicon itself.
 * The one kernel every occurrence-anchoring caller shares - the
 * assertion decomposer and the agent-stated claim verdicts both anchor
 * through it, and each vault-aware caller resolves the SAME store-learned
 * lexicon through `resolveEntityLexicon` and threads it in here
 * (recall-quality M6), so no path can drift back to anchoring a form
 * the creation gate refuses. `opts` threads the admission vocabularies
 * (denylist, store-learned lexicon) into the gate; with the handle
 * null (no fresh artifact) every verdict is byte-identical to the
 * pre-lexicon behavior.
 */
export function anchorEntityForms(
  text: string,
  forms: ReadonlyArray<string>,
  opts: ValidateEntityLabelOptions = {},
): ReadonlyArray<string> {
  const haystack = normalizeEntityName(text);
  // The haystack folds to its one canonical-join variant ONCE per call,
  // not once per form: the same hoist entityAnchors makes, so a decompose
  // pass over an assertion x entity grid tokenizes the text once per
  // call instead of per form per entity. Identical verdicts -
  // `matchFormOccursIn` is exactly this comparison with the expansion
  // inlined.
  const haystackVariants = matchFormVariants(haystack);
  return Object.freeze(
    entityMatchForms(forms, opts).filter(
      (f) =>
        f.length >= MIN_ANCHOR_FORM_LENGTH &&
        matchVariantsOccurIn(matchFormVariants(f), haystackVariants),
    ),
  );
}

/**
 * Whole-name equality over the same quality-gated, variant-folded match
 * forms the occurrence kernel anchors with - the resolution kernel for a
 * NAME that must name one entity and nothing looser. Counterparty name
 * resolution asks "who IS this name", not "does this name mention an
 * entity": a name that merely contains a form ("the Ada Lovelace team")
 * is a different party, so the pledge intake resolves names through this
 * predicate and not through {@link anchorEntityForms}. The same shape
 * gate applies: forms below {@link MIN_ANCHOR_FORM_LENGTH} never
 * participate. Pure: no I/O, no registry.
 */
export function entityNameEqualsAnchorForm(
  name: string,
  forms: ReadonlyArray<string>,
  opts: ValidateEntityLabelOptions = {},
): boolean {
  const nameVariants = matchFormVariants(normalizeEntityName(name));
  if (nameVariants.length === 0) return false;
  return entityMatchForms(forms, opts).some(
    (form) =>
      form.length >= MIN_ANCHOR_FORM_LENGTH &&
      matchFormVariants(form).some((variant) => nameVariants.includes(variant)),
  );
}

/**
 * Anchor an assertion against active registry entities, sorted ids. Labels
 * pass through the quality gate (sanitise decoration, then normalise, drop
 * structurally-invalid forms) before matching - the same shared kernel the
 * fact-extract path uses. Pure: no I/O, so a junk stored label is simply
 * skipped (the fact-extract path is the one that logs the skip).
 */
function anchorEntities(
  text: string,
  entities: ReadonlyArray<AtomicEntityLike>,
  opts: ValidateEntityLabelOptions = {},
): string[] {
  if (entities.length === 0) return [];
  // The text tokenizes ONCE for the whole entity loop (M2, PR 268
  // review): folding the haystack once per entity through
  // `anchorEntityForms` re-derived the identical variant per iteration.
  // The verdict is the same per-entity comparison, minus the re-fold.
  const haystackVariants = matchFormVariants(normalizeEntityName(text));
  const ids: string[] = [];
  for (const entity of entities) {
    if (!entityStatusInScope(entity.status, ENTITY_STATUS_SCOPE.canonical)) continue;
    const anchored = entityMatchForms([entity.name, ...entity.aliases], opts).some(
      (f) =>
        f.length >= MIN_ANCHOR_FORM_LENGTH &&
        matchVariantsOccurIn(matchFormVariants(f), haystackVariants),
    );
    if (anchored) ids.push(entity.id);
  }
  return ids.toSorted();
}

/**
 * Decompose markdown/prose text into atomic assertions. Deterministic:
 * same input and entity set, same output.
 */
export function decomposeAtomicFacts(text: string, opts: DecomposeOptions = {}): AtomicAssertion[] {
  if (!text || text.trim() === "") return [];
  const entities = opts.entities ?? [];
  const lines = text.split(/\r?\n/);

  const out: AtomicAssertion[] = [];
  // Heading stack: [level, text] pairs; path = texts in order.
  const headingStack: Array<{ level: number; text: string }> = [];
  let inFence = false;
  let inFrontmatter = false;

  const push = (raw: string, line: number, kind: AtomicAssertionKind): void => {
    const stripped = collapse(raw.replace(/`[^`\n]*`/g, " "));
    if (stripped.length < MIN_ASSERTION_CHARS) return;
    out.push(
      Object.freeze({
        text: stripped,
        line,
        headingPath: Object.freeze(headingStack.map((h) => h.text)),
        kind,
        entities: Object.freeze(anchorEntities(stripped, entities, { lexicon: opts.lexicon })),
      }),
    );
  };

  // Paragraph accumulator: prose lines buffer until a blank/structural
  // line, then split into sentences attributed to the paragraph start.
  let paragraph: string[] = [];
  let paragraphStart = 0;
  const flushParagraph = (): void => {
    if (paragraph.length === 0) return;
    const joined = paragraph.join(" ");
    for (const sentence of splitSentences(joined)) {
      push(sentence, paragraphStart, "sentence");
    }
    paragraph = [];
  };

  for (let i = 0; i < lines.length; i++) {
    const line = lines[i]!;
    const trimmed = line.trim();

    // Frontmatter: only an opening `---` on the very first line.
    if (i === 0 && trimmed === "---") {
      inFrontmatter = true;
      continue;
    }
    if (inFrontmatter) {
      if (trimmed === "---") inFrontmatter = false;
      continue;
    }

    if (trimmed.startsWith("```") || trimmed.startsWith("~~~")) {
      flushParagraph();
      inFence = !inFence;
      continue;
    }
    if (inFence) continue;

    if (trimmed === "") {
      flushParagraph();
      continue;
    }

    if (trimmed.startsWith(">")) {
      flushParagraph();
      continue;
    }

    const heading = HEADING_RE.exec(trimmed);
    if (heading !== null) {
      flushParagraph();
      const level = heading[1]!.length;
      while (headingStack.length > 0 && headingStack.at(-1)!.level >= level) {
        headingStack.pop();
      }
      headingStack.push({ level, text: collapse(heading[2]!) });
      continue;
    }

    const item = LIST_ITEM_RE.exec(line);
    if (item !== null) {
      flushParagraph();
      // A multi-sentence list item still decomposes; single-sentence
      // items (the common case) survive whole.
      const sentences = splitSentences(item[1]!);
      if (sentences.length <= 1) {
        push(item[1]!, i + 1, "list_item");
      } else {
        for (const sentence of sentences) push(sentence, i + 1, "list_item");
      }
      continue;
    }

    if (paragraph.length === 0) paragraphStart = i + 1;
    paragraph.push(trimmed);
  }
  flushParagraph();

  return out;
}