Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
34.9 KB
Raw Blame
/**
 * Canonicalization kernel - the one place entity identity is computed.
 *
 * Shared by the registry (duplicate refusal, alias resolution), the
 * doctor lints, search alias expansion, and the fact-extraction router,
 * so every consumer compares like with like. Same normalization shape
 * as `extractEntities` in src/core/search/entities.ts: NFC, lowercase,
 * collapsed whitespace - plus the quote-variant fold below.
 */

/** A run of whitespace of any kind, collapsed to one space. */
const WHITESPACE_RUN_RE = /\s+/g;

/**
 * The one form every typographic quote variant folds to in the IDENTITY
 * kernel. U+0027 is the ASCII apostrophe, so the fold never rewrites an
 * ASCII byte: a label that is already ASCII keys exactly as it did before
 * the fold existed.
 *
 * The asymmetry that follows, stated because it is not obvious and because
 * a consumer that cannot live with it needs to know to reach for
 * {@link foldQuoteVariantsByClass}: EVERY quote class lands here, the
 * double-width ones included. So `«war»` and `'war'` are one identity while
 * `"war"` is another - the ASCII double quote is the one quote form this
 * fold cannot reach, because reaching it would mean rewriting an ASCII byte.
 * See {@link foldQuoteVariants} for why the kernel keeps it that way.
 */
export const QUOTE_VARIANT_FOLD_TARGET = "'";

/**
 * The target the class-aware fold uses for double-width quote marks. Also
 * ASCII, so that fold rewrites no ASCII byte either.
 */
export const QUOTE_VARIANT_FOLD_TARGET_DOUBLE = '"';

/**
 * The quote variants whose width is UNAMBIGUOUSLY double.
 *
 * Enumerated rather than derived because Unicode publishes no property for
 * quote width - `\p{Pi}`/`\p{Pf}` say a mark opens or closes a quotation,
 * never whether it stands for one quote or two. The list is therefore
 * confined to the marks whose width no reading disputes; every other member
 * of the class, including the editorial half-brackets U+2E02…U+2E21 whose
 * width Unicode does not state, keeps {@link QUOTE_VARIANT_FOLD_TARGET}.
 *
 * That default is what makes the enumeration safe to age: a code point
 * added to `\p{Pi}`/`\p{Pf}` in a future Unicode version behaves exactly as
 * it does today rather than acquiring a new, wrong classification. The
 * footprint is asserted over the whole code point space in
 * `tests/core/brain/topic-key-quote-classes.test.ts`.
 */
const DOUBLE_WIDTH_QUOTE_VARIANTS: ReadonlySet<string> = new Set([
  "«", // « LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
  "»", // » RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
  "“", // “ LEFT DOUBLE QUOTATION MARK
  "”", // ” RIGHT DOUBLE QUOTATION MARK
  "‟", // ‟ DOUBLE HIGH-REVERSED-9 QUOTATION MARK
]);

/**
 * U+02BC MODIFIER LETTER APOSTROPHE. It is a LETTER by general category
 * (`\p{Lm}`), so it is outside the punctuation classes below and needs
 * its own entry - and it is an apostrophe by function, which is why an
 * editor or a paste can substitute it for U+0027 in a label.
 */
const MODIFIER_LETTER_APOSTROPHE = "ʼ";

/**
 * Every typographic quote variant, DERIVED from Unicode general
 * categories rather than enumerated by hand: initial quote punctuation
 * (`\p{Pi}`), final quote punctuation (`\p{Pf}`), and the modifier-letter
 * apostrophe. The engine resolves the membership, so the rule tracks the
 * Unicode tables it names instead of a list that ages.
 *
 * The exact footprint is asserted exhaustively over the whole code point
 * space in `tests/core/brain/entities/quote-variant-fold.test.ts`, so a
 * widened class cannot land unmeasured.
 */
const QUOTE_VARIANT_RE = new RegExp(`[\\p{Pi}\\p{Pf}${MODIFIER_LETTER_APOSTROPHE}]`, "gu");

/**
 * Fold every typographic quote variant onto {@link QUOTE_VARIANT_FOLD_TARGET}.
 *
 * No Unicode normal form does this: NFC, NFD, NFKC and NFKD all leave
 * U+2019 exactly where it is, so `Taylor's` and `Taylor’s` were two
 * canonical entities in one registry, differing only in which key a text
 * editor happened to insert. Identity is the one place that distinction
 * carries no information, so this is where it is discarded - the same
 * kind of deliberate loss as lowercasing and whitespace collapse.
 *
 * ## Why the single target stays, asymmetry and all
 *
 * Sending double-width marks to `"` instead (what
 * {@link foldQuoteVariantsByClass} does) would be the more faithful fold,
 * and it is NOT applied here, because this function's output is persisted:
 * `truth/store.ts` writes `normalizeEntityName(entity)` and `(aspect)` into
 * an append-only claim log. Re-targeting would leave every claim already
 * written under the old key and every new claim under the new one, splitting
 * a history in a log that by construction cannot be rewritten. It would also
 * mint NEW duplicate refusals in the registry between ASCII-double and
 * typographic-double labels that coexist on disk today - the same upgrade
 * pain the fold's own introduction caused, for a second time.
 *
 * A consumer whose keys are ephemeral pays none of that and can afford the
 * faithful fold; `dream-plan.ts`'s `topicKey` is the one such consumer today.
 * Both folds live here so quote knowledge keeps a single home.
 */
export function foldQuoteVariants(raw: string): string {
  return raw.replace(QUOTE_VARIANT_RE, QUOTE_VARIANT_FOLD_TARGET);
}

/**
 * Fold every typographic quote variant onto the ASCII mark of its own
 * WIDTH: double-width marks to `"`, everything else in the class to `'`.
 *
 * The same population as {@link foldQuoteVariants} - this is a re-targeting,
 * never a widening - and the same "rewrites no ASCII byte" guarantee, now
 * held for both ASCII quote forms rather than one. What it buys: `“strict”`
 * keys with `"strict"` rather than with `'strict'`, so two labels that use
 * different quote MARKS stay two subjects while two spellings of the same
 * mark become one.
 *
 * For a caller whose keys are computed per run and never written down. A
 * caller whose keys are persisted must use {@link normalizeEntityName}; see
 * its fold for what re-targeting a stored key would cost.
 */
export function foldQuoteVariantsByClass(raw: string): string {
  return raw.replace(QUOTE_VARIANT_RE, (mark) =>
    DOUBLE_WIDTH_QUOTE_VARIANTS.has(mark)
      ? QUOTE_VARIANT_FOLD_TARGET_DOUBLE
      : QUOTE_VARIANT_FOLD_TARGET,
  );
}

/**
 * The separators whose RUNS are interchangeable at match time, folded to
 * the one canonical join. Space, hyphen and underscore only: a
 * `notes/link-candidates.ts` basename is a slug, so those three are the
 * spellings one token sequence answers to in this store; dot and slash
 * carry structure (`Node.js`, a path) and are never folded.
 */
const MATCH_SEPARATOR_RUN_RE = /[-_ ]+/;

/**
 * Minimum normalized form length that participates in substring matching.
 * The anchoring kernels gate the FULL forms they compare with this floor,
 * and {@link matchFormVariants} gates the slug tail it emits - the tail is
 * an emitted needle in its own right, and an unfloored one would anchor
 * standalone words no full form ever matched (`AC/DC` must not mint `dc`).
 * One constant here so the kernels and the generator cannot drift.
 */
export const MIN_ANCHOR_FORM_LENGTH = 3;

/**
 * The entity-type folders of the canonical entity registry: the category
 * slugs entity pages (`Brain/entities/<category>/<id>.md`) and entity ids
 * (`ent-<category>-<slug>`) are minted from. Declared here - the leaf
 * kernel every anchoring consumer shares - so the registry names its own
 * folder vocabulary exactly once and every reader derives from the one
 * list: {@link matchFormVariants} gates the slug tail on it, and the
 * merge guard's default guarded categories ARE this list (re-exported
 * there under its guard-facing name).
 */
export const ENTITY_TYPE_FOLDERS: ReadonlyArray<string> = Object.freeze([
  "people",
  "person",
  "org",
  "organization",
  "organisation",
  "company",
  "team",
]);

/** Set form of {@link ENTITY_TYPE_FOLDERS} for the tail gate's lookups. */
const ENTITY_TYPE_FOLDER_SET: ReadonlySet<string> = new Set(ENTITY_TYPE_FOLDERS);

/**
 * Fold a string's separator runs to the ONE canonical join: the tokens
 * the string splits into, joined by single spaces (the tokens keep their
 * slashes and dots - only the three foldable separators run together).
 * Edge separator runs vanish with their empty edge tokens, exactly as
 * they always did.
 */
function canonicalMatchJoin(s: string): string {
  return s
    .split(MATCH_SEPARATOR_RUN_RE)
    .filter((token) => token !== "")
    .join(" ");
}

/**
 * Expand one ALREADY-NORMALIZED match form into the match variants the
 * anchoring kernels compare with (note-change truth maintenance, F3;
 * grid reduced per M2 of the PR 268 review). At most TWO variants:
 *
 *   - The form's CANONICAL JOIN ({@link canonicalMatchJoin}): runs of
 *     space, hyphen and underscore (in any mix) fold to one space, so
 *     `ada lovelace`, `ada-lovelace`, `ada_lovelace` and
 *     `ada -_ lovelace` all expand to the one same variant. The kernels
 *     fold the needle AND the haystack through the same function, and
 *     one space-joined comparison answers for the whole variant class:
 *     tokens never contain separators, every join character counts as a
 *     non-word neighbour at the leading edge, and both the old grid and
 *     the folded one compare token sequences - so the fold is verdict-
 *     equivalent, at one scan instead of nine.
 *   - When the form's LAST `/`-segment (the slug itself) has a registry
 *     entity-type folder for its parent segment ({@link
 *     ENTITY_TYPE_FOLDERS} - the categories entity pages and ids are
 *     minted from; the parent SEGMENT gates, so the registry's deeper
 *     spellings `entities/people/ada_lovelace` and
 *     `Brain/entities/people/ada_lovelace` mint the tail too), the
 *     CANONICAL JOIN of that segment, so `people/ada_lovelace` also
 *     occurs in a text that only says `ada lovelace`. The segment prefix
 *     is registry
 *     decoration, not part of the mentioned name - and because the tail
 *     is an emitted needle in its own right, the tail meets
 *     {@link MIN_ANCHOR_FORM_LENGTH}, the same floor the kernels apply
 *     to full forms (`AC/DC` emits no `dc` for a standalone word to
 *     match). The FULL form is never floored or gated here: gating it
 *     stays the caller's call, unchanged. A slash after any other prefix
 *     carries no registry decoration - "Warner/Chappell" is one name
 *     with a slash in it, not a directory path - so `Warner/Chappell`
 *     mints no `chappell` and `Sony/ATV` no `atv` (M1, PR 268 review).
 *
 * The output is deterministic (the folded full form first, the folded
 * tail after) and never empty for a non-empty separator-free form of at
 * least {@link MIN_ANCHOR_FORM_LENGTH}. Deliberately NOT part of
 * identity: the input is never re-normalized, `normalizeEntityName` is
 * untouched, and nothing this returns may be persisted - identity keys
 * keep their separators, which is exactly why two spellings can be two
 * identities and one match class. Pure: no I/O, no state.
 */
export function matchFormVariants(form: string): string[] {
  const out: string[] = [];
  const full = canonicalMatchJoin(form);
  if (full === "") return out;
  out.push(full);
  const lastSlash = form.lastIndexOf("/");
  if (lastSlash >= 0) {
    // The gate is the slug's OWN parent segment - the category folder an
    // entity page's id sits in - not the whole prefix before the slash:
    // a form that still carries the registry's deeper path
    // (`Brain/entities/people/ada_lovelace`) has `people` as its
    // immediate parent and mints its tail the same as the bare
    // `people/ada_lovelace`, while `Warner/Chappell` has `warner` there
    // and mints nothing (M1, PR 268 review).
    const prefix = form.slice(0, lastSlash);
    const folder = prefix.slice(prefix.lastIndexOf("/") + 1).toLowerCase();
    if (ENTITY_TYPE_FOLDER_SET.has(folder)) {
      const tail = canonicalMatchJoin(form.slice(lastSlash + 1));
      if (tail.length >= MIN_ANCHOR_FORM_LENGTH) out.push(tail);
    }
  }
  return out;
}

/**
 * A letter or digit in ANY script: the word characters a match must not
 * sit inside. The fold is deliberately NOT on this list - a separator is
 * exactly where one token ends and the next begins, so `ada` may anchor
 * inside `ada-lovelace` but never inside `canada`.
 */
const MATCH_WORD_CHAR_RE = /[\p{L}\p{N}]/u;

/**
 * The scripts written WITHOUT word spaces - Han, Hiragana, Katakana,
 * Thai, Lao, Khmer, Myanmar. In a run of such text every letter touches
 * a letter, so a LEADING letter is not evidence of a mid-word hit the
 * way it is in a spaced script: `田中太郎` never occurs in
 * `鈴木花子と田中太郎` at a word boundary, and a boundary rule applied
 * there verbatim would have silenced every CJK mention the pre-rule
 * literal `includes` matching anchored. For a leading neighbour from one
 * of these scripts the boundary requirement is therefore suspended - the
 * script keeps its literal behavior back, spaced scripts keep the rule
 * (a Latin neighbour blocks even inside otherwise-CJK text). Korean is
 * deliberately NOT on this list: it writes spaces between words, so a
 * leading Hangul letter keeps blocking - Korean attaches no prefixes,
 * and `박철수` must not anchor `철수`.
 */
const NO_SPACE_SCRIPT_RE =
  /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u;

/**
 * True when `codePoint` belongs to a script written without word spaces
 * (see {@link NO_SPACE_SCRIPT_RE}): a leading neighbour from one of
 * these scripts never blocks an occurrence, because adjacency there
 * carries no word-boundary information.
 */
function boundaryNotRequiredForScripts(codePoint: string): boolean {
  return NO_SPACE_SCRIPT_RE.test(codePoint);
}

function isHighSurrogate(code: number): boolean {
  return code >= 0xd800 && code <= 0xdbff;
}

function isLowSurrogate(code: number): boolean {
  return code >= 0xdc00 && code <= 0xdfff;
}

/**
 * The code point ending just before index `i`, or null at the string
 * edge. Steps over a surrogate pair so an astral letter comes back
 * whole rather than as a lone (always non-word) code unit.
 */
function codePointBefore(s: string, i: number): string | null {
  if (i <= 0) return null;
  const start =
    i >= 2 && isLowSurrogate(s.charCodeAt(i - 1)) && isHighSurrogate(s.charCodeAt(i - 2))
      ? i - 2
      : i - 1;
  return s.slice(start, i);
}

/**
 * True when `variant` occurs in one folded haystack string with a clean
 * LEADING edge: some occurrence of it has no letter or digit directly
 * before it - UNLESS that neighbour belongs to a script written without
 * word spaces (see {@link NO_SPACE_SCRIPT_RE}), whose adjacency suspends
 * the requirement. The trailing side is unchecked (H1, PR 268 review):
 * every language that attaches endings to a name with no space - Russian
 * `Ивана`, the German genitive `Annas`, a Korean particle run like
 * `김철수는` - lost those mentions under a two-sided rule, and a
 * trailing character is an inflection until proven otherwise. The
 * leading-side rule is what keeps the separator fold from planting
 * mid-word hits nothing anchored before (`canada` never anchors `ada`:
 * the letter before the hit blocks it, folded spellings included); an
 * occurrence beside a separator or the string edge is a token edge and
 * anchors.
 */
function variantOccursAtWordBoundary(variant: string, haystack: string): boolean {
  // An empty variant occurs everywhere and nowhere - refuse it rather
  // than let a boundary-less occurrence anchor.
  if (variant === "") return false;
  let at = haystack.indexOf(variant);
  while (at >= 0) {
    const before = codePointBefore(haystack, at);
    const blockedBefore =
      before !== null && MATCH_WORD_CHAR_RE.test(before) && !boundaryNotRequiredForScripts(before);
    if (!blockedBefore) return true;
    at = haystack.indexOf(variant, at + 1);
  }
  return false;
}

/**
 * True when ANY of the needle's match variants occurs with a clean
 * leading edge in ANY of the haystack's match variants - the one
 * comparison the anchoring kernels and their twins share. Both arrays
 * come from {@link matchFormVariants}: the needle's variants are
 * precomputed by the caller where it already precomputes forms, and the
 * haystack's variants are expanded once per text, never once per entity.
 * Pure: no I/O.
 */
export function matchVariantsOccurIn(
  formVariants: ReadonlyArray<string>,
  haystackVariants: ReadonlyArray<string>,
): boolean {
  return formVariants.some((variant) =>
    haystackVariants.some((haystack) => variantOccursAtWordBoundary(variant, haystack)),
  );
}

/**
 * One-shot convenience over {@link matchFormVariants} and
 * {@link matchVariantsOccurIn}: does the single normalized form `form`
 * occur, separator-folded and leading-edge-checked, in the normalized
 * haystack text? For callers that match one form at a time; a caller
 * holding precomputed variant lists calls {@link matchVariantsOccurIn}
 * directly so the haystack expands once, not once per form.
 */
export function matchFormOccursIn(form: string, normalizedHaystack: string): boolean {
  return matchVariantsOccurIn(matchFormVariants(form), matchFormVariants(normalizedHaystack));
}

/**
 * The kernel's shape pass, WITHOUT the quote fold: NFC, trim, collapse
 * whitespace runs, lowercase. Separated from {@link normalizeEntityName}
 * so {@link differsOnlyByQuoteVariant} can name the difference the fold
 * makes rather than re-deriving it from a second copy of these steps -
 * and exported so a caller that pairs it with
 * {@link foldQuoteVariantsByClass} (`dream-plan.ts`'s `topicKey`) shares
 * these steps rather than copying them.
 */
export function normalizeEntityShape(raw: string): string {
  return raw.normalize("NFC").trim().replace(WHITESPACE_RUN_RE, " ").toLowerCase();
}

/** NFC-normalise, trim, collapse whitespace runs, lowercase, fold quote variants. */
export function normalizeEntityName(raw: string): string {
  return foldQuoteVariants(normalizeEntityShape(raw));
}

/**
 * Registry code the quote-variant collision resolves its forward pointer
 * through, and the structural command that resolves it.
 *
 * Both live in this LEAF module rather than at the consumer, and the
 * `DIAGNOSTIC_SIGNALS` entry in `diagnostics.ts` is built from them, so
 * the string still has exactly one definition. The usual arrangement -
 * the consumer resolving `requireNextStep(code)` at module scope - is not
 * available to the REGISTRY: `next-step.ts` reads `diagnostics.ts`, which
 * reaches the entity registry through the doctor's entity checks and the
 * search query-expansion module, so a registry import of `next-step.ts`
 * closes an import cycle the architecture ratchet refuses
 * (`tests/core/architecture/import-cycles.test.ts`). This module imports
 * nothing, so both consumers can read it.
 *
 * The code's PRODUCER is `doctor/entity-checks.ts`, which spells it on the
 * finding for a collision the fold created; every doctor surface resolves
 * a finding's code through the rail (`nextCommandField` / `emitNextStep`),
 * which is how the other registered doctor codes reach an operator. The
 * registry's own refusals are thrown synchronously out of core and name
 * the command from {@link ENTITY_QUOTE_VARIANT_COLLISION_COMMAND} - the
 * same constant the registration is built from, so the two cannot drift.
 */
export const ENTITY_QUOTE_VARIANT_COLLISION_CODE = "entity-quote-variant-collision";
export const ENTITY_QUOTE_VARIANT_COLLISION_COMMAND = "o2b brain entity archive <name>";

/**
 * The one sentence naming WHY two labels that coexisted before the fold
 * cannot coexist after it. Written once here because three surfaces say
 * it - the registry's write refusal, the registry's ambiguity error, and
 * the doctor's finding - and an operator who meets it twice must read the
 * same words, or the two readings become two conditions.
 */
export const ENTITY_QUOTE_VARIANT_CAUSE =
  "the two differ only in typographic quote form and now resolve to one identity key";

/**
 * True when `a` and `b` are two spellings of one identity that ONLY the
 * quote fold unifies - they were distinct identities before it and are
 * one after it.
 *
 * This is the predicate a collision message asks so it can name the right
 * exit: a refusal an operator meets for the first time immediately after
 * an upgrade needs to say WHY two records that used to coexist no longer
 * can, and every other collision keeps the message it always had.
 */
export function differsOnlyByQuoteVariant(a: string, b: string): boolean {
  const shapeA = normalizeEntityShape(a);
  const shapeB = normalizeEntityShape(b);
  return shapeA !== shapeB && foldQuoteVariants(shapeA) === foldQuoteVariants(shapeB);
}

/**
 * True when ANY two of `labels` are one identity only the fold unifies.
 *
 * The n-ary form because a collision is a set of claimants, not a pair:
 * three files can claim one key, and a message that only ever compared
 * the first two would omit the cause on exactly the messiest vault.
 */
export function anyDiffersOnlyByQuoteVariant(labels: ReadonlyArray<string>): boolean {
  for (let i = 0; i < labels.length; i++) {
    for (let j = i + 1; j < labels.length; j++) {
      if (differsOnlyByQuoteVariant(labels[i]!, labels[j]!)) return true;
    }
  }
  return false;
}

/**
 * The tail a collision message carries when, and only when, the labels
 * were distinct identities before the quote fold and are one after it.
 * Empty for every other collision, so the messages an operator already
 * knows are unchanged and the new sentence means exactly what it says:
 * this refusal is new, and here is why.
 *
 * Lives beside the predicate rather than at the registry because the
 * doctor's finding needs the same words - see
 * {@link ENTITY_QUOTE_VARIANT_CAUSE}.
 */
export function quoteVariantCollisionTail(...labels: ReadonlyArray<string>): string {
  if (!anyDiffersOnlyByQuoteVariant(labels)) return "";
  return (
    ` - ${ENTITY_QUOTE_VARIANT_CAUSE}; ` +
    `run ${ENTITY_QUOTE_VARIANT_COLLISION_COMMAND} on the record you do not keep`
  );
}

/**
 * Validate an entity category: a lowercase kebab-ish slug with no path
 * separators, traversal, or whitespace. Lowercases the input so
 * `People` and `people` are the same category.
 */
export function validateEntityCategory(raw: string): string {
  const category = raw.normalize("NFC").trim().toLowerCase();
  if (!category) throw new Error("entity category must not be empty");
  if (!/^[a-z0-9][a-z0-9-]*$/.test(category)) {
    throw new Error(
      `entity category must be a lowercase slug ([a-z0-9-], starting alphanumeric): ${JSON.stringify(raw)}`,
    );
  }
  return category;
}

/**
 * The identity key one canonical entity owns: `<category>:<normalized name>`.
 * Two files claiming the same key are duplicates by definition.
 */
export function entityIdentityKey(category: string, name: string): string {
  return `${validateEntityCategory(category)}:${normalizeEntityName(name)}`;
}

// ----- Label quality gate (A1 / t_657b365e) ---------------------------------
//
// `normalizeEntityName` above stays byte-stable for every label that
// carries no typographic quote variant: it is the identity kernel and its
// output feeds `entityIdentityKey`, so such a label must key identically to
// before. The quote fold is the ONE deliberate exception, its footprint is
// exactly the Unicode classes `foldQuoteVariants` names, and the corpus
// proof lives in tests/core/brain/entities/quote-variant-fold.test.ts. The
// label QUALITY pass lives here as
// a separate step applied BEFORE `normalizeEntityName` at every label-intake
// boundary (entity creation, fact-extract anchoring, atomic-facts anchoring):
// it strips Markdown decoration and surrounding punctuation, then rejects
// structurally-junk labels. Junk detection is STRUCTURAL only (Unicode
// property classes, never a natural-language word list); the sole vocabulary
// source is the operator-supplied denylist threaded in by the caller.

/**
 * Upper bound on a sanitized label's length. A defensible ceiling: real
 * entity names are short, and anything past this is a pasted blob, not a
 * label. Exported so callers and tests share one source of truth.
 */
export const ENTITY_LABEL_MAX_LENGTH = 200;

/** Leading Markdown heading marker: one-to-six `#` then required space/tab. */
const HEADING_PREFIX_RE = /^#{1,6}[ \t]+/;

/**
 * Matched Markdown emphasis/code wrappers, longest opener first so `**`
 * is peeled before `*`. One matched pair is removed per sanitise pass;
 * the outer loop re-runs until the label stops changing.
 */
const EMPHASIS_WRAPPERS: ReadonlyArray<readonly [string, string]> = [
  ["**", "**"],
  ["__", "__"],
  ["*", "*"],
  ["_", "_"],
  ["`", "`"],
] as const;

/** A leading run of Unicode punctuation (surrounding, not internal). */
const LEADING_PUNCT_RE = /^\p{P}+/u;
/** A trailing run of Unicode punctuation (surrounding, not internal). */
const TRAILING_PUNCT_RE = /\p{P}+$/u;
/** At least one letter or digit in ANY script - the "carries meaning" test. */
const HAS_LETTER_OR_DIGIT_RE = /[\p{L}\p{N}]/u;

/**
 * One arm of the structural code-shape test: a character that only code
 * (or a path, or a numeric identifier) plants inside a label - underscore,
 * dot, slash, or a digit of ANY script. The letters are checked separately
 * so word-position can be honoured for case.
 */
const CODE_SHAPE_CHAR_RE = /[_./\p{N}]/u;

/** Peel one matched emphasis/code wrapper pair, if present. */
function stripSurroundingEmphasis(s: string): string {
  for (const [open, close] of EMPHASIS_WRAPPERS) {
    if (s.length >= open.length + close.length + 1 && s.startsWith(open) && s.endsWith(close)) {
      return s.slice(open.length, s.length - close.length).trim();
    }
  }
  return s;
}

/**
 * Strip surrounding Markdown emphasis/heading decoration and surrounding
 * (never internal) punctuation from a raw label, iterating until stable.
 *
 * Examples: `**Foo**` -> `Foo`, `# Heading` -> `Heading`, `_bar_` -> `bar`,
 * `(baz)` -> `baz`, `Foo.` -> `Foo`. Internal punctuation is preserved, so
 * `Node.js`, `e.g`, and `C++` survive. NFC-normalises and trims first so a
 * clean label (`Ada`, `Open Second Brain`, `café`, `Ада`) is returned
 * byte-identical to `raw.trim()` - the backward-compatibility guarantee for
 * every currently-clean label.
 */
export function sanitizeEntityLabel(raw: string): string {
  let s = raw.normalize("NFC").trim();
  let prev = "";
  while (s !== prev) {
    prev = s;
    s = s.replace(HEADING_PREFIX_RE, "").trim();
    s = stripSurroundingEmphasis(s);
    s = s.replace(LEADING_PUNCT_RE, "").replace(TRAILING_PUNCT_RE, "").trim();
  }
  return s;
}

/**
 * Why a label was rejected by {@link validateEntityLabel}.
 *
 * `shouted-heading` and `function-word-run` are the recall-quality growth:
 * case-based verdicts that fire ONLY behind an {@link EntityLexiconHandle}
 * (see {@link ValidateEntityLabelOptions.lexicon}) - with the handle null
 * or absent the predicate is byte-identical to its pre-lexicon behavior.
 */
export type EntityLabelInvalidReason =
  | "empty"
  | "too-long"
  | "no-letter-or-digit"
  | "denylisted"
  | "shouted-heading"
  | "function-word-run";

export interface EntityLabelValidation {
  readonly valid: boolean;
  readonly reason?: EntityLabelInvalidReason;
}

/**
 * The store-learned vocabulary the case-based verdicts consult, resolved
 * by the entity-lexicon module and threaded in beside the denylist.
 * Declared here - not imported - because this module is a leaf of the
 * import graph and must keep importing nothing.
 */
export interface EntityLexiconHandle {
  /**
   * False for a caseless-script corpus (a script whose letters carry no
   * upper/lower distinction): case statistics learned over such a corpus
   * are meaningless, so a false handle suppresses BOTH case-based
   * verdicts - `shouted-heading` and `function-word-run` never fire.
   */
  readonly caseful: boolean;
  /**
   * Whether `token` is a store-learned common word. Tokens are handed
   * over LOWERCASED by the predicate, so implementations do exact
   * lookups against their learned lowercase keys.
   */
  isCommonWord(token: string): boolean;
}

export interface ValidateEntityLabelOptions {
  /**
   * Operator-supplied denylist of exact labels, ALREADY normalised via
   * `normalizeEntityName` (the comparison is post-normalization). The only
   * vocabulary source; empty/absent means no name-based rejection.
   */
  readonly denylist?: ReadonlySet<string>;
  /**
   * The store-learned lexicon behind the case-based verdicts
   * (`shouted-heading`, `function-word-run`). Null or absent keeps the
   * predicate byte-identical to its pre-lexicon behavior: those two
   * reasons never fire.
   */
  readonly lexicon?: EntityLexiconHandle | null;
}

/**
 * True when the label has a structural CODE shape: it contains `_`, `.`,
 * `/`, any digit, or an INTERNAL uppercase letter - a camelCase hump, an
 * uppercase whose preceding character is a lowercase LETTER (`OpenAI`,
 * `TheAnd`, `McDonald`). An uppercase that starts the label or follows a
 * word boundary is Title Case, and the letters of an ALL-CAPS word follow
 * their own uppercase, so neither reads as code - `Open Second Brain`
 * stays a name and `MEETING NOTES` stays eligible for the case verdicts,
 * while `FTS5`, `foo_bar`, `Node.js` and `path/to/x` are identifiers.
 */
function hasStructuralCodeShape(sanitized: string): boolean {
  if (CODE_SHAPE_CHAR_RE.test(sanitized)) return true;
  for (let i = 1; i < sanitized.length; i++) {
    const ch = sanitized[i]!;
    const prev = sanitized[i - 1]!;
    if (ch !== ch.toLowerCase() && prev !== prev.toUpperCase()) return true;
  }
  return false;
}

/**
 * True when some letter carries case - `l !== l.toLowerCase()`. Only a
 * CASEFUL label is eligible for the case-based verdicts; a caseless one
 * (CJK, all-lowercase text, digits-only) admits on the structural gate
 * alone.
 */
function isCasefulLabel(sanitized: string): boolean {
  for (const ch of sanitized) {
    if (ch !== ch.toLowerCase()) return true;
  }
  return false;
}

/**
 * True when every token of the (already whitespace-collapsed) label is a
 * store-learned common word. Tokens are handed to the handle LOWERCASED,
 * so implementations do exact lookups against lowercase keys.
 */
function isCommonWordRun(sanitized: string, lexicon: EntityLexiconHandle): boolean {
  // `split` always yields at least one element (an empty string yields
  // one empty token), so there is no empty-result guard to write; the
  // per-token emptiness check below is what actually fires, on the empty
  // edge tokens a leading/trailing separator produces.
  const tokens = sanitized.split(WHITESPACE_RUN_RE);
  return tokens.every((token) => token.length > 0 && lexicon.isCommonWord(token.toLowerCase()));
}

/**
 * Validate an already-sanitized label. Structural rejections only, plus the
 * operator denylist:
 *   - `empty`               - nothing survived sanitisation;
 *   - `too-long`            - exceeds {@link ENTITY_LABEL_MAX_LENGTH};
 *   - `no-letter-or-digit`  - no `\p{L}`/`\p{N}` in any script (pure
 *                             punctuation/symbols);
 *   - `denylisted`          - normalised form is in the operator denylist.
 *
 * Behind a lexicon ({@link ValidateEntityLabelOptions.lexicon}) two
 * CASE-BASED verdicts join, in this order after the four above:
 *   - `shouted-heading`     - every letter is uppercase (a `# HEADING`
 *                             residue), checked first;
 *   - `function-word-run`   - every token is a store-learned common word
 *                             (a single common token is already a run).
 * Both run ONLY for a caseful label (some letter where `l !==
 * l.toLowerCase()`) under a caseful handle, and BOTH are bypassed by a
 * structural code shape ({@link hasStructuralCodeShape}) - `FTS5` and
 * `foo_bar` admit even when they shout. The four structural verdicts are
 * kept unconditionally; a caseless label admits on the structural gate
 * alone. With the lexicon null or absent the two new reasons never fire
 * and the verdicts are byte-identical to the pre-lexicon behavior.
 *
 * The single source of truth for label validity - creation (typed error),
 * anchoring (logged skip), doctor, and prune all route through it. The
 * hard-at-create vs advisory-heal POLICY over these verdicts lives in the
 * callers, not here.
 */
export function validateEntityLabel(
  sanitized: string,
  opts: ValidateEntityLabelOptions = {},
): EntityLabelValidation {
  if (sanitized.length === 0) return { valid: false, reason: "empty" };
  if (sanitized.length > ENTITY_LABEL_MAX_LENGTH) return { valid: false, reason: "too-long" };
  if (!HAS_LETTER_OR_DIGIT_RE.test(sanitized))
    return { valid: false, reason: "no-letter-or-digit" };
  if (opts.denylist && opts.denylist.has(normalizeEntityName(sanitized))) {
    return { valid: false, reason: "denylisted" };
  }
  const lexicon = opts.lexicon;
  if (!lexicon || !lexicon.caseful) return { valid: true };
  if (!isCasefulLabel(sanitized)) return { valid: true };
  if (hasStructuralCodeShape(sanitized)) return { valid: true };
  if (sanitized === sanitized.toUpperCase()) return { valid: false, reason: "shouted-heading" };
  if (isCommonWordRun(sanitized, lexicon)) return { valid: false, reason: "function-word-run" };
  return { valid: true };
}

/** Convenience boolean wrapper over {@link validateEntityLabel}. */
export function isValidEntityLabel(
  sanitized: string,
  opts: ValidateEntityLabelOptions = {},
): boolean {
  return validateEntityLabel(sanitized, opts).valid;
}

/** Typed error raised when a label is rejected at a creation boundary. */
export class InvalidEntityLabelError extends Error {
  readonly reason: EntityLabelInvalidReason;
  readonly raw: string;
  readonly sanitized: string;
  constructor(raw: string, sanitized: string, reason: EntityLabelInvalidReason) {
    super(
      `invalid entity label ${JSON.stringify(raw)} (${reason}): ` +
        `sanitised form ${JSON.stringify(sanitized)} does not pass the label quality gate`,
    );
    this.name = "InvalidEntityLabelError";
    this.reason = reason;
    this.raw = raw;
    this.sanitized = sanitized;
  }
}

/**
 * Sanitise `raw` and assert it passes the quality gate, returning the
 * sanitized label for storage. Throws {@link InvalidEntityLabelError} on
 * rejection - the creation-boundary contract (no silent drop).
 */
export function assertValidEntityLabel(raw: string, opts: ValidateEntityLabelOptions = {}): string {
  const sanitized = sanitizeEntityLabel(raw);
  const verdict = validateEntityLabel(sanitized, opts);
  if (!verdict.valid) {
    throw new InvalidEntityLabelError(raw, sanitized, verdict.reason!);
  }
  return sanitized;
}

/**
 * Normalised match forms for an entity's labels (name + aliases): sanitise
 * each, drop the structurally-invalid ones, then normalise the survivors.
 * The shared anchoring kernel so fact-extract and atomic-facts compare
 * facts against clean, valid label forms only. Pure - no I/O, no logging.
 */
export function entityMatchForms(
  rawForms: ReadonlyArray<string>,
  opts: ValidateEntityLabelOptions = {},
): string[] {
  const out: string[] = [];
  for (const raw of rawForms) {
    const sanitized = sanitizeEntityLabel(raw);
    if (!isValidEntityLabel(sanitized, opts)) continue;
    out.push(normalizeEntityName(sanitized));
  }
  return out;
}