Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Canonicalization kernel - the one place entity identity is computed.
*
* Shared by the registry (duplicate refusal, alias resolution), the
* doctor lints, search alias expansion, and the fact-extraction router,
* so every consumer compares like with like. Same normalization shape
* as `extractEntities` in src/core/search/entities.ts: NFC, lowercase,
* collapsed whitespace - plus the quote-variant fold below.
*/
/** A run of whitespace of any kind, collapsed to one space. */
const WHITESPACE_RUN_RE = /\s+/g;
/**
* The one form every typographic quote variant folds to in the IDENTITY
* kernel. U+0027 is the ASCII apostrophe, so the fold never rewrites an
* ASCII byte: a label that is already ASCII keys exactly as it did before
* the fold existed.
*
* The asymmetry that follows, stated because it is not obvious and because
* a consumer that cannot live with it needs to know to reach for
* {@link foldQuoteVariantsByClass}: EVERY quote class lands here, the
* double-width ones included. So `«war»` and `'war'` are one identity while
* `"war"` is another - the ASCII double quote is the one quote form this
* fold cannot reach, because reaching it would mean rewriting an ASCII byte.
* See {@link foldQuoteVariants} for why the kernel keeps it that way.
*/
export const QUOTE_VARIANT_FOLD_TARGET = "'";
/**
* The target the class-aware fold uses for double-width quote marks. Also
* ASCII, so that fold rewrites no ASCII byte either.
*/
export const QUOTE_VARIANT_FOLD_TARGET_DOUBLE = '"';
/**
* The quote variants whose width is UNAMBIGUOUSLY double.
*
* Enumerated rather than derived because Unicode publishes no property for
* quote width - `\p{Pi}`/`\p{Pf}` say a mark opens or closes a quotation,
* never whether it stands for one quote or two. The list is therefore
* confined to the marks whose width no reading disputes; every other member
* of the class, including the editorial half-brackets U+2E02…U+2E21 whose
* width Unicode does not state, keeps {@link QUOTE_VARIANT_FOLD_TARGET}.
*
* That default is what makes the enumeration safe to age: a code point
* added to `\p{Pi}`/`\p{Pf}` in a future Unicode version behaves exactly as
* it does today rather than acquiring a new, wrong classification. The
* footprint is asserted over the whole code point space in
* `tests/core/brain/topic-key-quote-classes.test.ts`.
*/
const DOUBLE_WIDTH_QUOTE_VARIANTS: ReadonlySet<string> = new Set([
"«", // « LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
"»", // » RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
"“", // “ LEFT DOUBLE QUOTATION MARK
"”", // ” RIGHT DOUBLE QUOTATION MARK
"‟", // ‟ DOUBLE HIGH-REVERSED-9 QUOTATION MARK
]);
/**
* U+02BC MODIFIER LETTER APOSTROPHE. It is a LETTER by general category
* (`\p{Lm}`), so it is outside the punctuation classes below and needs
* its own entry - and it is an apostrophe by function, which is why an
* editor or a paste can substitute it for U+0027 in a label.
*/
const MODIFIER_LETTER_APOSTROPHE = "ʼ";
/**
* Every typographic quote variant, DERIVED from Unicode general
* categories rather than enumerated by hand: initial quote punctuation
* (`\p{Pi}`), final quote punctuation (`\p{Pf}`), and the modifier-letter
* apostrophe. The engine resolves the membership, so the rule tracks the
* Unicode tables it names instead of a list that ages.
*
* The exact footprint is asserted exhaustively over the whole code point
* space in `tests/core/brain/entities/quote-variant-fold.test.ts`, so a
* widened class cannot land unmeasured.
*/
const QUOTE_VARIANT_RE = new RegExp(`[\\p{Pi}\\p{Pf}${MODIFIER_LETTER_APOSTROPHE}]`, "gu");
/**
* Fold every typographic quote variant onto {@link QUOTE_VARIANT_FOLD_TARGET}.
*
* No Unicode normal form does this: NFC, NFD, NFKC and NFKD all leave
* U+2019 exactly where it is, so `Taylor's` and `Taylor’s` were two
* canonical entities in one registry, differing only in which key a text
* editor happened to insert. Identity is the one place that distinction
* carries no information, so this is where it is discarded - the same
* kind of deliberate loss as lowercasing and whitespace collapse.
*
* ## Why the single target stays, asymmetry and all
*
* Sending double-width marks to `"` instead (what
* {@link foldQuoteVariantsByClass} does) would be the more faithful fold,
* and it is NOT applied here, because this function's output is persisted:
* `truth/store.ts` writes `normalizeEntityName(entity)` and `(aspect)` into
* an append-only claim log. Re-targeting would leave every claim already
* written under the old key and every new claim under the new one, splitting
* a history in a log that by construction cannot be rewritten. It would also
* mint NEW duplicate refusals in the registry between ASCII-double and
* typographic-double labels that coexist on disk today - the same upgrade
* pain the fold's own introduction caused, for a second time.
*
* A consumer whose keys are ephemeral pays none of that and can afford the
* faithful fold; `dream-plan.ts`'s `topicKey` is the one such consumer today.
* Both folds live here so quote knowledge keeps a single home.
*/
export function foldQuoteVariants(raw: string): string {
return raw.replace(QUOTE_VARIANT_RE, QUOTE_VARIANT_FOLD_TARGET);
}
/**
* Fold every typographic quote variant onto the ASCII mark of its own
* WIDTH: double-width marks to `"`, everything else in the class to `'`.
*
* The same population as {@link foldQuoteVariants} - this is a re-targeting,
* never a widening - and the same "rewrites no ASCII byte" guarantee, now
* held for both ASCII quote forms rather than one. What it buys: `“strict”`
* keys with `"strict"` rather than with `'strict'`, so two labels that use
* different quote MARKS stay two subjects while two spellings of the same
* mark become one.
*
* For a caller whose keys are computed per run and never written down. A
* caller whose keys are persisted must use {@link normalizeEntityName}; see
* its fold for what re-targeting a stored key would cost.
*/
export function foldQuoteVariantsByClass(raw: string): string {
return raw.replace(QUOTE_VARIANT_RE, (mark) =>
DOUBLE_WIDTH_QUOTE_VARIANTS.has(mark)
? QUOTE_VARIANT_FOLD_TARGET_DOUBLE
: QUOTE_VARIANT_FOLD_TARGET,
);
}
/**
* The separators whose RUNS are interchangeable at match time, folded to
* the one canonical join. Space, hyphen and underscore only: a
* `notes/link-candidates.ts` basename is a slug, so those three are the
* spellings one token sequence answers to in this store; dot and slash
* carry structure (`Node.js`, a path) and are never folded.
*/
const MATCH_SEPARATOR_RUN_RE = /[-_ ]+/;
/**
* Minimum normalized form length that participates in substring matching.
* The anchoring kernels gate the FULL forms they compare with this floor,
* and {@link matchFormVariants} gates the slug tail it emits - the tail is
* an emitted needle in its own right, and an unfloored one would anchor
* standalone words no full form ever matched (`AC/DC` must not mint `dc`).
* One constant here so the kernels and the generator cannot drift.
*/
export const MIN_ANCHOR_FORM_LENGTH = 3;
/**
* The entity-type folders of the canonical entity registry: the category
* slugs entity pages (`Brain/entities/<category>/<id>.md`) and entity ids
* (`ent-<category>-<slug>`) are minted from. Declared here - the leaf
* kernel every anchoring consumer shares - so the registry names its own
* folder vocabulary exactly once and every reader derives from the one
* list: {@link matchFormVariants} gates the slug tail on it, and the
* merge guard's default guarded categories ARE this list (re-exported
* there under its guard-facing name).
*/
export const ENTITY_TYPE_FOLDERS: ReadonlyArray<string> = Object.freeze([
"people",
"person",
"org",
"organization",
"organisation",
"company",
"team",
]);
/** Set form of {@link ENTITY_TYPE_FOLDERS} for the tail gate's lookups. */
const ENTITY_TYPE_FOLDER_SET: ReadonlySet<string> = new Set(ENTITY_TYPE_FOLDERS);
/**
* Fold a string's separator runs to the ONE canonical join: the tokens
* the string splits into, joined by single spaces (the tokens keep their
* slashes and dots - only the three foldable separators run together).
* Edge separator runs vanish with their empty edge tokens, exactly as
* they always did.
*/
function canonicalMatchJoin(s: string): string {
return s
.split(MATCH_SEPARATOR_RUN_RE)
.filter((token) => token !== "")
.join(" ");
}
/**
* Expand one ALREADY-NORMALIZED match form into the match variants the
* anchoring kernels compare with (note-change truth maintenance, F3;
* grid reduced per M2 of the PR 268 review). At most TWO variants:
*
* - The form's CANONICAL JOIN ({@link canonicalMatchJoin}): runs of
* space, hyphen and underscore (in any mix) fold to one space, so
* `ada lovelace`, `ada-lovelace`, `ada_lovelace` and
* `ada -_ lovelace` all expand to the one same variant. The kernels
* fold the needle AND the haystack through the same function, and
* one space-joined comparison answers for the whole variant class:
* tokens never contain separators, every join character counts as a
* non-word neighbour at the leading edge, and both the old grid and
* the folded one compare token sequences - so the fold is verdict-
* equivalent, at one scan instead of nine.
* - When the form's LAST `/`-segment (the slug itself) has a registry
* entity-type folder for its parent segment ({@link
* ENTITY_TYPE_FOLDERS} - the categories entity pages and ids are
* minted from; the parent SEGMENT gates, so the registry's deeper
* spellings `entities/people/ada_lovelace` and
* `Brain/entities/people/ada_lovelace` mint the tail too), the
* CANONICAL JOIN of that segment, so `people/ada_lovelace` also
* occurs in a text that only says `ada lovelace`. The segment prefix
* is registry
* decoration, not part of the mentioned name - and because the tail
* is an emitted needle in its own right, the tail meets
* {@link MIN_ANCHOR_FORM_LENGTH}, the same floor the kernels apply
* to full forms (`AC/DC` emits no `dc` for a standalone word to
* match). The FULL form is never floored or gated here: gating it
* stays the caller's call, unchanged. A slash after any other prefix
* carries no registry decoration - "Warner/Chappell" is one name
* with a slash in it, not a directory path - so `Warner/Chappell`
* mints no `chappell` and `Sony/ATV` no `atv` (M1, PR 268 review).
*
* The output is deterministic (the folded full form first, the folded
* tail after) and never empty for a non-empty separator-free form of at
* least {@link MIN_ANCHOR_FORM_LENGTH}. Deliberately NOT part of
* identity: the input is never re-normalized, `normalizeEntityName` is
* untouched, and nothing this returns may be persisted - identity keys
* keep their separators, which is exactly why two spellings can be two
* identities and one match class. Pure: no I/O, no state.
*/
export function matchFormVariants(form: string): string[] {
const out: string[] = [];
const full = canonicalMatchJoin(form);
if (full === "") return out;
out.push(full);
const lastSlash = form.lastIndexOf("/");
if (lastSlash >= 0) {
// The gate is the slug's OWN parent segment - the category folder an
// entity page's id sits in - not the whole prefix before the slash:
// a form that still carries the registry's deeper path
// (`Brain/entities/people/ada_lovelace`) has `people` as its
// immediate parent and mints its tail the same as the bare
// `people/ada_lovelace`, while `Warner/Chappell` has `warner` there
// and mints nothing (M1, PR 268 review).
const prefix = form.slice(0, lastSlash);
const folder = prefix.slice(prefix.lastIndexOf("/") + 1).toLowerCase();
if (ENTITY_TYPE_FOLDER_SET.has(folder)) {
const tail = canonicalMatchJoin(form.slice(lastSlash + 1));
if (tail.length >= MIN_ANCHOR_FORM_LENGTH) out.push(tail);
}
}
return out;
}
/**
* A letter or digit in ANY script: the word characters a match must not
* sit inside. The fold is deliberately NOT on this list - a separator is
* exactly where one token ends and the next begins, so `ada` may anchor
* inside `ada-lovelace` but never inside `canada`.
*/
const MATCH_WORD_CHAR_RE = /[\p{L}\p{N}]/u;
/**
* The scripts written WITHOUT word spaces - Han, Hiragana, Katakana,
* Thai, Lao, Khmer, Myanmar. In a run of such text every letter touches
* a letter, so a LEADING letter is not evidence of a mid-word hit the
* way it is in a spaced script: `田中太郎` never occurs in
* `鈴木花子と田中太郎` at a word boundary, and a boundary rule applied
* there verbatim would have silenced every CJK mention the pre-rule
* literal `includes` matching anchored. For a leading neighbour from one
* of these scripts the boundary requirement is therefore suspended - the
* script keeps its literal behavior back, spaced scripts keep the rule
* (a Latin neighbour blocks even inside otherwise-CJK text). Korean is
* deliberately NOT on this list: it writes spaces between words, so a
* leading Hangul letter keeps blocking - Korean attaches no prefixes,
* and `박철수` must not anchor `철수`.
*/
const NO_SPACE_SCRIPT_RE =
/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u;
/**
* True when `codePoint` belongs to a script written without word spaces
* (see {@link NO_SPACE_SCRIPT_RE}): a leading neighbour from one of
* these scripts never blocks an occurrence, because adjacency there
* carries no word-boundary information.
*/
function boundaryNotRequiredForScripts(codePoint: string): boolean {
return NO_SPACE_SCRIPT_RE.test(codePoint);
}
function isHighSurrogate(code: number): boolean {
return code >= 0xd800 && code <= 0xdbff;
}
function isLowSurrogate(code: number): boolean {
return code >= 0xdc00 && code <= 0xdfff;
}
/**
* The code point ending just before index `i`, or null at the string
* edge. Steps over a surrogate pair so an astral letter comes back
* whole rather than as a lone (always non-word) code unit.
*/
function codePointBefore(s: string, i: number): string | null {
if (i <= 0) return null;
const start =
i >= 2 && isLowSurrogate(s.charCodeAt(i - 1)) && isHighSurrogate(s.charCodeAt(i - 2))
? i - 2
: i - 1;
return s.slice(start, i);
}
/**
* True when `variant` occurs in one folded haystack string with a clean
* LEADING edge: some occurrence of it has no letter or digit directly
* before it - UNLESS that neighbour belongs to a script written without
* word spaces (see {@link NO_SPACE_SCRIPT_RE}), whose adjacency suspends
* the requirement. The trailing side is unchecked (H1, PR 268 review):
* every language that attaches endings to a name with no space - Russian
* `Ивана`, the German genitive `Annas`, a Korean particle run like
* `김철수는` - lost those mentions under a two-sided rule, and a
* trailing character is an inflection until proven otherwise. The
* leading-side rule is what keeps the separator fold from planting
* mid-word hits nothing anchored before (`canada` never anchors `ada`:
* the letter before the hit blocks it, folded spellings included); an
* occurrence beside a separator or the string edge is a token edge and
* anchors.
*/
function variantOccursAtWordBoundary(variant: string, haystack: string): boolean {
// An empty variant occurs everywhere and nowhere - refuse it rather
// than let a boundary-less occurrence anchor.
if (variant === "") return false;
let at = haystack.indexOf(variant);
while (at >= 0) {
const before = codePointBefore(haystack, at);
const blockedBefore =
before !== null && MATCH_WORD_CHAR_RE.test(before) && !boundaryNotRequiredForScripts(before);
if (!blockedBefore) return true;
at = haystack.indexOf(variant, at + 1);
}
return false;
}
/**
* True when ANY of the needle's match variants occurs with a clean
* leading edge in ANY of the haystack's match variants - the one
* comparison the anchoring kernels and their twins share. Both arrays
* come from {@link matchFormVariants}: the needle's variants are
* precomputed by the caller where it already precomputes forms, and the
* haystack's variants are expanded once per text, never once per entity.
* Pure: no I/O.
*/
export function matchVariantsOccurIn(
formVariants: ReadonlyArray<string>,
haystackVariants: ReadonlyArray<string>,
): boolean {
return formVariants.some((variant) =>
haystackVariants.some((haystack) => variantOccursAtWordBoundary(variant, haystack)),
);
}
/**
* One-shot convenience over {@link matchFormVariants} and
* {@link matchVariantsOccurIn}: does the single normalized form `form`
* occur, separator-folded and leading-edge-checked, in the normalized
* haystack text? For callers that match one form at a time; a caller
* holding precomputed variant lists calls {@link matchVariantsOccurIn}
* directly so the haystack expands once, not once per form.
*/
export function matchFormOccursIn(form: string, normalizedHaystack: string): boolean {
return matchVariantsOccurIn(matchFormVariants(form), matchFormVariants(normalizedHaystack));
}
/**
* The kernel's shape pass, WITHOUT the quote fold: NFC, trim, collapse
* whitespace runs, lowercase. Separated from {@link normalizeEntityName}
* so {@link differsOnlyByQuoteVariant} can name the difference the fold
* makes rather than re-deriving it from a second copy of these steps -
* and exported so a caller that pairs it with
* {@link foldQuoteVariantsByClass} (`dream-plan.ts`'s `topicKey`) shares
* these steps rather than copying them.
*/
export function normalizeEntityShape(raw: string): string {
return raw.normalize("NFC").trim().replace(WHITESPACE_RUN_RE, " ").toLowerCase();
}
/** NFC-normalise, trim, collapse whitespace runs, lowercase, fold quote variants. */
export function normalizeEntityName(raw: string): string {
return foldQuoteVariants(normalizeEntityShape(raw));
}
/**
* Registry code the quote-variant collision resolves its forward pointer
* through, and the structural command that resolves it.
*
* Both live in this LEAF module rather than at the consumer, and the
* `DIAGNOSTIC_SIGNALS` entry in `diagnostics.ts` is built from them, so
* the string still has exactly one definition. The usual arrangement -
* the consumer resolving `requireNextStep(code)` at module scope - is not
* available to the REGISTRY: `next-step.ts` reads `diagnostics.ts`, which
* reaches the entity registry through the doctor's entity checks and the
* search query-expansion module, so a registry import of `next-step.ts`
* closes an import cycle the architecture ratchet refuses
* (`tests/core/architecture/import-cycles.test.ts`). This module imports
* nothing, so both consumers can read it.
*
* The code's PRODUCER is `doctor/entity-checks.ts`, which spells it on the
* finding for a collision the fold created; every doctor surface resolves
* a finding's code through the rail (`nextCommandField` / `emitNextStep`),
* which is how the other registered doctor codes reach an operator. The
* registry's own refusals are thrown synchronously out of core and name
* the command from {@link ENTITY_QUOTE_VARIANT_COLLISION_COMMAND} - the
* same constant the registration is built from, so the two cannot drift.
*/
export const ENTITY_QUOTE_VARIANT_COLLISION_CODE = "entity-quote-variant-collision";
export const ENTITY_QUOTE_VARIANT_COLLISION_COMMAND = "o2b brain entity archive <name>";
/**
* The one sentence naming WHY two labels that coexisted before the fold
* cannot coexist after it. Written once here because three surfaces say
* it - the registry's write refusal, the registry's ambiguity error, and
* the doctor's finding - and an operator who meets it twice must read the
* same words, or the two readings become two conditions.
*/
export const ENTITY_QUOTE_VARIANT_CAUSE =
"the two differ only in typographic quote form and now resolve to one identity key";
/**
* True when `a` and `b` are two spellings of one identity that ONLY the
* quote fold unifies - they were distinct identities before it and are
* one after it.
*
* This is the predicate a collision message asks so it can name the right
* exit: a refusal an operator meets for the first time immediately after
* an upgrade needs to say WHY two records that used to coexist no longer
* can, and every other collision keeps the message it always had.
*/
export function differsOnlyByQuoteVariant(a: string, b: string): boolean {
const shapeA = normalizeEntityShape(a);
const shapeB = normalizeEntityShape(b);
return shapeA !== shapeB && foldQuoteVariants(shapeA) === foldQuoteVariants(shapeB);
}
/**
* True when ANY two of `labels` are one identity only the fold unifies.
*
* The n-ary form because a collision is a set of claimants, not a pair:
* three files can claim one key, and a message that only ever compared
* the first two would omit the cause on exactly the messiest vault.
*/
export function anyDiffersOnlyByQuoteVariant(labels: ReadonlyArray<string>): boolean {
for (let i = 0; i < labels.length; i++) {
for (let j = i + 1; j < labels.length; j++) {
if (differsOnlyByQuoteVariant(labels[i]!, labels[j]!)) return true;
}
}
return false;
}
/**
* The tail a collision message carries when, and only when, the labels
* were distinct identities before the quote fold and are one after it.
* Empty for every other collision, so the messages an operator already
* knows are unchanged and the new sentence means exactly what it says:
* this refusal is new, and here is why.
*
* Lives beside the predicate rather than at the registry because the
* doctor's finding needs the same words - see
* {@link ENTITY_QUOTE_VARIANT_CAUSE}.
*/
export function quoteVariantCollisionTail(...labels: ReadonlyArray<string>): string {
if (!anyDiffersOnlyByQuoteVariant(labels)) return "";
return (
` - ${ENTITY_QUOTE_VARIANT_CAUSE}; ` +
`run ${ENTITY_QUOTE_VARIANT_COLLISION_COMMAND} on the record you do not keep`
);
}
/**
* Validate an entity category: a lowercase kebab-ish slug with no path
* separators, traversal, or whitespace. Lowercases the input so
* `People` and `people` are the same category.
*/
export function validateEntityCategory(raw: string): string {
const category = raw.normalize("NFC").trim().toLowerCase();
if (!category) throw new Error("entity category must not be empty");
if (!/^[a-z0-9][a-z0-9-]*$/.test(category)) {
throw new Error(
`entity category must be a lowercase slug ([a-z0-9-], starting alphanumeric): ${JSON.stringify(raw)}`,
);
}
return category;
}
/**
* The identity key one canonical entity owns: `<category>:<normalized name>`.
* Two files claiming the same key are duplicates by definition.
*/
export function entityIdentityKey(category: string, name: string): string {
return `${validateEntityCategory(category)}:${normalizeEntityName(name)}`;
}
// ----- Label quality gate (A1 / t_657b365e) ---------------------------------
//
// `normalizeEntityName` above stays byte-stable for every label that
// carries no typographic quote variant: it is the identity kernel and its
// output feeds `entityIdentityKey`, so such a label must key identically to
// before. The quote fold is the ONE deliberate exception, its footprint is
// exactly the Unicode classes `foldQuoteVariants` names, and the corpus
// proof lives in tests/core/brain/entities/quote-variant-fold.test.ts. The
// label QUALITY pass lives here as
// a separate step applied BEFORE `normalizeEntityName` at every label-intake
// boundary (entity creation, fact-extract anchoring, atomic-facts anchoring):
// it strips Markdown decoration and surrounding punctuation, then rejects
// structurally-junk labels. Junk detection is STRUCTURAL only (Unicode
// property classes, never a natural-language word list); the sole vocabulary
// source is the operator-supplied denylist threaded in by the caller.
/**
* Upper bound on a sanitized label's length. A defensible ceiling: real
* entity names are short, and anything past this is a pasted blob, not a
* label. Exported so callers and tests share one source of truth.
*/
export const ENTITY_LABEL_MAX_LENGTH = 200;
/** Leading Markdown heading marker: one-to-six `#` then required space/tab. */
const HEADING_PREFIX_RE = /^#{1,6}[ \t]+/;
/**
* Matched Markdown emphasis/code wrappers, longest opener first so `**`
* is peeled before `*`. One matched pair is removed per sanitise pass;
* the outer loop re-runs until the label stops changing.
*/
const EMPHASIS_WRAPPERS: ReadonlyArray<readonly [string, string]> = [
["**", "**"],
["__", "__"],
["*", "*"],
["_", "_"],
["`", "`"],
] as const;
/** A leading run of Unicode punctuation (surrounding, not internal). */
const LEADING_PUNCT_RE = /^\p{P}+/u;
/** A trailing run of Unicode punctuation (surrounding, not internal). */
const TRAILING_PUNCT_RE = /\p{P}+$/u;
/** At least one letter or digit in ANY script - the "carries meaning" test. */
const HAS_LETTER_OR_DIGIT_RE = /[\p{L}\p{N}]/u;
/**
* One arm of the structural code-shape test: a character that only code
* (or a path, or a numeric identifier) plants inside a label - underscore,
* dot, slash, or a digit of ANY script. The letters are checked separately
* so word-position can be honoured for case.
*/
const CODE_SHAPE_CHAR_RE = /[_./\p{N}]/u;
/** Peel one matched emphasis/code wrapper pair, if present. */
function stripSurroundingEmphasis(s: string): string {
for (const [open, close] of EMPHASIS_WRAPPERS) {
if (s.length >= open.length + close.length + 1 && s.startsWith(open) && s.endsWith(close)) {
return s.slice(open.length, s.length - close.length).trim();
}
}
return s;
}
/**
* Strip surrounding Markdown emphasis/heading decoration and surrounding
* (never internal) punctuation from a raw label, iterating until stable.
*
* Examples: `**Foo**` -> `Foo`, `# Heading` -> `Heading`, `_bar_` -> `bar`,
* `(baz)` -> `baz`, `Foo.` -> `Foo`. Internal punctuation is preserved, so
* `Node.js`, `e.g`, and `C++` survive. NFC-normalises and trims first so a
* clean label (`Ada`, `Open Second Brain`, `café`, `Ада`) is returned
* byte-identical to `raw.trim()` - the backward-compatibility guarantee for
* every currently-clean label.
*/
export function sanitizeEntityLabel(raw: string): string {
let s = raw.normalize("NFC").trim();
let prev = "";
while (s !== prev) {
prev = s;
s = s.replace(HEADING_PREFIX_RE, "").trim();
s = stripSurroundingEmphasis(s);
s = s.replace(LEADING_PUNCT_RE, "").replace(TRAILING_PUNCT_RE, "").trim();
}
return s;
}
/**
* Why a label was rejected by {@link validateEntityLabel}.
*
* `shouted-heading` and `function-word-run` are the recall-quality growth:
* case-based verdicts that fire ONLY behind an {@link EntityLexiconHandle}
* (see {@link ValidateEntityLabelOptions.lexicon}) - with the handle null
* or absent the predicate is byte-identical to its pre-lexicon behavior.
*/
export type EntityLabelInvalidReason =
| "empty"
| "too-long"
| "no-letter-or-digit"
| "denylisted"
| "shouted-heading"
| "function-word-run";
export interface EntityLabelValidation {
readonly valid: boolean;
readonly reason?: EntityLabelInvalidReason;
}
/**
* The store-learned vocabulary the case-based verdicts consult, resolved
* by the entity-lexicon module and threaded in beside the denylist.
* Declared here - not imported - because this module is a leaf of the
* import graph and must keep importing nothing.
*/
export interface EntityLexiconHandle {
/**
* False for a caseless-script corpus (a script whose letters carry no
* upper/lower distinction): case statistics learned over such a corpus
* are meaningless, so a false handle suppresses BOTH case-based
* verdicts - `shouted-heading` and `function-word-run` never fire.
*/
readonly caseful: boolean;
/**
* Whether `token` is a store-learned common word. Tokens are handed
* over LOWERCASED by the predicate, so implementations do exact
* lookups against their learned lowercase keys.
*/
isCommonWord(token: string): boolean;
}
export interface ValidateEntityLabelOptions {
/**
* Operator-supplied denylist of exact labels, ALREADY normalised via
* `normalizeEntityName` (the comparison is post-normalization). The only
* vocabulary source; empty/absent means no name-based rejection.
*/
readonly denylist?: ReadonlySet<string>;
/**
* The store-learned lexicon behind the case-based verdicts
* (`shouted-heading`, `function-word-run`). Null or absent keeps the
* predicate byte-identical to its pre-lexicon behavior: those two
* reasons never fire.
*/
readonly lexicon?: EntityLexiconHandle | null;
}
/**
* True when the label has a structural CODE shape: it contains `_`, `.`,
* `/`, any digit, or an INTERNAL uppercase letter - a camelCase hump, an
* uppercase whose preceding character is a lowercase LETTER (`OpenAI`,
* `TheAnd`, `McDonald`). An uppercase that starts the label or follows a
* word boundary is Title Case, and the letters of an ALL-CAPS word follow
* their own uppercase, so neither reads as code - `Open Second Brain`
* stays a name and `MEETING NOTES` stays eligible for the case verdicts,
* while `FTS5`, `foo_bar`, `Node.js` and `path/to/x` are identifiers.
*/
function hasStructuralCodeShape(sanitized: string): boolean {
if (CODE_SHAPE_CHAR_RE.test(sanitized)) return true;
for (let i = 1; i < sanitized.length; i++) {
const ch = sanitized[i]!;
const prev = sanitized[i - 1]!;
if (ch !== ch.toLowerCase() && prev !== prev.toUpperCase()) return true;
}
return false;
}
/**
* True when some letter carries case - `l !== l.toLowerCase()`. Only a
* CASEFUL label is eligible for the case-based verdicts; a caseless one
* (CJK, all-lowercase text, digits-only) admits on the structural gate
* alone.
*/
function isCasefulLabel(sanitized: string): boolean {
for (const ch of sanitized) {
if (ch !== ch.toLowerCase()) return true;
}
return false;
}
/**
* True when every token of the (already whitespace-collapsed) label is a
* store-learned common word. Tokens are handed to the handle LOWERCASED,
* so implementations do exact lookups against lowercase keys.
*/
function isCommonWordRun(sanitized: string, lexicon: EntityLexiconHandle): boolean {
// `split` always yields at least one element (an empty string yields
// one empty token), so there is no empty-result guard to write; the
// per-token emptiness check below is what actually fires, on the empty
// edge tokens a leading/trailing separator produces.
const tokens = sanitized.split(WHITESPACE_RUN_RE);
return tokens.every((token) => token.length > 0 && lexicon.isCommonWord(token.toLowerCase()));
}
/**
* Validate an already-sanitized label. Structural rejections only, plus the
* operator denylist:
* - `empty` - nothing survived sanitisation;
* - `too-long` - exceeds {@link ENTITY_LABEL_MAX_LENGTH};
* - `no-letter-or-digit` - no `\p{L}`/`\p{N}` in any script (pure
* punctuation/symbols);
* - `denylisted` - normalised form is in the operator denylist.
*
* Behind a lexicon ({@link ValidateEntityLabelOptions.lexicon}) two
* CASE-BASED verdicts join, in this order after the four above:
* - `shouted-heading` - every letter is uppercase (a `# HEADING`
* residue), checked first;
* - `function-word-run` - every token is a store-learned common word
* (a single common token is already a run).
* Both run ONLY for a caseful label (some letter where `l !==
* l.toLowerCase()`) under a caseful handle, and BOTH are bypassed by a
* structural code shape ({@link hasStructuralCodeShape}) - `FTS5` and
* `foo_bar` admit even when they shout. The four structural verdicts are
* kept unconditionally; a caseless label admits on the structural gate
* alone. With the lexicon null or absent the two new reasons never fire
* and the verdicts are byte-identical to the pre-lexicon behavior.
*
* The single source of truth for label validity - creation (typed error),
* anchoring (logged skip), doctor, and prune all route through it. The
* hard-at-create vs advisory-heal POLICY over these verdicts lives in the
* callers, not here.
*/
export function validateEntityLabel(
sanitized: string,
opts: ValidateEntityLabelOptions = {},
): EntityLabelValidation {
if (sanitized.length === 0) return { valid: false, reason: "empty" };
if (sanitized.length > ENTITY_LABEL_MAX_LENGTH) return { valid: false, reason: "too-long" };
if (!HAS_LETTER_OR_DIGIT_RE.test(sanitized))
return { valid: false, reason: "no-letter-or-digit" };
if (opts.denylist && opts.denylist.has(normalizeEntityName(sanitized))) {
return { valid: false, reason: "denylisted" };
}
const lexicon = opts.lexicon;
if (!lexicon || !lexicon.caseful) return { valid: true };
if (!isCasefulLabel(sanitized)) return { valid: true };
if (hasStructuralCodeShape(sanitized)) return { valid: true };
if (sanitized === sanitized.toUpperCase()) return { valid: false, reason: "shouted-heading" };
if (isCommonWordRun(sanitized, lexicon)) return { valid: false, reason: "function-word-run" };
return { valid: true };
}
/** Convenience boolean wrapper over {@link validateEntityLabel}. */
export function isValidEntityLabel(
sanitized: string,
opts: ValidateEntityLabelOptions = {},
): boolean {
return validateEntityLabel(sanitized, opts).valid;
}
/** Typed error raised when a label is rejected at a creation boundary. */
export class InvalidEntityLabelError extends Error {
readonly reason: EntityLabelInvalidReason;
readonly raw: string;
readonly sanitized: string;
constructor(raw: string, sanitized: string, reason: EntityLabelInvalidReason) {
super(
`invalid entity label ${JSON.stringify(raw)} (${reason}): ` +
`sanitised form ${JSON.stringify(sanitized)} does not pass the label quality gate`,
);
this.name = "InvalidEntityLabelError";
this.reason = reason;
this.raw = raw;
this.sanitized = sanitized;
}
}
/**
* Sanitise `raw` and assert it passes the quality gate, returning the
* sanitized label for storage. Throws {@link InvalidEntityLabelError} on
* rejection - the creation-boundary contract (no silent drop).
*/
export function assertValidEntityLabel(raw: string, opts: ValidateEntityLabelOptions = {}): string {
const sanitized = sanitizeEntityLabel(raw);
const verdict = validateEntityLabel(sanitized, opts);
if (!verdict.valid) {
throw new InvalidEntityLabelError(raw, sanitized, verdict.reason!);
}
return sanitized;
}
/**
* Normalised match forms for an entity's labels (name + aliases): sanitise
* each, drop the structurally-invalid ones, then normalise the survivors.
* The shared anchoring kernel so fact-extract and atomic-facts compare
* facts against clean, valid label forms only. Pure - no I/O, no logging.
*/
export function entityMatchForms(
rawForms: ReadonlyArray<string>,
opts: ValidateEntityLabelOptions = {},
): string[] {
const out: string[] = [];
for (const raw of rawForms) {
const sanitized = sanitizeEntityLabel(raw);
if (!isValidEntityLabel(sanitized, opts)) continue;
out.push(normalizeEntityName(sanitized));
}
return out;
}