Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
57.7 KB
Raw Blame
/**
 * Auto-link entity mentions to known entity pages (orphan reduction, #1409;
 * entity recall, src/core/mentions/).
 *
 * `buildGazetteer` reads the brain's linkable entity pages (the pack-aware
 * per-source type set from `mentions/policy.ts`) and their names (titles plus
 * `page_aliases` rows: frontmatter aliases, declared aliases and title
 * subjects) into a token-Map lookup structure for body-text scanning.
 *
 * `findMentionedEntities` is a pure function that scans body text against
 * the gazetteer, applies the maximal-munch matcher (longest gazetteer
 * entry wins at each offset), case-sensitive entries, per-page ignore names,
 * the self-link guard, the cross-source guard, and the per-page
 * first-mention-only cap (1 link per (source_slug, target_slug)).
 *
 * Design decisions:
 *  - Linkable types come from the source's schema pack (primitive: entity
 *    types and their type aliases, product excluded) unioned with person,
 *    company, organization and entity.
 *  - Token-Map + multi-word phrase pass (no new deps, no regex alternation,
 *    no Aho-Corasick).
 *  - DB-source only — callers walk pages from the database.
 *  - `link_source='mentions'` writes are filtered out of backlink-count for
 *    search ranking (see postgres-engine.ts/pglite-engine.ts).
 *  - Self-link guard.
 *  - Ignore-list applied at gazetteer-build time, not match time. Built-in
 *    ambiguous tokens (Apple, Amazon, Square, Stripe, Box) are dropped from
 *    the gazetteer only when no corresponding entity page exists: a page the
 *    user created is trusted. `mentions.ignore` names are dropped always;
 *    pages listed in `mentions.exclude_slugs` get no entry at all (#5829).
 */

import { createHash } from 'crypto';
import type { BrainEngine } from './engine.ts';
import { isMissingSchemaObject } from './utils.ts';
import { CJK_SLUG_CHARS } from './cjk.ts';
import { stripCodeBlocks, isCrossSourceLinksEnabled } from './link-extraction.ts';
// #4222: shared generic-token reject list — same list gates enrichEntity
// minting and drives the junk_entity_hubs doctor check.
import { isGenericEntityToken } from './entity-name-quality.ts';
import { ALWAYS_LINKABLE_TYPES, canonicalTypeOf, linkableTypesFor, loadSourcePack, readMentionPolicy, type MentionPolicy, type PackTypes } from './mentions/policy.ts';
import { siblingVerdict, type SiblingCandidate } from './mentions/siblings.ts';
import { isShortCode } from './mentions/short-codes.ts';
import { bodyBlocks } from './page-body/segment-blocks.ts';
import { OperationError } from './ops/contract.ts';

/**
 * The four types that are always linkable, whatever the schema pack says. The
 * full per-source set is `linkableTypesFor` (mentions/policy.ts).
 */
export const LINKABLE_ENTITY_TYPES = ALWAYS_LINKABLE_TYPES;

let aliasGazetteerWarned = false;

/**
 * Minimum name length for gazetteer inclusion. Filters out 2-3 char names
 * (AI, YC, X, IBM) that produce dense false-positive auto-links in body text.
 */
const MIN_NAME_LENGTH = 4;
const MIN_CJK_NAME_LENGTH = 2;

/**
 * Built-in ignore list — common ambiguous tokens whose body-text mentions
 * are usually NOT references to the named brand/entity. Suppressed at
 * gazetteer-build time when no corresponding entity page exists.
 *
 * Per CK12 (codex outside-voice): if the user has explicitly created
 * `companies/apple` as a page, they want auto-link → ignore-list does
 * not override gazetteer presence. The list only suppresses entries
 * that would NOT otherwise be in the gazetteer.
 */
const DEFAULT_IGNORE_LIST = ['Apple', 'Amazon', 'Square', 'Stripe', 'Box', 'Meta', 'Target', 'Oracle'];

/** Where a gazetteer name came from: the page title or a `page_aliases` row's origin. */
export type GazetteerOrigin = 'title' | 'frontmatter' | 'declared' | 'subject';

export interface GazetteerEntry {
  /** Canonical page slug (e.g. `companies/acme-corp`). */
  slug: string;
  /** Source id (multi-source brains). 'default' for single-source. */
  source_id: string;
  /** Original title (preserved for the mention payload). */
  title: string;
  /** Lowercase title tokens in order. Length 1 = single-word entity. */
  tokens: string[];
  /** Absent on hand-built entries: a title. */
  origin?: GazetteerOrigin;
  /**
   * Set on case-sensitive entries (single-token declared aliases): the tokens
   * as written, NFC. A body token matches only when its original text equals.
   */
  caseTokens?: string[];
  /** Spelling being matched (the alias, not the display title, for aliases). */
  matchText?: string;
  /**
   * Set on a name two or three identity siblings claim (mentions/siblings.ts):
   * a match links to every sibling with the same tokens instead of one.
   */
  shared?: true;
  /**
   * #6158: the page is a connector's own contact rendering (`google_contact_id`
   * frontmatter), a near-empty stub of a person who may have a canonical page
   * elsewhere. Under the cross-source opt-in a mention in the stub's source
   * binds to that canonical twin instead (see findMentionedEntities).
   */
  connector_stub?: true;
  /** An X handle from the page's frontmatter: matches only where the text writes it as `@handle`. */
  atOnly?: true;
}

/** Frontmatter keys that hold a person's X handle. */
export const HANDLE_FRONTMATTER_KEYS = ['x_handle', 'twitter', 'twitter_handle', 'x'] as const;

const HANDLE_RE = /^[A-Za-z0-9_]{1,15}$/;
/** Values people write in a handle field when they do not know the handle. */
const PLACEHOLDER_HANDLES = new Set(['unknown', 'none', 'null', 'nil', 'na', 'tbd', 'todo', 'unset', 'undefined', 'private', 'redacted', 'anonymous', 'true', 'false', 'yes', 'no']);

/**
 * The X handle a frontmatter value names, lowercase without the `@`, or null
 * for anything that is not a handle: an empty value, a placeholder such as
 * `unknown`, an all-digit or all-zero value, a list or a longer text. An
 * `x.com/<handle>` or `twitter.com/<handle>` URL yields its handle.
 */
export function frontmatterHandle(value: string | null | undefined): string | null {
  const raw = value?.trim() ?? '';
  const url = /^(?:https?:\/\/)?(?:www\.)?(?:x|twitter)\.com\/([^/?#\s]+)/i.exec(raw);
  const v = (url ? url[1]! : raw).replace(/^@/, '');
  if (!v || !HANDLE_RE.test(v) || /^[\d_]+$/.test(v) || PLACEHOLDER_HANDLES.has(v.toLowerCase())) return null;
  return v.toLowerCase();
}

/** The name an entry is saved and diffed under: its tokens, case-preserved for a case-sensitive entry, `@`-prefixed for a handle. */
export function entryNameNorm(e: GazetteerEntry): string {
  return `${e.atOnly ? '@' : ''}${(e.caseTokens ?? e.tokens).join(' ')}`;
}

/**
 * Gazetteer is keyed by lowercase FIRST token. Multiple entries can
 * share a first token (e.g. "Acme" + "Acme Corp" + "Acme Foundation").
 * At match time, the scanner picks the entry with the most tokens that
 * matches the body-text token sequence at the current offset (maximal
 * munch).
 */
export type Gazetteer = Map<string, GazetteerEntry[]>;

/**
 * Fingerprint of every gazetteer entry (source_id, slug, title, tokens) for
 * the by-mention resume checkpoint. Hashing only the first-token bucket KEYS
 * missed a new "Acme Labs" beside an existing "Acme Corp", so resumed pages
 * silently skipped the new entity.
 */
export function hashGazetteer(gazetteer: Gazetteer): string {
  const entries: string[] = [];
  for (const bucket of gazetteer.values()) {
    for (const e of bucket) entries.push(`${e.source_id}\0${e.slug}\0${e.title}\0${e.tokens.join(' ')}\0${e.matchText ?? ''}${e.caseTokens ? `\0${e.caseTokens.join(' ')}` : ''}${e.atOnly ? '\0@' : ''}`);
  }
  // Matching semantics are part of the resume identity, not just DB contents.
  return createHash('sha256').update('katakana-boundaries-v5\n').update(entries.sort().join('\n')).digest('hex').slice(0, 8);
}

/** One row of the saved entry set the mention pass diffs (mention_gazetteer_entries). */
export interface GazetteerEntryKey {
  source_id: string;
  name_norm: string;
  target_slug: string;
  case_sensitive: boolean;
}

/** The gazetteer's entries as (source, normalized name, target, case flag), deduped and sorted. */
export function gazetteerEntryKeys(gazetteer: Gazetteer): GazetteerEntryKey[] {
  const out = new Map<string, GazetteerEntryKey>();
  for (const bucket of gazetteer.values()) {
    for (const e of bucket) {
      const key: GazetteerEntryKey = { source_id: e.source_id, name_norm: entryNameNorm(e), target_slug: e.slug, case_sensitive: !!e.caseTokens };
      out.set(`${key.source_id}\0${key.name_norm}\0${key.target_slug}\0${key.case_sensitive}`, key);
    }
  }
  return [...out.values()].sort((x, y) => x.source_id.localeCompare(y.source_id) || x.name_norm.localeCompare(y.name_norm)
    || x.target_slug.localeCompare(y.target_slug) || Number(x.case_sensitive) - Number(y.case_sensitive));
}

export interface Mention {
  /** Target page slug (the entity being mentioned). */
  slug: string;
  /** Target source id (cross-source guard). */
  source_id: string;
  /** Display name (original title). */
  name: string;
  /** Character offset in the ORIGINAL (un-stripped) body where the mention starts. */
  offset: number;
}

/** A name the gazetteer dropped, with the reason (read by `extract mentions --explain`). */
export interface DroppedName {
  source_id: string;
  slug: string;
  name: string;
  origin: GazetteerOrigin;
  reason: 'below_min_length' | 'generic_token' | 'alias_collision' | 'ambiguous_first_word' | 'ignored' | 'excluded_slug' | 'title_wins';
}

export interface BuildGazetteerOpts {
  /** Mention policy (types, `mentions.ignore`, `mentions.exclude_slugs`); read from config when absent. */
  policy?: MentionPolicy;
  /**
   * Authoritative build for the mention pass: a pack that fails to resolve or
   * an alias read that fails throws instead of degrading to a smaller set.
   */
  strict?: boolean;
  /** Collects every dropped name with its reason. */
  dropped?: DroppedName[];
}

export interface FindMentionsOpts {
  /** Source slug of the page being scanned. Used for self-link guard. */
  fromSlug: string;
  /** Source id of the page being scanned. Used for cross-source guard. */
  fromSourceId: string;
  /**
   * Lift the cross-source guard: a mention in source A of an entity that
   * lives only in source B produces a mention with the entity's own
   * `source_id`. Callers derive this from `link_resolution.cross_source`
   * (via `isCrossSourceLinksEnabled`) so the mention scan follows the same
   * operator opt-in as wikilink resolution. Default false — the historical
   * same-source-only posture.
   */
  allowCrossSource?: boolean;
  /**
   * Names the scanning page opts out of (its frontmatter `mention_ignore`),
   * compared on normalized tokens.
   */
  ignoreNames?: readonly string[];
}

// ============================================================
// Gazetteer construction
// ============================================================

/**
 * The CJK character set this module treats as char-level, declared ONCE.
 *
 * `CJK_SLUG_CHARS` (src/core/cjk.ts) is the repo-wide single source of truth
 * — Han U+4E00–9FFF, Hiragana, Katakana, Hangul syllables — and this module
 * now uses it verbatim.
 *
 * Note the deliberate behaviour change: the walkers here used to carry their
 * own copy of the ranges that also covered Han Extension A (U+3400–4DBF),
 * which cjk.ts scopes out repo-wide (see its header). Aligning on the shared
 * constant means Ext-A characters are no longer treated as CJK by
 * by-mention: they tokenize as word runs and, being a single sub-4-character
 * token, an Ext-A-only entity title now falls below MIN_NAME_LENGTH instead
 * of qualifying under MIN_CJK_NAME_LENGTH. Search, chunking and slug grammar
 * already ignore Ext-A, so this makes by-mention consistent with them rather
 * than being the one subsystem that disagrees.
 *
 * Everything below — TOKEN_RE, hasCJK(), cjkCharCount() and the two
 * per-character walkers — derives from this one import. There are no copies
 * of the ranges in this file.
 */
const CJK_CHAR_RE = new RegExp(`^[${CJK_SLUG_CHARS}]$`, 'u');

/**
 * Conservative code-point bounds for CJK_SLUG_CHARS, derived from the range
 * string itself (strip the `-` separators and the remaining characters are
 * exactly the range endpoints) so they can never drift from it. Used only
 * as a cheap pre-filter — Latin/Vietnamese text short-circuits before the
 * regex in the per-character walkers, which run over every body byte.
 */
const CJK_BOUNDS = ((): { min: number; max: number } => {
  let min = 0x10ffff;
  let max = 0;
  for (const ch of CJK_SLUG_CHARS.replace(/-/g, '')) {
    const cp = ch.codePointAt(0)!;
    if (cp < min) min = cp;
    if (cp > max) max = cp;
  }
  return { min, max };
})();

function isCJKChar(ch: string): boolean {
  const cp = ch.codePointAt(0) ?? 0;
  if (cp < CJK_BOUNDS.min || cp > CJK_BOUNDS.max) return false;
  return CJK_CHAR_RE.test(ch);
}

/**
 * Word-run tokenizer: a letter or ASCII digit, followed by any run of
 * letters, ASCII digits and combining marks — CJK excluded throughout, so
 * CJK keeps flowing through the per-character path in the walkers below.
 *
 * Latin scripts with diacritics tokenize as whole words instead of
 * fragmenting on every accented character — "Nguyễn" is one token, not
 * ["nguy","n"], and "Đà Nẵng" is ["đà","nẵng"], not ["n","ng"].
 *
 * Four deliberate boundaries, each of which was a real regression:
 *
 *  - The LEAD must be a letter or digit, so a token can never consist of
 *    combining marks alone. U+FE0F (VARIATION SELECTOR-16, category Mn)
 *    rides on most emoji, so a mark-only token would hijack the gazetteer
 *    key of every emoji-prefixed entity title ("❤️ Health Notes" keying on
 *    U+FE0F instead of "health") and collapse all of them into one shared,
 *    mutually-confusable bucket.
 *  - Combining marks ARE allowed after the lead. NFD Vietnamese is base
 *    letter + mark, so excluding \p{M} would re-fragment the exact names
 *    this tokenizer exists to keep whole.
 *  - Digits are ASCII-only, exactly as the previous /[a-zA-Z0-9]+/ was.
 *    \p{N} would additionally mint tokens for ¹ ½ 1 (Nl/No/non-ASCII Nd),
 *    and findMentionedEntities requires gazetteer tokens to be STRICTLY
 *    ADJACENT in the body — so a superscript between the words of
 *    "Acme Corp" would silently break a match that used to work.
 *  - Plain `u` flag, not `v`: the CJK exclusion is a negative lookahead
 *    over CJK_SLUG_CHARS, the same construction src/core/think/gather.ts
 *    already uses. No es2024 target requirement, no set-subtraction syntax.
 */
const TOKEN_RE = new RegExp(
  `(?![${CJK_SLUG_CHARS}])[\\p{L}0-9]` +
  `(?:(?![${CJK_SLUG_CHARS}])[\\p{L}\\p{M}0-9])*`,
  'gu',
);

/**
 * Canonical form for a single token. NFC only — canonical composition, no
 * compatibility folding — so an NFD body and an NFC gazetteer title produce
 * the same token, while diacritics stay significant ("Hồng" still must not
 * match "Hong").
 *
 * Applied PER TOKEN, never to the whole text: `Mention.offset` is contracted
 * to index into the ORIGINAL body (extract-ner.ts slices a context window
 * from it to infer the link verb), and normalizing the text up front would
 * silently shift every offset.
 */
function normalizeToken(s: string): string {
  return s.normalize('NFC').toLowerCase();
}

interface ScannedToken {
  text: string;       // lowercase
  offset: number;     // index in source
  length: number;     // original length (for span tracking)
}

/**
 * Body-text tokenizer. Returns `[token, offset]` pairs.
 *
 * Word runs: each TOKEN_RE match is one token, NFC-normalized and
 *   lowercased. Covers ASCII and diacritic Latin scripts like Vietnamese
 *   ("Nguyễn" → one token, not ["nguy","n"]).
 * CJK: each CJK character (Chinese/Japanese/Korean) is an individual
 *   token. This allows the normal maximal-munch scan path to reach CJK
 *   gazetteer entries without a separate substring pass.
 *
 * `offset` and `length` index into the ORIGINAL string — callers slice
 * context windows out of the untouched body with them.
 *
 * Possessive "Acme's" tokenizes as ['acme', 's'] (single-quote breaks the
 * run) — single-word "Acme" lookup succeeds at offset 0; the trailing 's'
 * is harmless noise.
 *
 * Exported so tests can assert on TOKENIZATION rather than only on the
 * resolved mention (see tokenizeTitle).
 */
export function tokenizeForScan(text: string): ScannedToken[] {
  const out: ScannedToken[] = [];
  TOKEN_RE.lastIndex = 0;
  let m: RegExpExecArray | null;

  // Collect word-run token spans first.
  const wordSpans: Array<{ start: number; end: number }> = [];
  while ((m = TOKEN_RE.exec(text)) !== null) {
    wordSpans.push({ start: m.index, end: m.index + m[0].length });
  }

  // Walk character-by-character: emit word-run tokens at their start
  // positions, then emit individual CJK characters for positions that fall
  // outside every word-run span.
  let spanIdx = 0;
  for (let i = 0; i < text.length;) {
    // Advance spanIdx past any spans that end before or at i.
    while (spanIdx < wordSpans.length && wordSpans[spanIdx]!.end <= i) {
      spanIdx++;
    }

    // If position i is inside a word-run span, emit the full token and jump
    // past it.
    if (spanIdx < wordSpans.length && i >= wordSpans[spanIdx]!.start && i < wordSpans[spanIdx]!.end) {
      const span = wordSpans[spanIdx]!;
      const token = text.slice(span.start, span.end);
      out.push({ text: normalizeToken(token), offset: span.start, length: token.length });
      i = span.end;
      spanIdx++;
      continue;
    }

    // CJK: emit as individual character token.
    const cp = text.codePointAt(i) ?? 0;
    const charLen = cp > 0xffff ? 2 : 1; // surrogate pair
    const charStr = text.slice(i, i + charLen);
    if (isCJKChar(charStr)) {
      out.push({ text: normalizeToken(charStr), offset: i, length: charLen });
      i += charLen;
    } else {
      i++;
    }
  }
  return out;
}

export function hasCJK(s: string): boolean {
  for (const ch of s) {
    if (isCJKChar(ch)) return true;
  }
  return false;
}

function cjkCharCount(s: string): number {
  let count = 0;
  for (const ch of s) {
    if (isCJKChar(ch)) count++;
  }
  return count;
}

/**
 * Tokenize a page title for gazetteer insertion.
 *
 * Word-run titles: TOKEN_RE tokenization, NFC-normalized and lowercased —
 *   ASCII plus diacritic Latin scripts (Vietnamese, etc.).
 * CJK titles (no word-run content): split into individual characters —
 *   e.g. "纳瓦尔" → ["纳","瓦","尔"]. This allows normal multi-token
 *   maximal-munch matching to work with character-level CJK tokens
 *   produced by `tokenizeForScan`.
 * Mixed CJK+word-run titles: word-run parts tokenized normally, CJK parts
 *   split into individual characters.
 *
 * Exported so tests can assert on TOKENIZATION rather than only on the
 * resolved mention — a mention-only assertion passes even with a tokenizer
 * that fragments the title and the body symmetrically.
 */
export function tokenizeTitle(title: string): string[] {
  const tokens: string[] = [];
  TOKEN_RE.lastIndex = 0;
  const hasWordRun = TOKEN_RE.test(title);
  if (hasWordRun) {
    // Mixed word-run+CJK or pure word-run: tokenize word runs normally,
    // then append individual CJK characters in order.
    TOKEN_RE.lastIndex = 0;
    let m: RegExpExecArray | null;
    const wordSpans: Array<{ start: number; end: number; text: string }> = [];
    while ((m = TOKEN_RE.exec(title)) !== null) {
      wordSpans.push({ start: m.index, end: m.index + m[0].length, text: normalizeToken(m[0]) });
    }
    let spanIdx = 0;
    for (let i = 0; i < title.length;) {
      while (spanIdx < wordSpans.length && wordSpans[spanIdx]!.end <= i) spanIdx++;
      if (spanIdx < wordSpans.length && i >= wordSpans[spanIdx]!.start && i < wordSpans[spanIdx]!.end) {
        tokens.push(wordSpans[spanIdx]!.text);
        i = wordSpans[spanIdx]!.end;
        spanIdx++;
        continue;
      }
      const cp = title.codePointAt(i) ?? 0;
      const charLen = cp > 0xffff ? 2 : 1;
      const charStr = title.slice(i, i + charLen);
      if (isCJKChar(charStr)) {
        tokens.push(normalizeToken(charStr));
        i += charLen;
      } else {
        i++;
      }
    }
    return tokens;
  }
  // Pure CJK (no word-run content): split into individual characters.
  if (hasCJK(title)) {
    for (let i = 0; i < title.length;) {
      const cp = title.codePointAt(i) ?? 0;
      const charLen = cp > 0xffff ? 2 : 1;
      const ch = title.slice(i, i + charLen);
      if (isCJKChar(ch)) tokens.push(normalizeToken(ch));
      i += charLen;
    }
    return tokens;
  }
  // Non-ASCII, non-CJK title (emoji, symbols, etc.) — empty set.
  return [];
}

/**
 * Build a token-Map gazetteer from every linkable entity page in the brain.
 *
 * Linkable types are resolved per source (`linkableTypesFor`: the four
 * always-linkable types plus the source pack's entity types and their type
 * aliases). Soft-deleted pages excluded. Each entity page contributes its
 * title and its `page_aliases` rows (frontmatter aliases, declared aliases,
 * title subjects; one row per alias with precedence frontmatter > declared >
 * subject). Guards on every name: the 4-character minimum, the generic-token
 * reject list for single-token person titles and every single-token alias,
 * the ignore lists. Guards on aliases only: an alias claimed by two pages of one source is dropped, an alias equal
 * to an existing title in the same source yields to the title, and a
 * single-token alias equal to the first word of another entity's multi-word
 * name in the source is dropped (an ambiguous first word never links alone).
 * Ignore-list applied per CK12: built-in ambiguous tokens dropped unless
 * the user has explicitly created the corresponding page; `mentions.ignore`
 * names always drop; a page in `mentions.exclude_slugs` contributes neither
 * its title nor its aliases.
 *
 * Returned gazetteer is keyed by lowercase first token; entries with the
 * same first token co-exist in the same bucket (e.g. "Acme" + "Acme Corp").
 */
export async function buildGazetteer(
  engine: BrainEngine,
  opts: BuildGazetteerOpts = {},
): Promise<Gazetteer> {
  const policy = opts.policy ?? await readMentionPolicy(engine);
  const dropped = opts.dropped;
  const drop = (d: DroppedName) => { dropped?.push(d); };
  const sources = await engine.executeRaw<{ id: string }>('SELECT DISTINCT source_id AS id FROM pages WHERE deleted_at IS NULL', []);
  const typesBySource = new Map<string, Set<string>>();
  const packBySource = new Map<string, PackTypes | null>();
  for (const { id } of sources) {
    const pack = await loadSourcePack(engine, id, { strict: opts.strict });
    packBySource.set(id, pack);
    typesBySource.set(id, new Set(linkableTypesFor(pack, policy)));
  }
  const allTypes = [...new Set([...typesBySource.values()].flatMap(set => [...set]))];
  const linkable = (sourceId: string | null, type: string | null) => !!type && (typesBySource.get(sourceId ?? 'default')?.has(type) ?? false);
  const rows = (await engine.executeRaw<{ slug: string; source_id: string | null; title: string | null; type: string | null; connector_stub: boolean;
    h1: string | null; h2: string | null; h3: string | null; h4: string | null }>(
    `SELECT slug, source_id, title, type, (frontmatter ->> 'google_contact_id') IS NOT NULL AS connector_stub,
            frontmatter ->> $2 AS h1, frontmatter ->> $3 AS h2, frontmatter ->> $4 AS h3, frontmatter ->> $5 AS h4
     FROM pages
     WHERE type = ANY($1::text[])
       AND deleted_at IS NULL`,
    [allTypes, ...HANDLE_FRONTMATTER_KEYS],
  )).filter(r => linkable(r.source_id, r.type));
  const stubPages = new Set(rows.filter(r => r.connector_stub).map(r => `${r.source_id ?? 'default'}\0${r.slug}`));
  const stubFlag = (src: string, slug: string) => stubPages.has(`${src}\0${slug}`) ? { connector_stub: true as const } : {};
  const kindOf = new Map(rows.map(r => [`${r.source_id ?? 'default'}\0${r.slug}`, canonicalTypeOf(r.type, packBySource.get(r.source_id ?? 'default') ?? null)]));
  const isPerson = (src: string, slug: string) => kindOf.get(`${src}\0${slug}`) === 'person';

  const ignoreSet = new Set<string>(DEFAULT_IGNORE_LIST);
  const userIgnore = new Set(policy.ignore.map(n => tokenizeTitle(n).join(' ')).filter(Boolean));
  const excludedSlugs = new Set(policy.excludeSlugs);

  // page_aliases rows joined to LIVE linkable entity pages (read before the
  // titles: frontmatter aliases are explicit claims a title can yield to).
  let aliasRows: Array<{ alias_norm: string; slug: string; source_id: string | null; title: string | null; type: string | null;
    origin: string | null; case_sensitive: boolean | null; alias_text: string | null; identity: unknown }> = [];
  try {
    aliasRows = (await engine.executeRaw<typeof aliasRows[number]>(
      `SELECT pa.alias_norm, pa.slug, pa.source_id, p.title, p.type, pa.origin, pa.case_sensitive, pa.alias_text, p.frontmatter->>'identity' AS identity
       FROM page_aliases pa
       JOIN pages p ON p.slug = pa.slug AND p.source_id = pa.source_id
       WHERE p.type = ANY($1::text[])
         AND p.deleted_at IS NULL`,
      [allTypes],
    )).filter(r => linkable(r.source_id, r.type));
  } catch (err) {
    // pre-v110 brains: no page_aliases table — titles-only gazetteer.
    // Any OTHER failure (connection blip, permission) warns once per process
    // (adversarial F12): a silently titles-only gazetteer under-links every
    // page processed until restart, and nobody would know why. A strict
    // (mention-pass) build throws instead, so nothing is reconciled against
    // a partial gazetteer.
    if (opts.strict) throw err;
    if (!isMissingSchemaObject(err, 'table', 'page_aliases') && !aliasGazetteerWarned) {
      aliasGazetteerWarned = true;
      console.error(`[gbrain] gazetteer alias load degraded (titles-only): ${err instanceof Error ? err.message : String(err)}`);
    }
  }

  // Explicit claims, per source and name: the pages whose frontmatter writes
  // the name as an alias or as their X handle. A page titled with a name that
  // exactly one other page of its kind claims, while it claims nothing itself,
  // is a thin page for that person or company: its title yields the name to
  // the claimant (a handle mention lands on the person, not on a page whose
  // slug is the handle) once the claimant's own entry for the name survives
  // the alias guards below. Two claimants keep the title.
  const handlesOf = (r: typeof rows[number]) => [...new Set([r.h1, r.h2, r.h3, r.h4].map(frontmatterHandle).filter((h): h is string => !!h))];
  const explicit = new Map<string, Set<string>>();
  const claim = (src: string, key: string, slug: string) => {
    if (!key) return;
    const k = `${src}\0${key}`;
    explicit.set(k, (explicit.get(k) ?? new Set<string>()).add(slug));
  };
  for (const a of aliasRows) if (a.origin === 'frontmatter' || a.origin === null) claim(a.source_id ?? 'default', tokenizeTitle(a.alias_norm ?? '').join(' '), a.slug);
  for (const r of rows) for (const h of handlesOf(r)) claim(r.source_id ?? 'default', tokenizeTitle(h).join(' '), r.slug);
  const claimant = (src: string, key: string, slug: string): string | null => {
    const slugs = explicit.get(`${src}\0${key}`);
    if (!slugs || slugs.size !== 1 || slugs.has(slug)) return null;
    const [other] = slugs;
    return kindOf.get(`${src}\0${other}`) === kindOf.get(`${src}\0${slug}`) ? other! : null;
  };
  /** `source\0name\0claimant` for every title that yields its name to a claimant whose entry survives. */
  const yielded = new Set<string>();
  const yielding: Array<{ entry: GazetteerEntry; base: Omit<DroppedName, 'reason'>; key: string }> = [];

  // Per source: the first word of every multi-word person name (titles and
  // aliases) and the pages that write it. A one-word person title equal to
  // another page's one is a bare first name: it never links alone, the same
  // rule single-token aliases follow below.
  const personFirstWords = new Map<string, Map<string, Set<string>>>();
  const notePersonName = (src: string, slug: string, name: string | null) => {
    const tokens = tokenizeTitle(name ?? '');
    if (tokens.length < 2 || !isPerson(src, slug)) return;
    const words = personFirstWords.get(src) ?? new Map<string, Set<string>>();
    words.set(tokens[0]!, (words.get(tokens[0]!) ?? new Set<string>()).add(slug));
    personFirstWords.set(src, words);
  };
  for (const r of rows) notePersonName(r.source_id ?? 'default', r.slug, r.title);
  for (const a of aliasRows) notePersonName(a.source_id ?? 'default', a.slug, a.alias_norm);

  const gazetteer: Gazetteer = new Map();
  const add = (entry: GazetteerEntry) => {
    const key = entry.tokens[0]!;
    const bucket = gazetteer.get(key);
    if (bucket) bucket.push(entry);
    else gazetteer.set(key, [entry]);
  };
  // Per source: the first token of every multi-word entity name (titles here,
  // aliases below), for the ambiguous-first-word guard on single-token aliases.
  const firstWords = new Map<string, Set<string>>();
  const noteFirstWord = (src: string, tokens: string[]) => {
    if (tokens.length < 2) return;
    const set = firstWords.get(src) ?? new Set<string>();
    set.add(tokens[0]!);
    firstWords.set(src, set);
  };
  for (const row of rows) {
    if (!row.title) continue;
    const src = row.source_id ?? 'default';
    const base = { source_id: src, slug: row.slug, name: row.title, origin: 'title' as const };
    if (excludedSlugs.has(row.slug)) { drop({ ...base, reason: 'excluded_slug' }); continue; }
    if (!hasCJK(row.title) && row.title.length < MIN_NAME_LENGTH) { drop({ ...base, reason: 'below_min_length' }); continue; }
    if (hasCJK(row.title) && cjkCharCount(row.title) < MIN_CJK_NAME_LENGTH) { drop({ ...base, reason: 'below_min_length' }); continue; }
    // CK12 policy: an ignore-listed name the user explicitly created a page
    // for is always allowed, so DEFAULT_IGNORE_LIST never drops a TITLE
    // (every row here is a real page); it bites on the ALIAS entries below,
    // which are not user-created pages. `mentions.ignore` and
    // `mentions.exclude_slugs` are the operator's levers for titles.

    const tokens = tokenizeTitle(row.title);
    if (tokens.length === 0) continue;
    if (tokens[0]!.length < MIN_NAME_LENGTH && tokens.length === 1) { drop({ ...base, reason: 'below_min_length' }); continue; }
    if (userIgnore.has(tokens.join(' '))) { drop({ ...base, reason: 'ignored' }); continue; }
    // #4222: a single-generic-token title ("Will", "Chief", "Info") is a
    // junk-hub magnet — every prose occurrence of the word would accrete
    // another mention edge onto a near-empty page. Dropped from the gazetteer
    // even though the page exists (unlike the CK12 ignore-list rule above,
    // which trusts user-created pages: these titles are overwhelmingly
    // extractor-minted, and the page itself stays intact — only the
    // auto-link accretion stops). #6958: every page type, not only person
    // (a company titled "Info" magnetized the same way). Multi-token titles
    // ("Will Smith") are unaffected; aliases of every type are checked below.
    if (tokens.length === 1 && isGenericEntityToken(tokens[0]!)) { drop({ ...base, reason: 'generic_token' }); continue; }
    noteFirstWord(src, tokens);
    if (tokens.length === 1 && isPerson(src, row.slug) && [...personFirstWords.get(src)?.get(tokens[0]!) ?? []].some(s => s !== row.slug)) { drop({ ...base, reason: 'ambiguous_first_word' }); continue; }
    const entry: GazetteerEntry = { slug: row.slug, source_id: src, title: row.title, tokens, origin: 'title', matchText: row.title, ...stubFlag(src, row.slug) };
    const owner = claimant(src, tokens.join(' '), row.slug);
    if (owner) { const key = `${src}\0${tokens.join(' ')}\0${owner}`; yielded.add(key); yielding.push({ entry, base, key }); continue; }
    add(entry);
  }

  // ── Alias entries ────────────────────────────────────────────────────────
  // page_aliases rows become additional gazetteer entries, so a body mention
  // of "saoirse" links to people/saoirse-x and "QUCO" links to the account
  // that declares it.
  // Guards (stricter than titles — aliases are not user-created pages):
  //   - one row per (source, alias, page) with origin precedence
  //     frontmatter > declared > subject
  //   - ignore-list applies CASE-INSENSITIVELY with NO existing-page escape
  //     (aliases store normalized lowercase; DEFAULT_IGNORE_LIST is cased)
  //   - an alias claimed by >1 page at its best origin within a source is
  //     dropped (ambiguous); a lower-precedence claim yields
  //   - aliases colliding with any existing page TITLE in the SAME source
  //     are skipped (the title entry wins; per-source scoping per R2-9),
  //     unless that title yielded the name to this page's explicit claim
  //   - MIN_NAME_LENGTH and the generic-token list apply to the alias string
  //   - a single-token alias equal to the first word of a multi-word entity
  //     name in the same source is dropped
  //   - a case-sensitive row (a single-token declared alias) matches only
  //     the original-case text
  const ignoreLc = new Set(Array.from(ignoreSet, (n) => n.toLowerCase()));
  // Per-source title index for alias-vs-title collision checks.
  const titleBySource = new Set<string>();
  for (const r of rows) {
    if (r.title) titleBySource.add(`${r.source_id ?? 'default'} ${r.title.toLowerCase()}`);
  }
  const rank = (origin: string | null) => origin === 'subject' ? 2 : origin === 'declared' ? 1 : 0;
  // Best (lowest) origin rank per (source, alias, slug), then per (source, alias).
  const bestPerPage = new Map<string, typeof aliasRows[number]>();
  for (const a of aliasRows) {
    const k = `${a.source_id ?? 'default'} ${a.alias_norm} ${a.slug}`;
    const prev = bestPerPage.get(k);
    if (!prev || rank(a.origin) < rank(prev.origin)) bestPerPage.set(k, a);
  }
  const claims = new Map<string, { rank: number; slugs: Set<string>; pages: SiblingCandidate[]; shared?: boolean }>();
  for (const a of bestPerPage.values()) {
    const k = `${a.source_id ?? 'default'} ${a.alias_norm}`;
    const r = rank(a.origin);
    const c = claims.get(k);
    const page = { slug: a.slug, title: a.title, type: a.type, identity: a.identity };
    if (!c || r < c.rank) claims.set(k, { rank: r, slugs: new Set([a.slug]), pages: [page] });
    else if (r === c.rank && !c.slugs.has(a.slug)) { c.slugs.add(a.slug); c.pages.push(page); }
  }
  // A name identity siblings share (the subject of "Account sheet: X" and
  // "CRM record: X", or a code both declare) links to every sibling.
  for (const [k, c] of claims) {
    if (c.slugs.size > 1) c.shared = siblingVerdict(c.pages, packBySource.get(k.slice(0, k.indexOf(' '))) ?? null, policy) === 'group';
  }
  const aliasEntries: Array<{ entry: GazetteerEntry; base: Omit<DroppedName, 'reason'> }> = [];
  for (const a of bestPerPage.values()) {
    const alias = a.alias_norm?.trim();
    if (!alias || !a.title) continue;
    const src = a.source_id ?? 'default';
    const origin = (a.origin ?? 'frontmatter') as GazetteerOrigin;
    const written = a.case_sensitive && a.alias_text ? a.alias_text : alias;
    const base = { source_id: src, slug: a.slug, name: written, origin };
    const claim = claims.get(`${src} ${alias}`)!;
    if (rank(a.origin) > claim.rank) continue;
    if (excludedSlugs.has(a.slug)) { drop({ ...base, reason: 'excluded_slug' }); continue; }
    if (claim.slugs.size > 1 && !claim.shared) { drop({ ...base, reason: 'alias_collision' }); continue; }
    // A declared 2-3 character code passed the own-page cue gate at derivation; it links only as written.
    const shortCode = a.origin === 'declared' && !!a.case_sensitive && !!a.alias_text && isShortCode(a.alias_text);
    if (alias.length < MIN_NAME_LENGTH && !hasCJK(alias) && !shortCode) { drop({ ...base, reason: 'below_min_length' }); continue; }
    if (hasCJK(alias) && cjkCharCount(alias) < MIN_CJK_NAME_LENGTH) { drop({ ...base, reason: 'below_min_length' }); continue; }
    if (ignoreLc.has(alias.toLowerCase())) { drop({ ...base, reason: 'ignored' }); continue; }
    const tokens = tokenizeTitle(alias);
    if (tokens.length === 0) continue;
    if (titleBySource.has(`${src} ${alias.toLowerCase()}`) && !yielded.has(`${src}\0${tokens.join(' ')}\0${a.slug}`)) { drop({ ...base, reason: 'title_wins' }); continue; }
    if (tokens[0]!.length < MIN_NAME_LENGTH && tokens.length === 1 && !shortCode) { drop({ ...base, reason: 'below_min_length' }); continue; }
    if (userIgnore.has(tokens.join(' '))) { drop({ ...base, reason: 'ignored' }); continue; }
    if (tokens.length === 1 && isGenericEntityToken(tokens[0]!)) { drop({ ...base, reason: 'generic_token' }); continue; }
    const caseTokens = a.case_sensitive && a.alias_text ? caseTokensOf(a.alias_text) : undefined;
    const entry: GazetteerEntry = { slug: a.slug, source_id: src, title: a.title, tokens, origin, matchText: alias,
      ...(caseTokens && caseTokens.length === tokens.length ? { caseTokens } : {}), ...(claim.shared ? { shared: true as const } : {}),
      ...stubFlag(src, a.slug) };
    noteFirstWord(src, tokens);
    aliasEntries.push({ entry, base });
  }

  // ── Handle entries ───────────────────────────────────────────────────────
  // A frontmatter X handle (x_handle, twitter, twitter_handle, x) links where
  // the text writes it as @handle. Where a thin page titled with the handle
  // yielded it above, the bare handle links to the claimant too, as it linked
  // to the thin page before. Two pages claiming one handle link neither.
  const aliasKeys = new Set(aliasEntries.map(({ entry }) => `${entry.source_id}\0${entry.tokens.join(' ')}\0${entry.slug}`));
  for (const row of rows) {
    if (!row.title) continue;
    const src = row.source_id ?? 'default';
    for (const handle of handlesOf(row)) {
      const tokens = tokenizeTitle(handle);
      const key = tokens.join(' ');
      const base = { source_id: src, slug: row.slug, name: `@${handle}`, origin: 'frontmatter' as const };
      if (tokens.length === 0) continue;
      if (excludedSlugs.has(row.slug)) { drop({ ...base, reason: 'excluded_slug' }); continue; }
      if ((explicit.get(`${src}\0${key}`)?.size ?? 0) > 1) { drop({ ...base, reason: 'alias_collision' }); continue; }
      if (handle.length < MIN_NAME_LENGTH) { drop({ ...base, reason: 'below_min_length' }); continue; }
      if (ignoreLc.has(handle) || userIgnore.has(key)) { drop({ ...base, reason: 'ignored' }); continue; }
      if (tokens.length === 1 && isGenericEntityToken(tokens[0]!)) { drop({ ...base, reason: 'generic_token' }); continue; }
      const entry: GazetteerEntry = { slug: row.slug, source_id: src, title: row.title, tokens, origin: 'frontmatter', matchText: handle, ...stubFlag(src, row.slug) };
      add({ ...entry, matchText: `@${handle}`, atOnly: true });
      const k = `${src}\0${key}\0${row.slug}`;
      if (yielded.has(k) && !aliasKeys.has(k)) { aliasKeys.add(k); aliasEntries.push({ entry, base: { ...base, name: handle } }); }
    }
  }
  const claimed = new Set<string>();
  for (const { entry, base } of aliasEntries) {
    if (entry.tokens.length === 1 && firstWords.get(entry.source_id)?.has(entry.tokens[0]!)) {
      drop({ ...base, reason: 'ambiguous_first_word' });
      continue;
    }
    add(entry);
    claimed.add(`${entry.source_id}\0${entry.tokens.join(' ')}\0${entry.slug}`);
  }
  // A title yields only to a claimant whose entry for the name survived;
  // otherwise it keeps the name, as it did before the claim.
  for (const { entry, base, key } of yielding) {
    if (claimed.has(key)) drop({ ...base, reason: 'alias_collision' });
    else add(entry);
  }

  // Sort each bucket by token-count DESC so maximal-munch walks longest-first;
  // a handle entry comes before a same-length name, so `@name` reaches the
  // page that claims the handle; ties break on (source_id, slug) so the pick
  // among same-name entries is deterministic instead of following DB row order.
  for (const bucket of gazetteer.values()) {
    bucket.sort((a, b) =>
      (b.tokens.length - a.tokens.length)
      || (Number(!!b.atOnly) - Number(!!a.atOnly))
      || a.source_id.localeCompare(b.source_id)
      || a.slug.localeCompare(b.slug));
  }
  return gazetteer;
}

/** Original-case tokens of a name (NFC, not lowercased), aligned with tokenizeTitle. */
function caseTokensOf(name: string): string[] {
  return tokenizeForScan(name).map(t => name.slice(t.offset, t.offset + t.length).normalize('NFC'));
}

// ============================================================
// Body-text scanner (pure)
// ============================================================

/** Suffixes that may attach to a Hangul name: an optional title or honorific,
 * then up to two particles or copula forms (지원씨는, 지원에게서, 지원이었다).
 * Any other attached Hangul continues a longer word (지원하는, 지원금), which
 * was 99% of 1,041 word-internal matches on a Korean corpus while this suffix
 * set kept 65 of 74 real name mentions (docs/designs/hangul-mention-boundaries.md).
 */
const HANGUL_NAME_TITLES = '씨 님 오빠 언니 누나 선배 후배 선생님 선생 교수 대표 사장 회장 팀장 부장 과장 이사 의원 기자 작가 감독 측';
const HANGUL_NAME_PARTICLES = '이 가 은 는 을 를 의 에 에게 에게서 한테 한테서 께 께서 와 과 랑 이랑 도 만 로 으로 로서 으로서 로부터 으로부터 에서 부터 까지 처럼 보다 마저 조차 뿐 밖에 이나 이든 이라도 라도 이여 이며 이고 이다 입니다 이에요 예요 이었다 였다 이었던 였던 이라는 라는 이라고 라고 이란 이야 야 아';
const hangulAlternation = (words: string) => `(?:${words.split(' ').sort((a, b) => b.length - a.length).join('|')})`;
const HANGUL_NAME_SUFFIX_RE = new RegExp(
  `^${hangulAlternation(HANGUL_NAME_TITLES)}?${hangulAlternation(HANGUL_NAME_PARTICLES)}{0,2}(?![가-힣])`, 'u',
);

/** Korean uses word spaces; Han/Kana retain character-substring matching.
 * Validate each candidate before maximal-munch selection so an invalid longer
 * phrase does not consume a valid shorter name. The name must start a word and
 * end at a non-Hangul character or an attached name suffix (above).
 */
function hasHangulMatchBoundary(
  text: string, tokens: ScannedToken[], start: number, entry: GazetteerEntry,
): boolean {
  if (!entry.tokens.every(t => /^[가-힣]+$/u.test(t))) return true;
  const first = tokens[start]!;
  if (first.offset > 0 && /[가-힣]/u.test(text[first.offset - 1]!)) return false;
  const last = tokens[start + entry.tokens.length - 1]!;
  const end = last.offset + last.length;
  const actual = text.slice(first.offset, end).replace(/\s+/gu, ' ');
  const expected = (entry.matchText ?? entry.tokens.join('')).trim().replace(/\s+/gu, ' ');
  return actual === expected && HANGUL_NAME_SUFFIX_RE.test(text.slice(end, end + 12));
}

/** A name written only in Katakana (ー included) must be a whole Katakana run,
 * so デン matches 「デン」, デンに, 喫茶デン and デン・カフェ but not インシデント
 * or デンタル. Katakana (small and supplementary kana included), ー and the
 * voicing marks continue a run; Hiragana, Han, Latin, punctuation and ・ end it.
 * This is a script-span rule, not morphology: a short name inside a real
 * Katakana compound is refused too, and the compound can be its own alias.
 */
const KATAKANA_NAME_TOKEN_RE = /^[\p{Script=Katakana}ー]$/u;
const KATAKANA_RUN_CHARS = '\\p{Script=Katakana}ー\\u3099-\\u309c\\uff70\\uff9e\\uff9f';
const KATAKANA_RUN_BEFORE_RE = new RegExp(`[${KATAKANA_RUN_CHARS}]$`, 'u');
const KATAKANA_RUN_AFTER_RE = new RegExp(`^[${KATAKANA_RUN_CHARS}]`, 'u');

function hasKatakanaMatchBoundary(
  text: string, tokens: ScannedToken[], start: number, entry: GazetteerEntry,
): boolean {
  if (!entry.tokens.every(t => KATAKANA_NAME_TOKEN_RE.test(t))) return true;
  const first = tokens[start]!;
  const last = tokens[start + entry.tokens.length - 1]!;
  const end = last.offset + last.length;
  return text.slice(first.offset, end) === entry.tokens.join('')
    && !KATAKANA_RUN_BEFORE_RE.test(text.slice(Math.max(0, first.offset - 2), first.offset))
    && !KATAKANA_RUN_AFTER_RE.test(text.slice(end, end + 2));
}

/**
 * Scan body text for mentions of gazetteer entities. Pure function — no
 * IO. Returns `Mention[]` ordered by offset, deduped per
 * `(fromSourceId, fromSlug → entry.source_id, entry.slug)` pair
 * (first-mention-only cap).
 *
 * Matcher is maximal-munch: at each token offset, the longest gazetteer
 * entry that matches the body-token sequence wins. Single-word entries
 * are length-1 maximal matches.
 *
 * Guards (deterministic):
 *  - D13 self-link: skip when BOTH `source_id` and `slug` match the
 *    scanning page. Slug uniqueness is `(source_id, slug)`, so once the
 *    cross-source guard is lifted a foreign namesake is a real target.
 *  - Cross-source: skip when `fromSourceId !== entry.source_id` UNLESS
 *    `opts.allowCrossSource` (the `link_resolution.cross_source` opt-in —
 *    the same switch wikilink resolution honours). A same-name entity in
 *    the scanning page's OWN source always outranks a cross-source twin,
 *    whichever way the switch is set (bucket order is length-only, so the
 *    first maximal match may be the foreign twin).
 *  - First-mention-only cap: dedup by `(entry.source_id, entry.slug)` (one
 *    link per target page regardless of how many body mentions there are).
 *
 * Code-block and HTML-comment stripping via `stripCodeBlocks` (preserves offsets, so the
 * returned mention offsets index into the ORIGINAL text not the stripped
 * text — useful for downstream debugging tools).
 */
export function findMentionedEntities(
  text: string,
  gazetteer: Gazetteer,
  opts: FindMentionsOpts,
): Mention[] {
  if (!text || gazetteer.size === 0) return [];
  const stripped = stripCodeBlocks(text, { onHtmlComment: () => {} });
  const tokens = tokenizeForScan(stripped);
  if (tokens.length === 0) return [];
  const ignored = new Set((opts.ignoreNames ?? []).map(n => tokenizeTitle(n).join(' ')).filter(Boolean));
  // A case-sensitive entry matches only the original-case body text.
  const caseMatches = (entry: GazetteerEntry, at: number) => !entry.caseTokens || entry.caseTokens.every((t, k) => {
    const tok = tokens[at + k]!;
    return stripped.slice(tok.offset, tok.offset + tok.length).normalize('NFC') === t;
  });
  // A handle entry matches only where the text writes it as @handle.
  const atMatches = (entry: GazetteerEntry, at: number) => !entry.atOnly || stripped[tokens[at]!.offset - 1] === '@';

  const out: Mention[] = [];
  const seenTargets = new Set<string>();
  let i = 0;

  while (i < tokens.length) {
    const head = tokens[i]!;
    const bucket = gazetteer.get(head.text);
    if (!bucket) {
      i++;
      continue;
    }

    // Maximal-munch: bucket is pre-sorted longest-first. Find the first
    // entry whose subsequent tokens all match the body sequence.
    let matched: GazetteerEntry | null = null;
    let matchedTokens = 0;
    for (const entry of bucket) {
      if (i + entry.tokens.length > tokens.length) continue;
      if (!hasHangulMatchBoundary(stripped, tokens, i, entry)) continue;
      if (!hasKatakanaMatchBoundary(stripped, tokens, i, entry)) continue;
      if (!caseMatches(entry, i)) continue;
      if (!atMatches(entry, i)) continue;
      if (entry.tokens.length === 1) {
        matched = entry;
        matchedTokens = 1;
        break;
      }
      // Multi-word: validate subsequent tokens.
      if (i + entry.tokens.length > tokens.length) continue;
      let allMatch = true;
      for (let k = 1; k < entry.tokens.length; k++) {
        if (tokens[i + k]!.text !== entry.tokens[k]) {
          allMatch = false;
          break;
        }
      }
      if (allMatch) {
        matched = entry;
        matchedTokens = entry.tokens.length;
        break;
      }
    }

    if (!matched) {
      i++;
      continue;
    }

    // Same-name twin in the scanning page's own source wins over a foreign
    // one: bucket order is length-only, so `matched` may be the cross-source
    // twin even when an own-source entry with identical tokens exists.
    const sameName = (e: GazetteerEntry, want: GazetteerEntry) =>
      e.tokens.length === want.tokens.length
        && e.tokens.every((t, k) => t === want.tokens[k])
        && caseMatches(e, i)
        && atMatches(e, i)
        && hasHangulMatchBoundary(stripped, tokens, i, e)
        && hasKatakanaMatchBoundary(stripped, tokens, i, e);
    if (matched.source_id !== opts.fromSourceId) {
      const want = matched;
      const own = bucket.find(e => e.source_id === opts.fromSourceId && sameName(e, want));
      if (own) matched = own;
    }
    // #6158: under the cross-source opt-in, a connector's contact stub (own
    // source or not) yields to the canonical same-name page in another
    // source: the one such page, or the one that shares its slug. Several
    // candidates and no slug match keep the stub (an honest tie).
    if (opts.allowCrossSource && matched.connector_stub) {
      const stub = matched;
      const canonical = bucket.filter(e => !e.connector_stub && e.source_id !== stub.source_id && sameName(e, stub));
      const pick = canonical.length === 1 ? canonical[0] : canonical.find(e => e.slug === stub.slug);
      if (pick && canonical.filter(e => e.slug === pick.slug).length === 1) matched = pick;
    }
    if (ignored.size > 0 && ignored.has(matched.tokens.join(' '))) {
      i += matchedTokens;
      continue;
    }

    // Guards.
    if (matched.source_id === opts.fromSourceId && matched.slug === opts.fromSlug) {
      i += matchedTokens;
      continue;
    }
    if (!opts.allowCrossSource && matched.source_id !== opts.fromSourceId) {
      i += matchedTokens;
      continue;
    }
    const target = `${matched.source_id}\0${matched.slug}`;
    if (seenTargets.has(target)) {
      i += matchedTokens;
      continue;
    }

    // A shared sibling name links every sibling, unless the scanning page is one of them.
    const want = matched.tokens;
    const group = matched.shared
      ? bucket.filter(e => e.shared && e.source_id === matched!.source_id && e.tokens.length === want.length
        && e.tokens.every((t, k) => t === want[k]) && caseMatches(e, i) && atMatches(e, i))
      : [matched];
    if (group.length > 1 && group.some(e => e.source_id === opts.fromSourceId && e.slug === opts.fromSlug)) {
      i += matchedTokens;
      continue;
    }
    for (const e of group) {
      const key = `${e.source_id}\0${e.slug}`;
      if (seenTargets.has(key)) continue;
      out.push({ slug: e.slug, source_id: e.source_id, name: e.title, offset: head.offset });
      seenTargets.add(key);
    }
    i += matchedTokens;
  }

  return out;
}

// ============================================================
// Stale-mention detection (read-only)
// ============================================================

/** One `link_source='mentions'` row the current gazetteer no longer produces. */
export interface StaleMention {
  from: string;
  to: string;
  /** `link_kind` as stored; NULL rows (legacy / pre-v98) report as 'plain'. */
  kind: string;
  /** `link_type` as stored (a verb on typed_ner rows), which `gbrain unlink --link-type` must name. */
  link_type: string;
}

/**
 * Mentions of a segmented page: the title, then each stored block, as the mention pass reads a body it cannot reuse
 * runs for. Reads only. Null when a block is too long for any block reader (the page is left out of the scan).
 */
async function segmentedBodyMentions(engine: BrainEngine, page: { id: number; slug: string; source_id: string; title: string | null },
  gazetteer: Gazetteer, opts: FindMentionsOpts): Promise<Mention[] | null> {
  const found = findMentionedEntities(page.title ?? '', gazetteer, opts);
  try {
    for await (const block of bodyBlocks(engine, page, { separator: '\n\n' })) found.push(...findMentionedEntities(block.text, gazetteer, opts));
  } catch (error) {
    if (error instanceof OperationError && error.canonicalCode === 'page_body_segmented') return null;
    throw error;
  }
  return found;
}

export interface StaleMentionsScan {
  /** Live pages carrying at least one `link_source='mentions'` row. */
  totalPagesWithMentions: number;
  /** Pages actually re-scanned (bounded by `opts.limit`). */
  pagesScanned: number;
  /** `mentions` rows on the scanned pages. */
  linksScanned: number;
  /** Of those, rows whose target the current gazetteer no longer produces. */
  staleLinks: number;
  /** Stale counts split by `link_kind` — `typed_ner` rows come from extract-ner. */
  staleByKind: Record<string, number>;
  /** True when the brain has NO linkable entity pages at all. */
  emptyGazetteer: boolean;
  /** First few stale rows, for an operator-facing message. */
  examples: StaleMention[];
}

const STALE_MENTIONS_DEFAULT_LIMIT = 500;
const STALE_MENTIONS_MAX_EXAMPLES = 5;

/**
 * Re-derive what `extract links --by-mention` would produce today and report
 * `link_source='mentions'` rows that no longer follow from the current
 * gazetteer + page bodies.
 *
 * STRICTLY READ-ONLY. This deletes nothing and writes nothing; it exists so
 * the drift is visible, because the scan's write path (`addLinksBatch`) is
 * purely additive. Re-running the scan adds today's correct links ALONGSIDE
 * rows left by an older gazetteer, an older tokenizer, or a body that has
 * since stopped mentioning the entity — nothing ever removes those.
 *
 * A row is counted stale when the current scan does not produce its
 * (source_id, slug) target from the page's body. That test is deliberately
 * target-based rather than kind-based, so it covers `link_kind='typed_ner'`
 * rows too: extract-ner derives those from the same mention set, so a target
 * this scan no longer yields cannot have a live verb-typed edge either. The
 * converse does NOT hold — a still-produced target says nothing about
 * whether the stored verb is still right — so `typed_ner` counts are
 * reported separately rather than folded into one number.
 *
 * Bounded by `limit` (default 500) in slug order for determinism. The
 * returned `totalPagesWithMentions` is the unbounded figure so callers can
 * say what was and was not covered instead of implying full coverage.
 */
export async function scanStaleMentions(
  engine: BrainEngine,
  opts: { limit?: number } = {},
): Promise<StaleMentionsScan> {
  const limit = opts.limit ?? STALE_MENTIONS_DEFAULT_LIMIT;

  const totalRow = await engine.executeRaw<{ count: number }>(
    `SELECT COUNT(DISTINCT l.from_page_id)::int AS count
       FROM links l
       JOIN pages p ON p.id = l.from_page_id
      WHERE l.link_source = 'mentions'
        AND p.deleted_at IS NULL`,
    [],
  );
  const totalPagesWithMentions = totalRow[0]?.count ?? 0;

  const empty: StaleMentionsScan = {
    totalPagesWithMentions,
    pagesScanned: 0,
    linksScanned: 0,
    staleLinks: 0,
    staleByKind: {},
    emptyGazetteer: false,
    examples: [],
  };
  if (totalPagesWithMentions === 0) return empty;

  const gazetteer = await buildGazetteer(engine);
  const allowCrossSource = await isCrossSourceLinksEnabled(engine);

  // Same bounded page set for both queries so the link rows and the bodies
  // can never describe different pages.
  const scannedCte =
    `WITH scanned AS (
       SELECT p.id
         FROM pages p
        WHERE p.deleted_at IS NULL
          AND EXISTS (
            SELECT 1 FROM links l
             WHERE l.from_page_id = p.id AND l.link_source = 'mentions'
          )
        ORDER BY p.slug
        LIMIT $1
     )`;

  const bodies = await engine.executeRaw<{
    id: number; slug: string; source_id: string | null; body: string; title: string | null; body_storage: string | null;
  }>(
    `${scannedCte}
     SELECT p.id, p.slug, p.source_id, p.title, p.body_storage,
            COALESCE(p.title, '') || E'\n\n' || COALESCE(p.compiled_truth, '') || E'\n\n' || COALESCE(p.timeline, '') AS body
       FROM pages p
       JOIN scanned s ON s.id = p.id
      ORDER BY p.slug`,
    [limit],
  );

  const rows = await engine.executeRaw<{
    from_id: number; from_slug: string; to_slug: string;
    to_source_id: string | null; link_kind: string | null; link_type: string;
  }>(
    `${scannedCte}
     SELECT l.from_page_id AS from_id, f.slug AS from_slug,
            t.slug AS to_slug, t.source_id AS to_source_id, l.link_kind, l.link_type
       FROM links l
       JOIN scanned s ON s.id = l.from_page_id
       JOIN pages f ON f.id = l.from_page_id
       JOIN pages t ON t.id = l.to_page_id
      WHERE l.link_source = 'mentions'
      ORDER BY f.slug, t.slug`,
    [limit],
  );

  const byPage = new Map<number, typeof rows>();
  for (const r of rows) {
    const bucket = byPage.get(r.from_id);
    if (bucket) bucket.push(r);
    else byPage.set(r.from_id, [r]);
  }

  const staleByKind: Record<string, number> = {};
  const examples: StaleMention[] = [];
  let staleLinks = 0;

  for (const page of bodies) {
    const stored = byPage.get(page.id);
    if (!stored || stored.length === 0) continue;

    const fromSourceId = page.source_id ?? 'default';
    const opts = { fromSlug: page.slug, fromSourceId, allowCrossSource };
    // #9121: a segmented page's body columns are empty; scan its stored blocks (read-only) or skip a page no block reader can read.
    const mentions = page.body_storage === 'segmented'
      ? await segmentedBodyMentions(engine, { id: page.id, slug: page.slug, source_id: fromSourceId, title: page.title }, gazetteer, opts)
      : findMentionedEntities(page.body, gazetteer, opts);
    if (!mentions) continue;
    const produced = new Set(mentions.map(m => `${m.source_id}::${m.slug}`));

    for (const row of stored) {
      const key = `${row.to_source_id ?? 'default'}::${row.to_slug}`;
      if (produced.has(key)) continue;
      staleLinks++;
      const kind = row.link_kind ?? 'plain';
      staleByKind[kind] = (staleByKind[kind] ?? 0) + 1;
      if (examples.length < STALE_MENTIONS_MAX_EXAMPLES) {
        examples.push({ from: page.slug, to: row.to_slug, kind, link_type: row.link_type });
      }
    }
  }

  return {
    totalPagesWithMentions,
    pagesScanned: bodies.length,
    linksScanned: rows.length,
    staleLinks,
    staleByKind,
    emptyGazetteer: gazetteer.size === 0,
    examples,
  };
}