Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
43.7 KB
Raw Blame
/**
 * The mention pass: keeps entity pages' derived aliases and every page's
 * `link_source='mentions'` links current, incrementally, from `gbrain extract
 * --stale` and the managed stale path (sync's inline extraction stays
 * link-only).
 *
 * One pass, per source in scope:
 *   1. Policy. `auto_link=false` or `mentions.auto_link=false` removes plain
 *      mention links and derived alias rows once and records `disabled`;
 *      re-enabling makes every page due. A change of the policy fingerprint
 *      (linkable type set, cross-source policy, ignore list, alias deny list,
 *      multi-word aliases, sibling merge) makes every page due once. The
 *      status row records the extractor versions its count reflects; when
 *      they differ from this binary's, `pending` is recounted once.
 *   2. Alias refresh: linkable entity pages whose derived aliases were
 *      refreshed at another content revision (or alias version) get their
 *      `declared` and `subject` rows rewritten; pages that stopped being
 *      entities lose them. Frontmatter rows are never touched. Managed
 *      brains write through the coordinated writer (`page_aliases` is guarded).
 *   3. Authoritative gazetteer: a pack or alias read that fails writes
 *      nothing and records `failed` with the error.
 *   4. Entry diff: the gazetteer's (source, name, target, case) entries are
 *      compared with the saved set under an advisory lock; on change the set
 *      is saved, the source generation bumps, and only pages the change can
 *      affect are marked due (pages linking to a removed or retargeted
 *      target, plus pages whose text matches an added name through the real
 *      tokenizer, prefiltered by a substring superset).
 *   5. Reconcile due pages (content revision, extractor version or an
 *      invalidation differs): per batch, publish only when the source
 *      generations and each page's content revision are unchanged since the
 *      scan read them (conditional publish), so an edit or a gazetteer change
 *      during the scan leaves the page due. Only plain mention rows are
 *      touched; `typed_ner` rows stay.
 *   6. Status row: state, pending pages (mention-due pages plus linkable
 *      entity pages with stale derived aliases, counted when the pass ends
 *      or its deadline stops it), last pass, generation. `coverage` on
 *      entity cards and get_backlinks reads it.
 *
 * MENTION_EXTRACTOR_VERSION is separate from LINK_EXTRACTOR_VERSION_TS, so a
 * gazetteer change never reruns link or timeline extraction.
 *
 * A segmented page (body in segments, NULL body columns) is scanned block by
 * block (page-body/segment-blocks.ts), the title first, keeping each target's
 * first mention, and incrementally: each run's mentions are stored on its
 * page_segment_derivations row keyed by its text, the gazetteer's digest and
 * the scan options, so only runs that changed are read again. A name whose
 * tokens straddle a segment cut (a blank line, heading, dated line or message
 * boundary) is not matched. A gazetteer change
 * marks every segmented page in scope due (no SQL prefilter reads its body).
 * Derived aliases are still refreshed for inline entity pages only.
 */

import { createHash } from 'node:crypto';
import type { BrainEngine, LinkBatchInput } from '../engine.ts';
import { pipelined } from '../page-state/transactions.ts';
import { isCrossSourceLinksEnabled } from '../link-extraction.ts';
import {
  buildGazetteer, entryNameNorm, findMentionedEntities, gazetteerEntryKeys, tokenizeTitle,
  type FindMentionsOpts, type Gazetteer, type GazetteerEntry, type GazetteerEntryKey, type Mention,
} from '../by-mention.ts';
import { bodyBlocks, detachedValue } from '../page-body/segment-blocks.ts';
import { commitRuns, derivationScope, keyDigest, planRuns, readRuns, SegmentDerivationRaced, type DeriveRun } from '../page-body/segment-derive.ts';
import { canonicalJson } from '../belief/keys.ts';
import { beliefCoveredSources, unmanagedBeliefPoint } from '../belief/projection-point.ts';
import { classifyItemFailure } from '../item-failure.ts';
import { linkableTypesFor, loadSourcePack, parseNameList, readMentionPolicy, type MentionPolicy, type PackTypes } from './policy.ts';
import { deriveEntityAliases, type DerivedAlias } from './aliases.ts';
import { normalizeAliasList } from '../search/alias-normalize.ts';
import { withCoordinatedWrite } from '../persistence/context.ts';
import { maintenanceAttribution } from '../persistence/attribution.ts';
import { managedPersistenceEnabled } from '../persistence/ownership.ts';

/** Bump to rescan every page's mentions once (gazetteer or scanner behavior change). */
export const MENTION_EXTRACTOR_VERSION = 7;
/** Bump to re-derive every entity page's declared aliases and title subject once. */
export const ALIAS_DERIVATION_VERSION = 4;

const RECONCILE_BATCH = 200;
const ALIAS_BATCH = 200;
const PREFILTER_BATCH = 500;

export type MentionState = 'complete' | 'pending' | 'disabled' | 'failed';

export interface MentionPassResult {
  /** Pages whose mention links were reconciled and published. */
  pages: number;
  created: number;
  removed: number;
  /** Pages left due because they (or the gazetteer) changed during the scan. */
  skipped: number;
  /** Entity pages whose derived alias rows changed. */
  aliasPages: number;
  /** Mention-due pages after the pass. */
  remaining: number;
  state: MentionState;
  error?: string;
  /** Pages whose own content the scanner refused (a run it must read whole is over its bound); they stay due and the pass goes on. */
  failed?: number;
  failures?: MentionPageFailure[];
}

export interface MentionPageFailure { slug: string; source_id: string; code: string; message: string }

/** The failed pages a pass lists; `failed` still counts every one. */
export const MENTION_FAILURE_REPORT_LIMIT = 20;

export interface MentionPassOpts {
  sourceId?: string;
  /** Epoch ms; the pass stops between batches after it. Absent = run to completion. */
  deadline?: number;
  signal?: AbortSignal;
  /** Test seam: runs after a batch is scanned and before it is published. */
  beforePublish?: (pageIds: number[]) => Promise<void>;
}

const DUE_PREDICATE = `(s.page_id IS NULL OR s.mention_revision IS DISTINCT FROM p.knowledge_revision OR s.mention_version IS DISTINCT FROM $2)`;

/** Mention-due live pages, per source (or one source). */
export async function countMentionDuePages(engine: Pick<BrainEngine, 'executeRaw'>, sourceId?: string): Promise<number> {
  const rows = await engine.executeRaw<{ n: number }>(
    `SELECT count(*)::int AS n FROM pages p LEFT JOIN page_mention_state s ON s.page_id = p.id
      WHERE p.deleted_at IS NULL AND ($1::text IS NULL OR p.source_id = $1) AND ${DUE_PREDICATE}`,
    [sourceId ?? null, MENTION_EXTRACTOR_VERSION]);
  return Number(rows[0]?.n ?? 0);
}

/**
 * Index-due live pages of one source: mention-due pages plus linkable entity
 * pages (`types`) whose derived aliases are behind their content or
 * ALIAS_DERIVATION_VERSION. The status row's `pending` holds this count.
 */
export async function countIndexDuePages(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string, types: string[]): Promise<number> {
  const rows = await engine.executeRaw<{ n: number }>(
    `SELECT count(*)::int AS n FROM pages p LEFT JOIN page_mention_state s ON s.page_id = p.id
      WHERE p.deleted_at IS NULL AND p.source_id = $1 AND (${DUE_PREDICATE}
        OR (p.type = ANY($3::text[]) AND (s.alias_revision IS DISTINCT FROM p.knowledge_revision OR s.alias_version IS DISTINCT FROM $4)))`,
    [sourceId, MENTION_EXTRACTOR_VERSION, types, ALIAS_DERIVATION_VERSION]);
  return Number(rows[0]?.n ?? 0);
}

/**
 * The extractor versions a status row's `pending` count reflects, stored after
 * the policy hash in `mention_index_status.policy_fingerprint`
 * (`<hash>;v<mention>.<alias>`). A row without it, or with other versions,
 * was counted by another binary.
 */
export const INDEX_VERSION_STAMP = `v${MENTION_EXTRACTOR_VERSION}.${ALIAS_DERIVATION_VERSION}`;

export function splitStoredFingerprint(stored: string | null | undefined): { hash: string | null; versions: string | null } {
  if (!stored) return { hash: null, versions: null };
  const at = stored.indexOf(';');
  return at < 0 ? { hash: stored, versions: null } : { hash: stored.slice(0, at), versions: stored.slice(at + 1) };
}

async function sourcesInScope(engine: BrainEngine, sourceId?: string): Promise<string[]> {
  if (sourceId) return [sourceId];
  const rows = await engine.executeRaw<{ id: string }>('SELECT id FROM sources ORDER BY id');
  return rows.map(r => r.id);
}

function policyFingerprint(types: string[], crossSource: boolean, policy: MentionPolicy): string {
  return createHash('sha256').update(JSON.stringify({
    types, crossSource, ignore: [...policy.ignore].map(n => n.toLowerCase()).sort(),
    aliasDeny: [...policy.aliasDeny].map(n => n.toLowerCase()).sort(), multiword: policy.multiwordAliases, siblings: policy.siblingMerge,
  })).digest('hex').slice(0, 16);
}

async function writeStatus(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string, patch: {
  state: MentionState; pending?: number; error?: string | null; lastPassAt?: string | null; bumpGeneration?: boolean; fingerprint?: string | null;
}): Promise<void> {
  await engine.executeRaw(
    `INSERT INTO mention_index_status (source_id, generation, state, pending, counted_at, last_pass_at, policy_fingerprint, error, updated_at)
     VALUES ($1, CASE WHEN $6::boolean THEN 1 ELSE 0 END, $2, COALESCE($3::int, 0), now(), $4::timestamptz, $7, $5, now())
     ON CONFLICT (source_id) DO UPDATE SET
       state = EXCLUDED.state,
       pending = COALESCE($3::int, mention_index_status.pending),
       counted_at = CASE WHEN $3::int IS NULL THEN mention_index_status.counted_at ELSE now() END,
       last_pass_at = COALESCE($4::timestamptz, mention_index_status.last_pass_at),
       error = $5,
       generation = mention_index_status.generation + CASE WHEN $6::boolean THEN 1 ELSE 0 END,
       policy_fingerprint = CASE WHEN $8::boolean THEN $7 ELSE mention_index_status.policy_fingerprint END,
       updated_at = now()`,
    [sourceId, patch.state, patch.pending ?? null, patch.lastPassAt ?? null, patch.error ?? null, patch.bumpGeneration === true,
      patch.fingerprint ?? null, patch.fingerprint !== undefined]);
}

/**
 * #9548: a disable (exclusive) and each reconcile batch reading the source's generation (shared) take this first, before page
 * keys or status rows: a journaled disable locks its origin pages, which a cross-source batch holding the status row links to.
 */
async function lockMentionSources(tx: Pick<BrainEngine, 'executeRaw'>, sourceIds: string[], shared: boolean): Promise<void> {
  await tx.executeRaw(`SELECT count(${shared ? 'pg_advisory_xact_lock_shared' : 'pg_advisory_xact_lock'}(hashtext('gbrain:mention-publish:' || s)::bigint))
    FROM (SELECT DISTINCT s FROM unnest($1::text[]) AS x(s) ORDER BY s) k`, [sourceIds]);
}

/** Run `fn` in a transaction; on a managed brain, as a coordinated write for `sourceId` (page_aliases is guarded). */
async function aliasWrite<T>(engine: BrainEngine, managed: boolean, sourceId: string, fn: (tx: BrainEngine) => Promise<T>): Promise<T> {
  if (!managed) return engine.transaction(fn);
  const attribution = await maintenanceAttribution(engine);
  return engine.transaction(tx => withCoordinatedWrite(tx, [sourceId], () => fn(tx), attribution));
}

/**
 * Replace one page's derived alias rows (declared, subject) with what its
 * current title and body yield, inside the caller's transaction (import and
 * `reindex --aliases`; on a managed brain the caller holds the coordinated
 * write). A page that is not a linkable entity, or linking that is off,
 * leaves no derived rows. Frontmatter rows are untouched.
 */
export async function writeDerivedAliases(tx: Pick<BrainEngine, 'executeRaw' | 'getConfig'>, sourceId: string,
  page: { slug: string; title: string | null; type: string | null; compiled_truth: string | null; timeline?: string | null; frontmatter?: Record<string, unknown> | null },
  opts: { pack?: PackTypes | null; policy?: MentionPolicy | null } = {}): Promise<number> {
  // A config read that fails (null policy) leaves the rows for the next sweep rather than failing the caller's write.
  const policy = opts.policy !== undefined ? opts.policy : await readMentionPolicy(tx).catch(() => null);
  if (!policy) return 0;
  const types = policy.enabled ? linkableTypesFor(opts.pack !== undefined ? opts.pack : await loadSourcePack(tx, sourceId), policy) : [];
  const aliases = types.includes(page.type ?? '') ? deriveEntityAliases(page, aliasOptsOf(policy)).aliases : [];
  await tx.executeRaw(`DELETE FROM page_aliases WHERE source_id = $1 AND slug = $2 AND origin IN ('declared','subject')`, [sourceId, page.slug]);
  if (aliases.length) {
    await tx.executeRaw(
      `INSERT INTO page_aliases (source_id, alias_norm, slug, origin, case_sensitive, alias_text)
       SELECT $1, a, $2, o, c::boolean, t FROM unnest($3::text[], $4::text[], $5::text[], $6::text[]) AS x(a, o, c, t) ON CONFLICT DO NOTHING`,
      [sourceId, page.slug, aliases.map(a => a.alias_norm), aliases.map(a => a.origin), aliases.map(a => String(a.case_sensitive)), aliases.map(a => a.alias_text)]);
  }
  return aliases.length;
}

/**
 * Import's alias projection: the page's frontmatter aliases and, for a
 * linkable entity page, its derived aliases. Import never loads a schema
 * pack itself: it uses the pack the caller already resolved (none: the four
 * always-linkable types), and the next sweep refreshes with the source's pack.
 * The caller reads the mention policy before its transaction opens.
 */
export async function writePageAliases(tx: BrainEngine, slug: string, sourceId: string,
  page: { title: string; type: string; compiled_truth: string; timeline: string; frontmatter: Record<string, unknown> }, pack: unknown,
  policy: MentionPolicy | null): Promise<void> {
  // #5984: the frontmatter and derived rows are independent; their statements are pipelined, in the caller's transaction.
  await pipelined(tx, [
    () => tx.setPageAliases(slug, sourceId, normalizeAliasList(page.frontmatter.aliases), { inline: true }),
    () => writeDerivedAliases(tx, sourceId, { slug, ...page }, { pack: (pack as PackTypes | undefined) ?? null, policy }),
  ]);
}

/** Turn the index off for one source: plain mention links, derived aliases, entries and page state go; typed_ner rows stay. */
async function disableSource(engine: BrainEngine, managed: boolean, sourceId: string): Promise<number> {
  let beliefs = null as ReturnType<typeof unmanagedBeliefPoint> | null;
  const removed = await aliasWrite(engine, managed, sourceId, async tx => {
    await lockMentionSources(tx, [sourceId], false);
    // #9548: on a belief-covered source the removed links are journaled like the reconcile's; only origins whose page keys are held are swept.
    const origins = (await beliefCoveredSources(tx, [sourceId])).has(sourceId) ? (await tx.executeRaw<{ slug: string }>(
      `SELECT DISTINCT f.slug FROM links l JOIN pages f ON f.id = l.from_page_id
        WHERE f.source_id = $1 AND l.link_source = 'mentions' AND (l.link_kind IS NULL OR l.link_kind = 'plain')`, [sourceId])).map(r => r.slug) : null;
    const unlink = async () => (await tx.executeRaw<{ id: number }>(
      `DELETE FROM links l USING pages f
        WHERE f.id = l.from_page_id AND f.source_id = $1 AND l.link_source = 'mentions'
          AND (l.link_kind IS NULL OR l.link_kind = 'plain') AND ($2::text[] IS NULL OR f.slug = ANY($2::text[])) RETURNING l.id`, [sourceId, origins])).length;
    if (origins?.length) beliefs = unmanagedBeliefPoint(sourceId, origins, 'mention_pass');
    const unlinked = beliefs ? await beliefs.run(tx, unlink) : await unlink();
    await tx.executeRaw(`DELETE FROM page_aliases WHERE source_id = $1 AND origin IN ('declared','subject')`, [sourceId]);
    await tx.executeRaw('DELETE FROM mention_gazetteer_entries WHERE source_id = $1', [sourceId]);
    await tx.executeRaw('DELETE FROM page_mention_state WHERE source_id = $1', [sourceId]);
    await writeStatus(tx, sourceId, { state: 'disabled', pending: 0, error: null, bumpGeneration: true, fingerprint: null });
    return unlinked;
  });
  await beliefs?.commit(engine);
  return removed;
}

interface DuePage { id: number; slug: string; source_id: string; revision: string; title: string | null; type: string | null;
  compiled_truth: string | null; timeline: string | null; mention_ignore: unknown; frontmatter?: Record<string, unknown> | null; body_storage?: string }

/** What one run of a segmented body holds for the mention pass: its UTF-16 length and each target's first mention in it (offset run-local). */
interface MentionRunOutput { chars: number; mentions: Mention[] }

const gazetteerDigests = new WeakMap<Gazetteer, string>();

/** A digest of every gazetteer entry (all fields), computed once per gazetteer object: the input a body run's mentions depend on. */
function gazetteerDigest(gazetteer: Gazetteer): string {
  let digest = gazetteerDigests.get(gazetteer);
  if (digest === undefined) {
    digest = keyDigest(canonicalJson([...gazetteer].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))));
    gazetteerDigests.set(gazetteer, digest);
  }
  return digest;
}

/**
 * The body mentions of a segmented page per run (spec 8.2 mentions), in page order: runs whose rows match the gazetteer,
 * the scan options and their text are reused, the others are scanned block by block and stored. Null when the page is
 * not segmented.
 */
async function segmentedRunMentions(engine: BrainEngine, p: { id: number; slug: string; source_id: string }, gazetteer: Gazetteer,
  opts: FindMentionsOpts): Promise<Array<{ run: DeriveRun; output: MentionRunOutput }> | null> {
  const scope = await derivationScope(engine, p);
  if (!scope) return null;
  const inputs = { extractor: MENTION_EXTRACTOR_VERSION, gazetteer: gazetteerDigest(gazetteer), opts, separator: '\n\n' };
  const plan = await planRuns<MentionRunOutput>(engine, 'mentions', scope, () => inputs);
  const fresh = new Map(plan.todo.map(run => [run.index, { chars: 0, mentions: [] as Mention[], seen: new Set<string>() }]));
  await readRuns(engine, scope, new Set(fresh.keys()), (run, block) => {
    const out = run && fresh.get(run.index);
    if (!out) return;
    for (const m of findMentionedEntities(block.text, gazetteer, opts)) {
      if (out.seen.has(`${m.source_id}\0${m.slug}`)) continue;
      out.seen.add(`${m.source_id}\0${m.slug}`);
      out.mentions.push(detachedValue({ ...m, offset: m.offset + out.chars }));
    }
    out.chars += block.text.length;
  }, { separator: '\n\n' });
  const outputs = new Map([...fresh].map(([index, { chars, mentions }]) => [index, { chars, mentions }]));
  await commitRuns(engine, 'mentions', scope, outputs, () => inputs);
  return scope.runs.map(run => ({ run, output: outputs.get(run.index) ?? plan.current.get(run.index)! }));
}

/**
 * findMentionedEntities over `${title}\n\n${compiled_truth}\n\n${timeline}`; a segmented body is scanned block by block,
 * incrementally (segmentedRunMentions), or in a full pass when its body moves during the read.
 */
export async function pageMentions(engine: BrainEngine,
  p: { id: number; slug: string; source_id: string; title: string | null; compiled_truth?: string | null; timeline?: string | null; body_storage?: string | null },
  gazetteer: Gazetteer, opts: FindMentionsOpts): Promise<Mention[]> {
  if (p.body_storage !== 'segmented') {
    return (p.title || p.compiled_truth || p.timeline) ? findMentionedEntities(`${p.title ?? ''}\n\n${p.compiled_truth ?? ''}\n\n${p.timeline ?? ''}`, gazetteer, opts) : [];
  }
  const found = new Map<string, Mention>();
  const add = (mentions: Mention[], at: number) => {
    for (const m of mentions) if (!found.has(`${m.source_id}\0${m.slug}`)) found.set(`${m.source_id}\0${m.slug}`, { ...m, offset: m.offset + at });
  };
  add(findMentionedEntities(p.title ?? '', gazetteer, opts), 0);
  const base = (p.title ?? '').length + 2;
  const runs = await segmentedRunMentions(engine, p, gazetteer, opts).catch(error => {
    if (error instanceof SegmentDerivationRaced) return null;
    throw error;
  });
  if (!runs) {
    for await (const block of bodyBlocks(engine, p, { separator: '\n\n' })) add(findMentionedEntities(block.text, gazetteer, opts), base + block.at);
    return [...found.values()];
  }
  let at = base;
  for (const { run, output } of runs) {
    if (run.region === 'timeline' && (run.index === 0 || runs[run.index - 1]!.run.region === 'truth')) at += 2;
    add(output.mentions, at);
    at += output.chars;
  }
  return [...found.values()];
}

/** Alias derivation options from the mention policy (`mentions.alias_deny`, `mentions.multiword_aliases`). */
export function aliasOptsOf(policy: Pick<MentionPolicy, 'aliasDeny' | 'multiwordAliases'>): { deny: string[]; multiword: boolean } {
  return { deny: policy.aliasDeny, multiword: policy.multiwordAliases };
}

/** Refresh derived alias rows of due entity pages in one source; returns pages whose rows changed. */
async function refreshAliases(engine: BrainEngine, managed: boolean, sourceId: string, types: string[], deadline: number, policy: MentionPolicy): Promise<number> {
  let changed = 0;
  // Pages that stopped being linkable entities (retype, pack change, deletion) lose their derived rows.
  const stale = await engine.executeRaw<{ slug: string }>(
    `SELECT DISTINCT pa.slug FROM page_aliases pa
       LEFT JOIN pages p ON p.slug = pa.slug AND p.source_id = pa.source_id
      WHERE pa.source_id = $1 AND pa.origin IN ('declared','subject')
        AND (p.id IS NULL OR p.deleted_at IS NOT NULL OR NOT (p.type = ANY($2::text[])))`, [sourceId, types]);
  if (stale.length) {
    await aliasWrite(engine, managed, sourceId, tx => tx.executeRaw(
      `DELETE FROM page_aliases WHERE source_id = $1 AND slug = ANY($2::text[]) AND origin IN ('declared','subject')`,
      [sourceId, stale.map(r => r.slug)]));
    changed += stale.length;
  }
  let after = 0;
  for (;;) {
    if (Date.now() > deadline) break;
    const pages = await engine.executeRaw<DuePage>(
      `SELECT p.id, p.slug, p.source_id, p.knowledge_revision::text AS revision, p.title, p.type, p.compiled_truth, p.timeline, NULL AS mention_ignore, p.frontmatter
         FROM pages p LEFT JOIN page_mention_state s ON s.page_id = p.id
        WHERE p.source_id = $1 AND p.deleted_at IS NULL AND p.body_storage = 'inline' AND p.type = ANY($2::text[]) AND p.id > $3
          AND (s.page_id IS NULL OR s.alias_revision IS DISTINCT FROM p.knowledge_revision OR s.alias_version IS DISTINCT FROM $4)
        ORDER BY p.id LIMIT $5`, [sourceId, types, after, ALIAS_DERIVATION_VERSION, ALIAS_BATCH]);
    if (!pages.length) break;
    after = pages[pages.length - 1]!.id;
    const desired = new Map(pages.map(p => [p.slug, deriveEntityAliases(p, aliasOptsOf(policy)).aliases]));
    const existing = await engine.executeRaw<{ slug: string; alias_norm: string; origin: string; case_sensitive: boolean; alias_text: string | null }>(
      `SELECT slug, alias_norm, origin, case_sensitive, alias_text FROM page_aliases
        WHERE source_id = $1 AND slug = ANY($2::text[]) AND origin IN ('declared','subject')`, [sourceId, pages.map(p => p.slug)]);
    const key = (a: { alias_norm: string; origin: string; case_sensitive: boolean; alias_text: string | null }) =>
      `${a.alias_norm}\0${a.origin}\0${a.case_sensitive}\0${a.case_sensitive ? a.alias_text : ''}`;
    const have = new Map<string, Set<string>>();
    for (const row of existing) {
      const set = have.get(row.slug) ?? new Set<string>();
      set.add(key(row));
      have.set(row.slug, set);
    }
    const rewrite = pages.filter(p => {
      const want = new Set((desired.get(p.slug) ?? []).map(a => key({ ...a, alias_text: a.alias_text })));
      const got = have.get(p.slug) ?? new Set<string>();
      return want.size !== got.size || [...want].some(k => !got.has(k));
    });
    await aliasWrite(engine, managed, sourceId, async tx => {
      const current = new Map((await tx.executeRaw<{ id: number; revision: string }>(
        `SELECT id, knowledge_revision::text AS revision FROM pages WHERE id = ANY($1::int[]) AND deleted_at IS NULL`,
        [pages.map(p => p.id)])).map(r => [Number(r.id), r.revision]));
      const fresh = pages.filter(p => current.get(Number(p.id)) === p.revision);
      const freshRewrite = rewrite.filter(p => current.get(Number(p.id)) === p.revision);
      if (freshRewrite.length) {
        await tx.executeRaw(`DELETE FROM page_aliases WHERE source_id = $1 AND slug = ANY($2::text[]) AND origin IN ('declared','subject')`,
          [sourceId, freshRewrite.map(p => p.slug)]);
        const rows = freshRewrite.flatMap(p => (desired.get(p.slug) ?? []).map((a: DerivedAlias) => ({ slug: p.slug, ...a })));
        if (rows.length) {
          await tx.executeRaw(
            `INSERT INTO page_aliases (source_id, alias_norm, slug, origin, case_sensitive, alias_text)
             SELECT $1, a, s, o, c::boolean, t FROM unnest($2::text[], $3::text[], $4::text[], $5::text[], $6::text[]) AS x(a, s, o, c, t)
             ON CONFLICT DO NOTHING`,
            [sourceId, rows.map(r => r.alias_norm), rows.map(r => r.slug), rows.map(r => r.origin), rows.map(r => String(r.case_sensitive)), rows.map(r => r.alias_text)]);
        }
      }
      if (fresh.length) {
        await tx.executeRaw(
          `INSERT INTO page_mention_state (page_id, source_id, alias_revision, alias_version, aliases_refreshed_at)
           SELECT id, $1, rev::uuid, $4, now() FROM unnest($2::int[], $3::text[]) AS x(id, rev)
           ON CONFLICT (page_id) DO UPDATE SET alias_revision = EXCLUDED.alias_revision, alias_version = EXCLUDED.alias_version,
             aliases_refreshed_at = EXCLUDED.aliases_refreshed_at`,
          [sourceId, fresh.map(p => p.id), fresh.map(p => p.revision), ALIAS_DERIVATION_VERSION]);
      }
      changed += freshRewrite.length;
    });
  }
  return changed;
}

/** Live pages in `sourceIds` whose text the mini-gazetteer of `added` matches (real tokenizer, substring-superset prefilter). */
async function pagesMatchingNames(engine: BrainEngine, sourceIds: string[], added: Gazetteer, allowCrossSource: boolean): Promise<number[]> {
  const longest = [...added.values()].flat().map(e => [...e.tokens].sort((a, b) => b.length - a.length)[0]!);
  // ASCII tokens: lower() in SQL agrees with the scanner's NFC+toLowerCase, so LIKE is a superset.
  // Otherwise every page in scope is scanned in JS.
  const ascii = longest.every(t => /^[\x20-\x7e]+$/.test(t));
  const patterns = [...new Set(longest)].map(t => `%${t.replace(/[\\%_]/g, m => `\\${m}`)}%`);
  const out: number[] = [];
  let after = 0;
  for (;;) {
    const rows = await engine.executeRaw<{ id: number; slug: string; source_id: string; body_storage: string; body: string; mention_ignore: unknown }>(
      `SELECT id, slug, source_id, body_storage, COALESCE(title, '') || E'\\n\\n' || COALESCE(compiled_truth, '') || E'\\n\\n' || COALESCE(timeline, '') AS body,
              frontmatter->'mention_ignore' AS mention_ignore
         FROM pages WHERE deleted_at IS NULL AND source_id = ANY($1::text[]) AND id > $2
          ${ascii ? `AND (body_storage = 'segmented' OR lower(COALESCE(title, '') || ' ' || COALESCE(compiled_truth, '') || ' ' || COALESCE(timeline, '')) LIKE ANY($4::text[]))` : ''}
        ORDER BY id LIMIT $3`,
      ascii ? [sourceIds, after, PREFILTER_BATCH, patterns] : [sourceIds, after, PREFILTER_BATCH]);
    if (!rows.length) break;
    after = rows[rows.length - 1]!.id;
    for (const r of rows) {
      if (r.body_storage === 'segmented') { out.push(Number(r.id)); continue; }
      const hits = findMentionedEntities(r.body, added, { fromSlug: r.slug, fromSourceId: r.source_id, allowCrossSource });
      if (hits.length) out.push(Number(r.id));
    }
  }
  return out;
}

function miniGazetteer(gazetteer: Gazetteer, keys: Set<string>): Gazetteer {
  const out: Gazetteer = new Map();
  for (const [first, bucket] of gazetteer) {
    const kept = bucket.filter(e => keys.has(entryKey(e)));
    if (kept.length) out.set(first, kept);
  }
  return out;
}

function entryKey(e: GazetteerEntry | GazetteerEntryKey): string {
  if ('target_slug' in e) return `${e.source_id}\0${e.name_norm}\0${e.target_slug}\0${e.case_sensitive}`;
  return `${e.source_id}\0${entryNameNorm(e)}\0${e.slug}\0${!!e.caseTokens}`;
}

/**
 * Diff the saved entry set of one source against `gazetteer`; on change, save
 * it, bump the generation and mark affected pages due, in one transaction under
 * the source's advisory lock. Returns whether the set changed.
 */
async function saveEntries(engine: BrainEngine, sourceId: string, gazetteer: Gazetteer, scopeSources: string[], allowCrossSource: boolean): Promise<boolean> {
  const current = gazetteerEntryKeys(gazetteer).filter(k => k.source_id === sourceId);
  const read = async (db: Pick<BrainEngine, 'executeRaw'>) => (await db.executeRaw<GazetteerEntryKey>(
    'SELECT source_id, name_norm, target_slug, case_sensitive FROM mention_gazetteer_entries WHERE source_id = $1', [sourceId]));
  const before = await read(engine);
  const beforeKeys = new Set(before.map(entryKey));
  const currentKeys = new Set(current.map(entryKey));
  const added = current.filter(k => !beforeKeys.has(entryKey(k)));
  const removed = before.filter(k => !currentKeys.has(entryKey(k)));
  if (!added.length && !removed.length) return false;
  // The first save for a source marks the whole candidate scope due (cheap); later saves only affected pages.
  const firstSave = before.length === 0;
  let candidates: number[] = [];
  if (!firstSave) {
    const linked = removed.length ? await engine.executeRaw<{ id: number }>(
      `SELECT DISTINCT l.from_page_id AS id FROM links l JOIN pages t ON t.id = l.to_page_id
        WHERE t.source_id = $1 AND t.slug = ANY($2::text[]) AND l.link_source = 'mentions'`,
      [sourceId, [...new Set(removed.map(k => k.target_slug))]]) : [];
    const matched = added.length ? await pagesMatchingNames(engine, scopeSources,
      miniGazetteer(gazetteer, new Set(added.map(entryKey))), allowCrossSource) : [];
    candidates = [...new Set([...linked.map(r => Number(r.id)), ...matched])];
  }
  return engine.transaction(async tx => {
    await tx.executeRaw('SELECT pg_advisory_xact_lock(hashtext($1)::bigint)', [`gbrain:mention-entries:${sourceId}`]);
    const locked = await read(tx);
    // Another sweep saved first: its pages and generation are authoritative; this pass's scan fails its conditional publish.
    if (locked.length !== before.length || locked.some(k => !beforeKeys.has(entryKey(k)))) return true;
    if (removed.length) {
      await tx.executeRaw(
        `DELETE FROM mention_gazetteer_entries e USING unnest($2::text[], $3::text[], $4::text[]) AS x(n, t, c)
          WHERE e.source_id = $1 AND e.name_norm = x.n AND e.target_slug = x.t AND e.case_sensitive = x.c::boolean`,
        [sourceId, removed.map(k => k.name_norm), removed.map(k => k.target_slug), removed.map(k => String(k.case_sensitive))]);
    }
    if (added.length) {
      await tx.executeRaw(
        `INSERT INTO mention_gazetteer_entries (source_id, name_norm, target_slug, case_sensitive)
         SELECT $1, n, t, c::boolean FROM unnest($2::text[], $3::text[], $4::text[]) AS x(n, t, c) ON CONFLICT DO NOTHING`,
        [sourceId, added.map(k => k.name_norm), added.map(k => k.target_slug), added.map(k => String(k.case_sensitive))]);
    }
    if (firstSave) {
      await tx.executeRaw('UPDATE page_mention_state SET mention_revision = NULL WHERE source_id = ANY($1::text[])', [scopeSources]);
    } else if (candidates.length) {
      await tx.executeRaw('UPDATE page_mention_state SET mention_revision = NULL WHERE page_id = ANY($1::int[])', [candidates]);
    }
    await writeStatus(tx, sourceId, { state: 'pending', bumpGeneration: true });
    return true;
  });
}

async function readGenerations(engine: Pick<BrainEngine, 'executeRaw'>, sourceIds: string[], lock: boolean): Promise<Map<string, string>> {
  const rows = await engine.executeRaw<{ source_id: string; generation: string }>(
    `SELECT source_id, generation::text AS generation FROM mention_index_status WHERE source_id = ANY($1::text[])${lock ? ' FOR SHARE' : ''}`,
    [sourceIds]);
  return new Map(rows.map(r => [r.source_id, r.generation]));
}

function sameGenerations(a: Map<string, string>, b: Map<string, string>): boolean {
  if (a.size !== b.size) return false;
  for (const [k, v] of a) if (b.get(k) !== v) return false;
  return true;
}

/** Reconcile due pages of one source; returns counts and whether a gazetteer change stopped it. */
async function reconcileSource(engine: BrainEngine, sourceId: string, gazetteer: Gazetteer, genSources: string[], allowCrossSource: boolean,
  opts: MentionPassOpts, deadline: number, result: MentionPassResult): Promise<void> {
  const generations = await readGenerations(engine, genSources, false);
  let after = 0;
  for (;;) {
    opts.signal?.throwIfAborted();
    if (Date.now() > deadline) return;
    const pages = await engine.executeRaw<DuePage>(
      `SELECT p.id, p.slug, p.source_id, p.knowledge_revision::text AS revision, p.title, p.type, p.compiled_truth, p.timeline,
              p.frontmatter->'mention_ignore' AS mention_ignore, p.body_storage
         FROM pages p LEFT JOIN page_mention_state s ON s.page_id = p.id
        WHERE p.source_id = $1 AND p.deleted_at IS NULL AND p.id > $3 AND ${DUE_PREDICATE}
        ORDER BY p.id LIMIT $4`, [sourceId, MENTION_EXTRACTOR_VERSION, after, RECONCILE_BATCH]);
    if (!pages.length) return;
    after = pages[pages.length - 1]!.id;
    const found = new Map<number, Mention[]>();
    const readable: DuePage[] = [];
    for (const p of pages) {
      try {
        found.set(Number(p.id), await pageMentions(engine, p, gazetteer,
          { fromSlug: p.slug, fromSourceId: p.source_id, allowCrossSource, ignoreNames: parseNameList(p.mention_ignore) }));
        readable.push(p);
      } catch (error) {
        const failure = classifyItemFailure(error);
        if (failure.scope !== 'item' || failure.class !== 'content') throw error;
        result.failed = (result.failed ?? 0) + 1;
        if ((result.failures ??= []).length < MENTION_FAILURE_REPORT_LIMIT) {
          result.failures.push({ slug: p.slug, source_id: p.source_id, code: failure.code, message: failure.message.slice(0, 500) });
        }
      }
    }
    if (!readable.length) continue;
    await opts.beforePublish?.(readable.map(p => Number(p.id)));
    // #7031: on a belief-covered source the link changes are an unmanaged projection point, like derived links; the
    // page keys are locked first, as every publisher locks them before the page rows.
    let beliefs = null as ReturnType<typeof unmanagedBeliefPoint> | null;
    const outcome = await engine.transaction(async tx => {
      await lockMentionSources(tx, genSources, true);
      const journaled = (await beliefCoveredSources(tx, [sourceId])).has(sourceId);
      if (journaled) await tx.lockPageKeys(readable.map(p => ({ sourceId: p.source_id, slug: p.slug })));
      if (!sameGenerations(generations, await readGenerations(tx, genSources, true))) return null;
      const current = new Map((await tx.executeRaw<{ id: number; revision: string }>(
        `SELECT id, knowledge_revision::text AS revision FROM pages WHERE id = ANY($1::int[]) AND deleted_at IS NULL FOR SHARE`,
        [readable.map(p => p.id)])).map(r => [Number(r.id), r.revision]));
      const fresh = readable.filter(p => current.get(Number(p.id)) === p.revision);
      if (!fresh.length) return { fresh: 0, created: 0, removed: 0 };
      const rows = await tx.executeRaw<{ id: number; from_page_id: number; to_slug: string; to_source_id: string; plain: boolean }>(
        `SELECT l.id, l.from_page_id, t.slug AS to_slug, t.source_id AS to_source_id,
                (l.link_source = 'mentions' AND (l.link_kind IS NULL OR l.link_kind = 'plain')) AS plain
           FROM links l JOIN pages t ON t.id = l.to_page_id WHERE l.from_page_id = ANY($1::int[])`,
        [fresh.map(p => p.id)]);
      const existing = rows.filter(r => r.plain);
      // One link per page pair: a pair another producer already links (markdown, frontmatter, typed_ner, manual) gets no plain mention.
      // A cross-source mention is also covered by an explicit link to the same slug in any source (the resolver may pick another twin).
      const covered = new Set(rows.filter(r => !r.plain).flatMap(r => [`${r.from_page_id}\0${r.to_source_id}\0${r.to_slug}`, `${r.from_page_id}\0*\0${r.to_slug}`]));
      const pageSource = new Map(fresh.map(p => [Number(p.id), p.source_id]));
      const want = new Set(fresh.flatMap(p => (found.get(Number(p.id)) ?? [])
        .filter(m => !covered.has(`${p.id}\0${m.source_id}\0${m.slug}`) && !(m.source_id !== pageSource.get(Number(p.id)) && covered.has(`${p.id}\0*\0${m.slug}`)))
        .map(m => `${p.id}\0${m.source_id}\0${m.slug}`)));
      const have = new Set(existing.map(r => `${r.from_page_id}\0${r.to_source_id}\0${r.to_slug}`));
      const drop = existing.filter(r => !want.has(`${r.from_page_id}\0${r.to_source_id}\0${r.to_slug}`)).map(r => Number(r.id));
      const add: LinkBatchInput[] = fresh.flatMap(p => (found.get(Number(p.id)) ?? [])
        .filter(m => want.has(`${p.id}\0${m.source_id}\0${m.slug}`) && !have.has(`${p.id}\0${m.source_id}\0${m.slug}`))
        .map(m => ({ from_slug: p.slug, to_slug: m.slug, link_type: 'mentions', link_source: 'mentions', context: m.name,
          from_source_id: p.source_id, to_source_id: m.source_id })));
      const write = async () => {
        if (drop.length) await tx.executeRaw('DELETE FROM links WHERE id = ANY($1::int[])', [drop]);
        return add.length ? await tx.addLinksBatch(add, { auditSite: 'extract.mentions' }) : 0; // gbrain-allow-direct-insert: mention pass — reconciling write of the scan's own plain mention rows
      };
      const dropped = new Set(existing.filter(r => drop.includes(Number(r.id))).map(r => Number(r.from_page_id)));
      const changed = fresh.filter(p => dropped.has(Number(p.id))).map(p => p.slug).concat(add.map(link => link.from_slug));
      if (journaled && changed.length) beliefs = unmanagedBeliefPoint(sourceId, changed, 'mention_pass');
      const created = beliefs ? await beliefs.run(tx, write) : await write();
      await tx.executeRaw(
        `INSERT INTO page_mention_state (page_id, source_id, mention_revision, mention_version, mention_generation, mentions_scanned_at)
         SELECT id, $1, rev::uuid, $4, $5::bigint, now() FROM unnest($2::int[], $3::text[]) AS x(id, rev)
         ON CONFLICT (page_id) DO UPDATE SET mention_revision = EXCLUDED.mention_revision, mention_version = EXCLUDED.mention_version,
           mention_generation = EXCLUDED.mention_generation, mentions_scanned_at = EXCLUDED.mentions_scanned_at`,
        [sourceId, fresh.map(p => p.id), fresh.map(p => p.revision), MENTION_EXTRACTOR_VERSION, generations.get(sourceId) ?? '0']);
      return { fresh: fresh.length, created, removed: drop.length };
    });
    await beliefs?.commit(engine);
    if (!outcome) { result.skipped += readable.length; return; }
    result.pages += outcome.fresh;
    result.skipped += readable.length - outcome.fresh;
    result.created += outcome.created;
    result.removed += outcome.removed;
  }
}

/**
 * Run the mention pass for one source or every source. Never throws for a
 * pack, alias or entry-table failure: it writes nothing and returns
 * `state: 'failed'` with the error (recorded on the status rows).
 */
export async function runMentionPass(engine: BrainEngine, opts: MentionPassOpts = {}): Promise<MentionPassResult> {
  const result: MentionPassResult = { pages: 0, created: 0, removed: 0, skipped: 0, aliasPages: 0, remaining: 0, state: 'complete' };
  const deadline = opts.deadline ?? Infinity;
  const passStart = new Date().toISOString();
  const policy = await readMentionPolicy(engine);
  const sources = await sourcesInScope(engine, opts.sourceId);
  const managed = await managedPersistenceEnabled(engine);
  if (!policy.enabled) {
    const status = new Map((await engine.executeRaw<{ source_id: string; state: string }>(
      'SELECT source_id, state FROM mention_index_status WHERE source_id = ANY($1::text[])', [sources])).map(r => [r.source_id, r.state]));
    for (const sourceId of sources) {
      if (status.get(sourceId) === 'disabled') continue;
      result.removed += await disableSource(engine, managed, sourceId);
    }
    result.state = 'disabled';
    return result;
  }
  const allowCrossSource = await isCrossSourceLinksEnabled(engine);
  const types = new Map<string, string[]>();
  let gazetteer: Gazetteer;
  try {
    for (const sourceId of sources) types.set(sourceId, linkableTypesFor(await loadSourcePack(engine, sourceId, { strict: true }), policy));
    gazetteer = await buildGazetteer(engine, { policy, strict: true });
  } catch (error) {
    const message = error instanceof Error ? error.message : String(error);
    for (const sourceId of sources) await writeStatus(engine, sourceId, { state: 'failed', error: message });
    return { ...result, state: 'failed', error: message, remaining: await countMentionDuePages(engine, opts.sourceId) };
  }
  const allSources = allowCrossSource ? (await sourcesInScope(engine)) : [];
  const fingerprints = new Map<string, string>(sources.map(s => [s, policyFingerprint(types.get(s)!, allowCrossSource, policy)]));
  const stored = new Map((await engine.executeRaw<{ source_id: string; policy_fingerprint: string | null }>(
    'SELECT source_id, policy_fingerprint FROM mention_index_status WHERE source_id = ANY($1::text[])', [sources]))
    .map(r => [r.source_id, r.policy_fingerprint]));
  // A changed policy makes every page due; a binary with other extractor versions only recounts (version-behind pages are already due).
  for (const sourceId of sources) {
    const { hash, versions } = splitStoredFingerprint(stored.get(sourceId));
    const policyChanged = hash !== fingerprints.get(sourceId);
    if (!policyChanged && versions === INDEX_VERSION_STAMP) continue;
    await engine.transaction(async tx => {
      if (policyChanged) await tx.executeRaw('UPDATE page_mention_state SET mention_revision = NULL, alias_revision = NULL WHERE source_id = $1', [sourceId]);
      await writeStatus(tx, sourceId, { state: 'pending', bumpGeneration: policyChanged, pending: await countIndexDuePages(tx, sourceId, types.get(sourceId)!),
        fingerprint: `${fingerprints.get(sourceId)!};${INDEX_VERSION_STAMP}` });
    });
  }
  for (const sourceId of sources) {
    if (Date.now() > deadline) break;
    result.aliasPages += await refreshAliases(engine, managed, sourceId, types.get(sourceId)!, deadline, policy);
  }
  // Aliases may have changed: the gazetteer the reconcile uses is built after the refresh.
  if (result.aliasPages > 0) {
    try { gazetteer = await buildGazetteer(engine, { policy, strict: true }); }
    catch (error) {
      const message = error instanceof Error ? error.message : String(error);
      for (const sourceId of sources) await writeStatus(engine, sourceId, { state: 'failed', error: message });
      return { ...result, state: 'failed', error: message, remaining: await countMentionDuePages(engine, opts.sourceId) };
    }
  }
  for (const sourceId of sources) {
    await saveEntries(engine, sourceId, gazetteer, allowCrossSource ? allSources : [sourceId], allowCrossSource);
  }
  for (const sourceId of sources) {
    await reconcileSource(engine, sourceId, gazetteer, allowCrossSource ? allSources : [sourceId], allowCrossSource, opts, deadline, result);
    const pending = await countIndexDuePages(engine, sourceId, types.get(sourceId)!);
    result.remaining += pending;
    await writeStatus(engine, sourceId, { state: pending > 0 ? 'pending' : 'complete', pending, error: null, lastPassAt: passStart });
  }
  result.state = result.remaining > 0 ? 'pending' : 'complete';
  return result;
}

/** The gazetteer's entries targeting one page, with their origins (for explain). */
export function entriesFor(gazetteer: Gazetteer, sourceId: string, slug: string): GazetteerEntry[] {
  return [...gazetteer.values()].flat().filter(e => e.source_id === sourceId && e.slug === slug);
}

/** Normalized token key of a name, as the scanner compares it. */
export function nameKey(name: string): string {
  return tokenizeTitle(name).join(' ');
}