Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
44.0 KB
Raw Blame
/**
 * Unit tests for src/core/by-mention.ts.
 *
 * Pure-function coverage of `findMentionedEntities` + `buildGazetteer`.
 * Hermetic via PGLite for buildGazetteer (needs engine); pure-fn cases
 * for findMentionedEntities (no engine needed).
 *
 * Covers all 20 cases enumerated in the v0.42.0.0 plan:
 *   1. Single-token title match
 *   2. Multi-word phrase pass ("Acme Corp" matches "Acme Corp" not "Acme")
 *   3. Case folding
 *   4. Whole-word boundary
 *   5. Possessive form
 *   6. Code-block stripping
 *   7. Min-length filter
 *   8. Ignore-list at gazetteer build (Apple suppressed when no page)
 *   9. Ignore-list inverse (Apple matches when page exists)
 *   10. First-mention-only cap
 *   11. Empty gazetteer
 *   12. Empty text
 *   13. All entity pages soft-deleted → empty gazetteer
 *   14. Multi-word shared first token (longest-match wins)
 *   15. Determinism across 10 calls
 *   16. Self-link guard (D13)
 *   17. Cross-source guard
 *   17b. Cross-source guard lifts under allowCrossSource
 *   17c. Own-source same-name twin outranks the cross-source twin
 *   18. Hardcoded type filter (meeting NOT in gazetteer)
 *   19. Min-length + ignore-list interaction
 *   20. Code-block + token interaction
 */

import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import {
  buildGazetteer,
  findMentionedEntities,
  tokenizeForScan,
  tokenizeTitle,
  LINKABLE_ENTITY_TYPES,
  type Gazetteer,
  type GazetteerEntry,
} from '../src/core/by-mention.ts';

let engine: PGLiteEngine;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
}, 60_000);

afterAll(async () => {
  await engine.disconnect();
});

beforeEach(async () => {
  await engine.executeRaw('DELETE FROM links');
  await engine.executeRaw('DELETE FROM pages');
});

// Tiny gazetteer builder for pure-fn cases that don't need engine.
//
// Deliberately calls the PRODUCTION `tokenizeTitle` rather than re-declaring
// the tokenizer. A duplicated copy makes every test here non-discriminating:
// reverting the source tokenizer would leave the fixture on the new one, so
// title and body would keep agreeing and the tests would pass either way.
function gazetteerFromEntries(entries: Omit<GazetteerEntry, 'tokens'>[]): Gazetteer {
  const g: Gazetteer = new Map();
  for (const raw of entries) {
    const tokens = tokenizeTitle(raw.title);
    if (tokens.length === 0) continue;
    const key = tokens[0]!;
    const entry: GazetteerEntry = { ...raw, tokens };
    const bucket = g.get(key);
    if (bucket) bucket.push(entry);
    else g.set(key, [entry]);
  }
  for (const bucket of g.values()) bucket.sort((a, b) => b.tokens.length - a.tokens.length);
  return g;
}

// ============================================================
// findMentionedEntities — pure unit tests
// ============================================================

describe('findMentionedEntities — pure cases', () => {
  test('text inside HTML comments is never a mention: a gbrain fence marker does not link a page titled gbrain (#6740)', () => {
    const g = gazetteerFromEntries([
      { slug: 'projects/gbrain', source_id: 'default', title: 'gbrain' },
      { slug: 'companies/acme', source_id: 'default', title: 'Acme Example' },
    ]);
    const body = '## Facts\n<!--- gbrain:facts:begin -->\n| claim |\n<!--- gbrain:facts:end -->\n<!-- Acme Example owns this -->\nNotes.';
    expect(findMentionedEntities(body, g, { fromSlug: 'notes/n1', fromSourceId: 'default' })).toEqual([]);
    const prose = '<!-- Acme Example --> We shipped gbrain with Acme Example.';
    const mentions = findMentionedEntities(prose, g, { fromSlug: 'notes/n1', fromSourceId: 'default' });
    expect(mentions.map(m => [m.slug, m.offset])).toEqual([
      ['projects/gbrain', prose.indexOf('gbrain')],
      ['companies/acme', prose.lastIndexOf('Acme Example')],
    ]);
  });

  test('1. single-token title match', () => {
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'default', title: 'Acme' },
    ]);
    const mentions = findMentionedEntities('Acme launched today.', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(1);
    expect(mentions[0]!.slug).toBe('companies/acme');
    expect(mentions[0]!.name).toBe('Acme');
    expect(mentions[0]!.offset).toBe(0);
  });

  test('2. multi-word phrase pass — "Acme Corp" matches multi-word, not single', () => {
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'default', title: 'Acme' },
      { slug: 'companies/acme-corp', source_id: 'default', title: 'Acme Corp' },
    ]);
    const mentions = findMentionedEntities('We met with Acme Corp last week.', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    // longest-match wins → only the multi-word target
    expect(mentions).toHaveLength(1);
    expect(mentions[0]!.slug).toBe('companies/acme-corp');
  });

  test('3. case folding — "iOS Engineer" title matches "ios engineer" in body', () => {
    const g = gazetteerFromEntries([
      { slug: 'people/ios-engineer', source_id: 'default', title: 'iOS Engineer' },
    ]);
    const mentions = findMentionedEntities('Looking to hire an ios engineer.', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(1);
    expect(mentions[0]!.slug).toBe('people/ios-engineer');
  });

  test('4. whole-word boundary — "Acme" matches "Acme." but NOT "Acmecorp"', () => {
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'default', title: 'Acme' },
    ]);
    // Should match (sentence-ending dot is a token break)
    const m1 = findMentionedEntities('We bought Acme.', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(m1).toHaveLength(1);
    // Should NOT match — "Acmecorp" tokenizes as single token "acmecorp"
    const m2 = findMentionedEntities('Acmecorp is unrelated.', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(m2).toHaveLength(0);
  });

  test('5. possessive form — "Acme\'s growth" → Acme matches', () => {
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'default', title: 'Acme' },
    ]);
    const mentions = findMentionedEntities("Acme's growth is impressive.", g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(1);
    expect(mentions[0]!.slug).toBe('companies/acme');
  });

  test('6. code-block stripping — mentions inside ``` blocks ignored', () => {
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'default', title: 'Acme' },
    ]);
    const body = '```\nAcme code\n```\nNothing here.';
    const mentions = findMentionedEntities(body, g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(0);
  });

  test('10. first-mention-only cap — 5 body mentions of same entity → 1 link', () => {
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'default', title: 'Acme' },
    ]);
    const body = 'Acme one. Acme two. Acme three. Acme four. Acme five.';
    const mentions = findMentionedEntities(body, g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(1);
  });

  test('11. empty gazetteer → empty result', () => {
    const g: Gazetteer = new Map();
    const mentions = findMentionedEntities('Anything goes here.', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toEqual([]);
  });

  test('12. empty text → empty result', () => {
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'default', title: 'Acme' },
    ]);
    const mentions = findMentionedEntities('', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toEqual([]);
  });

  test('14. multi-word shared first token — longest-match wins', () => {
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'default', title: 'Acme' },
      { slug: 'companies/acme-corp', source_id: 'default', title: 'Acme Corp' },
      { slug: 'companies/acme-foundation', source_id: 'default', title: 'Acme Foundation' },
    ]);
    const body = 'Acme Foundation announced. Then Acme Corp. Then plain Acme.';
    const mentions = findMentionedEntities(body, g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    // First offset: "Acme Foundation" — longest match wins.
    // Second occurrence of "Acme": multi-word "Acme Corp" matches → multi-word wins.
    // Third: plain "Acme" alone — single-word match.
    const slugs = mentions.map(m => m.slug);
    expect(slugs).toContain('companies/acme-foundation');
    expect(slugs).toContain('companies/acme-corp');
    expect(slugs).toContain('companies/acme');
  });

  test('15. determinism — same body + same gazetteer → identical output across 10 calls', () => {
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'default', title: 'Acme' },
      { slug: 'companies/acme-corp', source_id: 'default', title: 'Acme Corp' },
      { slug: 'people/alice', source_id: 'default', title: 'Alice Smith' },
    ]);
    const body = 'Acme Corp and Alice Smith and Acme met. Then Alice Smith again.';
    const refs = new Set<string>();
    for (let i = 0; i < 10; i++) {
      const mentions = findMentionedEntities(body, g, {
        fromSlug: 'writing/post-1', fromSourceId: 'default',
      });
      refs.add(JSON.stringify(mentions));
    }
    expect(refs.size).toBe(1);
  });

  test('16. self-link guard (D13) — entity page mentioning own title skips', () => {
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'default', title: 'Acme' },
    ]);
    // Page IS the Acme page; body mentions "Acme" → self-link guard skips.
    const mentions = findMentionedEntities('Acme has 500 customers.', g, {
      fromSlug: 'companies/acme', fromSourceId: 'default',
    });
    expect(mentions).toEqual([]);
  });

  test('17. cross-source guard — page in source A mentions entity in source B → no link', () => {
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'team-b', title: 'Acme' },
    ]);
    const mentions = findMentionedEntities('We met Acme today.', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'team-a', // different source
    });
    expect(mentions).toEqual([]);
  });

  test('17b. cross-source guard lifts under allowCrossSource — mention carries the entity\'s own source_id', () => {
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'team-b', title: 'Acme' },
    ]);
    const opts = { fromSlug: 'writing/post-1', fromSourceId: 'team-a' };
    expect(findMentionedEntities('We met Acme today.', g, opts)).toEqual([]);
    const mentions = findMentionedEntities('We met Acme today.', g, { ...opts, allowCrossSource: true });
    expect(mentions).toHaveLength(1);
    expect(mentions[0]!.slug).toBe('companies/acme');
    expect(mentions[0]!.source_id).toBe('team-b');
  });

  test('17c. same-name twin in the scanning page\'s own source outranks the cross-source twin', () => {
    // Bucket order is length-only, so the foreign twin sits first: without the
    // own-source preference the guard drops the mention (allowCrossSource off)
    // or links the foreign page (allowCrossSource on).
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'team-b', title: 'Acme' },
      { slug: 'companies/acme-local', source_id: 'team-a', title: 'Acme' },
    ]);
    for (const allowCrossSource of [false, true]) {
      const mentions = findMentionedEntities('We met Acme today.', g, {
        fromSlug: 'writing/post-1', fromSourceId: 'team-a', allowCrossSource,
      });
      expect(mentions).toHaveLength(1);
      expect(mentions[0]!.slug).toBe('companies/acme-local');
      expect(mentions[0]!.source_id).toBe('team-a');
    }
  });

  // Slug uniqueness is (source_id, slug): with the cross-source guard lifted,
  // a foreign page that merely shares the scanning page's slug is a real
  // target, not a self-link.
  test('17d. allowCrossSource — foreign namesakes of the scanning page are not self-links', () => {
    const g = gazetteerFromEntries([
      { slug: 'entities/shared', source_id: 'team-b', title: 'Beta Entity' },
      { slug: 'entities/shared', source_id: 'team-c', title: 'Gamma Entity' },
    ]);
    const mentions = findMentionedEntities('Beta Entity met Gamma Entity.', g, {
      fromSlug: 'entities/shared', fromSourceId: 'team-a', allowCrossSource: true,
    });
    expect(mentions.map(({ source_id, slug }) => [source_id, slug])).toEqual([
      ['team-b', 'entities/shared'],
      ['team-c', 'entities/shared'],
    ]);
  });

  // #6158 / #4845: a connector renders its contacts as near-empty person pages in its own source. With
  // cross_source on, a mention in that source binds to the canonical twin elsewhere, not the stub; with it
  // off the own-source posture is unchanged; two foreign twins (no same-slug one) keep the stub (honest tie).
  test('17f. a connector contact stub yields to the one canonical twin under allowCrossSource only', () => {
    const g = gazetteerFromEntries([
      { slug: 'people/dana-example', source_id: 'gmail-a', title: 'Dana Example', connector_stub: true },
      { slug: 'people/dana-example', source_id: 'workspace', title: 'Dana Example' },
    ]);
    const opts = { fromSlug: 'emails/2026/08/thread-1', fromSourceId: 'gmail-a' };
    expect(findMentionedEntities('Call with Dana Example.', g, opts)).toEqual([expect.objectContaining({ slug: 'people/dana-example', source_id: 'gmail-a' })]);
    expect(findMentionedEntities('Call with Dana Example.', g, { ...opts, allowCrossSource: true }))
      .toEqual([expect.objectContaining({ slug: 'people/dana-example', source_id: 'workspace' })]);
    const tie = gazetteerFromEntries([
      { slug: 'people/dana-example', source_id: 'gmail-a', title: 'Dana Example', connector_stub: true },
      { slug: 'people/dana-e', source_id: 'workspace', title: 'Dana Example' },
      { slug: 'people/dana-example-2', source_id: 'notes', title: 'Dana Example' },
    ]);
    expect(findMentionedEntities('Call with Dana Example.', tie, { ...opts, allowCrossSource: true }))
      .toEqual([expect.objectContaining({ slug: 'people/dana-example', source_id: 'gmail-a' })]);
    // A third source's page mentioning the name binds to the canonical twin, not the stub the bucket sorts first.
    expect(findMentionedEntities('Call with Dana Example.', g, { fromSlug: 'notes/x', fromSourceId: 'notes', allowCrossSource: true }))
      .toEqual([expect.objectContaining({ slug: 'people/dana-example', source_id: 'workspace' })]);
    const stubOnly = gazetteerFromEntries([
      { slug: 'people/dana-example', source_id: 'gmail-a', title: 'Dana Example', connector_stub: true },
      { slug: 'people/dana-example', source_id: 'gmail-b', title: 'Dana Example', connector_stub: true },
    ]);
    expect(findMentionedEntities('Call with Dana Example.', stubOnly, { ...opts, allowCrossSource: true }))
      .toEqual([expect.objectContaining({ slug: 'people/dana-example', source_id: 'gmail-a' })]);
  });

  test('17e. allowCrossSource — first-mention dedup keys on (source_id, slug), not bare slug', () => {
    const g = gazetteerFromEntries([
      { slug: 'entities/shared', source_id: 'team-b', title: 'Beta Entity' },
      { slug: 'entities/shared', source_id: 'team-c', title: 'Gamma Entity' },
    ]);
    const mentions = findMentionedEntities('Beta Entity met Gamma Entity.', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'team-a', allowCrossSource: true,
    });
    expect(mentions.map(({ source_id, slug }) => [source_id, slug])).toEqual([
      ['team-b', 'entities/shared'],
      ['team-c', 'entities/shared'],
    ]);
  });

  test('20. code-block + token interaction — body text outside block linked, inside skipped', () => {
    const g = gazetteerFromEntries([
      { slug: 'companies/acme', source_id: 'default', title: 'Acme' },
    ]);
    // Single backtick inline-code blocks the inner mention; outer mention fires.
    const body = 'Outside: Acme works. Inline `Acme inside` should skip. After.';
    const mentions = findMentionedEntities(body, g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(1); // first-mention-only cap
    expect(mentions[0]!.slug).toBe('companies/acme');
  });
});

// ============================================================
// CJK — entity extraction tests
// ============================================================

describe('findMentionedEntities — CJK cases', () => {
  test('CJK single-name match — "艾例丝" in body → matched', () => {
    const g = gazetteerFromEntries([
      { slug: 'people/alice-example', source_id: 'default', title: '艾例丝' },
    ]);
    const mentions = findMentionedEntities('我最近读了艾例丝的书。', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(1);
    expect(mentions[0]!.slug).toBe('people/alice-example');
    expect(mentions[0]!.name).toBe('艾例丝');
  });

  test('CJK multi-name — two different CJK entities in one body', () => {
    const g = gazetteerFromEntries([
      { slug: 'people/alice-example', source_id: 'default', title: '艾例丝' },
      { slug: 'people/bob-example', source_id: 'default', title: '鲍例布' },
    ]);
    const mentions = findMentionedEntities('艾例丝和鲍例布都是作家。', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(2);
    const slugs = mentions.map(m => m.slug);
    expect(slugs).toContain('people/alice-example');
    expect(slugs).toContain('people/bob-example');
  });

  test('CJK first-mention-only — repeated name → single link', () => {
    const g = gazetteerFromEntries([
      { slug: 'people/alice-example', source_id: 'default', title: '艾例丝' },
    ]);
    const mentions = findMentionedEntities('艾例丝说过。然后艾例丝又说过。', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(1);
  });

  test('CJK self-link guard — entity page mentioning itself is skipped', () => {
    const g = gazetteerFromEntries([
      { slug: 'people/alice-example', source_id: 'default', title: '艾例丝' },
    ]);
    const mentions = findMentionedEntities('艾例丝是一位投资人。', g, {
      fromSlug: 'people/alice-example', fromSourceId: 'default',
    });
    expect(mentions).toEqual([]);
  });

  test('CJK cross-source guard — entity in different source skipped', () => {
    const g = gazetteerFromEntries([
      { slug: 'people/alice-example', source_id: 'team-b', title: '艾例丝' },
    ]);
    const mentions = findMentionedEntities('艾例丝写了这本书。', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'team-a',
    });
    expect(mentions).toEqual([]);
  });

  test('CJK code-block stripping — CJK name inside ``` is skipped, outside matched', () => {
    const g = gazetteerFromEntries([
      { slug: 'people/alice-example', source_id: 'default', title: '艾例丝' },
    ]);
    // "艾例丝" only appears inside code block → should be skipped.
    const body = '```\n艾例丝\n```\n只有代码块里面有。';
    const mentions = findMentionedEntities(body, g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(0);
  });

  test('CJK determinism — same output across 10 calls', () => {
    const g = gazetteerFromEntries([
      { slug: 'people/alice-example', source_id: 'default', title: '艾例丝' },
      { slug: 'people/bob-example', source_id: 'default', title: '鲍例布' },
    ]);
    const body = '艾例丝和鲍例布。艾例丝再说一次。';
    const refs = new Set<string>();
    for (let i = 0; i < 10; i++) {
      const mentions = findMentionedEntities(body, g, {
        fromSlug: 'writing/post-1', fromSourceId: 'default',
      });
      refs.add(JSON.stringify(mentions));
    }
    expect(refs.size).toBe(1);
  });

  test('CJK mixed body — CJK entity matched in body with ASCII around it', () => {
    const g = gazetteerFromEntries([
      { slug: 'people/alice-example', source_id: 'default', title: '艾例丝' },
      { slug: 'companies/acme', source_id: 'default', title: 'Acme' },
    ]);
    const mentions = findMentionedEntities('Acme was founded by 艾例丝 in 2020.', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(2);
    const slugs = mentions.map(m => m.slug);
    expect(slugs).toContain('people/alice-example');
    expect(slugs).toContain('companies/acme');
  });

  test('CJK empty gazetteer — no false positives', () => {
    const g: Gazetteer = new Map();
    const mentions = findMentionedEntities('艾例丝和鲍例布。', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toEqual([]);
  });

  test('CJK empty text → empty result', () => {
    const g = gazetteerFromEntries([
      { slug: 'people/alice-example', source_id: 'default', title: '艾例丝' },
    ]);
    const mentions = findMentionedEntities('', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toEqual([]);
  });
});

// ============================================================
// Vietnamese (diacritic Latin) — entity extraction tests
// ============================================================

// Fictional Vietnamese names only (privacy rule: no real people in fixtures).
// "Đà Nẵng" is a public city, not a person, and is the canonical đ-diacritic case.
//
// Every case below asserts TOKENIZATION, not just the resolved mention. A
// mention-only assertion does not discriminate: the previous ASCII tokenizer
// fragmented the gazetteer title and the body symmetrically, so a 5-fragment
// entry still matched a 5-fragment body run, and `Mention.name` is copied from
// the untouched `title` column rather than derived from tokens.
describe('findMentionedEntities — Vietnamese cases', () => {
  test('VN multi-syllable name matches as a WHOLE (regression: no diacritic fragmentation)', () => {
    // Discriminating assertion: the ASCII tokenizer produced
    // ['nguy','n','v','n','c'] for this title.
    expect(tokenizeTitle('Nguyễn Văn Đức')).toEqual(['nguyễn', 'văn', 'đức']);
    const body = 'Hôm nay mình học bài của thầy Nguyễn Văn Đức.';
    expect(tokenizeForScan(body).map(t => t.text)).toContain('nguyễn');

    const g = gazetteerFromEntries([
      { slug: 'people/nguyen-van-duc', source_id: 'default', title: 'Nguyễn Văn Đức' },
    ]);
    const mentions = findMentionedEntities(body, g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(1);
    expect(mentions[0]!.slug).toBe('people/nguyen-van-duc');
    expect(mentions[0]!.name).toBe('Nguyễn Văn Đức');
  });

  test('VN place name with đ/diacritics — "Đà Nẵng" matched', () => {
    // Discriminating assertion: the ASCII tokenizer produced ['n','ng'],
    // which is what made this entity match 820 pages instead of 440.
    expect(tokenizeTitle('Đà Nẵng')).toEqual(['đà', 'nẵng']);
    const body = 'Gia đình mình chuyển tới Đà Nẵng năm ngoái.';
    expect(tokenizeForScan(body).map(t => t.text)).toContain('nẵng');

    const g = gazetteerFromEntries([
      { slug: 'places/da-nang', source_id: 'default', title: 'Đà Nẵng' },
    ]);
    const mentions = findMentionedEntities(body, g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(1);
    expect(mentions[0]!.slug).toBe('places/da-nang');
    expect(mentions[0]!.name).toBe('Đà Nẵng');
  });

  test('VN NFD body matches an NFC gazetteer title (and the reverse)', () => {
    const nfc = 'Nguyễn Văn';
    const nfd = nfc.normalize('NFD');
    expect(nfd).not.toBe(nfc);                       // fixture really is decomposed
    expect(tokenizeTitle(nfd)).toEqual(tokenizeTitle(nfc));
    expect(tokenizeTitle(nfd)).toEqual(['nguyễn', 'văn']);

    const opts = { fromSlug: 'writing/post-1', fromSourceId: 'default' };
    const gNfc = gazetteerFromEntries([{ slug: 'people/nvd', source_id: 'default', title: nfc }]);
    expect(findMentionedEntities(`Thầy ${nfd} nói.`, gNfc, opts)).toHaveLength(1);

    const gNfd = gazetteerFromEntries([{ slug: 'people/nvd', source_id: 'default', title: nfd }]);
    expect(findMentionedEntities(`Thầy ${nfc} nói.`, gNfd, opts)).toHaveLength(1);
  });

  test('VN diacritics are significant — "Hồng" title does NOT match diacritic-free "Hong"', () => {
    // The title must survive tokenization intact for this to mean anything:
    // under the ASCII tokenizer it became ['l','th','h','ng'] and missed for
    // the wrong reason.
    expect(tokenizeTitle('Lê Thị Hồng')).toEqual(['lê', 'thị', 'hồng']);
    const g = gazetteerFromEntries([
      { slug: 'people/le-thi-hong', source_id: 'default', title: 'Lê Thị Hồng' },
    ]);
    // Body uses the ASCII-typed variant "Le Thi Hong" — tokens differ, no false match.
    const mentions = findMentionedEntities('Gặp Le Thi Hong hôm qua.', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toEqual([]);
  });

  test('VN mixed with ASCII — Vietnamese name + ASCII company in one body', () => {
    const body = 'Phạm Quốc Bảo hợp tác với Acme.';
    // Discriminating: the ASCII tokenizer emitted ['ph','m','qu','c','b','o',
    // 'h','p','t','c','v','i','acme'] here — only the ASCII control survived.
    expect(tokenizeForScan(body).map(t => t.text))
      .toEqual(['phạm', 'quốc', 'bảo', 'hợp', 'tác', 'với', 'acme']);

    const g = gazetteerFromEntries([
      { slug: 'people/pham-quoc-bao', source_id: 'default', title: 'Phạm Quốc Bảo' },
      { slug: 'companies/acme', source_id: 'default', title: 'Acme' },
    ]);
    const mentions = findMentionedEntities(body, g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(2);
    const slugs = mentions.map(m => m.slug);
    expect(slugs).toContain('people/pham-quoc-bao');
    expect(slugs).toContain('companies/acme');
  });

  test('VN longest-match wins — "Nguyễn Văn Đức" beats a shorter "Nguyễn Văn" entry', () => {
    // Both entries must share a real first token for maximal-munch to be
    // exercised at all; under the ASCII tokenizer both keyed on 'nguy'.
    expect(tokenizeTitle('Nguyễn Văn')).toEqual(['nguyễn', 'văn']);
    expect(tokenizeTitle('Nguyễn Văn Đức')[0]).toBe('nguyễn');
    const g = gazetteerFromEntries([
      { slug: 'people/nguyen-van-duc', source_id: 'default', title: 'Nguyễn Văn Đức' },
      { slug: 'people/nguyen-van', source_id: 'default', title: 'Nguyễn Văn' },
    ]);
    const mentions = findMentionedEntities('Bài giảng của Nguyễn Văn Đức rất hay.', g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(1);
    expect(mentions[0]!.slug).toBe('people/nguyen-van-duc');
  });

  test('VN first-mention-only cap — repeated name → single link', () => {
    const g = gazetteerFromEntries([
      { slug: 'people/nguyen-van-duc', source_id: 'default', title: 'Nguyễn Văn Đức' },
    ]);
    const body = 'Nguyễn Văn Đức nói. Sau đó Nguyễn Văn Đức nói tiếp.';
    // The cap must be capping a WHOLE-name match, not a fragment run.
    expect(tokenizeForScan(body).map(t => t.text).slice(0, 3)).toEqual(['nguyễn', 'văn', 'đức']);
    const mentions = findMentionedEntities(body, g, {
      fromSlug: 'writing/post-1', fromSourceId: 'default',
    });
    expect(mentions).toHaveLength(1);
  });

  test('VN determinism — identical output across 10 calls', () => {
    const g = gazetteerFromEntries([
      { slug: 'people/nguyen-van-duc', source_id: 'default', title: 'Nguyễn Văn Đức' },
      { slug: 'places/da-nang', source_id: 'default', title: 'Đà Nẵng' },
    ]);
    const body = 'Thầy Nguyễn Văn Đức ở Đà Nẵng. Nguyễn Văn Đức lần nữa.';
    expect(tokenizeForScan(body).map(t => t.text))
      .toEqual(['thầy', 'nguyễn', 'văn', 'đức', 'ở', 'đà', 'nẵng', 'nguyễn', 'văn', 'đức', 'lần', 'nữa']);
    const refs = new Set<string>();
    for (let i = 0; i < 10; i++) {
      refs.add(JSON.stringify(findMentionedEntities(body, g, {
        fromSlug: 'writing/post-1', fromSourceId: 'default',
      })));
    }
    expect(refs.size).toBe(1);
  });
});

// ============================================================
// Tokenizer boundaries — non-word glyphs
// ============================================================

// Guards for the two ways a Unicode tokenizer regresses against the ASCII one
// it replaces. Both were found in review of the first version of this change,
// which used /[[\p{L}\p{M}\p{N}]--[CJK]]+/gv: \p{M} let a token consist of
// combining marks alone, and \p{N} minted tokens the ASCII regex never emitted.
describe('tokenizer boundaries — marks and non-ASCII numerics', () => {
  const opts = { fromSlug: 'writing/post-1', fromSourceId: 'default' };

  test('a token can never be combining marks alone (U+FE0F does not become a key)', () => {
    // VARIATION SELECTOR-16 is \p{Mn} and rides on most emoji. Allowing a
    // mark-only token made every emoji-prefixed entity title key on a bare
    // U+FE0F, collapsing them into one mutually-confusable bucket.
    expect(tokenizeTitle('❤️ Health Notes')).toEqual(['health', 'notes']);
    expect(tokenizeTitle('⭐️ Budget Notes')).toEqual(['budget', 'notes']);
    expect(tokenizeForScan('❤️').map(t => t.text)).toEqual([]);

    const g = gazetteerFromEntries([
      { slug: 'companies/health-notes', source_id: 'default', title: '❤️ Health Notes' },
      { slug: 'companies/budget-notes', source_id: 'default', title: '⭐️ Budget Notes' },
    ]);
    expect([...g.keys()].sort()).toEqual(['budget', 'health']);

    // The plain-text link survives...
    expect(findMentionedEntities('Plain health notes, no emoji.', g, opts).map(m => m.slug))
      .toEqual(['companies/health-notes']);
    // ...and an unrelated emoji in the body does not drag in the other entity.
    expect(findMentionedEntities('Sprint ⚠️ health notes were fine.', g, opts).map(m => m.slug))
      .toEqual(['companies/health-notes']);
  });

  test('non-ASCII numerics do not break strict token adjacency of an ASCII name', () => {
    // findMentionedEntities requires an entry's tokens to be STRICTLY
    // ADJACENT in the body, so any glyph that newly tokenizes between the
    // words of "Acme Corp" silently kills a match that used to work.
    const g = gazetteerFromEntries([
      { slug: 'companies/acme-corp', source_id: 'default', title: 'Acme Corp' },
    ]);
    for (const body of [
      'We met Acme Corp today.',    // control
      'We met Acme¹ Corp today.',   // U+00B9 superscript one (No)
      'Acme ½ Corp',                // U+00BD vulgar fraction (No)
      'Acme 1 Corp',                // U+FF11 fullwidth digit one (Nd)
      'Acme ❤️ Corp',          // emoji + VS16 (So + Mn)
    ]) {
      expect(findMentionedEntities(body, g, opts)).toHaveLength(1);
    }
    // ASCII digits still tokenize exactly as /[a-zA-Z0-9]+/ did.
    expect(tokenizeForScan('web3 and h2o').map(t => t.text)).toEqual(['web3', 'and', 'h2o']);
  });

  test('Han Extension A is no longer CJK here — aligned with cjk.ts scope', () => {
    // Deliberate behaviour change. by-mention's walkers used to carry their
    // own range copy covering Ext-A (U+3400–4DBF); cjk.ts scopes Ext-A out
    // repo-wide, and this module now uses CJK_SLUG_CHARS verbatim. Ext-A
    // therefore tokenizes as a word run instead of per character, and an
    // Ext-A-only title is one sub-MIN_NAME_LENGTH token rather than N
    // char-level ones. Search, chunking and slug grammar already ignore
    // Ext-A, so this removes by-mention as the lone subsystem that disagreed.
    expect(tokenizeForScan('㐀㐁').map(t => t.text)).toEqual(['㐀㐁']);
    expect(tokenizeTitle('㐀㐁')).toEqual(['㐀㐁']);
    // In-scope CJK is untouched: still char-level.
    expect(tokenizeTitle('艾例丝')).toEqual(['艾', '例', '丝']);
    expect(tokenizeForScan('艾例丝说').map(t => t.text)).toEqual(['艾', '例', '丝', '说']);
  });
});

// ============================================================
// buildGazetteer — engine-backed tests
// ============================================================

describe('buildGazetteer — engine integration', () => {
  test('7. min-length filter — title "AI" (length 2) not in gazetteer', async () => {
    await engine.putPage('companies/ai', {
      type: 'company', title: 'AI', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    await engine.putPage('companies/acme', {
      type: 'company', title: 'Acme', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    const g = await buildGazetteer(engine);
    expect(g.has('acme')).toBe(true);
    expect(g.has('ai')).toBe(false);
  });

  test('8. ignore-list at build — "Apple" suppressed when no companies/apple page', async () => {
    // Seed a different entity page named "Apple" — but importantly NO
    // companies/apple slug exists. Wait — actually the ignore-list keys
    // on TITLE not slug. So even a non-companies slug with title="Apple"
    // would be in the gazetteer because `existingTitles.has("Apple")` is true.
    // The ignore list only fires when NO row has title="Apple". To exercise
    // suppression: seed no entity with title="Apple" — duh, then there's
    // nothing in the gazetteer for Apple anyway. The ignore-list rule is
    // only meaningful if a HYPOTHETICAL entity named "Apple" would otherwise
    // appear; in practice, the ignore-list short-circuits ANY row whose
    // title is in the ignore set AND whose title isn't in existingTitles.
    // For a deterministic test: seed one entity with title="Apple" and
    // verify it IS in the gazetteer (per CK12 inverse rule); seed another
    // run with no Apple entity and verify the ignore-list doesn't add one.
    // Both behaviors covered by the existingTitles vs ignore_set logic.
    await engine.putPage('people/alice', {
      type: 'person', title: 'Alice Example', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    const g = await buildGazetteer(engine);
    // No Apple entity seeded → 'apple' not a gazetteer key (trivially).
    expect(g.has('apple')).toBe(false);
    // Alice IS in gazetteer.
    expect(g.has('alice')).toBe(true);
  });

  test('9. ignore-list inverse — title "Apple" matches when companies/apple exists (CK12)', async () => {
    await engine.putPage('companies/apple', {
      type: 'company', title: 'Apple', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    const g = await buildGazetteer(engine);
    // existingTitles has "Apple" so the ignore-list does NOT suppress;
    // gazetteer presence wins per CK12 rule.
    expect(g.has('apple')).toBe(true);
    expect(g.get('apple')![0]!.slug).toBe('companies/apple');
  });

  test('#6158: a page carrying google_contact_id builds a connector_stub entry; a hand-written page does not', async () => {
    await engine.executeRaw(`INSERT INTO sources (id, name, config) VALUES ('gmail-a', 'gmail-a', '{"kind":"google"}'::jsonb) ON CONFLICT (id) DO NOTHING`);
    await engine.putPage('people/dana-example', { type: 'person', title: 'Dana Example', compiled_truth: 'Profile.', timeline: '', frontmatter: {} });
    await engine.putPage('people/dana-example', { type: 'person', title: 'Dana Example', compiled_truth: 'Contact: [email protected].', timeline: '',
      frontmatter: { google_contact_id: 'people/c1' } }, { sourceId: 'gmail-a' });
    const g = await buildGazetteer(engine);
    const bucket = g.get('dana')!.filter((e) => e.origin === 'title');
    expect(bucket.find((e) => e.source_id === 'gmail-a')?.connector_stub).toBe(true);
    expect(bucket.find((e) => e.source_id === 'default')?.connector_stub).toBeUndefined();
  });

  test('alias entries (v0.46.15, #3801): page_aliases become gazetteer entries', async () => {
    await engine.putPage('people/saoirse-x', {
      type: 'person', title: 'Saoirse Example', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    await engine.setPageAliases('people/saoirse-x', 'default', ['saoirse']);
    const g = await buildGazetteer(engine);
    const bucket = g.get('saoirse');
    expect(bucket).toBeDefined();
    expect(bucket!.some((e) => e.slug === 'people/saoirse-x')).toBe(true);
  });

  test('REGRESSION (v0.46.15): ignore-list rejects ALIAS entries case-insensitively; CK12 title behavior unchanged', async () => {
    // Title side (unchanged CK12 policy): a real page titled "Apple" stays.
    await engine.putPage('companies/apple', {
      type: 'company', title: 'Apple', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    // Alias side (new teeth): an alias "apple" on an unrelated page is
    // suppressed — aliases are not user-created pages; the cased ignore list
    // must match the normalized-lowercase alias store.
    await engine.putPage('people/annie-p', {
      type: 'person', title: 'Annie P Example', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    await engine.setPageAliases('people/annie-p', 'default', ['apple']);
    const g = await buildGazetteer(engine);
    const bucket = g.get('apple') ?? [];
    expect(bucket.some((e) => e.slug === 'companies/apple')).toBe(true); // title entry survives
    expect(bucket.some((e) => e.slug === 'people/annie-p')).toBe(false); // alias entry rejected
  });

  test('alias entries: ambiguous alias (two slugs, same source) is skipped', async () => {
    await engine.putPage('people/sable-one', {
      type: 'person', title: 'Sable One Example', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    await engine.putPage('people/sable-two', {
      type: 'person', title: 'Sable Two Example', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    await engine.setPageAliases('people/sable-one', 'default', ['sable']);
    await engine.setPageAliases('people/sable-two', 'default', ['sable']);
    const g = await buildGazetteer(engine);
    // Title entries ("Sable One Example" etc.) legitimately share the key —
    // the ALIAS-shaped entries (single-token) must be absent.
    expect((g.get('sable') ?? []).filter((e) => e.tokens.length === 1)).toHaveLength(0);
  });

  test('alias entries: alias colliding with an existing page TITLE in the same source is skipped', async () => {
    await engine.putPage('companies/acme-corp', {
      type: 'company', title: 'Acme', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    await engine.putPage('people/andy-c', {
      type: 'person', title: 'Andy C Example', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    await engine.setPageAliases('people/andy-c', 'default', ['acme']);
    const g = await buildGazetteer(engine);
    const bucket = g.get('acme') ?? [];
    expect(bucket.some((e) => e.slug === 'companies/acme-corp')).toBe(true);
    expect(bucket.some((e) => e.slug === 'people/andy-c')).toBe(false);
  });

  test('alias entries: sub-MIN_NAME_LENGTH aliases are skipped', async () => {
    await engine.putPage('people/jt-example', {
      type: 'person', title: 'JT Example Person', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    await engine.setPageAliases('people/jt-example', 'default', ['jt']);
    const g = await buildGazetteer(engine);
    // The multi-token TITLE entry may share the key; the single-token alias
    // entry ('jt' < MIN_NAME_LENGTH) must be absent.
    expect((g.get('jt') ?? []).filter((e) => e.tokens.length === 1)).toHaveLength(0);
  });

  test('13. all entity pages soft-deleted → empty gazetteer', async () => {
    await engine.putPage('people/alice', {
      type: 'person', title: 'Alice Example', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    await engine.softDeletePage('people/alice');
    const g = await buildGazetteer(engine);
    expect(g.size).toBe(0);
  });

  test('18. hardcoded type filter — page with type=meeting NOT in gazetteer', async () => {
    await engine.putPage('meetings/2026-01-15', {
      type: 'meeting' as any, title: 'Weekly Sync',
      compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    await engine.putPage('people/bob', {
      type: 'person', title: 'Robert Builder', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    const g = await buildGazetteer(engine);
    expect(g.has('weekly')).toBe(false); // meeting type filtered out
    expect(g.has('robert')).toBe(true);  // person type included
  });

  test('19. min-length + ignore-list interaction — "YC" (2 chars) filtered by min-length BEFORE ignore-list', async () => {
    // YC isn't in DEFAULT_IGNORE_LIST. But "Box" (3 chars) is. "Box" length
    // = 3 < MIN_NAME_LENGTH (4), so it's filtered by min-length first. The
    // ignore-list never fires. We test the regression that the min-length
    // gate runs BEFORE the ignore-list (so adding Box to ignore-list
    // doesn't accidentally change the filter ordering).
    await engine.putPage('companies/box', {
      type: 'company', title: 'Box', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    const g = await buildGazetteer(engine);
    // "Box" is 3 chars → min-length filter drops it (whether or not in ignore-list).
    expect(g.has('box')).toBe(false);
  });

  test('operator levers for a title the CK12 rule keeps: mentions.exclude_slugs (the removed extraIgnore never applied to titles)', async () => {
    await engine.putPage('people/john', {
      type: 'person', title: 'John', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    const g1 = await buildGazetteer(engine);
    expect(g1.has('john')).toBe(true); // baseline: a user-created page's title is in the gazetteer
    await engine.setConfig('mentions.exclude_slugs', 'people/john');
    try {
      const g2 = await buildGazetteer(engine);
      expect(g2.has('john')).toBe(false);
    } finally {
      await engine.unsetConfig('mentions.exclude_slugs');
    }
  });

  test('LINKABLE_ENTITY_TYPES exposes the hardcoded contract', () => {
    // Regression: if anyone changes the hardcoded type list, this test
    // forces a deliberate change (and a corresponding test update).
    expect(LINKABLE_ENTITY_TYPES).toEqual(['person', 'company', 'organization', 'entity']);
  });

  // CJK — engine-backed tests
  test('CJK entity with 2-char title enters gazetteer with char-level tokens', async () => {
    await engine.putPage('people/alice-example', {
      type: 'person', title: '艾例丝', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    const g = await buildGazetteer(engine);
    // "艾例丝" tokenized as ["艾","例","丝"] → key is "艾"
    expect(g.has('艾')).toBe(true);
    const bucket = g.get('艾')!;
    expect(bucket.length).toBe(1);
    expect(bucket[0]!.tokens).toEqual(['艾', '例', '丝']);
    expect(bucket[0]!.slug).toBe('people/alice-example');
  });

  test('CJK single-char title (cjkCharCount < 2) excluded from gazetteer', async () => {
    await engine.putPage('people/x', {
      type: 'person', title: '谢', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    const g = await buildGazetteer(engine);
    expect(g.size).toBe(0);
  });

  test('VN person title enters gazetteer keyed on first diacritic-preserving token', async () => {
    await engine.putPage('people/nguyen-van-duc', {
      type: 'person', title: 'Nguyễn Văn Đức', compiled_truth: 'b', timeline: '', frontmatter: {},
    });
    const g = await buildGazetteer(engine);
    // "Nguyễn Văn Đức" → ["nguyễn","văn","đức"], keyed on "nguyễn" (NOT fragmented to "nguy").
    expect(g.has('nguyễn')).toBe(true);
    const bucket = g.get('nguyễn')!;
    expect(bucket[0]!.tokens).toEqual(['nguyễn', 'văn', 'đức']);
    expect(bucket[0]!.slug).toBe('people/nguyen-van-duc');
    // Regression guard: the old ASCII tokenizer would have keyed on "nguy".
    expect(g.has('nguy')).toBe(false);
  });
});