Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Find suggested links in CJK and Thai text (PRD-9409) (#6409)
Andrew Mikofalvy23b978db88d8
diff --git a/.changeset/suggest-links-unspaced-scripts.md b/.changeset/suggest-links-unspaced-scripts.mdnew file mode 100644index 000000000..3f207759d--- /dev/null+++ b/.changeset/suggest-links-unspaced-scripts.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Suggested links now find page mentions in Chinese, Japanese, Thai and other text written without spaces, and report the right position when the text contains characters such as `İ`.diff --git a/packages/server/src/suggest-links.test.ts b/packages/server/src/suggest-links.test.tsindex 996eb04f3..63595da4c 100644--- a/packages/server/src/suggest-links.test.ts+++ b/packages/server/src/suggest-links.test.ts@@ -529,6 +529,96 @@ describe('suggestLinks', () => {}});+ async function suggestAgainst(+ targetMarkdown: string,+ sourceMarkdown: string,+ ): Promise<Array<{ source: string; excerpt: string; offset: number }>> {+ const contentDir = mkdtempSync(join(tmpdir(), 'ok-suggest-links-unicode-'));+ const hocuspocus = new Hocuspocus({ quiet: true });+ try {+ writeFileSync(join(contentDir, 'target.md'), targetMarkdown, 'utf-8');+ writeFileSync(join(contentDir, 'notes.md'), sourceMarkdown, 'utf-8');+ const result = await suggestLinks({+ hocuspocus,+ fileIndex: buildFileIndex(contentDir, ['target', 'notes']),+ docName: 'target',+ });+ return result.mentions;+ } finally {+ rmSync(contentDir, { recursive: true, force: true });+ }+ }++ test.each([+ ['Japanese', '東京', '来週東京に行きます。'],+ ['Chinese', '北京', '我们下周去北京开会。'],+ ['Thai', 'กรุงเทพ', 'พรุ่งนี้ฉันจะไปกรุงเทพกับเพื่อน'],+ ['Lao', 'ລາວ', 'ດິນແດນລາວງາມ'],+ ['Katakana beside Latin', 'テスト', 'Unitテスト'],+ ['Khmer', 'ភ្នំពេញ', 'ភ្នំពេញជាទីក្រុង'],+ ['Myanmar', 'ဗမာ', 'ဗမာစကားပြော'],+ ['Latin beside Japanese', 'React', 'Reactは便利です。'],+ ['Japanese beside Latin', '日本語', 'Python日本語ガイド'],+ ])('finds a %s mention in text written without spaces', async (_script, title, source) => {+ const mentions = await suggestAgainst(`# ${title}\n`, `${source}\n`);+ expect(mentions).toEqual([{ source: 'notes', excerpt: source, offset: source.indexOf(title) }]);+ });++ test.each([+ ['Japanese', '京都', '東京都に住んでいます。'],+ ['Thai', 'ท', 'ที่นี่'],+ ])(+ 'rejects a %s mention that splits a word written without spaces',+ async (_script, title, source) => {+ const mentions = await suggestAgainst(`# ${title}\n`, `${source}\n`);+ expect(mentions).toEqual([]);+ },+ );++ test('still rejects a Latin title inside a longer Latin word', async () => {+ const mentions = await suggestAgainst('# Cat\n', 'The category is not a cat.\n');+ expect(mentions).toEqual([+ { source: 'notes', excerpt: 'The category is not a cat.', offset: 22 },+ ]);+ });++ test('reads a neighbouring character outside the Basic Multilingual Plane as one letter', async () => {+ const source = '𝐀cat and cat𝐀 and cat.';+ const mentions = await suggestAgainst('# Cat\n', `${source}\n`);+ expect(mentions.map((mention) => mention.offset)).toEqual([source.lastIndexOf('cat')]);+ });++ test.each([+ ['C++', 'We write C++ daily.', [9]],+ ['Node.js', 'Use Node.js, not Nodexjs.', [4]],+ ['(a|b)?', 'Pick (a|b)? or ab.', [5]],+ ])('matches the title %s literally', async (title, source, offsets) => {+ const mentions = await suggestAgainst(`# ${title}\n`, `${source}\n`);+ expect(mentions.map((mention) => mention.offset)).toEqual(offsets);+ });++ test('rejects a Latin title followed by a combining accent', async () => {+ const mentions = await suggestAgainst('# Cafe\n', 'Cafe\u0301 society.\n');+ expect(mentions).toEqual([]);+ });++ test('returns source offsets when lowercasing the text would change its length', async () => {+ const source = 'İİ notes mention Project Alpha here.';+ const mentions = await suggestAgainst('# Project Alpha\n', `${source}\n`);+ expect(mentions).toEqual([+ { source: 'notes', excerpt: source, offset: source.indexOf('Project Alpha') },+ ]);+ });++ test('matches a title whose lowercase form is longer than the title', async () => {+ const source = 'We flew to İzmir and to İZMIR again.';+ const mentions = await suggestAgainst('# İzmir\n', `${source}\n`);+ expect(mentions.map((mention) => mention.offset)).toEqual([+ source.indexOf('İzmir'),+ source.indexOf('İZMIR'),+ ]);+ });+test('returns an empty success result when no candidates exist', async () => {const projectDir = mkdtempSync(join(tmpdir(), 'ok-suggest-links-'));const contentDir = join(projectDir, 'content');diff --git a/packages/server/src/suggest-links.ts b/packages/server/src/suggest-links.tsindex 05f4a4c05..4e89d5801 100644--- a/packages/server/src/suggest-links.ts+++ b/packages/server/src/suggest-links.ts@@ -16,7 +16,12 @@ import { buildProjectWikiLinkLookup } from './project-wiki-link-lookup.ts';const log = getLogger('suggest-links');-const WORD_CHAR_RE = /[\p{L}\p{N}]/u;+const WORD_CHAR_RE = /[\p{L}\p{M}\p{N}]/u;+const UNSPACED_SCRIPT_CHAR_RE =+ /[\p{scx=Han}\p{scx=Hiragana}\p{scx=Katakana}\p{scx=Thai}\p{scx=Lao}\p{scx=Khmer}\p{scx=Myanmar}]/u;+const REGEXP_SYNTAX_CHAR_RE = /[\\^$.*+?()[\]{}|/]/g;++const WORD_SEGMENTER = new Intl.Segmenter('und', { granularity: 'word' });interface FenceState {char: '`' | '~';@@ -25,8 +30,7 @@ interface FenceState {interface SearchLabel {raw: string;- lower: string;- length: number;+ pattern: RegExp;}interface SegmentMatch {@@ -249,12 +253,57 @@ function readMarkdownImage(line: string, start: number): { alt: string; nextInde};}-function isWordBoundaryChar(char: string | undefined): boolean {- return !char || !WORD_CHAR_RE.test(char);+function charAt(text: string, index: number): string | undefined {+ const codePoint = text.codePointAt(index);+ return codePoint === undefined ? undefined : String.fromCodePoint(codePoint);+}++function charBefore(text: string, index: number): string | undefined {+ if (index <= 0) return undefined;+ const lowIndex = index - 1;+ const unit = text.charCodeAt(lowIndex);+ const isTrailSurrogate = unit >= 0xdc00 && unit <= 0xdfff;+ return charAt(text, isTrailSurrogate && lowIndex > 0 ? lowIndex - 1 : lowIndex);}-function hasWholeWordBoundaries(text: string, start: number, end: number): boolean {- return isWordBoundaryChar(text[start - 1]) && isWordBoundaryChar(text[end]);+function isUnspacedScriptChar(char: string | undefined): boolean {+ return char !== undefined && UNSPACED_SCRIPT_CHAR_RE.test(char);+}++type WordBoundaryLookup = (index: number) => boolean;++function createWordBoundaryLookup(text: string): WordBoundaryLookup {+ let boundaries: Set<number> | null = null;+ return (index) => {+ if (!boundaries) {+ boundaries = new Set([text.length]);+ for (const segment of WORD_SEGMENTER.segment(text)) boundaries.add(segment.index);+ }+ return boundaries.has(index);+ };+}++function isMatchBoundary(+ outside: string | undefined,+ inside: string | undefined,+ index: number,+ isWordBoundary: WordBoundaryLookup,+): boolean {+ if (outside === undefined || !WORD_CHAR_RE.test(outside)) return true;+ if (!isUnspacedScriptChar(outside) && !isUnspacedScriptChar(inside)) return false;+ return isWordBoundary(index);+}++function hasWholeWordBoundaries(+ text: string,+ start: number,+ end: number,+ isWordBoundary: WordBoundaryLookup,+): boolean {+ return (+ isMatchBoundary(charBefore(text, start), charAt(text, start), start, isWordBoundary) &&+ isMatchBoundary(charAt(text, end), charBefore(text, end), end, isWordBoundary)+ );}function prepareSearchLabels(identity: PageIdentity): SearchLabel[] {@@ -264,36 +313,35 @@ function prepareSearchLabels(identity: PageIdentity): SearchLabel[] {for (const label of identity.matchLabels) {const trimmed = label.trim();if (!trimmed) continue;- const lower = trimmed.toLocaleLowerCase();- if (seen.has(lower)) continue;- seen.add(lower);+ const key = trimmed.toLowerCase();+ if (seen.has(key)) continue;+ seen.add(key);labels.push({raw: trimmed,- lower,- length: trimmed.length,+ pattern: new RegExp(trimmed.replace(REGEXP_SYNTAX_CHAR_RE, '\\$&'), 'giu'),});}return labels.sort((left, right) => {- if (right.length !== left.length) return right.length - left.length;+ if (right.raw.length !== left.raw.length) return right.raw.length - left.raw.length;return left.raw.localeCompare(right.raw);});}function findSegmentMatches(text: string, labels: readonly SearchLabel[]): SegmentMatch[] {- const lowerText = text.toLocaleLowerCase();const candidates: SegmentMatch[] = [];+ const isWordBoundary = createWordBoundaryLookup(text);for (const label of labels) {- let startIndex = 0;- while (startIndex <= lowerText.length - label.length) {- const foundIndex = lowerText.indexOf(label.lower, startIndex);- if (foundIndex === -1) break;- const endIndex = foundIndex + label.length;- if (hasWholeWordBoundaries(text, foundIndex, endIndex)) {- candidates.push({ start: foundIndex, end: endIndex });+ const { pattern } = label;+ pattern.lastIndex = 0;+ for (let found = pattern.exec(text); found; found = pattern.exec(text)) {+ const start = found.index;+ const end = start + found[0].length;+ if (hasWholeWordBoundaries(text, start, end, isWordBoundary)) {+ candidates.push({ start, end });}- startIndex = foundIndex + 1;+ pattern.lastIndex = start + (charAt(text, start)?.length ?? 1);}}