Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Find suggested links in CJK and Thai text (PRD-9409) (#6409)

Andrew Mikofalvy23b978db88d8
diff --git a/.changeset/suggest-links-unspaced-scripts.md b/.changeset/suggest-links-unspaced-scripts.md
new file mode 100644
index 000000000..3f207759d
--- /dev/null
+++ b/.changeset/suggest-links-unspaced-scripts.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Suggested links now find page mentions in Chinese, Japanese, Thai and other text written without spaces, and report the right position when the text contains characters such as `İ`.
diff --git a/packages/server/src/suggest-links.test.ts b/packages/server/src/suggest-links.test.ts
index 996eb04f3..63595da4c 100644
--- a/packages/server/src/suggest-links.test.ts
+++ b/packages/server/src/suggest-links.test.ts
@@ -529,6 +529,96 @@ describe('suggestLinks', () => {
}
});
+ async function suggestAgainst(
+ targetMarkdown: string,
+ sourceMarkdown: string,
+ ): Promise<Array<{ source: string; excerpt: string; offset: number }>> {
+ const contentDir = mkdtempSync(join(tmpdir(), 'ok-suggest-links-unicode-'));
+ const hocuspocus = new Hocuspocus({ quiet: true });
+ try {
+ writeFileSync(join(contentDir, 'target.md'), targetMarkdown, 'utf-8');
+ writeFileSync(join(contentDir, 'notes.md'), sourceMarkdown, 'utf-8');
+ const result = await suggestLinks({
+ hocuspocus,
+ fileIndex: buildFileIndex(contentDir, ['target', 'notes']),
+ docName: 'target',
+ });
+ return result.mentions;
+ } finally {
+ rmSync(contentDir, { recursive: true, force: true });
+ }
+ }
+
+ test.each([
+ ['Japanese', '東京', '来週東京に行きます。'],
+ ['Chinese', '北京', '我们下周去北京开会。'],
+ ['Thai', 'กรุงเทพ', 'พรุ่งนี้ฉันจะไปกรุงเทพกับเพื่อน'],
+ ['Lao', 'ລາວ', 'ດິນແດນລາວງາມ'],
+ ['Katakana beside Latin', 'テスト', 'Unitテスト'],
+ ['Khmer', 'ភ្នំពេញ', 'ភ្នំពេញជាទីក្រុង'],
+ ['Myanmar', 'ဗမာ', 'ဗမာစကားပြော'],
+ ['Latin beside Japanese', 'React', 'Reactは便利です。'],
+ ['Japanese beside Latin', '日本語', 'Python日本語ガイド'],
+ ])('finds a %s mention in text written without spaces', async (_script, title, source) => {
+ const mentions = await suggestAgainst(`# ${title}\n`, `${source}\n`);
+ expect(mentions).toEqual([{ source: 'notes', excerpt: source, offset: source.indexOf(title) }]);
+ });
+
+ test.each([
+ ['Japanese', '京都', '東京都に住んでいます。'],
+ ['Thai', 'ท', 'ที่นี่'],
+ ])(
+ 'rejects a %s mention that splits a word written without spaces',
+ async (_script, title, source) => {
+ const mentions = await suggestAgainst(`# ${title}\n`, `${source}\n`);
+ expect(mentions).toEqual([]);
+ },
+ );
+
+ test('still rejects a Latin title inside a longer Latin word', async () => {
+ const mentions = await suggestAgainst('# Cat\n', 'The category is not a cat.\n');
+ expect(mentions).toEqual([
+ { source: 'notes', excerpt: 'The category is not a cat.', offset: 22 },
+ ]);
+ });
+
+ test('reads a neighbouring character outside the Basic Multilingual Plane as one letter', async () => {
+ const source = '𝐀cat and cat𝐀 and cat.';
+ const mentions = await suggestAgainst('# Cat\n', `${source}\n`);
+ expect(mentions.map((mention) => mention.offset)).toEqual([source.lastIndexOf('cat')]);
+ });
+
+ test.each([
+ ['C++', 'We write C++ daily.', [9]],
+ ['Node.js', 'Use Node.js, not Nodexjs.', [4]],
+ ['(a|b)?', 'Pick (a|b)? or ab.', [5]],
+ ])('matches the title %s literally', async (title, source, offsets) => {
+ const mentions = await suggestAgainst(`# ${title}\n`, `${source}\n`);
+ expect(mentions.map((mention) => mention.offset)).toEqual(offsets);
+ });
+
+ test('rejects a Latin title followed by a combining accent', async () => {
+ const mentions = await suggestAgainst('# Cafe\n', 'Cafe\u0301 society.\n');
+ expect(mentions).toEqual([]);
+ });
+
+ test('returns source offsets when lowercasing the text would change its length', async () => {
+ const source = 'İİ notes mention Project Alpha here.';
+ const mentions = await suggestAgainst('# Project Alpha\n', `${source}\n`);
+ expect(mentions).toEqual([
+ { source: 'notes', excerpt: source, offset: source.indexOf('Project Alpha') },
+ ]);
+ });
+
+ test('matches a title whose lowercase form is longer than the title', async () => {
+ const source = 'We flew to İzmir and to İZMIR again.';
+ const mentions = await suggestAgainst('# İzmir\n', `${source}\n`);
+ expect(mentions.map((mention) => mention.offset)).toEqual([
+ source.indexOf('İzmir'),
+ source.indexOf('İZMIR'),
+ ]);
+ });
+
test('returns an empty success result when no candidates exist', async () => {
const projectDir = mkdtempSync(join(tmpdir(), 'ok-suggest-links-'));
const contentDir = join(projectDir, 'content');
diff --git a/packages/server/src/suggest-links.ts b/packages/server/src/suggest-links.ts
index 05f4a4c05..4e89d5801 100644
--- a/packages/server/src/suggest-links.ts
+++ b/packages/server/src/suggest-links.ts
@@ -16,7 +16,12 @@ import { buildProjectWikiLinkLookup } from './project-wiki-link-lookup.ts';
const log = getLogger('suggest-links');
-const WORD_CHAR_RE = /[\p{L}\p{N}]/u;
+const WORD_CHAR_RE = /[\p{L}\p{M}\p{N}]/u;
+const UNSPACED_SCRIPT_CHAR_RE =
+ /[\p{scx=Han}\p{scx=Hiragana}\p{scx=Katakana}\p{scx=Thai}\p{scx=Lao}\p{scx=Khmer}\p{scx=Myanmar}]/u;
+const REGEXP_SYNTAX_CHAR_RE = /[\\^$.*+?()[\]{}|/]/g;
+
+const WORD_SEGMENTER = new Intl.Segmenter('und', { granularity: 'word' });
interface FenceState {
char: '`' | '~';
@@ -25,8 +30,7 @@ interface FenceState {
interface SearchLabel {
raw: string;
- lower: string;
- length: number;
+ pattern: RegExp;
}
interface SegmentMatch {
@@ -249,12 +253,57 @@ function readMarkdownImage(line: string, start: number): { alt: string; nextInde
};
}
-function isWordBoundaryChar(char: string | undefined): boolean {
- return !char || !WORD_CHAR_RE.test(char);
+function charAt(text: string, index: number): string | undefined {
+ const codePoint = text.codePointAt(index);
+ return codePoint === undefined ? undefined : String.fromCodePoint(codePoint);
+}
+
+function charBefore(text: string, index: number): string | undefined {
+ if (index <= 0) return undefined;
+ const lowIndex = index - 1;
+ const unit = text.charCodeAt(lowIndex);
+ const isTrailSurrogate = unit >= 0xdc00 && unit <= 0xdfff;
+ return charAt(text, isTrailSurrogate && lowIndex > 0 ? lowIndex - 1 : lowIndex);
}
-function hasWholeWordBoundaries(text: string, start: number, end: number): boolean {
- return isWordBoundaryChar(text[start - 1]) && isWordBoundaryChar(text[end]);
+function isUnspacedScriptChar(char: string | undefined): boolean {
+ return char !== undefined && UNSPACED_SCRIPT_CHAR_RE.test(char);
+}
+
+type WordBoundaryLookup = (index: number) => boolean;
+
+function createWordBoundaryLookup(text: string): WordBoundaryLookup {
+ let boundaries: Set<number> | null = null;
+ return (index) => {
+ if (!boundaries) {
+ boundaries = new Set([text.length]);
+ for (const segment of WORD_SEGMENTER.segment(text)) boundaries.add(segment.index);
+ }
+ return boundaries.has(index);
+ };
+}
+
+function isMatchBoundary(
+ outside: string | undefined,
+ inside: string | undefined,
+ index: number,
+ isWordBoundary: WordBoundaryLookup,
+): boolean {
+ if (outside === undefined || !WORD_CHAR_RE.test(outside)) return true;
+ if (!isUnspacedScriptChar(outside) && !isUnspacedScriptChar(inside)) return false;
+ return isWordBoundary(index);
+}
+
+function hasWholeWordBoundaries(
+ text: string,
+ start: number,
+ end: number,
+ isWordBoundary: WordBoundaryLookup,
+): boolean {
+ return (
+ isMatchBoundary(charBefore(text, start), charAt(text, start), start, isWordBoundary) &&
+ isMatchBoundary(charAt(text, end), charBefore(text, end), end, isWordBoundary)
+ );
}
function prepareSearchLabels(identity: PageIdentity): SearchLabel[] {
@@ -264,36 +313,35 @@ function prepareSearchLabels(identity: PageIdentity): SearchLabel[] {
for (const label of identity.matchLabels) {
const trimmed = label.trim();
if (!trimmed) continue;
- const lower = trimmed.toLocaleLowerCase();
- if (seen.has(lower)) continue;
- seen.add(lower);
+ const key = trimmed.toLowerCase();
+ if (seen.has(key)) continue;
+ seen.add(key);
labels.push({
raw: trimmed,
- lower,
- length: trimmed.length,
+ pattern: new RegExp(trimmed.replace(REGEXP_SYNTAX_CHAR_RE, '\\$&'), 'giu'),
});
}
return labels.sort((left, right) => {
- if (right.length !== left.length) return right.length - left.length;
+ if (right.raw.length !== left.raw.length) return right.raw.length - left.raw.length;
return left.raw.localeCompare(right.raw);
});
}
function findSegmentMatches(text: string, labels: readonly SearchLabel[]): SegmentMatch[] {
- const lowerText = text.toLocaleLowerCase();
const candidates: SegmentMatch[] = [];
+ const isWordBoundary = createWordBoundaryLookup(text);
for (const label of labels) {
- let startIndex = 0;
- while (startIndex <= lowerText.length - label.length) {
- const foundIndex = lowerText.indexOf(label.lower, startIndex);
- if (foundIndex === -1) break;
- const endIndex = foundIndex + label.length;
- if (hasWholeWordBoundaries(text, foundIndex, endIndex)) {
- candidates.push({ start: foundIndex, end: endIndex });
+ const { pattern } = label;
+ pattern.lastIndex = 0;
+ for (let found = pattern.exec(text); found; found = pattern.exec(text)) {
+ const start = found.index;
+ const end = start + found[0].length;
+ if (hasWholeWordBoundaries(text, start, end, isWordBoundary)) {
+ candidates.push({ start, end });
}
- startIndex = foundIndex + 1;
+ pattern.lastIndex = start + (charAt(text, start)?.length ?? 1);
}
}