Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(citations): match canonically equivalent Unicode text (#4560)

fengyue-xveec6325313fc0
diff --git a/frontend/src/utils/sourceLocator.test.ts b/frontend/src/utils/sourceLocator.test.ts
index 27b02c056..038afc1fe 100644
--- a/frontend/src/utils/sourceLocator.test.ts
+++ b/frontend/src/utils/sourceLocator.test.ts
@@ -113,3 +113,25 @@ test('EPUB paths resolve against their document and preview as epub', () => {
assert.equal(resolveEpubPath('OEBPS/content.opf', 'ch1.xhtml'), 'OEBPS/ch1.xhtml')
assert.equal(resolvePreviewKind('epub'), 'epub')
})
+
+
+test('citation matching treats composed and decomposed accents as the same text', () => {
+ assert.equal(normalizeForMatch('Café'), normalizeForMatch('Cafe\u0301'))
+ assert.notEqual(normalizeForMatch('Café'), normalizeForMatch('Cafe'))
+ assert.equal(normalizeForMatch('Ångström'), normalizeForMatch('A\u030Angstro\u0308m'))
+ assert.equal(normalizeForMatch('각도'), normalizeForMatch('\u1100\u1161\u11A8도'))
+})
+
+test('canonical matching returns exact original UTF-16 spans including combining marks', () => {
+ for (const [text, quote, expected] of [
+ ['前言 Cafe\u0301 后记', 'Café', 'Cafe\u0301'],
+ ['前言 Café 后记', 'Cafe\u0301', 'Café'],
+ ['𝒜前言 cafe\u0301 后记', 'CAFÉ', 'cafe\u0301'],
+ ['前言 A\u030Angstro\u0308m 后记', 'Ångström', 'A\u030Angstro\u0308m'],
+ ['前言 \u1100\u1161\u11A8도 后记', '각도', '\u1100\u1161\u11A8도'],
+ ]) {
+ const hit = findInText(text, quote)
+ assert.ok(hit, text)
+ assert.equal(text.slice(hit.start, hit.end), expected)
+ }
+})
diff --git a/frontend/src/utils/sourceLocator.ts b/frontend/src/utils/sourceLocator.ts
index c2a4abe01..6a9387022 100644
--- a/frontend/src/utils/sourceLocator.ts
+++ b/frontend/src/utils/sourceLocator.ts
@@ -55,6 +55,32 @@ export type SourceLocateRequest = {
// break matching.
const LETTER_OR_DIGIT = /[\p{L}\p{N}]/u
+const GRAPHEMES = typeof Intl.Segmenter === 'function'
+ ? new Intl.Segmenter(undefined, { granularity: 'grapheme' })
+ : undefined
+
+function* normalizationSpans(raw: string): Iterable<{ segment: string; index: number }> {
+ if (GRAPHEMES) {
+ yield* GRAPHEMES.segment(raw)
+ return
+ }
+ // Older browsers can normalize NFC without supporting Intl.Segmenter. Keep
+ // marks with their base and join adjacent starters only when they compose
+ // (notably Hangul L/V/T). These are normalization spans, not full graphemes.
+ let segment = '', index = 0
+ for (const ch of raw) {
+ if (segment && !/\p{M}/u.test(ch)) {
+ const boundary = segment.normalize('NFC') + ch.normalize('NFC')
+ if (boundary.normalize('NFC') === boundary) {
+ yield { segment, index }
+ index += segment.length
+ segment = ''
+ }
+ }
+ segment += ch
+ }
+ if (segment) yield { segment, index }
+}
export function foldChar(ch: string): string {
if (!LETTER_OR_DIGIT.test(ch)) return ''
@@ -66,10 +92,24 @@ export function normalizeForMatch(text: string): string {
return normalizeWithPositions(text).norm
}
-export function normalizeWithPositions(text: string): { norm: string; pos: number[] } {
- const chars = Array.from(String(text || ''))
+export function normalizeWithPositions(text: string): { norm: string; pos: number[]; endPos: number[] } {
+ const raw = String(text || '')
+ const canonical = raw.normalize('NFC')
+ const chars = Array.from(canonical)
+ const starts: number[] = [], ends: number[] = []
+ // Most text is already NFC. Only build the span map when composition or
+ // combining marks require a different span than one original code point.
+ if (canonical !== raw || /\p{M}/u.test(raw)) {
+ for (const { segment, index } of normalizationSpans(raw)) {
+ const length = Array.from(segment.normalize('NFC')).length
+ for (let j = 0; j < length; j++) {
+ starts.push(index)
+ ends.push(index + segment.length)
+ }
+ }
+ }
let norm = ''
- const pos: number[] = []
+ const pos: number[] = [], endPos: number[] = []
let offset = 0
for (let i = 0; i < chars.length; i++) {
const ch = chars[i]!
@@ -87,10 +127,13 @@ export function normalizeWithPositions(text: string): { norm: string; pos: numbe
}
norm += folded
// Offsets throughout this module are UTF-16, including supplementary letters.
- for (let j = 0; j < folded.length; j++) pos.push(offset)
+ for (let j = 0; j < folded.length; j++) {
+ pos.push(starts[i] ?? offset)
+ endPos.push(ends[i] ?? offset + ch.length)
+ }
offset += ch.length
}
- return { norm, pos }
+ return { norm, pos, endPos }
}
export type NormalizedMatch = { start: number; end: number; score: number }
@@ -132,11 +175,10 @@ export function findNormalized(haystack: string, quote: string): NormalizedMatch
}
export function findTextMatches(text: string, quote: string): TextMatch[] {
- const { norm, pos } = normalizeWithPositions(text)
- return findNormalizedMatches(norm, quote).map((m) => {
- const start = pos[m.start]!, last = pos[m.end - 1]!
- return { start, end: last + (text.codePointAt(last)! > 0xffff ? 2 : 1) }
- })
+ const { norm, pos, endPos } = normalizeWithPositions(text)
+ return findNormalizedMatches(norm, quote).map((m) => ({
+ start: pos[m.start]!, end: endPos[m.end - 1]!,
+ }))
}
/** Locate a quote in raw text; returns UTF-16 offsets into `text`. */
diff --git a/frontend/src/utils/sourceLocatorCompatibility.test.ts b/frontend/src/utils/sourceLocatorCompatibility.test.ts
new file mode 100644
index 000000000..393e6c9be
--- /dev/null
+++ b/frontend/src/utils/sourceLocatorCompatibility.test.ts
@@ -0,0 +1,78 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import test from 'node:test'
+import { runInNewContext } from 'node:vm'
+import ts from 'typescript'
+import type * as SourceLocator from './sourceLocator'
+import type * as TableSource from './tableSource'
+
+const compile = (file: string) => ts.transpileModule(
+ readFileSync(new URL(file, import.meta.url), 'utf8'),
+ { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } },
+).outputText
+const locatorCode = compile('./sourceLocator.ts')
+const tableCode = compile('./tableSource.ts')
+
+function loadModules(segmenter: typeof Intl.Segmenter | undefined) {
+ const locator = {} as typeof SourceLocator
+ // Evaluate the module itself with the API absent, without changing global Intl.
+ runInNewContext(locatorCode, { exports: locator, Intl: { Segmenter: segmenter } })
+ const table = {} as typeof TableSource
+ runInNewContext(tableCode, {
+ exports: table,
+ require: (name: string) => {
+ assert.equal(name, './sourceLocator')
+ return locator
+ },
+ })
+ return { ...locator, ...table }
+}
+
+for (const segmenter of [Intl.Segmenter, undefined]) {
+ const mode = segmenter ? 'with Intl.Segmenter' : 'without Intl.Segmenter'
+
+ test(`citation modules load and preserve ordinary matching ${mode}`, () => {
+ const { findInText, normalizeForMatch } = loadModules(segmenter)
+ const hit = findInText('hello world', 'world')
+ assert.ok(hit)
+ assert.equal(hit.start, 6)
+ assert.equal(hit.end, 11)
+ assert.equal(normalizeForMatch('8.14 Hello!'), '8.14hello')
+ assert.equal(findInText('hello world world', 'world'), null)
+ assert.equal(findInText('value 15', '1.5'), null)
+ })
+
+ test(`canonical matching preserves original UTF-16 spans ${mode}`, () => {
+ const { findInText, normalizeForMatch } = loadModules(segmenter)
+ assert.notEqual(normalizeForMatch('Café'), normalizeForMatch('Cafe'))
+ for (const [raw, quote] of [
+ ['Cafe\u0301', 'Café'],
+ ['Café', 'Cafe\u0301'],
+ ['A\u030Angstro\u0308m', 'Ångström'],
+ ['Ångström', 'Ångström'], // A single code point can also change under NFC.
+ ['\u1100\u1161\u11A8도', '각도'],
+ ['가\u11A8도', '각도'],
+ ['각도', '\u1100\u1161\u11A8도'],
+ ['ab\u0301', 'ab'], // Uncomposed marks must still be included in the span.
+ ['a\u0315\u0300b', 'à\u0315b'], // Canonical mark reordering.
+ ] as const) {
+ const text = `𝒜前言 ${raw} 后记`
+ assert.equal(normalizeForMatch(raw), normalizeForMatch(quote))
+ const hit = findInText(text, quote)
+ assert.ok(hit, text)
+ assert.equal(hit.start, text.indexOf(raw))
+ assert.equal(hit.end, text.indexOf(raw) + raw.length)
+ assert.equal(text.slice(hit.start, hit.end), raw)
+ }
+ })
+
+ test(`table highlights retain combining marks ${mode}`, () => {
+ const { findTableSource } = loadModules(segmenter)
+ const page = 'Maths Cafe\u0301\nPhysics Andre\u0301\nHistory Chloe\u0301\nBiology Renee\u0301'
+ const scope = '|Maths Physics History Biology|Café André Chloé Reneé|'
+ const hits = findTableSource([page], scope)
+ assert.equal(hits.length, 8)
+ assert.ok(hits.some(hit => hit.quote === 'Cafe\u0301'))
+ for (const hit of hits) assert.equal(page.slice(hit.start, hit.end), hit.quote)
+ })
+}
diff --git a/frontend/src/utils/tableSource.test.ts b/frontend/src/utils/tableSource.test.ts
index c2e8863fc..a78fe2c25 100644
--- a/frontend/src/utils/tableSource.test.ts
+++ b/frontend/src/utils/tableSource.test.ts
@@ -64,3 +64,13 @@ test('image-heavy steps recover exact instructions around font arrows and page g
assert.ok(r.targets.some(t => t.quote?.includes('申请选修')))
assert.ok(r.targets.every(t => !t.quote?.includes('额外说明')))
})
+
+
+test('table cell spans include original combining marks after canonical matching', () => {
+ const page = 'Maths Cafe\u0301\nPhysics Andre\u0301\nHistory Chloe\u0301\nBiology Renee\u0301'
+ const scope = '|Maths Physics History Biology|Café André Chloé Reneé|'
+ const hits = findTableSource([page], scope)
+ assert.equal(hits.length, 8)
+ assert.ok(hits.some(hit => hit.quote === 'Cafe\u0301'))
+ for (const hit of hits) assert.equal(page.slice(hit.start, hit.end), hit.quote)
+})
diff --git a/frontend/src/utils/tableSource.ts b/frontend/src/utils/tableSource.ts
index ac1b4fde2..6b373a548 100644
--- a/frontend/src/utils/tableSource.ts
+++ b/frontend/src/utils/tableSource.ts
@@ -2,7 +2,7 @@ import { findNormalizedMatches, normalizeForMatch, normalizeWithPositions, type
export type TableSourceMatch = TextMatch & { page: number; quote: string }
type Column = { text: string; ends: number[] }
-type Row = { text: string; page: number; start: number; norm: string; pos: number[] }
+type Row = { text: string; page: number; start: number; norm: string; pos: number[]; endPos: number[] }
type Step = { row: number; hits: TableSourceMatch[] }
function columns(scope: string): Column[] {
@@ -81,8 +81,7 @@ export function findTableSource(pages: string[], scope: string): TableSourceMatc
if (size < 2 || size > row.norm.length) continue
const hits = findNormalizedMatches(row.norm, column.text.slice(offset, end))
if (hits.length !== 1) continue
- const hit = hits[0]!, start = row.pos[hit.start]!, last = row.pos[hit.end - 1]!
- const finish = last + (row.text.codePointAt(last)! > 0xffff ? 2 : 1)
+ const hit = hits[0]!, start = row.pos[hit.start]!, finish = row.endPos[hit.end - 1]!
return { end, hit: { page: row.page, start: row.start + start, end: row.start + finish, quote: row.text.slice(start, finish) } }
}
return null