Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(citations): match canonically equivalent Unicode text (#4560)
fengyue-xveec6325313fc0
diff --git a/frontend/src/utils/sourceLocator.test.ts b/frontend/src/utils/sourceLocator.test.tsindex 27b02c056..038afc1fe 100644--- a/frontend/src/utils/sourceLocator.test.ts+++ b/frontend/src/utils/sourceLocator.test.ts@@ -113,3 +113,25 @@ test('EPUB paths resolve against their document and preview as epub', () => {assert.equal(resolveEpubPath('OEBPS/content.opf', 'ch1.xhtml'), 'OEBPS/ch1.xhtml')assert.equal(resolvePreviewKind('epub'), 'epub')})+++test('citation matching treats composed and decomposed accents as the same text', () => {+ assert.equal(normalizeForMatch('Café'), normalizeForMatch('Cafe\u0301'))+ assert.notEqual(normalizeForMatch('Café'), normalizeForMatch('Cafe'))+ assert.equal(normalizeForMatch('Ångström'), normalizeForMatch('A\u030Angstro\u0308m'))+ assert.equal(normalizeForMatch('각도'), normalizeForMatch('\u1100\u1161\u11A8도'))+})++test('canonical matching returns exact original UTF-16 spans including combining marks', () => {+ for (const [text, quote, expected] of [+ ['前言 Cafe\u0301 后记', 'Café', 'Cafe\u0301'],+ ['前言 Café 后记', 'Cafe\u0301', 'Café'],+ ['𝒜前言 cafe\u0301 后记', 'CAFÉ', 'cafe\u0301'],+ ['前言 A\u030Angstro\u0308m 后记', 'Ångström', 'A\u030Angstro\u0308m'],+ ['前言 \u1100\u1161\u11A8도 后记', '각도', '\u1100\u1161\u11A8도'],+ ]) {+ const hit = findInText(text, quote)+ assert.ok(hit, text)+ assert.equal(text.slice(hit.start, hit.end), expected)+ }+})diff --git a/frontend/src/utils/sourceLocator.ts b/frontend/src/utils/sourceLocator.tsindex c2a4abe01..6a9387022 100644--- a/frontend/src/utils/sourceLocator.ts+++ b/frontend/src/utils/sourceLocator.ts@@ -55,6 +55,32 @@ export type SourceLocateRequest = {// break matching.const LETTER_OR_DIGIT = /[\p{L}\p{N}]/u+const GRAPHEMES = typeof Intl.Segmenter === 'function'+ ? new Intl.Segmenter(undefined, { granularity: 'grapheme' })+ : undefined++function* normalizationSpans(raw: string): Iterable<{ segment: string; index: number }> {+ if (GRAPHEMES) {+ yield* GRAPHEMES.segment(raw)+ return+ }+ // Older browsers can normalize NFC without supporting Intl.Segmenter. Keep+ // marks with their base and join adjacent starters only when they compose+ // (notably Hangul L/V/T). These are normalization spans, not full graphemes.+ let segment = '', index = 0+ for (const ch of raw) {+ if (segment && !/\p{M}/u.test(ch)) {+ const boundary = segment.normalize('NFC') + ch.normalize('NFC')+ if (boundary.normalize('NFC') === boundary) {+ yield { segment, index }+ index += segment.length+ segment = ''+ }+ }+ segment += ch+ }+ if (segment) yield { segment, index }+}export function foldChar(ch: string): string {if (!LETTER_OR_DIGIT.test(ch)) return ''@@ -66,10 +92,24 @@ export function normalizeForMatch(text: string): string {return normalizeWithPositions(text).norm}-export function normalizeWithPositions(text: string): { norm: string; pos: number[] } {- const chars = Array.from(String(text || ''))+export function normalizeWithPositions(text: string): { norm: string; pos: number[]; endPos: number[] } {+ const raw = String(text || '')+ const canonical = raw.normalize('NFC')+ const chars = Array.from(canonical)+ const starts: number[] = [], ends: number[] = []+ // Most text is already NFC. Only build the span map when composition or+ // combining marks require a different span than one original code point.+ if (canonical !== raw || /\p{M}/u.test(raw)) {+ for (const { segment, index } of normalizationSpans(raw)) {+ const length = Array.from(segment.normalize('NFC')).length+ for (let j = 0; j < length; j++) {+ starts.push(index)+ ends.push(index + segment.length)+ }+ }+ }let norm = ''- const pos: number[] = []+ const pos: number[] = [], endPos: number[] = []let offset = 0for (let i = 0; i < chars.length; i++) {const ch = chars[i]!@@ -87,10 +127,13 @@ export function normalizeWithPositions(text: string): { norm: string; pos: numbe}norm += folded// Offsets throughout this module are UTF-16, including supplementary letters.- for (let j = 0; j < folded.length; j++) pos.push(offset)+ for (let j = 0; j < folded.length; j++) {+ pos.push(starts[i] ?? offset)+ endPos.push(ends[i] ?? offset + ch.length)+ }offset += ch.length}- return { norm, pos }+ return { norm, pos, endPos }}export type NormalizedMatch = { start: number; end: number; score: number }@@ -132,11 +175,10 @@ export function findNormalized(haystack: string, quote: string): NormalizedMatch}export function findTextMatches(text: string, quote: string): TextMatch[] {- const { norm, pos } = normalizeWithPositions(text)- return findNormalizedMatches(norm, quote).map((m) => {- const start = pos[m.start]!, last = pos[m.end - 1]!- return { start, end: last + (text.codePointAt(last)! > 0xffff ? 2 : 1) }- })+ const { norm, pos, endPos } = normalizeWithPositions(text)+ return findNormalizedMatches(norm, quote).map((m) => ({+ start: pos[m.start]!, end: endPos[m.end - 1]!,+ }))}/** Locate a quote in raw text; returns UTF-16 offsets into `text`. */diff --git a/frontend/src/utils/sourceLocatorCompatibility.test.ts b/frontend/src/utils/sourceLocatorCompatibility.test.tsnew file mode 100644index 000000000..393e6c9be--- /dev/null+++ b/frontend/src/utils/sourceLocatorCompatibility.test.ts@@ -0,0 +1,78 @@+import assert from 'node:assert/strict'+import { readFileSync } from 'node:fs'+import test from 'node:test'+import { runInNewContext } from 'node:vm'+import ts from 'typescript'+import type * as SourceLocator from './sourceLocator'+import type * as TableSource from './tableSource'++const compile = (file: string) => ts.transpileModule(+ readFileSync(new URL(file, import.meta.url), 'utf8'),+ { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } },+).outputText+const locatorCode = compile('./sourceLocator.ts')+const tableCode = compile('./tableSource.ts')++function loadModules(segmenter: typeof Intl.Segmenter | undefined) {+ const locator = {} as typeof SourceLocator+ // Evaluate the module itself with the API absent, without changing global Intl.+ runInNewContext(locatorCode, { exports: locator, Intl: { Segmenter: segmenter } })+ const table = {} as typeof TableSource+ runInNewContext(tableCode, {+ exports: table,+ require: (name: string) => {+ assert.equal(name, './sourceLocator')+ return locator+ },+ })+ return { ...locator, ...table }+}++for (const segmenter of [Intl.Segmenter, undefined]) {+ const mode = segmenter ? 'with Intl.Segmenter' : 'without Intl.Segmenter'++ test(`citation modules load and preserve ordinary matching ${mode}`, () => {+ const { findInText, normalizeForMatch } = loadModules(segmenter)+ const hit = findInText('hello world', 'world')+ assert.ok(hit)+ assert.equal(hit.start, 6)+ assert.equal(hit.end, 11)+ assert.equal(normalizeForMatch('8.14 Hello!'), '8.14hello')+ assert.equal(findInText('hello world world', 'world'), null)+ assert.equal(findInText('value 15', '1.5'), null)+ })++ test(`canonical matching preserves original UTF-16 spans ${mode}`, () => {+ const { findInText, normalizeForMatch } = loadModules(segmenter)+ assert.notEqual(normalizeForMatch('Café'), normalizeForMatch('Cafe'))+ for (const [raw, quote] of [+ ['Cafe\u0301', 'Café'],+ ['Café', 'Cafe\u0301'],+ ['A\u030Angstro\u0308m', 'Ångström'],+ ['Ångström', 'Ångström'], // A single code point can also change under NFC.+ ['\u1100\u1161\u11A8도', '각도'],+ ['가\u11A8도', '각도'],+ ['각도', '\u1100\u1161\u11A8도'],+ ['ab\u0301', 'ab'], // Uncomposed marks must still be included in the span.+ ['a\u0315\u0300b', 'à\u0315b'], // Canonical mark reordering.+ ] as const) {+ const text = `𝒜前言 ${raw} 后记`+ assert.equal(normalizeForMatch(raw), normalizeForMatch(quote))+ const hit = findInText(text, quote)+ assert.ok(hit, text)+ assert.equal(hit.start, text.indexOf(raw))+ assert.equal(hit.end, text.indexOf(raw) + raw.length)+ assert.equal(text.slice(hit.start, hit.end), raw)+ }+ })++ test(`table highlights retain combining marks ${mode}`, () => {+ const { findTableSource } = loadModules(segmenter)+ const page = 'Maths Cafe\u0301\nPhysics Andre\u0301\nHistory Chloe\u0301\nBiology Renee\u0301'+ const scope = '|Maths Physics History Biology|Café André Chloé Reneé|'+ const hits = findTableSource([page], scope)+ assert.equal(hits.length, 8)+ assert.ok(hits.some(hit => hit.quote === 'Cafe\u0301'))+ for (const hit of hits) assert.equal(page.slice(hit.start, hit.end), hit.quote)+ })+}diff --git a/frontend/src/utils/tableSource.test.ts b/frontend/src/utils/tableSource.test.tsindex c2e8863fc..a78fe2c25 100644--- a/frontend/src/utils/tableSource.test.ts+++ b/frontend/src/utils/tableSource.test.ts@@ -64,3 +64,13 @@ test('image-heavy steps recover exact instructions around font arrows and page gassert.ok(r.targets.some(t => t.quote?.includes('申请选修')))assert.ok(r.targets.every(t => !t.quote?.includes('额外说明')))})+++test('table cell spans include original combining marks after canonical matching', () => {+ const page = 'Maths Cafe\u0301\nPhysics Andre\u0301\nHistory Chloe\u0301\nBiology Renee\u0301'+ const scope = '|Maths Physics History Biology|Café André Chloé Reneé|'+ const hits = findTableSource([page], scope)+ assert.equal(hits.length, 8)+ assert.ok(hits.some(hit => hit.quote === 'Cafe\u0301'))+ for (const hit of hits) assert.equal(page.slice(hit.start, hit.end), hit.quote)+})diff --git a/frontend/src/utils/tableSource.ts b/frontend/src/utils/tableSource.tsindex ac1b4fde2..6b373a548 100644--- a/frontend/src/utils/tableSource.ts+++ b/frontend/src/utils/tableSource.ts@@ -2,7 +2,7 @@ import { findNormalizedMatches, normalizeForMatch, normalizeWithPositions, typeexport type TableSourceMatch = TextMatch & { page: number; quote: string }type Column = { text: string; ends: number[] }-type Row = { text: string; page: number; start: number; norm: string; pos: number[] }+type Row = { text: string; page: number; start: number; norm: string; pos: number[]; endPos: number[] }type Step = { row: number; hits: TableSourceMatch[] }function columns(scope: string): Column[] {@@ -81,8 +81,7 @@ export function findTableSource(pages: string[], scope: string): TableSourceMatcif (size < 2 || size > row.norm.length) continueconst hits = findNormalizedMatches(row.norm, column.text.slice(offset, end))if (hits.length !== 1) continue- const hit = hits[0]!, start = row.pos[hit.start]!, last = row.pos[hit.end - 1]!- const finish = last + (row.text.codePointAt(last)! > 0xffff ? 2 : 1)+ const hit = hits[0]!, start = row.pos[hit.start]!, finish = row.endPos[hit.end - 1]!return { end, hit: { page: row.page, start: row.start + start, end: row.start + finish, quote: row.text.slice(start, finish) } }}return null