Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

6.8 KB
Raw Blame
import test from 'node:test'
import assert from 'node:assert/strict'

import {
  findInText,
  findNormalized,
  locatorPages,
  normalizeForMatch,
  parseSourceLocators,
  selectLocatorsForSentence,
  selectQuoteForSentence,
  textFragmentUrl,
} from './sourceLocator.ts'
import { anchorWithContext, lastSentence } from './citationAnchor.ts'
import { resolveReferenceSource } from './referenceSources.ts'
import { resolveEpubPath } from './epubPreview.ts'
import { resolvePreviewKind } from './filePreview.ts'

test('normalization preserves numeric symbols while folding width and case', () => {
  assert.equal(normalizeForMatch('8.14 索夹,Hello!'), '8.14索夹hello')
})

test('findInText maps a normalized match back to the raw text', () => {
  const text = '8.14.11 索夹与吊索施工应符合下列规定:\n1 在满足施工需要的前提下,应减小猫道面层开孔面积。'
  const hit = findInText(text, '在满足施工需要的前提下,应减小猫道面层开孔面积')
  assert.ok(hit)
  assert.equal(text.slice(hit.start, hit.end), '在满足施工需要的前提下,应减小猫道面层开孔面积')
})

test('findNormalized tolerates only an omitted leading list label', () => {
  const hay = normalizeForMatch('前文。紧固同一索夹的螺栓时,各螺栓受力应均匀,并应按规定顺序施拧。后文')
  const m = findNormalized(hay, '(二)紧固同一索夹的螺栓时,各螺栓受力应均匀,并应按规定顺序施拧')
  assert.ok(m)
  assert.ok(m.score >= 0.5)
  assert.equal(findNormalized(hay, '完全无关的一段内容不应该被匹配到这里来吧'), null)
})



test('selectLocatorsForSentence narrows to the locator that supports the sentence', () => {
  const locs = [
    { type: 'pdf', page: 57, quote: '8.14.11 索夹与吊索施工应符合下列规定' },
    { type: 'pdf', page: 57, quote: '1 在满足施工需要的前提下,应减小猫道面层开孔面积,并应在开孔位置四周绑扎防滑木条,设立警示标志。' },
    { type: 'pdf', page: 57, quote: '2 索夹在主缆上定位后,应紧固螺栓。' },
  ]
  const picked = selectLocatorsForSentence(locs, '在满足施工需要的前提下,应减小猫道面层开孔面积,并应在开孔位置四周绑扎防滑木条,设立警示标志。')
  assert.equal(picked.length, 1)
  assert.equal(picked[0], locs[1])
  assert.equal(selectLocatorsForSentence(locs, '毫不相干的一句话').length, 3)
})

test('selectQuoteForSentence prefers the supporting sentence of the chunk', () => {
  const content = '索夹在主缆上定位后,应紧固螺栓。吊运物体时,作业人员不得沿主缆顶面行走。'
  const [first] = selectQuoteForSentence(content, '吊运物体时作业人员不得沿主缆顶面行走')
  assert.equal(first, '吊运物体时,作业人员不得沿主缆顶面行走')
  assert.equal(selectQuoteForSentence(content, '')[0], content)
})

test('parseSourceLocators and locatorPages', () => {
  const locs = parseSourceLocators([{ type: 'pdf', page: 3 }, null, { page: 2 }, { type: 'pdf', page: 3 }, { type: 'pdf', page: 1 }])
  assert.equal(locs.length, 3)
  assert.deepEqual(locatorPages(locs), [3, 1])
  assert.deepEqual(parseSourceLocators('nope'), [])
})

test('textFragmentUrl builds a scroll-to-text link', () => {
  assert.equal(
    textFragmentUrl('https://example.com/a#old', 'safety rules - part 1'),
    'https://example.com/a#:~:text=safety%20rules%20%2D%20part%201',
  )
  const long = textFragmentUrl('https://example.com/a', 'x'.repeat(50) + ' ' + 'y'.repeat(50))
  assert.match(long, /#:~:text=x+,.*y+$/)
})





test('lastSentence keeps the cited sentence and extends short ones', () => {
  assert.equal(lastSentence('第一句很长很长的内容。第二句也是完整的句子。'), '第二句也是完整的句子。')
  assert.equal(lastSentence('前一句足够长的内容。短句。'), '前一句足够长的内容。短句。')
})

test('anchorWithContext borrows the previous block for a marker on a label line', () => {
  const para = '在索夹吊装作业开始前,必须对主缆表面进行清理。'
  assert.equal(anchorWithContext('依据:', [para, '2. 清理主缆表面']), `${para}依据:`)
  assert.equal(anchorWithContext('索夹就位后应立即进行初拧。', [para]), '索夹就位后应立即进行初拧。')
  assert.equal(anchorWithContext('依据:', []), '依据:')
})

test('resolveReferenceSource picks the cited chunk of a document reference', () => {
  const refs = [
    { id: 'c1', knowledge_id: 'k1', knowledge_filename: 'a.pdf', knowledge_title: 'A', content: 'x', source_locators: [{ type: 'pdf', page: 2 }] },
    { id: 'c2', chunk_ids: ['c2', 'c3'], knowledge_id: 'k2', knowledge_title: 'B' },
    { id: 'w', chunk_type: 'web_search', knowledge_id: 'k3' },
    { id: 'f', chunk_type: 'faq', knowledge_id: 'k4' },
  ]
  const exact = resolveReferenceSource(refs, { chunkId: 'c1', anchorText: 's' })
  assert.equal(exact?.knowledgeId, 'k1')
  assert.equal(exact?.locators?.[0].page, 2)
  assert.equal(exact?.anchorText, 's')
  const grouped = resolveReferenceSource(refs, { chunkId: 'c3' })
  assert.equal(grouped?.knowledgeId, 'k2')
  assert.equal(grouped?.chunkId, 'c3')
  assert.equal(grouped?.locators, undefined, 'a grouped match belongs to another chunk')
  assert.equal(resolveReferenceSource(refs, { chunkId: 'w' }), null)
  assert.equal(resolveReferenceSource(refs, { chunkId: 'f' }), null)
  assert.equal(resolveReferenceSource(refs, { url: 'https://x' }), null)
})

test('EPUB paths resolve against their document and preview as epub', () => {
  assert.equal(resolveEpubPath('OEBPS/text/ch1.xhtml', '../images/a%20b.png#x'), 'OEBPS/images/a b.png')
  assert.equal(resolveEpubPath('OEBPS/content.opf', 'ch1.xhtml'), 'OEBPS/ch1.xhtml')
  assert.equal(resolvePreviewKind('epub'), 'epub')
})


test('citation matching treats composed and decomposed accents as the same text', () => {
  assert.equal(normalizeForMatch('Café'), normalizeForMatch('Cafe\u0301'))
  assert.notEqual(normalizeForMatch('Café'), normalizeForMatch('Cafe'))
  assert.equal(normalizeForMatch('Ångström'), normalizeForMatch('A\u030Angstro\u0308m'))
  assert.equal(normalizeForMatch('각도'), normalizeForMatch('\u1100\u1161\u11A8도'))
})

test('canonical matching returns exact original UTF-16 spans including combining marks', () => {
  for (const [text, quote, expected] of [
    ['前言 Cafe\u0301 后记', 'Café', 'Cafe\u0301'],
    ['前言 Café 后记', 'Cafe\u0301', 'Café'],
    ['𝒜前言 cafe\u0301 后记', 'CAFÉ', 'cafe\u0301'],
    ['前言 A\u030Angstro\u0308m 后记', 'Ångström', 'A\u030Angstro\u0308m'],
    ['前言 \u1100\u1161\u11A8도 后记', '각도', '\u1100\u1161\u11A8도'],
  ]) {
    const hit = findInText(text, quote)
    assert.ok(hit, text)
    assert.equal(text.slice(hit.start, hit.end), expected)
  }
})