Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

perf(sourceloc): index align matches in one document scan (#4544)

Sun Shihao0eb298467d07
diff --git a/internal/sourceloc/align.go b/internal/sourceloc/align.go
index f8e09f8c9..510a138c1 100644
--- a/internal/sourceloc/align.go
+++ b/internal/sourceloc/align.go
@@ -29,22 +29,19 @@ func Align(markdown string, units []Unit) []types.SourceBlock {
var blocks []types.SourceBlock
cursor, floor := 0, 0
counts := make(map[string]int)
- matches := make(map[string][]int)
for _, u := range units {
counts[normalizeKey(u.Text)]++
}
+ matches := indexNormalizedMatches(norm.text, counts)
for _, u := range units {
key := normalizeKey(u.Text)
if utf8.RuneCountInString(key) < alignMinRunes {
continue
}
- // Repeated source text is safe only when every occurrence is accounted
- // for by a source unit in reading order (not a TOC or an omitted block).
- positions, known := matches[key]
- if !known {
- positions = normalizedMatches(norm.text, key)
- matches[key] = positions
- }
+ // A unit may only be placed when every occurrence of its text in the
+ // document is accounted for by a source unit (not a TOC or an omitted
+ // block). The occurrence index is built in one pass over the document.
+ positions := matches[key]
if len(positions) != counts[key] {
continue
}
@@ -79,31 +76,61 @@ func Align(markdown string, units []Unit) []types.SourceBlock {
return blocks
}
-// normalizedMatches rejects a number embedded in a different numeric value.
-func normalizedMatches(text, key string) []int {
- var out []int
- if key == "" {
- return out
+// alignIndexPrefixBytes bounds the prefix used to bucket patterns. Long unit
+// keys in tables and office documents usually diverge in their first few
+// cells, so a fixed prefix gives an O(1) candidate lookup per document
+// position while keeping the full key comparison as the correctness check.
+const alignIndexPrefixBytes = 32
+
+// indexNormalizedMatches finds every occurrence of the requested normalized
+// keys in one pass. Align used to call normalizedMatches once per distinct
+// unit, so a table with one unit per row scanned the whole document once per
+// row. Long keys are bucketed by a fixed byte prefix; short keys are grouped
+// by their first rune and checked directly.
+func indexNormalizedMatches(text string, counts map[string]int) map[string][]int {
+ long := make(map[string][]string)
+ short := make(map[rune][]string)
+ for key := range counts {
+ if utf8.RuneCountInString(key) < alignMinRunes {
+ continue
+ }
+ if len(key) >= alignIndexPrefixBytes {
+ long[key[:alignIndexPrefixBytes]] = append(long[key[:alignIndexPrefixBytes]], key)
+ continue
+ }
+ first, _ := utf8.DecodeRuneInString(key)
+ short[first] = append(short[first], key)
}
- first, _ := utf8.DecodeRuneInString(key)
- last, _ := utf8.DecodeLastRuneInString(key)
- for from := 0; from <= len(text)-len(key); {
- i := strings.Index(text[from:], key)
- if i < 0 {
- break
+
+ matches := make(map[string][]int)
+ for at := 0; at < len(text); {
+ first, size := utf8.DecodeRuneInString(text[at:])
+ for _, key := range short[first] {
+ if strings.HasPrefix(text[at:], key) && normalizedBoundaryOK(text, at, key) {
+ matches[key] = append(matches[key], at)
+ }
}
- at := from + i
- before, _ := utf8.DecodeLastRuneInString(text[:at])
- after, _ := utf8.DecodeRuneInString(text[at+len(key):])
- left := unicode.IsDigit(first) && (unicode.IsDigit(before) || strings.ContainsRune(".,:/+−-", before))
- right := unicode.IsDigit(last) && (unicode.IsDigit(after) || strings.ContainsRune(".,:/%‰", after))
- if !left && !right {
- out = append(out, at)
+ if end := at + alignIndexPrefixBytes; end <= len(text) {
+ for _, key := range long[text[at:end]] {
+ if strings.HasPrefix(text[at:], key) && normalizedBoundaryOK(text, at, key) {
+ matches[key] = append(matches[key], at)
+ }
+ }
}
- _, size := utf8.DecodeRuneInString(text[at:])
- from = at + size
+ at += size
}
- return out
+ return matches
+}
+
+// normalizedBoundaryOK rejects a key embedded in a different numeric value.
+func normalizedBoundaryOK(text string, at int, key string) bool {
+ first, _ := utf8.DecodeRuneInString(key)
+ last, _ := utf8.DecodeLastRuneInString(key)
+ before, _ := utf8.DecodeLastRuneInString(text[:at])
+ after, _ := utf8.DecodeRuneInString(text[at+len(key):])
+ left := unicode.IsDigit(first) && (unicode.IsDigit(before) || strings.ContainsRune(".,:/+−-", before))
+ right := unicode.IsDigit(last) && (unicode.IsDigit(after) || strings.ContainsRune(".,:/%‰", after))
+ return !left && !right
}
// lineLeadStart moves pos back to the start of its line when everything
diff --git a/internal/sourceloc/alignment_index_test.go b/internal/sourceloc/alignment_index_test.go
new file mode 100644
index 000000000..7c97d38de
--- /dev/null
+++ b/internal/sourceloc/alignment_index_test.go
@@ -0,0 +1,52 @@
+package sourceloc
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// TestAlignIndexesUniqueRowsInOnePass locks the large-table path: every row is
+// a distinct unit, and all of them must still align despite the former
+// per-key full-document scan. The linearity itself is covered by the benchmark.
+func TestAlignIndexesUniqueRowsInOnePass(t *testing.T) {
+ const rows = 2000
+ var md strings.Builder
+ md.WriteString("| id | value |\n| --- | --- |\n")
+ units := make([]Unit, 0, rows)
+ for i := 0; i < rows; i++ {
+ text := fmt.Sprintf("row-%04d value-%04d", i, i)
+ fmt.Fprintf(&md, "| row-%04d | value-%04d |\n", i, i)
+ units = append(units, Unit{Text: text, Locator: types.SourceLocator{
+ Type: "sheet", Sheet: "CSV", RowStart: i + 2, RowEnd: i + 2,
+ }})
+ }
+ blocks := Align(md.String(), units)
+ if len(blocks) != len(units) {
+ t.Fatalf("aligned %d of %d unique rows", len(blocks), len(units))
+ }
+ for i, block := range blocks {
+ if block.Locator.RowStart != units[i].Locator.RowStart {
+ t.Fatalf("row %d mapped to source row %d", i, block.Locator.RowStart)
+ }
+ }
+}
+
+// Repeated keys must still be rejected unless the document and the source units
+// account for every occurrence. Prefix bucketing must not turn a TOC duplicate
+// into a false exact match.
+func TestAlignPrefixIndexKeepsDuplicateSafety(t *testing.T) {
+ units := []Unit{
+ {Text: "设备验收合格", Locator: types.SourceLocator{Type: "pdf", Page: 1}},
+ }
+ if blocks := Align("目录:设备验收合格。\n正文:设备验收合格。", units); len(blocks) != 0 {
+ t.Fatalf("unaccounted duplicate aligned: %#v", blocks)
+ }
+ units = append(units, Unit{Text: "设备验收合格", Locator: types.SourceLocator{Type: "pdf", Page: 2}})
+ blocks := Align("设备验收合格\n设备验收合格", units)
+ if len(blocks) != 2 || blocks[1].Locator.Page != 2 {
+ t.Fatalf("accounted duplicates = %#v", blocks)
+ }
+}