Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
perf(sourceloc): index align matches in one document scan (#4544)
Sun Shihao0eb298467d07
diff --git a/internal/sourceloc/align.go b/internal/sourceloc/align.goindex f8e09f8c9..510a138c1 100644--- a/internal/sourceloc/align.go+++ b/internal/sourceloc/align.go@@ -29,22 +29,19 @@ func Align(markdown string, units []Unit) []types.SourceBlock {var blocks []types.SourceBlockcursor, floor := 0, 0counts := make(map[string]int)- matches := make(map[string][]int)for _, u := range units {counts[normalizeKey(u.Text)]++}+ matches := indexNormalizedMatches(norm.text, counts)for _, u := range units {key := normalizeKey(u.Text)if utf8.RuneCountInString(key) < alignMinRunes {continue}- // Repeated source text is safe only when every occurrence is accounted- // for by a source unit in reading order (not a TOC or an omitted block).- positions, known := matches[key]- if !known {- positions = normalizedMatches(norm.text, key)- matches[key] = positions- }+ // A unit may only be placed when every occurrence of its text in the+ // document is accounted for by a source unit (not a TOC or an omitted+ // block). The occurrence index is built in one pass over the document.+ positions := matches[key]if len(positions) != counts[key] {continue}@@ -79,31 +76,61 @@ func Align(markdown string, units []Unit) []types.SourceBlock {return blocks}-// normalizedMatches rejects a number embedded in a different numeric value.-func normalizedMatches(text, key string) []int {- var out []int- if key == "" {- return out+// alignIndexPrefixBytes bounds the prefix used to bucket patterns. Long unit+// keys in tables and office documents usually diverge in their first few+// cells, so a fixed prefix gives an O(1) candidate lookup per document+// position while keeping the full key comparison as the correctness check.+const alignIndexPrefixBytes = 32++// indexNormalizedMatches finds every occurrence of the requested normalized+// keys in one pass. Align used to call normalizedMatches once per distinct+// unit, so a table with one unit per row scanned the whole document once per+// row. Long keys are bucketed by a fixed byte prefix; short keys are grouped+// by their first rune and checked directly.+func indexNormalizedMatches(text string, counts map[string]int) map[string][]int {+ long := make(map[string][]string)+ short := make(map[rune][]string)+ for key := range counts {+ if utf8.RuneCountInString(key) < alignMinRunes {+ continue+ }+ if len(key) >= alignIndexPrefixBytes {+ long[key[:alignIndexPrefixBytes]] = append(long[key[:alignIndexPrefixBytes]], key)+ continue+ }+ first, _ := utf8.DecodeRuneInString(key)+ short[first] = append(short[first], key)}- first, _ := utf8.DecodeRuneInString(key)- last, _ := utf8.DecodeLastRuneInString(key)- for from := 0; from <= len(text)-len(key); {- i := strings.Index(text[from:], key)- if i < 0 {- break++ matches := make(map[string][]int)+ for at := 0; at < len(text); {+ first, size := utf8.DecodeRuneInString(text[at:])+ for _, key := range short[first] {+ if strings.HasPrefix(text[at:], key) && normalizedBoundaryOK(text, at, key) {+ matches[key] = append(matches[key], at)+ }}- at := from + i- before, _ := utf8.DecodeLastRuneInString(text[:at])- after, _ := utf8.DecodeRuneInString(text[at+len(key):])- left := unicode.IsDigit(first) && (unicode.IsDigit(before) || strings.ContainsRune(".,:/+−-", before))- right := unicode.IsDigit(last) && (unicode.IsDigit(after) || strings.ContainsRune(".,:/%‰", after))- if !left && !right {- out = append(out, at)+ if end := at + alignIndexPrefixBytes; end <= len(text) {+ for _, key := range long[text[at:end]] {+ if strings.HasPrefix(text[at:], key) && normalizedBoundaryOK(text, at, key) {+ matches[key] = append(matches[key], at)+ }+ }}- _, size := utf8.DecodeRuneInString(text[at:])- from = at + size+ at += size}- return out+ return matches+}++// normalizedBoundaryOK rejects a key embedded in a different numeric value.+func normalizedBoundaryOK(text string, at int, key string) bool {+ first, _ := utf8.DecodeRuneInString(key)+ last, _ := utf8.DecodeLastRuneInString(key)+ before, _ := utf8.DecodeLastRuneInString(text[:at])+ after, _ := utf8.DecodeRuneInString(text[at+len(key):])+ left := unicode.IsDigit(first) && (unicode.IsDigit(before) || strings.ContainsRune(".,:/+−-", before))+ right := unicode.IsDigit(last) && (unicode.IsDigit(after) || strings.ContainsRune(".,:/%‰", after))+ return !left && !right}// lineLeadStart moves pos back to the start of its line when everythingdiff --git a/internal/sourceloc/alignment_index_test.go b/internal/sourceloc/alignment_index_test.gonew file mode 100644index 000000000..7c97d38de--- /dev/null+++ b/internal/sourceloc/alignment_index_test.go@@ -0,0 +1,52 @@+package sourceloc++import (+ "fmt"+ "strings"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+)++// TestAlignIndexesUniqueRowsInOnePass locks the large-table path: every row is+// a distinct unit, and all of them must still align despite the former+// per-key full-document scan. The linearity itself is covered by the benchmark.+func TestAlignIndexesUniqueRowsInOnePass(t *testing.T) {+ const rows = 2000+ var md strings.Builder+ md.WriteString("| id | value |\n| --- | --- |\n")+ units := make([]Unit, 0, rows)+ for i := 0; i < rows; i++ {+ text := fmt.Sprintf("row-%04d value-%04d", i, i)+ fmt.Fprintf(&md, "| row-%04d | value-%04d |\n", i, i)+ units = append(units, Unit{Text: text, Locator: types.SourceLocator{+ Type: "sheet", Sheet: "CSV", RowStart: i + 2, RowEnd: i + 2,+ }})+ }+ blocks := Align(md.String(), units)+ if len(blocks) != len(units) {+ t.Fatalf("aligned %d of %d unique rows", len(blocks), len(units))+ }+ for i, block := range blocks {+ if block.Locator.RowStart != units[i].Locator.RowStart {+ t.Fatalf("row %d mapped to source row %d", i, block.Locator.RowStart)+ }+ }+}++// Repeated keys must still be rejected unless the document and the source units+// account for every occurrence. Prefix bucketing must not turn a TOC duplicate+// into a false exact match.+func TestAlignPrefixIndexKeepsDuplicateSafety(t *testing.T) {+ units := []Unit{+ {Text: "设备验收合格", Locator: types.SourceLocator{Type: "pdf", Page: 1}},+ }+ if blocks := Align("目录:设备验收合格。\n正文:设备验收合格。", units); len(blocks) != 0 {+ t.Fatalf("unaccounted duplicate aligned: %#v", blocks)+ }+ units = append(units, Unit{Text: "设备验收合格", Locator: types.SourceLocator{Type: "pdf", Page: 2}})+ blocks := Align("设备验收合格\n设备验收合格", units)+ if len(blocks) != 2 || blocks[1].Locator.Page != 2 {+ t.Fatalf("accounted duplicates = %#v", blocks)+ }+}