Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
perf(chunker): avoid quadratic protected span sorting (#4633)
Captainb040b97d4708
diff --git a/internal/infrastructure/chunker/protected_spans_order_test.go b/internal/infrastructure/chunker/protected_spans_order_test.gonew file mode 100644index 000000000..f7dae2f04--- /dev/null+++ b/internal/infrastructure/chunker/protected_spans_order_test.go@@ -0,0 +1,94 @@+package chunker++import (+ "fmt"+ "reflect"+ "strings"+ "testing"+)++func TestProtectedSpansMixedPatternsInDocumentOrder(t *testing.T) {+ parts := []string{+ "`示例 [link](url)`",+ "",+ "$$x + y$$",+ "| name | value |\n| --- | --- |\n",+ "| cell | `code` |\n",+ "```go\n[inside](code)\n```",+ "[last](https://example.com)",+ }+ var docBuilder strings.Builder+ for _, part := range parts {+ docBuilder.WriteString(part)+ if !strings.HasSuffix(part, "\n") {+ docBuilder.WriteByte('\n')+ }+ }+ doc := docBuilder.String()+ var got []string+ lastEnd := 0+ for _, s := range protectedSpans(doc) {+ if s.start < lastEnd {+ t.Fatalf("overlapping or unordered span: %v after %d", s, lastEnd)+ }+ got = append(got, doc[s.start:s.end])+ lastEnd = s.end+ }+ if !reflect.DeepEqual(got, parts) {+ t.Fatalf("protected content = %q, want %q", got, parts)+ }+ // At the public entry point, overlapping inner matches must not split+ // an inline-code span, table header, image or fenced block apart.+ chunks := SplitText(doc, SplitterConfig{+ ChunkSize: 64, ChunkOverlap: 0, Separators: []string{"\n", " "},+ })+ for _, part := range parts {+ found := false+ for _, chunk := range chunks {+ if strings.Contains(chunk.Content, part) {+ found = true+ break+ }+ }+ if !found {+ t.Errorf("protected content split across chunks: %q", part)+ }+ }+}++func benchmarkMixedProtectedDocument(rows int) string {+ return strings.Repeat("[link](https://example.com) and `inline code`\n", rows)+}++func BenchmarkProtectedSpansMixedPatterns(b *testing.B) {+ for _, rows := range []int{1000, 4000, 16000} {+ b.Run(fmt.Sprintf("rows_%d", rows), func(b *testing.B) {+ doc := benchmarkMixedProtectedDocument(rows)+ b.SetBytes(int64(len(doc)))+ b.ReportAllocs()+ b.ResetTimer()+ for b.Loop() {+ if len(protectedSpans(doc)) != 2*rows {+ b.Fatal("missing protected spans")+ }+ }+ })+ }+}++func BenchmarkSplitTextMixedPatterns(b *testing.B) {+ for _, rows := range []int{1000, 4000, 16000} {+ b.Run(fmt.Sprintf("rows_%d", rows), func(b *testing.B) {+ doc := benchmarkMixedProtectedDocument(rows)+ cfg := DefaultConfig()+ b.SetBytes(int64(len(doc)))+ b.ReportAllocs()+ b.ResetTimer()+ for b.Loop() {+ if len(SplitText(doc, cfg)) == 0 {+ b.Fatal("missing chunks")+ }+ }+ })+ }+}diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.goindex 23b6596c7..99a86aa14 100644--- a/internal/infrastructure/chunker/splitter.go+++ b/internal/infrastructure/chunker/splitter.go@@ -4,7 +4,9 @@package chunkerimport (+ "cmp""regexp"+ "slices""strings""unicode/utf8"@@ -183,17 +185,14 @@ func protectedSpans(text string) []span {return nil}- // Sort by start, then by length descending- for i := 1; i < len(all); i++ {- for j := i; j > 0; j-- {- if all[j].start < all[j-1].start ||- (all[j].start == all[j-1].start && (all[j].end-all[j].start) > (all[j-1].end-all[j-1].start)) {- all[j], all[j-1] = all[j-1], all[j]- } else {- break- }+ // Matches are grouped by pattern, not document position. Avoid quadratic+ // insertion sorting when a long document interleaves different patterns.+ slices.SortFunc(all, func(a, b match) int {+ if a.start != b.start {+ return cmp.Compare(a.start, b.start)}- }+ return cmp.Compare(b.end, a.end)+ })// Remove overlapsvar result []span