Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

perf(deepdoc): bucket substring-dedup scan by page and column (#20122)

xugangqiang483b892bb528
diff --git a/internal/deepdoc/parser/pdf/layout/dedup_substring_opt_test.go b/internal/deepdoc/parser/pdf/layout/dedup_substring_opt_test.go
new file mode 100644
index 0000000000..d7a58fc63d
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/layout/dedup_substring_opt_test.go
@@ -0,0 +1,271 @@
+package layout
+
+import (
+ "math/rand"
+ "reflect"
+ "strings"
+ "testing"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// referenceDedupSubstringOverlaps is a faithful copy of the ORIGINAL O(n²)
+// double-loop implementation of DedupSubstringOverlaps. It is the brute-force
+// oracle for the equivalence tests below: the indexed rewrite must produce a
+// byte-identical kept-set, so any divergence (wrong bucket key, dropped j==i
+// guard, broken break semantics) is caught by TestDedupSubstringOverlapsEquivalence.
+func referenceDedupSubstringOverlaps(boxes []pdf.TextBox) []pdf.TextBox {
+ drop := make([]bool, len(boxes))
+ norm := make([]string, len(boxes))
+ for i, b := range boxes {
+ if b.IsOCR {
+ norm[i] = dedupNormText(strings.TrimSpace(b.Text))
+ }
+ }
+ for i := range boxes {
+ if drop[i] {
+ continue
+ }
+ if !boxes[i].IsOCR {
+ continue
+ }
+ ai := strings.TrimSpace(boxes[i].Text)
+ if ai == "" {
+ continue
+ }
+ for j := range boxes {
+ if i == j || drop[j] || boxes[i].PageNumber != boxes[j].PageNumber || !boxes[j].IsOCR {
+ continue
+ }
+ aj := strings.TrimSpace(boxes[j].Text)
+ if aj == "" || ai == aj {
+ continue
+ }
+ ni, nj := norm[i], norm[j]
+ if boxes[i].ColID != boxes[j].ColID {
+ continue
+ }
+ if len(ni) >= len(nj) && strings.Contains(ni, nj) {
+ if boxInsideTolerant(boxes[j], boxes[i]) {
+ drop[j] = true
+ }
+ } else if len(nj) > len(ni) && strings.Contains(nj, ni) {
+ if boxInsideTolerant(boxes[i], boxes[j]) {
+ drop[i] = true
+ break
+ }
+ }
+ }
+ }
+ out := boxes[:0]
+ for i, b := range boxes {
+ if drop[i] {
+ continue
+ }
+ out = append(out, b)
+ }
+ return out
+}
+
+func mkBox(text string, page, col int, x0, x1, top, bottom float64, isOCR bool) pdf.TextBox {
+ return pdf.TextBox{Text: text, PageNumber: page, ColID: col, X0: x0, X1: x1, Top: top, Bottom: bottom, IsOCR: isOCR}
+}
+
+// substringVocab has deliberate substring relationships so the random stream
+// exercises the containment branches (e.g. "ab" ⊂ "abc", "abc" ⊂ "abcd").
+var substringVocab = []string{
+ "a", "ab", "abc", "abcd", "abcde",
+ "hello", "hello world", "world",
+ "", "x", "xy", "xyz", "wxyz",
+ "RAG分词", "三国人物", "文章 中 提到", "文章中提到",
+}
+
+// randBoxes builds a synthetic document. PageNumber may be 0 (~20%) and ColID
+// may be 0, so the bucket key's zero-value paths are exercised. IsOCR is ~90%
+// true so OCR-vs-char-path and OCR-vs-OCR paths both appear.
+func randBoxes(rng *rand.Rand, pages, perPage, cols int) []pdf.TextBox {
+ boxes := make([]pdf.TextBox, 0, pages*perPage)
+ for p := 0; p < pages; p++ {
+ for k := 0; k < perPage; k++ {
+ text := substringVocab[rng.Intn(len(substringVocab))]
+ isOCR := rng.Intn(10) != 0
+ col := rng.Intn(cols + 1)
+ page := p
+ if rng.Intn(5) == 0 {
+ page = 0
+ }
+ top := float64(rng.Intn(800))
+ h := float64(10 + rng.Intn(30))
+ x0 := float64(rng.Intn(500))
+ x1 := x0 + float64(50+rng.Intn(400))
+ boxes = append(boxes, pdf.TextBox{
+ Text: text,
+ PageNumber: page,
+ ColID: col,
+ Top: top,
+ Bottom: top + h,
+ X0: x0,
+ X1: x1,
+ IsOCR: isOCR,
+ })
+ }
+ }
+ return boxes
+}
+
+// TestDedupSubstringOverlapsEquivalence pins the indexed rewrite to the
+// brute-force oracle across many random documents (varied page counts, boxes per
+// page, columns, PageNumber==0, ColID==0, mixed OCR). Any bucket-key or
+// control-flow divergence shows up as a kept-set mismatch.
+func TestDedupSubstringOverlapsEquivalence(t *testing.T) {
+ for seed := int64(1); seed <= 50; seed++ {
+ rng := rand.New(rand.NewSource(seed))
+ boxes := randBoxes(rng, 10+rng.Intn(15), 25+rng.Intn(35), 5)
+ // DedupSubstringOverlaps reuses the input slice's backing array
+ // (out := boxes[:0]), so each call must get its own copy.
+ inA := append([]pdf.TextBox(nil), boxes...)
+ inB := append([]pdf.TextBox(nil), boxes...)
+ got := DedupSubstringOverlaps(inA)
+ want := referenceDedupSubstringOverlaps(inB)
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("seed %d: indexed result differs from brute-force oracle (got %d boxes, want %d)", seed, len(got), len(want))
+ }
+ }
+ // A realistic large document (100 pages x 100 boxes) stresses the bucket
+ // path the way a real scan doc would.
+ for _, seed := range []int64{7, 23} {
+ rng := rand.New(rand.NewSource(seed))
+ boxes := randBoxes(rng, 100, 100, 5)
+ inA := append([]pdf.TextBox(nil), boxes...)
+ inB := append([]pdf.TextBox(nil), boxes...)
+ got := DedupSubstringOverlaps(inA)
+ want := referenceDedupSubstringOverlaps(inB)
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("large-doc seed %d: indexed result differs from brute-force oracle (got %d boxes, want %d)", seed, len(got), len(want))
+ }
+ }
+}
+
+func TestDedupSubstringOverlaps_EmptyAndSingle(t *testing.T) {
+ if got := DedupSubstringOverlaps(nil); got != nil {
+ t.Fatalf("nil input must return nil, got %d", len(got))
+ }
+ one := []pdf.TextBox{mkBox("hello world", 0, 0, 50, 500, 100, 130, true)}
+ if got := DedupSubstringOverlaps(one); len(got) != 1 {
+ t.Fatalf("single box must be kept, got %d", len(got))
+ }
+}
+
+// TestDedupSubstringOverlaps_CharPathContainerKeepsOCR: a char-path box (IsOCR
+// false) whose text contains an OCR box's text and geometrically contains it
+// must NOT cause the OCR box to be dropped — char-path boxes are never treated
+// as fragments.
+func TestDedupSubstringOverlaps_CharPathContainerKeepsOCR(t *testing.T) {
+ container := mkBox("hello world", 0, 0, 50, 500, 100, 130, false)
+ frag := mkBox("world", 0, 0, 100, 400, 105, 120, true)
+ got := DedupSubstringOverlaps([]pdf.TextBox{container, frag})
+ if len(got) != 2 {
+ t.Fatalf("OCR fragment under char-path container must be kept, got %d: %+v", len(got), got)
+ }
+}
+
+// TestDedupSubstringOverlaps_CrossPageKept: a substring box on a different page
+// is independent document text and must be kept.
+func TestDedupSubstringOverlaps_CrossPageKept(t *testing.T) {
+ a := mkBox("hello world", 5, 0, 50, 500, 100, 130, true)
+ b := mkBox("world", 7, 0, 100, 400, 105, 120, true)
+ got := DedupSubstringOverlaps([]pdf.TextBox{a, b})
+ if len(got) != 2 {
+ t.Fatalf("cross-page substring must be kept, got %d", len(got))
+ }
+}
+
+// TestDedupSubstringOverlaps_CrossColumnKept: a substring box in a DIFFERENT
+// column, even when geometrically inside the container, must be kept by the
+// ColID guard.
+func TestDedupSubstringOverlaps_CrossColumnGeomInsideKept(t *testing.T) {
+ a := mkBox("hello world", 5, 0, 50, 500, 100, 130, true)
+ b := mkBox("world", 5, 1, 100, 400, 105, 120, true) // inside A geometry, different column
+ got := DedupSubstringOverlaps([]pdf.TextBox{a, b})
+ if len(got) != 2 {
+ t.Fatalf("cross-column substring (geom inside) must be kept by ColID guard, got %d: %+v", len(got), got)
+ }
+}
+
+// TestDedupSubstringOverlaps_PageZeroSemantics: PageNumber 0 boxes compare with
+// other PageNumber 0 boxes (0==0) but not with PageNumber>0 boxes. So an
+// in-geometry fragment on page 0 is dropped, while an identical in-geometry
+// fragment on page 5 is kept.
+func TestDedupSubstringOverlaps_PageZeroSemantics(t *testing.T) {
+ c := mkBox("hello world", 0, 0, 50, 500, 100, 130, true)
+ f := mkBox("world", 0, 0, 100, 400, 105, 120, true) // page 0 -> dropped
+ f2 := mkBox("world", 5, 0, 100, 400, 105, 120, true) // page 5 -> kept
+ got := DedupSubstringOverlaps([]pdf.TextBox{c, f, f2})
+ if len(got) != 2 {
+ t.Fatalf("want 2 kept (C and F2), got %d: %+v", len(got), got)
+ }
+ for _, b := range got {
+ if b.PageNumber == 0 && b.Text == "world" {
+ t.Fatalf("page-0 fragment should have been dropped")
+ }
+ }
+}
+
+// TestDedupSubstringOverlaps_ColIDZeroChain: with the default ColID 0, a
+// container and two nested fragments on the same page collapse to the container.
+func TestDedupSubstringOverlaps_ColIDZeroChain(t *testing.T) {
+ c := mkBox("abcde", 3, 0, 50, 500, 100, 130, true)
+ f1 := mkBox("abc", 3, 0, 100, 400, 105, 120, true)
+ f2 := mkBox("cde", 3, 0, 100, 400, 105, 120, true)
+ got := DedupSubstringOverlaps([]pdf.TextBox{c, f1, f2})
+ if len(got) != 1 || got[0].Text != "abcde" {
+ t.Fatalf("want only the container 'abcde', got %d: %+v", len(got), got)
+ }
+}
+
+// TestDedupSubstringOverlaps_TallerFragmentKept: a box whose text is a
+// substring but which is geometrically NOT contained (taller, poking out on
+// both axes beyond the 3pt tolerance) must be kept.
+func TestDedupSubstringOverlaps_TallerFragmentKeptOpt(t *testing.T) {
+ c := mkBox("hello world", 0, 0, 50, 500, 100, 130, true) // height 30
+ f := mkBox("world", 0, 0, 100, 400, 95, 135, true) // top/bottom exceed tolerance
+ got := DedupSubstringOverlaps([]pdf.TextBox{c, f})
+ if len(got) != 2 {
+ t.Fatalf("geometrically-disjoint substring must be kept, got %d", len(got))
+ }
+}
+
+// TestDedupSubstringOverlaps_FragmentDropped: the happy-path case — an OCR
+// fragment contained in its container is dropped.
+func TestDedupSubstringOverlaps_FragmentDropped(t *testing.T) {
+ c := mkBox("hello world", 0, 0, 50, 500, 100, 130, true)
+ f := mkBox("world", 0, 0, 100, 400, 105, 120, true)
+ got := DedupSubstringOverlaps([]pdf.TextBox{c, f})
+ if len(got) != 1 || got[0].Text != "hello world" {
+ t.Fatalf("fragment must be dropped, got %d: %+v", len(got), got)
+ }
+}
+
+func buildSyntheticBoxes(pages, perPage int) []pdf.TextBox {
+ rng := rand.New(rand.NewSource(99))
+ return randBoxes(rng, pages, perPage, 4)
+}
+
+// Both benchmarks run on the SAME synthetic document (fixed seed) so the per-op
+// numbers are directly comparable: the indexed path is O(Σ bucket²) while the
+// brute oracle is O(n²) over the whole slice.
+func BenchmarkDedupSubstringOverlapsIndexed(b *testing.B) {
+ boxes := buildSyntheticBoxes(120, 100) // 12000 boxes
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ _ = DedupSubstringOverlaps(boxes)
+ }
+}
+
+func BenchmarkDedupSubstringOverlapsBrute(b *testing.B) {
+ boxes := buildSyntheticBoxes(120, 100) // same input as Indexed
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ _ = referenceDedupSubstringOverlaps(boxes)
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/layout/layout.go b/internal/deepdoc/parser/pdf/layout/layout.go
index 1916f52350..9708d228bc 100644
--- a/internal/deepdoc/parser/pdf/layout/layout.go
+++ b/internal/deepdoc/parser/pdf/layout/layout.go
@@ -145,17 +145,38 @@ func DedupIdenticalText(boxes []pdf.TextBox) []pdf.TextBox {
// RAG分词, 三国人物). Geometry (boxInsideTolerant) remains the actual
// containment proof; whitespace normalization only makes the fragment check
// robust to the char-vs-OCR space divergence.
+// dedupBucket groups box indices by (page, column) so the substring-overlap scan
+// only walks boxes that can actually be OCR double-detection fragments of one
+// another. The original O(n²) loop skipped every pair whose PageNumber or ColID
+// differed, so restricting each i to its own bucket is exactly equivalent while
+// cutting the inner-loop count from n² to Σ_bucket size²: OCR fragments always
+// share their container's page and column (see the ColID guard below), so no
+// valid comparison pair is lost.
+func dedupBucket(boxes []pdf.TextBox) map[[2]int][]int {
+ byPageCol := make(map[[2]int][]int, len(boxes))
+ for i, b := range boxes {
+ key := [2]int{b.PageNumber, b.ColID}
+ byPageCol[key] = append(byPageCol[key], i)
+ }
+ return byPageCol
+}
+
func DedupSubstringOverlaps(boxes []pdf.TextBox) []pdf.TextBox {
drop := make([]bool, len(boxes))
- // Precompute the whitespace-normalized text once per box. The inner loop
- // compares every OCR pair, so recomputing norm there would make the O(n²)
- // loop O(n²·len) in the worst case (a large scan document hits ~3k boxes).
+ // Precompute the whitespace-normalized text once per box. The pair scan
+ // still touches many OCR boxes, so recomputing norm per comparison would
+ // make the worst case O(bucketSize²·len).
norm := make([]string, len(boxes))
for i, b := range boxes {
if b.IsOCR {
norm[i] = dedupNormText(strings.TrimSpace(b.Text))
}
}
+ // Index boxes by (page, column) so each i only compares against same-page,
+ // same-column boxes instead of the whole slice — equivalent to the original
+ // per-pair PageNumber/ColID guards, but O(n) to build and Σ_bucket size² to
+ // scan.
+ byPageCol := dedupBucket(boxes)
for i := range boxes {
if drop[i] {
continue
@@ -169,15 +190,19 @@ func DedupSubstringOverlaps(boxes []pdf.TextBox) []pdf.TextBox {
if ai == "" {
continue
}
- for j := range boxes {
- if i == j || drop[j] || boxes[i].PageNumber != boxes[j].PageNumber || !boxes[j].IsOCR {
+ ni := norm[i]
+ // Candidates are the boxes sharing i's (page, column). The bucket keeps
+ // the original slice order, so the early `break` (i is a substring of a
+ // larger box) fires at the same point as in the unindexed loop.
+ for _, j := range byPageCol[[2]int{boxes[i].PageNumber, boxes[i].ColID}] {
+ if i == j || drop[j] || !boxes[j].IsOCR {
continue
}
aj := strings.TrimSpace(boxes[j].Text)
if aj == "" || ai == aj {
continue
}
- ni, nj := norm[i], norm[j]
+ nj := norm[j]
// Never collapse a substring across columns. OCR double-detection
// fragments always share the SAME column as their container; a
// substring in a DIFFERENT column (e.g. the opposite column of a