Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(deepdoc): dedup OCR substring fragments regardless of whitespace divergence (#18568)

Jackf793a44f9981
diff --git a/internal/deepdoc/parser/pdf/layout/dedup_test.go b/internal/deepdoc/parser/pdf/layout/dedup_test.go
index 67adfdf820..752dea94e6 100644
--- a/internal/deepdoc/parser/pdf/layout/dedup_test.go
+++ b/internal/deepdoc/parser/pdf/layout/dedup_test.go
@@ -242,6 +242,30 @@ func TestDedupSubstringOverlaps_PartialYOverlapKept(t *testing.T) {
}
}
+// TestDedupSubstringOverlaps_HorizontalOverhangKept locks the horizontal-
+// containment boundary: a box FULLY inside in Y but extending horizontally
+// beyond the containing box (to either side) must be KEPT — it is not an OCR
+// fragment of the container (e.g. an adjacent-column line whose text happens
+// to be a whitespace-normalized substring of the paragraph). Only a box fully
+// inside on BOTH axes is collapsed. This pins the boxInside hardening that the
+// whitespace-insensitive match otherwise leaves exposed (a plain horizontal
+// intersect used to pass the X check).
+func TestDedupSubstringOverlaps_HorizontalOverhangKept(t *testing.T) {
+ outer := pdf.TextBox{
+ Text: "the quick brown fox jumps over the lazy dog near the river bank", PageNumber: 0, Top: 100, Bottom: 130, X0: 60, X1: 260, IsOCR: true,
+ }
+ right := pdf.TextBox{
+ Text: "brown fox jumps over the lazy", PageNumber: 0, Top: 105, Bottom: 120, X0: 240, X1: 320, IsOCR: true, // X1 extends past outer.X1
+ }
+ left := pdf.TextBox{
+ Text: "lazy dog near the river", PageNumber: 0, Top: 105, Bottom: 120, X0: 40, X1: 120, IsOCR: true, // X0 extends past outer.X0
+ }
+ got := DedupSubstringOverlaps([]pdf.TextBox{outer, right, left})
+ if len(got) != 3 {
+ t.Fatalf("horizontally-overhanging substrings must be kept, got %d boxes", len(got))
+ }
+}
+
// TestDedupSubstringOverlaps_TallerFragmentKept locks the defensive invariant:
// a PHYSICALLY TALLER box whose SHORT text is a substring of a shorter, contained
// box's longer text is NOT silently dropped. Collapse requires the substring-text
@@ -261,6 +285,30 @@ func TestDedupSubstringOverlaps_TallerFragmentKept(t *testing.T) {
}
}
+// TestDedupSubstringOverlaps_WhitespaceInsensitive_EqualAfterNorm locks the
+// `>=` branch: two boxes whose text differs ONLY by whitespace placement
+// ("-name:" vs "- name:") normalize to the SAME text and must collapse when
+// geometrically contained. The legacy `len(ai) == len(aj)` skip would have
+// kept the duplicate; whitespace normalization makes the two look identical,
+// and boxInside decides the containment.
+func TestDedupSubstringOverlaps_WhitespaceInsensitive_EqualAfterNorm(t *testing.T) {
+ outer := pdf.TextBox{
+ Text: "-name:", // OCR recognizer stripped the space after '-'
+ PageNumber: 0, Top: 100, Bottom: 120, X0: 60, X1: 120, IsOCR: true,
+ }
+ inner := pdf.TextBox{
+ Text: "- name:", // char-layer text kept the space
+ PageNumber: 0, Top: 105, Bottom: 115, X0: 60, X1: 120, IsOCR: true, // fully inside outer
+ }
+ got := DedupSubstringOverlaps([]pdf.TextBox{outer, inner})
+ if len(got) != 1 {
+ t.Fatalf("whitespace-equal contained fragment must be dropped, got %d boxes", len(got))
+ }
+ if got[0].Text != outer.Text {
+ t.Fatalf("outer box must be kept, got %q", got[0].Text)
+ }
+}
+
// TestDedupSubstringOverlaps_CharPathKept locks that a char-path box whose text
// is a substring of another char-path box is NEVER collapsed — e.g. a repeated
// heading inside another paragraph's text range on a digital PDF. Only OCR
@@ -277,3 +325,98 @@ func TestDedupSubstringOverlaps_CharPathKept(t *testing.T) {
t.Fatalf("char-path substring must be kept, got %d boxes (lost content?)", len(got))
}
}
+
+// TestDedupSubstringOverlaps_WhitespaceInsensitive locks that a substring box
+// whose char-derived text preserves the PDF's original spaces is STILL
+// collapsed when the containing OCR box carries a space-stripped recognition.
+// ocrMergeChars fills inner line boxes with char-layer text that keeps spaces
+// ("- name: SSL_CERT_FILE", "⽂章 中 提到") while the outer paragraph/OCR box
+// carries the recognizer's joined text ("-name: SSL_CERT_FILE",
+// "⽂章中提到"); the two are no longer contiguous substrings of each other
+// byte-wise, so dedup must compare whitespace-normalized text. This is the
+// root cause of the ocr_real text gaps (plugin-daemon/RAG分词/三国人物
+// duplicated lines after vertical merge).
+func TestDedupSubstringOverlaps_WhitespaceInsensitive(t *testing.T) {
+ outer := pdf.TextBox{
+ Text: "直接⽤rag分词建⽴索引,这时⽤分词1来查询,服务体系?都会保留,因为根据rag分词不会删除标点。但是,在原⽂中,并没有服务体系?这样的⽂字,因此这个短语查询⽆法命中。",
+ PageNumber: 0, Top: 100, Bottom: 400, X0: 60, X1: 520, IsOCR: true,
+ }
+ inner := pdf.TextBox{
+ Text: "直接⽤ rag 分词建⽴索引,这时⽤分词 1 来查询", // char layer kept the PDF's original spaces
+ PageNumber: 0, Top: 120, Bottom: 135, X0: 60, X1: 520, IsOCR: true, // fully inside outer
+ }
+ got := DedupSubstringOverlaps([]pdf.TextBox{outer, inner})
+ if len(got) != 1 {
+ t.Fatalf("whitespace-divergent substring fragment must be dropped, got %d boxes: %+v", len(got), got)
+ }
+ if got[0].Text != outer.Text {
+ t.Fatalf("outer paragraph must be kept, got %q", got[0].Text)
+ }
+}
+
+// TestDedupSubstringOverlaps_WhitespaceInsensitive_CJK uses the CJK case from
+// RAG分词召回分析.pdf: the outer OCR box joined CJK without spaces while the
+// inner char-derived fragment kept per-word spaces ("⽂章 中 提到" vs "⽂章中提到").
+func TestDedupSubstringOverlaps_WhitespaceInsensitive_CJK(t *testing.T) {
+ outer := pdf.TextBox{
+ Text: "⽤Python⽣成的分词1为:⽂章中提到了哪些健康服务体系?",
+ PageNumber: 0, Top: 100, Bottom: 400, X0: 60, X1: 520, IsOCR: true,
+ }
+ inner := pdf.TextBox{
+ Text: "⽂章 中 提到 了 哪些 健康 服务体系", // char layer preserved spaces between CJK words
+ PageNumber: 0, Top: 150, Bottom: 165, X0: 60, X1: 520, IsOCR: true,
+ }
+ got := DedupSubstringOverlaps([]pdf.TextBox{outer, inner})
+ if len(got) != 1 {
+ t.Fatalf("CJK whitespace-divergent fragment must be dropped, got %d boxes", len(got))
+ }
+}
+
+// TestDedupSubstringOverlaps_WhitespaceInsensitive_DifferentColumnKept locks
+// the boundary guard: whitespace normalization must NOT let a substring box in
+// a DIFFERENT column (disjoint X) be dropped — the geometry check still
+// governs, only the text comparison became whitespace-insensitive.
+func TestDedupSubstringOverlaps_WhitespaceInsensitive_DifferentColumnKept(t *testing.T) {
+ colA := pdf.TextBox{
+ Text: "line xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", PageNumber: 0, Top: 100, Bottom: 115, X0: 60, X1: 260, IsOCR: true,
+ }
+ colB := pdf.TextBox{
+ Text: "line x x x x x x x x x", PageNumber: 0, Top: 100, Bottom: 115, X0: 320, X1: 520, IsOCR: true, // disjoint X
+ }
+ got := DedupSubstringOverlaps([]pdf.TextBox{colA, colB})
+ if len(got) != 2 {
+ t.Fatalf("different-column whitespace-divergent substring must be kept, got %d boxes", len(got))
+ }
+}
+
+// TestDedupSubstringOverlaps_WhitespaceInsensitive_DisjointYKept locks that a
+// whitespace-divergent substring at a DISJOINT Y position is kept — a real
+// repeated heading, not an OCR fragment.
+func TestDedupSubstringOverlaps_WhitespaceInsensitive_DisjointYKept(t *testing.T) {
+ full := pdf.TextBox{
+ Text: "Conclusion summary of the whole document body text", PageNumber: 0, Top: 100, Bottom: 115, IsOCR: true,
+ }
+ repeat := pdf.TextBox{
+ Text: "C o n c l u s i o n", PageNumber: 0, Top: 300, Bottom: 315, IsOCR: true, // disjoint Y
+ }
+ got := DedupSubstringOverlaps([]pdf.TextBox{full, repeat})
+ if len(got) != 2 {
+ t.Fatalf("disjoint-Y whitespace-divergent substring must be kept, got %d boxes", len(got))
+ }
+}
+
+// TestDedupSubstringOverlaps_WhitespaceInsensitive_CharPathKept locks that a
+// char-path (IsOCR=false) box is never collapsed even when its whitespace-
+// normalized text is a substring of another char-path box.
+func TestDedupSubstringOverlaps_WhitespaceInsensitive_CharPathKept(t *testing.T) {
+ full := pdf.TextBox{
+ Text: "本协议终止后保密条款继续有效双方仍应承担保密义务", PageNumber: 0, Top: 100, Bottom: 120,
+ }
+ repeat := pdf.TextBox{
+ Text: "保密 条款 继续 有效", PageNumber: 0, Top: 110, Bottom: 118, // nested substring, char path
+ }
+ got := DedupSubstringOverlaps([]pdf.TextBox{full, repeat})
+ if len(got) != 2 {
+ t.Fatalf("char-path whitespace-divergent substring must be kept, got %d boxes", len(got))
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/layout/layout.go b/internal/deepdoc/parser/pdf/layout/layout.go
index 06eb409b3d..27c0dd011c 100644
--- a/internal/deepdoc/parser/pdf/layout/layout.go
+++ b/internal/deepdoc/parser/pdf/layout/layout.go
@@ -131,8 +131,28 @@ func DedupIdenticalText(boxes []pdf.TextBox) []pdf.TextBox {
// text is legal. The containment test is bound to the substring box (not just
// the shorter-height box) so a physically taller box whose short text is a
// substring of a neighbour is never silently dropped.
+//
+// The substring comparison is whitespace-insensitive: ocrMergeChars fills line
+// fragments with char-layer text that preserves the PDF's original spaces
+// ("- name: SSL_CERT_FILE", "⽂章 中 提到") while the containing paragraph OCR
+// box carries the recognizer's joined text ("-name: SSL_CERT_FILE",
+// "⽂章中提到"). Byte-wise matching would miss the fragment relation and the
+// inner box survives to NaiveVerticalMerge, which concatenates it into the
+// paragraph — duplicating text (the ocr_real text gaps: plugin-daemon,
+// RAG分词, 三国人物). Geometry (boxInside) remains the actual containment
+// proof; whitespace normalization only makes the fragment check robust to the
+// char-vs-OCR space divergence.
func DedupSubstringOverlaps(boxes []pdf.TextBox) []pdf.TextBox {
drop := make([]bool, len(boxes))
+ // Precompute the whitespace-normalized text once per box. The inner loop
+ // compares every OCR pair, so recomputing norm there would make the O(n²)
+ // loop O(n²·len) in the worst case (a large scan document hits ~3k boxes).
+ norm := make([]string, len(boxes))
+ for i, b := range boxes {
+ if b.IsOCR {
+ norm[i] = dedupNormText(strings.TrimSpace(b.Text))
+ }
+ }
for i := range boxes {
if drop[i] {
continue
@@ -151,21 +171,22 @@ func DedupSubstringOverlaps(boxes []pdf.TextBox) []pdf.TextBox {
continue
}
aj := strings.TrimSpace(boxes[j].Text)
- if aj == "" || len(ai) == len(aj) {
+ if aj == "" || ai == aj {
continue
}
+ ni, nj := norm[i], norm[j]
// Collapse only when the SUBSTRING-text box is geometrically CONTAINED
// in the text-containing box. Binding the geometry to the actual
// substring (not merely the shorter-height box) avoids silently
// dropping a physically taller box whose short text happens to be a
// substring of a neighbour — OCR double-detection fragments are always
// the smaller, contained box, so the taller container is kept.
- if len(ai) > len(aj) && strings.Contains(ai, aj) {
+ if len(ni) >= len(nj) && strings.Contains(ni, nj) {
// j's text is a substring of i's -> drop j only if j sits inside i.
if boxInside(boxes[j], boxes[i]) {
drop[j] = true
}
- } else if len(aj) > len(ai) && strings.Contains(aj, ai) {
+ } else if len(nj) > len(ni) && strings.Contains(nj, ni) {
// i's text is a substring of j's -> drop i only if i sits inside j.
if boxInside(boxes[i], boxes[j]) {
drop[i] = true
@@ -184,14 +205,33 @@ func DedupSubstringOverlaps(boxes []pdf.TextBox) []pdf.TextBox {
return out
}
-// boxInside reports whether inner is fully contained within outer in Y and
-// overlaps it in X. It confirms an OCR substring fragment sits inside the box
-// whose text contains it (not merely shares a Y band at a different column).
+// dedupNormText strips all whitespace for substring comparison. Spaces are the
+// only divergence between the char-derived fragment text and the OCR box text,
+// and whitespace carries no content identity here — the containment geometry is
+// the real proof (see DedupSubstringOverlaps).
+func dedupNormText(s string) string {
+ var b strings.Builder
+ b.Grow(len(s))
+ for _, r := range s {
+ if !unicode.IsSpace(r) {
+ b.WriteRune(r)
+ }
+ }
+ return b.String()
+}
+
+// boxInside reports whether inner is FULLY contained within outer on BOTH
+// axes. It confirms an OCR substring fragment sits inside the box whose text
+// contains it — not merely sharing a Y band at a different column, nor poking
+// out horizontally beyond the container. Requiring horizontal containment
+// stops a whitespace-normalized substring match from dropping a box that
+// extends past the container's X range (e.g. an adjacent-column line whose
+// text happens to be a substring of the paragraph's).
func boxInside(inner, outer pdf.TextBox) bool {
if inner.Top < outer.Top || inner.Bottom > outer.Bottom {
return false
}
- if outer.X1 <= inner.X0 || inner.X1 <= outer.X0 {
+ if inner.X0 < outer.X0 || inner.X1 > outer.X1 {
return false
}
return true