Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(deepdoc): close ocr_real text-gap divergences and pin char-to-box matching (#18630)

Jack6483f84d0c63
diff --git a/internal/common/environments.go b/internal/common/environments.go
index 20b3120831..c1a3059845 100644
--- a/internal/common/environments.go
+++ b/internal/common/environments.go
@@ -141,6 +141,8 @@ const (
EnvOSSDeepDocURL = "OSSDEEPDOC_URL"
EnvUpdateGolden = "UPDATE_GOLDEN"
EnvBatchParityFilter = "BATCH_PARITY_FILTER"
+ EnvBatchParityVariant = "BATCH_PARITY_VARIANT"
+ EnvBatchParityDataRoot = "BATCH_PARITY_DATA_ROOT"
EnvDumpCount = "DUMP_COUNT"
EnvBatchCSV = "BATCH_CSV"
EnvESTest = "ES_TEST"
diff --git a/internal/deepdoc/parser/pdf/layout/dedup_test.go b/internal/deepdoc/parser/pdf/layout/dedup_test.go
index 752dea94e6..6d4ea42acb 100644
--- a/internal/deepdoc/parser/pdf/layout/dedup_test.go
+++ b/internal/deepdoc/parser/pdf/layout/dedup_test.go
@@ -175,6 +175,30 @@ func TestDedupSubstringOverlaps(t *testing.T) {
}
}
+// TestDedupSubstringOverlaps_YOvershootFragmentDropped locks root-cause-A:
+// an OCR double-detection fragment whose text is a whitespace-normalized
+// substring of the container but whose Y bounds overshoot by a few points of
+// detection noise (beyond boxInsideTolerant's 3pt Y tolerance) must STILL be collapsed. This
+// is the exact geometry Rag Flow Usage / 三国人物 produce and that leaks
+// duplicated text into the Go output without it.
+func TestDedupSubstringOverlaps_YOvershootFragmentDropped(t *testing.T) {
+ full := pdf.TextBox{
+ Text: "We'resoextoseeyou again",
+ PageNumber: 0, Top: 224.8, Bottom: 233.3, X0: 477, X1: 599, IsOCR: true,
+ }
+ frag := pdf.TextBox{
+ Text: "toseeyou again", // substring of full; Y overshoots 0.5pt top / 1.0pt bottom
+ PageNumber: 0, Top: 224.3, Bottom: 234.3, X0: 537, X1: 599, IsOCR: true,
+ }
+ got := DedupSubstringOverlaps([]pdf.TextBox{full, frag})
+ if len(got) != 1 {
+ t.Fatalf("Y-overshoot substring fragment must be dropped, got %d boxes", len(got))
+ }
+ if got[0].Text != full.Text {
+ t.Fatalf("the full box must be kept, got %q", got[0].Text)
+ }
+}
+
// TestDedupSubstringOverlaps_DisjointYKept ensures a substring box at a
// DISJOINT Y position is kept — a real repeated heading or sentence is legal.
func TestDedupSubstringOverlaps_DisjointYKept(t *testing.T) {
@@ -226,7 +250,7 @@ func TestDedupSubstringOverlaps_AdjacentLinesKept(t *testing.T) {
// TestDedupSubstringOverlaps_PartialYOverlapKept locks the Y-containment
// boundary: a substring box that PARTIALLY overlaps the containing box in Y
// (extends below it) is KEPT — it is an adjacent-line fragment, not a contained
-// duplicate. Only a fragment FULLY inside the containing box (boxInside) is
+// duplicate. Only a fragment contained in the box (boxInsideTolerant) is
// collapsed. With the height-vs-text decoupled guard this case is also kept;
// the test pins the boundary against future over-collapsing.
func TestDedupSubstringOverlaps_PartialYOverlapKept(t *testing.T) {
@@ -247,7 +271,7 @@ func TestDedupSubstringOverlaps_PartialYOverlapKept(t *testing.T) {
// beyond the containing box (to either side) must be KEPT — it is not an OCR
// fragment of the container (e.g. an adjacent-column line whose text happens
// to be a whitespace-normalized substring of the paragraph). Only a box fully
-// inside on BOTH axes is collapsed. This pins the boxInside hardening that the
+// inside on BOTH axes is collapsed. This pins the boxInsideTolerant hardening that the
// whitespace-insensitive match otherwise leaves exposed (a plain horizontal
// intersect used to pass the X check).
func TestDedupSubstringOverlaps_HorizontalOverhangKept(t *testing.T) {
@@ -290,7 +314,7 @@ func TestDedupSubstringOverlaps_TallerFragmentKept(t *testing.T) {
// ("-name:" vs "- name:") normalize to the SAME text and must collapse when
// geometrically contained. The legacy `len(ai) == len(aj)` skip would have
// kept the duplicate; whitespace normalization makes the two look identical,
-// and boxInside decides the containment.
+// and boxInsideTolerant decides the containment.
func TestDedupSubstringOverlaps_WhitespaceInsensitive_EqualAfterNorm(t *testing.T) {
outer := pdf.TextBox{
Text: "-name:", // OCR recognizer stripped the space after '-'
diff --git a/internal/deepdoc/parser/pdf/layout/layout.go b/internal/deepdoc/parser/pdf/layout/layout.go
index 27c0dd011c..6ee978e5bd 100644
--- a/internal/deepdoc/parser/pdf/layout/layout.go
+++ b/internal/deepdoc/parser/pdf/layout/layout.go
@@ -139,9 +139,9 @@ func DedupIdenticalText(boxes []pdf.TextBox) []pdf.TextBox {
// "⽂章中提到"). Byte-wise matching would miss the fragment relation and the
// inner box survives to NaiveVerticalMerge, which concatenates it into the
// paragraph — duplicating text (the ocr_real text gaps: plugin-daemon,
-// RAG分词, 三国人物). Geometry (boxInside) remains the actual containment
-// proof; whitespace normalization only makes the fragment check robust to the
-// char-vs-OCR space divergence.
+// RAG分词, 三国人物). Geometry (boxInsideTolerant) remains the actual
+// containment proof; whitespace normalization only makes the fragment check
+// robust to the char-vs-OCR space divergence.
func DedupSubstringOverlaps(boxes []pdf.TextBox) []pdf.TextBox {
drop := make([]bool, len(boxes))
// Precompute the whitespace-normalized text once per box. The inner loop
@@ -183,12 +183,12 @@ func DedupSubstringOverlaps(boxes []pdf.TextBox) []pdf.TextBox {
// the smaller, contained box, so the taller container is kept.
if len(ni) >= len(nj) && strings.Contains(ni, nj) {
// j's text is a substring of i's -> drop j only if j sits inside i.
- if boxInside(boxes[j], boxes[i]) {
+ if boxInsideTolerant(boxes[j], boxes[i]) {
drop[j] = true
}
} else if len(nj) > len(ni) && strings.Contains(nj, ni) {
// i's text is a substring of j's -> drop i only if i sits inside j.
- if boxInside(boxes[i], boxes[j]) {
+ if boxInsideTolerant(boxes[i], boxes[j]) {
drop[i] = true
break
}
@@ -220,18 +220,32 @@ func dedupNormText(s string) string {
return b.String()
}
-// boxInside reports whether inner is FULLY contained within outer on BOTH
-// axes. It confirms an OCR substring fragment sits inside the box whose text
-// contains it — not merely sharing a Y band at a different column, nor poking
-// out horizontally beyond the container. Requiring horizontal containment
-// stops a whitespace-normalized substring match from dropping a box that
-// extends past the container's X range (e.g. an adjacent-column line whose
-// text happens to be a substring of the paragraph's).
-func boxInside(inner, outer pdf.TextBox) bool {
- if inner.Top < outer.Top || inner.Bottom > outer.Bottom {
+// dedupYTolerancePt tolerates a small Y-boundary overshoot of an OCR
+// double-detection fragment. Such fragments sit on the SAME text line as their
+// container but their detected Y bounds jitter by ~0.5-2pt of detection noise
+// (observed on Rag Flow Usage / 三国人物); strict Y containment then misses
+// them and the duplicate text leaks into the output after TextMerge. We only
+// relax Y (never X) and only inside DedupSubstringOverlaps, where the
+// text-substring precondition already proves the box is an OCR duplicate — so
+// a few points of Y noise must not keep it. The tolerance stays far below a
+// full line height, so genuine adjacent-line or partial-overlap boxes (kept by
+// the #18568 guards) are never collapsed here.
+const dedupYTolerancePt = 3.0
+
+// boxInsideTolerant reports whether inner is contained within outer: X is
+// strict, Y allows up to dedupYTolerancePt of overshoot. It confirms an OCR
+// substring fragment sits inside the box whose text contains it — not merely
+// sharing a Y band at a different column, nor poking out horizontally beyond
+// the container. Requiring horizontal containment stops a whitespace-
+// normalized substring match from dropping a box that extends past the
+// container's X range (e.g. an adjacent-column line whose text happens to be a
+// substring of the paragraph's). The Y tolerance absorbs double-detection
+// noise; X stays strict so adjacent-column duplicates remain kept.
+func boxInsideTolerant(inner, outer pdf.TextBox) bool {
+ if inner.X0 < outer.X0 || inner.X1 > outer.X1 {
return false
}
- if inner.X0 < outer.X0 || inner.X1 > outer.X1 {
+ if inner.Top < outer.Top-dedupYTolerancePt || inner.Bottom > outer.Bottom+dedupYTolerancePt {
return false
}
return true
diff --git a/internal/deepdoc/parser/pdf/match_chars_test.go b/internal/deepdoc/parser/pdf/match_chars_test.go
new file mode 100644
index 0000000000..05683957ee
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/match_chars_test.go
@@ -0,0 +1,221 @@
+//go:build cgo
+
+package pdf
+
+import (
+ "strings"
+ "testing"
+
+ pdftype "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// TestMatchCharsToBoxes_AreaTieBreak guards the fix for ocr_real RAG分词
+// text doubling. When an OCR detector over-segments a line into a full-line
+// container box AND a smaller fragment box fully contained in it, a glyph
+// that is fully inside both boxes yields an equal overlap ratio (1.0) for
+// each. The greedy assignment must prefer the LARGER (container) box on a
+// tie, so the fragment cannot steal the glyph and truncate the container.
+//
+// The previous `>=`-with-last-wins rule let the smaller fragment win, which
+// truncated the container (dropped the trailing 么), after which
+// DedupSubstringOverlaps could no longer recognise the fragment as a
+// substring and NaiveVerticalMerge glued it back on — duplicating the text.
+func TestMatchCharsToBoxes_AreaTieBreak(t *testing.T) {
+ // Container: full-line box (large area).
+ container := ocrDetectBox{
+ box: pdftype.TextBox{X0: 10, X1: 500, Top: 560, Bottom: 589},
+ x0: 10, y0: 560, x1: 500, y1: 589,
+ }
+ // Fragment: smaller box fully contained in the container.
+ fragment := ocrDetectBox{
+ box: pdftype.TextBox{X0: 10, X1: 500, Top: 577, Bottom: 589},
+ x0: 10, y0: 577, x1: 500, y1: 589,
+ }
+ boxes := []ocrDetectBox{container, fragment}
+
+ // Chars: one belongs only to the container (above the fragment's top),
+ // and one (么) is fully inside BOTH boxes -> overlap ratio 1.0 for each.
+ // Glyph heights (~12pt) are realistic: the container box is ~29pt tall, so
+ // the existing char-height filter (>=0.7 height mismatch) does NOT drop the
+ // glyph — only the area tie-break decides which box wins.
+ chars := []pdftype.TextChar{
+ {Text: "在", X0: 20, X1: 32, Top: 565, Bottom: 575}, // container only
+ {Text: "么", X0: 100, X1: 112, Top: 577, Bottom: 589}, // inside both
+ }
+
+ got := matchCharsToBoxes(boxes, chars)
+
+ if len(got) != 2 {
+ t.Fatalf("expected 2 box char groups, got %d", len(got))
+ }
+
+ // The trailing glyph must land in the container (box 0), never the
+ // fragment (box 1).
+ var inContainer, inFragment bool
+ for _, c := range got[0] {
+ if c.Text == "么" {
+ inContainer = true
+ }
+ }
+ for _, c := range got[1] {
+ if c.Text == "么" {
+ inFragment = true
+ }
+ }
+
+ if !inContainer {
+ t.Errorf("trailing glyph 么 was NOT assigned to the container box; doubling regression risk")
+ }
+ if inFragment {
+ t.Errorf("trailing glyph 么 was assigned to the smaller fragment box; this is exactly the over-segmentation tie-break bug")
+ }
+
+ // The container-only char stays in the container.
+ var containerOnly bool
+ for _, c := range got[0] {
+ if c.Text == "在" {
+ containerOnly = true
+ }
+ }
+ if !containerOnly {
+ t.Errorf("container-only glyph 在 was not assigned to the container box")
+ }
+}
+
+// TestMatchCharsToBoxes_FullyContainedSmallChar guards the fix for the
+// ocr_real plugin-daemon "certifi missing" divergence. A small inline code
+// span (e.g. "certifi", ~8pt ink height) sits fully inside a tall detect box
+// (~36pt, spanning two text lines). The char-height filter
+// (parser_ocr.go:263, mirroring pdf_parser.py:798) drops it because
+// |8-36|/36 >= 0.7 — but the char is geometrically fully inside the box, so
+// it cannot belong to an adjacent line and must be kept. The Python golden
+// keeps such glyphs (plugin-daemon box[16] carries "certifi"), so Go must
+// too.
+//
+// Fix: a fully-contained small glyph (overlap ≈ 1.0, ratio < 0.9) is deferred
+// and re-kept when the box ALSO carries a non-space normal-height char — the
+// box is a real text line with an inline span. Cross-line chars only partially
+// overlap a tall box (ratio << 1.0), so they are still filtered — no bleed.
+func TestMatchCharsToBoxes_FullyContainedSmallChar(t *testing.T) {
+ // Tall detect box (height ~35.7), like plugin-daemon box[16].
+ tallBox := ocrDetectBox{
+ box: pdftype.TextBox{X0: 70, X1: 763, Top: 492.6, Bottom: 528.3},
+ x0: 70, y0: 492.6, x1: 763, y1: 528.3,
+ }
+ boxes := []ocrDetectBox{tallBox}
+
+ // `在`: body text, ~12pt, fully inside the tall box. Passes the filter
+ // regardless (|12-35.7|/35.7 = 0.66 < 0.7) — control case.
+ // `c`/`e`: small inline code span, ~8.3pt, ALSO fully inside the tall box.
+ // |8.3-35.7|/35.7 = 0.77 >= 0.7 -> dropped by the raw filter, but it is
+ // fully contained, so the fix must keep it (mirrors Python).
+ chars := []pdftype.TextChar{
+ {Text: "在", X0: 100, X1: 112, Top: 500, Bottom: 512}, // h=12, inside
+ {Text: "c", X0: 667, X1: 675, Top: 497.4, Bottom: 505.7}, // h=8.3, inside (certifi)
+ {Text: "e", X0: 680, X1: 688, Top: 497.4, Bottom: 505.7}, // h=8.3, inside (certifi)
+ }
+
+ got := matchCharsToBoxes(boxes, chars)
+
+ if len(got) != 1 {
+ t.Fatalf("expected 1 box char group, got %d", len(got))
+ }
+
+ gotTexts := make(map[string]bool)
+ for _, c := range got[0] {
+ gotTexts[c.Text] = true
+ }
+
+ if !gotTexts["在"] {
+ t.Errorf("body glyph 在 was dropped from the box")
+ }
+ // Pre-fix: `c` and `e` are dropped by the height filter (regression guard).
+ if !gotTexts["c"] {
+ t.Errorf("small inline glyph c (fully contained, h=8.3) was dropped; certifi-style text loss regression")
+ }
+ if !gotTexts["e"] {
+ t.Errorf("small inline glyph e (fully contained, h=8.3) was dropped; certifi-style text loss regression")
+ }
+}
+
+// TestMatchCharsToBoxes_FullyContainedTinyDropped locks the other boundary of
+// the containment guard: a FULLY CONTAINED char that is extremely tiny
+// (height <= 10% of the box, ratio >= 0.9) must STILL be dropped by the
+// height gate — it is baseline noise (e.g. a 1pt glyph in a 40pt box), not
+// inline content. This preserves TestOCRMergeChars_HeightGate's contract.
+func TestMatchCharsToBoxes_FullyContainedTinyDropped(t *testing.T) {
+ box := ocrDetectBox{
+ box: pdftype.TextBox{X0: 0, X1: 90, Top: 0, Bottom: 120},
+ x0: 0, y0: 0, x1: 90, y1: 120,
+ }
+ boxes := []ocrDetectBox{box}
+ // h=1 vs bh=120 -> ratio 0.992 >= 0.9, fully contained (overlap 1.0).
+ chars := []pdftype.TextChar{
+ {X0: 5, X1: 10, Top: 40, Bottom: 41, Text: "t"},
+ }
+ got := matchCharsToBoxes(boxes, chars)
+ if len(got) != 1 {
+ t.Fatalf("expected 1 box group, got %d", len(got))
+ }
+ if len(got[0]) != 0 {
+ t.Errorf("extremely tiny fully-contained glyph must be dropped by the height gate, got %+v", got[0])
+ }
+}
+
+// TestMatchCharsToBoxes_DeferredSpaceOnlyNormalDropped locks the boundary that
+// kept dell-configuration-services-sd-zh-tw (and the other real-PDF set) from
+// regressing: a box whose ONLY normal-height chars are SPACES must NOT absorb
+// deferred inline glyphs. The real line text lives in a tighter neighbor box
+// there, so the small glyphs are another line's content — leaving the box
+// empty lets buildTextBoxes' OCR fallback recognize the true line from the
+// image. Without this guard, plugin-daemon's certifi fix added stray glyphs to
+// space-only boxes and dropped gridSim from 100% to 95.9% on that PDF.
+func TestMatchCharsToBoxes_DeferredSpaceOnlyNormalDropped(t *testing.T) {
+ box := ocrDetectBox{
+ box: pdftype.TextBox{X0: 0, X1: 90, Top: 0, Bottom: 40},
+ x0: 0, y0: 0, x1: 90, y1: 40,
+ }
+ boxes := []ocrDetectBox{box}
+ // Space char: ratio < 0.7 (normal), but carries no content.
+ // "x": small glyph, fully contained, ratio in [0.7, 0.9) -> deferred.
+ chars := []pdftype.TextChar{
+ {X0: 5, X1: 6, Top: 5, Bottom: 20, Text: " "},
+ {X0: 10, X1: 16, Top: 6, Bottom: 12, Text: "x"}, // h=6 vs bh=40 -> ratio 0.85
+ }
+ got := matchCharsToBoxes(boxes, chars)
+ if len(got) != 1 {
+ t.Fatalf("expected 1 box group, got %d", len(got))
+ }
+ for _, c := range got[0] {
+ if strings.TrimSpace(c.Text) != "" {
+ t.Errorf("deferred glyph %q must be dropped when the box's only normal chars are spaces", c.Text)
+ }
+ }
+}
+
+// TestMatchCharsToBoxes_PartialOverlapHeightFiltered locks that a char which
+// only PARTIALLY overlaps its assigned box (overlap < 0.95) is STILL dropped
+// by the height gate when its height differs from the box by >=70% — it is an
+// adjacent-line glyph poking into the box, not inline content. The deferred
+// inline-glyph path requires full containment (bestOverlap >= 0.95), so a
+// partial overlap must never be re-kept there.
+func TestMatchCharsToBoxes_PartialOverlapHeightFiltered(t *testing.T) {
+ box := ocrDetectBox{
+ box: pdftype.TextBox{X0: 0, X1: 90, Top: 0, Bottom: 40},
+ x0: 0, y0: 0, x1: 90, y1: 40,
+ }
+ boxes := []ocrDetectBox{box}
+ // h=5 vs bh=40 -> ratio 0.875 >= 0.7; the char spans top=-2..3, so only
+ // 3 of its 5pt vertically overlap the box (top 0..3) -> overlap 0.6 < 0.95,
+ // i.e. NOT fully contained -> deferred path must not absorb it.
+ chars := []pdftype.TextChar{
+ {X0: 5, X1: 12, Top: -2, Bottom: 3, Text: "x"},
+ }
+ got := matchCharsToBoxes(boxes, chars)
+ if len(got) != 1 {
+ t.Fatalf("expected 1 box group, got %d", len(got))
+ }
+ if len(got[0]) != 0 {
+ t.Errorf("partial-overlap height-mismatched glyph must be dropped by the height gate, got %+v", got[0])
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/parser_ocr.go b/internal/deepdoc/parser/pdf/parser_ocr.go
index fd9e2ccf62..3922b57623 100644
--- a/internal/deepdoc/parser/pdf/parser_ocr.go
+++ b/internal/deepdoc/parser/pdf/parser_ocr.go
@@ -226,13 +226,36 @@ func (p *Parser) detectBoxes(ctx context.Context, pageImg image.Image, doc pdf.D
func matchCharsToBoxes(boxes []ocrDetectBox, chars []pdf.TextChar) [][]pdf.TextChar {
boxChars := make([][]pdf.TextChar, len(boxes))
+ // deferred holds fully-contained small glyphs (candidates for inline text)
+ // until we know whether the box also carries any normal-height content. A
+ // box whose ONLY char-layer chars are small (ratio >= 0.7) must stay empty
+ // so buildTextBoxes' OCR fallback can recognize the full line from the
+ // image — keeping the small glyphs alone would emit a partial fragment and
+ // suppress the OCR fill (三国人物/反间谍法 regressed under that rule).
+ deferred := make([][]pdf.TextChar, len(boxes))
for _, c := range chars {
bestIdx := -1
bestOverlap := 1e-6
+ bestArea := 0.0
for i := range boxes {
overlap := charBoxOverlapRatio(c, boxes[i].x0, boxes[i].x1, boxes[i].y0, boxes[i].y1)
- if overlap >= bestOverlap {
+ if overlap < bestOverlap {
+ continue
+ }
+ area := (boxes[i].x1 - boxes[i].x0) * (boxes[i].y1 - boxes[i].y0)
+ // Tie-break: when a char is fully inside several boxes (a full-line
+ // box and a contained OCR fragment that over-segments it), prefer
+ // the LARGER container so the fragment cannot steal the glyph and
+ // truncate the container. Mirrors Python's Recognizer.find_overlapped
+ // (recognizer.py:223), which keeps the max-overlapped (largest-area)
+ // box on ties via a strict `>`. The previous `>=`-with-last-wins
+ // rule let the smaller fragment win, truncating the container; after
+ // DedupSubstringOverlaps could no longer recognise the fragment as a
+ // substring, NaiveVerticalMerge glued it back on and duplicated text
+ // (ocr_real RAG分词 doubling).
+ if overlap > bestOverlap || area > bestArea {
bestOverlap = overlap
+ bestArea = area
bestIdx = i
}
}
@@ -244,10 +267,37 @@ func matchCharsToBoxes(boxes []ocrDetectBox, chars []pdf.TextChar) [][]pdf.TextC
ch = 1
}
bh := boxes[bestIdx].y1 - boxes[bestIdx].y0
- if math.Abs(ch-bh)/math.Max(ch, bh) >= 0.7 && c.Text != " " {
- continue
+ // Char-height filter (mirrors Python pdf_parser.py:798): drop chars
+ // whose height differs greatly from the box height — they belong to
+ // another line. A fully-contained small glyph (overlap >= 0.95,
+ // ratio < 0.9) is an inline-text candidate (e.g. a code span like
+ // "certifi", ~8pt, inside a tall two-line detect box ~36pt — the Python
+ // golden keeps it, plugin-daemon box[16]): it cannot be from an
+ // adjacent line because it lies almost entirely inside the box. It is
+ // deferred and re-kept only when the box also carries normal-height
+ // content, so an isolated small glyph cannot suppress the OCR fallback.
+ ratio := math.Abs(ch-bh) / math.Max(ch, bh)
+ if ratio < 0.7 || c.Text == " " {
+ boxChars[bestIdx] = append(boxChars[bestIdx], c)
+ } else if bestOverlap >= 0.95 && ratio < 0.9 {
+ deferred[bestIdx] = append(deferred[bestIdx], c)
+ }
+ }
+ for i := range boxChars {
+ // Re-keep the deferred inline glyphs only when the box actually carries
+ // a non-space normal-height char: a box whose only normal chars are
+ // spaces (the real line text lives in a tighter neighbor box) must not
+ // absorb the small glyphs — they are another line's content there.
+ hasText := false
+ for _, c := range boxChars[i] {
+ if strings.TrimSpace(c.Text) != "" {
+ hasText = true
+ break
+ }
+ }
+ if hasText {
+ boxChars[i] = append(boxChars[i], deferred[i]...)
}
- boxChars[bestIdx] = append(boxChars[bestIdx], c)
}
return boxChars
}
diff --git a/internal/deepdoc/parser/pdf/parser_ocr_test.go b/internal/deepdoc/parser/pdf/parser_ocr_test.go
index 529e019844..63e1fe5d5d 100644
--- a/internal/deepdoc/parser/pdf/parser_ocr_test.go
+++ b/internal/deepdoc/parser/pdf/parser_ocr_test.go
@@ -305,12 +305,9 @@ func TestOCRMergeChars_OverlappingBoxes(t *testing.T) {
p := NewParser(pdf.DefaultParserConfig())
// Box A: PDF x=0..20, y=0..20. Box B: PDF x=10..30, y=0..20.
// Overlap zone: x=10..20.
- // Char "Y" at PDF x=2..8 → Box A only.
- // Char "X" at PDF x=12..18 → overlap zone (both boxes).
- // Char "Z" at PDF x=22..28 → Box B only.
- //
- // Old box-perspective: Box A gets [Y,X], Box B gets [X,Z].
- // New char-perspective: Box A gets [Y,X] (best overlap), Box B gets [Z].
+ // Char "甲" at PDF x=2..8 → Box A only.
+ // Char "乙" at PDF x=12..18 → overlap zone (both boxes).
+ // Char "丙" at PDF x=22..28 → Box B only.
mock := &MockDocAnalyzer{
Healthy: true,
OCRBoxes: []pdf.OCRBox{
@@ -327,14 +324,18 @@ func TestOCRMergeChars_OverlappingBoxes(t *testing.T) {
if len(boxes) != 2 {
t.Fatalf("expected 2 boxes, got %d", len(boxes))
}
- // Tie on equal overlap → later box wins (matching Python's >=).
- // "乙" goes to Box B (both overlap=1.0, Box B checked later).
- // Box A → "甲", Box B → "乙丙" (sorted by X).
- if boxes[0].Text != "甲" {
- t.Errorf("box A: expected '甲', got %q", boxes[0].Text)
- }
- if boxes[1].Text != "乙丙" {
- t.Errorf("box B: expected '乙丙', got %q", boxes[1].Text)
+ // Tie on equal overlap → FIRST box wins. matchCharsToBoxes keeps the first
+ // box that reaches the max overlap via a strict `>` (preferring the larger
+ // area on ties), mirroring Python's Recognizer.find_overlapped
+ // (recognizer.py:223 `if ov <= max_overlapped: continue` — first-wins;
+ // verified empirically). "乙" has overlap 1.0 with both equal-area boxes,
+ // so it goes to Box A (the earlier one).
+ // Box A → "甲乙", Box B → "丙" (sorted by X).
+ if boxes[0].Text != "甲乙" {
+ t.Errorf("box A: expected '甲乙', got %q", boxes[0].Text)
+ }
+ if boxes[1].Text != "丙" {
+ t.Errorf("box B: expected '丙', got %q", boxes[1].Text)
}
}
diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_test.go
index a5fb2250ff..590a0ac62a 100644
--- a/internal/deepdoc/parser/pdf/pipeline_parity_test.go
+++ b/internal/deepdoc/parser/pdf/pipeline_parity_test.go
@@ -44,12 +44,15 @@ import (
// Python, e.g. table segmentation). Any genuine Go content regression is a
// FAIL.
func TestPipelineParity(t *testing.T) {
- charspyDir := filepath.Join("testdata", "charspy")
- pyTextDir := filepath.Join("testdata", "output", "py", "ocr", "text")
- dlaDir := filepath.Join("testdata", "output", "py", "ocr", "dla")
- tsrDir := filepath.Join("testdata", "output", "py", "ocr", "tsr_raw")
- ocrDir := filepath.Join("testdata", "output", "py", "ocr", "ocr")
- tablesDir := filepath.Join("testdata", "output", "py", "ocr", "tables")
+ // Dataset variant: "" (default) or "ocr" resolve to the built-in layout
+ // (charspy/, output/py/ocr/...) for the original 35-PDF fixture set; a
+ // custom variant such as "ocr_real" reads its own isolated directories
+ // (charspy_ocr_real/, output/py/ocr_real/...), so a second dataset (e.g.
+ // real_pdfs/) gets its own dumps and verdicts without touching the default
+ // set. See tool.ParityDirsFor.
+ dirs := tool.ParityDirsFor(common.GetEnv(common.EnvBatchParityVariant))
+ charspyDir, pyTextDir := dirs.Charspy, dirs.Text
+ dlaDir, tsrDir, ocrDir, tablesDir := dirs.DLA, dirs.TSRRaw, dirs.OCR, dirs.Tables
entries, err := os.ReadDir(charspyDir)
if err != nil {
@@ -158,7 +161,7 @@ func TestPipelineParity(t *testing.T) {
// Debug aid: BATCH_PARITY_DUMP_GO=1 writes the Go output text next to
// the Python golden so failing PDFs can be diffed case by case.
if common.GetEnv("BATCH_PARITY_DUMP_GO") != "" {
- dumpDir := filepath.Join("testdata", "output", "go", "ocr", "text")
+ dumpDir := dirs.GoText
_ = os.MkdirAll(dumpDir, 0o755)
_ = os.WriteFile(filepath.Join(dumpDir, name+".txt"), []byte(goText.String()), 0o644)
}
diff --git a/internal/deepdoc/parser/pdf/tool/parity_dirs.go b/internal/deepdoc/parser/pdf/tool/parity_dirs.go
new file mode 100644
index 0000000000..3c3e8ee50e
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/tool/parity_dirs.go
@@ -0,0 +1,66 @@
+package tool
+
+import (
+ "path/filepath"
+
+ "ragflow/internal/common"
+)
+
+// ParityDirs holds every testdata directory the pipeline-parity harness
+// reads/writes for one dataset variant. The variant isolates a second set of
+// PDFs (e.g. real_pdfs/) from the default 35-PDF fixture set so dumps and
+// parity verdicts never collide.
+type ParityDirs struct {
+ // Charspy is the Python pdfplumber chars dump directory.
+ Charspy string
+ // Text is Python's per-box text golden.
+ Text string
+ // DLA is Python's layout intermediate replay input.
+ DLA string
+ // TSRRaw is Python's table-structure-recognition replay input.
+ TSRRaw string
+ // OCR is Python's raw OCR detect replay input.
+ OCR string
+ // Tables is Python's table grid golden.
+ Tables string
+ // GoText is where the harness dumps Go output text for diffing (BATCH_PARITY_DUMP_GO).
+ GoText string
+}
+
+// ParityDirsFor maps a dataset variant to its testdata directories. The
+// default variant ("" or "ocr") resolves to the built-in layout (charspy/,
+// output/py/ocr/...); a custom variant such as "ocr_real" moves every artifact
+// under a variant suffix (charspy_ocr_real/, output/py/ocr_real/...) so two
+// datasets never share a file.
+//
+// For a custom variant the root directory can be redirected via
+// BATCH_PARITY_DATA_ROOT (e.g. a shared directory outside the worktree so
+// multiple worktrees reuse one dump). The default variant always stays under
+// the local "testdata".
+func ParityDirsFor(variant string) ParityDirs {
+ if variant == "" {
+ variant = "ocr"
+ }
+ root := "testdata"
+ if variant != "ocr" {
+ if r := common.GetEnv(common.EnvBatchParityDataRoot); r != "" {
+ root = r
+ }
+ }
+ // The charspy dir is the only artifact whose name does not carry the
+ // variant suffix for the default variant (charspy/), matching the layout
+ // the dumps were generated with; custom variants append the suffix.
+ charspyDir := "charspy"
+ if variant != "ocr" {
+ charspyDir = "charspy_" + variant
+ }
+ return ParityDirs{
+ Charspy: filepath.Join(root, charspyDir),
+ Text: filepath.Join(root, "output", "py", variant, "text"),
+ DLA: filepath.Join(root, "output", "py", variant, "dla"),
+ TSRRaw: filepath.Join(root, "output", "py", variant, "tsr_raw"),
+ OCR: filepath.Join(root, "output", "py", variant, "ocr"),
+ Tables: filepath.Join(root, "output", "py", variant, "tables"),
+ GoText: filepath.Join(root, "output", "go", variant, "text"),
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/tool/parity_dirs_test.go b/internal/deepdoc/parser/pdf/tool/parity_dirs_test.go
new file mode 100644
index 0000000000..788c5af266
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/tool/parity_dirs_test.go
@@ -0,0 +1,73 @@
+package tool
+
+import (
+ "path/filepath"
+ "testing"
+)
+
+// TestParityDirsForDefaultVariant pins the default directory layout: the
+// default variant ("" or "ocr") must resolve exactly to the built-in paths
+// (charspy/, output/py/ocr/...).
+func TestParityDirsForDefaultVariant(t *testing.T) {
+ for _, v := range []string{"", "ocr"} {
+ d := ParityDirsFor(v)
+ want := ParityDirs{
+ Charspy: filepath.Join("testdata", "charspy"),
+ Text: filepath.Join("testdata", "output", "py", "ocr", "text"),
+ DLA: filepath.Join("testdata", "output", "py", "ocr", "dla"),
+ TSRRaw: filepath.Join("testdata", "output", "py", "ocr", "tsr_raw"),
+ OCR: filepath.Join("testdata", "output", "py", "ocr", "ocr"),
+ Tables: filepath.Join("testdata", "output", "py", "ocr", "tables"),
+ GoText: filepath.Join("testdata", "output", "go", "ocr", "text"),
+ }
+ if d != want {
+ t.Errorf("ParityDirsFor(%q) = %+v, want %+v", v, d, want)
+ }
+ }
+}
+
+// TestParityDirsForCustomVariant pins the variant-isolated layout: a custom
+// variant moves every artifact under its own subdirectory so a second dataset
+// (e.g. real_pdfs) never collides with the default 35-PDF set.
+func TestParityDirsForCustomVariant(t *testing.T) {
+ d := ParityDirsFor("ocr_real")
+ want := ParityDirs{
+ Charspy: filepath.Join("testdata", "charspy_ocr_real"),
+ Text: filepath.Join("testdata", "output", "py", "ocr_real", "text"),
+ DLA: filepath.Join("testdata", "output", "py", "ocr_real", "dla"),
+ TSRRaw: filepath.Join("testdata", "output", "py", "ocr_real", "tsr_raw"),
+ OCR: filepath.Join("testdata", "output", "py", "ocr_real", "ocr"),
+ Tables: filepath.Join("testdata", "output", "py", "ocr_real", "tables"),
+ GoText: filepath.Join("testdata", "output", "go", "ocr_real", "text"),
+ }
+ if d != want {
+ t.Errorf("ParityDirsFor(\"ocr_real\") = %+v, want %+v", d, want)
+ }
+}
+
+// TestParityDirsForCustomVariantDataRoot pins the shared-data-root redirect:
+// when BATCH_PARITY_DATA_ROOT is set, a custom variant resolves every
+// artifact under that root (so multiple worktrees reuse one dump). The
+// default variant must ignore the env and stay under local testdata.
+func TestParityDirsForCustomVariantDataRoot(t *testing.T) {
+ t.Setenv("BATCH_PARITY_DATA_ROOT", "/srv/parity-data")
+ d := ParityDirsFor("ocr_real")
+ want := ParityDirs{
+ Charspy: filepath.Join("/srv/parity-data", "charspy_ocr_real"),
+ Text: filepath.Join("/srv/parity-data", "output", "py", "ocr_real", "text"),
+ DLA: filepath.Join("/srv/parity-data", "output", "py", "ocr_real", "dla"),
+ TSRRaw: filepath.Join("/srv/parity-data", "output", "py", "ocr_real", "tsr_raw"),
+ OCR: filepath.Join("/srv/parity-data", "output", "py", "ocr_real", "ocr"),
+ Tables: filepath.Join("/srv/parity-data", "output", "py", "ocr_real", "tables"),
+ GoText: filepath.Join("/srv/parity-data", "output", "go", "ocr_real", "text"),
+ }
+ if d != want {
+ t.Errorf("ParityDirsFor(\"ocr_real\") with BATCH_PARITY_DATA_ROOT = %+v, want %+v", d, want)
+ }
+
+ // Default variant must NOT be redirected by the env.
+ def := ParityDirsFor("")
+ if def.Charspy != filepath.Join("testdata", "charspy") {
+ t.Errorf("default variant Charspy = %q, want %q (env must be ignored)", def.Charspy, filepath.Join("testdata", "charspy"))
+ }
+}