RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(deepdoc): close ocr_real text-gap divergences and pin char-to-box matching (#18630)
Jack6483f84d0c63
diff --git a/internal/common/environments.go b/internal/common/environments.goindex 20b3120831..c1a3059845 100644--- a/internal/common/environments.go+++ b/internal/common/environments.go@@ -141,6 +141,8 @@ const (EnvOSSDeepDocURL = "OSSDEEPDOC_URL"EnvUpdateGolden = "UPDATE_GOLDEN"EnvBatchParityFilter = "BATCH_PARITY_FILTER"+ EnvBatchParityVariant = "BATCH_PARITY_VARIANT"+ EnvBatchParityDataRoot = "BATCH_PARITY_DATA_ROOT"EnvDumpCount = "DUMP_COUNT"EnvBatchCSV = "BATCH_CSV"EnvESTest = "ES_TEST"diff --git a/internal/deepdoc/parser/pdf/layout/dedup_test.go b/internal/deepdoc/parser/pdf/layout/dedup_test.goindex 752dea94e6..6d4ea42acb 100644--- a/internal/deepdoc/parser/pdf/layout/dedup_test.go+++ b/internal/deepdoc/parser/pdf/layout/dedup_test.go@@ -175,6 +175,30 @@ func TestDedupSubstringOverlaps(t *testing.T) {}}+// TestDedupSubstringOverlaps_YOvershootFragmentDropped locks root-cause-A:+// an OCR double-detection fragment whose text is a whitespace-normalized+// substring of the container but whose Y bounds overshoot by a few points of+// detection noise (beyond boxInsideTolerant's 3pt Y tolerance) must STILL be collapsed. This+// is the exact geometry Rag Flow Usage / 三国人物 produce and that leaks+// duplicated text into the Go output without it.+func TestDedupSubstringOverlaps_YOvershootFragmentDropped(t *testing.T) {+ full := pdf.TextBox{+ Text: "We'resoextoseeyou again",+ PageNumber: 0, Top: 224.8, Bottom: 233.3, X0: 477, X1: 599, IsOCR: true,+ }+ frag := pdf.TextBox{+ Text: "toseeyou again", // substring of full; Y overshoots 0.5pt top / 1.0pt bottom+ PageNumber: 0, Top: 224.3, Bottom: 234.3, X0: 537, X1: 599, IsOCR: true,+ }+ got := DedupSubstringOverlaps([]pdf.TextBox{full, frag})+ if len(got) != 1 {+ t.Fatalf("Y-overshoot substring fragment must be dropped, got %d boxes", len(got))+ }+ if got[0].Text != full.Text {+ t.Fatalf("the full box must be kept, got %q", got[0].Text)+ }+}+// TestDedupSubstringOverlaps_DisjointYKept ensures a substring box at a// DISJOINT Y position is kept — a real repeated heading or sentence is legal.func TestDedupSubstringOverlaps_DisjointYKept(t *testing.T) {@@ -226,7 +250,7 @@ func TestDedupSubstringOverlaps_AdjacentLinesKept(t *testing.T) {// TestDedupSubstringOverlaps_PartialYOverlapKept locks the Y-containment// boundary: a substring box that PARTIALLY overlaps the containing box in Y// (extends below it) is KEPT — it is an adjacent-line fragment, not a contained-// duplicate. Only a fragment FULLY inside the containing box (boxInside) is+// duplicate. Only a fragment contained in the box (boxInsideTolerant) is// collapsed. With the height-vs-text decoupled guard this case is also kept;// the test pins the boundary against future over-collapsing.func TestDedupSubstringOverlaps_PartialYOverlapKept(t *testing.T) {@@ -247,7 +271,7 @@ func TestDedupSubstringOverlaps_PartialYOverlapKept(t *testing.T) {// beyond the containing box (to either side) must be KEPT — it is not an OCR// fragment of the container (e.g. an adjacent-column line whose text happens// to be a whitespace-normalized substring of the paragraph). Only a box fully-// inside on BOTH axes is collapsed. This pins the boxInside hardening that the+// inside on BOTH axes is collapsed. This pins the boxInsideTolerant hardening that the// whitespace-insensitive match otherwise leaves exposed (a plain horizontal// intersect used to pass the X check).func TestDedupSubstringOverlaps_HorizontalOverhangKept(t *testing.T) {@@ -290,7 +314,7 @@ func TestDedupSubstringOverlaps_TallerFragmentKept(t *testing.T) {// ("-name:" vs "- name:") normalize to the SAME text and must collapse when// geometrically contained. The legacy `len(ai) == len(aj)` skip would have// kept the duplicate; whitespace normalization makes the two look identical,-// and boxInside decides the containment.+// and boxInsideTolerant decides the containment.func TestDedupSubstringOverlaps_WhitespaceInsensitive_EqualAfterNorm(t *testing.T) {outer := pdf.TextBox{Text: "-name:", // OCR recognizer stripped the space after '-'diff --git a/internal/deepdoc/parser/pdf/layout/layout.go b/internal/deepdoc/parser/pdf/layout/layout.goindex 27c0dd011c..6ee978e5bd 100644--- a/internal/deepdoc/parser/pdf/layout/layout.go+++ b/internal/deepdoc/parser/pdf/layout/layout.go@@ -139,9 +139,9 @@ func DedupIdenticalText(boxes []pdf.TextBox) []pdf.TextBox {// "⽂章中提到"). Byte-wise matching would miss the fragment relation and the// inner box survives to NaiveVerticalMerge, which concatenates it into the// paragraph — duplicating text (the ocr_real text gaps: plugin-daemon,-// RAG分词, 三国人物). Geometry (boxInside) remains the actual containment-// proof; whitespace normalization only makes the fragment check robust to the-// char-vs-OCR space divergence.+// RAG分词, 三国人物). Geometry (boxInsideTolerant) remains the actual+// containment proof; whitespace normalization only makes the fragment check+// robust to the char-vs-OCR space divergence.func DedupSubstringOverlaps(boxes []pdf.TextBox) []pdf.TextBox {drop := make([]bool, len(boxes))// Precompute the whitespace-normalized text once per box. The inner loop@@ -183,12 +183,12 @@ func DedupSubstringOverlaps(boxes []pdf.TextBox) []pdf.TextBox {// the smaller, contained box, so the taller container is kept.if len(ni) >= len(nj) && strings.Contains(ni, nj) {// j's text is a substring of i's -> drop j only if j sits inside i.- if boxInside(boxes[j], boxes[i]) {+ if boxInsideTolerant(boxes[j], boxes[i]) {drop[j] = true}} else if len(nj) > len(ni) && strings.Contains(nj, ni) {// i's text is a substring of j's -> drop i only if i sits inside j.- if boxInside(boxes[i], boxes[j]) {+ if boxInsideTolerant(boxes[i], boxes[j]) {drop[i] = truebreak}@@ -220,18 +220,32 @@ func dedupNormText(s string) string {return b.String()}-// boxInside reports whether inner is FULLY contained within outer on BOTH-// axes. It confirms an OCR substring fragment sits inside the box whose text-// contains it — not merely sharing a Y band at a different column, nor poking-// out horizontally beyond the container. Requiring horizontal containment-// stops a whitespace-normalized substring match from dropping a box that-// extends past the container's X range (e.g. an adjacent-column line whose-// text happens to be a substring of the paragraph's).-func boxInside(inner, outer pdf.TextBox) bool {- if inner.Top < outer.Top || inner.Bottom > outer.Bottom {+// dedupYTolerancePt tolerates a small Y-boundary overshoot of an OCR+// double-detection fragment. Such fragments sit on the SAME text line as their+// container but their detected Y bounds jitter by ~0.5-2pt of detection noise+// (observed on Rag Flow Usage / 三国人物); strict Y containment then misses+// them and the duplicate text leaks into the output after TextMerge. We only+// relax Y (never X) and only inside DedupSubstringOverlaps, where the+// text-substring precondition already proves the box is an OCR duplicate — so+// a few points of Y noise must not keep it. The tolerance stays far below a+// full line height, so genuine adjacent-line or partial-overlap boxes (kept by+// the #18568 guards) are never collapsed here.+const dedupYTolerancePt = 3.0++// boxInsideTolerant reports whether inner is contained within outer: X is+// strict, Y allows up to dedupYTolerancePt of overshoot. It confirms an OCR+// substring fragment sits inside the box whose text contains it — not merely+// sharing a Y band at a different column, nor poking out horizontally beyond+// the container. Requiring horizontal containment stops a whitespace-+// normalized substring match from dropping a box that extends past the+// container's X range (e.g. an adjacent-column line whose text happens to be a+// substring of the paragraph's). The Y tolerance absorbs double-detection+// noise; X stays strict so adjacent-column duplicates remain kept.+func boxInsideTolerant(inner, outer pdf.TextBox) bool {+ if inner.X0 < outer.X0 || inner.X1 > outer.X1 {return false}- if inner.X0 < outer.X0 || inner.X1 > outer.X1 {+ if inner.Top < outer.Top-dedupYTolerancePt || inner.Bottom > outer.Bottom+dedupYTolerancePt {return false}return truediff --git a/internal/deepdoc/parser/pdf/match_chars_test.go b/internal/deepdoc/parser/pdf/match_chars_test.gonew file mode 100644index 0000000000..05683957ee--- /dev/null+++ b/internal/deepdoc/parser/pdf/match_chars_test.go@@ -0,0 +1,221 @@+//go:build cgo++package pdf++import (+ "strings"+ "testing"++ pdftype "ragflow/internal/deepdoc/parser/pdf/type"+)++// TestMatchCharsToBoxes_AreaTieBreak guards the fix for ocr_real RAG分词+// text doubling. When an OCR detector over-segments a line into a full-line+// container box AND a smaller fragment box fully contained in it, a glyph+// that is fully inside both boxes yields an equal overlap ratio (1.0) for+// each. The greedy assignment must prefer the LARGER (container) box on a+// tie, so the fragment cannot steal the glyph and truncate the container.+//+// The previous `>=`-with-last-wins rule let the smaller fragment win, which+// truncated the container (dropped the trailing 么), after which+// DedupSubstringOverlaps could no longer recognise the fragment as a+// substring and NaiveVerticalMerge glued it back on — duplicating the text.+func TestMatchCharsToBoxes_AreaTieBreak(t *testing.T) {+ // Container: full-line box (large area).+ container := ocrDetectBox{+ box: pdftype.TextBox{X0: 10, X1: 500, Top: 560, Bottom: 589},+ x0: 10, y0: 560, x1: 500, y1: 589,+ }+ // Fragment: smaller box fully contained in the container.+ fragment := ocrDetectBox{+ box: pdftype.TextBox{X0: 10, X1: 500, Top: 577, Bottom: 589},+ x0: 10, y0: 577, x1: 500, y1: 589,+ }+ boxes := []ocrDetectBox{container, fragment}++ // Chars: one belongs only to the container (above the fragment's top),+ // and one (么) is fully inside BOTH boxes -> overlap ratio 1.0 for each.+ // Glyph heights (~12pt) are realistic: the container box is ~29pt tall, so+ // the existing char-height filter (>=0.7 height mismatch) does NOT drop the+ // glyph — only the area tie-break decides which box wins.+ chars := []pdftype.TextChar{+ {Text: "在", X0: 20, X1: 32, Top: 565, Bottom: 575}, // container only+ {Text: "么", X0: 100, X1: 112, Top: 577, Bottom: 589}, // inside both+ }++ got := matchCharsToBoxes(boxes, chars)++ if len(got) != 2 {+ t.Fatalf("expected 2 box char groups, got %d", len(got))+ }++ // The trailing glyph must land in the container (box 0), never the+ // fragment (box 1).+ var inContainer, inFragment bool+ for _, c := range got[0] {+ if c.Text == "么" {+ inContainer = true+ }+ }+ for _, c := range got[1] {+ if c.Text == "么" {+ inFragment = true+ }+ }++ if !inContainer {+ t.Errorf("trailing glyph 么 was NOT assigned to the container box; doubling regression risk")+ }+ if inFragment {+ t.Errorf("trailing glyph 么 was assigned to the smaller fragment box; this is exactly the over-segmentation tie-break bug")+ }++ // The container-only char stays in the container.+ var containerOnly bool+ for _, c := range got[0] {+ if c.Text == "在" {+ containerOnly = true+ }+ }+ if !containerOnly {+ t.Errorf("container-only glyph 在 was not assigned to the container box")+ }+}++// TestMatchCharsToBoxes_FullyContainedSmallChar guards the fix for the+// ocr_real plugin-daemon "certifi missing" divergence. A small inline code+// span (e.g. "certifi", ~8pt ink height) sits fully inside a tall detect box+// (~36pt, spanning two text lines). The char-height filter+// (parser_ocr.go:263, mirroring pdf_parser.py:798) drops it because+// |8-36|/36 >= 0.7 — but the char is geometrically fully inside the box, so+// it cannot belong to an adjacent line and must be kept. The Python golden+// keeps such glyphs (plugin-daemon box[16] carries "certifi"), so Go must+// too.+//+// Fix: a fully-contained small glyph (overlap ≈ 1.0, ratio < 0.9) is deferred+// and re-kept when the box ALSO carries a non-space normal-height char — the+// box is a real text line with an inline span. Cross-line chars only partially+// overlap a tall box (ratio << 1.0), so they are still filtered — no bleed.+func TestMatchCharsToBoxes_FullyContainedSmallChar(t *testing.T) {+ // Tall detect box (height ~35.7), like plugin-daemon box[16].+ tallBox := ocrDetectBox{+ box: pdftype.TextBox{X0: 70, X1: 763, Top: 492.6, Bottom: 528.3},+ x0: 70, y0: 492.6, x1: 763, y1: 528.3,+ }+ boxes := []ocrDetectBox{tallBox}++ // `在`: body text, ~12pt, fully inside the tall box. Passes the filter+ // regardless (|12-35.7|/35.7 = 0.66 < 0.7) — control case.+ // `c`/`e`: small inline code span, ~8.3pt, ALSO fully inside the tall box.+ // |8.3-35.7|/35.7 = 0.77 >= 0.7 -> dropped by the raw filter, but it is+ // fully contained, so the fix must keep it (mirrors Python).+ chars := []pdftype.TextChar{+ {Text: "在", X0: 100, X1: 112, Top: 500, Bottom: 512}, // h=12, inside+ {Text: "c", X0: 667, X1: 675, Top: 497.4, Bottom: 505.7}, // h=8.3, inside (certifi)+ {Text: "e", X0: 680, X1: 688, Top: 497.4, Bottom: 505.7}, // h=8.3, inside (certifi)+ }++ got := matchCharsToBoxes(boxes, chars)++ if len(got) != 1 {+ t.Fatalf("expected 1 box char group, got %d", len(got))+ }++ gotTexts := make(map[string]bool)+ for _, c := range got[0] {+ gotTexts[c.Text] = true+ }++ if !gotTexts["在"] {+ t.Errorf("body glyph 在 was dropped from the box")+ }+ // Pre-fix: `c` and `e` are dropped by the height filter (regression guard).+ if !gotTexts["c"] {+ t.Errorf("small inline glyph c (fully contained, h=8.3) was dropped; certifi-style text loss regression")+ }+ if !gotTexts["e"] {+ t.Errorf("small inline glyph e (fully contained, h=8.3) was dropped; certifi-style text loss regression")+ }+}++// TestMatchCharsToBoxes_FullyContainedTinyDropped locks the other boundary of+// the containment guard: a FULLY CONTAINED char that is extremely tiny+// (height <= 10% of the box, ratio >= 0.9) must STILL be dropped by the+// height gate — it is baseline noise (e.g. a 1pt glyph in a 40pt box), not+// inline content. This preserves TestOCRMergeChars_HeightGate's contract.+func TestMatchCharsToBoxes_FullyContainedTinyDropped(t *testing.T) {+ box := ocrDetectBox{+ box: pdftype.TextBox{X0: 0, X1: 90, Top: 0, Bottom: 120},+ x0: 0, y0: 0, x1: 90, y1: 120,+ }+ boxes := []ocrDetectBox{box}+ // h=1 vs bh=120 -> ratio 0.992 >= 0.9, fully contained (overlap 1.0).+ chars := []pdftype.TextChar{+ {X0: 5, X1: 10, Top: 40, Bottom: 41, Text: "t"},+ }+ got := matchCharsToBoxes(boxes, chars)+ if len(got) != 1 {+ t.Fatalf("expected 1 box group, got %d", len(got))+ }+ if len(got[0]) != 0 {+ t.Errorf("extremely tiny fully-contained glyph must be dropped by the height gate, got %+v", got[0])+ }+}++// TestMatchCharsToBoxes_DeferredSpaceOnlyNormalDropped locks the boundary that+// kept dell-configuration-services-sd-zh-tw (and the other real-PDF set) from+// regressing: a box whose ONLY normal-height chars are SPACES must NOT absorb+// deferred inline glyphs. The real line text lives in a tighter neighbor box+// there, so the small glyphs are another line's content — leaving the box+// empty lets buildTextBoxes' OCR fallback recognize the true line from the+// image. Without this guard, plugin-daemon's certifi fix added stray glyphs to+// space-only boxes and dropped gridSim from 100% to 95.9% on that PDF.+func TestMatchCharsToBoxes_DeferredSpaceOnlyNormalDropped(t *testing.T) {+ box := ocrDetectBox{+ box: pdftype.TextBox{X0: 0, X1: 90, Top: 0, Bottom: 40},+ x0: 0, y0: 0, x1: 90, y1: 40,+ }+ boxes := []ocrDetectBox{box}+ // Space char: ratio < 0.7 (normal), but carries no content.+ // "x": small glyph, fully contained, ratio in [0.7, 0.9) -> deferred.+ chars := []pdftype.TextChar{+ {X0: 5, X1: 6, Top: 5, Bottom: 20, Text: " "},+ {X0: 10, X1: 16, Top: 6, Bottom: 12, Text: "x"}, // h=6 vs bh=40 -> ratio 0.85+ }+ got := matchCharsToBoxes(boxes, chars)+ if len(got) != 1 {+ t.Fatalf("expected 1 box group, got %d", len(got))+ }+ for _, c := range got[0] {+ if strings.TrimSpace(c.Text) != "" {+ t.Errorf("deferred glyph %q must be dropped when the box's only normal chars are spaces", c.Text)+ }+ }+}++// TestMatchCharsToBoxes_PartialOverlapHeightFiltered locks that a char which+// only PARTIALLY overlaps its assigned box (overlap < 0.95) is STILL dropped+// by the height gate when its height differs from the box by >=70% — it is an+// adjacent-line glyph poking into the box, not inline content. The deferred+// inline-glyph path requires full containment (bestOverlap >= 0.95), so a+// partial overlap must never be re-kept there.+func TestMatchCharsToBoxes_PartialOverlapHeightFiltered(t *testing.T) {+ box := ocrDetectBox{+ box: pdftype.TextBox{X0: 0, X1: 90, Top: 0, Bottom: 40},+ x0: 0, y0: 0, x1: 90, y1: 40,+ }+ boxes := []ocrDetectBox{box}+ // h=5 vs bh=40 -> ratio 0.875 >= 0.7; the char spans top=-2..3, so only+ // 3 of its 5pt vertically overlap the box (top 0..3) -> overlap 0.6 < 0.95,+ // i.e. NOT fully contained -> deferred path must not absorb it.+ chars := []pdftype.TextChar{+ {X0: 5, X1: 12, Top: -2, Bottom: 3, Text: "x"},+ }+ got := matchCharsToBoxes(boxes, chars)+ if len(got) != 1 {+ t.Fatalf("expected 1 box group, got %d", len(got))+ }+ if len(got[0]) != 0 {+ t.Errorf("partial-overlap height-mismatched glyph must be dropped by the height gate, got %+v", got[0])+ }+}diff --git a/internal/deepdoc/parser/pdf/parser_ocr.go b/internal/deepdoc/parser/pdf/parser_ocr.goindex fd9e2ccf62..3922b57623 100644--- a/internal/deepdoc/parser/pdf/parser_ocr.go+++ b/internal/deepdoc/parser/pdf/parser_ocr.go@@ -226,13 +226,36 @@ func (p *Parser) detectBoxes(ctx context.Context, pageImg image.Image, doc pdf.Dfunc matchCharsToBoxes(boxes []ocrDetectBox, chars []pdf.TextChar) [][]pdf.TextChar {boxChars := make([][]pdf.TextChar, len(boxes))+ // deferred holds fully-contained small glyphs (candidates for inline text)+ // until we know whether the box also carries any normal-height content. A+ // box whose ONLY char-layer chars are small (ratio >= 0.7) must stay empty+ // so buildTextBoxes' OCR fallback can recognize the full line from the+ // image — keeping the small glyphs alone would emit a partial fragment and+ // suppress the OCR fill (三国人物/反间谍法 regressed under that rule).+ deferred := make([][]pdf.TextChar, len(boxes))for _, c := range chars {bestIdx := -1bestOverlap := 1e-6+ bestArea := 0.0for i := range boxes {overlap := charBoxOverlapRatio(c, boxes[i].x0, boxes[i].x1, boxes[i].y0, boxes[i].y1)- if overlap >= bestOverlap {+ if overlap < bestOverlap {+ continue+ }+ area := (boxes[i].x1 - boxes[i].x0) * (boxes[i].y1 - boxes[i].y0)+ // Tie-break: when a char is fully inside several boxes (a full-line+ // box and a contained OCR fragment that over-segments it), prefer+ // the LARGER container so the fragment cannot steal the glyph and+ // truncate the container. Mirrors Python's Recognizer.find_overlapped+ // (recognizer.py:223), which keeps the max-overlapped (largest-area)+ // box on ties via a strict `>`. The previous `>=`-with-last-wins+ // rule let the smaller fragment win, truncating the container; after+ // DedupSubstringOverlaps could no longer recognise the fragment as a+ // substring, NaiveVerticalMerge glued it back on and duplicated text+ // (ocr_real RAG分词 doubling).+ if overlap > bestOverlap || area > bestArea {bestOverlap = overlap+ bestArea = areabestIdx = i}}@@ -244,10 +267,37 @@ func matchCharsToBoxes(boxes []ocrDetectBox, chars []pdf.TextChar) [][]pdf.TextCch = 1}bh := boxes[bestIdx].y1 - boxes[bestIdx].y0- if math.Abs(ch-bh)/math.Max(ch, bh) >= 0.7 && c.Text != " " {- continue+ // Char-height filter (mirrors Python pdf_parser.py:798): drop chars+ // whose height differs greatly from the box height — they belong to+ // another line. A fully-contained small glyph (overlap >= 0.95,+ // ratio < 0.9) is an inline-text candidate (e.g. a code span like+ // "certifi", ~8pt, inside a tall two-line detect box ~36pt — the Python+ // golden keeps it, plugin-daemon box[16]): it cannot be from an+ // adjacent line because it lies almost entirely inside the box. It is+ // deferred and re-kept only when the box also carries normal-height+ // content, so an isolated small glyph cannot suppress the OCR fallback.+ ratio := math.Abs(ch-bh) / math.Max(ch, bh)+ if ratio < 0.7 || c.Text == " " {+ boxChars[bestIdx] = append(boxChars[bestIdx], c)+ } else if bestOverlap >= 0.95 && ratio < 0.9 {+ deferred[bestIdx] = append(deferred[bestIdx], c)+ }+ }+ for i := range boxChars {+ // Re-keep the deferred inline glyphs only when the box actually carries+ // a non-space normal-height char: a box whose only normal chars are+ // spaces (the real line text lives in a tighter neighbor box) must not+ // absorb the small glyphs — they are another line's content there.+ hasText := false+ for _, c := range boxChars[i] {+ if strings.TrimSpace(c.Text) != "" {+ hasText = true+ break+ }+ }+ if hasText {+ boxChars[i] = append(boxChars[i], deferred[i]...)}- boxChars[bestIdx] = append(boxChars[bestIdx], c)}return boxChars}diff --git a/internal/deepdoc/parser/pdf/parser_ocr_test.go b/internal/deepdoc/parser/pdf/parser_ocr_test.goindex 529e019844..63e1fe5d5d 100644--- a/internal/deepdoc/parser/pdf/parser_ocr_test.go+++ b/internal/deepdoc/parser/pdf/parser_ocr_test.go@@ -305,12 +305,9 @@ func TestOCRMergeChars_OverlappingBoxes(t *testing.T) {p := NewParser(pdf.DefaultParserConfig())// Box A: PDF x=0..20, y=0..20. Box B: PDF x=10..30, y=0..20.// Overlap zone: x=10..20.- // Char "Y" at PDF x=2..8 → Box A only.- // Char "X" at PDF x=12..18 → overlap zone (both boxes).- // Char "Z" at PDF x=22..28 → Box B only.- //- // Old box-perspective: Box A gets [Y,X], Box B gets [X,Z].- // New char-perspective: Box A gets [Y,X] (best overlap), Box B gets [Z].+ // Char "甲" at PDF x=2..8 → Box A only.+ // Char "乙" at PDF x=12..18 → overlap zone (both boxes).+ // Char "丙" at PDF x=22..28 → Box B only.mock := &MockDocAnalyzer{Healthy: true,OCRBoxes: []pdf.OCRBox{@@ -327,14 +324,18 @@ func TestOCRMergeChars_OverlappingBoxes(t *testing.T) {if len(boxes) != 2 {t.Fatalf("expected 2 boxes, got %d", len(boxes))}- // Tie on equal overlap → later box wins (matching Python's >=).- // "乙" goes to Box B (both overlap=1.0, Box B checked later).- // Box A → "甲", Box B → "乙丙" (sorted by X).- if boxes[0].Text != "甲" {- t.Errorf("box A: expected '甲', got %q", boxes[0].Text)- }- if boxes[1].Text != "乙丙" {- t.Errorf("box B: expected '乙丙', got %q", boxes[1].Text)+ // Tie on equal overlap → FIRST box wins. matchCharsToBoxes keeps the first+ // box that reaches the max overlap via a strict `>` (preferring the larger+ // area on ties), mirroring Python's Recognizer.find_overlapped+ // (recognizer.py:223 `if ov <= max_overlapped: continue` — first-wins;+ // verified empirically). "乙" has overlap 1.0 with both equal-area boxes,+ // so it goes to Box A (the earlier one).+ // Box A → "甲乙", Box B → "丙" (sorted by X).+ if boxes[0].Text != "甲乙" {+ t.Errorf("box A: expected '甲乙', got %q", boxes[0].Text)+ }+ if boxes[1].Text != "丙" {+ t.Errorf("box B: expected '丙', got %q", boxes[1].Text)}}diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_test.goindex a5fb2250ff..590a0ac62a 100644--- a/internal/deepdoc/parser/pdf/pipeline_parity_test.go+++ b/internal/deepdoc/parser/pdf/pipeline_parity_test.go@@ -44,12 +44,15 @@ import (// Python, e.g. table segmentation). Any genuine Go content regression is a// FAIL.func TestPipelineParity(t *testing.T) {- charspyDir := filepath.Join("testdata", "charspy")- pyTextDir := filepath.Join("testdata", "output", "py", "ocr", "text")- dlaDir := filepath.Join("testdata", "output", "py", "ocr", "dla")- tsrDir := filepath.Join("testdata", "output", "py", "ocr", "tsr_raw")- ocrDir := filepath.Join("testdata", "output", "py", "ocr", "ocr")- tablesDir := filepath.Join("testdata", "output", "py", "ocr", "tables")+ // Dataset variant: "" (default) or "ocr" resolve to the built-in layout+ // (charspy/, output/py/ocr/...) for the original 35-PDF fixture set; a+ // custom variant such as "ocr_real" reads its own isolated directories+ // (charspy_ocr_real/, output/py/ocr_real/...), so a second dataset (e.g.+ // real_pdfs/) gets its own dumps and verdicts without touching the default+ // set. See tool.ParityDirsFor.+ dirs := tool.ParityDirsFor(common.GetEnv(common.EnvBatchParityVariant))+ charspyDir, pyTextDir := dirs.Charspy, dirs.Text+ dlaDir, tsrDir, ocrDir, tablesDir := dirs.DLA, dirs.TSRRaw, dirs.OCR, dirs.Tablesentries, err := os.ReadDir(charspyDir)if err != nil {@@ -158,7 +161,7 @@ func TestPipelineParity(t *testing.T) {// Debug aid: BATCH_PARITY_DUMP_GO=1 writes the Go output text next to// the Python golden so failing PDFs can be diffed case by case.if common.GetEnv("BATCH_PARITY_DUMP_GO") != "" {- dumpDir := filepath.Join("testdata", "output", "go", "ocr", "text")+ dumpDir := dirs.GoText_ = os.MkdirAll(dumpDir, 0o755)_ = os.WriteFile(filepath.Join(dumpDir, name+".txt"), []byte(goText.String()), 0o644)}diff --git a/internal/deepdoc/parser/pdf/tool/parity_dirs.go b/internal/deepdoc/parser/pdf/tool/parity_dirs.gonew file mode 100644index 0000000000..3c3e8ee50e--- /dev/null+++ b/internal/deepdoc/parser/pdf/tool/parity_dirs.go@@ -0,0 +1,66 @@+package tool++import (+ "path/filepath"++ "ragflow/internal/common"+)++// ParityDirs holds every testdata directory the pipeline-parity harness+// reads/writes for one dataset variant. The variant isolates a second set of+// PDFs (e.g. real_pdfs/) from the default 35-PDF fixture set so dumps and+// parity verdicts never collide.+type ParityDirs struct {+ // Charspy is the Python pdfplumber chars dump directory.+ Charspy string+ // Text is Python's per-box text golden.+ Text string+ // DLA is Python's layout intermediate replay input.+ DLA string+ // TSRRaw is Python's table-structure-recognition replay input.+ TSRRaw string+ // OCR is Python's raw OCR detect replay input.+ OCR string+ // Tables is Python's table grid golden.+ Tables string+ // GoText is where the harness dumps Go output text for diffing (BATCH_PARITY_DUMP_GO).+ GoText string+}++// ParityDirsFor maps a dataset variant to its testdata directories. The+// default variant ("" or "ocr") resolves to the built-in layout (charspy/,+// output/py/ocr/...); a custom variant such as "ocr_real" moves every artifact+// under a variant suffix (charspy_ocr_real/, output/py/ocr_real/...) so two+// datasets never share a file.+//+// For a custom variant the root directory can be redirected via+// BATCH_PARITY_DATA_ROOT (e.g. a shared directory outside the worktree so+// multiple worktrees reuse one dump). The default variant always stays under+// the local "testdata".+func ParityDirsFor(variant string) ParityDirs {+ if variant == "" {+ variant = "ocr"+ }+ root := "testdata"+ if variant != "ocr" {+ if r := common.GetEnv(common.EnvBatchParityDataRoot); r != "" {+ root = r+ }+ }+ // The charspy dir is the only artifact whose name does not carry the+ // variant suffix for the default variant (charspy/), matching the layout+ // the dumps were generated with; custom variants append the suffix.+ charspyDir := "charspy"+ if variant != "ocr" {+ charspyDir = "charspy_" + variant+ }+ return ParityDirs{+ Charspy: filepath.Join(root, charspyDir),+ Text: filepath.Join(root, "output", "py", variant, "text"),+ DLA: filepath.Join(root, "output", "py", variant, "dla"),+ TSRRaw: filepath.Join(root, "output", "py", variant, "tsr_raw"),+ OCR: filepath.Join(root, "output", "py", variant, "ocr"),+ Tables: filepath.Join(root, "output", "py", variant, "tables"),+ GoText: filepath.Join(root, "output", "go", variant, "text"),+ }+}diff --git a/internal/deepdoc/parser/pdf/tool/parity_dirs_test.go b/internal/deepdoc/parser/pdf/tool/parity_dirs_test.gonew file mode 100644index 0000000000..788c5af266--- /dev/null+++ b/internal/deepdoc/parser/pdf/tool/parity_dirs_test.go@@ -0,0 +1,73 @@+package tool++import (+ "path/filepath"+ "testing"+)++// TestParityDirsForDefaultVariant pins the default directory layout: the+// default variant ("" or "ocr") must resolve exactly to the built-in paths+// (charspy/, output/py/ocr/...).+func TestParityDirsForDefaultVariant(t *testing.T) {+ for _, v := range []string{"", "ocr"} {+ d := ParityDirsFor(v)+ want := ParityDirs{+ Charspy: filepath.Join("testdata", "charspy"),+ Text: filepath.Join("testdata", "output", "py", "ocr", "text"),+ DLA: filepath.Join("testdata", "output", "py", "ocr", "dla"),+ TSRRaw: filepath.Join("testdata", "output", "py", "ocr", "tsr_raw"),+ OCR: filepath.Join("testdata", "output", "py", "ocr", "ocr"),+ Tables: filepath.Join("testdata", "output", "py", "ocr", "tables"),+ GoText: filepath.Join("testdata", "output", "go", "ocr", "text"),+ }+ if d != want {+ t.Errorf("ParityDirsFor(%q) = %+v, want %+v", v, d, want)+ }+ }+}++// TestParityDirsForCustomVariant pins the variant-isolated layout: a custom+// variant moves every artifact under its own subdirectory so a second dataset+// (e.g. real_pdfs) never collides with the default 35-PDF set.+func TestParityDirsForCustomVariant(t *testing.T) {+ d := ParityDirsFor("ocr_real")+ want := ParityDirs{+ Charspy: filepath.Join("testdata", "charspy_ocr_real"),+ Text: filepath.Join("testdata", "output", "py", "ocr_real", "text"),+ DLA: filepath.Join("testdata", "output", "py", "ocr_real", "dla"),+ TSRRaw: filepath.Join("testdata", "output", "py", "ocr_real", "tsr_raw"),+ OCR: filepath.Join("testdata", "output", "py", "ocr_real", "ocr"),+ Tables: filepath.Join("testdata", "output", "py", "ocr_real", "tables"),+ GoText: filepath.Join("testdata", "output", "go", "ocr_real", "text"),+ }+ if d != want {+ t.Errorf("ParityDirsFor(\"ocr_real\") = %+v, want %+v", d, want)+ }+}++// TestParityDirsForCustomVariantDataRoot pins the shared-data-root redirect:+// when BATCH_PARITY_DATA_ROOT is set, a custom variant resolves every+// artifact under that root (so multiple worktrees reuse one dump). The+// default variant must ignore the env and stay under local testdata.+func TestParityDirsForCustomVariantDataRoot(t *testing.T) {+ t.Setenv("BATCH_PARITY_DATA_ROOT", "/srv/parity-data")+ d := ParityDirsFor("ocr_real")+ want := ParityDirs{+ Charspy: filepath.Join("/srv/parity-data", "charspy_ocr_real"),+ Text: filepath.Join("/srv/parity-data", "output", "py", "ocr_real", "text"),+ DLA: filepath.Join("/srv/parity-data", "output", "py", "ocr_real", "dla"),+ TSRRaw: filepath.Join("/srv/parity-data", "output", "py", "ocr_real", "tsr_raw"),+ OCR: filepath.Join("/srv/parity-data", "output", "py", "ocr_real", "ocr"),+ Tables: filepath.Join("/srv/parity-data", "output", "py", "ocr_real", "tables"),+ GoText: filepath.Join("/srv/parity-data", "output", "go", "ocr_real", "text"),+ }+ if d != want {+ t.Errorf("ParityDirsFor(\"ocr_real\") with BATCH_PARITY_DATA_ROOT = %+v, want %+v", d, want)+ }++ // Default variant must NOT be redirected by the env.+ def := ParityDirsFor("")+ if def.Charspy != filepath.Join("testdata", "charspy") {+ t.Errorf("default variant Charspy = %q, want %q (env must be ignored)", def.Charspy, filepath.Join("testdata", "charspy"))+ }+}