RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(deepdoc): Python-intermediate replay parity harness + table cell-fill fixes (#18516)
Jackc90a0b7a8798
diff --git a/internal/deepdoc/parser/pdf/layout/dedup_test.go b/internal/deepdoc/parser/pdf/layout/dedup_test.gonew file mode 100644index 0000000000..95aa5eaa29--- /dev/null+++ b/internal/deepdoc/parser/pdf/layout/dedup_test.go@@ -0,0 +1,252 @@+package layout++import (+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// TestDedupIdenticalText locks that boxes carrying TRIM-EQUAL text (identical+// after strings.TrimSpace — trailing whitespace normalized) on the SAME page+// with DISJOINT Y bands are collapsed to one. OCR repeatedly detects the same+// text at different Y positions (09_crosspage_paragraph detects each paragraph+// 5-6x per page, all disjoint), and Python's downstream merge collapses these;+// Go must too, or the replay output duplicates paragraphs.+func TestDedupIdenticalText(t *testing.T) {+ long := "paragraph one with enough words to qualify as a real paragraph duplicate text for collapsing"+ boxes := []pdf.TextBox{+ {Text: long, PageNumber: 0, Top: 100, Bottom: 115, X0: 60, X1: 520, IsOCR: true},+ {Text: long, PageNumber: 0, Top: 300, Bottom: 315, X0: 60, X1: 520, IsOCR: true}, // disjoint dup -> drop+ {Text: long, PageNumber: 1, Top: 100, Bottom: 115, X0: 60, X1: 520, IsOCR: true}, // other page -> keep+ {Text: "paragraph two", PageNumber: 0, Top: 200, Bottom: 215, IsOCR: true},+ }+ got := DedupIdenticalText(boxes)+ if len(got) != 3 {+ t.Fatalf("want 3 boxes (1 disjoint dup dropped), got %d: %+v", len(got), got)+ }+ if got[0].Text != long || got[1].Text != long {+ t.Fatalf("page-0 and page-1 copies must both be kept in order")+ }+ if got[2].Text != "paragraph two" {+ t.Fatalf("want 'paragraph two' third, got %q", got[2].Text)+ }+}++// TestDedupIdenticalText_YOverlap ensures overlapping boxes are kept: two+// columns / adjacent lines on the same page legitimately share text (e.g.+// eval_three_wide has 3 columns at the same Y), so only disjoint duplicates+// are collapsed.+func TestDedupIdenticalText_YOverlap(t *testing.T) {+ boxes := []pdf.TextBox{+ {Text: "line xxxxxxxxxxxxx", PageNumber: 0, Top: 104, Bottom: 116, X0: 60, X1: 260, IsOCR: true},+ {Text: "line xxxxxxxxxxxxx", PageNumber: 0, Top: 104, Bottom: 116, X0: 320, X1: 520, IsOCR: true}, // 2nd column -> keep+ {Text: "line xxxxxxxxxxxxx", PageNumber: 0, Top: 118, Bottom: 130, X0: 60, X1: 260, IsOCR: true}, // overlapping neighbor -> keep+ }+ got := DedupIdenticalText(boxes)+ if len(got) != 3 {+ t.Fatalf("overlapping same-text boxes must all be kept, got %d", len(got))+ }+}++// TestDedupIdenticalText_WhitespaceSensitive ensures trimming does not merge+// boxes that differ only by trailing spaces into a false duplicate.+func TestDedupIdenticalText_WhitespaceSensitive(t *testing.T) {+ long := "a sufficiently long repeated sentence that qualifies as a paragraph"+ boxes := []pdf.TextBox{+ {Text: long, PageNumber: 0, Top: 10, Bottom: 20, X0: 60, X1: 520, IsOCR: true},+ {Text: long + " ", PageNumber: 0, Top: 90, Bottom: 100, X0: 60, X1: 520, IsOCR: true}, // 80pt gap > 4x height+ }+ got := DedupIdenticalText(boxes)+ if len(got) != 1 {+ t.Fatalf("trimmed-equal texts should be treated as duplicates, got %d", len(got))+ }+}++// TestDedupIdenticalText_ShortTextKept locks that SHORT identical texts+// (e.g. the repeated keyword 'Transformer' in 16_dense_cjk) are NOT collapsed —+// short repeated content is real document text, not an OCR paragraph duplicate.+func TestDedupIdenticalText_ShortTextKept(t *testing.T) {+ boxes := []pdf.TextBox{+ {Text: "Transformer", PageNumber: 0, Top: 100, Bottom: 112, X0: 60, X1: 120, IsOCR: true},+ {Text: "Transformer", PageNumber: 0, Top: 300, Bottom: 312, X0: 60, X1: 120, IsOCR: true}, // disjoint Y, short -> keep+ }+ got := DedupIdenticalText(boxes)+ if len(got) != 2 {+ t.Fatalf("short identical text must be kept, got %d boxes", len(got))+ }+}++// TestDedupIdenticalText_AdjacentRepeatsKept locks that identical lines only+// ~1x their height apart (adjacent rows) are NOT collapsed — they are real+// document content (eval_two_narrow_gutter has 'linexxx' rows 44pt apart),+// unlike OCR pseudo-duplicates detected with a large rolling stride (89-136pt).+func TestDedupIdenticalText_AdjacentRepeatsKept(t *testing.T) {+ row := "line xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxX"+ boxes := []pdf.TextBox{+ {Text: row, PageNumber: 0, Top: 106, Bottom: 121, X0: 60, X1: 260, IsOCR: true},+ {Text: row, PageNumber: 0, Top: 150, Bottom: 165, X0: 60, X1: 260, IsOCR: true}, // 44pt gap (< 4x height) -> keep+ }+ got := DedupIdenticalText(boxes)+ if len(got) != 2 {+ t.Fatalf("adjacent identical rows must be kept, got %d boxes", len(got))+ }+}++// TestDedupIdenticalText_StridedPseudoDuplicate collapses identical text far+// apart (>4x height) at the same X — the OCR rolling-stride duplicate+// (eval_single_wide / 09_crosspage_paragraph).+func TestDedupIdenticalText_StridedPseudoDuplicate(t *testing.T) {+ long := "a sufficiently long repeated sentence that qualifies as a paragraph duplicate"+ boxes := []pdf.TextBox{+ {Text: long, PageNumber: 0, Top: 100, Bottom: 115, X0: 60, X1: 520, IsOCR: true},+ {Text: long, PageNumber: 0, Top: 300, Bottom: 315, X0: 60, X1: 520, IsOCR: true}, // 200pt gap -> drop+ }+ got := DedupIdenticalText(boxes)+ if len(got) != 1 {+ t.Fatalf("strided pseudo-duplicate must be dropped, got %d boxes", len(got))+ }+}++// TestDedupIdenticalText_CharPathKept locks the key invariant: char-path+// digital-PDF boxes (IsOCR=false) are NEVER de-duplicated, even when they are+// byte-identical, far apart, and in the same column. Dropping them would+// silently lose legitimate repeated content (repeated clauses / headings) that+// Python's char path keeps — a regression the IsOCR scoping must prevent.+// With the scoping removed this test fails (gets 1 instead of 2).+func TestDedupIdenticalText_CharPathKept(t *testing.T) {+ clause := "保密条款:双方应对在合作中知悉的商业秘密承担保密义务直至保密期限届满"+ boxes := []pdf.TextBox{+ {Text: clause, PageNumber: 0, Top: 100, Bottom: 120, X0: 60, X1: 520}, // char path+ {Text: clause, PageNumber: 0, Top: 1000, Bottom: 1020, X0: 60, X1: 520}, // far-apart repeat, same column+ }+ got := DedupIdenticalText(boxes)+ if len(got) != 2 {+ t.Fatalf("char-path identical repeats must be kept, got %d boxes (lost content?)", len(got))+ }+}++// TestDedupSubstringOverlaps locks that a box whose text is a CONTIGUOUS+// SUBSTRING of another same-page box, and whose Y band overlaps it, is+// collapsed — OCR detects both a full paragraph and its middle fragment (e.g.+// 01_english_simple box1 y=(105,166) full paragraph + box2 y=(119,132)+// "language models. When a user asks..."), and Python drops the fragment.+func TestDedupSubstringOverlaps(t *testing.T) {+ full := pdf.TextBox{+ Text: "Retrieval-Augmented Generation (RAG) is a technique that combines information retrieval with large language models. When a user asks a question",+ PageNumber: 0, Top: 100, Bottom: 160, X0: 60, X1: 520, IsOCR: true,+ }+ frag := pdf.TextBox{+ Text: "language models. When a user asks a question",+ PageNumber: 0, Top: 115, Bottom: 130, X0: 60, X1: 520, IsOCR: true, // overlaps the full box+ }+ got := DedupSubstringOverlaps([]pdf.TextBox{full, frag})+ if len(got) != 1 {+ t.Fatalf("overlapping substring fragment must be dropped, got %d boxes", len(got))+ }+ if got[0].Text != full.Text {+ t.Fatalf("the full paragraph must be kept, got %q", got[0].Text)+ }+}++// TestDedupSubstringOverlaps_DisjointYKept ensures a substring box at a+// DISJOINT Y position is kept — a real repeated heading or sentence is legal.+func TestDedupSubstringOverlaps_DisjointYKept(t *testing.T) {+ full := pdf.TextBox{+ Text: "Conclusion summary of the whole document body text", PageNumber: 0, Top: 100, Bottom: 115, IsOCR: true,+ }+ repeat := pdf.TextBox{+ Text: "Conclusion summary", PageNumber: 0, Top: 300, Bottom: 315, IsOCR: true, // disjoint Y+ }+ got := DedupSubstringOverlaps([]pdf.TextBox{full, repeat})+ if len(got) != 2 {+ t.Fatalf("disjoint-Y substring must be kept, got %d boxes", len(got))+ }+}++// TestDedupSubstringOverlaps_DifferentColumnKept ensures a substring-like text+// in a DIFFERENT column (disjoint X) is kept — two columns can carry similar+// 'linexxx' fragments at the same Y (eval_two_wide_gutter). Only fragments at+// the same X location (true OCR duplicates) are collapsed.+func TestDedupSubstringOverlaps_DifferentColumnKept(t *testing.T) {+ colA := pdf.TextBox{+ Text: "line xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", PageNumber: 0, Top: 100, Bottom: 115, X0: 60, X1: 260, IsOCR: true,+ }+ colB := pdf.TextBox{+ Text: "line xxxxxxxxxxxxxxxxxxxxxxxxxxxx", PageNumber: 0, Top: 100, Bottom: 115, X0: 320, X1: 520, IsOCR: true, // disjoint X+ }+ got := DedupSubstringOverlaps([]pdf.TextBox{colA, colB})+ if len(got) != 2 {+ t.Fatalf("same-Y different-column substring must be kept, got %d boxes", len(got))+ }+}++// TestDedupSubstringOverlaps_AdjacentLinesKept ensures two 'linexxx' boxes on+// ADJACENT lines (Y only touches at the boundary, overlap << 80%) are kept —+// they are distinct rows, not an OCR fragment of one another.+func TestDedupSubstringOverlaps_AdjacentLinesKept(t *testing.T) {+ a := pdf.TextBox{+ Text: "line xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", PageNumber: 0, Top: 100, Bottom: 112, X0: 60, X1: 260, IsOCR: true,+ }+ b := pdf.TextBox{+ Text: "line xxxxxxxxxxxxxxxxxxxxxxxxxxxx", PageNumber: 0, Top: 113, Bottom: 125, X0: 60, X1: 260, IsOCR: true, // 1pt overlap, 0.1 < 0.8+ }+ got := DedupSubstringOverlaps([]pdf.TextBox{a, b})+ if len(got) != 2 {+ t.Fatalf("adjacent-line substring must be kept, got %d boxes", len(got))+ }+}++// TestDedupSubstringOverlaps_PartialYOverlapKept locks the Y-containment+// boundary: a substring box that PARTIALLY overlaps the containing box in Y+// (extends below it) is KEPT — it is an adjacent-line fragment, not a contained+// duplicate. Only a fragment FULLY inside the containing box (boxInside) is+// collapsed. With the height-vs-text decoupled guard this case is also kept;+// the test pins the boundary against future over-collapsing.+func TestDedupSubstringOverlaps_PartialYOverlapKept(t *testing.T) {+ full := pdf.TextBox{+ Text: "the quick brown fox jumps over the lazy dog near the river bank", PageNumber: 0, Top: 100, Bottom: 130, X0: 60, X1: 520, IsOCR: true,+ }+ frag := pdf.TextBox{+ Text: "near the river", PageNumber: 0, Top: 120, Bottom: 150, X0: 60, X1: 520, IsOCR: true, // extends below full -> not inside+ }+ got := DedupSubstringOverlaps([]pdf.TextBox{full, frag})+ if len(got) != 2 {+ t.Fatalf("partial-Y-overlap substring must be kept (not fully inside), got %d boxes", len(got))+ }+}++// TestDedupSubstringOverlaps_TallerFragmentKept locks the defensive invariant:+// a PHYSICALLY TALLER box whose SHORT text is a substring of a shorter, contained+// box's longer text is NOT silently dropped. Collapse requires the substring-text+// box to be geometrically INSIDE the text-containing box; here the substring box+// is the taller CONTAINER, so it is kept. This guards the height-vs-text decoupling+// fix (a taller box must not be dropped just because its text is a substring).+func TestDedupSubstringOverlaps_TallerFragmentKept(t *testing.T) {+ tall := pdf.TextBox{+ Text: "X", PageNumber: 0, Top: 100, Bottom: 160, X0: 60, X1: 520, IsOCR: true, // taller container+ }+ wide := pdf.TextBox{+ Text: "prefix X suffix", PageNumber: 0, Top: 115, Bottom: 130, X0: 60, X1: 520, IsOCR: true, // shorter, inside tall, contains "X"+ }+ got := DedupSubstringOverlaps([]pdf.TextBox{tall, wide})+ if len(got) != 2 {+ t.Fatalf("taller substring box must be kept (only contained fragments are dropped), got %d boxes", len(got))+ }+}++// TestDedupSubstringOverlaps_CharPathKept locks that a char-path box whose text+// is a substring of another char-path box is NEVER collapsed — e.g. a repeated+// heading inside another paragraph's text range on a digital PDF. Only OCR+// pseudo-fragments (IsOCR=true) are collapsed.+func TestDedupSubstringOverlaps_CharPathKept(t *testing.T) {+ full := pdf.TextBox{+ Text: "本协议终止后保密条款继续有效双方仍应承担保密义务", PageNumber: 0, Top: 100, Bottom: 120,+ }+ repeat := pdf.TextBox{+ Text: "保密条款继续有效", PageNumber: 0, Top: 110, Bottom: 118, // nested substring, char path+ }+ got := DedupSubstringOverlaps([]pdf.TextBox{full, repeat})+ if len(got) != 2 {+ t.Fatalf("char-path substring must be kept, got %d boxes (lost content?)", len(got))+ }+}diff --git a/internal/deepdoc/parser/pdf/layout/layout.go b/internal/deepdoc/parser/pdf/layout/layout.goindex 387247e570..9d08343ce2 100644--- a/internal/deepdoc/parser/pdf/layout/layout.go+++ b/internal/deepdoc/parser/pdf/layout/layout.go@@ -19,6 +19,170 @@ import (// ---- Text merge (horizontal) ----+// DedupIdenticalText collapses boxes whose text is byte-identical on the SAME+// page AND whose Y bands are pairwise DISJOINT, keeping the first+// (reading-order) occurrence. OCR repeatedly detects the same text at disjoint+// Y positions (09_crosspage_paragraph detects each paragraph 5-6x per page);+// Python's downstream merge collapses these, so Go must too or the replay+// output duplicates every paragraph. Boxes that OVERLAP in Y are kept: multiple+// columns / adjacent lines on one page legitimately share text (e.g.+// eval_three_wide has 3 columns at the same Y). Different pages keep their own+// copies (a cross-page paragraph appears once per page).+func DedupIdenticalText(boxes []pdf.TextBox) []pdf.TextBox {+ type key struct {+ page int+ text string+ }+ groups := make(map[key][]int, len(boxes))+ for i, b := range boxes {+ // Only OCR boxes are de-duplicated: char-path digital-PDF boxes may+ // legitimately repeat the same text (clauses, headings) and must be+ // kept verbatim — dropping them would silently lose content.+ if !b.IsOCR {+ continue+ }+ t := strings.TrimSpace(b.Text)+ if t == "" {+ continue+ }+ groups[key{b.PageNumber, t}] = append(groups[key{b.PageNumber, t}], i)+ }++ drop := make(map[int]bool, len(boxes))+ for _, idxs := range groups {+ if len(idxs) < 2 {+ continue+ }+ // Short identical texts (e.g. the repeated keyword 'Transformer' in+ // 16_dense_cjk) are real document content, not OCR paragraph+ // duplicates — only paragraphs (>= 20 runes) are collapsed.+ if utf8.RuneCountInString(strings.TrimSpace(boxes[idxs[0]].Text)) < 20 {+ continue+ }+ // OCR pseudo-duplicate: the same text detected repeatedly at disjoint+ // Y positions of the SAME X location, separated by MORE than 4x the box+ // height (a rolling-stride re-detection). Adjacent identical rows+ // (~1x height apart, e.g. eval_two_narrow_gutter) and different columns+ // are real content and are kept.+ gapThreshold := 4 * (boxes[idxs[0]].Bottom - boxes[idxs[0]].Top)+ if gapThreshold <= 0 {+ continue+ }+ hasOverlapY := false+ allOverlapX := true+ allGapOK := true+ for a := 0; a < len(idxs) && !hasOverlapY; a++ {+ for c := a + 1; c < len(idxs); c++ {+ ba, bc := boxes[idxs[a]], boxes[idxs[c]]+ if ba.Bottom > bc.Top && bc.Bottom > ba.Top {+ hasOverlapY = true+ }+ if bc.X1 <= ba.X0 || ba.X1 <= bc.X0 {+ allOverlapX = false+ }+ gap := ba.Top - bc.Top+ if gap < 0 {+ gap = -gap+ }+ if gap <= gapThreshold {+ allGapOK = false+ }+ }+ }+ if !hasOverlapY && allOverlapX && allGapOK {+ for _, idx := range idxs[1:] {+ drop[idx] = true+ }+ }+ }++ out := boxes[:0]+ for i, b := range boxes {+ if drop[i] {+ continue+ }+ out = append(out, b)+ }+ return out+}++// DedupSubstringOverlaps collapses a box whose text is a CONTIGUOUS SUBSTRING+// of another same-page box AND is geometrically contained in that box (X AND Y+// inside), keeping the longer box. OCR detects both a full paragraph and its+// middle fragment at the same location (e.g. 01_english_simple: full paragraph+// y=(105,166) plus fragment "language models. When a user asks..." y=(119,132));+// Python drops the fragment, Go must too or the merged paragraph repeats it. A+// substring box at a disjoint Y OR X (different column, e.g.+// eval_two_wide_gutter) is kept — a real repeated heading or another column's+// text is legal. The containment test is bound to the substring box (not just+// the shorter-height box) so a physically taller box whose short text is a+// substring of a neighbour is never silently dropped.+func DedupSubstringOverlaps(boxes []pdf.TextBox) []pdf.TextBox {+ drop := make([]bool, len(boxes))+ for i := range boxes {+ if drop[i] {+ continue+ }+ // Only OCR-vs-OCR pairs are collapsed (see DedupIdenticalText): a+ // char-path box is never treated as a fragment of another.+ if !boxes[i].IsOCR {+ continue+ }+ ai := strings.TrimSpace(boxes[i].Text)+ if ai == "" {+ continue+ }+ for j := range boxes {+ if i == j || drop[j] || boxes[i].PageNumber != boxes[j].PageNumber || !boxes[j].IsOCR {+ continue+ }+ aj := strings.TrimSpace(boxes[j].Text)+ if aj == "" || len(ai) == len(aj) {+ continue+ }+ // Collapse only when the SUBSTRING-text box is geometrically CONTAINED+ // in the text-containing box. Binding the geometry to the actual+ // substring (not merely the shorter-height box) avoids silently+ // dropping a physically taller box whose short text happens to be a+ // substring of a neighbour — OCR double-detection fragments are always+ // the smaller, contained box, so the taller container is kept.+ if len(ai) > len(aj) && strings.Contains(ai, aj) {+ // j's text is a substring of i's -> drop j only if j sits inside i.+ if boxInside(boxes[j], boxes[i]) {+ drop[j] = true+ }+ } else if len(aj) > len(ai) && strings.Contains(aj, ai) {+ // i's text is a substring of j's -> drop i only if i sits inside j.+ if boxInside(boxes[i], boxes[j]) {+ drop[i] = true+ break+ }+ }+ }+ }+ out := boxes[:0]+ for i, b := range boxes {+ if drop[i] {+ continue+ }+ out = append(out, b)+ }+ return out+}++// boxInside reports whether inner is fully contained within outer in Y and+// overlaps it in X. It confirms an OCR substring fragment sits inside the box+// whose text contains it (not merely shares a Y band at a different column).+func boxInside(inner, outer pdf.TextBox) bool {+ if inner.Top < outer.Top || inner.Bottom > outer.Bottom {+ return false+ }+ if outer.X1 <= inner.X0 || inner.X1 <= outer.X0 {+ return false+ }+ return true+}+// TextMerge horizontally merges adjacent boxes at similar vertical positions.//// Python: pdf_parser.py:888 _text_merge()@@ -85,7 +249,17 @@ func NaiveVerticalMerge(boxes []pdf.TextBox, medianHeights map[int]float64, medi}mh := medianHeights[pg]- if mh <= 0 {+ if pageEnglish[pg] {+ // Python: for is_english documents chars are cleared so+ // mean_height becomes 0 and _naive_vertical_merge skips every+ // pair (gap > 0). Mirror that for English pages DIRECTLY — do not+ // fall back to the (positive) char-derived median height, or real+ // English pages would still merge and 'linexxx' rows (eval_*)+ // concatenate into one giant line. The old guard `if mh <= 0`+ // never fired for real pages because their char-derived median+ // height is always positive.+ mh = 0+ } else if mh <= 0 {mh = util.MedianHeight(bxs)}mw := medianWidths[pg]diff --git a/internal/deepdoc/parser/pdf/layout/vmerge_test.go b/internal/deepdoc/parser/pdf/layout/vmerge_test.gonew file mode 100644index 0000000000..1da48ef7dd--- /dev/null+++ b/internal/deepdoc/parser/pdf/layout/vmerge_test.go@@ -0,0 +1,59 @@+package layout++import (+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// TestVerticalMergeEnglishNoMerge locks that an ENGLISH page with median+// height 0 does NOT vertically merge adjacent lines. Python clears chars for+// is_english documents so mean_height becomes 0 and _naive_vertical_merge+// skips every pair (gap > 0); Go must mirror that, otherwise 'linexxx' rows+// (eval_two_wide_gutter) concatenate into one giant line.+func TestVerticalMergeEnglishNoMerge(t *testing.T) {+ boxes := []pdf.TextBox{+ {Text: "line xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", PageNumber: 0, Top: 100, Bottom: 112, X0: 60, X1: 260},+ {Text: "line xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", PageNumber: 0, Top: 113, Bottom: 125, X0: 60, X1: 260},+ {Text: "line xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", PageNumber: 0, Top: 126, Bottom: 138, X0: 60, X1: 260},+ }+ // English page, median height explicitly 0 -> no vertical merge.+ got := NaiveVerticalMerge(boxes, map[int]float64{0: 0}, map[int]float64{0: 8}, map[int]bool{0: true})+ if len(got) != 3 {+ t.Fatalf("english page with mh=0 must keep 3 separate lines, got %d", len(got))+ }+}++// TestVerticalMergeEnglishNoMergePositiveMedian locks the REAL production path:+// an ENGLISH page whose char-derived median height is POSITIVE (the common case+// for digital English PDFs, where Go keeps embedded chars) must STILL NOT+// vertically merge. The old guard `if mh <= 0 { if pageEnglish ... }` never+// fired for real pages (mh > 0), so English lines were wrongly merged and+// 'linexxx' rows concatenated into one giant line. This reproduces that gap and+// is the regression lock for the fix (mirror Python is_english -> chars=[] ->+// mean_height 0 -> no merge).+func TestVerticalMergeEnglishNoMergePositiveMedian(t *testing.T) {+ boxes := []pdf.TextBox{+ {Text: "line xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", PageNumber: 0, Top: 100, Bottom: 112, X0: 60, X1: 260},+ {Text: "line xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", PageNumber: 0, Top: 113, Bottom: 125, X0: 60, X1: 260},+ {Text: "line xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", PageNumber: 0, Top: 126, Bottom: 138, X0: 60, X1: 260},+ }+ // English page, POSITIVE median height (real char-path case) -> no merge.+ got := NaiveVerticalMerge(boxes, map[int]float64{0: 15}, map[int]float64{0: 8}, map[int]bool{0: true})+ if len(got) != 3 {+ t.Fatalf("english page with positive median height must keep 3 separate lines, got %d", len(got))+ }+}++// TestVerticalMergeNonEnglishKeepsMerge ensures non-English pages still merge+// close adjacent lines (median height > 0), so paragraph assembly is intact.+func TestVerticalMergeNonEnglishKeepsMerge(t *testing.T) {+ boxes := []pdf.TextBox{+ {Text: "first paragraph line one", PageNumber: 0, Top: 100, Bottom: 115, X0: 60, X1: 520},+ {Text: "second line of the same paragraph", PageNumber: 0, Top: 116, Bottom: 131, X0: 60, X1: 520},+ }+ got := NaiveVerticalMerge(boxes, map[int]float64{0: 15}, map[int]float64{0: 8}, map[int]bool{0: false})+ if len(got) != 1 {+ t.Fatalf("non-english page should merge close lines, got %d boxes", len(got))+ }+}diff --git a/internal/deepdoc/parser/pdf/parity_lock.go b/internal/deepdoc/parser/pdf/parity_lock.gonew file mode 100644index 0000000000..02aad7d412--- /dev/null+++ b/internal/deepdoc/parser/pdf/parity_lock.go@@ -0,0 +1,18 @@+package pdf++// parityLockedGridPDFs returns the set of table PDFs that MUST hold full grid+// (cell-content) parity with Python. The pipeline-parity harness reads this to+// error if a locked PDF's gridSim drops below 100%.+//+// Only PDFs that actually reach gridSim=100% (HTML-format-only divergence)+// belong here. Known-unfixed content gaps (e.g. 13_crosspage_table,+// 14_text_table_interleaved) must NOT be locked, or the harness both accepts+// and forbids the same gap — a contradiction. Keep this set in lock-step with+// known_diffs.json: a PDF listed here must have gridSim=100% (or be exempted+// as go_intentional), never an open/accepted_at_merge failure.+func parityLockedGridPDFs() map[string]bool {+ return map[string]bool{+ "06_table_content.pdf": true,+ "18_table_caption.pdf": true,+ }+}diff --git a/internal/deepdoc/parser/pdf/parity_lock_test.go b/internal/deepdoc/parser/pdf/parity_lock_test.gonew file mode 100644index 0000000000..f348e0da9c--- /dev/null+++ b/internal/deepdoc/parser/pdf/parity_lock_test.go@@ -0,0 +1,28 @@+package pdf++import "testing"++// TestParityLockedGridExcludesUnfixed locks that known-unfixed content gaps are+// NOT in the locked grid-parity set. Locking a PDF whose gridSim is below 100%+// (13_crosspage_table open gap, 14_text_table_interleaved accepted_at_merge)+// makes the harness simultaneously accept and forbid the same gap. Only PDFs at+// gridSim=100% (HTML-format-only divergence: 06, 18) belong in the lock.+func TestParityLockedGridExcludesUnfixed(t *testing.T) {+ locked := parityLockedGridPDFs()++ // Known-unfixed content gaps must not be locked.+ if locked["13_crosspage_table.pdf"] {+ t.Error("13_crosspage_table is an open grid gap and must not be locked")+ }+ if locked["14_text_table_interleaved.pdf"] {+ t.Error("14_text_table_interleaved is accepted_at_merge and must not be locked")+ }++ // PDFs that reach gridSim=100% (HTML-format-only divergence) stay locked.+ if !locked["06_table_content.pdf"] {+ t.Error("06_table_content reaches grid parity and must stay locked")+ }+ if !locked["18_table_caption.pdf"] {+ t.Error("18_table_caption reaches grid parity and must stay locked")+ }+}diff --git a/internal/deepdoc/parser/pdf/parser.go b/internal/deepdoc/parser/pdf/parser.goindex 70a500bb1c..fe82f0cb77 100644--- a/internal/deepdoc/parser/pdf/parser.go+++ b/internal/deepdoc/parser/pdf/parser.go@@ -179,6 +179,12 @@ func (p *Parser) extractOutlines(engine pdf.PDFEngine) []pdf.Outline {func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,docAnalyzer pdf.DocAnalyzer, tb pdf.TableBuilder,) pageResult {+ // Stamp the page number up front so replay DocAnalyzers can route their+ // per-page lookups in BOTH the OCR path (processPageBoxes -> OCRDetect/+ // OCRRecognize) and the DLA/TSR path (enrichOnePageWithDeepDoc also stamps+ // it later; this covers the earlier OCR calls). Production analyzers ignore+ // the key, so this is behavior-neutral.+ ctx = context.WithValue(ctx, pageNumCtxKey, pg)chars, extractErr := engine.ExtractChars(pg)if extractErr != nil {slog.Warn("processPage: ExtractChars failed", "page", pg, "err", extractErr)@@ -342,6 +348,14 @@ func (p *Parser) processPageBoxes(ctx context.Context, pageImg image.Image, char}}+ // Mark OCR-derived boxes so OCR-only post-processing (layout.Dedup*)+ // can scope itself to them and never de-duplicate char-path content.+ if ocrUsed {+ for i := range ocrBoxes {+ ocrBoxes[i].IsOCR = true+ }+ }+return ocrBoxes, chars, ocrUsed}@@ -493,6 +507,12 @@ func (p *Parser) buildLayout(ctx context.Context,) error {result.Metrics.BoxesInitial = len(boxes)+ // Collapse OCR duplicates BEFORE any merge step: overlapping same-text /+ // substring boxes must be dropped while still independent, otherwise+ // TextMerge/NaiveVerticalMerge concatenate them into duplicated text.+ boxes = lyt.DedupIdenticalText(boxes)+ boxes = lyt.DedupSubstringOverlaps(boxes)+boxes = lyt.AssignColumn(boxes)boxes = lyt.TextMerge(boxes, medianHeights)result.Metrics.BoxesTextMerge = len(boxes)diff --git a/internal/deepdoc/parser/pdf/parser_ocr.go b/internal/deepdoc/parser/pdf/parser_ocr.goindex 4e3515c1b3..fd9e2ccf62 100644--- a/internal/deepdoc/parser/pdf/parser_ocr.go+++ b/internal/deepdoc/parser/pdf/parser_ocr.go@@ -29,7 +29,7 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image,imgH := float64(pageImg.Bounds().Dy()) / pdf.DlaScalevar result []pdf.TextBox- for _, b := range boxes {+ for i, b := range boxes {x0 := int(math.Min(b.X0, math.Min(b.X1, math.Min(b.X2, b.X3))))y0 := int(math.Min(b.Y0, math.Min(b.Y1, math.Min(b.Y2, b.Y3))))x1 := int(math.Max(b.X0, math.Max(b.X1, math.Max(b.X2, b.X3))))@@ -50,7 +50,11 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image,{X: b.X2, Y: b.Y2},{X: b.X3, Y: b.Y3},})- texts, recErr := p.ocrRecognizeWithRotation(ctx, doc, cropped)+ // Stamp the detect-box index so a replay DocAnalyzer can route this+ // recognition back to the Python-dumped text for the same box. The+ // production analyzer ignores the key.+ recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, i)+ texts, recErr := p.ocrRecognizeWithRotation(recCtx, doc, cropped)if recErr != nil {slog.Warn(logLabel+" OCR recognize failed", "page", pageNum, "err", recErr)continue@@ -150,6 +154,10 @@ func ocrBestScore(texts []pdf.OCRText) float64 {type ocrDetectBox struct {box pdf.TextBoxx0, y0, x1, y1 float64+ // srcIdx is the box's index in the OCRDetect result (before detectBoxes+ // re-sorts). It routes per-box OCR fallback (buildTextBoxes) back to the+ // same Python-dumped box in replay, matching ocrDetectAndRecognize.+ srcIdx int}func (p *Parser) ocrMergeChars(ctx context.Context, pageImg image.Image, chars []pdf.TextChar, doc pdf.DocAnalyzer, pageNum int) []pdf.TextBox {@@ -174,7 +182,7 @@ func (p *Parser) detectBoxes(ctx context.Context, pageImg image.Image, doc pdf.DimgH := float64(imgBounds.Dy()) / scaleboxes := make([]ocrDetectBox, 0, len(ocrDetectBoxes))- for _, b := range ocrDetectBoxes {+ for i, b := range ocrDetectBoxes {x0 := min(b.X0, b.X1, b.X2, b.X3) / scaley0 := min(b.Y0, b.Y1, b.Y2, b.Y3) / scalex1 := max(b.X0, b.X1, b.X2, b.X3) / scale@@ -196,7 +204,7 @@ func (p *Parser) detectBoxes(ctx context.Context, pageImg image.Image, doc pdf.D}boxes = append(boxes, ocrDetectBox{box: pdf.TextBox{X0: x0, X1: x1, Top: y0, Bottom: y1, PageNumber: pageNum,- }, x0: x0, y0: y0, x1: x1, y1: y1})+ }, x0: x0, y0: y0, x1: x1, y1: y1, srcIdx: i})}if len(boxes) > 1 {@@ -335,7 +343,12 @@ func (p *Parser) buildTextBoxes(ctx context.Context, pageImg image.Image,{X: boxes[idx].x1 * scale, Y: boxes[idx].y1 * scale},{X: boxes[idx].x0 * scale, Y: boxes[idx].y1 * scale},})- texts, err := p.ocrRecognizeWithRotation(ctx, doc, cropped)+ // Stamp the source detect-box index so a replay DocAnalyzer routes+ // this fallback to the same Python-dumped box (detectBoxes may have+ // re-sorted, so use srcIdx, not the loop index). The production+ // analyzer ignores the key.+ recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, boxes[idx].srcIdx)+ texts, err := p.ocrRecognizeWithRotation(recCtx, doc, cropped)if err != nil {slog.Warn("ocr merge: recognize failed", "page", pageNum, "err", err)continuediff --git a/internal/deepdoc/parser/pdf/pipeline_parity_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_test.gonew file mode 100644index 0000000000..36a3bc829d--- /dev/null+++ b/internal/deepdoc/parser/pdf/pipeline_parity_test.go@@ -0,0 +1,363 @@+//go:build cgo && manual++package pdf++import (+ "encoding/json"+ "fmt"+ "os"+ "path"+ "path/filepath"+ "strings"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/deepdoc/parser/pdf/tool"+ pdf "ragflow/internal/deepdoc/parser/pdf/type"+ util "ragflow/internal/deepdoc/parser/pdf/util"+)++// TestPipelineParity verifies Go pipeline logic equivalence with Python.+// It loads Python pdfplumber chars (from charspy/), replays Python's DLA ++// TSR intermediates (output/py/ocr/{dla,tsr_raw}) through Go's assembly, and+// compares the resulting sections against Python's output/py/ocr/text/.+//+// Phase 1 (char→section assembly) expects CharSim 100%. Phase 2 (DLA/TSR+// replay) also builds tables from Python's inference; table section HTML may+// differ in whitespace/join formatting, so table PDFs are additionally+// checked by grid (rows) against output/py/ocr/tables/. Phase 3 (OCR replay)+// feeds Python's raw per-page OCR detect results so image-only PDFs get their+// text from the same inference Python used. A PDF without an OCR dump (the+// dump script has not been run with an OCR service available) falls back to+// the char-derived path exactly as before.+//+// Result classification (both sims are always reported):+// - textSim==100% -> PASS (fully aligned, includes table HTML byte-identical)+// - table PDF, gridSim==100% but textSim<100% -> GRID_OK: cell content+// matches Python, the residual gap is the HTML serialization layer only.+// Reported separately (not labeled PASS) and not counted as a content+// failure; classified as go_intentional/go_bug via known_diffs.json.+// - table PDF, gridSim<100% (or non-table textSim<100%) -> FAIL: Go assembly+// content genuinely diverges from Python.+func TestPipelineParity(t *testing.T) {+ charspyDir := filepath.Join("testdata", "charspy")+ pyTextDir := filepath.Join("testdata", "output", "py", "ocr", "text")+ dlaDir := filepath.Join("testdata", "output", "py", "ocr", "dla")+ tsrDir := filepath.Join("testdata", "output", "py", "ocr", "tsr_raw")+ ocrDir := filepath.Join("testdata", "output", "py", "ocr", "ocr")+ tablesDir := filepath.Join("testdata", "output", "py", "ocr", "tables")++ entries, err := os.ReadDir(charspyDir)+ if err != nil {+ t.Skipf("charspy/ not found: %v", err)+ }++ filter := common.GetEnv(common.EnvBatchParityFilter)++ // TSR-replay table PDFs that must hold full grid content parity — guards+ // the replay TableIndex json tag / per-page index mapping / cumulative-Y+ // handling against regressions. table_rotation_test.pdf is exempted: its+ // gridSim<100% gap is the table segmentation divergence documented as+ // go_intentional (rule table-rotation-split-vs-merged-grid in+ // known_diffs.json), where Go's split matches the physical PDF and+ // Python's merged 8x7 grid is a segmentation defect.+ lockedGridPDFs := parityLockedGridPDFs()++ // go_intentional rules exempt their applies_to PDFs from the FAIL count:+ // a documented, deliberate divergence where Go is judged at least as good+ // as Python (registry in table/testdata/parity/known_diffs.json). Only a+ // rule that names this exact PDF (no glob, e.g.+ // table-rotation-split-vs-merged-grid) may exempt it; a wildcard rule+ // like table-cell-fill-filled-threshold-0.85 documents a global threshold+ // divergence and must NOT mask a specific PDF's gridSim gap (13/14 stay+ // locked FAILs).+ knownDiffRules := loadPdfKnownDiffs(t)+ intentionalPDF := func(name string) bool {+ for _, r := range knownDiffRules {+ if r.Tag == "go_intentional" && r.matchesExactly(name) {+ return true+ }+ }+ return false+ }++ // Phase 2: replay Python's DLA + TSR intermediates through Go's assembly+ // so tables are built from Python's inference, not a mock. The factory is+ // keyed on the analyzer type, so non-replay parses are unaffected.+ RegisterReplayTableBuilder()++ total, passed, htmlDivergent, intentional, failed := 0, 0, 0, 0, 0+ for _, e := range entries {+ if e.IsDir() || !strings.HasSuffix(e.Name(), ".json") {+ continue+ }+ name := strings.TrimSuffix(e.Name(), ".json")+ if filter != "" && !strings.Contains(e.Name(), filter) {+ continue+ }++ // Load Python chars+ jsonPath := filepath.Join(charspyDir, e.Name())+ engine, err := tool.LoadPythonChars(jsonPath)+ if err != nil {+ t.Errorf("%s: tool.LoadPythonChars: %v", name, err)+ continue+ }++ // English documents: Python (is_english -> chars=[]) consumes pure OCR+ // output, not pdfplumber chars. Mirror that so both sides replay the+ // same input — clear the replayed chars and let processPageBoxes fall+ // through to the OCR-replay path (Phase 3 OCR dump). dims are kept so+ // the placeholder page image still carries the correct size.+ //+ // Use the is_english verdict captured from the SAME Python run that+ // produced the text golden (stable); fall back to DetectEnglish only+ // for older charspy dumps that omit it.+ isEnglish := false+ if v := engine.IsEnglish(); v != nil {+ isEnglish = *v+ } else if pages, _ := engine.PageCount(); util.DetectEnglish(engine.PageChars(), pages, nil) {+ isEnglish = true+ }+ if isEnglish {+ engine.ClearChars()+ }++ // Run Go pipeline with Python's DLA/TSR/OCR replayed.+ cfg := pdf.DefaultParserConfig()+ cfg.SortByTop = true+ analyzer := NewPythonIntermediateDocAnalyzer(name, dlaDir, tsrDir, ocrDir, engine.PageDims())+ p := NewParser(cfg)+ result, err := p.ParseRaw(t.Context(), engine, analyzer)+ if err != nil {+ t.Errorf("%s: Parse: %v", name, err)+ continue+ }++ // Read Python sections+ pyPath := filepath.Join(pyTextDir, name+".txt")+ pyData, err := os.ReadFile(pyPath)+ if err != nil {+ t.Logf("%s: no Python reference at %s — skip", name, pyPath)+ continue+ }++ // Build Go text+ var goText strings.Builder+ for _, s := range result.Sections {+ goText.WriteString(s.Text)+ goText.WriteByte('\n')+ }++ // Debug aid: BATCH_PARITY_DUMP_GO=1 writes the Go output text next to+ // the Python golden so failing PDFs can be diffed case by case.+ if common.GetEnv("BATCH_PARITY_DUMP_GO") != "" {+ dumpDir := filepath.Join("testdata", "output", "go", "ocr", "text")+ _ = os.MkdirAll(dumpDir, 0o755)+ _ = os.WriteFile(filepath.Join(dumpDir, name+".txt"), []byte(goText.String()), 0o644)+ }++ // Compare+ sim := tool.CharSimilarity(goText.String(), tool.StripMeta(string(pyData)))++ // Phase 2: tables are built from Python's replayed TSR, so compare the+ // reconstructed grid content (cell text) rather than the rendered HTML+ // (which differs in whitespace/join/header markup). Grid content parity+ // isolates Go's GroupCells + FillCellTextFromBoxes assembly; HTML-only+ // differences are expected and not a Go logic bug.+ pyRows, pyHasTables := loadPythonTables(t, filepath.Join(tablesDir, name+".json"))+ goRows := goTableRows(result)+ goHasTables := len(goRows) > 0++ total+++ status, detail := "", ""+ switch {+ case sim >= 100.0:+ // True full-output alignment (text, or table HTML byte-identical).+ passed+++ status = "PASS"+ detail = fmt.Sprintf("textSim=%.1f%%", sim)+ case pyHasTables || goHasTables:+ // Table PDFs not textSim=100% are judged on cell content first.+ // gridSim==100% means Go's GroupCells + FillCellTextFromBoxes+ // assembly matches Python's grid; the residual textSim gap is the+ // HTML serialization layer (whitespace/join/header markup). That is+ // a real divergence but HTML-only, so it is reported as GRID_OK —+ // never mislabeled PASS — and classified via known_diffs.json.+ gridSim := tool.CharSimilarity(joinGrid(goRows), joinGrid(pyRows))+ if gridSim >= 100.0 {+ htmlDivergent+++ status = "GRID_OK"+ detail = fmt.Sprintf("gridSim=%.1f%% textSim=%.1f%% (HTML-format divergence)", gridSim, sim)+ } else if intentionalPDF(name) {+ // go_intentional rule covers this PDF: the gridSim gap is a+ // documented, deliberate divergence (Go at least as good as+ // Python, e.g. table segmentation), so it is reported as+ // INTENTIONAL and not counted as a content failure. It is+ // still logged so a reader can see the gap.+ intentional+++ status = "INTENTIONAL"+ detail = fmt.Sprintf("gridSim=%.1f%% textSim=%.1f%% (go_intentional: %s)", gridSim, sim, intentionalRuleID(knownDiffRules, name))+ } else {+ failed+++ status = "FAIL"+ detail = fmt.Sprintf("gridSim=%.1f%% textSim=%.1f%% (table grid content differs)", gridSim, sim)+ }+ // Regression lock: TSR-replay table PDFs that reached full grid+ // content parity must stay there (protects against regressions in+ // the replay index/Y mapping).+ if lockedGridPDFs[name] && gridSim < 100.0 {+ t.Errorf("LOCKED %s: gridSim=%.1f%% regressed below 100%%", name, gridSim)+ }+ default:+ failed+++ status = "FAIL"+ detail = fmt.Sprintf("textSim=%.1f%% (must be 100%%)", sim)+ }+ t.Logf("%s %s: %s tables:%d boxes:%d->%d->%d->%d",+ status, name, detail, len(goRows), result.Metrics.BoxesInitial, result.Metrics.BoxesTextMerge, result.Metrics.BoxesVertMerge, len(result.Sections))+ }++ if total == 0 {+ t.Skip("no charspy/ files found")+ }+ t.Logf("Pipeline parity: aligned=%d html-divergent=%d intentional=%d failed=%d total=%d", passed, htmlDivergent, intentional, failed, total)+ if failed > 0 {+ t.Errorf("%d parity failures — Go pipeline content differs from Python (grid/text)", failed)+ }+}++// ── known_diffs.json integration (same schema as chunker) ────────────────++const pdfKnownDiffsPath = "table/testdata/parity/known_diffs.json"++// pdfDiffRule is one entry in known_diffs.json: an accepted, documented+// divergence from the Python baseline. Mirrors chunker's diffRule so a PDF+// whose gridSim gap is tagged go_intentional is exempted from the FAIL count+// (reported as INTENTIONAL instead).+type pdfDiffRule struct {+ ID string `json:"id"`+ Tag string `json:"tag"`+ Kind string `json:"kind"`+ AppliesTo []string `json:"applies_to"`+ Fields []string `json:"fields"`+}++func (r pdfDiffRule) matches(caseID string) bool {+ for _, pattern := range r.AppliesTo {+ if ok, err := path.Match(pattern, caseID); err == nil && ok {+ return true+ }+ }+ return false+}++// matchesExactly reports whether the rule names caseID with a literal (glob-+// free) pattern. Used for the FAIL-exemption: only a rule that explicitly+// names the PDF may waive its gridSim gap; a wildcard rule such as+// applies_to ["*"] documents a global divergence and must not mask a+// specific PDF's failure.+func (r pdfDiffRule) matchesExactly(caseID string) bool {+ for _, pattern := range r.AppliesTo {+ if strings.ContainsAny(pattern, "*?[") {+ continue+ }+ if pattern == caseID {+ return true+ }+ }+ return false+}++func loadPdfKnownDiffs(t *testing.T) []pdfDiffRule {+ t.Helper()+ raw, err := os.ReadFile(pdfKnownDiffsPath)+ if err != nil {+ t.Fatalf("read %s: %v", pdfKnownDiffsPath, err)+ }+ var registry struct {+ Version int `json:"version"`+ Rules []pdfDiffRule `json:"rules"`+ }+ if err := json.Unmarshal(raw, ®istry); err != nil {+ t.Fatalf("parse %s: %v", pdfKnownDiffsPath, err)+ }+ return registry.Rules+}++// intentionalRuleID returns the id of the first go_intentional rule naming+// this exact PDF, for the INTENTIONAL status detail line.+func intentionalRuleID(rules []pdfDiffRule, name string) string {+ for _, r := range rules {+ if r.Tag == "go_intentional" && r.matchesExactly(name) {+ return r.ID+ }+ }+ return "unknown"+}++// ── table-grid parity helpers ──────────────────────────────────────────++type pyTableDump struct {+ Results []struct {+ Positions []struct {+ Page int `json:"page"`+ Left float64 `json:"left"`+ Right float64 `json:"right"`+ Top float64 `json:"top"`+ Bottom float64 `json:"bottom"`+ } `json:"positions"`+ Rows [][]string `json:"rows"`+ } `json:"results"`+}++// loadPythonTables returns the flattened concatenation of every table's row+// grid from output/py/ocr/tables/{name}.json, plus whether any table exists.+// loadPythonTables reads Python's dumped table grid for a PDF. A missing golden+// and a corrupt golden are reported distinctly so neither silently masquerades+// as a Go content failure (both return (nil, false) so the table branch treats+// them as 'no Python tables').+func loadPythonTables(t *testing.T, jsonPath string) ([][]string, bool) {+ t.Helper()+ data, err := os.ReadFile(jsonPath)+ if err != nil {+ t.Errorf("%s: missing Python tables golden (environment gap, NOT a Go content fail): %v", jsonPath, err)+ return nil, false+ }+ var td pyTableDump+ if err := json.Unmarshal(data, &td); err != nil {+ t.Errorf("%s: corrupt Python tables golden (bad input, NOT a Go content fail): %v", jsonPath, err)+ return nil, false+ }+ var rows [][]string+ for _, r := range td.Results {+ rows = append(rows, r.Rows...)+ }+ return rows, len(rows) > 0+}++// goTableRows flattens every Go result table's grid (cell text) into rows.+func goTableRows(result *pdf.ParseResult) [][]string {+ var rows [][]string+ for _, t := range result.Tables {+ for _, r := range t.Grid {+ row := make([]string, len(r))+ for j, c := range r {+ row[j] = strings.TrimSpace(c.Text)+ }+ rows = append(rows, row)+ }+ }+ return rows+}++// joinGrid renders a grid as newline-joined, space-joined cell text so+// CharSimilarity can compare two grids' content ignoring row/cell order.+func joinGrid(grid [][]string) string {+ var b strings.Builder+ for _, row := range grid {+ b.WriteString(strings.Join(row, " "))+ b.WriteByte('\n')+ }+ return b.String()+}diff --git a/internal/deepdoc/parser/pdf/replay_analyzer_test.go b/internal/deepdoc/parser/pdf/replay_analyzer_test.gonew file mode 100644index 0000000000..92b076819a--- /dev/null+++ b/internal/deepdoc/parser/pdf/replay_analyzer_test.go@@ -0,0 +1,238 @@+//go:build cgo && manual++package pdf++import (+ "context"+ "image"+ "path/filepath"+ "sort"++ "ragflow/internal/deepdoc/parser/pdf/table"+ "ragflow/internal/deepdoc/parser/pdf/tool"+ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// PythonIntermediateDocAnalyzer replays Python's pre-computed DLA layout+// regions so the parity harness can exercise Go's assembly logic against+// Python's inference output. It is the DocAnalyzer half of the+// intermediate-replay paradigm (see pipeline_parity_test.go).+//+// DLA and page-level OCR are wired here. TSR is delivered by the paired+// PythonIntermediateTableBuilder (selected via the TableBuilder factory),+// and table-rotation OCR (self.ocr(img) directly in Python) is out of scope.+// OCRDetect returns the Python-dumped detect boxes for the page; OCRRecognize+// returns the Python-dumped recognized text for the box identified by the+// ocrBoxIdxCtxKey stamped in ocrDetectAndRecognize.+type PythonIntermediateDocAnalyzer struct {+ // name is the PDF base name INCLUDING the .pdf extension, matching the+ // dump file naming (e.g. "06_table_content.pdf").+ name string+ dlaDir string+ tsrDir string+ ocrDir string+ // dims carries per-page image dimensions [w,h] in ZM pixels (from+ // charspy), used by TSR replay to derive Python's page_cum_height.+ dims map[int][2]int+ healthy bool+}++// NewPythonIntermediateDocAnalyzer builds a replay analyzer rooted at the+// given dump directories. dlaDir/tsrDir/ocrDir point at+// output/py/ocr/{dla,tsr_raw,ocr}; dims is the charspy per-page image+// dimensions (PythonCharEngine.PageDims).+func NewPythonIntermediateDocAnalyzer(name, dlaDir, tsrDir, ocrDir string, dims map[int][2]int) *PythonIntermediateDocAnalyzer {+ return &PythonIntermediateDocAnalyzer{name: name, dlaDir: dlaDir, tsrDir: tsrDir, ocrDir: ocrDir, dims: dims, healthy: true}+}++func (a *PythonIntermediateDocAnalyzer) DLA(ctx context.Context, _ image.Image) ([]pdf.DLARegion, error) {+ pg, _ := ctx.Value(pageNumCtxKey).(int)+ pages, err := tool.LoadPythonDLA(filepath.Join(a.dlaDir, a.name+".json"))+ if err != nil {+ return nil, err+ }+ for _, p := range pages {+ if p.Page != pg {+ continue+ }+ out := make([]pdf.DLARegion, len(p.Regions))+ for i, r := range p.Regions {+ out[i] = r.ToDLARegion()+ }+ return out, nil+ }+ return nil, nil+}++// TSR is unused: the pipeline reaches TSR via the TableBuilder, not here.+func (a *PythonIntermediateDocAnalyzer) TSR(_ context.Context, _ image.Image) ([]pdf.TSRCell, error) {+ return nil, nil+}++// OCRDetect returns the Python-dumped raw detect boxes for the current page+// (image-pixel space, matching the page image handed to the analyzer).+// Missing dump or page → nil, letting the pipeline fall back to the+// char-derived path just as it does when OCR returns nothing.+func (a *PythonIntermediateDocAnalyzer) OCRDetect(ctx context.Context, _ image.Image) ([]pdf.OCRBox, error) {+ pg, _ := ctx.Value(pageNumCtxKey).(int)+ pages, err := tool.LoadPythonOCR(filepath.Join(a.ocrDir, a.name+".json"))+ if err != nil {+ return nil, err+ }+ // OCR dump Y is page-cumulative (page_cum_height added in __ocr), like+ // the TSR dump; derive the cumulative offset from charspy page dims and+ // subtract so boxes land on the current page before the crop shift.+ cumOffsetPx := 0.0+ for p := 0; p < pg; p++ {+ if d, ok := a.dims[p]; ok {+ cumOffsetPx += float64(d[1])+ }+ }+ // The OCR dump keys pages 1-based (Python's __ocr receives i+1), while+ // ctx pageNumCtxKey is 0-based — match pg+1.+ for _, p := range pages {+ if p.Page != pg+1 {+ continue+ }+ out := make([]pdf.OCRBox, len(p.Boxes))+ for i, b := range p.Boxes {+ out[i] = b.ToOCRBox(cumOffsetPx)+ }+ return out, nil+ }+ return nil, nil+}++// OCRRecognize returns the Python-dumped recognized text for the detect box+// whose index is stamped in ctx (ocrBoxIdxCtxKey) by ocrDetectAndRecognize.+// All layer-2 rotation candidates resolve to the same text, so the+// score-based rotation selection is a no-op over identical inputs.+func (a *PythonIntermediateDocAnalyzer) OCRRecognize(ctx context.Context, _ image.Image) ([]pdf.OCRText, error) {+ pg, _ := ctx.Value(pageNumCtxKey).(int)+ bi, _ := ctx.Value(ocrBoxIdxCtxKey).(int)+ if bi < 0 {+ return nil, nil+ }+ pages, err := tool.LoadPythonOCR(filepath.Join(a.ocrDir, a.name+".json"))+ if err != nil {+ return nil, err+ }+ // OCR dump pages are 1-based; pageNumCtxKey is 0-based (see OCRDetect).+ for _, p := range pages {+ if p.Page != pg+1 {+ continue+ }+ if bi >= len(p.Boxes) {+ return nil, nil+ }+ b := p.Boxes[bi]+ return []pdf.OCRText{{Text: b.Text, Confidence: b.Conf}}, nil+ }+ return nil, nil+}+func (a *PythonIntermediateDocAnalyzer) Health() bool { return a.healthy }++// PythonIntermediateTableBuilder replays Python's raw TSR components for a+// single (page, table) identified via ctx. DetectCells returns the+// components in Go's crop space; GroupCells delegates to the production+// grouping so the parity harness measures Go's assembly, not a re-implementaion.+type PythonIntermediateTableBuilder struct {+ analyzer *PythonIntermediateDocAnalyzer+}++// DetectCells returns the Python TSR components for the (page, table_index)+// carried in ctx, mapped from PDF-point space into Go's crop-image space+// using the crop origin stamped by processOneTable.+//+// Two Python→Go mapping quirks are handled here:+// 1. Python's table_index is a document-global counter (pdf_parser.py:590),+// while Go's tableIdxCtxKey is the per-page ordinal (table_extract.go:97).+// The ordinal is mapped to the Nth distinct global index on this page.+// 2. Python adds page_cum_height to TSR Y (pdf_parser.py:572-573), so the+// cumulative offset is derived from charspy page dims and subtracted+// before the crop shift, keeping cells in page-local space.+func (b *PythonIntermediateTableBuilder) DetectCells(ctx context.Context, _ image.Image) ([]pdf.TSRCell, error) {+ pg, _ := ctx.Value(pageNumCtxKey).(int)+ ti, _ := ctx.Value(tableIdxCtxKey).(int)+ cropOffX, _ := ctx.Value(cropOffXKey).(float64)+ cropOffY, _ := ctx.Value(cropOffYKey).(float64)++ cells, err := tool.LoadPythonTSR(filepath.Join(b.analyzer.tsrDir, b.analyzer.name+".json"))+ if err != nil {+ return nil, err+ }++ // Map the per-page ordinal ti to the Nth distinct global table index on+ // this page. The dump's global indices are monotonic per page, so sorting+ // and de-duplicating yields ordinal order.+ var globalIdx []int+ for _, c := range cells {+ if c.Page == pg {+ globalIdx = append(globalIdx, c.TableIndex)+ }+ }+ sort.Ints(globalIdx)+ globalIdx = compactInts(globalIdx)+ if ti < 0 || ti >= len(globalIdx) {+ return nil, nil+ }+ wantIdx := globalIdx[ti]++ // page_cum_height[pg] = sum of prior page image heights ÷ ZM+ // (pdf_parser.py:1690). dims hold the ZM-pixel page image heights, so the+ // cumulative offset in image pixels is the raw sum.+ cumOffsetPx := 0.0+ for p := 0; p < pg; p++ {+ if d, ok := b.analyzer.dims[p]; ok {+ cumOffsetPx += float64(d[1])+ }+ }++ var out []pdf.TSRCell+ for _, c := range cells {+ if c.Page != pg || c.TableIndex != wantIdx {+ continue+ }+ out = append(out, c.ToTSRCell(cropOffX, cropOffY, cumOffsetPx))+ }+ return out, nil+}++// compactInts removes consecutive duplicates from a sorted slice.+func compactInts(in []int) []int {+ out := in[:0]+ var prev int+ first := true+ for _, v := range in {+ if first || v != prev {+ out = append(out, v)+ prev = v+ first = false+ }+ }+ return out+}++// GroupCells uses the production cross-product grouping so the replay+// measures Go's grid assembly against Python's raw TSR components.+func (b *PythonIntermediateTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {+ // DeepDocTableBuilder.GroupCells is receiver-independent (it only reads+ // the passed cells), so a zero builder delegates to production logic.+ return (&table.DeepDocTableBuilder{}).GroupCells(cells)+}++func (b *PythonIntermediateTableBuilder) Name() string { return "py-intermediate" }++// RegisterReplayTableBuilder installs a TableBuilder factory that returns the+// replay builder whenever the DocAnalyzer is a PythonIntermediateDocAnalyzer,+// and otherwise falls back to the production DeepDoc builder. Safe to call+// repeatedly; the factory is keyed on the analyzer type so it never alters+// behavior for non-replay parses.+func RegisterReplayTableBuilder() {+ RegisterTableBuilder(func(doc pdf.DocAnalyzer) pdf.TableBuilder {+ if a, ok := doc.(*PythonIntermediateDocAnalyzer); ok {+ return &PythonIntermediateTableBuilder{analyzer: a}+ }+ return table.NewDeepDocTableBuilder(doc)+ })+}diff --git a/internal/deepdoc/parser/pdf/table/crosspage_cleanup_test.go b/internal/deepdoc/parser/pdf/table/crosspage_cleanup_test.gonew file mode 100644index 0000000000..f3dd3556be--- /dev/null+++ b/internal/deepdoc/parser/pdf/table/crosspage_cleanup_test.go@@ -0,0 +1,56 @@+package table++import (+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// TestMergeTablesAcrossPages_PostMergeCleanup locks the production behavior+// flagged as untested in review: when two same-column tables on consecutive+// pages are merged, the rebuilt cross-page grid must run the post-GroupCells+// cleanup (DropAllEmptyRows + CleanupOrphanColumns + CleanupOrphanRows) at the+// merge site. Without it, an empty "table row" detected next to a header on a+// continuation page leaks into the merged grid as a row of empty cells,+// inflating item.Grid and breaking gridSim (13_crosspage_table.pdf page 2).+//+// Uses synthetic grids only — no Python dump, CI-runnable.+func TestMergeTablesAcrossPages_PostMergeCleanup(t *testing.T) {+ anchor := pdf.TableItem{+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 0, Right: 200, Top: 100, Bottom: 200}},+ Grid: [][]pdf.TSRCell{+ {{X0: 0, Y0: 100, X1: 100, Y1: 130, Text: "h1"}, {X0: 100, Y0: 100, X1: 200, Y1: 130, Text: "h2"}},+ {{X0: 0, Y0: 130, X1: 100, Y1: 160, Text: "a"}, {X0: 100, Y0: 130, X1: 200, Y1: 160, Text: "b"}},+ // Empty orphan row (e.g. a stray "table row" next to a header on+ // the continuation page): must be dropped by the merge-site cleanup.+ {{X0: 0, Y0: 160, X1: 100, Y1: 190}, {X0: 100, Y0: 160, X1: 200, Y1: 190}},+ },+ }+ continuation := pdf.TableItem{+ Positions: []pdf.Position{{PageNumbers: []int{1}, Left: 0, Right: 200, Top: 210, Bottom: 260}},+ Grid: [][]pdf.TSRCell{+ {{X0: 0, Y0: 210, X1: 100, Y1: 240, Text: "c"}, {X0: 100, Y0: 210, X1: 200, Y1: 240, Text: "d"}},+ },+ }++ out := MergeTablesAcrossPages([]pdf.TableItem{anchor, continuation}, nil)+ if len(out) != 1 {+ t.Fatalf("expected 1 merged table, got %d", len(out))+ }+ merged := out[0]++ // Stacked grid = 3 anchor rows + 1 continuation row = 4; the empty orphan+ // row must be removed, leaving 3 rows (h, a, c). Continuation rows preserved.+ if len(merged.Grid) != 3 {+ t.Fatalf("expected 3 rows after empty-row cleanup, got %d: %+v", len(merged.Grid), merged.Grid)+ }+ // The cleaned grid must be reflected in the public Rows field.+ if len(merged.Rows) != 3 {+ t.Errorf("merged.Rows should have 3 rows after cleanup, got %d", len(merged.Rows))+ }+ // Spot-check the continuation content survived the merge.+ last := merged.Grid[len(merged.Grid)-1]+ if len(last) < 2 || last[0].Text != "c" || last[1].Text != "d" {+ t.Errorf("continuation row content lost: %+v", last)+ }+}diff --git a/internal/deepdoc/parser/pdf/table/merge_captions.go b/internal/deepdoc/parser/pdf/table/merge_captions.goindex 418e326d0c..b7323f38f4 100644--- a/internal/deepdoc/parser/pdf/table/merge_captions.go+++ b/internal/deepdoc/parser/pdf/table/merge_captions.go@@ -22,8 +22,19 @@ func MergeCaptions(sections []pdf.Section, figures []pdf.Section) []pdf.Section} else {sections[target].Text = s.Text}+ captions = append(captions, i)+ continue+ }+ // No merge target. A FIGURE caption is kept as its own section: a pure+ // image figure has no text section (BoxesToSections skips empty figure+ // boxes), so removing it would drop caption text that Python keeps+ // (07_mixed_content 'Figure 1/2'). A TABLE caption without a table+ // section is a DLA mislabel (e.g. rotate_270's rotated text labeled+ // "table") — keep the historical removal so rotated-page text is not+ // duplicated.+ if captionType != pdf.LayoutTypeFigure {+ captions = append(captions, i)}- captions = append(captions, i)}// Remove caption sections in reverse order.n := len(sections)diff --git a/internal/deepdoc/parser/pdf/table/merge_captions_test.go b/internal/deepdoc/parser/pdf/table/merge_captions_test.goindex 5587985146..d818498779 100644--- a/internal/deepdoc/parser/pdf/table/merge_captions_test.go+++ b/internal/deepdoc/parser/pdf/table/merge_captions_test.go@@ -72,3 +72,26 @@ func TestMergeCaptions_EuclideanDistance(t *testing.T) {t.Errorf("figure Text should contain caption 'close', got %q", result[0].Text)}}++// TestMergeCaptions_FigureCaptionNoTargetKept locks the production behavior+// change flagged as untested in review: a figure caption with NO nearby figure+// section is KEPT as its own section. A pure-image figure has no text section+// to merge into, so removing the caption (the old behavior) would silently drop+// caption text that Python keeps (07_mixed_content 'Figure 1/2').+func TestMergeCaptions_FigureCaptionNoTargetKept(t *testing.T) {+ sections := []pdf.Section{+ {Text: "Figure 2: system architecture overview", LayoutType: "figure caption",+ Positions: []pdf.Position{{PageNumbers: []int{0, 0}, Left: 40, Right: 160, Top: 300, Bottom: 315}}},+ }+ figures := pdf.CollectFigures(sections) // empty: no "figure" section present+ result := MergeCaptions(sections, figures)+ if len(result) != 1 {+ t.Fatalf("figure caption with no parent must be kept, got %d sections", len(result))+ }+ if result[0].LayoutType != "figure caption" {+ t.Errorf("expected figure caption kept as its own section, got LayoutType %q", result[0].LayoutType)+ }+ if result[0].Text != "Figure 2: system architecture overview" {+ t.Errorf("caption text must be preserved, got %q", result[0].Text)+ }+}diff --git a/internal/deepdoc/parser/pdf/table/table_cells.go b/internal/deepdoc/parser/pdf/table/table_cells.goindex c087cd7769..add723ef9c 100644--- a/internal/deepdoc/parser/pdf/table/table_cells.go+++ b/internal/deepdoc/parser/pdf/table/table_cells.go@@ -95,6 +95,31 @@ func GroupTSRCellsToRows(cells []pdf.TSRCell) [][]pdf.TSRCell {// picked, matching Python. See go_intentional rule// table-cell-fill-filled-threshold-0.85.func FillCellTextFromBoxes(cells []pdf.TSRCell, boxes []pdf.TextBox) {+ FillCellTextFromBoxesWithRows(cells, boxes, nil)+}++// FillCellTextFromBoxesWithRows is FillCellTextFromBoxes with the per-row+// strip X range taken from the original TSR "table row" component bboxes+// instead of the grid column union.+//+// Python matches a box to a row via find_overlapped_with_threshold(box,+// rows) where rows are the raw TSR row components (gather(".* (row|header)")+// in pdf_parser.py:602) and overlapped_area divides by the row's own bbox.+// Go's grid is a TSR-row × column cross-product; its cells always span the+// full column union, so the old strip X range was wider than the row+// component's true bbox. When TSR emits a row whose line does not cover a+// table edge (e.g. 13_crosspage_table.pdf page 2 row 44: x0=106.9 vs the+// grid's x0=90.8), a col-0 box straddling that row and the one above it+// can pass the 0.3 threshold against the full-width strip while Python+// rejects it against the true bbox — so Go assigned the box one row lower+// than Python.+//+// rowStrips are the TSR "table row" components (data rows only — header /+// projrowheader components are excluded by the caller). Each is matched to a+// grid row band by Y coordinate via matchRowStrip, so the override applies to+// every data row regardless of whether the table has a header. Rows with no+// matching TSR row component (e.g. header rows) keep the grid-union strip X.+func FillCellTextFromBoxesWithRows(cells []pdf.TSRCell, boxes []pdf.TextBox, rowStrips []pdf.TSRCell) {slog.Debug("fillCellTextFromBoxes", "cells", len(cells), "boxes", len(boxes))if len(cells) == 0 || len(boxes) == 0 {return@@ -156,6 +181,20 @@ func FillCellTextFromBoxes(cells []pdf.TSRCell, boxes []pdf.TextBox) {return cells[rows[ri].cells[a]].X0 < cells[rows[ri].cells[b]].X0})}+ // Replace each grid row's strip X with the matching TSR "table row"+ // component's own bbox X (see FillCellTextFromBoxesWithRows doc). Matching+ // is by Y band, not positional index, so the override applies to every+ // data row independently of whether the table has a header: header /+ // projrowheader components are simply absent from rowStrips and fall back+ // to the grid-union strip X, which is correct for header rows. This+ // replaces the former all-or-nothing `len(rowStrips) == len(rows)` guard,+ // which silently disabled the override for any table that had a header.+ for ri := range rows {+ if sx0, sx1, ok := matchRowStrip(rowStrips, rows[ri].y0); ok {+ rows[ri].stripX0 = sx0+ rows[ri].stripX1 = sx1+ }+ }// Accumulate box text per target cell so multiple boxes in one cell join.cellText := make([]string, len(cells))@@ -280,6 +319,24 @@ func BoxMatchesCell(cell pdf.TSRCell, box pdf.TextBox, cellIsEmpty bool) bool {return inter/boxArea >= 0.85}+// matchRowStrip finds the TSR "table row" component whose Y0 matches the grid+// row band y0 within yTol, returning its X range. A header / projrowheader+// component (not collected into rowStrips by the caller) returns ok=false, so+// that row keeps the grid-union strip X — correct for header rows. Matching by+// Y (not positional index) lets the strip-X override apply to every data row+// even when the table also has a header, instead of the former+// all-or-nothing `len(rowStrips) == len(rows)` guard that silently disabled+// the override for any header-bearing table.+func matchRowStrip(rowStrips []pdf.TSRCell, y0 float64) (float64, float64, bool) {+ const yTol = 1e-6+ for _, rs := range rowStrips {+ if math.Abs(rs.Y0-y0) <= yTol {+ return rs.X0, rs.X1, true+ }+ }+ return 0, 0, false+}+// isCaptionBox checks if a text box is a table/figure caption,// matching Python is_caption(). Captions should not enter table cells.var reCaption = regexp.MustCompile(`^[图表]+[ 0-9::]{2,}|(?i)Fig\.?\s*\d+|(?i)Figure\s+\d+|(?i)Table\s+\d+`)diff --git a/internal/deepdoc/parser/pdf/table/table_cells_group_test.go b/internal/deepdoc/parser/pdf/table/table_cells_group_test.goindex f13492f09a..7619ead3b4 100644--- a/internal/deepdoc/parser/pdf/table/table_cells_group_test.go+++ b/internal/deepdoc/parser/pdf/table/table_cells_group_test.go@@ -233,3 +233,88 @@ func TestFillCellTextFromBoxes_EmptyBoxIgnored(t *testing.T) {t.Errorf("whitespace text should produce empty, got %q", cells[0].Text)}}++func TestFillCellTextFromBoxes_RowStripUsesTSRRowBBox(t *testing.T) {+ // 13_crosspage_table root cause: TSR "table row" 44 has a narrower X+ // bbox (x0=106.9) than the grid column union (x0=90.8). A col-0 box+ // spanning rows 43/44 ("2024-43 2024-44", x0=87) overlaps row 44's true+ // bbox by only ~21% (Python's find_overlapped_with_threshold rejects it,+ // thr=0.3) but overlaps the full-width grid strip by ~50% — so Go picked+ // row 44 while Python fell back to row 43. The rowStrip variant must use+ // the TSR row bbox so the box lands in the upper row.+ grid := [][]pdf.TSRCell{+ {+ {X0: 0, Y0: 0, X1: 1500, Y1: 54},+ {X0: 1500, Y0: 0, X1: 3000, Y1: 54},+ },+ {+ {X0: 0, Y0: 54, X1: 1500, Y1: 108},+ {X0: 1500, Y0: 54, X1: 3000, Y1: 108},+ },+ }+ // TSR row bboxes (sorted top-to-bottom): row 1 is narrower on the left,+ // like table_rotation/13's second-row-of-a-pair. Without rowStrips the+ // full-width strip makes the box overlap row 1 at ~51% (> row 0's 49%)+ // and Go wrongly assigns it to the lower row.+ rowStrips := []pdf.TSRCell{+ {X0: 0, Y0: 0, X1: 3000, Y1: 54},+ {X0: 57, Y0: 54, X1: 1400, Y1: 108}, // narrow left edge: col-0 box partially outside+ }+ // Box spans both rows, x0=0 is inside row 0's strip but outside row 1's+ // narrow strip (x0=57), so only row 0 gets >= 0.3 overlap.+ boxes := []pdf.TextBox{{X0: 0, X1: 100, Top: 20, Bottom: 90, Text: "AB"}}++ flat := FlattenGrid(grid)+ FillCellTextFromBoxesWithRows(flat, boxes, rowStrips)++ if flat[0].Text != "AB" { // grid row0 col0+ t.Fatalf("row0 col0 = %q, want AB (box must reject narrow row1 bbox)", flat[0].Text)+ }+ if flat[2].Text != "" { // grid row1 col0+ t.Errorf("row1 col0 = %q, want empty (box belongs to row 0)", flat[2].Text)+ }+}++// TestFillCellTextFromBoxes_RowStripHeaderTable locks that the TSR row-strip+// override applies to data rows even when the table HAS a header. The caller+// (table_extract.go) collects only "table row" components into rowStrips and+// excludes header / projrowheader; the former all-or-nothing guard+// (len(rowStrips) == len(rows)) disabled the override for any header-bearing+// table. Matching is now by Y, so data rows get the narrow strip X while the+// header row keeps the grid-union X and is unaffected.+func TestFillCellTextFromBoxes_RowStripHeaderTable(t *testing.T) {+ // 3 grid rows: header + 2 data rows. Header is wider in Y only.+ grid := [][]pdf.TSRCell{+ {+ {X0: 0, Y0: 0, X1: 3000, Y1: 50}, // header row+ },+ {+ {X0: 0, Y0: 50, X1: 3000, Y1: 100}, // data row A (full width)+ },+ {+ {X0: 0, Y0: 100, X1: 3000, Y1: 150}, // data row B (narrow on left)+ },+ }+ // Only data rows are collected into rowStrips (header excluded), matching+ // production: data A full-width, data B narrow (x0=57).+ rowStrips := []pdf.TSRCell{+ {X0: 0, Y0: 50, X1: 3000, Y1: 100},+ {X0: 57, Y0: 100, X1: 1400, Y1: 150}, // narrow left edge+ }+ // Box spans both data rows but is outside data row B's narrow strip+ // (x0=57), so it must land in data row A.+ boxes := []pdf.TextBox{{X0: 0, X1: 100, Top: 60, Bottom: 140, Text: "AB"}}++ flat := FlattenGrid(grid)+ FillCellTextFromBoxesWithRows(flat, boxes, rowStrips)++ if flat[1].Text != "AB" { // header=row0, data A=row1+ t.Fatalf("data row A (flat[1]) = %q, want AB (override must apply to header tables)", flat[1].Text)+ }+ if flat[2].Text != "" { // data B=row2+ t.Errorf("data row B (flat[2]) = %q, want empty (box rejected by narrow strip)", flat[2].Text)+ }+ if flat[0].Text != "" { // header row: box does not overlap it+ t.Errorf("header row (flat[0]) = %q, want empty", flat[0].Text)+ }+}diff --git a/internal/deepdoc/parser/pdf/table/table_construct.go b/internal/deepdoc/parser/pdf/table/table_construct.goindex 073a748745..35b0d14ac4 100644--- a/internal/deepdoc/parser/pdf/table/table_construct.go+++ b/internal/deepdoc/parser/pdf/table/table_construct.go@@ -59,6 +59,19 @@ func ConstructTable(cells []pdf.TSRCell, boxes []pdf.TextBox, caption string, it// column cleanup, leaving it stale; item.Grid also went stale because// CleanupOrphanRows returns a re-sliced header).rows = CleanupOrphanColumns(rows)+ // Drop all-empty rows. Python's construct_table groups boxes by their+ // R annotation (tbl[i] is the boxes for row i; a TSR row with no+ // overlapping box contributes no row). Its HTML emitter also skips+ // rows whose rendered HTML stays at "<tr>" (no cell wrote anything).+ // Go's grid is a TSR-row cross-product, so an extra "table row" that+ // the model emits alongside a "table projected row header" (or any+ // other TSR component with no OCR/text overlap) leaks through as a+ // 5-cell row of empty strings, inflating item.Rows and the HTML. See+ // 13_crosspage_table.pdf page 2: y0=885 "table row" sits on top of+ // a "table projected row header" with no box overlap and survives+ // every TSR cleanup pass. Drop it here so rows/Rows/HTML all match+ // Python.+ rows = DropAllEmptyRows(rows)rows = CleanupOrphanRows(rows)hdrs := HeaderSetWithBlockType(rows, boxes)if item != nil {@@ -364,6 +377,30 @@ func CleanupOrphanRows(rows [][]pdf.TSRCell) [][]pdf.TSRCell {return rows}+// DropAllEmptyRows removes rows whose cells are all whitespace/empty.+// Python's construct_table never emits a row that has no boxes assigned+// to any (R,C), so the parity target is to keep only rows that contribute+// at least one non-empty cell. Runs BEFORE CleanupOrphanRows so the+// downstream "exactly one populated cell" check is not silently masking+// a TSR false positive (e.g. a duplicate row component detected on top+// of a "table projected row header").+func DropAllEmptyRows(rows [][]pdf.TSRCell) [][]pdf.TSRCell {+ out := make([][]pdf.TSRCell, 0, len(rows))+ for _, r := range rows {+ empty := true+ for j := range r {+ if strings.TrimSpace(r[j].Text) != "" {+ empty = false+ break+ }+ }+ if !empty {+ out = append(out, r)+ }+ }+ return out+}+// mergeOrphanCell appends the lone cell at (from, col) into the target cell at// (to, col). Matches Python's tbl[to][col].extend(tbl[from][col]): when the// target already has text, the orphan text is appended after it. The targetdiff --git a/internal/deepdoc/parser/pdf/table/table_construct_test.go b/internal/deepdoc/parser/pdf/table/table_construct_test.goindex 97d7a62ed5..e14b97cd1c 100644--- a/internal/deepdoc/parser/pdf/table/table_construct_test.go+++ b/internal/deepdoc/parser/pdf/table/table_construct_test.go@@ -48,6 +48,48 @@ func TestConstructTable_Simple3x2(t *testing.T) {t.Logf("HTML:\n%s", html)}+// TestConstructTable_DropsAllEmptyRow pins Python parity: when the+// TSR-derived grid carries a row that no text box landed in+// (e.g. an extra "table row" detected next to a "table projected row+// header" on a cross-page table's continuation page), Python's+// construct_table builds rows from box.R grouping and never emits that+// row, while Go's grid carries every TSR row and previously kept the+// empty row — leaking it into item.Rows and the HTML table.+//+// Repro fixture from 13_crosspage_table.pdf page 2: the first+// "table row" (y0=885) sits on top of a "table projected row header"+// with no OCR box overlap. Go's GroupCells keeps it as a 5-cell+// row, all empty; this test pins the drop.+func TestConstructTable_DropsAllEmptyRow(t *testing.T) {+ mkRow := func(y0 float64, texts ...string) []pdf.TSRCell {+ row := make([]pdf.TSRCell, 5)+ for c := range row {+ row[c] = pdf.TSRCell{X0: float64(c * 100), Y0: y0, X1: float64(c*100 + 100), Y1: y0 + 20}+ }+ for c, t := range texts {+ row[c].Text = t+ }+ return row+ }+ item := &pdf.TableItem{+ Grid: [][]pdf.TSRCell{+ mkRow(0, "r0c0", "r0c1", "r0c2", "r0c3", "r0c4"),+ mkRow(20), // all empty: TSR-detected row with no OCR box+ mkRow(40, "r2c0", "r2c1", "r2c2", "r2c3", "r2c4"),+ },+ }+ _ = ConstructTable(nil, nil, "", item)+ if len(item.Rows) != 2 {+ t.Fatalf("expected 2 rows after dropping all-empty row, got %d:\n%v", len(item.Rows), item.Rows)+ }+ if item.Rows[0][0] != "r0c0" || item.Rows[1][0] != "r2c0" {+ t.Errorf("row content wrong: %v", item.Rows)+ }+ if len(item.Grid) != 2 {+ t.Errorf("expected item.Grid also to drop the empty row, got len=%d", len(item.Grid))+ }+}+func TestConstructTable_EmptyCells(t *testing.T) {html := ConstructTable(nil, nil, "", nil)if html != "" {diff --git a/internal/deepdoc/parser/pdf/table/table_html.go b/internal/deepdoc/parser/pdf/table/table_html.goindex 947ff18338..cc7db28a47 100644--- a/internal/deepdoc/parser/pdf/table/table_html.go+++ b/internal/deepdoc/parser/pdf/table/table_html.go@@ -8,6 +8,13 @@ import (pdf "ragflow/internal/deepdoc/parser/pdf/type")+// RowsToHTML renders a table grid to HTML. The output is intentionally+// compact (no inter-tag whitespace) and the caption is html-escaped. This+// diverges from Python's __html_table, which emits newlines between rows and+// leaves the caption unescaped; the difference is HTML serialization only and+// never affects cell content (gridSim=100% in the parity harness). Registered+// as go_intentional in table/testdata/parity/known_diffs.json+// (rule "table-html-emission-format").func RowsToHTML(rows [][]pdf.TSRCell, caption string, headerRows map[int]bool, spanInfo map[[2]int][2]int, covered map[[2]int]bool) string {var b strings.Builderb.WriteString("<table>")diff --git a/internal/deepdoc/parser/pdf/table/table_merge.go b/internal/deepdoc/parser/pdf/table/table_merge.goindex f429cbd76d..8a73f9fae2 100644--- a/internal/deepdoc/parser/pdf/table/table_merge.go+++ b/internal/deepdoc/parser/pdf/table/table_merge.go@@ -164,6 +164,24 @@ func MergeTablesAcrossPages(tables []pdf.TableItem, medianHeights map[int]float6anchor.Grid = padGridCols(rebuilt, uniCols)}}+ // Re-run the post-GroupCells cleanup that processOneTable would+ // otherwise have applied per-page: stackGrids rebuilds the grid+ // from raw (un-cleaned) per-page cells, so the empty / orphan+ // cleanup done inside ConstructTable never runs on the merged+ // grid. Without it, an extra "table row" detected next to a+ // "table projected row header" on a cross-page continuation+ // page (e.g. 13_crosspage_table.pdf page 2 y0=885) leaks into+ // the merged grid as a row of empty cells, inflating+ // item.Grid and breaking gridSim against Python's box.R+ // grouping which never produces such a row. See+ // table_construct.go dropAllEmptyRows for the matching+ // per-page fix.+ if len(anchor.Grid) > 0 && HasText(anchor.Grid) {+ anchor.Grid = DropAllEmptyRows(anchor.Grid)+ anchor.Grid = CleanupOrphanColumns(anchor.Grid)+ anchor.Grid = CleanupOrphanRows(anchor.Grid)+ anchor.Rows = RowsToStrings(anchor.Grid)+ }}result = append(result, anchor)}diff --git a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.jsonindex 57a456586d..b2d2add670 100644--- a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json+++ b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json@@ -45,6 +45,55 @@"resolution": "FillCellTextFromBoxes assigns each box to the single tightest column, so a box overlapping several cells lands in only the tightest one (matching Python, which keeps the single best via its ov/_ov ordering). Regression: TestFillCellTextFromBoxes_EqualBoxRatioSingleAssignment.","tracking": "When several cells qualify for a box at equal box-area ratio, keep only the single best (max inter/boxArea, tie-broken by inter/cellArea, mirroring Python's ov/_ov ordering) instead of filling all.","reason": "Python's find_overlapped_with_threshold orders candidates by (ov=inter/boxArea, _ov=inter/cellArea) and returns the single best, using cell-area ratio as a tie-break. Go's FillCellTextFromBoxes filled EVERY qualifying cell and never considered inter/cellArea. So a box overlapping several cells at the same box-area ratio was duplicated into all of them in Go, whereas Python keeps only the one it best fills (highest cell-area ratio). Implementation divergence only (same grid construction on both sides). Exposed by the now-fixed TestFillCellTextFromBoxes_EqualBoxRatioSingleAssignment."+ },+ {+ "id": "table-html-emission-format",+ "tag": "go_intentional",+ "kind": "serialization_format",+ "applies_to": ["06_table_content.pdf", "18_table_caption.pdf"],+ "fields": ["RowsToHTML"],+ "permanent": true,+ "reason": "Pipeline-parity textSim<100% on these two PDFs is entirely the HTML serialization layer; gridSim=100% confirms cell text / row-col structure / span coverage are identical to Python. Go's RowsToHTML (table_html.go) emits compact HTML: no newlines between rows or before </table>, a <th> for every header row, covered cells skipped via the covered map, caption from item.Caption. Python's __html_table (deepdoc/vision/table_structure_recognizer.py:356) emits '\\n' before each row and before </table>, <td></td>/<th></th> for empty non-covered cells (Go emits '<td ></td>'), and DROPS an entire header row when every cell text in it already appeared in a previous header row ('if all([t in hdset for t in txts]): continue' — hdset mixes row indices and cell texts, so a duplicated-text header row silently vanishes in Python; Go keeps it). Covered-cell handling is identical on both sides (Python 'arr is None: continue' == Go covered-skip). Downstream (internal/parser/parser/pdf_postprocess.go:76, chunker) treats the table HTML as opaque section text; the divergence does not alter cell content or structure, and Go's compact HTML is token-friendlier for embedding while preserving all header rows Python would drop. Deliberate divergence, not a regression target."+ },+ {+ "id": "english-page-ocr-source-divergence",+ "tag": "go_intentional",+ "kind": "input_source_divergence",+ "applies_to": ["*"],+ "fields": ["processPageBoxes", "DetectEnglish", "hasCleanChars"],+ "permanent": true,+ "reason": "Pipeline-parity textSim<100% on English-document PDFs is dominated by an input-source divergence, not assembly logic. Python (deepdoc/parser/pdf_parser.py:1687) clears page chars for documents whose majority vote is English (is_english -> chars=[]) and runs pure OCR (__ocr: detect + recognize_batch); Go (internal/deepdoc/parser/pdf/parser.go:312 hasCleanChars) keeps the extracted chars and runs the char-merge path, so the two sides consume different inputs for the same PDF. Go already mirrors the detection (internal/deepdoc/parser/pdf/util/eng_detect.go DetectEnglish, document-level majority vote) but only records it as the IsEnglish metadata flag. Classified go_intentional: for text-based English PDFs Go's embedded-char extraction is more accurate than OCR (OCR introduces recognition/spacing variance), and scanned/noisy English pages are covered by Go's isScanNoise/IsGarbledPage fallback into ocrDetectAndRecognize. A follow-up can evaluate an OCR fallback for English pages where pdfplumber extraction is poor. Note: the parity harness already clears the replayed chars for is_english documents (engine.ClearChars in pipeline_parity_test.go), so both sides consume the same OCR input in replay; that is a measurement alignment, not a production behavior change."+ },+ {+ "id": "ocr-replay-full-vs-python-merge-truncation",+ "tag": "go_intentional",+ "kind": "output_completeness",+ "applies_to": ["eval_single_wide.pdf", "eval_two_overlap_no_gutter.pdf"],+ "fields": ["ocrDetectAndRecognize", "NaiveVerticalMerge"],+ "permanent": true,+ "reason": "Go's OCR-replay output is the FULL OCR dump (eval_single_wide: 18 boxes / 2936 chars; eval_two_overlap_no_gutter: 17 boxes / 2912 chars), while the Python text golden contains only the FIRST box's content (1519 / 1512 chars, i.e. ~48% of the OCR content is missing; verified py == dump_join prefix). The loss happens in Python's downstream assembly: for English documents Python clears chars (is_english -> chars=[], pdf_parser.py:1687) so mean_height becomes 0, and _naive_vertical_merge (pdf_parser.py:1007) then merges the overlapping OCR boxes into one box whose text retains only the first box — a Python-side truncation. Go keeps the complete OCR content, which is strictly more complete for downstream retrieval, so the divergence is deliberate (output completeness), not a Go regression. Python-side root cause is tracked as an issue (not fixed here per no-Python-change policy)."+ },+ {+ "id": "table-cell-fill-pre-vs-post-merge-order",+ "tag": "go_bug",+ "kind": "pipeline_order",+ "applies_to": ["14_text_table_interleaved.pdf"],+ "fields": ["FillCellTextFromBoxes", "processOneTable", "NaiveVerticalMerge"],+ "owner_fix_side": "go",+ "status": "open",+ "accepted_at_merge": true,+ "note": "KNOWN-UNFIXED AND ACCEPTED AT MERGE of PR #18516 (2026-08-19). The pre-vs-post-merge pipeline-order divergence on 14_text_table_interleaved is documented and tracked below but is NOT fixed in this PR. It is a go_bug content gap (gridSim < 100%) and will continue to report as FAIL in the manual (cgo && manual) pipeline-parity harness — which does not run in CI — so it does not block merge. It is the single remaining content-real-divergence and is scoped to this one PDF. Follow-up: instrument Python's self.boxes mid-pipeline (requires DEEPDOC_URL + py dump regeneration) to replicate the collapse-before-cell-fill exactly; see tracking below.",+ "tracking": "Align Go's table cell fill with Python's pipeline order: Python runs _naive_vertical_merge (and friends) BEFORE construct_table marks R/C and fills cells, so overlapping OCR boxes are collapsed first; Go's processOneTable -> FillCellTextFromBoxes runs before buildLayout's NaiveVerticalMerge, so overlapping boxes (e.g. OCR boxes 'Software Hardware' y=(270,300) and 'Hardware' y=(287.7,300)) are both matched to the same row and their text concatenated ('Software Hardware Hardware') where Python collapses them to one ('Software Hardware' in row1, row2 empty). Replicating the collapse before cell fill (or matching Python's R/C marking) should close the 14 gridSim gap.",+ "reason": "Go cell text fill (internal/deepdoc/parser/pdf/table/table_cells.go FillCellTextFromBoxes) runs on the raw OCR boxes inside processOneTable, BEFORE buildLayout's NaiveVerticalMerge. Python fills cells inside construct_table (deepdoc/vision/table_structure_recognizer.py:156) AFTER _naive_vertical_merge, so boxes overlapping in Y are merged first. For 14_text_table_interleaved table 2, OCR boxes 'Software Hardware' (y 270-300) and 'Hardware' (y 287.7-300) overlap; both go to Go's row band 2 (vertical-overlap ratio 0.53 vs 0.44) and concatenate to 'Software Hardware Hardware', while Python collapses them and the final HTML keeps 'Software Hardware' in row 1. Replicated Python find_overlapped_with_threshold + gather/layouts_cleanup (rows unchanged, R=row2) but the final HTML differs — the divergence stems from the box set that reach construct_table after Python's downstream merge, which dump data cannot fully reproduce; instrumenting Python's self.boxes mid-pipeline is required to close it exactly."+ },+ {+ "id": "table-rotation-split-vs-merged-grid",+ "tag": "go_intentional",+ "kind": "table_segmentation",+ "applies_to": ["table_rotation_test.pdf"],+ "fields": ["processOneTable", "MergeTablesAcrossPages", "GroupCells"],+ "permanent": true,+ "reason": "table_rotation_test.pdf contains three physical tables: page 1 has two 5x3 tables (grey header 'Header A/B/C', blue header), page 2 has the blue-header table rotated 90 degrees clockwise. Go emits three TableItems (5x3 + 5x3 + 3x6); the rotated table is content-correct (each row = Cell 4A, Cell 3A, Cell 2A, Cell 1A, Header A, plus one padding column). Python's golden emits TWO tables: the first 5x3 is identical, but the second is an 8x7 grid that MERGES the page-1 blue-header table with the page-2 rotated table — first 5 rows padded to 7 columns (4 empty columns + 3 data columns), last 3 rows the rotated content, with garbled cell text on both sides ('Header B Cell 1B', 'Cell 2A Cell 3A') inherited from the shared TSR dump (OCR box vs TSR cell boundary offset). gridSim 97.9% is therefore table segmentation + column-count divergence, NOT Go cell-content error: Go's split matches the physical PDF structure, and the residual text misplacement is identical on both sides (same TSR input), not a Go assembly bug. Python's merged 8x7 grid is a segmentation defect; Go's split is deliberately retained. Locked by TestPipelineParity which now exempts go_intentional rules via known_diffs.json."}]}diff --git a/internal/deepdoc/parser/pdf/table_extract.go b/internal/deepdoc/parser/pdf/table_extract.goindex d7ff1656e2..413662ecce 100644--- a/internal/deepdoc/parser/pdf/table_extract.go+++ b/internal/deepdoc/parser/pdf/table_extract.go@@ -5,6 +5,7 @@ import ("image""log/slog""math"+ "strings"lyt "ragflow/internal/deepdoc/parser/pdf/layout"tbl "ragflow/internal/deepdoc/parser/pdf/table"@@ -12,6 +13,31 @@ import (util "ragflow/internal/deepdoc/parser/pdf/util")+// pageNumCtxKey / tableIdxCtxKey let a DocAnalyzer that replays+// pre-computed Python intermediates (the pipeline-parity harness) recover+// which page, and which per-page table, a DLA/TSR call refers to. The+// production DeepDoc analyzer ignores them; they matter only for replay+// tests, where the DocAnalyzer interface cannot otherwise carry page+// context (pages are processed concurrently and TSR receives a cropped+// image with no page identifier).+type replayCtxKey int++const (+ pageNumCtxKey replayCtxKey = iota+ tableIdxCtxKey+ // cropOffXKey / cropOffYKey carry the crop origin (image pixels) of the+ // table region that processOneTable hands to TSR. A replay TableBuilder+ // reads them to map Python's page-space TSR cells into the exact crop+ // space Go uses, so cells and boxInCrop share one coordinate frame.+ cropOffXKey+ cropOffYKey+ // ocrBoxIdxCtxKey carries the index of the OCR detect box that a+ // per-crop OCRRecognize call belongs to (stamped by ocrDetectAndRecognize+ // before recognition). The Phase 3 replay analyzer reads it to return the+ // Python-dumped recognized text for that box instead of running OCR.+ ocrBoxIdxCtxKey+)+// enrichOnePageWithDeepDoc runs DLA+TSR for a single page and returns// worker-local artifacts. Boxes may be empty (image-only pages); the// function still runs DLA/TSR if pageImg is available so a page can@@ -49,6 +75,9 @@ func (p *Parser) enrichOnePageWithDeepDoc(ctx context.Context,if docAnalyzer == nil || !docAnalyzer.Health() || renderErr != nil || pageImg == nil {return pageBoxes, nil, nil}+ // Stamp the page number before DLA so a replay analyzer can map the call+ // back to the correct Python DLA page. The production analyzer ignores it.+ ctx = context.WithValue(ctx, pageNumCtxKey, pg)regions, err := p.inferDLA(ctx, docAnalyzer, pageImg)if err != nil {slog.Warn("DLA failed", "page", pg, "err", err)@@ -64,8 +93,11 @@ func (p *Parser) enrichOnePageWithDeepDoc(ctx context.Context,tableMatches := tbl.MatchTableRegions(annotated, regions, scale)var items []pdf.TableItem- for _, tm := range tableMatches {- item := p.processOneTable(ctx, pageImg, annotated, pg, docAnalyzer, tb, tm, scale)+ for i, tm := range tableMatches {+ // Stamp the per-page table index so a replay analyzer can map a+ // TSR call back to the correct Python intermediate table.+ tctx := context.WithValue(ctx, tableIdxCtxKey, i)+ item := p.processOneTable(tctx, pageImg, annotated, pg, docAnalyzer, tb, tm, scale)if item.ImageB64 != "" || len(item.Cells) > 0 || len(item.Positions) > 0 {items = append(items, item)}@@ -81,6 +113,8 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxesif cropErr != nil {return pdf.TableItem{}}+ cropOffX := math.Max(0, tm.Region.X0-util.TSRRegionMarginPx)+ cropOffY := math.Max(0, tm.Region.Y0-util.TSRRegionMarginPx)autoRotate := p.Config.AutoRotateTables != nil && *p.Config.AutoRotateTablesbestAngle := 0origW, origH := cropped.Bounds().Dx(), cropped.Bounds().Dy()@@ -94,12 +128,15 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxesif encErr != nil {slog.Warn("table PNG encode failed", "page", pageNum, "err", encErr)}- cells, tsrErr := p.inferTSR(ctx, tb, tsrImg)+ // Hand the crop origin to TSR so a replay TableBuilder can map Python+ // page-space TSR cells into this exact crop frame. Production callers+ // (DeepDocTableBuilder) ignore the value and use the cropped image pixels.+ tsrCtx := context.WithValue(ctx, cropOffXKey, cropOffX)+ tsrCtx = context.WithValue(tsrCtx, cropOffYKey, cropOffY)+ cells, tsrErr := p.inferTSR(tsrCtx, tb, tsrImg)if tsrErr != nil {slog.Warn("TSR failed", "page", pageNum, "err", tsrErr)}- cropOffX := math.Max(0, tm.Region.X0-util.TSRRegionMarginPx)- cropOffY := math.Max(0, tm.Region.Y0-util.TSRRegionMarginPx)var boxInCrop []pdf.TextBoxif tsrErr == nil && len(cells) > 0 {if bestAngle != 0 {@@ -117,6 +154,14 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxesif firstCellTop == 1e9 {firstCellTop = cells[0].Y0}+ // Collapse overlapping/adjacent OCR boxes before cell-fill, mirroring+ // Python's pipeline order — _naive_vertical_merge runs before+ // construct_table, so overlapping boxes are merged before they reach+ // cell assignment. Go runs table cell-fill per-page (here), before the+ // document-wide vertical merge in buildLayout, so it must run its own+ // collapse on this table's box subset first or overlapping OCR boxes+ // duplicate text across cells (e.g. 13_crosspage_table page 2:+ // '2024-43 2024-44' y=(1014,1045) + nested '2024-44' y=(1032,1045)).tableBoxes := make([]pdf.TextBox, 0, len(tm.BoxIdx))for _, idx := range tm.BoxIdx {b := boxes[idx]@@ -125,13 +170,6 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxes}tableBoxes = append(tableBoxes, b)}- // Collapse overlapping/adjacent OCR boxes before cell-fill, mirroring- // Python's pipeline order: _naive_vertical_merge runs before- // construct_table, so overlapping boxes are merged before they reach- // cell assignment. Go runs table cell-fill per-page (here), before the- // document-wide vertical merge in buildLayout, so it must run its own- // collapse on this table's box subset first or overlapping OCR boxes- // duplicate text across cells.tableBoxes = lyt.NaiveVerticalMerge(tableBoxes, nil, nil, nil)boxInCrop = make([]pdf.TextBox, 0, len(tableBoxes))for _, b := range tableBoxes {@@ -150,8 +188,24 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxesif len(cells) > 0 {grid = tb.GroupCells(cells)if len(grid) > 0 {+ // Pass the original TSR "table row" bboxes to cell fill so the+ // box→row matching uses the row component's own X range, exactly+ // like Python's find_overlapped_with_threshold over the raw row+ // components. Using the grid column union instead (wider X) can+ // push a col-0 box that straddles a row pair into the lower row+ // (13_crosspage_table page 2 rows 43/44: row 44's line starts at+ // x=106.9 while the grid union starts at 90.8, so '2024-43+ // 2024-44' overlaps row 44 ~21% against the true bbox but ~50%+ // against the union; Python keeps it in row 43).+ tsrRows := make([]pdf.TSRCell, 0, len(cells))+ for _, c := range cells {+ if strings.HasSuffix(c.Label, "table row") {+ tsrRows = append(tsrRows, c)+ }+ }+ tbl.SortYFirstly(tsrRows, 10)flat := tbl.FlattenGrid(grid)- tbl.FillCellTextFromBoxes(flat, boxInCrop)+ tbl.FillCellTextFromBoxesWithRows(flat, boxInCrop, tsrRows)idx := 0for ri := range grid {for ci := range grid[ri] {diff --git a/internal/deepdoc/parser/pdf/tool/char_adapter.go b/internal/deepdoc/parser/pdf/tool/char_adapter.goindex 8e2f30aee4..6a92bed42e 100644--- a/internal/deepdoc/parser/pdf/tool/char_adapter.go+++ b/internal/deepdoc/parser/pdf/tool/char_adapter.go@@ -14,8 +14,10 @@ import (// It is used for pipeline parity testing — same input chars as Python,// so any difference in pipeline output is a Go pipeline logic bug.type PythonCharEngine struct {- chars map[int][]pdf.TextChar // pageNum → chars- pages int+ chars map[int][]pdf.TextChar // pageNum → chars+ dims map[int][2]int // pageNum → [w,h] in ZM/image pixels (optional)+ isEnglish *bool // document-level verdict captured from the same Python run+ pages int}// LoadPythonChars loads chars from a charspy/{name}.json file.@@ -34,12 +36,35 @@ func LoadPythonChars(jsonPath string) (*PythonCharEngine, error) {FontName string `json:"fontname"`Size float64 `json:"size"`} `json:"pages"`+ // dims carries per-page image dimensions in ZM/image pixels, written+ // by the Phase 3 dump script so image-only pages (no embedded chars)+ // get a correctly sized placeholder page image. REQUIRED: a missing+ // dim would fall back to a 1x1 placeholder, corrupting pageHeight and+ // pushing every box off-page — LoadPythonChars fails fast instead.+ Dims [][]int `json:"dims"`+ // IsEnglish is the document-level english verdict from the SAME Python+ // run that produced the text golden. Capturing it (instead of+ // re-deriving via random sampling) keeps the replay input identical to+ // Python's. Optional: older dumps omit it.+ IsEnglish *bool `json:"is_english"`}if err := json.Unmarshal(data, &wrapper); err != nil {return nil, fmt.Errorf("parse charspy json: %w", err)}+ // dims are required (1:1 with pages) so image-only pages never fall back+ // to a degenerate placeholder that corrupts pageHeight. Fail fast on a bad+ // dump instead of silently degrading parity.+ if len(wrapper.Dims) != len(wrapper.Pages) {+ return nil, fmt.Errorf("charspy json dims count %d != pages %d (regenerate with dump_py_results.py)", len(wrapper.Dims), len(wrapper.Pages))+ }+ for i, d := range wrapper.Dims {+ if len(d) < 2 || d[0] <= 0 || d[1] <= 0 {+ return nil, fmt.Errorf("charspy json page %d invalid dims: %v", i, d)+ }+ }chars := make(map[int][]pdf.TextChar, len(wrapper.Pages))+ dims := make(map[int][2]int, len(wrapper.Dims))for pg, pageChars := range wrapper.Pages {result := make([]pdf.TextChar, len(pageChars))for i, c := range pageChars {@@ -56,7 +81,12 @@ func LoadPythonChars(jsonPath string) (*PythonCharEngine, error) {}chars[pg] = result}- return &PythonCharEngine{chars: chars, pages: len(wrapper.Pages)}, nil+ for pg, d := range wrapper.Dims {+ if len(d) >= 2 {+ dims[pg] = [2]int{d[0], d[1]}+ }+ }+ return &PythonCharEngine{chars: chars, dims: dims, isEnglish: wrapper.IsEnglish, pages: len(wrapper.Pages)}, nil}// ExtractChars returns all characters for the given page (0-indexed).@@ -67,14 +97,54 @@ func (e *PythonCharEngine) ExtractChars(pageNum int) ([]pdf.TextChar, error) {return e.chars[pageNum], nil}-// RenderPage returns a 1x1 placeholder PNG (not used in parity tests).+// PageChars returns the loaded per-page chars so callers can run document-level+// detection (e.g. util.DetectEnglish) before the pipeline consumes them.+func (e *PythonCharEngine) PageChars() map[int][]pdf.TextChar { return e.chars }++// IsEnglish returns the document-level english verdict captured from the same+// Python run that produced the dump, or nil when the dump omits it (older+// charspy files). Prefer it over re-deriving via random sampling, which is+// unstable on documents with small english runs.+func (e *PythonCharEngine) IsEnglish() *bool { return e.isEnglish }++// PageDims returns per-page image dimensions [w,h] in ZM/image pixels. They+// size the placeholder page image for image-only pages AND let TSR replay+// derive Python's page_cum_height (sum of prior page image heights), which+// the tsr_raw dump does not carry.+func (e *PythonCharEngine) PageDims() map[int][2]int { return e.dims }++// ClearChars empties every page's chars while keeping page count and dims.+// Python clears chars for is_english documents (pdf_parser.py:1687) and feeds+// pure OCR output instead, so replaying an English document must do the same —+// processPageBoxes then falls through to ocrDetectAndRecognize, which consumes+// the Phase 3 OCR dump. dims are untouched so image-only pages still get a+// correctly sized placeholder page image.+func (e *PythonCharEngine) ClearChars() {+ for pg := range e.chars {+ e.chars[pg] = nil+ }+}++// RenderPage is unused for parity — this engine supplies pre-loaded chars+// only. Return nil bytes (not an error) so callers that treat a nil render+// as "skip rendering" behave correctly.func (e *PythonCharEngine) RenderPage(pageNum int, dpi float64) ([]byte, error) {- return nil, fmt.Errorf("PythonCharEngine: RenderPage not supported")+ return nil, nil}-// RenderPageImage returns a 1x1 placeholder image (not used in parity tests).+// RenderPageImage returns a blank placeholder image (no error) sized from the+// dumped per-page image dims. This engine has no PDF bytes to rasterize; the+// DocAnalyzer ignores the pixels anyway, but processPage derives+// pageHeight/pageWidth from the placeholder size, so a wrong size would+// confound the assembly-parity measurement with bogus geometry. dims are+// required (LoadPythonChars fails fast when a dump omits them); a missing page+// here is a hard error, never a degenerate 1x1 placeholder.func (e *PythonCharEngine) RenderPageImage(pageNum int, dpi float64) (image.Image, error) {- return nil, fmt.Errorf("PythonCharEngine: RenderPageImage not supported")+ d, ok := e.dims[pageNum]+ if !ok || d[0] <= 0 || d[1] <= 0 {+ return nil, fmt.Errorf("page %d has no valid image dims (regenerate dump with dump_py_results.py)", pageNum)+ }+ return image.NewRGBA(image.Rect(0, 0, d[0], d[1])), nil}// PageCount returns the number of pages.diff --git a/internal/deepdoc/parser/pdf/tool/py_intermediates.go b/internal/deepdoc/parser/pdf/tool/py_intermediates.gonew file mode 100644index 0000000000..63a2b41f38--- /dev/null+++ b/internal/deepdoc/parser/pdf/tool/py_intermediates.go@@ -0,0 +1,170 @@+package tool++import (+ "encoding/json"+ "fmt"+ "os"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// ── Python intermediate loaders ────────────────────────────────────────────+//+// These parse the JSON dumps produced by dump_py_results.py+// (output/py/ocr/{dla,tsr_raw}/{name}.pdf.json). The coordinates in those+// dumps are in PDF-point space (÷DlaScale); replay adapters multiply by+// pdf.DlaScale before feeding Go, which expects image-pixel space.++// PythonDLAPage holds one page's DLA layout regions.+type PythonDLAPage struct {+ Page int+ Regions []PythonDLARegion+}++// PythonDLARegion mirrors a single DLA region in the dump.+// Coordinates are PDF points (top/left origin), NOT image pixels.+type PythonDLARegion struct {+ Type string+ X0 float64+ X1 float64+ Top float64+ Bottom float64+}++// LoadPythonDLA parses output/py/ocr/dla/{name}.pdf.json into per-page+// DLA regions. The file name uses the .pdf.json suffix produced by the+// dump script (name already includes the .pdf extension).+func LoadPythonDLA(jsonPath string) ([]PythonDLAPage, error) {+ data, err := os.ReadFile(jsonPath)+ if err != nil {+ return nil, fmt.Errorf("read dla json: %w", err)+ }+ var pages []PythonDLAPage+ if err := json.Unmarshal(data, &pages); err != nil {+ return nil, fmt.Errorf("parse dla json: %w", err)+ }+ return pages, nil+}++// PythonTSRCell mirrors one raw TSR component in the dump (table / table+// column / table row / table column header / table spanning cell ...).+// Coordinates are PDF points (top/left origin), NOT image pixels.+//+// Note: Y coordinates are PAGE-CUMULATIVE — Python's+// _map_tsr_component_to_page_space (pdf_parser.py:572-573) adds+// page_cum_height[page] to top/bottom. Replay adapters must subtract the+// cumulative offset (derived from charspy page dims) before mapping into+// Go's crop space.+type PythonTSRCell struct {+ TableIndex int `json:"table_index"`+ Page int `json:"page"`+ Label string `json:"label"`+ X0 float64 `json:"x0"`+ Y0 float64 `json:"y0"`+ X1 float64 `json:"x1"`+ Y1 float64 `json:"y1"`+ Text string `json:"text"`+}++// LoadPythonTSR parses output/py/ocr/tsr_raw/{name}.pdf.json into the raw+// TSR component list. The replay TableBuilder filters by (page, table_index)+// and maps each component into Go's crop space.+func LoadPythonTSR(jsonPath string) ([]PythonTSRCell, error) {+ data, err := os.ReadFile(jsonPath)+ if err != nil {+ return nil, fmt.Errorf("read tsr json: %w", err)+ }+ var cells []PythonTSRCell+ if err := json.Unmarshal(data, &cells); err != nil {+ return nil, fmt.Errorf("parse tsr json: %w", err)+ }+ return cells, nil+}++// ToDLARegion converts a Python PDF-point DLA region into a Go image-pixel+// DLARegion (×DlaScale). Confidence is pinned high so the replay does not+// trip the garbage-layout gate that Python never applied to these dumps.+func (r PythonDLARegion) ToDLARegion() pdf.DLARegion {+ return pdf.DLARegion{+ X0: r.X0 * pdf.DlaScale,+ Y0: r.Top * pdf.DlaScale,+ X1: r.X1 * pdf.DlaScale,+ Y1: r.Bottom * pdf.DlaScale,+ Label: r.Type,+ Confidence: 1.0,+ }+}++// ToTSRCell converts a Python PDF-point TSR component into a Go TSRCell in+// crop space. Python's Y is page-cumulative (page_cum_height added in+// _map_tsr_component_to_page_space), so it must first be reduced by+// cumOffsetPx — the sum of prior page image heights in image pixels, i.e.+// page_cum_height × DlaScale — to land in page-local points, then ×DlaScale+// and shifted by the crop origin (image pixels) so it shares the frame of+// Go's boxInCrop.+func (c PythonTSRCell) ToTSRCell(cropOffX, cropOffY, cumOffsetPx float64) pdf.TSRCell {+ return pdf.TSRCell{+ X0: c.X0*pdf.DlaScale - cropOffX,+ Y0: c.Y0*pdf.DlaScale - cumOffsetPx - cropOffY,+ X1: c.X1*pdf.DlaScale - cropOffX,+ Y1: c.Y1*pdf.DlaScale - cumOffsetPx - cropOffY,+ Label: c.Label,+ Text: c.Text,+ }+}++// ── Phase 3: OCR replay ────────────────────────────────────────────────────++// PythonOCRPage holds one page's final OCR-derived text boxes (the box list+// __ocr appends to parser.boxes). The coordinates are PAGE-POINTS but+// PAGE-CUMULATIVE — Python's __ocr runs on page images stacked at+// page_cum_height offsets, so box Y includes the sum of prior page heights.+// Replay adapters must subtract the cumulative offset (derived from charspy+// page dims) before mapping into Go's page-local image space.+type PythonOCRPage struct {+ Page int+ Boxes []PythonOCRBox+}++// PythonOCRBox mirrors one final OCR text box: axis-aligned bbox in+// page-cumulative points, plus the assembled recognized text. Confidence is+// not preserved in the dump (the assembled boxes do not carry it), so replay+// uses 0.+type PythonOCRBox struct {+ X0 float64+ Y0 float64+ X1 float64+ Y1 float64+ Text string+ Conf float64+}++// LoadPythonOCR parses output/py/ocr/ocr/{name}.pdf.json into per-page final+// OCR text boxes.+func LoadPythonOCR(jsonPath string) ([]PythonOCRPage, error) {+ data, err := os.ReadFile(jsonPath)+ if err != nil {+ return nil, fmt.Errorf("read ocr json: %w", err)+ }+ var pages []PythonOCRPage+ if err := json.Unmarshal(data, &pages); err != nil {+ return nil, fmt.Errorf("parse ocr json: %w", err)+ }+ return pages, nil+}++// ToOCRBox converts a Python page-cumulative-point OCR box into a Go OCRBox+// quad in page-local image-pixel space. The cumulative offset (sum of prior+// page image heights in image pixels, i.e. page_cum_height × DlaScale) is+// subtracted from Y first so boxes land on the current page, then ×DlaScale.+// The quad is the axis-aligned rectangle itself, so WarpCrop receives an+// identity de-skew and the emitted bounds match the dump's bbox after the+// /DlaScale conversion in ocrDetectAndRecognize.+func (b PythonOCRBox) ToOCRBox(cumOffsetPx float64) pdf.OCRBox {+ return pdf.OCRBox{+ X0: b.X0 * pdf.DlaScale, Y0: b.Y0*pdf.DlaScale - cumOffsetPx,+ X1: b.X1 * pdf.DlaScale, Y1: b.Y0*pdf.DlaScale - cumOffsetPx,+ X2: b.X1 * pdf.DlaScale, Y2: b.Y1*pdf.DlaScale - cumOffsetPx,+ X3: b.X0 * pdf.DlaScale, Y3: b.Y1*pdf.DlaScale - cumOffsetPx,+ }+}diff --git a/internal/deepdoc/parser/pdf/tool/testdata b/internal/deepdoc/parser/pdf/tool/testdatanew file mode 120000index 0000000000..4f7c116840--- /dev/null+++ b/internal/deepdoc/parser/pdf/tool/testdata@@ -0,0 +1 @@+../testdata\ No newline at end of filediff --git a/internal/deepdoc/parser/pdf/vm_whitespace_bridge_test.go b/internal/deepdoc/parser/pdf/vm_whitespace_bridge_test.gonew file mode 100644index 0000000000..9972d8780a--- /dev/null+++ b/internal/deepdoc/parser/pdf/vm_whitespace_bridge_test.go@@ -0,0 +1,55 @@+package pdf++import (+ "testing"++ lyt "ragflow/internal/deepdoc/parser/pdf/layout"+ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// TestVMWhitespaceGapBridge locks production lyt.NaiveVerticalMerge's inline+// whitespace gap bridge on the exact RAG PDF divergence:+//+// A (339.35-382.39) content+// WS (396.39-406.79) " " U+00A0 non-breaking space, non-zero width+// B (420.16-431.19) content+// C (436.16-447.20) content+//+// mh=9.361 so the merge threshold is mh*1.5=14.04. The whitespace box sits+// 14.00 below A — just under the threshold — so mergeColumnBoxes bridges it+// into A (extending A.Bottom to 406.79, OverlapX(A,WS)=1.0). That flips B's+// gap (420.16-406.79=13.37 < 14.04) from reject to merge, then C cascades+// (gap 4.97): the whole column collapses to ONE section. Without the bridge+// (whitespace ignored) A→B gap is 37.77 > 14.04 and B→C still merge, yielding+// two sections — so the count 1 specifically proves the bridge fired.+//+// The literal count is asserted directly (no hand-written replica of the merge+// algorithm) so the test locks production behavior itself, not+// production-vs-a-copy. Moved out of the cgo && manual parity file so it runs+// in CI: it uses only synthetic boxes and the production merge, no Python dump.+func TestVMWhitespaceGapBridge(t *testing.T) {+ boxes := []pdf.TextBox{+ // Content A: merged result of 3 preceding lines+ {X0: 37.6, X1: 491.0, Top: 339.35, Bottom: 382.39,+ Text: "生成文本再用standard分词建立索引", PageNumber: 1},+ // Whitespace: U+00A0 non-breaking space, has non-zero width+ {X0: 37.6, X1: 40.3, Top: 396.39, Bottom: 406.79,+ Text: " ", PageNumber: 1},+ // Content B: would be rejected without whitespace gap bridge+ {X0: 37.6, X1: 543.3, Top: 420.16, Bottom: 431.19,+ Text: "直接用rag分词建立索引", PageNumber: 1},+ // Content C: cascades after B merges+ {X0: 37.6, X1: 526.4, Top: 436.16, Bottom: 447.20,+ Text: "是在原文中并没有这样的文字", PageNumber: 1},+ }++ mhMap := map[int]float64{1: 9.361}+ mwMap := map[int]float64{1: 5}+ vmResult := lyt.NaiveVerticalMerge(boxes, mhMap, mwMap, nil)+ if len(vmResult) != 1 {+ t.Fatalf("whitespace gap bridge must collapse the 4-box column to 1 section, got %d", len(vmResult))+ }+ if vmResult[0].Bottom < 447.20 {+ t.Fatalf("bridged box must cascade-merge content C (bottom 447.20), got bottom %.2f", vmResult[0].Bottom)+ }+}diff --git a/internal/deepdoc/parser/type/types.go b/internal/deepdoc/parser/type/types.goindex 3b57e8e390..7b13430113 100644--- a/internal/deepdoc/parser/type/types.go+++ b/internal/deepdoc/parser/type/types.go@@ -99,6 +99,11 @@ type TextBox struct {LayoutNo stringColID intR int+ // IsOCR marks a box produced by an OCR pass (ocrDetectAndRecognize /+ // ocrMergeChars), as opposed to one built from embedded PDF chars+ // (CharsToBoxes). It scopes OCR-only post-processing (see layout.Dedup*)+ // so char-path digital PDFs are never silently de-duplicated.+ IsOCR bool// Post-TSR table annotation fields (Python: R/H/C/SP tags)RTop, RBott float64HTop, HBott float64