RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(deepdoc): drop nested OCR boxes before table cell fill + expose 13/14 parity gaps (#18524)
Jack20ee2dad2204
diff --git a/internal/deepdoc/parser/pdf/grid_structure_helpers_test.go b/internal/deepdoc/parser/pdf/grid_structure_helpers_test.gonew file mode 100644index 0000000000..f866eca122--- /dev/null+++ b/internal/deepdoc/parser/pdf/grid_structure_helpers_test.go@@ -0,0 +1,77 @@+//go:build cgo++package pdf++import (+ "fmt"+ "strings"+)++// gridShape returns a compact STRUCTURAL signature of a grid, ignoring all+// cell text: the row count and, when every row shares the same column count,+// "RxC"; otherwise "Rx[colcounts]" so divergent shapes stay distinguishable.+// Examples: "3x6", "8x7", "2x[2,1]". Two grids with identical shape share the+// same signature even when their cell text differs.+func gridShape(g [][]string) string {+ if len(g) == 0 {+ return "0x0"+ }+ cols := make([]int, len(g))+ allSame := true+ first := len(g[0])+ for i, row := range g {+ cols[i] = len(row)+ if cols[i] != first {+ allSame = false+ }+ }+ if allSame {+ return fmt.Sprintf("%dx%d", len(g), first)+ }+ parts := make([]string, len(cols))+ for i, c := range cols {+ parts[i] = fmt.Sprintf("%d", c)+ }+ return fmt.Sprintf("%dx[%s]", len(g), strings.Join(parts, ","))+}++// gridStructureSimilarity reports how closely two grids' ROW/COLUMN structure+// matches, independent of cell text. Unlike gridSim (CharSimilarity over+// joined cell text, which is order- and shape-blind), this catches table+// segmentation divergences: Go splitting a rotated table into 3x6 while Python+// merges it into 8x7 yields gridSim=100% (identical cell text) but+// structureSim<100% (different shape).+//+// Returns (similarity, detail) where detail is "<goShape> vs <pyShape>" for+// logging. Similarity is 100 when both grids are empty or have identical row+// counts and identical per-row column counts; otherwise it is graded by the+// fraction of structural elements that match (1 unit for matching row count ++// 1 unit per row whose column count matches, normalized).+func gridStructureSimilarity(goRows, pyRows [][]string) (float64, string) {+ gs, ps := gridShape(goRows), gridShape(pyRows)+ if gs == ps {+ return 100, gs+ }+ n := len(goRows)+ if len(pyRows) > n {+ n = len(pyRows)+ }+ if n == 0 {+ return 100, gs + " vs " + ps // both empty -> identical+ }+ match := 0+ if len(goRows) == len(pyRows) {+ match+++ }+ compared := len(goRows)+ if len(pyRows) < compared {+ compared = len(pyRows)+ }+ for i := 0; i < compared; i++ {+ if len(goRows[i]) == len(pyRows[i]) {+ match+++ }+ }+ total := 1 + compared+ return float64(match) / float64(total) * 100, gs + " vs " + ps+}diff --git a/internal/deepdoc/parser/pdf/grid_structure_test.go b/internal/deepdoc/parser/pdf/grid_structure_test.gonew file mode 100644index 0000000000..eb721a1416--- /dev/null+++ b/internal/deepdoc/parser/pdf/grid_structure_test.go@@ -0,0 +1,91 @@+//go:build cgo++package pdf++import (+ "math"+ "testing"+)++// mkgrid builds an r×c grid of empty cells (cell text is irrelevant to the+// structural metric under test).+func mkgrid(r, c int) [][]string {+ g := make([][]string, r)+ for i := range g {+ g[i] = make([]string, c)+ }+ return g+}++// TestGridStructureSimilarity verifies gridStructureSimilarity is SHAPE-AWARE:+// it catches table-segmentation divergences (e.g. Go splitting a rotated table+// into 3x6 while Python merges it into 8x7) even when both grids contain the+// identical cell text — which gridSim (CharSimilarity over joined text) cannot+// see. This is the regression guard behind surfacing table_rotation_test.pdf+// as INTENTIONAL instead of mislabeling it html-divergent.+func TestGridStructureSimilarity(t *testing.T) {+ mkTextGrid := func(rows ...[]string) [][]string {+ return rows+ }+ cases := []struct {+ name string+ goRows [][]string+ pyRows [][]string+ wantSim float64+ wantDetail string+ }{+ {+ name: "identical 3x6",+ goRows: mkgrid(3, 6),+ pyRows: mkgrid(3, 6),+ wantSim: 100, wantDetail: "3x6",+ },+ {+ name: "rotated 3x6 vs merged 8x7",+ goRows: mkgrid(3, 6),+ pyRows: mkgrid(8, 7),+ wantSim: 0,+ wantDetail: "3x6 vs 8x7",+ },+ {+ name: "both empty",+ goRows: nil,+ pyRows: nil,+ wantSim: 100, wantDetail: "0x0",+ },+ {+ name: "row count differs 5x3 vs 8x7",+ goRows: mkgrid(5, 3),+ pyRows: mkgrid(8, 7),+ wantSim: 0,+ wantDetail: "5x3 vs 8x7",+ },+ {+ name: "uneven column counts",+ goRows: mkTextGrid([]string{"a", "b"}, []string{"c"}),+ pyRows: mkTextGrid([]string{"a", "b"}, []string{"c", "d"}),+ wantSim: 66.67,+ wantDetail: "2x[2,1] vs 2x2",+ },+ {+ // Shape matches -> 100 even though cell TEXT differs. Confirms the+ // metric is purely structural, not a content re-test.+ name: "same shape, different text",+ goRows: mkTextGrid([]string{"x", "y"}, []string{"z", "w"}),+ pyRows: mkTextGrid([]string{"p", "q"}, []string{"r", "s"}),+ wantSim: 100,+ wantDetail: "2x2",+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ sim, detail := gridStructureSimilarity(tc.goRows, tc.pyRows)+ if math.Abs(sim-tc.wantSim) > 0.01 {+ t.Errorf("sim = %.2f, want %.2f", sim, tc.wantSim)+ }+ if detail != tc.wantDetail {+ t.Errorf("detail = %q, want %q", detail, tc.wantDetail)+ }+ })+ }+}diff --git a/internal/deepdoc/parser/pdf/layout/dedup_test.go b/internal/deepdoc/parser/pdf/layout/dedup_test.goindex 95aa5eaa29..67adfdf820 100644--- a/internal/deepdoc/parser/pdf/layout/dedup_test.go+++ b/internal/deepdoc/parser/pdf/layout/dedup_test.go@@ -6,23 +6,24 @@ import (pdf "ragflow/internal/deepdoc/parser/pdf/type")-// TestDedupIdenticalText locks that boxes carrying TRIM-EQUAL text (identical-// after strings.TrimSpace — trailing whitespace normalized) on the SAME page-// with DISJOINT Y bands are collapsed to one. OCR repeatedly detects the same-// text at different Y positions (09_crosspage_paragraph detects each paragraph-// 5-6x per page, all disjoint), and Python's downstream merge collapses these;-// Go must too, or the replay output duplicates paragraphs.+// TestDedupIdenticalText locks that a rolling-stride CHAIN of identical-text+// boxes (>= pseudoDupChainMin disjoint same-X copies, e.g. 09_crosspage_paragraph+// detects each paragraph 14-18x per page) is collapsed to one, while a+// cross-page copy and unrelated text are kept.func TestDedupIdenticalText(t *testing.T) {long := "paragraph one with enough words to qualify as a real paragraph duplicate text for collapsing"boxes := []pdf.TextBox{{Text: long, PageNumber: 0, Top: 100, Bottom: 115, X0: 60, X1: 520, IsOCR: true},- {Text: long, PageNumber: 0, Top: 300, Bottom: 315, X0: 60, X1: 520, IsOCR: true}, // disjoint dup -> drop+ {Text: long, PageNumber: 0, Top: 200, Bottom: 215, X0: 60, X1: 520, IsOCR: true},+ {Text: long, PageNumber: 0, Top: 300, Bottom: 315, X0: 60, X1: 520, IsOCR: true},+ {Text: long, PageNumber: 0, Top: 400, Bottom: 415, X0: 60, X1: 520, IsOCR: true},+ {Text: long, PageNumber: 0, Top: 500, Bottom: 515, X0: 60, X1: 520, IsOCR: true}, // 5-copy chain -> collapse{Text: long, PageNumber: 1, Top: 100, Bottom: 115, X0: 60, X1: 520, IsOCR: true}, // other page -> keep- {Text: "paragraph two", PageNumber: 0, Top: 200, Bottom: 215, IsOCR: true},+ {Text: "paragraph two", PageNumber: 0, Top: 600, Bottom: 615, IsOCR: true},}got := DedupIdenticalText(boxes)if len(got) != 3 {- t.Fatalf("want 3 boxes (1 disjoint dup dropped), got %d: %+v", len(got), got)+ t.Fatalf("want 3 boxes (5-copy chain collapsed to 1), got %d: %+v", len(got), got)}if got[0].Text != long || got[1].Text != long {t.Fatalf("page-0 and page-1 copies must both be kept in order")@@ -49,16 +50,20 @@ func TestDedupIdenticalText_YOverlap(t *testing.T) {}// TestDedupIdenticalText_WhitespaceSensitive ensures trimming does not merge-// boxes that differ only by trailing spaces into a false duplicate.+// boxes that differ only by trailing spaces into a false duplicate — the+// trimmed-equal copies are still grouped and a full CHAIN collapses.func TestDedupIdenticalText_WhitespaceSensitive(t *testing.T) {long := "a sufficiently long repeated sentence that qualifies as a paragraph"boxes := []pdf.TextBox{{Text: long, PageNumber: 0, Top: 10, Bottom: 20, X0: 60, X1: 520, IsOCR: true},- {Text: long + " ", PageNumber: 0, Top: 90, Bottom: 100, X0: 60, X1: 520, IsOCR: true}, // 80pt gap > 4x height+ {Text: long + " ", PageNumber: 0, Top: 90, Bottom: 100, X0: 60, X1: 520, IsOCR: true},+ {Text: long, PageNumber: 0, Top: 170, Bottom: 180, X0: 60, X1: 520, IsOCR: true},+ {Text: long + " ", PageNumber: 0, Top: 250, Bottom: 260, X0: 60, X1: 520, IsOCR: true},+ {Text: long, PageNumber: 0, Top: 330, Bottom: 340, X0: 60, X1: 520, IsOCR: true}, // 5-copy trimmed-equal chain}got := DedupIdenticalText(boxes)if len(got) != 1 {- t.Fatalf("trimmed-equal texts should be treated as duplicates, got %d", len(got))+ t.Fatalf("trimmed-equal 5-copy chain should collapse to 1, got %d", len(got))}}@@ -92,18 +97,40 @@ func TestDedupIdenticalText_AdjacentRepeatsKept(t *testing.T) {}}-// TestDedupIdenticalText_StridedPseudoDuplicate collapses identical text far-// apart (>4x height) at the same X — the OCR rolling-stride duplicate-// (eval_single_wide / 09_crosspage_paragraph).+// TestDedupIdenticalText_StridedPseudoDuplicate collapses a rolling-stride+// CHAIN of identical text far apart (>4x height) at the same X — the OCR+// rolling-stride duplicate (eval_single_wide / 09_crosspage_paragraph). A+// chain needs pseudoDupChainMin copies; short pairs are real content and are+// kept (see TestDedupIdenticalText_ShortPairKept).func TestDedupIdenticalText_StridedPseudoDuplicate(t *testing.T) {long := "a sufficiently long repeated sentence that qualifies as a paragraph duplicate"- boxes := []pdf.TextBox{- {Text: long, PageNumber: 0, Top: 100, Bottom: 115, X0: 60, X1: 520, IsOCR: true},- {Text: long, PageNumber: 0, Top: 300, Bottom: 315, X0: 60, X1: 520, IsOCR: true}, // 200pt gap -> drop+ var boxes []pdf.TextBox+ for i := 0; i < pseudoDupChainMin; i++ {+ top := float64(100 + i*200)+ boxes = append(boxes, pdf.TextBox{+ Text: long, PageNumber: 0, Top: top, Bottom: top + 15, X0: 60, X1: 520, IsOCR: true,+ })}got := DedupIdenticalText(boxes)if len(got) != 1 {- t.Fatalf("strided pseudo-duplicate must be dropped, got %d boxes", len(got))+ t.Fatalf("5-copy strided pseudo-duplicate chain must collapse to 1, got %d boxes", len(got))+ }+}++// TestDedupIdenticalText_ShortPairKept locks the eval_two_* fix: a same-text+// group of only 2-4 copies (distinct physical lines that happen to share text,+// e.g. the template rows of eval_two_wide_gutter / eval_two_indented_first_para)+// is NOT a rolling-stride OCR pseudo-duplicate — every copy is a real line and+// must be kept verbatim. Dropping any copy silently loses document content.+func TestDedupIdenticalText_ShortPairKept(t *testing.T) {+ row := "line xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxXx"+ boxes := []pdf.TextBox{+ {Text: row, PageNumber: 0, Top: 165, Bottom: 176, X0: 54, X1: 229, IsOCR: true},+ {Text: row, PageNumber: 0, Top: 300, Bottom: 311, X0: 54, X1: 229, IsOCR: true}, // far-apart identical row -> keep+ }+ got := DedupIdenticalText(boxes)+ if len(got) != 2 {+ t.Fatalf("2-copy identical row pair must be kept (real content), got %d boxes", len(got))}}diff --git a/internal/deepdoc/parser/pdf/layout/layout.go b/internal/deepdoc/parser/pdf/layout/layout.goindex 9d08343ce2..06eb409b3d 100644--- a/internal/deepdoc/parser/pdf/layout/layout.go+++ b/internal/deepdoc/parser/pdf/layout/layout.go@@ -22,12 +22,21 @@ import (// DedupIdenticalText collapses boxes whose text is byte-identical on the SAME// page AND whose Y bands are pairwise DISJOINT, keeping the first// (reading-order) occurrence. OCR repeatedly detects the same text at disjoint-// Y positions (09_crosspage_paragraph detects each paragraph 5-6x per page);+// Y positions (09_crosspage_paragraph detects each paragraph 14-18x per page);// Python's downstream merge collapses these, so Go must too or the replay// output duplicates every paragraph. Boxes that OVERLAP in Y are kept: multiple// columns / adjacent lines on one page legitimately share text (e.g.// eval_three_wide has 3 columns at the same Y). Different pages keep their own// copies (a cross-page paragraph appears once per page).+//+// A same-text group is treated as a pseudo-duplicate only when it forms a+// REGULAR rolling-stride CHAIN of at least pseudoDupChainMin boxes (same X,+// pairwise disjoint Y, gaps > 4x height). Shorter groups (2-4 copies, e.g. the+// identical template rows of eval_two_wide_gutter / eval_two_indented_first_para)+// are real document content — distinct physical lines that happen to share+// text — and are kept verbatim.+const pseudoDupChainMin = 5+func DedupIdenticalText(boxes []pdf.TextBox) []pdf.TextBox {type key struct {page int@@ -50,7 +59,12 @@ func DedupIdenticalText(boxes []pdf.TextBox) []pdf.TextBox {drop := make(map[int]bool, len(boxes))for _, idxs := range groups {- if len(idxs) < 2 {+ // A pseudo-duplicate needs a rolling-stride CHAIN of at least+ // pseudoDupChainMin detections. Isolated pairs / short groups of+ // identical text are real repeated lines and must be kept (e.g. the+ // eval_two_* template rows detected 2-4x, where dropping any copy+ // loses a real line).+ if len(idxs) < pseudoDupChainMin {continue}// Short identical texts (e.g. the repeated keyword 'Transformer' indiff --git a/internal/deepdoc/parser/pdf/parity_lock.go b/internal/deepdoc/parser/pdf/parity_lock.goindex 02aad7d412..80cf79dad6 100644--- a/internal/deepdoc/parser/pdf/parity_lock.go+++ b/internal/deepdoc/parser/pdf/parity_lock.go@@ -1,18 +1,23 @@package pdf// parityLockedGridPDFs returns the set of table PDFs that MUST hold full grid-// (cell-content) parity with Python. The pipeline-parity harness reads this to-// error if a locked PDF's gridSim drops below 100%.+// (cell-content AND structure) parity with Python. The pipeline-parity harness+// reads this to error if a locked PDF's gridSim OR structureSim drops below+// 100% (both must hold — gridSim alone is shape-blind and would miss a+// segmentation divergence with identical cell text).//-// Only PDFs that actually reach gridSim=100% (HTML-format-only divergence)-// belong here. Known-unfixed content gaps (e.g. 13_crosspage_table,-// 14_text_table_interleaved) must NOT be locked, or the harness both accepts-// and forbids the same gap — a contradiction. Keep this set in lock-step with-// known_diffs.json: a PDF listed here must have gridSim=100% (or be exempted-// as go_intentional), never an open/accepted_at_merge failure.+// Only PDFs that actually reach gridSim=100% AND structureSim=100% (the only+// divergence left being non-cell-text: caption/body/format outside table cells)+// belong here. Known-unfixed content/structure gaps (e.g. table_rotation_test)+// must NOT be locked, or the harness both accepts and forbids the same gap — a+// contradiction. Keep this set in lock-step with known_diffs.json: a PDF listed+// here must have gridSim=100% and structureSim=100% (or be exempted as+// go_intentional), never an open/accepted_at_merge failure.func parityLockedGridPDFs() map[string]bool {return map[string]bool{- "06_table_content.pdf": true,- "18_table_caption.pdf": true,+ "06_table_content.pdf": true,+ "13_crosspage_table.pdf": true,+ "14_text_table_interleaved.pdf": true,+ "18_table_caption.pdf": true,}}diff --git a/internal/deepdoc/parser/pdf/parity_lock_test.go b/internal/deepdoc/parser/pdf/parity_lock_test.goindex f348e0da9c..c4c70985b4 100644--- a/internal/deepdoc/parser/pdf/parity_lock_test.go+++ b/internal/deepdoc/parser/pdf/parity_lock_test.go@@ -3,26 +3,31 @@ package pdfimport "testing"// TestParityLockedGridExcludesUnfixed locks that known-unfixed content gaps are-// NOT in the locked grid-parity set. Locking a PDF whose gridSim is below 100%-// (13_crosspage_table open gap, 14_text_table_interleaved accepted_at_merge)-// makes the harness simultaneously accept and forbid the same gap. Only PDFs at-// gridSim=100% (HTML-format-only divergence: 06, 18) belong in the lock.+// NOT in the locked grid-parity set, and that every PDF at full grid+structure+// parity (06/13/14/18) IS locked. Locking a PDF whose gridSim is below 100%+// (e.g. table_rotation_test, INTENTIONAL structSim=35.7%) makes the harness+// simultaneously accept and forbid the same gap; only PDFs at gridSim=100% AND+// structureSim=100% (the remaining divergence is non-cell-text) belong in the+// lock. 13/14 were added once their grid+structure parity was reached (cell-fill+// and caption fixes); keep this in lock-step with parity_lock.go.func TestParityLockedGridExcludesUnfixed(t *testing.T) {locked := parityLockedGridPDFs()- // Known-unfixed content gaps must not be locked.- if locked["13_crosspage_table.pdf"] {- t.Error("13_crosspage_table is an open grid gap and must not be locked")- }- if locked["14_text_table_interleaved.pdf"] {- t.Error("14_text_table_interleaved is accepted_at_merge and must not be locked")+ // Known-unfixed structural gap must not be locked.+ if locked["table_rotation_test.pdf"] {+ t.Error("table_rotation_test has structSim<100 (go_intentional segmentation) and must not be locked")}- // PDFs that reach gridSim=100% (HTML-format-only divergence) stay locked.- if !locked["06_table_content.pdf"] {- t.Error("06_table_content reaches grid parity and must stay locked")- }- if !locked["18_table_caption.pdf"] {- t.Error("18_table_caption reaches grid parity and must stay locked")+ // PDFs that reach grid+structure parity (only non-cell-text divergence+ // left) stay locked.+ for _, name := range []string{+ "06_table_content.pdf",+ "13_crosspage_table.pdf",+ "14_text_table_interleaved.pdf",+ "18_table_caption.pdf",+ } {+ if !locked[name] {+ t.Errorf("%s reaches grid+structure parity and must stay locked", name)+ }}}diff --git a/internal/deepdoc/parser/pdf/parser_test.go b/internal/deepdoc/parser/pdf/parser_test.goindex fcef7c4845..c189c1f14f 100644--- a/internal/deepdoc/parser/pdf/parser_test.go+++ b/internal/deepdoc/parser/pdf/parser_test.go@@ -461,8 +461,10 @@ func TestOCR_MergeChars(t *testing.T) {})}-// TestTableSectionCaptionInHTML verifies mergeCaptions prepends table-// caption text before the HTML table, matching Python's caption handling.+// TestTableSectionCaptionInHTML verifies mergeCaptions retains a table+// caption by injecting it as a <caption> element INSIDE the table's HTML+// (matching Python's __html_table), instead of dropping it or emitting+// malformed pre-table text. The standalone caption section is removed.func TestTableSectionCaptionInHTML(t *testing.T) {// Simulate pipeline order: extractTableAndReplace → boxesToSections → mergeCaptions@@ -489,12 +491,21 @@ func TestTableSectionCaptionInHTML(t *testing.T) {Text: "表1: 交通工具等级",})- // Step 2: mergeCaptions prepends caption before HTML+ // Step 2: mergeCaptions injects caption as a <caption> element inside <table>figures := pdf.CollectFigures(sections)sections = tbl.MergeCaptions(sections, figures)- if !strings.HasPrefix(sections[0].Text, "表1: 交通工具等级<table") {- t.Errorf("expected caption before table HTML, got %q", sections[0].Text)+ got := sections[0].Text+ if !strings.Contains(got, "<caption>表1: 交通工具等级</caption>") {+ t.Errorf("expected a <caption> element inside the table HTML, got %q", got)+ }+ // <caption> must be a proper child of <table>, immediately after <table>.+ if i := strings.Index(got, "<table>"); i < 0 || !strings.Contains(got[i:i+len("<table><caption>")], "<caption>") {+ t.Errorf("expected <caption> immediately after <table>, got %q", got)+ }+ // The standalone caption section must be removed (no raw pre-table text).+ if strings.HasPrefix(got, "表1: 交通工具等级<table") {+ t.Errorf("caption must be a <caption> child of <table>, not pre-table text, got %q", got)}}diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_13_crosspage_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_13_crosspage_test.gonew file mode 100644index 0000000000..0266c9728a--- /dev/null+++ b/internal/deepdoc/parser/pdf/pipeline_parity_13_crosspage_test.go@@ -0,0 +1,186 @@+//go:build cgo && manual++package pdf++import (+ "encoding/json"+ "fmt"+ "os"+ "path/filepath"+ "strings"+ "testing"++ "ragflow/internal/deepdoc/parser/pdf/tool"+ pdf "ragflow/internal/deepdoc/parser/pdf/type"+ util "ragflow/internal/deepdoc/parser/pdf/util"+)++// TestPipelineParity13CrosspageSeam is the focused, cell-level counterpart to+// the harness divergence for 13_crosspage_table.pdf (tracked as go_bug rule+// table-crosspage-merge-seam-duplication in+// table/testdata/parity/known_diffs.json, now resolved).+//+// 13 is a multi-page table: Python's golden is ONE 81x5 grid. The divergence is+// NOT the cross-page merge — each per-page grid is correct and MergeTablesAcrossPages+// (table/table_merge.go) stacks them correctly. The real cause was+// FillCellTextFromBoxesWithRows (table/table_cells.go) picking the row with the+// MAX 2D-overlap, which placed a box spanning two adjacent data rows in the+// LOWER row. Python's construct_table groups boxes into rows by each box's TSR+// row label b["R"] — the row the box STARTS in — so the same box lands in the+// UPPER row (pdf_parser.py construct_table:176-192).+//+// Concretely the cell-level diff showed, at the page seam, Go emitting+//+// c0 = "2024-06 2024-07 2024-07" (3 tokens) one row too low+//+// while Python emits a single cell+//+// c0 = "2024-06 2024-07" (2 tokens)+//+// The fix prefers the topmost row band whose Y range CONTAINS the box's TOP edge+// (top-containment), matching Python's R-grouping. The residual+// leading/trailing-whitespace diffs are normalized by the harness gridSim guard+// and are not the real bug; after the fix the dominant divergence (seam+// duplication) is gone and gridSim=100.0%.+//+// This test asserts the seam-duplication bug signature (a Go cell whose+// whitespace-split tokens contain a repeated token, e.g. "2024-07 2024-07") is+// absent AND gridSim==100. It is the green regression guard for rule+// table-crosspage-merge-seam-duplication; if FillCellTextFromBoxesWithRows+// regresses to max-overlap row selection, this test fails again.+func TestPipelineParity13CrosspageSeam(t *testing.T) {+ name := "13_crosspage_table.pdf"+ base := filepath.Join("testdata", "output", "py", "ocr")+ charspyDir := filepath.Join("testdata", "charspy")+ dlaDir := filepath.Join(base, "dla")+ tsrDir := filepath.Join(base, "tsr_raw")+ tablesDir := filepath.Join(base, "tables")+ ocrDir := filepath.Join(base, "ocr")++ engine, err := tool.LoadPythonChars(filepath.Join(charspyDir, name+".json"))+ if err != nil {+ t.Fatal(err)+ }+ // Mirror the harness: English documents clear chars and fall through to+ // the OCR-replay path so both sides replay the same input.+ isEnglish := false+ if v := engine.IsEnglish(); v != nil {+ isEnglish = *v+ } else if pages, _ := engine.PageCount(); util.DetectEnglish(engine.PageChars(), pages, nil) {+ isEnglish = true+ }+ if isEnglish {+ engine.ClearChars()+ }++ // Register the replay TableBuilder so Go builds its grid from Python's+ // replayed TSR intermediates (not a fresh analysis). Without this the+ // parser falls back to the production builder and result.Tables is empty.+ RegisterReplayTableBuilder()+ cfg := pdf.DefaultParserConfig()+ cfg.SortByTop = true+ analyzer := NewPythonIntermediateDocAnalyzer(name, dlaDir, tsrDir, ocrDir, engine.PageDims())+ p := NewParser(cfg)+ result, err := p.ParseRaw(t.Context(), engine, analyzer)+ if err != nil {+ t.Fatal(err)+ }++ raw, err := os.ReadFile(filepath.Join(tablesDir, name+".json"))+ if err != nil {+ t.Fatal(err)+ }+ var td pyTableDump+ if err := json.Unmarshal(raw, &td); err != nil {+ t.Fatal(err)+ }++ goRows := goTableRows(result)+ pyRows, pyHasTables := loadPythonTables(t, filepath.Join(tablesDir, name+".json"))+ if !pyHasTables {+ t.Fatal("Python golden has no tables")+ }++ // Log the full cell-level diff so the divergence is fully exposed.+ n := len(goRows)+ if len(pyRows) > n {+ n = len(pyRows)+ }+ type cellDiff struct {+ row, col int+ goVal, pyVal string+ }+ var seamDup []cellDiff+ otherCount := 0+ gridSim := tool.CharSimilarity(joinGrid(goRows), joinGrid(pyRows))+ for i := 0; i < n; i++ {+ g := rowOrEmpty(goRows, i)+ py := rowOrEmpty(pyRows, i)+ cn := len(g)+ if len(py) > cn {+ cn = len(py)+ }+ for c := 0; c < cn; c++ {+ gc := cellOrEmpty(g, c)+ pyc := cellOrEmpty(py, c)+ if gc == pyc {+ continue+ }+ d := cellDiff{i, c, gc, pyc}+ if hasRepeatedToken(gc) {+ seamDup = append(seamDup, d)+ } else {+ // goVal is already TrimSpace'd by goTableRows; a cell whose+ // Python value only differs by surrounding whitespace is not a+ // content gap — gridSim (CharSimilarity, whitespace/order+ // insensitive) below 100 is the content-parity guard below.+ otherCount+++ }+ t.Logf("ROW %d c%d: GO=%q PY=%q", i, c, gc, pyc)+ }+ }+ t.Logf("13_crosspage_table seam-duplication cells=%d other-diff cells=%d gridSim=%.1f%%",+ len(seamDup), otherCount, gridSim)++ // Regression guard: the seam-duplication signature must not exist, and the+ // grid must retain full content parity (gridSim==100). A partial fix that+ // drops or adds cell content drops gridSim below 100 even if no+ // repeated-token cell remains.+ if len(seamDup) > 0 || gridSim < 100 {+ ex := "no cells"+ if len(seamDup) > 0 {+ ex = fmt.Sprintf("e.g. %q vs Python %q", seamDup[0].goVal, seamDup[0].pyVal)+ }+ t.Errorf("REGRESSION table-crosspage-merge-seam-duplication (resolved): %d Go cells contain a duplicated seam token, gridSim=%.1f%% (%s). "++ "FillCellTextFromBoxesWithRows must place a straddling box in the topmost row band containing its top edge (top-containment), matching Python's R-grouping, not max-overlap.",+ len(seamDup), gridSim, ex)+ }+}++// hasRepeatedToken reports whether s, split on whitespace, contains the same+// non-empty token twice consecutively — the signature of a box placed one row+// too low by FillCellTextFromBoxesWithRows, duplicating a value across the+// seam (e.g. "2024-07 2024-07").+func hasRepeatedToken(s string) bool {+ toks := strings.Fields(s)+ for i := 1; i < len(toks); i++ {+ if toks[i] != "" && toks[i] == toks[i-1] {+ return true+ }+ }+ return false+}++func rowOrEmpty(grid [][]string, i int) []string {+ if i < len(grid) {+ return grid[i]+ }+ return nil+}++func cellOrEmpty(row []string, c int) string {+ if c < len(row) {+ return row[c]+ }+ return ""+}diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_14_interleaved_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_14_interleaved_test.gonew file mode 100644index 0000000000..0b0842a8a8--- /dev/null+++ b/internal/deepdoc/parser/pdf/pipeline_parity_14_interleaved_test.go@@ -0,0 +1,142 @@+//go:build cgo && manual++package pdf++import (+ "encoding/json"+ "fmt"+ "os"+ "path/filepath"+ "testing"++ "ragflow/internal/deepdoc/parser/pdf/tool"+ pdf "ragflow/internal/deepdoc/parser/pdf/type"+ util "ragflow/internal/deepdoc/parser/pdf/util"+)++// TestPipelineParity14TextTableInterleaved is the regression guard for the+// 14_text_table_interleaved.pdf go_bug table-cell-fill-pre-vs-post-merge-order+// (table/testdata/parity/known_diffs.json — RESOLVED by this PR).+//+// The bug: processOneTable pre-merged the table's OCR boxes with+// NaiveVerticalMerge before cell fill, and a strictly-NESTED re-detected box+// ('Hardware' fully inside 'Software Hardware') was CONCATENATED by+// mergeTwoBoxes into 'Software Hardware Hardware'; Python keeps the longer+// box only. Fixed by dedupNestedBoxes (table_extract.go), which drops a box+// strictly nested inside another box whose trimmed text contains it, before+// the merge.+//+// The bug signature is a Go cell whose whitespace-split tokens contain a+// repeated token (e.g. 'Hardware Hardware'), which hasRepeatedToken detects.+// This test asserts that signature is absent AND that no NON-whitespace cell+// difference remains (14 now reaches gridSim=100%, i.e. cell char-multiset+// parity; whitespace-only diffs are tolerated because Python's golden keeps+// surrounding spaces that the Go pipeline trims).+func TestPipelineParity14TextTableInterleaved(t *testing.T) {+ name := "14_text_table_interleaved.pdf"+ base := filepath.Join("testdata", "output", "py", "ocr")+ charspyDir := filepath.Join("testdata", "charspy")+ dlaDir := filepath.Join(base, "dla")+ tsrDir := filepath.Join(base, "tsr_raw")+ tablesDir := filepath.Join(base, "tables")+ ocrDir := filepath.Join(base, "ocr")++ engine, err := tool.LoadPythonChars(filepath.Join(charspyDir, name+".json"))+ if err != nil {+ t.Fatal(err)+ }+ // Mirror the harness: English documents clear chars and fall through to+ // the OCR-replay path so both sides replay the same input.+ isEnglish := false+ if v := engine.IsEnglish(); v != nil {+ isEnglish = *v+ } else if pages, _ := engine.PageCount(); util.DetectEnglish(engine.PageChars(), pages, nil) {+ isEnglish = true+ }+ if isEnglish {+ engine.ClearChars()+ }++ // Register the replay TableBuilder so Go builds its grid from Python's+ // replayed TSR intermediates (not a fresh analysis).+ RegisterReplayTableBuilder()+ cfg := pdf.DefaultParserConfig()+ cfg.SortByTop = true+ analyzer := NewPythonIntermediateDocAnalyzer(name, dlaDir, tsrDir, ocrDir, engine.PageDims())+ p := NewParser(cfg)+ result, err := p.ParseRaw(t.Context(), engine, analyzer)+ if err != nil {+ t.Fatal(err)+ }++ raw, err := os.ReadFile(filepath.Join(tablesDir, name+".json"))+ if err != nil {+ t.Fatal(err)+ }+ var td pyTableDump+ if err := json.Unmarshal(raw, &td); err != nil {+ t.Fatal(err)+ }++ goRows := goTableRows(result)+ pyRows, pyHasTables := loadPythonTables(t, filepath.Join(tablesDir, name+".json"))+ if !pyHasTables {+ t.Fatal("Python golden has no tables")+ }++ // Log the full cell-level diff so the divergence is fully exposed.+ n := len(goRows)+ if len(pyRows) > n {+ n = len(pyRows)+ }+ type cellDiff struct {+ row, col int+ goVal, pyVal string+ }+ var dup []cellDiff+ otherCount := 0+ gridSim := tool.CharSimilarity(joinGrid(goRows), joinGrid(pyRows))+ for i := 0; i < n; i++ {+ g := rowOrEmpty(goRows, i)+ py := rowOrEmpty(pyRows, i)+ cn := len(g)+ if len(py) > cn {+ cn = len(py)+ }+ for c := 0; c < cn; c++ {+ gc := cellOrEmpty(g, c)+ pyc := cellOrEmpty(py, c)+ if gc == pyc {+ continue+ }+ d := cellDiff{i, c, gc, pyc}+ if hasRepeatedToken(gc) {+ dup = append(dup, d)+ } else {+ // goVal is already TrimSpace'd by goTableRows; a cell whose+ // Python value only differs by surrounding whitespace (or a+ // row-index shift that preserves the char multiset) is not a+ // content gap — gridSim (CharSimilarity, whitespace/order+ // insensitive) below 100 is the content-parity guard below.+ otherCount+++ }+ t.Logf("ROW %d c%d: GO=%q PY=%q", i, c, gc, pyc)+ }+ }+ t.Logf("14_text_table_interleaved dup-token cells=%d other-diff cells=%d gridSim=%.1f%%",+ len(dup), otherCount, gridSim)++ // Regression guard: the overlapping-box concatenation signature must not+ // exist, and the grid must retain full content parity (gridSim==100). A+ // partial fix that drops or adds cell content drops gridSim below 100 even+ // if no repeated-token cell remains.+ if len(dup) > 0 || gridSim < 100 {+ ex := "no cells"+ if len(dup) > 0 {+ ex = fmt.Sprintf("e.g. %q vs Python %q", dup[0].goVal, dup[0].pyVal)+ }+ t.Errorf("table-cell-fill-pre-vs-post-merge-order regression: %d Go cells contain a duplicated token from overlapping-box concatenation, gridSim=%.1f%% (%s). "++ "Go must merge overlapping OCR boxes before cell fill, matching Python's construct_table order.",+ len(dup), gridSim, ex)+ }+}diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_caption_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_caption_test.gonew file mode 100644index 0000000000..0d4d36457c--- /dev/null+++ b/internal/deepdoc/parser/pdf/pipeline_parity_caption_test.go@@ -0,0 +1,181 @@+//go:build cgo && manual++package pdf++import (+ "os"+ "path/filepath"+ "regexp"+ "strings"+ "testing"++ "ragflow/internal/deepdoc/parser/pdf/tool"+ pdf "ragflow/internal/deepdoc/parser/pdf/type"+ util "ragflow/internal/deepdoc/parser/pdf/util"+)++// replayPipelineText runs Go's replay pipeline for name (Python DLA/TSR/OCR+// replayed, mirroring the parity harness) and returns Go's assembled section+// text plus Python's text golden (metadata stripped). It is the shared setup+// for the caption / interleaved-paragraph divergence tests below.+func replayPipelineText(t *testing.T, name string) (goText, pyText string) {+ t.Helper()+ // Python chars are the raw replay input under testdata/charspy/; Python's+ // DLA/TSR/OCR intermediates and text golden live under testdata/output/py/ocr/.+ charspyDir := filepath.Join("testdata", "charspy")+ base := filepath.Join("testdata", "output", "py", "ocr")+ dlaDir := filepath.Join(base, "dla")+ tsrDir := filepath.Join(base, "tsr_raw")+ ocrDir := filepath.Join(base, "ocr")++ engine, err := tool.LoadPythonChars(filepath.Join(charspyDir, name+".json"))+ if err != nil {+ t.Fatal(err)+ }+ // Mirror the harness: English documents clear chars and fall through to+ // the OCR-replay path so both sides replay the same input.+ isEnglish := false+ if v := engine.IsEnglish(); v != nil {+ isEnglish = *v+ } else if pages, _ := engine.PageCount(); util.DetectEnglish(engine.PageChars(), pages, nil) {+ isEnglish = true+ }+ if isEnglish {+ engine.ClearChars()+ }++ RegisterReplayTableBuilder()+ cfg := pdf.DefaultParserConfig()+ cfg.SortByTop = true+ analyzer := NewPythonIntermediateDocAnalyzer(name, dlaDir, tsrDir, ocrDir, engine.PageDims())+ p := NewParser(cfg)+ result, err := p.ParseRaw(t.Context(), engine, analyzer)+ if err != nil {+ t.Fatal(err)+ }++ var b strings.Builder+ for _, s := range result.Sections {+ b.WriteString(s.Text)+ b.WriteByte('\n')+ }+ goText = b.String()++ pyData, err := os.ReadFile(filepath.Join(base, "text", name+".txt"))+ if err != nil {+ t.Fatalf("read Python text golden: %v", err)+ }+ pyText = tool.StripMeta(string(pyData))+ return goText, pyText+}++// TestPipelineParity14InterleavedParagraphDropped is the end-to-end regression+// guard for the resolved go_bug table-text-interleaved-paragraph-dropped:+// 14_text_table_interleaved.pdf has a body paragraph interleaved BEFORE Table 1+// ("... Table 1 shows revenuebycategory."). Python extracts it as a text+// section; Go must keep it too. The paragraph was previously dropped because an+// unanchored "Table N" caption regex mislabeled it a caption and MergeCaptions+// removed it. This test is GREEN now (Go retains the paragraph) and turns RED+// again if that content loss ever regresses.+func TestPipelineParity14InterleavedParagraphDropped(t *testing.T) {+ name := "14_text_table_interleaved.pdf"+ goText, pyText := replayPipelineText(t, name)+ const want = "revenuebycategory"+ if !strings.Contains(pyText, want) {+ t.Fatalf("test setup error: Python golden does not contain %q", want)+ }+ if !strings.Contains(goText, want) {+ t.Errorf("REGRESSION go_bug table-text-interleaved-paragraph-dropped: Go output is missing the interleaved body paragraph containing %q (Python keeps it). Real content loss, not HTML format.", want)+ }+}++// TestPipelineParityCaptionEmitted locks the fix for the content-loss half of+// go_bug table-html-emission-format: Go used to DROP the standalone caption+// section entirely, so the caption text vanished from output. Go now retains+// it by injecting the caption text as a <caption> element inside the target+// table's HTML (MergeCaptions -> injectCaption). This guard fails if the+// retention ever regresses (caption dropped -> no <caption> emitted).+//+// Scope: PDFs that carry DLA "table caption" boxes (06/13/14/18). For those,+// Go now emits <caption>; the emitted text is real PDF content, so it must+// also appear somewhere in Python's full extracted text (proving Go did not+// fabricate it). Go and Python CHOOSE/ORDER caption sentences differently (a+// caption-selection divergence - a separate, accepted go_intentional+// residual), so we assert each emitted caption SENTENCE, not the whole+// combined string byte-equally.+//+// Two regression guards:+// 1. VALID HTML: no <table> may contain more than one <caption> (HTML allows+// only one; extras are dropped by browsers/Markdown converters). A table+// with caption text both above and below used to emit multiple <caption>+// elements - that is the bug this guard locks against.+// 2. RETENTION: every caption sentence Go extracted must survive in the+// output AND be real PDF content (present in Python's full text).+func TestPipelineParityCaptionEmitted(t *testing.T) {+ pdfs := []string{+ "06_table_content.pdf",+ "13_crosspage_table.pdf",+ "14_text_table_interleaved.pdf",+ "18_table_caption.pdf",+ }+ // Expected caption sentences Go extracts for each PDF (whitespace collapsed).+ // These are the DLA "table caption" box texts; Go combines them into one+ // <caption> per table. They differ in wording/order from Python's <caption>+ // (selection divergence) but must all be retained as real content. 13's two+ // caption boxes (page 0 title + page 2 footer) now attach to the tall+ // cross-page table after the edge-distance fix.+ expected := map[string][]string{+ "06_table_content.pdf": {+ "Table 1: Quarterly sales by product category (in USD)",+ "The following table summarizes the quarterly sales performance across different product categories.",+ },+ "13_crosspage_table.pdf": {+ "Extended Financial Report",+ "Table: Monthly financial summary FY2024",+ },+ "14_text_table_interleaved.pdf": {+ "Table 1: Revenue",+ "Analysis continues with additional metrics.",+ "Table 2: Satisfaction Analysis continues with additional metrics.",+ },+ "18_table_caption.pdf": {+ "Table 1: Product specification comparison (2024 Q2)",+ "Product Specifications",+ },+ }+ captionRe := regexp.MustCompile(`(?is)<caption>(.*?)</caption>`)+ tableRe := regexp.MustCompile(`(?is)<table>.*?</table>`)+ wsRe := regexp.MustCompile(`\s+`)+ for _, name := range pdfs {+ t.Run(name, func(t *testing.T) {+ goText, pyText := replayPipelineText(t, name)+ // Guard 1: valid HTML - at most one <caption> per <table>.+ for _, tbl := range tableRe.FindAllString(goText, -1) {+ if n := strings.Count(tbl, "<caption>"); n > 1 {+ t.Errorf("REGRESSION table-html-emission-format: a <table> contains %d <caption> elements (HTML allows only one; extras are dropped by consumers): %q", n, tbl)+ }+ }+ caps := captionRe.FindAllStringSubmatch(goText, -1)+ if len(caps) == 0 {+ t.Fatalf("REGRESSION table-html-emission-format: Go emitted NO <caption> for %s - the standalone caption section was dropped (Python keeps caption text)", name)+ }+ // Guard 2: every expected sentence retained AND real content.+ var allGo strings.Builder+ for _, c := range caps {+ allGo.WriteString(wsRe.ReplaceAllString(strings.TrimSpace(c[1]), " "))+ allGo.WriteByte(' ')+ }+ goMerged := allGo.String()+ for _, w := range expected[name] {+ want := wsRe.ReplaceAllString(strings.TrimSpace(w), " ")+ if !strings.Contains(goMerged, want) {+ t.Errorf("REGRESSION table-html-emission-format: Go dropped caption sentence %q for %s", w, name)+ }+ if !strings.Contains(pyText, want) {+ t.Errorf("REGRESSION table-html-emission-format: Go's caption sentence %q for %s not found in Python text - may be fabricated, not real PDF content", w, name)+ }+ }+ t.Logf("%s: Go emits %d <caption> block(s), all sentences retained and real", name, len(caps))+ })+ }+}diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_eval_two_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_eval_two_test.gonew file mode 100644index 0000000000..cf18d8174d--- /dev/null+++ b/internal/deepdoc/parser/pdf/pipeline_parity_eval_two_test.go@@ -0,0 +1,117 @@+//go:build cgo && manual++package pdf++import (+ "os"+ "path/filepath"+ "strings"+ "testing"++ "ragflow/internal/deepdoc/parser/pdf/tool"+ pdf "ragflow/internal/deepdoc/parser/pdf/type"+ util "ragflow/internal/deepdoc/parser/pdf/util"+)++// runReplayText runs Go's pipeline for a PDF with Python's DLA/TSR/OCR+// replayed (mirroring the parity harness) and returns the assembled section+// text.+func runReplayText(t *testing.T, name string) string {+ t.Helper()+ base := filepath.Join("testdata", "output", "py", "ocr")+ engine, err := tool.LoadPythonChars(filepath.Join("testdata", "charspy", name+".json"))+ if err != nil {+ t.Fatal(err)+ }+ isEnglish := false+ if v := engine.IsEnglish(); v != nil {+ isEnglish = *v+ } else if pages, _ := engine.PageCount(); util.DetectEnglish(engine.PageChars(), pages, nil) {+ isEnglish = true+ }+ if isEnglish {+ engine.ClearChars()+ }+ RegisterReplayTableBuilder()+ cfg := pdf.DefaultParserConfig()+ cfg.SortByTop = true+ analyzer := NewPythonIntermediateDocAnalyzer(name, filepath.Join(base, "dla"), filepath.Join(base, "tsr_raw"), filepath.Join(base, "ocr"), engine.PageDims())+ p := NewParser(cfg)+ result, err := p.ParseRaw(t.Context(), engine, analyzer)+ if err != nil {+ t.Fatal(err)+ }+ var sb strings.Builder+ for _, s := range result.Sections {+ sb.WriteString(s.Text)+ sb.WriteByte('\n')+ }+ return sb.String()+}++// loadPyGoldenText returns the Python golden section text (meta line+// stripped) for a PDF.+func loadPyGoldenText(t *testing.T, name string) string {+ t.Helper()+ raw, err := os.ReadFile(filepath.Join("testdata", "output", "py", "ocr", "text", name+".txt"))+ if err != nil {+ t.Fatal(err)+ }+ return stripMetaLine(string(raw))+}++func stripMetaLine(s string) string {+ if i := strings.LastIndex(s, "\n#@meta"); i >= 0 {+ return s[:i]+ }+ return s+}++// assertTextParity asserts Go's assembled section text is structurally and+// content-identical to Python's golden: textSim==100% (same non-space char+// multiset, i.e. no content lost or added) AND same section count (line+// structure preserved). Both sides consume the same replayed intermediate, so+// equivalent processing logic must yield identical output.+func assertTextParity(t *testing.T, name string) {+ t.Helper()+ goText := runReplayText(t, name)+ pyText := stripMetaLine(loadPyGoldenText(t, name))++ goSections := nonEmptyLines(goText)+ pySections := nonEmptyLines(pyText)+ sim := tool.CharSimilarity(goText, pyText)++ if sim < 100 || len(goSections) != len(pySections) {+ t.Errorf("OPEN go_bug (eval_two_* content/structure loss): %s sections GO=%d PY=%d textSim=%.1f%%",+ name, len(goSections), len(pySections), sim)+ t.Logf("GO text:\n%s", goText)+ t.Logf("PY golden:\n%s", pyText)+ }+}++func nonEmptyLines(s string) []string {+ var out []string+ for _, l := range strings.Split(s, "\n") {+ if strings.TrimSpace(l) != "" {+ out = append(out, l)+ }+ }+ return out+}++// TestPipelineParityEvalTwoWideGutter: PDF = two columns (16 lines each).+// Go's text assembly DROPS 2 real lines (one '...Xx' and one '...xXx',+// 30 vs 32 line-fragments), while Python keeps all 32. Tracks the content+// loss reported by the harness (textSim 96.7%).+func TestPipelineParityEvalTwoWideGutter(t *testing.T) {+ assertTextParity(t, "eval_two_wide_gutter.pdf")+}++// TestPipelineParityEvalTwoIndentedFirstPara: PDF = left column 8 independent+// lines (overlapping middle text), right column 16. Go COLLAPSES the left+// column's 8 lines into ONE section (3 vs 16 sections, 23 vs 24+// line-fragments), losing the per-line structure Python preserves. Tracks+// the harness textSim 97.9% gap.+func TestPipelineParityEvalTwoIndentedFirstPara(t *testing.T) {+ assertTextParity(t, "eval_two_indented_first_para.pdf")+}diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_test.goindex 36a3bc829d..3acc659845 100644--- a/internal/deepdoc/parser/pdf/pipeline_parity_test.go+++ b/internal/deepdoc/parser/pdf/pipeline_parity_test.go@@ -33,12 +33,16 @@ import (//// Result classification (both sims are always reported):// - textSim==100% -> PASS (fully aligned, includes table HTML byte-identical)-// - table PDF, gridSim==100% but textSim<100% -> GRID_OK: cell content-// matches Python, the residual gap is the HTML serialization layer only.-// Reported separately (not labeled PASS) and not counted as a content-// failure; classified as go_intentional/go_bug via known_diffs.json.-// - table PDF, gridSim<100% (or non-table textSim<100%) -> FAIL: Go assembly-// content genuinely diverges from Python.+// - table PDF, gridSim==100% AND structSim==100% but textSim<100% ->+// NONCELL_TEXT: cell content AND structure match Python, the residual gap+// is non-cell-text (caption/body paragraph Go omits, and/or HTML tag/format+// differences). Reported separately (not labeled PASS) and not counted as a+// content failure; classified as go_intentional/go_bug via known_diffs.json.+// - table PDF, gridSim<100% OR structSim<100% (or non-table textSim<100%) ->+// FAIL if not exempted by a go_intentional rule; INTENTIONAL if a+// go_intentional rule names this exact PDF (Go judged at least as good as+// Python, e.g. table segmentation). Any genuine Go content regression is a+// FAIL.func TestPipelineParity(t *testing.T) {charspyDir := filepath.Join("testdata", "charspy")pyTextDir := filepath.Join("testdata", "output", "py", "ocr", "text")@@ -54,13 +58,15 @@ func TestPipelineParity(t *testing.T) {filter := common.GetEnv(common.EnvBatchParityFilter)- // TSR-replay table PDFs that must hold full grid content parity — guards- // the replay TableIndex json tag / per-page index mapping / cumulative-Y- // handling against regressions. table_rotation_test.pdf is exempted: its- // gridSim<100% gap is the table segmentation divergence documented as- // go_intentional (rule table-rotation-split-vs-merged-grid in- // known_diffs.json), where Go's split matches the physical PDF and- // Python's merged 8x7 grid is a segmentation defect.+ // TSR-replay table PDFs that must hold full grid content + structure+ // parity — guards the replay TableIndex json tag / per-page index mapping+ // / cumulative-Y handling / segmentation against regressions.+ // table_rotation_test.pdf is exempted (not in the lock set): its+ // STRUCTURAL divergence (Go 3x6 split vs Python 8x7 merge, structSim<100%+ // though gridSim=100% because cell text is identical) is the table+ // segmentation divergence documented as go_intentional (rule+ // table-rotation-split-vs-merged-grid in known_diffs.json), where Go's+ // split matches the physical PDF and Python's merged grid is a defect.lockedGridPDFs := parityLockedGridPDFs()// go_intentional rules exempt their applies_to PDFs from the FAIL count:@@ -86,7 +92,7 @@ func TestPipelineParity(t *testing.T) {// keyed on the analyzer type, so non-replay parses are unaffected.RegisterReplayTableBuilder()- total, passed, htmlDivergent, intentional, failed := 0, 0, 0, 0, 0+ total, passed, noncellText, intentional, failed := 0, 0, 0, 0, 0for _, e := range entries {if e.IsDir() || !strings.HasSuffix(e.Name(), ".json") {continue@@ -161,10 +167,12 @@ func TestPipelineParity(t *testing.T) {sim := tool.CharSimilarity(goText.String(), tool.StripMeta(string(pyData)))// Phase 2: tables are built from Python's replayed TSR, so compare the- // reconstructed grid content (cell text) rather than the rendered HTML- // (which differs in whitespace/join/header markup). Grid content parity- // isolates Go's GroupCells + FillCellTextFromBoxes assembly; HTML-only- // differences are expected and not a Go logic bug.+ // reconstructed grid content (cell text) rather than the rendered HTML.+ // Grid content parity (gridSim + structSim) isolates Go's GroupCells ++ // FillCellTextFromBoxes assembly. When gridSim AND structSim are 100%+ // but textSim<100%, the residual gap is NON-CELL-TEXT: a <caption> Go+ // omits, an interleaved body paragraph Go drops, and/or HTML tag/format+ // differences — none of which is a table-cell-assembly bug.pyRows, pyHasTables := loadPythonTables(t, filepath.Join(tablesDir, name+".json"))goRows := goTableRows(result)goHasTables := len(goRows) > 0@@ -178,36 +186,59 @@ func TestPipelineParity(t *testing.T) {status = "PASS"detail = fmt.Sprintf("textSim=%.1f%%", sim)case pyHasTables || goHasTables:- // Table PDFs not textSim=100% are judged on cell content first.- // gridSim==100% means Go's GroupCells + FillCellTextFromBoxes- // assembly matches Python's grid; the residual textSim gap is the- // HTML serialization layer (whitespace/join/header markup). That is- // a real divergence but HTML-only, so it is reported as GRID_OK —- // never mislabeled PASS — and classified via known_diffs.json.+ // Table PDFs not textSim=100% are judged on cell content AND+ // structure. gridSim (CharSimilarity over joined cell text) is+ // order- and shape-blind, so a structural divergence (e.g. Go+ // splitting a rotated table into 3x6 while Python merges it into+ // 8x7) reads as gridSim=100% even though the grids are not the+ // same table. gridStructureSimilarity is SHAPE-AWARE: it compares+ // row/column structure ignoring cell text, so it catches exactly+ // that class of divergence.gridSim := tool.CharSimilarity(joinGrid(goRows), joinGrid(pyRows))- if gridSim >= 100.0 {- htmlDivergent++- status = "GRID_OK"- detail = fmt.Sprintf("gridSim=%.1f%% textSim=%.1f%% (HTML-format divergence)", gridSim, sim)+ structureSim, shapeDetail := gridStructureSimilarity(goRows, pyRows)+ // Full content parity = identical cell text AND identical+ // structure. Only then is the residual textSim gap purely the+ // non-cell-text layer (caption/body/format outside table cells),+ // reported as NONCELL_TEXT.+ contentMatch := gridSim >= 100.0 && structureSim >= 100.0+ if contentMatch {+ // Cells + structure match Python, but the full text still+ // diverges — the gap is OUTSIDE table cells: caption text+ // Go omits, an interleaved body paragraph Go drops, and/or+ // HTML tag/format differences. Label it noncell-text (not+ // "html-divergent", which wrongly implies content matches)+ // so the divergence is named and not hidden.+ noncellText+++ status = "NONCELL_TEXT"+ detail = fmt.Sprintf("gridSim=%.1f%% structSim=%.1f%% (%s) textSim=%.1f%% (non-cell-text divergence: caption/body/format outside table cells)", gridSim, structureSim, shapeDetail, sim)} else if intentionalPDF(name) {- // go_intentional rule covers this PDF: the gridSim gap is a- // documented, deliberate divergence (Go at least as good as- // Python, e.g. table segmentation), so it is reported as- // INTENTIONAL and not counted as a content failure. It is- // still logged so a reader can see the gap.+ // go_intentional rule covers this PDF: the divergence (cell+ // content and/or structure) is a documented, deliberate one+ // where Go is judged at least as good as Python (e.g. table+ // segmentation). Reported as INTENTIONAL and not counted as a+ // content failure. Still logged so a reader can see the gap.intentional++status = "INTENTIONAL"- detail = fmt.Sprintf("gridSim=%.1f%% textSim=%.1f%% (go_intentional: %s)", gridSim, sim, intentionalRuleID(knownDiffRules, name))+ detail = fmt.Sprintf("gridSim=%.1f%% structSim=%.1f%% (%s) textSim=%.1f%% (go_intentional: %s)", gridSim, structureSim, shapeDetail, sim, intentionalRuleID(knownDiffRules, name))} else {failed++status = "FAIL"- detail = fmt.Sprintf("gridSim=%.1f%% textSim=%.1f%% (table grid content differs)", gridSim, sim)+ detail = fmt.Sprintf("gridSim=%.1f%% structSim=%.1f%% (%s) textSim=%.1f%% (table grid content/structure differs)", gridSim, structureSim, shapeDetail, sim)+ }+ // Regression guard: a go_intentional PDF whose divergence is+ // STRUCTURAL (Go's table segmentation differs from Python's) must+ // surface as INTENTIONAL, never be hidden under gridSim=100% as+ // html-divergent. This is what caught table_rotation_test.pdf (Go+ // 3x6 split vs Python 8x7 merge) being mislabeled before the+ // structure metric existed.+ if intentionalPDF(name) && structureSim < 100.0 && status != "INTENTIONAL" {+ t.Errorf("REGRESSION %s: go_intentional structural divergence misclassified %s (must be INTENTIONAL)", name, status)}// Regression lock: TSR-replay table PDFs that reached full grid- // content parity must stay there (protects against regressions in- // the replay index/Y mapping).- if lockedGridPDFs[name] && gridSim < 100.0 {- t.Errorf("LOCKED %s: gridSim=%.1f%% regressed below 100%%", name, gridSim)+ // content + structure parity must stay there (protects against+ // regressions in the replay index/Y mapping or segmentation).+ if lockedGridPDFs[name] && (gridSim < 100.0 || structureSim < 100.0) {+ t.Errorf("LOCKED %s: gridSim=%.1f%% structSim=%.1f%% regressed below 100%%", name, gridSim, structureSim)}default:failed++@@ -221,7 +252,7 @@ func TestPipelineParity(t *testing.T) {if total == 0 {t.Skip("no charspy/ files found")}- t.Logf("Pipeline parity: aligned=%d html-divergent=%d intentional=%d failed=%d total=%d", passed, htmlDivergent, intentional, failed, total)+ t.Logf("Pipeline parity: aligned=%d noncell-text=%d intentional=%d failed=%d total=%d", passed, noncellText, intentional, failed, total)if failed > 0 {t.Errorf("%d parity failures — Go pipeline content differs from Python (grid/text)", failed)}diff --git a/internal/deepdoc/parser/pdf/table/caption_kind_test.go b/internal/deepdoc/parser/pdf/table/caption_kind_test.gonew file mode 100644index 0000000000..59dc97a0a8--- /dev/null+++ b/internal/deepdoc/parser/pdf/table/caption_kind_test.go@@ -0,0 +1,87 @@+package table++import (+ "strings"+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// TestCaptionKind_MidSentenceTableMentionNotCaption locks the root cause of+// go_bug table-text-interleaved-paragraph-dropped: a body paragraph that only+// MENTIONS "Table 1" mid-sentence must NOT be classified as a table caption.+// Python's is_caption uses re.match (start-anchored), so it keeps such a+// paragraph as a normal text section; Go must match that. Before the fix,+// reTableCaptionText matched "Table N" anywhere in the string, so the+// paragraph was mislabeled a caption and silently dropped by MergeCaptions.+func TestCaptionKind_MidSentenceTableMentionNotCaption(t *testing.T) {+ para := "The following analysis compares product performance across different market segments. Table 1 shows revenuebycategory."+ s := pdf.Section{Text: para, LayoutType: pdf.LayoutTypeText}+ if got := CaptionKind(s); got != "" {+ t.Errorf("CaptionKind(%q) = %q, want \"\" (mid-sentence 'Table 1' is not a caption)", para, got)+ }+}++// TestCaptionKind_StartsWithTableIsCaption confirms the legitimate case still+// classifies a line that BEGINS with "Table N" as a table caption (mirrors+// Python re.match).+func TestCaptionKind_StartsWithTableIsCaption(t *testing.T) {+ s := pdf.Section{Text: "Table 1: Revenue", LayoutType: pdf.LayoutTypeText}+ if got := CaptionKind(s); got != pdf.LayoutTypeTable {+ t.Errorf("CaptionKind(%q) = %q, want %q", s.Text, got, pdf.LayoutTypeTable)+ }+ s2 := pdf.Section{Text: "Figure 2: system architecture overview", LayoutType: pdf.LayoutTypeText}+ if got := CaptionKind(s2); got != pdf.LayoutTypeFigure {+ t.Errorf("CaptionKind(%q) = %q, want %q", s2.Text, got, pdf.LayoutTypeFigure)+ }+}++// TestIsCaptionBox_MidSentenceTableMention confirms IsCaptionBox (via reCaption)+// also keeps mid-sentence "Table N" as non-caption, mirroring Python re.match.+func TestIsCaptionBox_MidSentenceTableMention(t *testing.T) {+ para := "The following analysis compares product performance across different market segments. Table 1 shows revenuebycategory."+ if IsCaptionBox(para, pdf.LayoutTypeText) {+ t.Errorf("IsCaptionBox(%q) = true, want false (mid-sentence 'Table 1' is not a caption)", para)+ }+ if !IsCaptionBox("Table 1: Revenue", pdf.LayoutTypeText) {+ t.Errorf("IsCaptionBox(%q) = false, want true (starts with 'Table 1')", "Table 1: Revenue")+ }+}++// TestMergeCaptions_KeepsInterleavedParagraph locks the end-to-end behavior:+// an interleaved body paragraph that mentions "Table 1" must survive+// MergeCaptions as its own text section (Python keeps it too), not be dropped+// or merged into the table.+func TestMergeCaptions_KeepsInterleavedParagraph(t *testing.T) {+ sections := []pdf.Section{+ {Text: "Product Analysis Report", LayoutType: pdf.LayoutTypeTitle,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 192, Right: 404, Top: 65, Bottom: 82}}},+ {Text: "The following analysis compares product performance across different market segments. Table 1 shows revenuebycategory.",+ LayoutType: pdf.LayoutTypeText,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 60, Right: 524, Top: 101, Bottom: 124}}},+ {Text: "<table><tr><td >Category</td></tr></table>", LayoutType: pdf.LayoutTypeTable,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 159, Right: 393, Top: 130, Bottom: 336}}},+ }+ figures := pdf.CollectFigures(sections)+ result := MergeCaptions(sections, figures)+ if len(result) != 3 {+ t.Fatalf("expected 3 sections (title + paragraph + table), got %d: %v", len(result), textsOf(result))+ }+ found := false+ for _, s := range result {+ if strings.Contains(s.Text, "revenuebycategory") {+ found = true+ }+ }+ if !found {+ t.Errorf("interleaved paragraph dropped; sections = %v", textsOf(result))+ }+}++func textsOf(secs []pdf.Section) []string {+ out := make([]string, 0, len(secs))+ for _, s := range secs {+ out = append(out, s.Text)+ }+ return out+}diff --git a/internal/deepdoc/parser/pdf/table/merge_captions.go b/internal/deepdoc/parser/pdf/table/merge_captions.goindex b7323f38f4..cafe8c2e78 100644--- a/internal/deepdoc/parser/pdf/table/merge_captions.go+++ b/internal/deepdoc/parser/pdf/table/merge_captions.go@@ -1,11 +1,31 @@package tableimport (+ "html"+ "sort"+ "strings"+pdf "ragflow/internal/deepdoc/parser/pdf/type")+// captionText is a caption box's text plus its top edge, used to order+// multiple captions of one table in READING order (top→bottom) before+// concatenation. Section order is not guaranteed to match the PDF layout+// (e.g. 06's lower caption box precedes the upper one in sections), so the+// top coordinate is carried explicitly.+type captionText struct {+ top float64+ text string+}+func MergeCaptions(sections []pdf.Section, figures []pdf.Section) []pdf.Section {captions := make([]int, 0, 4)+ // Group caption texts by the target section index they attach to, so+ // multiple caption boxes for the SAME table/figure collapse into a single+ // <caption> element. HTML allows only one <caption> per <table>; injecting+ // one per box would emit invalid HTML whose extra <caption>s consumers+ // (browsers, HTML→Markdown converters) silently drop — a real content loss.+ byTarget := make(map[int][]captionText)for i, s := range sections {captionType := CaptionKind(s)if captionType == "" {@@ -13,15 +33,16 @@ func MergeCaptions(sections []pdf.Section, figures []pdf.Section) []pdf.Section}target := findNearestParent(i, s, sections, figures, captionType)if target >= 0 {- // For table sections, prepend caption before the HTML table- // (matching Python's _extract_table_figure caption->construct_table).- if sections[target].LayoutType == pdf.LayoutTypeTable && sections[target].Text != "" {- sections[target].Text = s.Text + sections[target].Text- } else if sections[target].Text != "" {- sections[target].Text += " " + s.Text- } else {- sections[target].Text = s.Text+ // Emit the caption inside the target table's HTML as a <caption>+ // element (matching Python's __html_table) and drop the standalone+ // caption section. Retaining the caption text closes the previous+ // content-loss go_bug table-html-emission-format; it is NOT a+ // table-assembly change (cell content/structure are untouched).+ top := 1e9+ if len(s.Positions) > 0 {+ top = s.Positions[0].Top}+ byTarget[target] = append(byTarget[target], captionText{top: top, text: s.Text})captions = append(captions, i)continue}@@ -36,6 +57,25 @@ func MergeCaptions(sections []pdf.Section, figures []pdf.Section) []pdf.Sectioncaptions = append(captions, i)}}+ // Inject one combined <caption> per target. Captions of the same table are+ // ordered by top edge (reading order, top→bottom) before concatenation.+ for idx, entries := range byTarget {+ sort.SliceStable(entries, func(i, j int) bool { return entries[i].top < entries[j].top })+ texts := make([]string, len(entries))+ for i, e := range entries {+ texts[i] = e.text+ }+ if sections[idx].LayoutType == pdf.LayoutTypeTable {+ injectCaption(§ions[idx], texts)+ continue+ }+ // Non-table target (figure): keep the historical raw-text+ // concatenation. The <caption> element is table-specific; wrapping a+ // figure section's text in it would emit meaningless HTML in a figure+ // section (which carries an image, not a table). Figure captions are+ // out of this PR's scope, so preserve their pre-existing behavior.+ appendRawCaptions(§ions[idx], texts)+ }// Remove caption sections in reverse order.n := len(sections)out := make([]pdf.Section, 0, n-len(captions))@@ -103,10 +143,153 @@ func findNearestParent(captionIdx int, caption pdf.Section, sections []pdf.Secti}}if captionType == pdf.LayoutTypeTable {- idx, dist := find(sections, captionIdx)- if idx >= 0 && dist < maxCaptionGap && sections[idx].LayoutType == pdf.LayoutTypeTable {+ idx, dist := findTables(sections, caption)+ if idx >= 0 && dist < maxCaptionGap {return idx}}return -1}++// findTables returns the nearest section whose LayoutType is table, by+// distance from the caption to the table's NEAREST EDGE (top/bottom) plus+// horizontal center offset. Restricting to table sections (not all sections)+// prevents a caption sitting in the left margin — far from the table's+// horizontal center — from matching a nearer non-table section (e.g. another+// caption) and being wrongly dropped.+//+// The distance is edge-based, NOT center-based: a cross-page table is one tall+// merged section, so its CENTER is far from a caption sitting just above/below+// it — center-distance exceeded maxCaptionGap and the caption was dropped+// (real content loss, e.g. 13's 'Extended Financial Report' / 14's 'Table 1:+// Revenue'). A caption just above the table (gap to its top) or just below+// (gap to its bottom) has a small edge distance and attaches; one that+// vertically overlaps the table has gap 0.+func findTables(sections []pdf.Section, caption pdf.Section) (int, float64) {+ bestIdx := -1+ bestDist := 1e9+ if len(caption.Positions) == 0 {+ return bestIdx, bestDist+ }+ cp := caption.Positions[0]+ ccx := (cp.Left + cp.Right) / 2+ // Page-scope guard: a caption on a DIFFERENT page may attach only when it+ // VERTICALLY OVERLAPS the table's Y band (gapY==0). That is the cross-page+ // table continuation case — a merged cross-page table's Positions[0] keeps+ // only the FIRST page's geometry (pages=[0]), so a caption on a later page+ // of the same table lands inside that band in page-local coordinates (13's+ // page-2 caption 'Table: Monthly financial summary FY2024' at page-local+ // Y=154 sits inside the page-0 band 98-777). A caption clearly ABOVE or+ // BELOW a table on a different page (gapY>0) is a false attachment:+ // page-local Y ranges repeat on every page, so such a caption only+ // "happens" to line up with an unrelated single-page table.+ capKnown := len(cp.PageNumbers) > 0+ capPage := 0+ if capKnown {+ capPage = cp.PageNumbers[0]+ }+ for i, t := range sections {+ if t.LayoutType != pdf.LayoutTypeTable || len(t.Positions) == 0 {+ continue+ }+ tp := t.Positions[0]+ // Vertical gap to the table's nearest edge. A caption fully above the+ // table measures the gap to the top; fully below, to the bottom;+ // vertically overlapping the table, the gap is 0.+ gapY := 0.0+ if cp.Bottom <= tp.Top {+ gapY = tp.Top - cp.Bottom+ } else if cp.Top >= tp.Bottom {+ gapY = cp.Top - tp.Bottom+ }+ if capKnown {+ onPage := len(tp.PageNumbers) == 0 // unknown table page → don't restrict+ for _, p := range tp.PageNumbers {+ if p == capPage {+ onPage = true+ break+ }+ }+ // Different page: reject unless vertically overlapping (cross-page+ // table continuation). A caption above/below on another page is a+ // false attachment (see function comment).+ if !onPage && gapY > 0 {+ continue+ }+ }+ // Horizontal center offset: a caption far to the side ranks worse.+ dx := (tp.Left+tp.Right)/2 - ccx+ if dx < 0 {+ dx = -dx+ }+ dist := gapY*gapY + dx*dx+ if dist < bestDist {+ bestDist = dist+ bestIdx = i+ }+ }+ return bestIdx, bestDist+}++// appendRawCaptions concatenates caption texts onto a non-table section+// (figure) as raw text, preserving the historical behavior before the+// <caption> injection existed. The <caption> element is table-specific and+// would be meaningless in a figure section's text.+func appendRawCaptions(target *pdf.Section, captions []string) {+ var b strings.Builder+ for _, c := range captions {+ t := strings.TrimSpace(c)+ if t == "" {+ continue+ }+ if b.Len() > 0 {+ b.WriteByte(' ')+ }+ b.WriteString(t)+ }+ if b.Len() == 0 {+ return+ }+ if target.Text != "" {+ target.Text += " " + b.String()+ } else {+ target.Text = b.String()+ }+}++// injectCaption concatenates the given caption texts (already grouped per+// target by MergeCaptions) into a SINGLE <caption> element inserted+// immediately after the table's opening <table> tag (matching Python's+// __html_table). This keeps the caption text inside the table HTML instead of+// losing it as a dropped standalone section, and emits valid HTML (one+// <caption> per table) rather than one <caption> per caption box. If the+// target has no <table> tag the <caption> is prepended so the text is at least+// preserved.+func injectCaption(table *pdf.Section, captions []string) {+ var b strings.Builder+ for _, c := range captions {+ t := strings.TrimSpace(c)+ if t == "" {+ continue+ }+ if b.Len() > 0 {+ b.WriteByte(' ')+ }+ b.WriteString(html.EscapeString(t))+ }+ if b.Len() == 0 {+ return+ }+ escaped := "<caption>" + b.String() + "</caption>"+ if table.Text == "" {+ table.Text = escaped+ return+ }+ const open = "<table>"+ if idx := strings.Index(table.Text, open); idx >= 0 {+ at := idx + len(open)+ table.Text = table.Text[:at] + escaped + table.Text[at:]+ return+ }+ table.Text = escaped + table.Text+}diff --git a/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.go b/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.gonew file mode 100644index 0000000000..072f4f4ef6--- /dev/null+++ b/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.go@@ -0,0 +1,243 @@+package table++import (+ "strings"+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// TestMergeCaptions_EmitsCaptionTag locks the fix for go_bug+// table-html-emission-format: a "table caption" section must be emitted as a+// <caption> element INSIDE the target table's HTML (matching Python's+// __html_table), and the standalone caption section removed — NOT dropped+// (its previous behavior) and NOT duplicated. This retains the caption text+// that Go used to silently lose.+func TestMergeCaptions_EmitsCaptionTag(t *testing.T) {+ sections := []pdf.Section{+ {Text: "<table><tr><td >Category</td></tr></table>", LayoutType: pdf.LayoutTypeTable,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 159, Right: 393, Top: 130, Bottom: 336}}},+ {Text: "Table 1: Revenue", LayoutType: pdf.DLALabelTableCaption,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 61, Right: 144, Top: 219, Bottom: 231}}},+ }+ figures := pdf.CollectFigures(sections)+ result := MergeCaptions(sections, figures)++ if len(result) != 1 {+ t.Fatalf("expected 1 section (table with caption, standalone caption removed), got %d: %v", len(result), textsOf(result))+ }+ got := result[0].Text+ if !strings.Contains(got, "<caption>Table 1: Revenue</caption>") {+ t.Errorf("table HTML missing <caption> element; got %q", got)+ }+ // Caption must sit INSIDE the table, right after <table>.+ if i := strings.Index(got, "<table>"); i < 0 || !strings.Contains(got[:i+len("<table><caption>")], "<caption>") {+ t.Errorf("expected <caption> immediately after <table>, got %q", got)+ }+ // No duplicate: the bare caption text must not also appear as a separate section+ // (it is consumed into the table).+ if strings.Count(got, "Table 1: Revenue") != 1 {+ t.Errorf("caption text should appear exactly once (inside <caption>), got %q", got)+ }+}++// TestMergeCaptions_LeftMarginCaptionAttaches locks that a caption positioned in+// the left margin (far from the table's horizontal center) still attaches to+// its table, not to a nearer non-table section. Before the fix,+// findNearestParent used pure Euclidean distance over ALL sections and returned+// -1 (nearest was another caption / non-table), so the caption was dropped.+func TestMergeCaptions_LeftMarginCaptionAttaches(t *testing.T) {+ sections := []pdf.Section{+ {Text: "Product Analysis Report", LayoutType: pdf.LayoutTypeTitle,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 192, Right: 404, Top: 65, Bottom: 82}}},+ {Text: "<table><tr><td >Category</td></tr></table>", LayoutType: pdf.LayoutTypeTable,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 159, Right: 393, Top: 130, Bottom: 336}}},+ // Caption sits in the left margin, far from the table's horizontal center.+ {Text: "The following table summarizes the quarterly sales performance", LayoutType: pdf.DLALabelTableCaption,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 61, Right: 144, Top: 150, Bottom: 170}}},+ }+ figures := pdf.CollectFigures(sections)+ result := MergeCaptions(sections, figures)+ if len(result) != 2 {+ t.Fatalf("expected 2 sections (title + table-with-caption), got %d: %v", len(result), textsOf(result))+ }+ found := false+ for _, s := range result {+ if strings.Contains(s.Text, "<caption>The following table summarizes the quarterly sales performance</caption>") {+ found = true+ }+ }+ if !found {+ t.Errorf("left-margin caption dropped; sections = %v", textsOf(result))+ }+}++// TestMergeCaptions_SingleCaptionPerTable locks the fix for invalid HTML where+// a table with MORE THAN ONE caption box (e.g. a sentence above AND below the+// table) produced multiple <caption> elements inside one <table>. The HTML+// spec allows only ONE <caption> per table, so browsers/HTML parsers/Markdown+// converters keep only the first and silently drop the rest — a real content+// loss. MergeCaptions must collapse all captions attaching to the SAME table+// into a SINGLE <caption> element (texts concatenated), retaining every+// sentence.+func TestMergeCaptions_SingleCaptionPerTable(t *testing.T) {+ sections := []pdf.Section{+ {Text: "<table><tr><td >Category</td></tr></table>", LayoutType: pdf.LayoutTypeTable,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 159, Right: 393, Top: 130, Bottom: 336}}},+ // Caption above (left margin) AND caption below the table — both attach to the same table.+ {Text: "Table 1: Quarterly sales by product category (in USD)", LayoutType: pdf.DLALabelTableCaption,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 61, Right: 144, Top: 219, Bottom: 231}}},+ {Text: "The following table summarizes the quarterly sales performance", LayoutType: pdf.DLALabelTableCaption,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 200, Right: 380, Top: 350, Bottom: 370}}},+ }+ figures := pdf.CollectFigures(sections)+ result := MergeCaptions(sections, figures)+ if len(result) != 1 {+ t.Fatalf("expected 1 section (table with combined caption), got %d: %v", len(result), textsOf(result))+ }+ got := result[0].Text+ if n := strings.Count(got, "<caption>"); n != 1 {+ t.Errorf("expected exactly ONE <caption> per table (HTML allows only one), got %d: %q", n, got)+ }+ // Both caption sentences must survive inside the single <caption>.+ if !strings.Contains(got, "Table 1: Quarterly sales by product category (in USD)") {+ t.Errorf("first caption sentence lost in combined <caption>: %q", got)+ }+ if !strings.Contains(got, "The following table summarizes the quarterly sales performance") {+ t.Errorf("second caption sentence lost in combined <caption>: %q", got)+ }+}++// TestMergeCaptions_ReadingOrderByTop locks that multiple captions attached to+// the SAME table are concatenated in READING order (top→bottom), matching the+// PDF layout and Python's construct_table. Real case 06_table_content.pdf has+// "The following table summarizes..." ABOVE the table (top=105) and "Table 1:+// Quarterly sales..." BELOW it (top=248), but Go's sections list carries the+// lower one first — so a pure section-order concatenation produces a reversed+// (unnatural) caption. Sorting by the caption box's top edge restores reading+// order.+func TestMergeCaptions_ReadingOrderByTop(t *testing.T) {+ sections := []pdf.Section{+ {Text: "<table><tr><td >Category</td></tr></table>", LayoutType: pdf.LayoutTypeTable,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 159, Right: 393, Top: 130, Bottom: 336}}},+ // Lower caption box FIRST in section order, despite being BELOW the+ // table's caption ("Table 1..." sits inside the table band, top=248).+ {Text: "Table 1: Quarterly sales by product category (in USD)", LayoutType: pdf.DLALabelTableCaption,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 62, Right: 303, Top: 248, Bottom: 261}}},+ // Upper caption box SECOND in section order, though it is ABOVE the+ // table (top=105). Reading order must place it FIRST.+ {Text: "The following table summarizes the quarterly sales performance", LayoutType: pdf.DLALabelTableCaption,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 107, Right: 501, Top: 105, Bottom: 118}}},+ }+ figures := pdf.CollectFigures(sections)+ result := MergeCaptions(sections, figures)+ if len(result) != 1 {+ t.Fatalf("expected 1 section (table with combined caption), got %d: %v", len(result), textsOf(result))+ }+ want := "<caption>The following table summarizes the quarterly sales performance Table 1: Quarterly sales by product category (in USD)</caption>"+ if !strings.Contains(result[0].Text, want) {+ t.Errorf("caption must be concatenated in reading order (top→bottom):\n want %q\n got %q", want, result[0].Text)+ }+}++// TestMergeCaptions_TallTableCaptionNearEdgeAttaches locks the fix for the+// cross-page-table caption drop (13/14): a caption sitting just above or below+// a TALL table (a cross-page table's section is one tall merged region) was+// REJECTED by findNearestParent because the distance was measured to the+// table's CENTER — for a tall table the center is far from the edge, so+// center-distance exceeded maxCaptionGap and the caption was dropped (real+// content loss, e.g. 13's 'Extended Financial Report' and 14's 'Table 1:+// Revenue'). The distance must be measured to the table's NEAREST EDGE.+func TestMergeCaptions_TallTableCaptionNearEdgeAttaches(t *testing.T) {+ table := pdf.Section{+ Text: "<table><tr><th >Month</th><th >Revenue</th></tr></table>", LayoutType: pdf.LayoutTypeTable,+ // Cross-page merged table: Positions[0] keeps ONLY the first page's+ // geometry (pages=[0], the page-0 band), exactly like 13's real merged+ // table. A caption on a LATER page of the same table lands inside this+ // band in page-local coordinates (gapY=0) and must still attach.+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 82, Right: 513, Top: 98, Bottom: 777}},+ }+ for _, c := range []struct {+ name string+ caption pdf.Section+ }{+ {"above", pdf.Section{+ Text: "Extended Financial Report", LayoutType: pdf.DLALabelTableCaption,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 183, Right: 412, Top: 64, Bottom: 82}}}},+ {"later-page-inside-band", pdf.Section{+ Text: "Table: Monthly financial summary FY2024", LayoutType: pdf.DLALabelTableCaption,+ Positions: []pdf.Position{{PageNumbers: []int{2}, Left: 62, Right: 250, Top: 154, Bottom: 167}}}},+ } {+ t.Run(c.name, func(t *testing.T) {+ sections := []pdf.Section{table, c.caption}+ figures := pdf.CollectFigures(sections)+ result := MergeCaptions(sections, figures)+ if len(result) != 1 {+ t.Fatalf("expected 1 section (table with caption), got %d: %v", len(result), textsOf(result))+ }+ want := "<caption>" + c.caption.Text + "</caption>"+ if !strings.Contains(result[0].Text, want) {+ t.Errorf("caption near tall-table edge must attach (was dropped by center-distance): want %q, got %q", want, result[0].Text)+ }+ })+ }+}++// TestMergeCaptions_CaptionOtherPageDoesNotAttach locks the page-scope guard:+// a caption clearly ABOVE or BELOW a table's Y band on a DIFFERENT page must+// NOT attach. The edge-distance alone is page-local-coordinate blind — a+// page-N caption whose page-local Y is above a page-0 table's band gets a+// small edge gap (gapY>0) and would wrongly attach. Only captions on a+// different page that VERTICALLY OVERLAP the band (gapY==0, the cross-page+// table continuation case, see TallTableCaptionNearEdgeAttaches) may attach.+func TestMergeCaptions_CaptionOtherPageDoesNotAttach(t *testing.T) {+ table := pdf.Section{+ Text: "<table><tr><th >Month</th><th >Revenue</th></tr></table>", LayoutType: pdf.LayoutTypeTable,+ // SINGLE-page table on page 0 only.+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 82, Right: 513, Top: 98, Bottom: 777}},+ }+ // Page-1 caption ABOVE the page-0 table's Y band (page-local Y 64-82 is+ // above band top 98) — on a different page, gapY>0 must reject it.+ caption := pdf.Section{+ Text: "Table 1: Revenue", LayoutType: pdf.DLALabelTableCaption,+ Positions: []pdf.Position{{PageNumbers: []int{1}, Left: 183, Right: 412, Top: 64, Bottom: 82}},+ }+ sections := []pdf.Section{table, caption}+ figures := pdf.CollectFigures(sections)+ result := MergeCaptions(sections, figures)+ // No table on the caption's page -> orphaned caption (dropped), table unchanged.+ if len(result) != 1 {+ t.Fatalf("expected 1 section (table unchanged, orphaned caption dropped), got %d: %v", len(result), textsOf(result))+ }+ if strings.Contains(result[0].Text, "<caption>") {+ t.Errorf("caption above a table on a DIFFERENT page must not attach: %q", result[0].Text)+ }+}++// TestMergeCaptions_FigureCaptionRawText locks that a figure caption attaching+// to its figure section is concatenated as RAW text, NOT wrapped in a+// <caption> element. The <caption> element is table-specific (matching+// Python's __html_table); a figure section carries an image, not a table, so+// wrapping its text in <caption> would emit meaningless HTML. Figure caption+// handling is outside this PR's scope, so the pre-existing raw-text behavior+// must be preserved.+func TestMergeCaptions_FigureCaptionRawText(t *testing.T) {+ sections := []pdf.Section{+ {Text: "", LayoutType: pdf.LayoutTypeFigure,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 400, Top: 300, Bottom: 500}}},+ {Text: "Figure 1: Revenue trend by quarter", LayoutType: pdf.DLALabelFigureCaption,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 400, Top: 510, Bottom: 525}}},+ }+ figures := pdf.CollectFigures(sections)+ result := MergeCaptions(sections, figures)+ if len(result) != 1 {+ t.Fatalf("expected 1 section (figure with caption), got %d: %v", len(result), textsOf(result))+ }+ got := result[0]+ if got.Text != "Figure 1: Revenue trend by quarter" {+ t.Errorf("figure caption must be appended as raw text, got %q", got.Text)+ }+ if strings.Contains(got.Text, "<caption>") {+ t.Errorf("figure section must NOT be wrapped in a <caption> element (table-specific): %q", got.Text)+ }+}diff --git a/internal/deepdoc/parser/pdf/table/table_cells.go b/internal/deepdoc/parser/pdf/table/table_cells.goindex add723ef9c..66d99d8481 100644--- a/internal/deepdoc/parser/pdf/table/table_cells.go+++ b/internal/deepdoc/parser/pdf/table/table_cells.go@@ -3,11 +3,12 @@ package tableimport ("log/slog""math"- pdf "ragflow/internal/deepdoc/parser/pdf/type"- "ragflow/internal/deepdoc/parser/pdf/util""regexp""sort""strings"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+ "ragflow/internal/deepdoc/parser/pdf/util")// ── TSR cell grouping ──────────────────────────────────────────────────@@ -210,7 +211,36 @@ func FillCellTextFromBoxesWithRows(cells []pdf.TSRCell, boxes []pdf.TextBox, rowif boxArea <= 0 {continue}- // 1. Best row by vertical-overlap ratio (>= 0.3), tie-broken by _ov.+ // 1. Best row: prefer the row the box BEGINS in.+ //+ // Python's construct_table groups boxes into rows by each box's TSR+ // row label (b["R"]) — the row the box STARTS in — not by spatial+ // overlap (pdf_parser.py construct_table:176-192). When a label or+ // value box spans two adjacent data rows (e.g. a Month cell merged+ // across a pair of rows, or a Margin cell repeated across a seam),+ // Go must place it in the UPPER row to match Python. Without this,+ // FillCellTextFromBoxes picked the MAX-overlap row (usually the+ // lower one) and the doubled text landed one row too low, which is+ // the 13_crosspage_table.pdf divergence (tracked as go_bug+ // table-crosspage-merge-seam-duplication, whose root cause is this+ // row-selection rule, not the cross-page merge itself).+ //+ // We therefore prefer the topmost row band whose Y range CONTAINS the+ // box's TOP edge. Top-containment is a stronger, non-spurious signal+ // than the 0.3 area ratio (a box whose top sits inside a band clearly+ // belongs to that row even when its 2D overlap ratio is below 0.3, as+ // happens for a narrow Month box whose width gives a small area+ // ratio). Only when no band contains the top edge do we fall back to+ // the original max-overlap (ov, ov2) rule, so single-row boxes and+ // genuinely lower-row boxes are unchanged.+ topR := -1+ for ri := range rows {+ rb := &rows[ri]+ if b.Top >= rb.y0 && b.Top <= rb.y1 {+ topR = ri+ break+ }+ }bestR := -1bestOv, bestOv2 := 0.3, 0.0for ri := range rows {@@ -221,16 +251,21 @@ func FillCellTextFromBoxesWithRows(cells []pdf.TSRCell, boxes []pdf.TextBox, rowstrip := pdf.TSRCell{X0: rb.stripX0, Y0: rb.y0, X1: rb.stripX1, Y1: rb.y1}inter := util.OverlapInter(&strip, &b)ov := inter / boxArea+ if ov < 0.3 {+ continue+ }ov2 := 0.0if a := util.Area(&strip); a > 0 {ov2 = inter / a}- // Skip unless strictly better than the current best, mirroring- // Python's (ov, _ov) tuple ordering in find_overlapped_with_threshold.- if !(ov > bestOv || (ov == bestOv && ov2 > bestOv2)) {- continue+ // Fallback: keep the max-overlap (ov, ov2) best, mirroring+ // Python's find_overlapped_with_threshold tuple ordering.+ if ov > bestOv || (ov == bestOv && ov2 > bestOv2) {+ bestR, bestOv, bestOv2 = ri, ov, ov2}- bestR, bestOv, bestOv2 = ri, ov, ov2+ }+ if topR >= 0 {+ bestR = topR}if bestR < 0 {continue@@ -339,7 +374,11 @@ func matchRowStrip(rowStrips []pdf.TSRCell, y0 float64) (float64, float64, bool)// isCaptionBox checks if a text box is a table/figure caption,// matching Python is_caption(). Captions should not enter table cells.-var reCaption = regexp.MustCompile(`^[图表]+[ 0-9::]{2,}|(?i)Fig\.?\s*\d+|(?i)Figure\s+\d+|(?i)Table\s+\d+`)+// reCaption backs IsCaptionBox and must mirror Python's start-anchored+// is_caption: only a line BEGINNING with a caption marker counts. The English+// alternatives are start-anchored with ^ for the same reason as+// reTableCaptionText/reFigureCaptionText.+var reCaption = regexp.MustCompile(`^[图表]+[ 0-9::]{2,}|(?i)^Fig\.?\s*\d+|(?i)^Figure\s+\d+|(?i)^Table\s+\d+`)func IsCaptionBox(text string, layoutType string) bool {if strings.Contains(layoutType, "caption") {@@ -349,11 +388,16 @@ func IsCaptionBox(text string, layoutType string) bool {}// reTableCaptionText matches text patterns that indicate a table caption-// (as opposed to a figure caption). Python is_caption uses the same set.-var reTableCaptionText = regexp.MustCompile(`^表|(?i)Table\s+\d+`)+// (as opposed to a figure caption). Python is_caption uses re.match, which is+// start-anchored, so a body paragraph that merely MENTIONS "Table N"+// mid-sentence is NOT a caption. The English alternatives below are therefore+// start-anchored with ^: an unanchored alternative wrongly classifies such a+// paragraph as a caption and MergeCaptions then drops it (go_bug+// table-text-interleaved-paragraph-dropped).+var reTableCaptionText = regexp.MustCompile(`^表|(?i)^Table\s+\d+`)// reFigureCaptionText matches text patterns that indicate a figure caption.-var reFigureCaptionText = regexp.MustCompile(`^图|(?i)Fig\.?\s*\d+|(?i)Figure\s+\d+`)+var reFigureCaptionText = regexp.MustCompile(`^图|(?i)^Fig\.?\s*\d+|(?i)^Figure\s+\d+`)// captionKind returns "table" if the section is a table caption,// "figure" if a figure caption, or "" if not a caption.diff --git a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.jsonindex b2d2add670..e4f7be679d 100644--- a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json+++ b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json@@ -5,8 +5,12 @@"id": "table-cell-fill-filled-threshold-0.85","tag": "go_intentional","kind": "threshold",- "applies_to": ["*"],- "fields": ["BoxMatchesCell"],+ "applies_to": [+ "*"+ ],+ "fields": [+ "BoxMatchesCell"+ ],"permanent": true,"reason": "BoxMatchesCell (internal/deepdoc/parser/pdf/table/table_cells.go) uses a two-stage overlap threshold: empty cell -> inter/boxArea >= 0.3 (matches Python find_overlapped_with_threshold default thr=0.3, which fills cells from overlapping PDF boxes uniformly); filled cell -> >= 0.85. The 0.85 branch has NO Python equivalent. It is reached only in the rotated-table path: table_extract.go calls ocrTableCells (parser_ocr.go) to pre-fill each cell with per-cell OCR text BEFORE FillCellTextFromBoxes runs (table_extract.go), and the 0.85 bar stops a weakly-overlapping detected text box from corrupting/overriding that OCR result. Python has no per-cell OCR at this stage, so it never raises the threshold and would join a 30-85%-overlapping secondary box (e.g. cell='Total' + a box '元' overlapping 60% -> Python 'Total 元', Go 'Total'). This is a deliberate go_intentional divergence, not a regression target. It is locked by TestFillCellTextFromBoxes_PrefilledCellDropsSecondaryBox and TestBoxMatchesCell_FilledCellWeakOverlapRejected in table_cell_spatial_test.go; the Go-only rationale is documented in the BoxMatchesCell doc comment (table_cells.go). NOTE: if we later unify the pipeline to always run FillCellTextFromBoxes (empty cell, 0.3) before ocrTableCells fills only remaining empties — matching the non-rotated path — this threshold can be removed entirely and full parity with Python restored. Until then it stays permanent."},@@ -14,8 +18,13 @@"id": "table-cell-fill-multi-assignment","tag": "go_bug","kind": "assignment_cardinality",- "applies_to": ["*"],- "fields": ["FillCellTextFromBoxes", "BoxMatchesCell"],+ "applies_to": [+ "*"+ ],+ "fields": [+ "FillCellTextFromBoxes",+ "BoxMatchesCell"+ ],"owner_fix_side": "go","status": "resolved","resolution": "FillCellTextFromBoxes now assigns each box to exactly ONE cell (best row by vertical-overlap ratio >= 0.3, tie-broken by inter/rowArea; then tightest column by horizontal edge/center distance). The many-to-many 2-D cell-overlap filter was removed. Regression: TestFillCellTextFromBoxes_BoxOverlappingTwoCells_SingleAssignment.",@@ -26,8 +35,13 @@"id": "table-cell-fill-column-algorithm","tag": "go_bug","kind": "match_primitive",- "applies_to": ["*"],- "fields": ["FillCellTextFromBoxes", "BoxMatchesCell"],+ "applies_to": [+ "*"+ ],+ "fields": [+ "FillCellTextFromBoxes",+ "BoxMatchesCell"+ ],"owner_fix_side": "go","status": "resolved","resolution": "FillCellTextFromBoxes selects the target cell via best row (vertical-overlap ratio >= 0.3 on the full-width row strip) intersected with tightest column (find_horizontally_tightest_fit, no threshold). The single 2-D cell-intersection 0.3 test is gone. Regression: TestFillCellTextFromBoxes_PythonAssignsGoRejects_2DThreshold_FIXED.",@@ -38,8 +52,13 @@"id": "table-cell-fill-no-best-match-tiebreak","tag": "go_bug","kind": "selection",- "applies_to": ["*"],- "fields": ["FillCellTextFromBoxes", "BoxMatchesCell"],+ "applies_to": [+ "*"+ ],+ "fields": [+ "FillCellTextFromBoxes",+ "BoxMatchesCell"+ ],"owner_fix_side": "go","status": "resolved","resolution": "FillCellTextFromBoxes assigns each box to the single tightest column, so a box overlapping several cells lands in only the tightest one (matching Python, which keeps the single best via its ov/_ov ordering). Regression: TestFillCellTextFromBoxes_EqualBoxRatioSingleAssignment.",@@ -50,17 +69,47 @@"id": "table-html-emission-format","tag": "go_intentional","kind": "serialization_format",- "applies_to": ["06_table_content.pdf", "18_table_caption.pdf"],- "fields": ["RowsToHTML"],+ "applies_to": [+ "06_table_content.pdf",+ "13_crosspage_table.pdf"+ ],+ "fields": [+ "RowsToHTML"+ ],"permanent": true,- "reason": "Pipeline-parity textSim<100% on these two PDFs is entirely the HTML serialization layer; gridSim=100% confirms cell text / row-col structure / span coverage are identical to Python. Go's RowsToHTML (table_html.go) emits compact HTML: no newlines between rows or before </table>, a <th> for every header row, covered cells skipped via the covered map, caption from item.Caption. Python's __html_table (deepdoc/vision/table_structure_recognizer.py:356) emits '\\n' before each row and before </table>, <td></td>/<th></th> for empty non-covered cells (Go emits '<td ></td>'), and DROPS an entire header row when every cell text in it already appeared in a previous header row ('if all([t in hdset for t in txts]): continue' — hdset mixes row indices and cell texts, so a duplicated-text header row silently vanishes in Python; Go keeps it). Covered-cell handling is identical on both sides (Python 'arr is None: continue' == Go covered-skip). Downstream (internal/parser/parser/pdf_postprocess.go:76, chunker) treats the table HTML as opaque section text; the divergence does not alter cell content or structure, and Go's compact HTML is token-friendlier for embedding while preserving all header rows Python would drop. Deliberate divergence, not a regression target."+ "reason": "This rule's caption/content-loss half is FULLY RESOLVED; the residual is the HTML serialization format only. (1) RESOLVED — caption content loss: MergeCaptions used to DROP the standalone 'table caption' section (findNearestParent returned -1 and the no-target branch removed it), then multiple caption boxes for one table emitted MULTIPLE <caption> elements in one <table> (invalid HTML — consumers keep only the first), and captions of cross-page tables were rejected because the distance was measured to the merged table's CENTER (far from a caption near the edge). All three are fixed: caption text is injected as a single <caption> per table (injectCaption); caption sentences are concatenated in READING order (top->bottom, matching Python's construct_table order); and the table match uses edge distance (findTables: vertical gap to the table's nearest top/bottom edge + horizontal offset), so 13/14's captions near a tall cross-page table now attach. Result: 06/13/14/18 all retain every caption sentence and their <caption> text now equals Python's (verified: 06 'The following table summarizes... Table 1: Quarterly sales by product category (in USD)' and 13 'Extended Financial Report Table: Monthly financial summary FY2024' match byte-for-byte; 14's 'Table 1: Revenue' is restored). 14 and 18 are now ALIGNED (textSim=100%). Regression-guarded by TestMergeCaptions_EmitsCaptionTag, TestMergeCaptions_LeftMarginCaptionAttaches, TestMergeCaptions_SingleCaptionPerTable, TestMergeCaptions_ReadingOrderByTop, TestMergeCaptions_TallTableCaptionNearEdgeAttaches, and the end-to-end TestPipelineParityCaptionEmitted (at most one <caption> per <table>, every sentence retained + real). (2) RESIDUAL — serialization format (go_intentional, this rule): the remaining textSim<100% on 06/13 comes ONLY from HTML tag/whitespace formatting: Go emits <th> for detected header rows (Python emits <td> everywhere — Go deliberately keeps the header semantics, see the <th> analysis) and compact single-space tags (<td >) vs Python's double-space <td > + leading/trailing cell spaces. Cell text and row/col structure match exactly (gridSim=100%, structSim=100%); the gap is NON-CELL-TEXT only and deliberate."+ },+ {+ "id": "table-text-interleaved-paragraph-dropped",+ "tag": "go_bug",+ "kind": "content_loss",+ "applies_to": [+ "14_text_table_interleaved.pdf"+ ],+ "fields": [+ "CaptionKind",+ "reTableCaptionText",+ "reCaption",+ "MergeCaptions"+ ],+ "owner_fix_side": "go",+ "status": "resolved",+ "resolution": "reTableCaptionText and reCaption now start-anchor their English 'Table N' / 'Figure N' / 'Fig N' alternatives (^), mirroring Python's is_caption which uses re.match (start-anchored). A body paragraph that only MENTIONS 'Table 1' mid-sentence ('...Table 1 shows revenuebycategory.') is no longer misclassified as a caption, so MergeCaptions no longer drops it; 14 now emits title + paragraph + 2 tables (textSim 81.7% -> 90.0%, boxes ...->4). Regression: TestCaptionKind_MidSentenceTableMentionNotCaption, TestIsCaptionBox_MidSentenceTableMention, TestMergeCaptions_KeepsInterleavedParagraph, plus the end-to-end TestPipelineParity14InterleavedParagraphDropped (now GREEN).",+ "tracking": "Original fields guessed the loss in processPageBoxes/NaiveVerticalMerge/BoxesToSections. Step-through of buildLayout proved the paragraph box survives every stage (Dedup, AssignColumn, TextMerge, FinalReadingOrderMerge, NaiveVerticalMerge, ExtractTableAndReplace, ConsolidateFigures, BoxesToSections) and is only dropped in MergeCaptions: CaptionKind mislabeled it a table caption (unanchored 'Table N' match), findNearestParent returned -1 (nearest section is the title, not a table), so the caption-without-target branch removed it. Root cause is caption DETECTION, not assembly.",+ "reason": "14_text_table_interleaved.pdf has a body paragraph ('The following analysis compares product performance across different market segments. Table 1 shows revenuebycategory.') interleaved BEFORE Table 1. Python extracts it as a separate text section; Go's pipeline dropped it entirely (Go output had only the title + the two tables). This was a real CONTENT loss, not an HTML-emission/format difference: the text was absent from Go's sections, and gridSim=100% still held because the paragraph is outside any table cell. RESOLVED: the drop was caused by unanchored 'Table N' caption detection (reTableCaptionText/reCaption matched 'Table 1' anywhere in the string), mislabeling the paragraph a caption that MergeCaptions then removed; anchoring to the start aligns Go with Python's re.match is_caption and retains the paragraph. 14's RESIDUAL gap is now caption emission only (rule table-html-emission-format). Originally exposed by TestPipelineParity14InterleavedParagraphDropped (was RED, now GREEN)."},{"id": "english-page-ocr-source-divergence","tag": "go_intentional","kind": "input_source_divergence",- "applies_to": ["*"],- "fields": ["processPageBoxes", "DetectEnglish", "hasCleanChars"],+ "applies_to": [+ "*"+ ],+ "fields": [+ "processPageBoxes",+ "DetectEnglish",+ "hasCleanChars"+ ],"permanent": true,"reason": "Pipeline-parity textSim<100% on English-document PDFs is dominated by an input-source divergence, not assembly logic. Python (deepdoc/parser/pdf_parser.py:1687) clears page chars for documents whose majority vote is English (is_english -> chars=[]) and runs pure OCR (__ocr: detect + recognize_batch); Go (internal/deepdoc/parser/pdf/parser.go:312 hasCleanChars) keeps the extracted chars and runs the char-merge path, so the two sides consume different inputs for the same PDF. Go already mirrors the detection (internal/deepdoc/parser/pdf/util/eng_detect.go DetectEnglish, document-level majority vote) but only records it as the IsEnglish metadata flag. Classified go_intentional: for text-based English PDFs Go's embedded-char extraction is more accurate than OCR (OCR introduces recognition/spacing variance), and scanned/noisy English pages are covered by Go's isScanNoise/IsGarbledPage fallback into ocrDetectAndRecognize. A follow-up can evaluate an OCR fallback for English pages where pdfplumber extraction is poor. Note: the parity harness already clears the replayed chars for is_english documents (engine.ClearChars in pipeline_parity_test.go), so both sides consume the same OCR input in replay; that is a measurement alignment, not a production behavior change."},@@ -68,8 +117,14 @@"id": "ocr-replay-full-vs-python-merge-truncation","tag": "go_intentional","kind": "output_completeness",- "applies_to": ["eval_single_wide.pdf", "eval_two_overlap_no_gutter.pdf"],- "fields": ["ocrDetectAndRecognize", "NaiveVerticalMerge"],+ "applies_to": [+ "eval_single_wide.pdf",+ "eval_two_overlap_no_gutter.pdf"+ ],+ "fields": [+ "ocrDetectAndRecognize",+ "NaiveVerticalMerge"+ ],"permanent": true,"reason": "Go's OCR-replay output is the FULL OCR dump (eval_single_wide: 18 boxes / 2936 chars; eval_two_overlap_no_gutter: 17 boxes / 2912 chars), while the Python text golden contains only the FIRST box's content (1519 / 1512 chars, i.e. ~48% of the OCR content is missing; verified py == dump_join prefix). The loss happens in Python's downstream assembly: for English documents Python clears chars (is_english -> chars=[], pdf_parser.py:1687) so mean_height becomes 0, and _naive_vertical_merge (pdf_parser.py:1007) then merges the overlapping OCR boxes into one box whose text retains only the first box — a Python-side truncation. Go keeps the complete OCR content, which is strictly more complete for downstream retrieval, so the divergence is deliberate (output completeness), not a Go regression. Python-side root cause is tracked as an issue (not fixed here per no-Python-change policy)."},@@ -77,23 +132,51 @@"id": "table-cell-fill-pre-vs-post-merge-order","tag": "go_bug","kind": "pipeline_order",- "applies_to": ["14_text_table_interleaved.pdf"],- "fields": ["FillCellTextFromBoxes", "processOneTable", "NaiveVerticalMerge"],+ "applies_to": [+ "14_text_table_interleaved.pdf"+ ],+ "fields": [+ "FillCellTextFromBoxes",+ "processOneTable",+ "NaiveVerticalMerge"+ ],"owner_fix_side": "go",- "status": "open",- "accepted_at_merge": true,- "note": "KNOWN-UNFIXED AND ACCEPTED AT MERGE of PR #18516 (2026-08-19). The pre-vs-post-merge pipeline-order divergence on 14_text_table_interleaved is documented and tracked below but is NOT fixed in this PR. It is a go_bug content gap (gridSim < 100%) and will continue to report as FAIL in the manual (cgo && manual) pipeline-parity harness — which does not run in CI — so it does not block merge. It is the single remaining content-real-divergence and is scoped to this one PDF. Follow-up: instrument Python's self.boxes mid-pipeline (requires DEEPDOC_URL + py dump regeneration) to replicate the collapse-before-cell-fill exactly; see tracking below.",- "tracking": "Align Go's table cell fill with Python's pipeline order: Python runs _naive_vertical_merge (and friends) BEFORE construct_table marks R/C and fills cells, so overlapping OCR boxes are collapsed first; Go's processOneTable -> FillCellTextFromBoxes runs before buildLayout's NaiveVerticalMerge, so overlapping boxes (e.g. OCR boxes 'Software Hardware' y=(270,300) and 'Hardware' y=(287.7,300)) are both matched to the same row and their text concatenated ('Software Hardware Hardware') where Python collapses them to one ('Software Hardware' in row1, row2 empty). Replicating the collapse before cell fill (or matching Python's R/C marking) should close the 14 gridSim gap.",+ "status": "resolved",+ "resolution": "processOneTable now calls dedupNestedBoxes on the table's OCR boxes BEFORE NaiveVerticalMerge (internal/deepdoc/parser/pdf/table_extract.go). dedupNestedBoxes drops a box that is strictly nested inside another box whose trimmed text contains it (e.g. the re-detected 'Hardware' box fully inside 'Software Hardware'), so the subsequent vertical merge no longer concatenates them into 'Software Hardware Hardware'. After the fix Go's grid content matches Python's exactly (gridSim=100%, cell char-multiset parity; the residual textSim gap is HTML-format/whitespace only, so 14 now classifies GRID_OK like 06/18). Regression: TestPipelineParity14TextTableInterleaved asserts no Go cell contains a duplicated token from overlapping-box concatenation.",+ "tracking": "Original hypothesis was a pipeline-order difference (Python merges before fill, Go after). Root cause was narrower: the two OCR boxes are nested (one fully inside the other with a substring text), so the per-column NaiveVerticalMerge concatenated them. Dropping the nested duplicate box before merge matches Python's effective behavior without reordering the whole pipeline.","reason": "Go cell text fill (internal/deepdoc/parser/pdf/table/table_cells.go FillCellTextFromBoxes) runs on the raw OCR boxes inside processOneTable, BEFORE buildLayout's NaiveVerticalMerge. Python fills cells inside construct_table (deepdoc/vision/table_structure_recognizer.py:156) AFTER _naive_vertical_merge, so boxes overlapping in Y are merged first. For 14_text_table_interleaved table 2, OCR boxes 'Software Hardware' (y 270-300) and 'Hardware' (y 287.7-300) overlap; both go to Go's row band 2 (vertical-overlap ratio 0.53 vs 0.44) and concatenate to 'Software Hardware Hardware', while Python collapses them and the final HTML keeps 'Software Hardware' in row 1. Replicated Python find_overlapped_with_threshold + gather/layouts_cleanup (rows unchanged, R=row2) but the final HTML differs — the divergence stems from the box set that reach construct_table after Python's downstream merge, which dump data cannot fully reproduce; instrumenting Python's self.boxes mid-pipeline is required to close it exactly."},+ {+ "id": "table-crosspage-merge-seam-duplication",+ "tag": "go_bug",+ "kind": "cross_page_merge",+ "applies_to": [+ "13_crosspage_table.pdf"+ ],+ "fields": [+ "MergeTablesAcrossPages",+ "processOneTable",+ "FillCellTextFromBoxes"+ ],+ "owner_fix_side": "go",+ "status": "resolved",+ "tracking": "Originally hypothesized as a MergeTablesAcrossPages page-seam bug, but dumping the per-page grids and comparing them against Python's construct_table proved the per-page grids were ALREADY correct — MergeTablesAcrossPages stacks them correctly and is NOT the root cause. The real cause is FillCellTextFromBoxesWithRows (table/table_cells.go) row-selection: it picked the row with the MAX 2D-overlap (inter/boxArea >= 0.3), which placed a box spanning two adjacent data rows in the LOWER row. Python's construct_table groups boxes into rows by each box's TSR row label b['R'] — the row the box STARTS in — not by spatial overlap (pdf_parser.py construct_table:176-192), so it places the same box in the UPPER row. Fix: prefer the topmost row band whose Y range CONTAINS the box's TOP edge (top-containment), falling back to max-overlap only when no band contains the top. This matches Python's R-grouping and resolves both the seam-duplication and the other row-shift diffs: TestPipelineParity13CrosspageSeam is now green (seam-dup=0, gridSim=100.0%) and 13 moved from harness failed to html-divergent. The ~10 residual other-diff cells are leading/trailing-whitespace only, absorbed by the harness gridSim guard.",+ "reason": "13_crosspage_table.pdf is a multi-page table (Python golden = single 81x5 grid). The divergence did NOT come from the cross-page merge: each per-page grid is correct and MergeTablesAcrossPages stacks them correctly. It came from FillCellTextFromBoxesWithRows assigning OCR/value boxes to the wrong grid ROW. Go used MAX-2D-overlap (inter/boxArea >= 0.3) to pick the row, so a box spanning two adjacent data rows (e.g. a Month cell merged across two rows, or a value box repeated across a seam) landed in the LOWER row; Python's construct_table groups by each box's TSR row label b['R'] (the row the box STARTS in), so it lands in the UPPER row. That one-row-low placement is what produced the duplicated/shifted seam cells. After the fix (top-containment: place the box in the topmost band containing its top edge), Go matches Python and the cell-level diff drops to 0 seam-duplication cells with gridSim=100.0% (TestPipelineParity13CrosspageSeam green). The residual ~10 cells differ only by leading/trailing whitespace and are absorbed by the harness gridSim guard."+ },{"id": "table-rotation-split-vs-merged-grid","tag": "go_intentional","kind": "table_segmentation",- "applies_to": ["table_rotation_test.pdf"],- "fields": ["processOneTable", "MergeTablesAcrossPages", "GroupCells"],+ "applies_to": [+ "table_rotation_test.pdf"+ ],+ "fields": [+ "processOneTable",+ "MergeTablesAcrossPages",+ "GroupCells"+ ],"permanent": true,- "reason": "table_rotation_test.pdf contains three physical tables: page 1 has two 5x3 tables (grey header 'Header A/B/C', blue header), page 2 has the blue-header table rotated 90 degrees clockwise. Go emits three TableItems (5x3 + 5x3 + 3x6); the rotated table is content-correct (each row = Cell 4A, Cell 3A, Cell 2A, Cell 1A, Header A, plus one padding column). Python's golden emits TWO tables: the first 5x3 is identical, but the second is an 8x7 grid that MERGES the page-1 blue-header table with the page-2 rotated table — first 5 rows padded to 7 columns (4 empty columns + 3 data columns), last 3 rows the rotated content, with garbled cell text on both sides ('Header B Cell 1B', 'Cell 2A Cell 3A') inherited from the shared TSR dump (OCR box vs TSR cell boundary offset). gridSim 97.9% is therefore table segmentation + column-count divergence, NOT Go cell-content error: Go's split matches the physical PDF structure, and the residual text misplacement is identical on both sides (same TSR input), not a Go assembly bug. Python's merged 8x7 grid is a segmentation defect; Go's split is deliberately retained. Locked by TestPipelineParity which now exempts go_intentional rules via known_diffs.json."+ "reason": "table_rotation_test.pdf contains three physical tables: page 1 has two tables (grey header 'Header A/B/C', blue header), page 2 has the blue-header table rotated 90 degrees clockwise. Go emits three TableItems (6x3 + 6x3 + 3x6, 15 grid rows total); the rotated table is content-correct (each row = Cell 4A, Cell 3A, Cell 2A, Cell 1A, Header A, plus one padding column). Python's golden emits TWO tables: the first 5x3 is close, but the second is an 8x7 grid that MERGES the page-1 blue-header table with the page-2 rotated table — first 5 rows padded to 7 columns (4 empty columns + 3 data columns), last 3 rows the rotated content, with garbled cell text on both sides ('Header B Cell 1B', 'Cell 2A Cell 3A') inherited from the shared TSR dump (OCR box vs TSR cell boundary offset). The divergence is therefore TABLE SEGMENTATION + column-count, NOT Go cell-content error: Go's split matches the physical PDF structure, and the residual text misplacement is identical on both sides (same TSR input), not a Go assembly bug. Because the cell TEXT is identical on both sides, the old shape-blind gridSim reads 100% — it could NOT see the 15x[...3x12,6x3] vs 13x[3x5,7x8] structural gap. The harness now adds a SHAPE-AWARE structure metric (gridStructureSimilarity): gridSim=100% AND structSim=100% is required for a PDF to be 'content matched' (html-divergent); a go_intentional PDF whose structSim<100 (like this one, structSim=35.7%) is classified INTENTIONAL instead of being hidden under gridSim=100%. Python's merged 8x7 grid is a segmentation defect; Go's split is deliberately retained."}]}diff --git a/internal/deepdoc/parser/pdf/table_extract.go b/internal/deepdoc/parser/pdf/table_extract.goindex 413662ecce..77f0f5bd2b 100644--- a/internal/deepdoc/parser/pdf/table_extract.go+++ b/internal/deepdoc/parser/pdf/table_extract.go@@ -170,6 +170,13 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxes}tableBoxes = append(tableBoxes, b)}+ // Drop OCR boxes that are strictly nested inside another box whose+ // text contains theirs (e.g. a re-detected "Hardware" box fully+ // inside "Software Hardware"). Without this, NaiveVerticalMerge+ // concatenates the two into "Software Hardware Hardware" while+ // Python's construct_table keeps the longer box only. Mirrors+ // Python's effective behavior: the nested duplicate is not merged.+ tableBoxes = dedupNestedBoxes(tableBoxes)tableBoxes = lyt.NaiveVerticalMerge(tableBoxes, nil, nil, nil)boxInCrop = make([]pdf.TextBox, 0, len(tableBoxes))for _, b := range tableBoxes {@@ -224,3 +231,49 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxestbl.WriteTableAnnotations(boxes, tm.BoxIdx, cells, scale, cropOffX, cropOffY, tb)return item}++// dedupNestedBoxes drops OCR boxes that are strictly nested inside another+// box whose trimmed text contains the nested box's trimmed text. This removes+// re-detected duplicate boxes (e.g. a "Hardware" detection fully inside a+// "Software Hardware" box) so the subsequent NaiveVerticalMerge does not+// concatenate them into "Software Hardware Hardware". Python's+// construct_table keeps the longer box only, so this matches Python's output.+//+// Containment requires the nested box's bbox to lie entirely within the other+// box's bbox (same or narrower X, same or shorter Y). Side-by-side or+// half-overlapping boxes are never nested, so they are preserved and merged+// normally.+func dedupNestedBoxes(boxes []pdf.TextBox) []pdf.TextBox {+ keep := make([]bool, len(boxes))+ for i := range boxes {+ keep[i] = strings.TrimSpace(boxes[i].Text) != ""+ }+ for i := 0; i < len(boxes); i++ {+ if !keep[i] {+ continue+ }+ at := strings.TrimSpace(boxes[i].Text)+ for j := 0; j < len(boxes); j++ {+ if i == j || !keep[j] {+ continue+ }+ bt := strings.TrimSpace(boxes[j].Text)+ if bt == "" || !strings.Contains(at, bt) {+ continue+ }+ // Drop the nested (shorter) box when it sits fully inside the+ // outer box's bbox.+ if boxes[j].X0 >= boxes[i].X0 && boxes[j].X1 <= boxes[i].X1 &&+ boxes[j].Top >= boxes[i].Top && boxes[j].Bottom <= boxes[i].Bottom {+ keep[j] = false+ }+ }+ }+ out := make([]pdf.TextBox, 0, len(boxes))+ for i := range boxes {+ if keep[i] {+ out = append(out, boxes[i])+ }+ }+ return out+}diff --git a/internal/deepdoc/parser/pdf/table_extract_test.go b/internal/deepdoc/parser/pdf/table_extract_test.goindex ab0587e4c3..8e43cf2022 100644--- a/internal/deepdoc/parser/pdf/table_extract_test.go+++ b/internal/deepdoc/parser/pdf/table_extract_test.go@@ -5,6 +5,7 @@ import ("fmt""image""math"+ "strings""testing"tbl "ragflow/internal/deepdoc/parser/pdf/table"@@ -296,3 +297,26 @@ func TestProcessOneTable_CollapsesOverlappingBoxesBeforeCellFill(t *testing.T) {t.Fatalf("nonEmpty cells = %d (row0=%q row1=%q), want exactly 1: overlapping boxes must merge into a single cell assignment instead of spreading duplicated text across rows", nonEmpty, row0, row1)}}++// TestDedupNestedBoxes_IdenticalBBoxKeepsContent locks that two boxes with+// IDENTICAL bbox AND identical text are not BOTH dropped. The containment+// check drops the inner box when its bbox lies fully inside the outer's and+// the outer text contains the inner's — but for identical bboxes each box is+// "inside" the other, so both get marked for removal and the cell content+// vanishes. Only STRICT containment may drop; identical-bbox boxes must keep+// at least one copy.+func TestDedupNestedBoxes_IdenticalBBoxKeepsContent(t *testing.T) {+ boxes := []pdf.TextBox{+ {X0: 100, X1: 300, Top: 100, Bottom: 120, Text: "Revenue", IsOCR: true},+ {X0: 100, X1: 300, Top: 100, Bottom: 120, Text: "Revenue", IsOCR: true},+ }+ got := dedupNestedBoxes(boxes)+ if len(got) == 0 {+ t.Fatal("identical bbox+text boxes were BOTH dropped — content loss; at least one copy must survive")+ }+ for _, b := range got {+ if strings.TrimSpace(b.Text) != "Revenue" {+ t.Errorf("surviving box text = %q, want %q", b.Text, "Revenue")+ }+ }+}