RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(deepdoc): preserve covered cells through span injection so span tables keep all TSR rows (#18574)
Jack295dc12cc5ee
diff --git a/internal/deepdoc/parser/pdf/parity_lock.go b/internal/deepdoc/parser/pdf/parity_lock.goindex 80cf79dad6..410efc8f88 100644--- a/internal/deepdoc/parser/pdf/parity_lock.go+++ b/internal/deepdoc/parser/pdf/parity_lock.go@@ -19,5 +19,6 @@ func parityLockedGridPDFs() map[string]bool {"13_crosspage_table.pdf": true,"14_text_table_interleaved.pdf": true,"18_table_caption.pdf": true,+ "1.pdf": true,}}diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_test.goindex 3acc659845..a5fb2250ff 100644--- a/internal/deepdoc/parser/pdf/pipeline_parity_test.go+++ b/internal/deepdoc/parser/pdf/pipeline_parity_test.go@@ -368,13 +368,20 @@ func loadPythonTables(t *testing.T, jsonPath string) ([][]string, bool) {}// goTableRows flattens every Go result table's grid (cell text) into rows.+// Cells covered by a spanning cell (marked "table covered" by ConstructTable)+// are dropped, matching Python's golden rows, which come from the rendered+// HTML and omit covered <td> entirely — so per-row column counts match+// Python's variable-width grids (e.g. 11x[3,2,3,...]).func goTableRows(result *pdf.ParseResult) [][]string {var rows [][]stringfor _, t := range result.Tables {for _, r := range t.Grid {- row := make([]string, len(r))- for j, c := range r {- row[j] = strings.TrimSpace(c.Text)+ var row []string+ for _, c := range r {+ if strings.Contains(c.Label, "covered") {+ continue+ }+ row = append(row, strings.TrimSpace(c.Text))}rows = append(rows, row)}diff --git a/internal/deepdoc/parser/pdf/table/deepdoc_table_builder.go b/internal/deepdoc/parser/pdf/table/deepdoc_table_builder.goindex 854288a27a..da072e5d88 100644--- a/internal/deepdoc/parser/pdf/table/deepdoc_table_builder.go+++ b/internal/deepdoc/parser/pdf/table/deepdoc_table_builder.go@@ -132,9 +132,14 @@ func (b *DeepDocTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {grid[first.r][first.c].X1 = sp.X1grid[first.r][first.c].Y1 = sp.Y1grid[first.r][first.c].Label = sp.Label- for _, idx := range covered[1:] {- grid[idx.r][idx.c] = pdf.TSRCell{}- }+ // The remaining covered cells KEEP their row×column bbox. Python's+ // construct_table assigns every box to a cell by its TSR row/column+ // label (R/C) regardless of spanning cells, then __cal_spans folds the+ // covered cells' text into the span origin cell at render time. If Go+ // zeroed these bboxes here, FillCellTextFromBoxesWithRows would skip+ // them, their text would never reach the span cell, and orphan-row/+ // column cleanup would drop the covered rows/columns — the+ // dell/prodeploy/screenshot real-PDF parity gap (11→6 rows).}return griddiff --git a/internal/deepdoc/parser/pdf/table/deepdoc_table_builder_test.go b/internal/deepdoc/parser/pdf/table/deepdoc_table_builder_test.goindex dd57675166..29261a54e1 100644--- a/internal/deepdoc/parser/pdf/table/deepdoc_table_builder_test.go+++ b/internal/deepdoc/parser/pdf/table/deepdoc_table_builder_test.go@@ -108,9 +108,17 @@ func TestDeepDocTableBuildService_GroupCells_SpanInjection(t *testing.T) {t.Errorf("grid[0][0] X range = (%.0f,%.0f), want (0,200)", spanCell.X0, spanCell.X1)}- // grid[0][1] should be covered (bbox zeroed).- if !isZeroCell(grid[0][1]) {- t.Errorf("grid[0][1] should be covered (zero bbox), got (%.0f,%.0f,%.0f,%.0f)",+ // grid[0][1] is covered by the span but keeps its own row×col bbox.+ // Python keeps every covered cell's geometry — construct_table folds+ // covered text into the span cell only at render time (__cal_spans), so+ // zeroing the bbox here would starve the covered cell of box text and+ // make downstream orphan-row/column cleanup drop the row/column+ // (dell/prodeploy real-PDF parity gap).+ if isZeroCell(grid[0][1]) {+ t.Errorf("grid[0][1] must keep its row×col bbox (covered by span), got zero bbox")+ }+ if grid[0][1].X0 != 100 || grid[0][1].Y0 != 0 || grid[0][1].X1 != 200 || grid[0][1].Y1 != 50 {+ t.Errorf("grid[0][1] bbox = (%.0f,%.0f,%.0f,%.0f), want (100,0,200,50)",grid[0][1].X0, grid[0][1].Y0, grid[0][1].X1, grid[0][1].Y1)}diff --git a/internal/deepdoc/parser/pdf/table/span_grid_fill_test.go b/internal/deepdoc/parser/pdf/table/span_grid_fill_test.gonew file mode 100644index 0000000000..4cddf07520--- /dev/null+++ b/internal/deepdoc/parser/pdf/table/span_grid_fill_test.go@@ -0,0 +1,205 @@+package table++import (+ "strings"+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// spanGridFixture builds a 3×3 TSR grid with a spanning cell whose bbox+// covers rows 0-1 × cols 0-1 and whose top edge (0.3) sits a fraction of a+// pixel above the other cells of its own grid row (row 0 starts at 0).+//+// This mirrors the real-PDF failure (dell/prodeploy/screenshot parity):+// GroupCells extends the span cell's bbox across the covered region, so its+// Y0 differs from the sibling cells' Y0. The Y0-based row-banding in+// FillCellTextFromBoxesWithRows then splits the span cell into its own row+// band; because that band spans several TSR rows, the top-containment rule+// swallows every box in the covered region into the span cell, emptying the+// real rows (which are then dropped by orphan-row cleanup: 11→6 rows).+func spanGridFixture() (grid [][]pdf.TSRCell, boxes []pdf.TextBox, rowStrips []pdf.TSRCell) {+ cells := []pdf.TSRCell{+ {X0: 0, Y0: 0, X1: 150, Y1: 120, Label: "table"},+ // rows+ {X0: 0, Y0: 0, X1: 150, Y1: 40, Label: "table row"}, // r0+ {X0: 0, Y0: 40, X1: 150, Y1: 80, Label: "table row"}, // r1+ {X0: 0, Y0: 80, X1: 150, Y1: 120, Label: "table row"}, // r2+ // cols+ {X0: 0, Y0: 0, X1: 50, Y1: 120, Label: "table column"}, // c0+ {X0: 50, Y0: 0, X1: 100, Y1: 120, Label: "table column"}, // c1+ {X0: 100, Y0: 0, X1: 150, Y1: 120, Label: "table column"}, // c2+ // spanning cell covering r0+r1 in c0+c1; top edge 0.3px above r0+ {X0: 0, Y0: 0.3, X1: 100, Y1: 80, Label: "table spanning cell"},+ }+ grid = (&DeepDocTableBuilder{}).GroupCells(cells)+ boxes = []pdf.TextBox{+ {X0: 5, X1: 30, Top: 5, Bottom: 35, Text: "A"}, // r0 c0 area+ {X0: 55, X1: 95, Top: 5, Bottom: 35, Text: "B"}, // r0 c1 area+ {X0: 5, X1: 30, Top: 45, Bottom: 75, Text: "C"}, // r1 c0 area+ {X0: 105, X1: 145, Top: 45, Bottom: 75, Text: "D"}, // r1 c2 area+ {X0: 5, X1: 30, Top: 85, Bottom: 115, Text: "E"}, // r2 c0 area+ }+ for _, c := range cells {+ if strings.HasSuffix(c.Label, "table row") {+ rowStrips = append(rowStrips, c)+ }+ }+ SortYFirstly(rowStrips, 10)+ return grid, boxes, rowStrips+}++// fillGrid copies flat cell text back onto the 2-D grid (same copy-back as+// table_extract.go processOneTable).+func fillGrid(grid [][]pdf.TSRCell, flat []pdf.TSRCell) {+ idx := 0+ for ri := range grid {+ for ci := range grid[ri] {+ grid[ri][ci].Text = flat[idx].Text+ idx+++ }+ }+}++// TestFillCellTextFromBoxes_SpanCellDoesNotSwallowOtherRows guards the+// primary span-grid failure: a spanning cell must NOT form its own row band+// (its bbox spans several TSR rows), so boxes in the covered rows stay in+// their own rows instead of being swallowed into the span cell.+func TestFillCellTextFromBoxes_SpanCellDoesNotSwallowOtherRows(t *testing.T) {+ grid, boxes, rowStrips := spanGridFixture()+ flat := FlattenGrid(grid)+ FillCellTextFromBoxesWithRows(flat, boxes, rowStrips)+ fillGrid(grid, flat)++ // r1 (covered by the span) keeps its own boxes.+ if got := grid[1][0].Text; got != "C" {+ t.Errorf("r1 c0 text = %q, want %q (box swallowed into span band)", got, "C")+ }+ if got := grid[1][2].Text; got != "D" {+ t.Errorf("r1 c2 text = %q, want %q (box swallowed into span band)", got, "D")+ }+ // r2 keeps its box.+ if got := grid[2][0].Text; got != "E" {+ t.Errorf("r2 c0 text = %q, want %q", got, "E")+ }+ // r0 c1 (covered by the span) keeps box B.+ if got := grid[0][1].Text; got != "B" {+ t.Errorf("r0 c1 text = %q, want %q", got, "B")+ }+}++// TestCalSpans_NoSpanningLabelNoCovered guards the regression where+// CalSpans computed cs/rs for EVERY grid cell: a normal (unlabeled) cell+// whose bbox happens to straddle the next column's center was misclassified+// as a span, and MarkCoveredCells then dropped its covered neighbours from+// the rendered rows — turning 4-column rows into 3 (公司差旅费管理办法+// regression). Only "table spanning cell" / "table header" labels may+// trigger coverage; Python's __cal_spans iterates SP/H-annotated boxes only.+func TestCalSpans_NoSpanningLabelNoCovered(t *testing.T) {+ rows := [][]pdf.TSRCell{+ {+ {X0: 0, Y0: 0, X1: 200, Y1: 30, Text: "a", Label: "table row"}, // wide bbox: X covers col0 + col1 center+ {X0: 100, Y0: 0, X1: 200, Y1: 30, Text: "b", Label: "table row"},+ {X0: 200, Y0: 0, X1: 300, Y1: 30, Text: "c", Label: "table row"},+ },+ {+ {X0: 0, Y0: 35, X1: 100, Y1: 65, Text: "1", Label: "table row"},+ {X0: 100, Y0: 35, X1: 200, Y1: 65, Text: "2", Label: "table row"},+ {X0: 200, Y0: 35, X1: 300, Y1: 65, Text: "3", Label: "table row"},+ },+ }+ _, covered := CalSpans(rows)+ if len(covered) != 0 {+ t.Errorf("no spanning-labeled cell, expected 0 covered, got %v", covered)+ }+}++// TestConstructTable_SpanTableKeepsAllRows guards that the full assembly+// (GroupCells + fill + orphan cleanup) retains every TSR row of a+// span-bearing table. The old span handling emptied covered rows and dropped+// them (e.g. dell-configuration 11→6, screenshot 11→6 rows).+func TestConstructTable_SpanTableKeepsAllRows(t *testing.T) {+ grid, boxes, rowStrips := spanGridFixture()+ flat := FlattenGrid(grid)+ FillCellTextFromBoxesWithRows(flat, boxes, rowStrips)+ fillGrid(grid, flat)++ item := &pdf.TableItem{Grid: grid}+ ConstructTable(flat, boxes, "", item)+ if item.Grid == nil {+ t.Fatal("item.Grid is nil")+ }+ if len(item.Grid) != 3 {+ t.Errorf("expected 3 rows retained (every TSR row), got %d", len(item.Grid))+ }+ // Every row must contribute at least one non-empty cell.+ for ri, row := range item.Grid {+ has := false+ for _, c := range row {+ if strings.TrimSpace(c.Text) != "" {+ has = true+ break+ }+ }+ if !has {+ t.Errorf("row %d is empty after ConstructTable", ri)+ }+ }+}++// TestConstructTable_SpanCellFoldsCoveredText checks that the full assembly+// folds the covered cells' text into the span origin cell, matching Python's+// __cal_spans (the covered row boxes "C"/"D" end up inside the span cell's+// text rather than vanishing because their bboxes are no longer zeroed).+func TestConstructTable_SpanCellFoldsCoveredText(t *testing.T) {+ grid, boxes, rowStrips := spanGridFixture()+ flat := FlattenGrid(grid)+ FillCellTextFromBoxesWithRows(flat, boxes, rowStrips)+ fillGrid(grid, flat)++ item := &pdf.TableItem{Grid: grid}+ ConstructTable(flat, boxes, "", item)++ // span cell is r0 c0; its text must contain the covered region's boxes+ // (A=r0c0 self, B=r0c1 covered, C=r1c0 covered) but NOT boxes outside the+ // span (D=r1c2, E=r2c0).+ spanText := ""+ if len(item.Grid) > 0 && len(item.Grid[0]) > 0 {+ spanText = item.Grid[0][0].Text+ }+ for _, want := range []string{"A", "B", "C"} {+ if !strings.Contains(spanText, want) {+ t.Errorf("span cell text = %q, want it to contain %q (covered text folded)", spanText, want)+ }+ }+ for _, notWant := range []string{"D", "E"} {+ if strings.Contains(spanText, notWant) {+ t.Errorf("span cell text = %q, must NOT contain %q (outside covered region)", spanText, notWant)+ }+ }+}++// TestRowsToStrings_SkipsCovered guards that RowsToStrings drops cells marked+// "table covered", so item.Rows / anchor.Rows match Python's variable-width+// rendered rows instead of carrying phantom columns.+func TestRowsToStrings_SkipsCovered(t *testing.T) {+ rows := [][]pdf.TSRCell{+ {+ {X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "a"},+ {X0: 100, Y0: 0, X1: 200, Y1: 30, Text: "b", Label: "table covered"},+ {X0: 200, Y0: 0, X1: 300, Y1: 30, Text: "c"},+ },+ {+ {X0: 0, Y0: 35, X1: 100, Y1: 65, Text: "1"},+ {X0: 100, Y0: 35, X1: 200, Y1: 65, Text: "2"},+ {X0: 200, Y0: 35, X1: 300, Y1: 65, Text: "3"},+ },+ }+ got := RowsToStrings(rows)+ if len(got[0]) != 2 || got[0][0] != "a" || got[0][1] != "c" {+ t.Errorf("row 0 = %v, want [a c] (covered cell dropped)", got[0])+ }+ if len(got[1]) != 3 || got[1][2] != "3" {+ t.Errorf("row 1 = %v, want [1 2 3] (no covered cell)", got[1])+ }+}diff --git a/internal/deepdoc/parser/pdf/table/table_cells.go b/internal/deepdoc/parser/pdf/table/table_cells.goindex 66d99d8481..3057a680ef 100644--- a/internal/deepdoc/parser/pdf/table/table_cells.go+++ b/internal/deepdoc/parser/pdf/table/table_cells.go@@ -130,6 +130,14 @@ func FillCellTextFromBoxesWithRows(cells []pdf.TSRCell, boxes []pdf.TextBox, row// row×column cross-product, so every cell in a row shares the same Y band.// A row band spans the full table width (union of its cells), matching// Python's full-width "table row" components.+ //+ // Spanning cells are folded into the nearest band by Y0 instead of opening+ // their own band. GroupCells extends a span cell's bbox across the covered+ // region, so its Y0 differs from the sibling cells' Y0; a separate band+ // would span several TSR rows and the top-containment rule would swallow+ // every covered box into it, emptying the real rows (11→6 in dell/+ // screenshot real-PDF parity). Python assigns boxes to rows by TSR row+ // component overlap only — spanning cells never define a row.type rowBand struct {y0, y1 float64stripX0 float64@@ -142,11 +150,15 @@ func FillCellTextFromBoxesWithRows(cells []pdf.TSRCell, boxes []pdf.TextBox, row// Y0 value, and distinct rows differ by at least a row height, so a tiny// epsilon is enough and never merges two real rows.const yTol = 1e-6+ // Pass 1: build bands from non-spanning cells only.for i := range cells {c := &cells[i]if c.X1 <= c.X0 || c.Y1 <= c.Y0 {continue // degenerate / span-covered cell: not a fill target}+ if strings.Contains(c.Label, "spanning") {+ continue+ }rb := &rowBand{}found := falsefor ri := range rows {@@ -173,6 +185,34 @@ func FillCellTextFromBoxesWithRows(cells []pdf.TSRCell, boxes []pdf.TextBox, row}rb.cells = append(rb.cells, i)}+ // Pass 2: fold spanning cells into the nearest band by Y0. GroupCells+ // extends a span cell's bbox across the covered region, so its Y0 differs+ // from its own row's other cells; a separate band would span several TSR+ // rows and the top-containment rule would swallow every covered box into+ // it, emptying the real rows (dell/screenshot 11→6). Python assigns boxes+ // to rows by TSR row component overlap only — spanning cells never define+ // a row. The span cell stays a fill target in that band (its covered+ // boxes' R/C labels still point at the span cell's column), but it must+ // NOT extend the band's Y/X geometry — its bbox already covers the span.+ for i := range cells {+ c := &cells[i]+ if c.X1 <= c.X0 || c.Y1 <= c.Y0 {+ continue+ }+ if !strings.Contains(c.Label, "spanning") {+ continue+ }+ best, bestD := -1, math.Inf(1)+ for ri := range rows {+ if d := math.Abs(rows[ri].y0 - c.Y0); d < bestD {+ best, bestD = ri, d+ }+ }+ if best < 0 {+ continue+ }+ rows[best].cells = append(rows[best].cells, i)+ }// Stable ordering: rows top-to-bottom, cells left-to-right (matches// Python's first-wins tie-breaking in find_overlapped_with_threshold /// find_horizontally_tightest_fit).diff --git a/internal/deepdoc/parser/pdf/table/table_construct.go b/internal/deepdoc/parser/pdf/table/table_construct.goindex 35b0d14ac4..861e59a4f5 100644--- a/internal/deepdoc/parser/pdf/table/table_construct.go+++ b/internal/deepdoc/parser/pdf/table/table_construct.go@@ -79,6 +79,9 @@ func ConstructTable(cells []pdf.TSRCell, boxes []pdf.TextBox, caption string, ititem.Rows = RowsToStrings(rows)}spanInfo, covered := CalSpans(rows)+ if item != nil {+ MarkCoveredCells(item.Grid, covered)+ }return RowsToHTML(rows, caption, hdrs, spanInfo, covered)}// Fallback: boxes with R/C annotations.diff --git a/internal/deepdoc/parser/pdf/table/table_html.go b/internal/deepdoc/parser/pdf/table/table_html.goindex cc7db28a47..62b735d81c 100644--- a/internal/deepdoc/parser/pdf/table/table_html.go+++ b/internal/deepdoc/parser/pdf/table/table_html.go@@ -103,13 +103,22 @@ func SimpleRowsToHTML(rows [][]string) string {return b.String()}+// RowsToStrings converts a grid to a string matrix. Cells covered by a span+// (Label contains "covered") are omitted, matching Python's rendered output,+// which drops covered cells entirely (HTML <td> skipped; desc_table sees+// tbl[r][c] = None) — so a covered cell never appears in the row. Span-free+// grids are unaffected.func RowsToStrings(rows [][]pdf.TSRCell) [][]string {out := make([][]string, len(rows))for ri, row := range rows {- out[ri] = make([]string, len(row))- for ci, c := range row {- out[ri][ci] = c.Text+ var cells []string+ for _, c := range row {+ if strings.Contains(c.Label, "covered") {+ continue+ }+ cells = append(cells, c.Text)}+ out[ri] = cells}return out}diff --git a/internal/deepdoc/parser/pdf/table/table_spans.go b/internal/deepdoc/parser/pdf/table/table_spans.goindex bf145ac310..3b7b492824 100644--- a/internal/deepdoc/parser/pdf/table/table_spans.go+++ b/internal/deepdoc/parser/pdf/table/table_spans.go@@ -65,11 +65,24 @@ func CalSpans(rows [][]pdf.TSRCell) (map[[2]int][2]int, map[[2]int]bool) {}// For each spanning cell, compute how many cols/rows it covers.+ // Only cells that actually represent a span participate: a "table+ // spanning cell" (GroupCells injects this label for TSR span components)+ // or a "table header" cell whose HLeft/HRight bbox straddles neighbour+ // columns (GroupBoxesByRC path, e.g. TestRowsToHTML_Colspan). Python's+ // __cal_spans iterates boxes carrying an SP/H annotation, never every+ // grid cell. Computing cs/rs for arbitrary cells is unsafe: a normal cell+ // whose bbox happens to straddle the next column's center would be+ // misclassified as a span, and its covered neighbours would then be+ // dropped from the rendered rows (a real regression for span-free tables+ // like 公司差旅费管理办法).for i, row := range rows {for j, cell := range row {if j >= nCols || covered[[2]int{i, j}] {continue}+ if !strings.Contains(cell.Label, "spanning") && !strings.Contains(cell.Label, "table header") {+ continue+ }// Skip cells without position data (they can't span).if cell.X0 == 0 && cell.X1 == 0 && cell.Y0 == 0 && cell.Y1 == 0 {continue@@ -99,20 +112,69 @@ func CalSpans(rows [][]pdf.TSRCell) (map[[2]int][2]int, map[[2]int]bool) {}if cs > 1 || rs > 1 {spanInfo[[2]int{i, j}] = [2]int{cs, rs}- // Mark covered cells.+ // Mark covered cells first, unconditionally — a covered cell+ // is dropped from rendered output whether or not it carries+ // text (Python sets tbl[r][c] = None for every covered cell).for ri := i; ri < i+rs && ri < nRows; ri++ {for cj := j; cj < j+cs && cj < nCols; cj++ {+ if cj >= len(rows[ri]) {+ continue // ragged row (post-cleanup grid)+ }if ri != i || cj != j {covered[[2]int{ri, cj}] = true}}}+ // Fold the covered cells' text into the span origin cell,+ // matching Python's __cal_spans (table_structure_recognizer.py:+ // 530-577): it walks the covered region row-major and extends+ // the span cell's text with every covered cell's text (skipping+ // already-folded duplicates via join(arr)). Because Go's+ // GroupCells no longer zeroes covered bboxes, those cells hold+ // their own box text by fill time; without this fold the span+ // cell would render empty and the covered text would be lost.+ var merged []string+ seen := map[string]bool{}+ for ri := i; ri < i+rs && ri < nRows; ri++ {+ for cj := j; cj < j+cs && cj < nCols; cj++ {+ if cj >= len(rows[ri]) {+ continue+ }+ txt := strings.TrimSpace(rows[ri][cj].Text)+ if txt == "" || seen[txt] {+ continue+ }+ seen[txt] = true+ merged = append(merged, txt)+ }+ }+ if len(merged) > 0 {+ rows[i][j].Text = strings.Join(merged, " ")+ }}}}return spanInfo, covered}+// MarkCoveredCells tags every cell covered by a span with a "covered" label+// so downstream consumers can drop them, matching Python's HTML output, which+// omits covered <td> entirely (construct_table __html_table skips arr is+// None). The parity harness reads this to reproduce Python's per-row column+// counts (a span merges covered columns into one rendered cell).+func MarkCoveredCells(rows [][]pdf.TSRCell, covered map[[2]int]bool) {+ for pos := range covered {+ r, c := pos[0], pos[1]+ if r >= len(rows) || c >= len(rows[r]) {+ continue+ }+ if rows[r][c].Label != "" {+ rows[r][c].Label += " "+ }+ rows[r][c].Label += "table covered"+ }+}+// flattenGrid flattens a 2D grid into a 1D slice for fillCellTextFromBoxes.func FlattenGrid(grid [][]pdf.TSRCell) []pdf.TSRCell {n := 0diff --git a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.jsonindex e4f7be679d..6269976b23 100644--- a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json+++ b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json@@ -177,6 +177,30 @@],"permanent": true,"reason": "table_rotation_test.pdf contains three physical tables: page 1 has two tables (grey header 'Header A/B/C', blue header), page 2 has the blue-header table rotated 90 degrees clockwise. Go emits three TableItems (6x3 + 6x3 + 3x6, 15 grid rows total); the rotated table is content-correct (each row = Cell 4A, Cell 3A, Cell 2A, Cell 1A, Header A, plus one padding column). Python's golden emits TWO tables: the first 5x3 is close, but the second is an 8x7 grid that MERGES the page-1 blue-header table with the page-2 rotated table — first 5 rows padded to 7 columns (4 empty columns + 3 data columns), last 3 rows the rotated content, with garbled cell text on both sides ('Header B Cell 1B', 'Cell 2A Cell 3A') inherited from the shared TSR dump (OCR box vs TSR cell boundary offset). The divergence is therefore TABLE SEGMENTATION + column-count, NOT Go cell-content error: Go's split matches the physical PDF structure, and the residual text misplacement is identical on both sides (same TSR input), not a Go assembly bug. Because the cell TEXT is identical on both sides, the old shape-blind gridSim reads 100% — it could NOT see the 15x[...3x12,6x3] vs 13x[3x5,7x8] structural gap. The harness now adds a SHAPE-AWARE structure metric (gridStructureSimilarity): gridSim=100% AND structSim=100% is required for a PDF to be 'content matched' (html-divergent); a go_intentional PDF whose structSim<100 (like this one, structSim=35.7%) is classified INTENTIONAL instead of being hidden under gridSim=100%. Python's merged 8x7 grid is a segmentation defect; Go's split is deliberately retained."+ },+ {+ "id": "table-span-follow-tsr-geometry",+ "tag": "go_intentional",+ "kind": "span_coverage",+ "applies_to": [+ "screenshot.pdf",+ "asset-recovery-services-sd-zh-tw.pdf",+ "asset-recovery-services-data-sanitization-for-enterprise-and-data-destruction-for-enterprise-sb-zh-tw.pdf",+ "data-migration-services-for-cloud-sb-zh-tw.pdf",+ "BookRAG A Hierarchical Structure-aware Index-based Approach.pdf",+ "qa.pdf",+ "lazards-lcoeplus-june-2025.pdf",+ "prodeploy-plus-dell-powervault-jbod-2u-4u-expansion-for-servers-sb-zh-tw.pdf"+ ],+ "fields": [+ "GroupCells",+ "CalSpans",+ "MarkCoveredCells",+ "goTableRows",+ "RowsToStrings"+ ],+ "permanent": true,+ "reason": "These real-PDF tables carry TSR-detected 'table spanning cell' components (verified against the physical documents: screenshot's double-layer 泡点MPa column spans 4 rows; lazards' multi-level column-group headers span columns; dell/asset-recovery/prodeploy marketing templates merge the 服務附件/排除項目 column across rows; asset-recovery-sd's cross-page Terms table merges its first column rows 1-2 and first row cols 2-3). Go's GroupCells labels the span-origin cell 'table spanning cell', keeps every covered cell's row x column bbox, CalSpans folds the covered text into the span cell and tags covered cells, and downstream (goTableRows / RowsToStrings) drops covered cells so per-row column counts follow the span geometry. Python's __cal_spans only folds when a BOX carries an SP annotation, assigned by find_overlapped_with_threshold(box, spans, thr=0.3); when no box in the merged region overlaps the span component by >=30% (empty merged region, or small text boxes inside a large span bbox), Python emits NO colspan/rowspan and renders a flat grid with the merged columns repeated as empty cells. For these PDFs the Python golden shows zero colspan while the TSR spans are real, so Go's structure (merged cells, fewer columns per covered row) is judged at least as good as Python's flat grid. Deliberate divergence, not a regression target."}]}