Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(deepdoc): preserve missed PDF table rows and merged headers (#22073)

jay777218ba216b1c198
diff --git a/internal/deepdoc/parser/pdf/parity_lock.go b/internal/deepdoc/parser/pdf/parity_lock.go
index ecba002f07..3adfc7778a 100644
--- a/internal/deepdoc/parser/pdf/parity_lock.go
+++ b/internal/deepdoc/parser/pdf/parity_lock.go
@@ -15,7 +15,7 @@ package pdf
// go_intentional), never an open/accepted_at_merge failure.
//
// 13_crosspage_table.pdf is NOT locked: the production R/C-assembly path
-// (AnnotateBoxesWithGrid + GroupBoxesByRC) renders its first row at 3 columns
+// (AnnotateBoxesWithGridAndRows + GroupBoxesByRC) renders its first row at 3 columns
// vs Python's 5 (a cross-page merged table whose first-page header boxes get
// re-assigned C indices after dedup/vertical-merge) — structSim 98.8%, a
// documented follow-up, so locking it would forbid a known gap.
diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_test.go
index f11c7d98c7..137460bdd0 100644
--- a/internal/deepdoc/parser/pdf/pipeline_parity_test.go
+++ b/internal/deepdoc/parser/pdf/pipeline_parity_test.go
@@ -160,9 +160,9 @@ func TestPipelineParity(t *testing.T) {
continue
}
- // Production path now derives per-char R/C itself (AnnotateBoxesWithGrid
- // + GroupBoxesByRC); the replay harness measures that production output
- // directly against Python's golden.
+ // Production path now derives per-char R/C itself
+ // (AnnotateBoxesWithGridAndRows + GroupBoxesByRC); the replay harness
+ // measures that production output directly against Python's golden.
// Read Python sections
pyPath := filepath.Join(pyTextDir, name+".txt")
diff --git a/internal/deepdoc/parser/pdf/production_smoke_test.go b/internal/deepdoc/parser/pdf/production_smoke_test.go
index 2bbd31f187..c7c7e9002a 100644
--- a/internal/deepdoc/parser/pdf/production_smoke_test.go
+++ b/internal/deepdoc/parser/pdf/production_smoke_test.go
@@ -14,7 +14,7 @@ import (
// TestProductionAssemblySmoke parses real PDFs through the PRODUCTION path —
// real in-process DLA/TSR/OCR inference (MODEL_DIR; ONNX Runtime statically linked), no replay dump — and checks
-// the converged R/C grid assembly (AnnotateBoxesWithGrid + GroupBoxesByRC)
+// the converged R/C grid assembly (AnnotateBoxesWithGridAndRows + GroupBoxesByRC)
// stays robust: parse does not crash, detected tables are not fully empty, and
// every table has content rows. This is the non-replay sanity check for the
// production assembly convergence.
diff --git a/internal/deepdoc/parser/pdf/table/deepdoc_table_builder.go b/internal/deepdoc/parser/pdf/table/deepdoc_table_builder.go
index df21daa22e..ceaef1c078 100644
--- a/internal/deepdoc/parser/pdf/table/deepdoc_table_builder.go
+++ b/internal/deepdoc/parser/pdf/table/deepdoc_table_builder.go
@@ -3,6 +3,7 @@ package table
import (
"context"
"image"
+ "math"
"strings"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
@@ -34,24 +35,22 @@ func (b *DeepDocTableBuilder) DetectCells(ctx context.Context, cropped image.Ima
// 1. Extract row boundaries from "table row" cells, sort by Y.
// 2. Extract column boundaries from "table column" cells, sort by X.
// 3. Cross-product: grid[r][c].X0/Y0/X1/Y1 = col[c] × row[r].
-// 4. Header propagation: rows overlapping the header cell's Y range
-// get Label = "table column header".
-// 5. Span injection: for each "table spanning cell", find grid cells
-// whose center falls inside the span bbox. The top-left cell gets
-// the span label + extended bbox; remaining cells are zeroed (covered).
+// 4. Header propagation: rows substantially covered by the header cell get
+// Label = "table column header".
+// 5. Span injection: for each "table spanning cell", find the first grid cell
+// whose center falls inside the span bbox. That cell gets the span label
+// and the span's full bbox; the covered cells keep their own geometry.
func (b *DeepDocTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {
if len(cells) == 0 {
return nil
}
// 1. Collect and sort structural elements.
- var rows, cols, spans []pdf.TSRCell
+ var cols, spans []pdf.TSRCell
var header *pdf.TSRCell
for _, c := range cells {
switch {
- case strings.HasSuffix(c.Label, "table row"):
- rows = append(rows, c)
case strings.HasSuffix(c.Label, "table column"):
cols = append(cols, c)
case strings.Contains(strings.ToLower(c.Label), "spanning"):
@@ -62,23 +61,15 @@ func (b *DeepDocTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {
}
}
+ rows := GridRowBands(cells)
if len(rows) == 0 {
return nil
}
- SortYFirstly(rows, 10)
SortXFirstly(cols, 10)
- // Python's _table_transformer_job de-duplicates the structure lines before
- // grouping: rows via gather (layouts_cleanup far=5 thr=0.6), columns via
- // layouts_cleanup far=5 thr=0.5. Overlapping duplicate row/column lines are
- // collapsed (higher detection score wins), so the cross-product grid rows
- // match the rows Python's per-char R/C view counts.
- rows = layoutCleanup(rows, nil, 5, 0.6)
+ // De-duplicate overlapping column lines, matching Python's layouts_cleanup.
cols = layoutCleanup(cols, nil, 5, 0.5)
- if len(rows) == 0 {
- return nil
- }
// 2. If no column cells, synthesize one wide column from row extents.
if len(cols) == 0 {
@@ -104,8 +95,9 @@ func (b *DeepDocTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {
// 4. Header propagation.
if header != nil {
for ri := range rows {
- if rows[ri].Y0 >= header.Y0 && rows[ri].Y1 <= header.Y1 ||
- overlapsY(rows[ri], *header) {
+ // Require substantial coverage: a header component that merely
+ // touches the next row must not mark it as a header.
+ if rowOverlapRatio(rows[ri], *header) >= 0.5 {
for cj := range grid[ri] {
grid[ri][cj].Label = "table column header"
}
@@ -115,37 +107,21 @@ func (b *DeepDocTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {
// 5. Span injection.
//
- // For each TSR "table spanning cell", locate the top-left grid cell of
- // the covered rectangle (the lexicographically smallest (row, col) whose
- // centre lies inside the span bbox) and extend its bbox to the span's
- // bbox, tagging it with the span label. A cell centre ∈ span bbox iff
- // its column centre ∈ [sp.X0,sp.X1] AND its row centre ∈ [sp.Y0,sp.Y1],
- // because every cell in a row shares the row boundary and every cell in
- // a column shares the column boundary. We track the minimum (r, c)
- // directly instead of allocating a covered slice and sorting it — the
- // covered set is recomputed later by CalSpans from this extended bbox,
- // so that slice was dead work. The origin must stay the covered
- // rectangle's top-left: CalSpans derives colspan/rowspan and the
- // `covered` (hidden) map from it, so a non-top-left origin would make
- // the rendered cell span a different rectangle than the one hidden.
- //
- // The remaining covered cells KEEP their row×column bbox. Python's
- // construct_table assigns every box to a cell by its TSR row/column
- // label (R/C) regardless of spanning cells, then __cal_spans folds the
- // covered cells' text into the span origin cell at render time. If Go
- // zeroed these bboxes here, FillCellTextFromBoxesWithRows would skip
- // them, their text would never reach the span cell, and orphan-row/
- // column cleanup would drop the covered rows/columns — the
- // dell/prodeploy/screenshot real-PDF parity gap (11→6 rows).
+ // Find the top-left grid cell whose centre lies inside the span bbox and
+ // extend it to the span's full extent; CalSpans later derives the covered
+ // rectangle from it. Covered cells keep their own bbox so their text can
+ // still be folded into the span origin at render time.
for _, sp := range spans {
firstR, firstC := -1, -1
for ri := range grid {
- cy := (grid[ri][0].Y0 + grid[ri][0].Y1) / 2
+ // Use raw structure lines for the centre: an earlier span may have
+ // widened a grid cell and shifted its centre outside this span.
+ cy := (rows[ri].Y0 + rows[ri].Y1) / 2
if cy < sp.Y0 || cy > sp.Y1 {
continue
}
for cj := range grid[ri] {
- cx := (grid[ri][cj].X0 + grid[ri][cj].X1) / 2
+ cx := (cols[cj].X0 + cols[cj].X1) / 2
if cx >= sp.X0 && cx <= sp.X1 {
if firstR == -1 || ri < firstR || (ri == firstR && cj < firstC) {
firstR, firstC = ri, cj
@@ -153,13 +129,8 @@ func (b *DeepDocTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {
}
}
}
- // A span box that covers only a single grid cell (e.g. a TSR
- // spanning cell whose box only reaches one row's center because the
- // model's box edge missed the neighbour row) is still a real TSR
- // span and must be labelled, not silently dropped. A 1-cell span
- // renders as rowspan=1/colspan=1 (harmless) but keeps the TSR
- // recognition visible downstream. Only skip when no grid cell centre
- // falls inside the box at all (genuine coordinate mismatch).
+ // Keep 1-cell spans: they render harmlessly and preserve TSR
+ // recognition. Skip only when no grid cell centre falls inside.
if firstR == -1 {
continue
}
@@ -173,9 +144,46 @@ func (b *DeepDocTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {
return grid
}
-// overlapsY reports whether two cells overlap in the Y dimension.
-func overlapsY(a, b pdf.TSRCell) bool {
- return a.Y0 < b.Y1 && a.Y1 > b.Y0
+// GridRowBands returns the TSR "table row" lines used to build the grid.
+// The header component is handled separately by GroupCells.
+func GridRowBands(cells []pdf.TSRCell) []pdf.TSRCell {
+ return rowBands(cells, func(label string) bool {
+ return strings.HasSuffix(strings.TrimSpace(strings.ToLower(label)), "table row")
+ })
+}
+
+// AnnotationRowBands returns the TSR row and header lines used for R
+// annotation, matching Python's gather(r".* (row|header)").
+func AnnotationRowBands(cells []pdf.TSRCell) []pdf.TSRCell {
+ return rowBands(cells, func(label string) bool {
+ label = strings.TrimSpace(strings.ToLower(label))
+ return strings.HasSuffix(label, " row") || strings.HasSuffix(label, " header")
+ })
+}
+
+func rowBands(cells []pdf.TSRCell, match func(string) bool) []pdf.TSRCell {
+ var rows []pdf.TSRCell
+ for _, c := range cells {
+ if match(c.Label) {
+ rows = append(rows, c)
+ }
+ }
+ SortYFirstly(rows, 10)
+ return layoutCleanup(rows, nil, 5, 0.6)
+}
+
+// rowOverlapRatio reports the fraction of row's height covered by header.
+func rowOverlapRatio(row, header pdf.TSRCell) float64 {
+ h := row.Y1 - row.Y0
+ if h <= 0 {
+ return 0
+ }
+ top := math.Max(row.Y0, header.Y0)
+ btm := math.Min(row.Y1, header.Y1)
+ if btm <= top {
+ return 0
+ }
+ return (btm - top) / h
}
// FinalizeGrid returns annotGrid unchanged. The DeepDoc builder folds OCR
diff --git a/internal/deepdoc/parser/pdf/table/deepdoc_table_builder_test.go b/internal/deepdoc/parser/pdf/table/deepdoc_table_builder_test.go
index dd7d644fa9..6faacc1ec7 100644
--- a/internal/deepdoc/parser/pdf/table/deepdoc_table_builder_test.go
+++ b/internal/deepdoc/parser/pdf/table/deepdoc_table_builder_test.go
@@ -79,6 +79,69 @@ func TestDeepDocTableBuildService_GroupCells_HeaderPropagation(t *testing.T) {
}
}
+func TestDeepDocTableBuildService_HeaderPropagationIgnoresTouchingDataRow(t *testing.T) {
+ b := &DeepDocTableBuilder{}
+
+ // The header box ends at 50.2 while the first data row starts at 50.0.
+ // The old any-overlap check marked that data row as a header too.
+ cells := []pdf.TSRCell{
+ {X0: 0, Y0: 0, X1: 200, Y1: 150, Label: "table"},
+ {X0: 0, Y0: 0, X1: 200, Y1: 50, Label: "table row"},
+ {X0: 0, Y0: 50, X1: 200, Y1: 100, Label: "table row"},
+ {X0: 0, Y0: 100, X1: 200, Y1: 150, Label: "table row"},
+ {X0: 0, Y0: 0, X1: 100, Y1: 150, Label: "table column"},
+ {X0: 100, Y0: 0, X1: 200, Y1: 150, Label: "table column"},
+ {X0: 0, Y0: 0, X1: 200, Y1: 50.2, Label: "table column header"},
+ }
+
+ grid := b.GroupCells(cells)
+
+ if len(grid) != 3 {
+ t.Fatalf("expected 3 rows, got %d", len(grid))
+ }
+ if grid[0][0].Label != "table column header" {
+ t.Errorf("row 0 label = %q, want table column header", grid[0][0].Label)
+ }
+ if grid[1][0].Label != "" {
+ t.Errorf("row 1 (data, only touching the header by 0.2px) label = %q, want empty", grid[1][0].Label)
+ }
+}
+
+func TestDeepDocTableBuildService_SpanInjectionUsesRawRowCenters(t *testing.T) {
+ b := &DeepDocTableBuilder{}
+
+ // Row 0 carries a column-spanning header and a 3-row vertical span in
+ // column 0. Extending grid[0][0] to y=0..150 first makes its centre
+ // (75) fall outside the 0..50 column span; the old code derived the row
+ // centre from grid[ri][0] and silently dropped the colspan.
+ cells := []pdf.TSRCell{
+ {X0: 0, Y0: 0, X1: 300, Y1: 150, Label: "table"},
+ {X0: 0, Y0: 0, X1: 300, Y1: 50, Label: "table row"},
+ {X0: 0, Y0: 50, X1: 300, Y1: 100, Label: "table row"},
+ {X0: 0, Y0: 100, X1: 300, Y1: 150, Label: "table row"},
+ {X0: 0, Y0: 0, X1: 100, Y1: 150, Label: "table column"},
+ {X0: 100, Y0: 0, X1: 200, Y1: 150, Label: "table column"},
+ {X0: 200, Y0: 0, X1: 300, Y1: 150, Label: "table column"},
+ // rowspan in column 0, listed before the colspan on purpose
+ {X0: 0, Y0: 0, X1: 100, Y1: 150, Label: "table spanning cell"},
+ // colspan across columns 1-2 in row 0
+ {X0: 100, Y0: 0, X1: 300, Y1: 50, Label: "table spanning cell"},
+ }
+
+ grid := b.GroupCells(cells)
+
+ if len(grid) != 3 || len(grid[0]) != 3 {
+ t.Fatalf("expected 3×3 grid, got %d×%d", len(grid), len(grid[0]))
+ }
+ if !strings.Contains(grid[0][1].Label, "spanning") {
+ t.Errorf("colspan at [0,1] was dropped: label = %q (grid row0 labels: %q/%q/%q)",
+ grid[0][1].Label, grid[0][0].Label, grid[0][1].Label, grid[0][2].Label)
+ }
+ if grid[0][1].X0 != 100 || grid[0][1].X1 != 300 {
+ t.Errorf("colspan bbox = (%.0f,%.0f), want (100,300)", grid[0][1].X0, grid[0][1].X1)
+ }
+}
+
func TestDeepDocTableBuildService_GroupCells_SpanInjection(t *testing.T) {
b := &DeepDocTableBuilder{}
diff --git a/internal/deepdoc/parser/pdf/table/group_boxes.go b/internal/deepdoc/parser/pdf/table/group_boxes.go
index 5a794d6c65..dff0f76478 100644
--- a/internal/deepdoc/parser/pdf/table/group_boxes.go
+++ b/internal/deepdoc/parser/pdf/table/group_boxes.go
@@ -1,6 +1,7 @@
package table
import (
+ "math"
"sort"
"strings"
@@ -9,52 +10,113 @@ import (
// rb is a row-box entry that holds cell data during grid construction.
type rb struct {
- row, col int
txt string
x0, y0, x1, y1 float64
label string
}
// GroupBoxesByRC groups text boxes into a cell grid by R/C annotations.
-// Matches Python's construct_table: sort by R, sort by C within each row,
-// merge nearby columns by X proximity.
+// Rows split on R changes; missing-R boxes keep their Y position and also
+// split when Y advances, so a row missed by TSR is still assembled.
func GroupBoxesByRC(boxes []pdf.TextBox) [][]pdf.TSRCell {
if len(boxes) == 0 {
return nil
}
- // If no real R/C annotations (maxR <= 0), fall back to YX coordinate
- // grouping — matching Python's construct_table when all R=-1.
- maxR := 0
+ // Distinguish an annotated table from raw boxes. R/C zero values are the
+ // Go default, so R=0 alone is not evidence of annotation.
+ hasAnnotations := false
for _, b := range boxes {
- if b.R > maxR {
- maxR = b.R
+ if b.HasTableAnnotations || b.R > 0 || b.C > 0 || b.H > 0 || b.SP > 0 {
+ hasAnnotations = true
+ break
}
}
- if maxR <= 0 {
+ if !hasAnnotations {
return GroupBoxesByYX(boxes)
}
- // Sort by R index first (Python: sort_R_firstly), then Y, then X.
- sort.Slice(boxes, func(i, j int) bool {
- if boxes[i].R != boxes[j].R {
- return boxes[i].R < boxes[j].R
- }
- if boxes[i].Top != boxes[j].Top {
- return boxes[i].Top < boxes[j].Top
+
+ sortRFirstly(boxes)
+
+ rowOf := assignRows(boxes)
+ compressed := rowOf[len(rowOf)-1]
+
+ cmap := collectBoxesPerRow(boxes, rowOf)
+ cCompressed, maxCol := compressColIndices(boxes, rowOf, compressed)
+
+ return buildGrid(cmap, cCompressed, maxCol, compressed)
+}
+
+// sortRFirstly mirrors Python's Recognizer.sort_R_firstly: sort by Y first
+// (with a fuzzy row-height threshold), then bubble boxes that carry an R label
+// into R order. Boxes without an R remain at their Y position and act as
+// barriers, so a missing-R row stays between the rows that surround it.
+func sortRFirstly(boxes []pdf.TextBox) {
+ rowh := 0.0
+ for _, b := range boxes {
+ if b.R >= 0 && b.RBott > b.RTop {
+ h := b.RBott - b.RTop
+ if rowh == 0 || h < rowh {
+ rowh = h
+ }
}
- return boxes[i].X0 < boxes[j].X0
- })
+ }
- // Compress R indices: Python's sort_R_firstly grouping.
- rowMap, compressed := compressRowIndices(boxes)
+ sortYFirstlyTextBoxes(boxes, rowh/2)
- // Collect boxes per row.
- cmap, _ := collectBoxesPerRow(boxes, rowMap)
+ for i := 0; i < len(boxes)-1; i++ {
+ for j := i; j >= 0; j-- {
+ a, b := boxes[j], boxes[j+1]
+ if a.R < 0 || b.R < 0 {
+ continue
+ }
+ if b.R < a.R || (b.R == a.R && b.X0 < a.X0) {
+ boxes[j], boxes[j+1] = b, a
+ }
+ }
+ }
+}
- // Compress C indices per row.
- cCompressed, cMaxCol := compressColIndices(boxes, rowMap, compressed)
+// sortYFirstlyTextBoxes is Python's Recognizer.sort_Y_firstly for text boxes:
+// primary key is the top edge, with X as the tie-breaker when two boxes are
+// within threshold. Pages are ordered first because Go boxes carry page-local
+// Y coordinates, while Python's Y values are page-cumulative.
+func sortYFirstlyTextBoxes(boxes []pdf.TextBox, threshold float64) {
+ sort.SliceStable(boxes, func(i, j int) bool {
+ if boxes[i].PageNumber != boxes[j].PageNumber {
+ return boxes[i].PageNumber < boxes[j].PageNumber
+ }
+ diff := boxes[i].Top - boxes[j].Top
+ if math.Abs(diff) < threshold {
+ return boxes[i].X0 < boxes[j].X0
+ }
+ return diff < 0
+ })
+}
- // Build grid.
- return buildGrid(cmap, cCompressed, cMaxCol, compressed)
+// assignRows mirrors Python's construct_table row loop: a new row starts when
+// the R label changes. When R is missing, a box whose top has advanced past
+// the current row's running bottom also starts a new row (Python's -1/-2
+// sentinel comparison); same-R lines stay together so a multi-line cell is not
+// split. The running bottom is averaged across the row's boxes, matching
+// Python's `btm = (btm + b["bottom"]) / 2`.
+func assignRows(boxes []pdf.TextBox) []int {
+ rowOf := make([]int, len(boxes))
+ if len(boxes) == 0 {
+ return rowOf
+ }
+ btm := boxes[0].Bottom
+ for i := 1; i < len(boxes); i++ {
+ if boxes[i].PageNumber != boxes[i-1].PageNumber ||
+ boxes[i-1].R != boxes[i].R ||
+ (boxes[i].R < 0 && boxes[i].Top >= btm-3) {
+ rowOf[i] = rowOf[i-1] + 1
+ btm = boxes[i].Bottom
+ continue
+ }
+ rowOf[i] = rowOf[i-1]
+ btm = (btm + boxes[i].Bottom) / 2
+ }
+ return rowOf
}
// GroupBoxesByYX groups boxes into a cell grid by Y/X coordinates,
@@ -163,42 +225,20 @@ func GroupBoxesByYX(boxes []pdf.TextBox) [][]pdf.TSRCell {
}
// cellPosFromBox returns the position coordinates and label for a cell
-// derived from a text box. Header cells use HLeft/HRight/HTop/HBott
-// for spanning-aware positions; regular cells use the box's own bounds.
+// derived from a text box. Header and span cells use the H* bounds copied
+// from the matched TSR cell; regular cells use the box's own bounds.
func cellPosFromBox(b pdf.TextBox) (x0, y0, x1, y1 float64, label string) {
x0, y0, x1, y1 = b.X0, b.Top, b.X1, b.Bottom
- if b.H > 0 {
- label = "table header"
- if b.HLeft != 0 || b.HRight != 0 {
- if b.HLeft != 0 {
- x0 = b.HLeft
- }
- if b.HRight != 0 {
- x1 = b.HRight
- }
- }
- if b.HTop != 0 {
- y0 = b.HTop
- }
- if b.HBott != 0 {
- y1 = b.HBott
+ // SP wins over H: CalSpans only spans cells labelled "table spanning
+ // cell", so a header cell that is also a span origin must keep that label.
+ if b.SP > 0 || b.H > 0 {
+ if b.SP > 0 {
+ label = "table spanning cell"
+ } else {
+ label = "table header"
}
- } else if b.SP > 0 {
- label = "table spanning cell"
- // A pure-SP box (H==0) still carries the span's full extent in
- // HLeft/HRight/HTop/HBott, copied together from the TSR spanning cell
- // by AnnotateTableBoxes (PR #18707). Rebuild the cell from those bounds
- // so CalSpans covers every column/row the span crosses — matching
- // Python's _annotate_table_boxes. Without this, the span cell falls
- // back to the box's own narrow text bounds, under-covers by one
- // column, and the header row keeps an extra empty cell (real_pdfs/1.pdf
- // row0 = 4 vs Python's 3; see known_diffs rule
- // table-1pdf-colspan-assembly-loss, resolved).
- //
- // HLeft/HRight (and HTop/HBott) are set together, so treat an axis as
- // propagated whenever either edge is set. This preserves a real edge at
- // coordinate 0 (a nonzero check would mistake it for "unset" and fall
- // back to the box's own bounds, dropping the span's left/top edge).
+ // Either edge being set means the axis carries the TSR extent; a
+ // real edge may be exactly 0.
if b.HLeft != 0 || b.HRight != 0 {
x0, x1 = b.HLeft, b.HRight
}
@@ -209,51 +249,19 @@ func cellPosFromBox(b pdf.TextBox) (x0, y0, x1, y1 float64, label string) {
return
}
-// cellLabelFromBox returns the TSR label for a box based on H/SP annotations.
-// Used when merging multiple boxes into one cell — preserves the spanning label.
-func cellLabelFromBox(b pdf.TextBox) string {
- if b.H > 0 {
- return "table header"
- }
- if b.SP > 0 {
- return "table spanning cell"
- }
- return ""
-}
-
-// compressRowIndices compresses R values into contiguous row indices.
-// Returns rowMap (original R → compressed index) and the maximum compressed index.
-// Boxes must already be sorted by R, Y, X.
-func compressRowIndices(boxes []pdf.TextBox) (map[int]int, int) {
- rowMap := make(map[int]int) // original R → compressed row index
- compressed := 0
- rowMap[boxes[0].R] = 0
- lastR := boxes[0].R
- for i := 1; i < len(boxes); i++ {
- if boxes[i].R != lastR {
- compressed++
- rowMap[boxes[i].R] = compressed
- lastR = boxes[i].R
- } else {
- rowMap[boxes[i].R] = compressed
- }
- }
- return rowMap, compressed
-}
-
// collectBoxesPerRow collects boxes into row groups, merging boxes in the same cell.
-// Returns cmap (row → col → entry) and maxCols (max column index per row).
-func collectBoxesPerRow(boxes []pdf.TextBox, rowMap map[int]int) (map[int]map[int]*rb, map[int]int) {
+// rowOf holds the row index assigned to each box by assignRows.
+// Returns cmap (row → col → entry).
+func collectBoxesPerRow(boxes []pdf.TextBox, rowOf []int) map[int]map[int]*rb {
cmap := make(map[int]map[int]*rb) // row → col → entry
- maxCols := make(map[int]int)
- for _, b := range boxes {
+ for i, b := range boxes {
t := strings.TrimSpace(b.Text)
// Keep boxes with SP/H annotations even if text is empty —
// their coordinates are needed for colspan/rowspan calculation.
if t == "" && b.H <= 0 && b.SP <= 0 {
continue
}
- r := rowMap[b.R]
+ r := rowOf[i]
c := b.C
if cmap[r] == nil {
cmap[r] = make(map[int]*rb)
@@ -263,9 +271,14 @@ func collectBoxesPerRow(boxes []pdf.TextBox, rowMap map[int]int) (map[int]map[in
if t != "" {
v.txt += " " + t
}
- // Merge spanning coordinates (use widest extent).
+ // Keep the spanning label if any box in this cell is a span;
+ // CalSpans only spans cells labelled "table spanning cell".
+ if b.SP > 0 {
+ v.label = "table spanning cell"
+ } else if b.H > 0 && v.label != "table spanning cell" {
+ v.label = "table header"
+ }
if b.H > 0 || b.SP > 0 {
- v.label = cellLabelFromBox(b)
if v.x0 > x0 {
v.x0 = x0
}
@@ -280,27 +293,16 @@ func collectBoxesPerRow(boxes []pdf.TextBox, rowMap map[int]int) (map[int]map[in
}
}
} else {
- cmap[r][c] = &rb{r, c, t, x0, y0, x1, y1, label}
- }
- if c > maxCols[r] {
- maxCols[r] = c
+ cmap[r][c] = &rb{txt: t, x0: x0, y0: y0, x1: x1, y1: y1, label: label}
}
}
- return cmap, maxCols
+ return cmap
}
-// compressColIndices assigns every box a column from a GLOBAL rank of the
-// table's distinct C labels, mirroring Python's construct_table which groups
-// boxes into columns by their per-char C assignment (each distinct C yields a
-// column, in C order; a box whose C is the previous box's C+1 always starts a
-// new column — Python never merges adjacent C values by X overlap).
-//
-// A table-wide mapping keeps the same C in the same column across ALL rows: a
-// per-row re-compression would misalign cell text (a C=4 box landing in
-// column 3 in one row and column 4 in another). Every row is sized to the
-// table-wide column count, matching Python's uniform-width construct_table
-// output (rows missing a C render empty cells).
-func compressColIndices(boxes []pdf.TextBox, rowMap map[int]int, compressed int) (map[int]map[int]int, map[int]int) {
+// compressColIndices maps the table's distinct C labels to contiguous column
+// indices. The mapping is table-wide so the same C always lands in the same
+// column across rows; rows without a C render as empty cells.
+func compressColIndices(boxes []pdf.TextBox, rowOf []int, compressed int) (map[int]map[int]int, int) {
distinctC := make(map[int]bool)
for _, b := range boxes {
if b.C >= 0 {
@@ -319,11 +321,10 @@ func compressColIndices(boxes []pdf.TextBox, rowMap map[int]int, compressed int)
maxCol := len(cs) - 1
cCompressed := make(map[int]map[int]int) // row → (original C → compressed col)
- cMaxCol := make(map[int]int) // every row spans the table-wide column count
for ri := 0; ri <= compressed; ri++ {
cMap := make(map[int]int)
- for _, b := range boxes {
- if rowMap[b.R] != ri {
+ for i, b := range boxes {
+ if rowOf[i] != ri {
continue
}
if strings.TrimSpace(b.Text) == "" && b.H <= 0 && b.SP <= 0 {
@@ -334,20 +335,18 @@ func compressColIndices(boxes []pdf.TextBox, rowMap map[int]int, compressed int)
}
}
cCompressed[ri] = cMap
- cMaxCol[ri] = maxCol
}
- return cCompressed, cMaxCol
+ return cCompressed, maxCol
}
// buildGrid builds the final cell grid from the collected and compressed data.
-func buildGrid(cmap map[int]map[int]*rb, cCompressed map[int]map[int]int, cMaxCol map[int]int, compressed int) [][]pdf.TSRCell {
+func buildGrid(cmap map[int]map[int]*rb, cCompressed map[int]map[int]int, maxCol, compressed int) [][]pdf.TSRCell {
rows := make([][]pdf.TSRCell, compressed+1)
for ri := 0; ri <= compressed; ri++ {
- maxC := cMaxCol[ri]
- rows[ri] = make([]pdf.TSRCell, maxC+1)
+ rows[ri] = make([]pdf.TSRCell, maxCol+1)
for ci, v := range cmap[ri] {
cci := cCompressed[ri][ci]
- if cci <= maxC {
+ if cci <= maxCol {
if rows[ri][cci].Text == "" {
rows[ri][cci].Text = v.txt
rows[ri][cci].X0 = v.x0
@@ -381,17 +380,3 @@ func buildGrid(cmap map[int]map[int]*rb, cCompressed map[int]map[int]int, cMaxCo
}
return rows
}
-
-func BoxesHaveAnnotations(boxes []pdf.TextBox) bool {
- maxR, maxC := 0, 0
- for _, b := range boxes {
- if b.R > maxR {
- maxR = b.R
- }
- if b.C > maxC {
- maxC = b.C
- }
- }
- // True if at least 2 rows or 2 cols (R/C are 0-based, so maxR>0 means ≥2 rows).
- return maxR > 0 || maxC > 0
-}
diff --git a/internal/deepdoc/parser/pdf/table/group_boxes_test.go b/internal/deepdoc/parser/pdf/table/group_boxes_test.go
index cc69dc1e9b..190c6e4b2e 100644
--- a/internal/deepdoc/parser/pdf/table/group_boxes_test.go
+++ b/internal/deepdoc/parser/pdf/table/group_boxes_test.go
@@ -1,7 +1,7 @@
package table
import (
- "sort"
+ "reflect"
"strings"
"testing"
@@ -68,6 +68,58 @@ func TestGroupBoxesByRC_RDiffSplitsRow(t *testing.T) {
}
}
+func TestGroupBoxesByRC_MissingRowStaysInYOrder(t *testing.T) {
+ // Mirrors the 01_两行合并表头 PDF: the TSR model missed the 成都公司 row,
+ // so its boxes have no R (-1), while the two header rows carry the raw
+ // TSR row indices 0 and 1. Python's construct_table keeps missing-R boxes
+ // at their Y position, so 成都 must land between 深圳 and 武汉 — not inside
+ // the header row.
+ boxes := []pdf.TextBox{
+ {X0: 10, X1: 90, Top: 0, Bottom: 20, Text: "单位名称", R: 0, C: 0},
+ {X0: 110, X1: 190, Top: 0, Bottom: 20, Text: "集团公司采用", R: 0, C: 1},
+ {X0: 110, X1: 190, Top: 25, Bottom: 45, Text: "政务", R: 1, C: 1},
+ {X0: 210, X1: 290, Top: 25, Bottom: 45, Text: "政工", R: 1, C: 2},
+ {X0: 10, X1: 90, Top: 50, Bottom: 70, Text: "深圳公司", R: 5, C: 0},
+ {X0: 10, X1: 90, Top: 100, Bottom: 120, Text: "成都公司", R: -1, C: 0},
+ {X0: 110, X1: 190, Top: 100, Bottom: 120, Text: "20(1)", R: -1, C: 1},
+ {X0: 10, X1: 90, Top: 150, Bottom: 170, Text: "武汉公司", R: 6, C: 0},
+ }
+
+ rows := GroupBoxesByRC(boxes)
+
+ if len(rows) != 5 {
+ t.Fatalf("expected 5 rows (header0, header1, 深圳, 成都, 武汉), got %d: %v", len(rows), rowTexts(rows))
+ }
+ want := []string{"单位名称", "政务", "深圳公司", "成都公司", "武汉公司"}
+ for i, w := range want {
+ got := ""
+ for _, c := range rows[i] {
+ if c.Text != "" {
+ got = c.Text
+ break
+ }
+ }
+ if got != w {
+ t.Errorf("row %d first text = %q, want %q (rows=%v)", i, got, w, rowTexts(rows))
+ }
+ }
+ if rows[3][0].Text != "成都公司" || rows[3][1].Text != "20(1)" {
+ t.Errorf("成都 row cells = %v, want [成都公司 20(1)]", rowTexts(rows[3:4]))
+ }
+}
+
+func rowTexts(rows [][]pdf.TSRCell) [][]string {
+ out := make([][]string, len(rows))
+ for i, row := range rows {
+ for _, c := range row {
+ if c.Text != "" {
+ out[i] = append(out[i], c.Text)
+ }
+ }
+ }
+ return out
+}
+
func TestFillCellTextFromBoxes_RCOnly(t *testing.T) {
cells := []pdf.TSRCell{
{X0: 0, Y0: 0, X1: 100, Y1: 50, Label: "table"},
@@ -97,6 +149,29 @@ func TestFillCellTextFromBoxes_RCOnly(t *testing.T) {
}
}
+func TestGroupBoxesByRC_ZeroValueBoxesFallBackToYX(t *testing.T) {
+ // Zero-value TextBox has R=0, C=0. That must not be mistaken for an
+ // annotated first row: these four boxes form a 2x2 grid by geometry.
+ boxes := []pdf.TextBox{
+ {X0: 0, X1: 50, Top: 0, Bottom: 10, Text: "A"},
+ {X0: 60, X1: 110, Top: 0, Bottom: 10, Text: "B"},
+ {X0: 0, X1: 50, Top: 20, Bottom: 30, Text: "C"},
+ {X0: 60, X1: 110, Top: 20, Bottom: 30, Text: "D"},
+ }
+
+ rows := GroupBoxesByRC(boxes)
+
+ if len(rows) != 2 {
+ t.Fatalf("expected 2 rows from unannotated boxes, got %d: %v", len(rows), rowTexts(rows))
+ }
+ if len(rows[0]) < 2 || rows[0][0].Text != "A" || rows[0][1].Text != "B" {
+ t.Errorf("row 0 = %v, want [A B]", rowTexts(rows[0:1]))
+ }
+ if len(rows[1]) < 2 || rows[1][0].Text != "C" || rows[1][1].Text != "D" {
+ t.Errorf("row 1 = %v, want [C D]", rowTexts(rows[1:2]))
+ }
+}
+
func TestGroupBoxesByRC_FallbackToYXWhenNoAnnotations(t *testing.T) {
// When all boxes have R=-1 (Python's case: regex didn't match "table" label),
// groupBoxesByRC should fall back to YX coordinate grouping.
@@ -107,7 +182,7 @@ func TestGroupBoxesByRC_FallbackToYXWhenNoAnnotations(t *testing.T) {
{X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "D", R: -1, C: -1},
}
rows := GroupBoxesByRC(boxes)
- // R=-1 for all → maxR = -1 → grid would be 0 rows. Must fall back to YX.
+ // R=-1 for all → no box carries an R annotation → must fall back to YX.
if len(rows) == 0 {
t.Fatal("groupBoxesByRC returned 0 rows when R=-1 — no YX fallback")
}
@@ -134,61 +209,75 @@ func TestGroupBoxesByRC_ColspanMissing(t *testing.T) {
_ = rows
}
-func TestCompressRowIndices(t *testing.T) {
+func TestAssignRows_RChanges(t *testing.T) {
// 6 boxes with 6 different R values → 6 rows.
boxes := []pdf.TextBox{
- {X0: 10, X1: 90, Top: 0, Bottom: 30, Text: "A", R: 0, C: 0},
- {X0: 110, X1: 190, Top: 0, Bottom: 30, Text: "B", R: 1, C: 1},
- {X0: 210, X1: 290, Top: 0, Bottom: 30, Text: "C", R: 2, C: 2},
- {X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "D", R: 3, C: 0},
- {X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "E", R: 4, C: 1},
- {X0: 210, X1: 290, Top: 35, Bottom: 65, Text: "F", R: 5, C: 2},
- }
- // Sort first (as GroupBoxesByRC does)
- sort.Slice(boxes, func(i, j int) bool {
- if boxes[i].R != boxes[j].R {
- return boxes[i].R < boxes[j].R
- }
- if boxes[i].Top != boxes[j].Top {
- return boxes[i].Top < boxes[j].Top
+ {X0: 10, X1: 90, Top: 0, Bottom: 30, Text: "A", R: 0, C: 0, RTop: 0, RBott: 30},
+ {X0: 110, X1: 190, Top: 0, Bottom: 30, Text: "B", R: 1, C: 1, RTop: 0, RBott: 30},
+ {X0: 210, X1: 290, Top: 0, Bottom: 30, Text: "C", R: 2, C: 2, RTop: 0, RBott: 30},
+ {X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "D", R: 3, C: 0, RTop: 35, RBott: 65},
+ {X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "E", R: 4, C: 1, RTop: 35, RBott: 65},
+ {X0: 210, X1: 290, Top: 35, Bottom: 65, Text: "F", R: 5, C: 2, RTop: 35, RBott: 65},
+ }
+ sortRFirstly(boxes)
+ rowOf := assignRows(boxes)
+ want := []int{0, 1, 2, 3, 4, 5}
+ if len(rowOf) != len(want) {
+ t.Fatalf("rowOf len = %d, want %d", len(rowOf), len(want))
+ }
+ for i := range want {
+ if rowOf[i] != want[i] {
+ t.Errorf("rowOf = %v, want %v", rowOf, want)
+ break
}
- return boxes[i].X0 < boxes[j].X0
- })
-
- rowMap, compressed := compressRowIndices(boxes)
- if compressed != 5 { // 0-based, 6 elements → max index 5
- t.Errorf("compressed = %d, want 5", compressed)
}
- if rowMap[0] != 0 || rowMap[1] != 1 || rowMap[2] != 2 || rowMap[3] != 3 || rowMap[4] != 4 || rowMap[5] != 5 {
- t.Errorf("rowMap mapping incorrect: %v", rowMap)
+}
+
+func TestAssignRows_SameR(t *testing.T) {
+ // Multiple boxes with same R and overlapping Y → same row.
+ boxes := []pdf.TextBox{
+ {X0: 10, X1: 90, Top: 0, Bottom: 30, Text: "A", R: 0, C: 0, RTop: 0, RBott: 30},
+ {X0: 110, X1: 190, Top: 0, Bottom: 30, Text: "B", R: 0, C: 1, RTop: 0, RBott: 30},
+ {X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "C", R: 1, C: 0, RTop: 35, RBott: 65},
+ {X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "D", R: 1, C: 1, RTop: 35, RBott: 65},
+ }
+ sortRFirstly(boxes)
+ rowOf := assignRows(boxes)
+ want := []int{0, 0, 1, 1}
+ for i := range want {
+ if rowOf[i] != want[i] {
+ t.Errorf("rowOf = %v, want %v", rowOf, want)
+ break
+ }
}
}
-func TestCompressRowIndices_SameR(t *testing.T) {
- // Multiple boxes with same R → same compressed row.
+func TestAssignRows_PageChangeStartsNewRow(t *testing.T) {
+ // Per-page R numbering can repeat; a page boundary must still start a new
+ // row even when R and Y are unchanged.
boxes := []pdf.TextBox{
- {X0: 10, X1: 90, Top: 0, Bottom: 30, Text: "A", R: 0, C: 0},
- {X0: 110, X1: 190, Top: 0, Bottom: 30, Text: "B", R: 0, C: 1},
- {X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "C", R: 1, C: 0},
- {X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "D", R: 1, C: 1},
+ {X0: 0, X1: 40, Top: 0, Bottom: 10, Text: "A", R: 0, C: 0, RTop: 0, RBott: 10},
+ {X0: 0, X1: 40, Top: 0, Bottom: 10, Text: "B", R: 0, C: 0, PageNumber: 1, RTop: 0, RBott: 10},
}
- // Sort first
- sort.Slice(boxes, func(i, j int) bool {
- if boxes[i].R != boxes[j].R {
- return boxes[i].R < boxes[j].R
- }
- if boxes[i].Top != boxes[j].Top {
- return boxes[i].Top < boxes[j].Top
- }
- return boxes[i].X0 < boxes[j].X0
- })
+ sortRFirstly(boxes)
+ rowOf := assignRows(boxes)
+ if !reflect.DeepEqual(rowOf, []int{0, 1}) {
+ t.Errorf("rowOf = %v, want [0 1]", rowOf)
+ }
+}
- rowMap, compressed := compressRowIndices(boxes)
- if compressed != 1 { // 0-based, 2 rows → max index 1
- t.Errorf("compressed = %d, want 1", compressed)
+func TestAssignRows_SameRMultilineStaysOneRow(t *testing.T) {
+ // Two text boxes in the same cell on separate lines share R; Python keeps
+ // them in one row because the Y-advance branch only applies when R is
+ // missing. Splitting here would break multi-line OCR cells.
+ boxes := []pdf.TextBox{
+ {X0: 0, X1: 40, Top: 5, Bottom: 10, Text: "R/W", R: 0, C: 0, RTop: 0, RBott: 60},
+ {X0: 0, X1: 40, Top: 12, Bottom: 17, Text: "R/W", R: 0, C: 0, RTop: 0, RBott: 60},
}
- if rowMap[0] != 0 || rowMap[1] != 1 {
- t.Errorf("rowMap mapping incorrect: %v", rowMap)
+ sortRFirstly(boxes)
+ rowOf := assignRows(boxes)
+ if !reflect.DeepEqual(rowOf, []int{0, 0}) {
+ t.Errorf("rowOf = %v, want [0 0] (same-R lines stay in one row)", rowOf)
}
}
@@ -199,19 +288,9 @@ func TestCollectBoxesPerRow(t *testing.T) {
{X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "C", R: 1, C: 0},
{X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "D", R: 1, C: 1},
}
- // Sort first
- sort.Slice(boxes, func(i, j int) bool {
- if boxes[i].R != boxes[j].R {
- return boxes[i].R < boxes[j].R
- }
- if boxes[i].Top != boxes[j].Top {
- return boxes[i].Top < boxes[j].Top
- }
- return boxes[i].X0 < boxes[j].X0
- })
-
- rowMap, _ := compressRowIndices(boxes)
- cmap, maxCols := collectBoxesPerRow(boxes, rowMap)
+ sortRFirstly(boxes)
+ rowOf := assignRows(boxes)
+ cmap := collectBoxesPerRow(boxes, rowOf)
if len(cmap) != 2 {
t.Errorf("cmap has %d rows, want 2", len(cmap))
@@ -222,9 +301,6 @@ func TestCollectBoxesPerRow(t *testing.T) {
if cmap[1][0].txt != "C" || cmap[1][1].txt != "D" {
t.Errorf("row 1 incorrect: %v, %v", cmap[1][0], cmap[1][1])
}
- if maxCols[0] != 1 || maxCols[1] != 1 {
- t.Errorf("maxCols incorrect: %v", maxCols)
- }
}
func TestCollectBoxesPerRow_MergeSameCell(t *testing.T) {
@@ -232,19 +308,9 @@ func TestCollectBoxesPerRow_MergeSameCell(t *testing.T) {
{X0: 10, X1: 50, Top: 0, Bottom: 30, Text: "Hello", R: 0, C: 0},
{X0: 50, X1: 90, Top: 0, Bottom: 30, Text: "World", R: 0, C: 0},
}
- // Sort first
- sort.Slice(boxes, func(i, j int) bool {
- if boxes[i].R != boxes[j].R {
- return boxes[i].R < boxes[j].R
- }
- if boxes[i].Top != boxes[j].Top {
- return boxes[i].Top < boxes[j].Top
- }
- return boxes[i].X0 < boxes[j].X0
- })
-
- rowMap, _ := compressRowIndices(boxes)
- cmap, _ := collectBoxesPerRow(boxes, rowMap)
+ sortRFirstly(boxes)
+ rowOf := assignRows(boxes)
+ cmap := collectBoxesPerRow(boxes, rowOf)
if cmap[0][0].txt != "Hello World" {
t.Errorf("merged text incorrect: %q", cmap[0][0].txt)
@@ -258,19 +324,10 @@ func TestCompressColIndices(t *testing.T) {
{X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "C", R: 1, C: 0},
{X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "D", R: 1, C: 1},
}
- // Sort first
- sort.Slice(boxes, func(i, j int) bool {
- if boxes[i].R != boxes[j].R {
- return boxes[i].R < boxes[j].R
- }
- if boxes[i].Top != boxes[j].Top {
- return boxes[i].Top < boxes[j].Top
- }
- return boxes[i].X0 < boxes[j].X0
- })
-
- rowMap, compressed := compressRowIndices(boxes)
- cCompressed, cMaxCol := compressColIndices(boxes, rowMap, compressed)
+ sortRFirstly(boxes)
+ rowOf := assignRows(boxes)
+ compressed := rowOf[len(rowOf)-1]
+ cCompressed, maxCol := compressColIndices(boxes, rowOf, compressed)
if cCompressed[0][0] != 0 || cCompressed[0][1] != 1 {
t.Errorf("row 0 compression incorrect: %v", cCompressed[0])
@@ -278,8 +335,8 @@ func TestCompressColIndices(t *testing.T) {
if cCompressed[1][0] != 0 || cCompressed[1][1] != 1 {
t.Errorf("row 1 compression incorrect: %v", cCompressed[1])
}
- if cMaxCol[0] != 1 || cMaxCol[1] != 1 {
- t.Errorf("cMaxCol incorrect: %v", cMaxCol)
+ if maxCol != 1 {
+ t.Errorf("maxCol = %d, want 1", maxCol)
}
}
@@ -292,27 +349,18 @@ func TestCompressColIndices_AdjacentCAlwaysNewColumn(t *testing.T) {
{X0: 10, X1: 100, Top: 0, Bottom: 30, Text: "A", R: 0, C: 0},
{X0: 90, X1: 190, Top: 0, Bottom: 30, Text: "B", R: 0, C: 1}, // X-overlaps A but C is adjacent
}
- // Sort first
- sort.Slice(boxes, func(i, j int) bool {
- if boxes[i].R != boxes[j].R {
- return boxes[i].R < boxes[j].R
- }
- if boxes[i].Top != boxes[j].Top {
- return boxes[i].Top < boxes[j].Top
- }
- return boxes[i].X0 < boxes[j].X0
- })
-
- rowMap, compressed := compressRowIndices(boxes)
- cCompressed, cMaxCol := compressColIndices(boxes, rowMap, compressed)
+ sortRFirstly(boxes)
+ rowOf := assignRows(boxes)
+ compressed := rowOf[len(rowOf)-1]
+ cCompressed, maxCol := compressColIndices(boxes, rowOf, compressed)
// C=0 → column 0, C=1 → column 1 (distinct C rank, never merged).
if cCompressed[0][0] != 0 || cCompressed[0][1] != 1 {
t.Errorf("adjacent C must split into separate columns: %v", cCompressed[0])
}
- // Table-wide column count = 2 → cMaxCol == 1.
- if cMaxCol[0] != 1 {
- t.Errorf("cMaxCol incorrect: got %d, want 1 (2 distinct C → 2 columns)", cMaxCol[0])
+ // Table-wide column count = 2 → maxCol == 1.
+ if maxCol != 1 {
+ t.Errorf("maxCol = %d, want 1 (2 distinct C → 2 columns)", maxCol)
}
}
@@ -323,22 +371,13 @@ func TestBuildGrid(t *testing.T) {
{X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "C", R: 1, C: 0},
{X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "D", R: 1, C: 1},
}
- // Sort first
- sort.Slice(boxes, func(i, j int) bool {
- if boxes[i].R != boxes[j].R {
- return boxes[i].R < boxes[j].R
- }
- if boxes[i].Top != boxes[j].Top {
- return boxes[i].Top < boxes[j].Top
- }
- return boxes[i].X0 < boxes[j].X0
- })
-
- rowMap, compressed := compressRowIndices(boxes)
- cmap, _ := collectBoxesPerRow(boxes, rowMap)
- cCompressed, cMaxCol := compressColIndices(boxes, rowMap, compressed)
+ sortRFirstly(boxes)
+ rowOf := assignRows(boxes)
+ compressed := rowOf[len(rowOf)-1]
+ cmap := collectBoxesPerRow(boxes, rowOf)
+ cCompressed, maxCol := compressColIndices(boxes, rowOf, compressed)
- rows := buildGrid(cmap, cCompressed, cMaxCol, compressed)
+ rows := buildGrid(cmap, cCompressed, maxCol, compressed)
if len(rows) != 2 {
t.Fatalf("expected 2 rows, got %d", len(rows))
diff --git a/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.go b/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.go
index 88e879a426..10afbf8a6a 100644
--- a/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.go
+++ b/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.go
@@ -34,9 +34,8 @@ func TestGroupBoxesByRCSpanHeaderRowCellCount(t *testing.T) {
// (H==0, SP>0) cellPosFromBox must use those bounds, otherwise the span
// cell falls back to [0,450] and covers only 5 columns.
//
- // R starts at 1 (not 0): GroupBoxesByRC falls back to YX grouping when
- // maxR<=0, which would skip cellPosFromBox entirely; a real table has
- // multiple rows so maxR>0 and the R/C path runs.
+ // R starts at 1 (not 0) so the box carries a positive annotation;
+ // all-zero R/C boxes are treated as annotation-free and use YX grouping.
boxes = append(boxes, pdf.TextBox{
X0: 0, X1: 450, Top: 0, Bottom: 20,
Text: "HEADER", R: 1, C: 0,
diff --git a/internal/deepdoc/parser/pdf/table/table_annotate.go b/internal/deepdoc/parser/pdf/table/table_annotate.go
index eb2bdfed2e..cadfd73ab0 100644
--- a/internal/deepdoc/parser/pdf/table/table_annotate.go
+++ b/internal/deepdoc/parser/pdf/table/table_annotate.go
@@ -505,9 +505,10 @@ func WriteTableAnnotations(boxes []pdf.TextBox, boxIdx []int, cells []pdf.TSRCel
}
}
annotGrid := tb.GroupCells(tableCells)
- AnnotateTableBoxes(tblBoxes, annotGrid)
+ AnnotateBoxesWithGridAndRows(tblBoxes, annotGrid, AnnotationRowBands(tableCells))
for k, idx := range boxIdx {
bp := &tblBoxes[k]
+ boxes[idx].HasTableAnnotations = bp.HasTableAnnotations
boxes[idx].R = bp.R
boxes[idx].RTop = bp.RTop / scale
boxes[idx].RBott = bp.RBott / scale
diff --git a/internal/deepdoc/parser/pdf/table/table_annotate_test.go b/internal/deepdoc/parser/pdf/table/table_annotate_test.go
index 14d9b28815..a702f5ae84 100644
--- a/internal/deepdoc/parser/pdf/table/table_annotate_test.go
+++ b/internal/deepdoc/parser/pdf/table/table_annotate_test.go
@@ -658,10 +658,31 @@ func TestWriteTableAnnotations_EmptyCells(t *testing.T) {
tb := &MockTableBuilder{GroupCellsFn: func(cells []pdf.TSRCell) [][]pdf.TSRCell {
return nil
}}
- // Should not panic with empty cells.
+ // Should not panic with empty cells; R stays missing and C uses the
+ // X-order fallback.
WriteTableAnnotations(boxes, BoxIdx, cells, 3.0, 0, 0, tb)
- if boxes[0].R != 0 || boxes[0].C != 0 {
- t.Errorf("empty cells: R=%d C=%d, want 0,0", boxes[0].R, boxes[0].C)
+ if boxes[0].R != -1 || boxes[0].C != 0 {
+ t.Errorf("empty cells: R=%d C=%d, want -1,0", boxes[0].R, boxes[0].C)
+ }
+}
+
+func TestWriteTableAnnotations_EmptyGridKeepsCFallback(t *testing.T) {
+ boxes := []pdf.TextBox{
+ {X0: 100, X1: 200, Top: 10, Bottom: 30, Text: "B", LayoutType: "table"},
+ {X0: 0, X1: 100, Top: 10, Bottom: 30, Text: "A", LayoutType: "table"},
+ }
+ BoxIdx := []int{0, 1}
+
+ tb := &MockTableBuilder{GroupCellsFn: func(cells []pdf.TSRCell) [][]pdf.TSRCell {
+ return nil
+ }}
+ WriteTableAnnotations(boxes, BoxIdx, nil, 3.0, 0, 0, tb)
+
+ if boxes[1].C != 0 || boxes[0].C != 1 {
+ t.Errorf("C fallback = A:%d B:%d, want A:0 B:1", boxes[1].C, boxes[0].C)
+ }
+ if boxes[0].R != -1 || boxes[1].R != -1 {
+ t.Errorf("empty grid R = %d/%d, want -1/-1", boxes[0].R, boxes[1].R)
}
}
diff --git a/internal/deepdoc/parser/pdf/table/table_construct.go b/internal/deepdoc/parser/pdf/table/table_construct.go
index e33272da45..c058219a07 100644
--- a/internal/deepdoc/parser/pdf/table/table_construct.go
+++ b/internal/deepdoc/parser/pdf/table/table_construct.go
@@ -76,25 +76,10 @@ func constructTableCore(cells []pdf.TSRCell, boxes []pdf.TextBox, caption string
rows = GroupTSRCellsToRows(cells)
}
if len(rows) > 0 && HasText(rows) {
- // Clean up orphan columns then orphan rows (Python order: columns at
- // construct_table:224-277, then rows at :279-333). Both passes mutate
- // the same grid and may drop rows/columns, so item.Grid and item.Rows
- // must be re-derived AFTER them (the old code set item.Rows before
- // column cleanup, leaving it stale; item.Grid also went stale because
- // CleanupOrphanRows returns a re-sliced header).
+ // Cleanup may drop rows/columns, so recompute item.Grid and item.Rows
+ // after both passes.
rows = CleanupOrphanColumns(rows)
- // Drop all-empty rows. Python's construct_table groups boxes by their
- // R annotation (tbl[i] is the boxes for row i; a TSR row with no
- // overlapping box contributes no row). Its HTML emitter also skips
- // rows whose rendered HTML stays at "<tr>" (no cell wrote anything).
- // Go's grid is a TSR-row cross-product, so an extra "table row" that
- // the model emits alongside a "table projected row header" (or any
- // other TSR component with no OCR/text overlap) leaks through as a
- // 5-cell row of empty strings, inflating item.Rows and the HTML. See
- // 13_crosspage_table.pdf page 2: y0=885 "table row" sits on top of
- // a "table projected row header" with no box overlap and survives
- // every TSR cleanup pass. Drop it here so rows/Rows/HTML all match
- // Python.
+ // TSR can emit an empty row component; drop it before orphan cleanup.
rows = DropAllEmptyRows(rows)
rows = CleanupOrphanRows(rows)
hdrs := HeaderSetWithBlockType(rows, boxes)
@@ -108,8 +93,9 @@ func constructTableCore(cells []pdf.TSRCell, boxes []pdf.TextBox, caption string
}
return tableutil.RenderTableHTML(ToTableData(rows, caption, hdrs, spanInfo, covered))
}
- // Fallback: boxes with R/C annotations.
- if len(boxes) > 0 && BoxesHaveAnnotations(boxes) {
+ // Fallback: rebuild from boxes. GroupBoxesByRC uses R/C when present and
+ // falls back to Y/X grouping otherwise.
+ if len(boxes) > 0 {
rows := GroupBoxesByRC(boxes)
if HasText(rows) {
if item != nil {
@@ -119,18 +105,6 @@ func constructTableCore(cells []pdf.TSRCell, boxes []pdf.TextBox, caption string
return tableutil.RenderTableHTML(ToTableData(rows, caption, BoxHeaderSet(rows, boxes), spanInfo, covered))
}
}
- // Test-only: Y/X coordinate grouping (matching Python construct_table).
- // Used by table_parity_test.go to verify pipeline with Python boxes.
- if len(boxes) > 0 && !BoxesHaveAnnotations(boxes) {
- rows := GroupBoxesByYX(boxes)
- if HasText(rows) {
- if item != nil {
- item.Rows = RowsToStrings(rows)
- }
- spanInfo, covered := CalSpans(rows)
- return tableutil.RenderTableHTML(ToTableData(rows, caption, BoxHeaderSet(rows, boxes), spanInfo, covered))
- }
- }
return ""
}
diff --git a/internal/deepdoc/parser/pdf/table/table_construct_test.go b/internal/deepdoc/parser/pdf/table/table_construct_test.go
index 0d84dac366..c068be7f94 100644
--- a/internal/deepdoc/parser/pdf/table/table_construct_test.go
+++ b/internal/deepdoc/parser/pdf/table/table_construct_test.go
@@ -31,6 +31,31 @@ func constructTableData(t *testing.T, cells []pdf.TSRCell, boxes []pdf.TextBox,
return td
}
+func TestConstructTable_ZeroValueBoxesFallback(t *testing.T) {
+ // R=0/C=0 is the Go zero value, not proof of annotation. ConstructTable
+ // must keep the Y/X fallback for raw boxes.
+ boxes := []pdf.TextBox{
+ {X0: 0, X1: 50, Top: 0, Bottom: 10, Text: "A"},
+ {X0: 60, X1: 110, Top: 0, Bottom: 10, Text: "B"},
+ {X0: 0, X1: 50, Top: 20, Bottom: 30, Text: "C"},
+ {X0: 60, X1: 110, Top: 20, Bottom: 30, Text: "D"},
+ }
+
+ td := constructTableData(t, nil, boxes, "", nil)
+ if td == nil {
+ t.Fatal("expected non-nil TableData")
+ }
+ if len(td.Grid) != 2 {
+ t.Fatalf("expected 2 rows, got %d: %v", len(td.Grid), td.Grid)
+ }
+ if cellTextAt(td, 0, 0) != "A" || cellTextAt(td, 0, 1) != "B" {
+ t.Errorf("row 0 = %q/%q, want A/B", cellTextAt(td, 0, 0), cellTextAt(td, 0, 1))
+ }
+ if cellTextAt(td, 1, 0) != "C" || cellTextAt(td, 1, 1) != "D" {
+ t.Errorf("row 1 = %q/%q, want C/D", cellTextAt(td, 1, 0), cellTextAt(td, 1, 1))
+ }
+}
+
func TestCellTexts(t *testing.T) {
cells := []pdf.TSRCell{
{Text: "A"}, {Text: "B"}, {Text: "C"},
diff --git a/internal/deepdoc/parser/pdf/table/table_coords.go b/internal/deepdoc/parser/pdf/table/table_coords.go
index 423b048ff6..cc7b55f0d0 100644
--- a/internal/deepdoc/parser/pdf/table/table_coords.go
+++ b/internal/deepdoc/parser/pdf/table/table_coords.go
@@ -43,6 +43,7 @@ func BoxToCropSpace(b pdf.TextBox, scale, cropOffX, cropOffY float64) pdf.TextBo
// CopyBoxAnnotations copies the DLA/TSR annotation fields from src to dst.
func CopyBoxAnnotations(dst, src *pdf.TextBox) {
+ dst.HasTableAnnotations = src.HasTableAnnotations
dst.R = src.R
dst.C = src.C
dst.RTop = src.RTop
diff --git a/internal/deepdoc/parser/pdf/table/table_coords_test.go b/internal/deepdoc/parser/pdf/table/table_coords_test.go
index 4132931d55..3db2286d14 100644
--- a/internal/deepdoc/parser/pdf/table/table_coords_test.go
+++ b/internal/deepdoc/parser/pdf/table/table_coords_test.go
@@ -42,10 +42,13 @@ func TestBoxToCropSpace(t *testing.T) {
}
func TestCopyBoxAnnotations(t *testing.T) {
- src := &pdf.TextBox{R: 1, C: 2, RTop: 10, RBott: 20, H: 3, HTop: 30, HBott: 40,
+ src := &pdf.TextBox{HasTableAnnotations: true, R: 1, C: 2, RTop: 10, RBott: 20, H: 3, HTop: 30, HBott: 40,
HLeft: 50, HRight: 60, CLeft: 70, CRight: 80, SP: 4}
dst := &pdf.TextBox{}
CopyBoxAnnotations(dst, src)
+ if !dst.HasTableAnnotations {
+ t.Error("annotation flag not copied")
+ }
if dst.R != 1 || dst.C != 2 || dst.RTop != 10 || dst.RBott != 20 {
t.Error("R/C fields not copied")
}
diff --git a/internal/deepdoc/parser/pdf/table/table_layout.go b/internal/deepdoc/parser/pdf/table/table_layout.go
index a8d37e8798..cd1eee489f 100644
--- a/internal/deepdoc/parser/pdf/table/table_layout.go
+++ b/internal/deepdoc/parser/pdf/table/table_layout.go
@@ -174,79 +174,109 @@ func findHorizontallyTightestFit(box pdf.TextBox, clmns []pdf.TSRCell) int {
return best
}
-// AnnotateBoxesWithGrid derives per-box R/C/H/SP annotations in the SAME
-// coordinate frame as grid (e.g. a table's crop space), using Python's
-// _table_transformer_job semantics. It is the production entry point for
-// deriving R/C so the grid can be rebuilt from them (GroupBoxesByRC).
-func AnnotateBoxesWithGrid(boxes []pdf.TextBox, grid [][]pdf.TSRCell) {
- AnnotateTableBoxes(boxes, grid)
+// spanTopRowIndex returns the first physical table row whose centre lies
+// inside the span's Y range — the top row of the merged rectangle (Python
+// __cal_spans minR). Header components are skipped: they are not rows, and
+// using their index would put the span in a different R space from the row
+// cells. Returns -1 when no table row centre falls inside the span.
+func spanTopRowIndex(rows []pdf.TSRCell, sp pdf.TSRCell) int {
+ for i, r := range rows {
+ if isHeaderLabel(r.Label) {
+ continue
+ }
+ cy := (r.Y0 + r.Y1) / 2
+ if cy >= sp.Y0 && cy <= sp.Y1 {
+ return i
+ }
+ }
+ return -1
}
-// annotateTableBoxes tags table boxes with row/header/column indices using
-// TSR cell labels. Matching Python's R/H/C/SP annotation logic.
-//
-// Python: pdf_parser.py:518-554
+// AnnotateBoxesWithGridAndRows derives R/C/H/SP for boxes in the same
+// coordinate frame as grid. R is matched against the raw TSR row bands, not
+// span-extended grid rows.
+func AnnotateBoxesWithGridAndRows(boxes []pdf.TextBox, grid [][]pdf.TSRCell, rows []pdf.TSRCell) {
+ annotateTableBoxes(boxes, grid, rows)
+}
+
+// AnnotateTableBoxes is the fallback when raw TSR rows are unavailable. It
+// matches R against grid rows, which can over-merge a rowspan header;
+// production uses AnnotateBoxesWithGridAndRows.
func AnnotateTableBoxes(boxes []pdf.TextBox, grid [][]pdf.TSRCell) {
- // grid[0] is the header row. Spans are computed by calSpans later.
+ annotateTableBoxes(boxes, grid, nil)
+}
+
+func annotateTableBoxes(boxes []pdf.TextBox, grid [][]pdf.TSRCell, rows []pdf.TSRCell) {
var headers, spans []pdf.TSRCell
var clmns []pdf.TSRCell
- // Python t_recognizer.py: headers = gather(r".*header$") — the set of layout
- // cells whose label ends in "header", NOT the first grid row. Collect them
- // from every grid row so a header that sits on a row other than 0 is still
- // matched and tagged with H>0 (fixes the grid[0] approximation).
+ // Collect header and spanning cells from every grid row. A header is not
+ // necessarily in row 0, and box.SP must be set or CalSpans drops the span.
for _, row := range grid {
for _, cell := range row {
if isHeaderLabel(cell.Label) {
headers = append(headers, cell)
}
- // Collect spanning cells so the SP annotation is propagated to
- // overlapping boxes (Python _table_transformer_job appends every
- // "SP" cell to its `spans` list and matches boxes against it at
- // pdf_parser.py:518-554). Without this, box.SP stays 0, the
- // rebuilt grid (GroupBoxesByRC) loses the span, and
- // ConstructTable/CalSpans drops the colspan/rowspan — Go emits
- // independent empty <th> where Python emits <th colspan=6>
- // (e.g. real_pdfs/1.pdf).
if strings.Contains(cell.Label, "spanning") {
spans = append(spans, cell)
}
}
}
if len(grid) > 0 && len(grid[0]) > 0 {
- // Python's clmns are the "table column" lines: vertical bboxes spanning
- // the whole table height. Derive them from the grid (each column's X
- // range from the first row, Y range from the table's top/bottom rows).
+ // Recover column edges from non-spanning cells: grid[0] alone can be
+ // widened by a span origin and would absorb neighbouring columns.
+ colLeft, colRight, _, _ := gridBounds(grid)
+ nCols := len(colLeft)
tableTop := grid[0][0].Y0
tableBot := grid[len(grid)-1][0].Y1
- clmns = make([]pdf.TSRCell, len(grid[0]))
- for ci := range grid[0] {
- clmns[ci] = pdf.TSRCell{X0: grid[0][ci].X0, Y0: tableTop, X1: grid[0][ci].X1, Y1: tableBot}
+ if len(rows) > 0 {
+ tableTop = rows[0].Y0
+ tableBot = rows[len(rows)-1].Y1
+ }
+ clmns = make([]pdf.TSRCell, nCols)
+ for ci := 0; ci < nCols; ci++ {
+ if colLeft[ci] == 1e9 {
+ // No non-spanning cell in this column: keep the grid bounds.
+ colLeft[ci], colRight[ci] = grid[0][ci].X0, grid[0][ci].X1
+ }
+ clmns[ci] = pdf.TSRCell{X0: colLeft[ci], Y0: tableTop, X1: colRight[ci], Y1: tableBot}
}
}
SortYFirstly(headers, 10)
SortXFirstly(clmns, 10)
for i := range boxes {
- // Python processes only boxes whose layout_type is "table"; callers
- // (processOneTable / WriteTableAnnotations) already pass the table
- // region's box subset, so an empty LayoutType (e.g. OCR-replay boxes
- // that carry no DLA annotation) is treated as table content too.
+ // Empty LayoutType is table content too (e.g. OCR replay boxes).
if boxes[i].LayoutType != pdf.LayoutTypeTable && boxes[i].LayoutType != "" {
continue
}
- // R: Python find_overlapped_with_threshold(box, rows, 0.3) over the
- // WHOLE row line — the grid row's bbox spans the table width (the row
- // line's own X range), not individual grid cells.
- for ri, row := range grid {
- if len(row) == 0 {
- continue
- }
- rowBBox := pdf.TSRCell{X0: row[0].X0, Y0: row[0].Y0, X1: row[len(row)-1].X1, Y1: row[0].Y1}
- if findOverlappedWithThreshold(boxes[i], []pdf.TSRCell{rowBBox}, 0.3) >= 0 {
+ boxes[i].HasTableAnnotations = true
+ // R: Python find_overlapped_with_threshold(box, rows, 0.3). Python
+ // leaves R unset when no row line matches; using Go's zero value would
+ // silently put that box in row 0, so start from -1 and only write a
+ // row when the match succeeds.
+ boxes[i].R = -1
+ boxes[i].RTop = 0
+ boxes[i].RBott = 0
+ if len(rows) > 0 {
+ if ri := findOverlappedWithThreshold(boxes[i], rows, 0.3); ri >= 0 {
boxes[i].R = ri
- boxes[i].RTop = row[0].Y0
- boxes[i].RBott = row[0].Y1
- break
+ boxes[i].RTop = rows[ri].Y0
+ boxes[i].RBott = rows[ri].Y1
+ }
+ } else {
+ // Fallback: match against grid rows. A span-extended row can
+ // over-merge; production passes raw rows to avoid it.
+ for ri, row := range grid {
+ if len(row) == 0 {
+ continue
+ }
+ rowBBox := pdf.TSRCell{X0: row[0].X0, Y0: row[0].Y0, X1: row[len(row)-1].X1, Y1: row[0].Y1}
+ if findOverlappedWithThreshold(boxes[i], []pdf.TSRCell{rowBBox}, 0.3) >= 0 {
+ boxes[i].R = ri
+ boxes[i].RTop = row[0].Y0
+ boxes[i].RBott = row[0].Y1
+ break
+ }
}
}
if idx := findOverlappedWithThreshold(boxes[i], headers, 0.3); idx >= 0 {
@@ -254,11 +284,8 @@ func AnnotateTableBoxes(boxes []pdf.TextBox, grid [][]pdf.TSRCell) {
boxes[i].HBott = headers[idx].Y1
boxes[i].HLeft = headers[idx].X0
boxes[i].HRight = headers[idx].X1
- // Offset by 1: store idx+1 so a box matching the FIRST header cell
- // (idx == 0) is distinguishable from "no header overlap" (the
- // default H == 0). All readers check H > 0, so this keeps the
- // boolean semantics while fixing single-column / first-column
- // header detection (parity #4, asymmetry 1).
+ // Store idx+1: H>0 means matched, and an index of 0 is still
+ // distinguishable from "no header".
boxes[i].H = idx + 1
}
// C: Python find_horizontally_tightest_fit(box, clmns).
@@ -270,27 +297,28 @@ func AnnotateTableBoxes(boxes []pdf.TextBox, grid [][]pdf.TSRCell) {
}
}
if idx := findOverlappedWithThreshold(boxes[i], spans, 0.3); idx >= 0 {
- // Offset by 1 so a box matching the FIRST spanning cell
- // (idx == 0) is distinguishable from "no span overlap" (the
- // default SP == 0). All readers check SP > 0, matching Python's
- // boolean SP semantics (pdf_parser.py:518-554).
+ // Store idx+1: SP>0 means matched, including span index 0.
boxes[i].SP = idx + 1
- // Python _annotate_table_boxes (pdf_parser.py:632-635) copies the
- // spanning cell's bbox onto the box as H_top/H_bott/H_left/H_right.
- // GroupBoxesByRC then builds the span cell from these full extents
- // (cellPosFromBox uses HLeft/HRight when H>0), so CalSpans covers
- // every column the span crosses. Without this, the span cell falls
- // back to the box's own narrow bounds and Go emits colspan=5 where
- // Python emits colspan=6 (real_pdfs/1.pdf).
+ // Copy the span's full extent so cellPosFromBox can build a cell
+ // that covers every row/column the span crosses.
boxes[i].HTop = spans[idx].Y0
boxes[i].HBott = spans[idx].Y1
boxes[i].HLeft = spans[idx].X0
boxes[i].HRight = spans[idx].X1
+ // A rowspan cell's text is vertically centred and may sit in the
+ // lower row; move R to the span's top row (Python __cal_spans
+ // places the merged cell at tbl[minR][minC]).
+ if len(rows) > 0 {
+ if sr := spanTopRowIndex(rows, spans[idx]); sr >= 0 {
+ boxes[i].R = sr
+ boxes[i].RTop = rows[sr].Y0
+ boxes[i].RBott = rows[sr].Y1
+ }
+ }
}
}
- // Two-pass C fallback: after all R values are assigned, compute C by X-order within each row.
- // This matches Python's behavior when TSR provides few "table column" cells.
+ // No usable column structure: assign C by X order within each R group.
if len(clmns) <= 1 {
// Collect all table boxes grouped by R (LayoutType empty → table content).
rBoxes := make(map[int][]int)
diff --git a/internal/deepdoc/parser/pdf/table/table_layout_test.go b/internal/deepdoc/parser/pdf/table/table_layout_test.go
index d64fad730d..829587121c 100644
--- a/internal/deepdoc/parser/pdf/table/table_layout_test.go
+++ b/internal/deepdoc/parser/pdf/table/table_layout_test.go
@@ -166,6 +166,226 @@ func TestAnnotateTableBoxes(t *testing.T) {
}
}
+// TestAnnotateTableBoxes_NoRowMatchLeavesRMissing pins Python's missing-R
+// state: a box that overlaps no row line must stay unassigned (-1), not fall
+// back to Go's zero value (row 0). The old behavior silently folded a row the
+// TSR model missed into the header row.
+func TestAnnotateTableBoxes_NoRowMatchLeavesRMissing(t *testing.T) {
+ grid := [][]pdf.TSRCell{
+ {{X0: 0, Y0: 0, X1: 100, Y1: 20, Label: "table row"}},
+ }
+ boxes := []pdf.TextBox{
+ {X0: 0, X1: 100, Top: 100, Bottom: 120, Text: "成都公司", LayoutType: pdf.LayoutTypeTable},
+ }
+
+ AnnotateTableBoxes(boxes, grid)
+
+ if boxes[0].R != -1 {
+ t.Errorf("unmatched box R = %d, want -1 (missing); a zero value makes GroupBoxesByRC fold it into row 0", boxes[0].R)
+ }
+}
+
+// TestAnnotateTableBoxes_RawRowBandsOverrideSpanExtendedGrid pins that R is
+// matched against the raw TSR row lines, not the span-extended grid cells.
+// grid[0] covers both physical rows because a rowspan cell was extended over
+// them; a box in the second physical row must still get R=1.
+func TestAnnotateTableBoxes_RawRowBandsOverrideSpanExtendedGrid(t *testing.T) {
+ grid := [][]pdf.TSRCell{
+ // The first row's bbox was extended to y=100 by a rowspan cell, but
+ // the cell itself is not a span (so it must not pull R up by span
+ // origin relocation either).
+ {{X0: 0, Y0: 0, X1: 100, Y1: 100, Label: "table row"}},
+ {{X0: 0, Y0: 20, X1: 100, Y1: 40, Label: "table row"}},
+ }
+ rows := []pdf.TSRCell{
+ {X0: 0, Y0: 0, X1: 100, Y1: 20, Label: "table row"},
+ {X0: 0, Y0: 20, X1: 100, Y1: 40, Label: "table row"},
+ }
+ boxes := []pdf.TextBox{
+ {X0: 0, X1: 100, Top: 20, Bottom: 40, Text: "第二行", LayoutType: pdf.LayoutTypeTable},
+ }
+
+ AnnotateBoxesWithGridAndRows(boxes, grid, rows)
+
+ if boxes[0].R != 1 {
+ t.Errorf("second physical row R = %d, want 1; the span-extended grid row 0 must not swallow it", boxes[0].R)
+ }
+ if boxes[0].RTop != 20 || boxes[0].RBott != 40 {
+ t.Errorf("R band = (%v, %v), want (20, 40)", boxes[0].RTop, boxes[0].RBott)
+ }
+}
+
+// TestAnnotateTableBoxes_ColumnBoundsIgnoreSpanExtension pins that C is
+// matched against the real column boundaries, recovered from non-spanning
+// cells. Using grid[0] directly would let a wide spanning header cover every
+// column and assign all boxes under it to the span's origin column.
+func TestAnnotateTableBoxes_ColumnBoundsIgnoreSpanExtension(t *testing.T) {
+ grid := [][]pdf.TSRCell{
+ {
+ {X0: 0, Y0: 0, X1: 300, Y1: 40, Label: "table spanning cell"},
+ {X0: 100, Y0: 0, X1: 200, Y1: 20, Label: "table column header"},
+ {X0: 200, Y0: 0, X1: 300, Y1: 20, Label: "table column header"},
+ },
+ {
+ {X0: 0, Y0: 20, X1: 100, Y1: 40, Label: "table row"},
+ {X0: 100, Y0: 20, X1: 200, Y1: 40, Label: "table row"},
+ {X0: 200, Y0: 20, X1: 300, Y1: 40, Label: "table row"},
+ },
+ }
+ boxes := []pdf.TextBox{
+ {X0: 250, X1: 300, Top: 0, Bottom: 20, Text: "第三列", LayoutType: pdf.LayoutTypeTable},
+ }
+
+ AnnotateTableBoxes(boxes, grid)
+
+ if boxes[0].C != 2 {
+ t.Errorf("box in the third column got C=%d, want 2; a span-extended grid[0] column must not absorb it", boxes[0].C)
+ }
+}
+
+// TestAnnotateTableBoxes_MovesRowspanOriginToTopRow pins that a rowspan cell
+// whose text is vertically centred in the lower row is moved to the span's
+// top row, matching Python __cal_spans (tbl[minR][minC]).
+func TestAnnotateTableBoxes_MovesRowspanOriginToTopRow(t *testing.T) {
+ grid := [][]pdf.TSRCell{
+ {{X0: 0, Y0: 0, X1: 100, Y1: 40, Label: "table spanning cell"}},
+ {{X0: 0, Y0: 20, X1: 100, Y1: 40, Label: "table row"}},
+ }
+ rows := []pdf.TSRCell{
+ {X0: 0, Y0: 0, X1: 100, Y1: 20, Label: "table row"},
+ {X0: 0, Y0: 20, X1: 100, Y1: 40, Label: "table row"},
+ }
+ boxes := []pdf.TextBox{
+ // Text is centred in the lower row, but the span starts at row 0.
+ {X0: 0, X1: 100, Top: 25, Bottom: 35, Text: "单位名称", LayoutType: pdf.LayoutTypeTable},
+ }
+
+ AnnotateBoxesWithGridAndRows(boxes, grid, rows)
+
+ if boxes[0].R != 0 {
+ t.Errorf("rowspan origin R = %d, want 0 (top row of the span)", boxes[0].R)
+ }
+ if boxes[0].SP <= 0 {
+ t.Errorf("span not detected: SP=%d", boxes[0].SP)
+ }
+}
+
+func TestAnnotateBoxesWithGridAndRows_SpanUsesRowNotHeaderIndex(t *testing.T) {
+ rows := []pdf.TSRCell{
+ {X0: 0, Y0: 0, X1: 100, Y1: 100, Label: "table column header"}, // index 0
+ {X0: 0, Y0: 0, X1: 100, Y1: 40, Label: "table row"}, // index 1
+ {X0: 0, Y0: 40, X1: 100, Y1: 80, Label: "table row"}, // index 2
+ }
+ grid := [][]pdf.TSRCell{
+ {
+ {X0: 0, Y0: 0, X1: 50, Y1: 80, Label: "table spanning cell"},
+ {X0: 50, Y0: 0, X1: 100, Y1: 40, Label: "table row"},
+ },
+ {
+ {X0: 0, Y0: 40, X1: 50, Y1: 80, Label: "table row"},
+ {X0: 50, Y0: 40, X1: 100, Y1: 80, Label: "table row"},
+ },
+ }
+ boxes := []pdf.TextBox{
+ // Span text sits in the lower row but the span starts at row 1.
+ {X0: 10, X1: 40, Top: 45, Bottom: 55, Text: "span", LayoutType: pdf.LayoutTypeTable},
+ // Normal cell in the first header row.
+ {X0: 60, X1: 90, Top: 5, Bottom: 15, Text: "normal", LayoutType: pdf.LayoutTypeTable},
+ }
+
+ AnnotateBoxesWithGridAndRows(boxes, grid, rows)
+
+ if boxes[0].SP == 0 {
+ t.Fatal("span box was not annotated")
+ }
+ if boxes[0].R != 1 {
+ t.Errorf("span R = %d, want 1 (table row index); the header component is index 0", boxes[0].R)
+ }
+ if boxes[1].R != 1 {
+ t.Errorf("normal header cell R = %d, want 1", boxes[1].R)
+ }
+}
+
+func TestAnnotateBoxesWithGridAndRows_RealPDFSpanRIndex(t *testing.T) {
+ // 1.pdf header coordinates from TestGroupCellsInjectsSpanning:
+ // header band 110.9..231.0, row0 111.4..138.9, row1 138.3..231.4,
+ // rowspan span #18 135.3..229.7.
+ cells := []pdf.TSRCell{
+ {X0: 81.1, Y0: 110.4, X1: 212.6, Y1: 290.6, Label: "table column"},
+ {X0: 211.5, Y0: 110.4, X1: 249.5, Y1: 290.6, Label: "table column"},
+ {X0: 249.4, Y0: 110.4, X1: 286.1, Y1: 290.6, Label: "table column"},
+ {X0: 287.2, Y0: 110.4, X1: 331.3, Y1: 290.6, Label: "table column"},
+ {X0: 80.3, Y0: 111.4, X1: 503.0, Y1: 138.9, Label: "table row"},
+ {X0: 80.3, Y0: 138.3, X1: 502.2, Y1: 231.4, Label: "table row"},
+ {X0: 80.3, Y0: 230.3, X1: 501.8, Y1: 290.6, Label: "table row"},
+ {X0: 79.1, Y0: 110.9, X1: 503.6, Y1: 231.0, Label: "table column header"},
+ {X0: 210.3, Y0: 135.3, X1: 249.7, Y1: 229.7, Label: "table spanning cell"},
+ }
+
+ grid := (&DeepDocTableBuilder{}).GroupCells(cells)
+ if grid == nil {
+ t.Fatal("GroupCells returned nil")
+ }
+
+ boxes := []pdf.TextBox{
+ {X0: 90, X1: 200, Top: 112, Bottom: 138, Text: "h0c0", LayoutType: pdf.LayoutTypeTable},
+ {X0: 250, X1: 280, Top: 112, Bottom: 138, Text: "h0c2", LayoutType: pdf.LayoutTypeTable},
+ {X0: 290, X1: 325, Top: 112, Bottom: 138, Text: "h0c3", LayoutType: pdf.LayoutTypeTable},
+ {X0: 90, X1: 200, Top: 140, Bottom: 229, Text: "h1c0", LayoutType: pdf.LayoutTypeTable},
+ {X0: 250, X1: 280, Top: 140, Bottom: 229, Text: "h1c2", LayoutType: pdf.LayoutTypeTable},
+ {X0: 290, X1: 325, Top: 140, Bottom: 229, Text: "h1c3", LayoutType: pdf.LayoutTypeTable},
+ {X0: 220, X1: 240, Top: 160, Bottom: 180, Text: "名称", LayoutType: pdf.LayoutTypeTable},
+ {X0: 90, X1: 200, Top: 240, Bottom: 280, Text: "d0", LayoutType: pdf.LayoutTypeTable},
+ {X0: 250, X1: 280, Top: 240, Bottom: 280, Text: "d2", LayoutType: pdf.LayoutTypeTable},
+ {X0: 290, X1: 325, Top: 240, Bottom: 280, Text: "d3", LayoutType: pdf.LayoutTypeTable},
+ }
+
+ rowsBands := AnnotationRowBands(cells)
+ AnnotateBoxesWithGridAndRows(boxes, grid, rowsBands)
+
+ spanIdx, row1Idx := 6, 3
+ if boxes[spanIdx].SP == 0 {
+ t.Fatal("span box was not annotated with SP")
+ }
+ if got := spanTopRowIndex(rowsBands, cells[len(cells)-1]); got != boxes[row1Idx].R {
+ t.Errorf("span top row = %d, row1 R = %d; they must use the same annotation index", got, boxes[row1Idx].R)
+ }
+ if boxes[spanIdx].R != boxes[row1Idx].R {
+ t.Errorf("span R = %d, row1 cell R = %d; mixed R index spaces", boxes[spanIdx].R, boxes[row1Idx].R)
+ }
+
+ rows := GroupBoxesByRC(boxes)
+ if len(rows) != 3 {
+ t.Fatalf("expected 3 logical rows (2 header + 1 data), got %d: %v", len(rows), rowTexts(rows))
+ }
+ if len(rows[1]) != 4 || len(rows[2]) != 4 {
+ t.Errorf("header/data row widths = %d/%d, want 4/4", len(rows[1]), len(rows[2]))
+ }
+}
+
+func TestRowBandsSplitGridAndAnnotationSemantics(t *testing.T) {
+ cells := []pdf.TSRCell{
+ {X0: 0, Y0: 0, X1: 100, Y1: 100, Label: "table column header"},
+ {X0: 0, Y0: 0, X1: 100, Y1: 20, Label: "table row"},
+ {X0: 0, Y0: 0, X1: 10, Y1: 100, Label: "table column"},
+ {X0: 0, Y0: 20, X1: 100, Y1: 40, Label: "table row"},
+ {X0: 0, Y0: 40, X1: 100, Y1: 60, Label: "table projected row header"},
+ }
+
+ gridRows := GridRowBands(cells)
+ if len(gridRows) != 2 {
+ t.Fatalf("GridRowBands returned %d rows, want 2 (table row only): %+v", len(gridRows), gridRows)
+ }
+ if gridRows[0].Y0 != 0 || gridRows[1].Y0 != 20 {
+ t.Errorf("grid row bands = (%v, %v), want (0, 20)", gridRows[0].Y0, gridRows[1].Y0)
+ }
+
+ annotationRows := AnnotationRowBands(cells)
+ if len(annotationRows) != 4 {
+ t.Fatalf("AnnotationRowBands returned %d rows, want 4 (row + header): %+v", len(annotationRows), annotationRows)
+ }
+}
+
// ── GroupTSRCellsToRows ─────────────────────────────────────────
func TestGroupTSRCellsToRowsLabeled(t *testing.T) {
diff --git a/internal/deepdoc/parser/pdf/table/table_spans.go b/internal/deepdoc/parser/pdf/table/table_spans.go
index c2237f2e95..8032f709ce 100644
--- a/internal/deepdoc/parser/pdf/table/table_spans.go
+++ b/internal/deepdoc/parser/pdf/table/table_spans.go
@@ -6,46 +6,31 @@ import (
pdf "ragflow/internal/deepdoc/parser/pdf/type"
)
-// calSpans computes colspan and rowspan for spanning cells in the grid.
-// Returns spanInfo (row,col → colspan,rowspan) and covered (cells hidden by spans).
-// Matches Python's __cal_spans (table_structure_recognizer.py:535).
-func CalSpans(rows [][]pdf.TSRCell) (map[[2]int][2]int, map[[2]int]bool) {
- spanInfo := make(map[[2]int][2]int)
- covered := make(map[[2]int]bool)
+// gridBounds returns the per-column and per-row geometry from non-spanning
+// cells. Spanning cells are excluded because their bbox is extended over the
+// covered region; zero-position padding cells are excluded because they would
+// drag boundaries to the origin. Both span detection and R/C annotation use
+// these bounds, so the two stages agree on the table's real column edges.
+func gridBounds(rows [][]pdf.TSRCell) (colLeft, colRight, rowTop, rowBott []float64) {
if len(rows) == 0 || len(rows[0]) == 0 {
- return spanInfo, covered
+ return nil, nil, nil, nil
}
-
- // Compute column center positions.
- nCols := len(rows[0])
- colLeft := make([]float64, nCols)
- colRight := make([]float64, nCols)
+ nCols, nRows := len(rows[0]), len(rows)
+ colLeft = make([]float64, nCols)
+ colRight = make([]float64, nCols)
+ rowTop = make([]float64, nRows)
+ rowBott = make([]float64, nRows)
for j := 0; j < nCols; j++ {
- colLeft[j] = 1e9
- colRight[j] = -1e9
+ colLeft[j], colRight[j] = 1e9, -1e9
}
- nRows := len(rows)
- rowTop := make([]float64, nRows)
- rowBott := make([]float64, nRows)
for i := 0; i < nRows; i++ {
- rowTop[i] = 1e9
- rowBott[i] = -1e9
+ rowTop[i], rowBott[i] = 1e9, -1e9
}
-
for i, row := range rows {
for j, cell := range row {
- if j >= nCols {
- continue
- }
- // Exclude spanning cells from column/row boundary calculations.
- // Use label-based detection (O(1), no dependency on column midpoints).
- if strings.Contains(cell.Label, "spanning") {
+ if j >= nCols || strings.Contains(cell.Label, "spanning") {
continue
}
- // Cells without position data (e.g. the zero-coordinate cells
- // padded by MergeTablesAcrossPages to align per-page column
- // counts) must not define column/row geometry, or they would drag
- // boundaries to the origin and corrupt span detection.
if cell.X0 == 0 && cell.X1 == 0 && cell.Y0 == 0 && cell.Y1 == 0 {
continue
}
@@ -63,24 +48,29 @@ func CalSpans(rows [][]pdf.TSRCell) (map[[2]int][2]int, map[[2]int]bool) {
}
}
}
+ return colLeft, colRight, rowTop, rowBott
+}
+
+// calSpans computes colspan and rowspan for spanning cells in the grid.
+// Returns spanInfo (row,col → colspan,rowspan) and covered (cells hidden by spans).
+// Matches Python's __cal_spans (table_structure_recognizer.py:535).
+func CalSpans(rows [][]pdf.TSRCell) (map[[2]int][2]int, map[[2]int]bool) {
+ spanInfo := make(map[[2]int][2]int)
+ covered := make(map[[2]int]bool)
+ if len(rows) == 0 || len(rows[0]) == 0 {
+ return spanInfo, covered
+ }
+
+ colLeft, colRight, rowTop, rowBott := gridBounds(rows)
+ if colLeft == nil {
+ return spanInfo, covered
+ }
+ nCols := len(colLeft)
+ nRows := len(rowTop)
- // For each spanning cell, compute how many cols/rows it covers.
- // Only cells explicitly recognized as a span participate. Python's
- // __cal_spans (table_structure_recognizer.py:500-528) iterates boxes that
- // carry an "SP" annotation (`if "SP" not in b: continue`) and never
- // treats an ordinary "table header" as a span origin. In Go the TSR
- // "table spanning cell" component is the SP equivalent and is labelled
- // "spanning" by GroupBoxesByRC/GroupCells, so that is the only branch we
- // honour here.
- //
- // A previous Go-only branch also spanned "table header" cells whose
- // HLeft/HRight bbox straddled a neighbour column. That diverges from
- // Python and is unsafe: when a cross-page merge widens a header cell's
- // bbox (e.g. icbccs "类型" after MergeTablesAcrossPages stacks two pages'
- // grids), its X1 can reach past the neighbour column's center and get
- // misclassified as a colspan, folding the neighbour's text in and dropping
- // a column from the rendered row. Removing the header branch makes Go
- // match Python and avoids that regression.
+ // Only cells explicitly labelled "spanning" participate (Python's SP).
+ // A plain header must never become a span origin: a cross-page merge can
+ // widen a header bbox over a neighbouring column.
for i, row := range rows {
for j, cell := range row {
if j >= nCols || covered[[2]int{i, j}] {
@@ -165,14 +155,9 @@ func CalSpans(rows [][]pdf.TSRCell) (map[[2]int][2]int, map[[2]int]bool) {
}
}
}
- // Fold the covered cells' text into the span origin cell,
- // matching Python's __cal_spans (table_structure_recognizer.py:
- // 530-577): it walks the covered region row-major and extends
- // the span cell's text with every covered cell's text (skipping
- // already-folded duplicates via join(arr)). Because Go's
- // GroupCells no longer zeroes covered bboxes, those cells hold
- // their own box text by fill time; without this fold the span
- // cell would render empty and the covered text would be lost.
+ // Fold covered-cell text into the span origin, matching Python's
+ // __cal_spans. Covered cells keep their bbox and text, so this is
+ // where their text reaches the rendered span cell.
var merged []string
seen := map[string]bool{}
for ri := minR; ri <= maxR && ri < nRows; ri++ {
diff --git a/internal/deepdoc/parser/pdf/table_extract.go b/internal/deepdoc/parser/pdf/table_extract.go
index 754484bf6f..3e9ec89741 100644
--- a/internal/deepdoc/parser/pdf/table_extract.go
+++ b/internal/deepdoc/parser/pdf/table_extract.go
@@ -185,9 +185,10 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxes
// gather) is used ONLY to derive per-box R/C annotations.
annotGrid := tb.GroupCells(cells)
if len(annotGrid) > 0 {
- // Derive R/C/H/SP with Python's _table_transformer_job semantics
- // (whole-row/whole-column line matching), in crop space.
- tbl.AnnotateBoxesWithGrid(boxInCrop, annotGrid)
+ // Derive R/C/H/SP in crop space. R uses the raw TSR row bands,
+ // not the span-extended grid rows, so a rowspan header does not
+ // pull the next physical row into its origin row.
+ tbl.AnnotateBoxesWithGridAndRows(boxInCrop, annotGrid, tbl.AnnotationRowBands(cells))
boxInCrop = dedupSameCellOCRBoxes(boxInCrop)
// Rebuild the grid from the derived per-char R/C, exactly like
// Python's construct_table groups boxes by their R/C labels —
diff --git a/internal/deepdoc/parser/pdf/tool/py_intermediates.go b/internal/deepdoc/parser/pdf/tool/py_intermediates.go
index cf58407e85..5230310a44 100644
--- a/internal/deepdoc/parser/pdf/tool/py_intermediates.go
+++ b/internal/deepdoc/parser/pdf/tool/py_intermediates.go
@@ -213,18 +213,19 @@ func LoadPythonTableBoxes(jsonPath string) ([]pdf.TextBox, error) {
boxes := make([]pdf.TextBox, 0, len(dumped))
for _, b := range dumped {
boxes = append(boxes, pdf.TextBox{
- X0: b.X0,
- X1: b.X1,
- Top: b.Top,
- Bottom: b.Bottom,
- Text: b.Text,
- R: b.R,
- C: b.C,
- H: b.H,
- SP: b.SP,
- PageNumber: b.PageNumber,
- LayoutNo: b.LayoutNo,
- LayoutType: b.LayoutType,
+ X0: b.X0,
+ X1: b.X1,
+ Top: b.Top,
+ Bottom: b.Bottom,
+ Text: b.Text,
+ HasTableAnnotations: true,
+ R: b.R,
+ C: b.C,
+ H: b.H,
+ SP: b.SP,
+ PageNumber: b.PageNumber,
+ LayoutNo: b.LayoutNo,
+ LayoutType: b.LayoutType,
})
}
return boxes, nil
diff --git a/internal/deepdoc/parser/type/types.go b/internal/deepdoc/parser/type/types.go
index 60ba30e21e..000d4f93cb 100644
--- a/internal/deepdoc/parser/type/types.go
+++ b/internal/deepdoc/parser/type/types.go
@@ -120,7 +120,11 @@ type TextBox struct {
// FigureBounds retains DLA geometry in PDF points until figure consolidation.
FigureBounds *Position
ColID int
- R int
+ // HasTableAnnotations marks that the post-TSR table annotation pass ran
+ // for this box. R/C zero values are otherwise indistinguishable from
+ // "not annotated".
+ HasTableAnnotations bool
+ R int
// IsOCR marks a box produced by an OCR pass (ocrDetectAndRecognize /
// ocrMergeChars), as opposed to one built from embedded PDF chars
// (CharsToBoxes). It scopes OCR-only post-processing (see layout.Dedup*)