RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(deepdoc): preserve missed PDF table rows and merged headers (#22073)
jay777218ba216b1c198
diff --git a/internal/deepdoc/parser/pdf/parity_lock.go b/internal/deepdoc/parser/pdf/parity_lock.goindex ecba002f07..3adfc7778a 100644--- a/internal/deepdoc/parser/pdf/parity_lock.go+++ b/internal/deepdoc/parser/pdf/parity_lock.go@@ -15,7 +15,7 @@ package pdf// go_intentional), never an open/accepted_at_merge failure.//// 13_crosspage_table.pdf is NOT locked: the production R/C-assembly path-// (AnnotateBoxesWithGrid + GroupBoxesByRC) renders its first row at 3 columns+// (AnnotateBoxesWithGridAndRows + GroupBoxesByRC) renders its first row at 3 columns// vs Python's 5 (a cross-page merged table whose first-page header boxes get// re-assigned C indices after dedup/vertical-merge) — structSim 98.8%, a// documented follow-up, so locking it would forbid a known gap.diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_test.goindex f11c7d98c7..137460bdd0 100644--- a/internal/deepdoc/parser/pdf/pipeline_parity_test.go+++ b/internal/deepdoc/parser/pdf/pipeline_parity_test.go@@ -160,9 +160,9 @@ func TestPipelineParity(t *testing.T) {continue}- // Production path now derives per-char R/C itself (AnnotateBoxesWithGrid- // + GroupBoxesByRC); the replay harness measures that production output- // directly against Python's golden.+ // Production path now derives per-char R/C itself+ // (AnnotateBoxesWithGridAndRows + GroupBoxesByRC); the replay harness+ // measures that production output directly against Python's golden.// Read Python sectionspyPath := filepath.Join(pyTextDir, name+".txt")diff --git a/internal/deepdoc/parser/pdf/production_smoke_test.go b/internal/deepdoc/parser/pdf/production_smoke_test.goindex 2bbd31f187..c7c7e9002a 100644--- a/internal/deepdoc/parser/pdf/production_smoke_test.go+++ b/internal/deepdoc/parser/pdf/production_smoke_test.go@@ -14,7 +14,7 @@ import (// TestProductionAssemblySmoke parses real PDFs through the PRODUCTION path —// real in-process DLA/TSR/OCR inference (MODEL_DIR; ONNX Runtime statically linked), no replay dump — and checks-// the converged R/C grid assembly (AnnotateBoxesWithGrid + GroupBoxesByRC)+// the converged R/C grid assembly (AnnotateBoxesWithGridAndRows + GroupBoxesByRC)// stays robust: parse does not crash, detected tables are not fully empty, and// every table has content rows. This is the non-replay sanity check for the// production assembly convergence.diff --git a/internal/deepdoc/parser/pdf/table/deepdoc_table_builder.go b/internal/deepdoc/parser/pdf/table/deepdoc_table_builder.goindex df21daa22e..ceaef1c078 100644--- a/internal/deepdoc/parser/pdf/table/deepdoc_table_builder.go+++ b/internal/deepdoc/parser/pdf/table/deepdoc_table_builder.go@@ -3,6 +3,7 @@ package tableimport ("context""image"+ "math""strings"pdf "ragflow/internal/deepdoc/parser/pdf/type"@@ -34,24 +35,22 @@ func (b *DeepDocTableBuilder) DetectCells(ctx context.Context, cropped image.Ima// 1. Extract row boundaries from "table row" cells, sort by Y.// 2. Extract column boundaries from "table column" cells, sort by X.// 3. Cross-product: grid[r][c].X0/Y0/X1/Y1 = col[c] × row[r].-// 4. Header propagation: rows overlapping the header cell's Y range-// get Label = "table column header".-// 5. Span injection: for each "table spanning cell", find grid cells-// whose center falls inside the span bbox. The top-left cell gets-// the span label + extended bbox; remaining cells are zeroed (covered).+// 4. Header propagation: rows substantially covered by the header cell get+// Label = "table column header".+// 5. Span injection: for each "table spanning cell", find the first grid cell+// whose center falls inside the span bbox. That cell gets the span label+// and the span's full bbox; the covered cells keep their own geometry.func (b *DeepDocTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {if len(cells) == 0 {return nil}// 1. Collect and sort structural elements.- var rows, cols, spans []pdf.TSRCell+ var cols, spans []pdf.TSRCellvar header *pdf.TSRCellfor _, c := range cells {switch {- case strings.HasSuffix(c.Label, "table row"):- rows = append(rows, c)case strings.HasSuffix(c.Label, "table column"):cols = append(cols, c)case strings.Contains(strings.ToLower(c.Label), "spanning"):@@ -62,23 +61,15 @@ func (b *DeepDocTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {}}+ rows := GridRowBands(cells)if len(rows) == 0 {return nil}- SortYFirstly(rows, 10)SortXFirstly(cols, 10)- // Python's _table_transformer_job de-duplicates the structure lines before- // grouping: rows via gather (layouts_cleanup far=5 thr=0.6), columns via- // layouts_cleanup far=5 thr=0.5. Overlapping duplicate row/column lines are- // collapsed (higher detection score wins), so the cross-product grid rows- // match the rows Python's per-char R/C view counts.- rows = layoutCleanup(rows, nil, 5, 0.6)+ // De-duplicate overlapping column lines, matching Python's layouts_cleanup.cols = layoutCleanup(cols, nil, 5, 0.5)- if len(rows) == 0 {- return nil- }// 2. If no column cells, synthesize one wide column from row extents.if len(cols) == 0 {@@ -104,8 +95,9 @@ func (b *DeepDocTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {// 4. Header propagation.if header != nil {for ri := range rows {- if rows[ri].Y0 >= header.Y0 && rows[ri].Y1 <= header.Y1 ||- overlapsY(rows[ri], *header) {+ // Require substantial coverage: a header component that merely+ // touches the next row must not mark it as a header.+ if rowOverlapRatio(rows[ri], *header) >= 0.5 {for cj := range grid[ri] {grid[ri][cj].Label = "table column header"}@@ -115,37 +107,21 @@ func (b *DeepDocTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {// 5. Span injection.//- // For each TSR "table spanning cell", locate the top-left grid cell of- // the covered rectangle (the lexicographically smallest (row, col) whose- // centre lies inside the span bbox) and extend its bbox to the span's- // bbox, tagging it with the span label. A cell centre ∈ span bbox iff- // its column centre ∈ [sp.X0,sp.X1] AND its row centre ∈ [sp.Y0,sp.Y1],- // because every cell in a row shares the row boundary and every cell in- // a column shares the column boundary. We track the minimum (r, c)- // directly instead of allocating a covered slice and sorting it — the- // covered set is recomputed later by CalSpans from this extended bbox,- // so that slice was dead work. The origin must stay the covered- // rectangle's top-left: CalSpans derives colspan/rowspan and the- // `covered` (hidden) map from it, so a non-top-left origin would make- // the rendered cell span a different rectangle than the one hidden.- //- // The remaining covered cells KEEP their row×column bbox. Python's- // construct_table assigns every box to a cell by its TSR row/column- // label (R/C) regardless of spanning cells, then __cal_spans folds the- // covered cells' text into the span origin cell at render time. If Go- // zeroed these bboxes here, FillCellTextFromBoxesWithRows would skip- // them, their text would never reach the span cell, and orphan-row/- // column cleanup would drop the covered rows/columns — the- // dell/prodeploy/screenshot real-PDF parity gap (11→6 rows).+ // Find the top-left grid cell whose centre lies inside the span bbox and+ // extend it to the span's full extent; CalSpans later derives the covered+ // rectangle from it. Covered cells keep their own bbox so their text can+ // still be folded into the span origin at render time.for _, sp := range spans {firstR, firstC := -1, -1for ri := range grid {- cy := (grid[ri][0].Y0 + grid[ri][0].Y1) / 2+ // Use raw structure lines for the centre: an earlier span may have+ // widened a grid cell and shifted its centre outside this span.+ cy := (rows[ri].Y0 + rows[ri].Y1) / 2if cy < sp.Y0 || cy > sp.Y1 {continue}for cj := range grid[ri] {- cx := (grid[ri][cj].X0 + grid[ri][cj].X1) / 2+ cx := (cols[cj].X0 + cols[cj].X1) / 2if cx >= sp.X0 && cx <= sp.X1 {if firstR == -1 || ri < firstR || (ri == firstR && cj < firstC) {firstR, firstC = ri, cj@@ -153,13 +129,8 @@ func (b *DeepDocTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {}}}- // A span box that covers only a single grid cell (e.g. a TSR- // spanning cell whose box only reaches one row's center because the- // model's box edge missed the neighbour row) is still a real TSR- // span and must be labelled, not silently dropped. A 1-cell span- // renders as rowspan=1/colspan=1 (harmless) but keeps the TSR- // recognition visible downstream. Only skip when no grid cell centre- // falls inside the box at all (genuine coordinate mismatch).+ // Keep 1-cell spans: they render harmlessly and preserve TSR+ // recognition. Skip only when no grid cell centre falls inside.if firstR == -1 {continue}@@ -173,9 +144,46 @@ func (b *DeepDocTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {return grid}-// overlapsY reports whether two cells overlap in the Y dimension.-func overlapsY(a, b pdf.TSRCell) bool {- return a.Y0 < b.Y1 && a.Y1 > b.Y0+// GridRowBands returns the TSR "table row" lines used to build the grid.+// The header component is handled separately by GroupCells.+func GridRowBands(cells []pdf.TSRCell) []pdf.TSRCell {+ return rowBands(cells, func(label string) bool {+ return strings.HasSuffix(strings.TrimSpace(strings.ToLower(label)), "table row")+ })+}++// AnnotationRowBands returns the TSR row and header lines used for R+// annotation, matching Python's gather(r".* (row|header)").+func AnnotationRowBands(cells []pdf.TSRCell) []pdf.TSRCell {+ return rowBands(cells, func(label string) bool {+ label = strings.TrimSpace(strings.ToLower(label))+ return strings.HasSuffix(label, " row") || strings.HasSuffix(label, " header")+ })+}++func rowBands(cells []pdf.TSRCell, match func(string) bool) []pdf.TSRCell {+ var rows []pdf.TSRCell+ for _, c := range cells {+ if match(c.Label) {+ rows = append(rows, c)+ }+ }+ SortYFirstly(rows, 10)+ return layoutCleanup(rows, nil, 5, 0.6)+}++// rowOverlapRatio reports the fraction of row's height covered by header.+func rowOverlapRatio(row, header pdf.TSRCell) float64 {+ h := row.Y1 - row.Y0+ if h <= 0 {+ return 0+ }+ top := math.Max(row.Y0, header.Y0)+ btm := math.Min(row.Y1, header.Y1)+ if btm <= top {+ return 0+ }+ return (btm - top) / h}// FinalizeGrid returns annotGrid unchanged. The DeepDoc builder folds OCRdiff --git a/internal/deepdoc/parser/pdf/table/deepdoc_table_builder_test.go b/internal/deepdoc/parser/pdf/table/deepdoc_table_builder_test.goindex dd7d644fa9..6faacc1ec7 100644--- a/internal/deepdoc/parser/pdf/table/deepdoc_table_builder_test.go+++ b/internal/deepdoc/parser/pdf/table/deepdoc_table_builder_test.go@@ -79,6 +79,69 @@ func TestDeepDocTableBuildService_GroupCells_HeaderPropagation(t *testing.T) {}}+func TestDeepDocTableBuildService_HeaderPropagationIgnoresTouchingDataRow(t *testing.T) {+ b := &DeepDocTableBuilder{}++ // The header box ends at 50.2 while the first data row starts at 50.0.+ // The old any-overlap check marked that data row as a header too.+ cells := []pdf.TSRCell{+ {X0: 0, Y0: 0, X1: 200, Y1: 150, Label: "table"},+ {X0: 0, Y0: 0, X1: 200, Y1: 50, Label: "table row"},+ {X0: 0, Y0: 50, X1: 200, Y1: 100, Label: "table row"},+ {X0: 0, Y0: 100, X1: 200, Y1: 150, Label: "table row"},+ {X0: 0, Y0: 0, X1: 100, Y1: 150, Label: "table column"},+ {X0: 100, Y0: 0, X1: 200, Y1: 150, Label: "table column"},+ {X0: 0, Y0: 0, X1: 200, Y1: 50.2, Label: "table column header"},+ }++ grid := b.GroupCells(cells)++ if len(grid) != 3 {+ t.Fatalf("expected 3 rows, got %d", len(grid))+ }+ if grid[0][0].Label != "table column header" {+ t.Errorf("row 0 label = %q, want table column header", grid[0][0].Label)+ }+ if grid[1][0].Label != "" {+ t.Errorf("row 1 (data, only touching the header by 0.2px) label = %q, want empty", grid[1][0].Label)+ }+}++func TestDeepDocTableBuildService_SpanInjectionUsesRawRowCenters(t *testing.T) {+ b := &DeepDocTableBuilder{}++ // Row 0 carries a column-spanning header and a 3-row vertical span in+ // column 0. Extending grid[0][0] to y=0..150 first makes its centre+ // (75) fall outside the 0..50 column span; the old code derived the row+ // centre from grid[ri][0] and silently dropped the colspan.+ cells := []pdf.TSRCell{+ {X0: 0, Y0: 0, X1: 300, Y1: 150, Label: "table"},+ {X0: 0, Y0: 0, X1: 300, Y1: 50, Label: "table row"},+ {X0: 0, Y0: 50, X1: 300, Y1: 100, Label: "table row"},+ {X0: 0, Y0: 100, X1: 300, Y1: 150, Label: "table row"},+ {X0: 0, Y0: 0, X1: 100, Y1: 150, Label: "table column"},+ {X0: 100, Y0: 0, X1: 200, Y1: 150, Label: "table column"},+ {X0: 200, Y0: 0, X1: 300, Y1: 150, Label: "table column"},+ // rowspan in column 0, listed before the colspan on purpose+ {X0: 0, Y0: 0, X1: 100, Y1: 150, Label: "table spanning cell"},+ // colspan across columns 1-2 in row 0+ {X0: 100, Y0: 0, X1: 300, Y1: 50, Label: "table spanning cell"},+ }++ grid := b.GroupCells(cells)++ if len(grid) != 3 || len(grid[0]) != 3 {+ t.Fatalf("expected 3×3 grid, got %d×%d", len(grid), len(grid[0]))+ }+ if !strings.Contains(grid[0][1].Label, "spanning") {+ t.Errorf("colspan at [0,1] was dropped: label = %q (grid row0 labels: %q/%q/%q)",+ grid[0][1].Label, grid[0][0].Label, grid[0][1].Label, grid[0][2].Label)+ }+ if grid[0][1].X0 != 100 || grid[0][1].X1 != 300 {+ t.Errorf("colspan bbox = (%.0f,%.0f), want (100,300)", grid[0][1].X0, grid[0][1].X1)+ }+}+func TestDeepDocTableBuildService_GroupCells_SpanInjection(t *testing.T) {b := &DeepDocTableBuilder{}diff --git a/internal/deepdoc/parser/pdf/table/group_boxes.go b/internal/deepdoc/parser/pdf/table/group_boxes.goindex 5a794d6c65..dff0f76478 100644--- a/internal/deepdoc/parser/pdf/table/group_boxes.go+++ b/internal/deepdoc/parser/pdf/table/group_boxes.go@@ -1,6 +1,7 @@package tableimport (+ "math""sort""strings"@@ -9,52 +10,113 @@ import (// rb is a row-box entry that holds cell data during grid construction.type rb struct {- row, col inttxt stringx0, y0, x1, y1 float64label string}// GroupBoxesByRC groups text boxes into a cell grid by R/C annotations.-// Matches Python's construct_table: sort by R, sort by C within each row,-// merge nearby columns by X proximity.+// Rows split on R changes; missing-R boxes keep their Y position and also+// split when Y advances, so a row missed by TSR is still assembled.func GroupBoxesByRC(boxes []pdf.TextBox) [][]pdf.TSRCell {if len(boxes) == 0 {return nil}- // If no real R/C annotations (maxR <= 0), fall back to YX coordinate- // grouping — matching Python's construct_table when all R=-1.- maxR := 0+ // Distinguish an annotated table from raw boxes. R/C zero values are the+ // Go default, so R=0 alone is not evidence of annotation.+ hasAnnotations := falsefor _, b := range boxes {- if b.R > maxR {- maxR = b.R+ if b.HasTableAnnotations || b.R > 0 || b.C > 0 || b.H > 0 || b.SP > 0 {+ hasAnnotations = true+ break}}- if maxR <= 0 {+ if !hasAnnotations {return GroupBoxesByYX(boxes)}- // Sort by R index first (Python: sort_R_firstly), then Y, then X.- sort.Slice(boxes, func(i, j int) bool {- if boxes[i].R != boxes[j].R {- return boxes[i].R < boxes[j].R- }- if boxes[i].Top != boxes[j].Top {- return boxes[i].Top < boxes[j].Top++ sortRFirstly(boxes)++ rowOf := assignRows(boxes)+ compressed := rowOf[len(rowOf)-1]++ cmap := collectBoxesPerRow(boxes, rowOf)+ cCompressed, maxCol := compressColIndices(boxes, rowOf, compressed)++ return buildGrid(cmap, cCompressed, maxCol, compressed)+}++// sortRFirstly mirrors Python's Recognizer.sort_R_firstly: sort by Y first+// (with a fuzzy row-height threshold), then bubble boxes that carry an R label+// into R order. Boxes without an R remain at their Y position and act as+// barriers, so a missing-R row stays between the rows that surround it.+func sortRFirstly(boxes []pdf.TextBox) {+ rowh := 0.0+ for _, b := range boxes {+ if b.R >= 0 && b.RBott > b.RTop {+ h := b.RBott - b.RTop+ if rowh == 0 || h < rowh {+ rowh = h+ }}- return boxes[i].X0 < boxes[j].X0- })+ }- // Compress R indices: Python's sort_R_firstly grouping.- rowMap, compressed := compressRowIndices(boxes)+ sortYFirstlyTextBoxes(boxes, rowh/2)- // Collect boxes per row.- cmap, _ := collectBoxesPerRow(boxes, rowMap)+ for i := 0; i < len(boxes)-1; i++ {+ for j := i; j >= 0; j-- {+ a, b := boxes[j], boxes[j+1]+ if a.R < 0 || b.R < 0 {+ continue+ }+ if b.R < a.R || (b.R == a.R && b.X0 < a.X0) {+ boxes[j], boxes[j+1] = b, a+ }+ }+ }+}- // Compress C indices per row.- cCompressed, cMaxCol := compressColIndices(boxes, rowMap, compressed)+// sortYFirstlyTextBoxes is Python's Recognizer.sort_Y_firstly for text boxes:+// primary key is the top edge, with X as the tie-breaker when two boxes are+// within threshold. Pages are ordered first because Go boxes carry page-local+// Y coordinates, while Python's Y values are page-cumulative.+func sortYFirstlyTextBoxes(boxes []pdf.TextBox, threshold float64) {+ sort.SliceStable(boxes, func(i, j int) bool {+ if boxes[i].PageNumber != boxes[j].PageNumber {+ return boxes[i].PageNumber < boxes[j].PageNumber+ }+ diff := boxes[i].Top - boxes[j].Top+ if math.Abs(diff) < threshold {+ return boxes[i].X0 < boxes[j].X0+ }+ return diff < 0+ })+}- // Build grid.- return buildGrid(cmap, cCompressed, cMaxCol, compressed)+// assignRows mirrors Python's construct_table row loop: a new row starts when+// the R label changes. When R is missing, a box whose top has advanced past+// the current row's running bottom also starts a new row (Python's -1/-2+// sentinel comparison); same-R lines stay together so a multi-line cell is not+// split. The running bottom is averaged across the row's boxes, matching+// Python's `btm = (btm + b["bottom"]) / 2`.+func assignRows(boxes []pdf.TextBox) []int {+ rowOf := make([]int, len(boxes))+ if len(boxes) == 0 {+ return rowOf+ }+ btm := boxes[0].Bottom+ for i := 1; i < len(boxes); i++ {+ if boxes[i].PageNumber != boxes[i-1].PageNumber ||+ boxes[i-1].R != boxes[i].R ||+ (boxes[i].R < 0 && boxes[i].Top >= btm-3) {+ rowOf[i] = rowOf[i-1] + 1+ btm = boxes[i].Bottom+ continue+ }+ rowOf[i] = rowOf[i-1]+ btm = (btm + boxes[i].Bottom) / 2+ }+ return rowOf}// GroupBoxesByYX groups boxes into a cell grid by Y/X coordinates,@@ -163,42 +225,20 @@ func GroupBoxesByYX(boxes []pdf.TextBox) [][]pdf.TSRCell {}// cellPosFromBox returns the position coordinates and label for a cell-// derived from a text box. Header cells use HLeft/HRight/HTop/HBott-// for spanning-aware positions; regular cells use the box's own bounds.+// derived from a text box. Header and span cells use the H* bounds copied+// from the matched TSR cell; regular cells use the box's own bounds.func cellPosFromBox(b pdf.TextBox) (x0, y0, x1, y1 float64, label string) {x0, y0, x1, y1 = b.X0, b.Top, b.X1, b.Bottom- if b.H > 0 {- label = "table header"- if b.HLeft != 0 || b.HRight != 0 {- if b.HLeft != 0 {- x0 = b.HLeft- }- if b.HRight != 0 {- x1 = b.HRight- }- }- if b.HTop != 0 {- y0 = b.HTop- }- if b.HBott != 0 {- y1 = b.HBott+ // SP wins over H: CalSpans only spans cells labelled "table spanning+ // cell", so a header cell that is also a span origin must keep that label.+ if b.SP > 0 || b.H > 0 {+ if b.SP > 0 {+ label = "table spanning cell"+ } else {+ label = "table header"}- } else if b.SP > 0 {- label = "table spanning cell"- // A pure-SP box (H==0) still carries the span's full extent in- // HLeft/HRight/HTop/HBott, copied together from the TSR spanning cell- // by AnnotateTableBoxes (PR #18707). Rebuild the cell from those bounds- // so CalSpans covers every column/row the span crosses — matching- // Python's _annotate_table_boxes. Without this, the span cell falls- // back to the box's own narrow text bounds, under-covers by one- // column, and the header row keeps an extra empty cell (real_pdfs/1.pdf- // row0 = 4 vs Python's 3; see known_diffs rule- // table-1pdf-colspan-assembly-loss, resolved).- //- // HLeft/HRight (and HTop/HBott) are set together, so treat an axis as- // propagated whenever either edge is set. This preserves a real edge at- // coordinate 0 (a nonzero check would mistake it for "unset" and fall- // back to the box's own bounds, dropping the span's left/top edge).+ // Either edge being set means the axis carries the TSR extent; a+ // real edge may be exactly 0.if b.HLeft != 0 || b.HRight != 0 {x0, x1 = b.HLeft, b.HRight}@@ -209,51 +249,19 @@ func cellPosFromBox(b pdf.TextBox) (x0, y0, x1, y1 float64, label string) {return}-// cellLabelFromBox returns the TSR label for a box based on H/SP annotations.-// Used when merging multiple boxes into one cell — preserves the spanning label.-func cellLabelFromBox(b pdf.TextBox) string {- if b.H > 0 {- return "table header"- }- if b.SP > 0 {- return "table spanning cell"- }- return ""-}--// compressRowIndices compresses R values into contiguous row indices.-// Returns rowMap (original R → compressed index) and the maximum compressed index.-// Boxes must already be sorted by R, Y, X.-func compressRowIndices(boxes []pdf.TextBox) (map[int]int, int) {- rowMap := make(map[int]int) // original R → compressed row index- compressed := 0- rowMap[boxes[0].R] = 0- lastR := boxes[0].R- for i := 1; i < len(boxes); i++ {- if boxes[i].R != lastR {- compressed++- rowMap[boxes[i].R] = compressed- lastR = boxes[i].R- } else {- rowMap[boxes[i].R] = compressed- }- }- return rowMap, compressed-}-// collectBoxesPerRow collects boxes into row groups, merging boxes in the same cell.-// Returns cmap (row → col → entry) and maxCols (max column index per row).-func collectBoxesPerRow(boxes []pdf.TextBox, rowMap map[int]int) (map[int]map[int]*rb, map[int]int) {+// rowOf holds the row index assigned to each box by assignRows.+// Returns cmap (row → col → entry).+func collectBoxesPerRow(boxes []pdf.TextBox, rowOf []int) map[int]map[int]*rb {cmap := make(map[int]map[int]*rb) // row → col → entry- maxCols := make(map[int]int)- for _, b := range boxes {+ for i, b := range boxes {t := strings.TrimSpace(b.Text)// Keep boxes with SP/H annotations even if text is empty —// their coordinates are needed for colspan/rowspan calculation.if t == "" && b.H <= 0 && b.SP <= 0 {continue}- r := rowMap[b.R]+ r := rowOf[i]c := b.Cif cmap[r] == nil {cmap[r] = make(map[int]*rb)@@ -263,9 +271,14 @@ func collectBoxesPerRow(boxes []pdf.TextBox, rowMap map[int]int) (map[int]map[inif t != "" {v.txt += " " + t}- // Merge spanning coordinates (use widest extent).+ // Keep the spanning label if any box in this cell is a span;+ // CalSpans only spans cells labelled "table spanning cell".+ if b.SP > 0 {+ v.label = "table spanning cell"+ } else if b.H > 0 && v.label != "table spanning cell" {+ v.label = "table header"+ }if b.H > 0 || b.SP > 0 {- v.label = cellLabelFromBox(b)if v.x0 > x0 {v.x0 = x0}@@ -280,27 +293,16 @@ func collectBoxesPerRow(boxes []pdf.TextBox, rowMap map[int]int) (map[int]map[in}}} else {- cmap[r][c] = &rb{r, c, t, x0, y0, x1, y1, label}- }- if c > maxCols[r] {- maxCols[r] = c+ cmap[r][c] = &rb{txt: t, x0: x0, y0: y0, x1: x1, y1: y1, label: label}}}- return cmap, maxCols+ return cmap}-// compressColIndices assigns every box a column from a GLOBAL rank of the-// table's distinct C labels, mirroring Python's construct_table which groups-// boxes into columns by their per-char C assignment (each distinct C yields a-// column, in C order; a box whose C is the previous box's C+1 always starts a-// new column — Python never merges adjacent C values by X overlap).-//-// A table-wide mapping keeps the same C in the same column across ALL rows: a-// per-row re-compression would misalign cell text (a C=4 box landing in-// column 3 in one row and column 4 in another). Every row is sized to the-// table-wide column count, matching Python's uniform-width construct_table-// output (rows missing a C render empty cells).-func compressColIndices(boxes []pdf.TextBox, rowMap map[int]int, compressed int) (map[int]map[int]int, map[int]int) {+// compressColIndices maps the table's distinct C labels to contiguous column+// indices. The mapping is table-wide so the same C always lands in the same+// column across rows; rows without a C render as empty cells.+func compressColIndices(boxes []pdf.TextBox, rowOf []int, compressed int) (map[int]map[int]int, int) {distinctC := make(map[int]bool)for _, b := range boxes {if b.C >= 0 {@@ -319,11 +321,10 @@ func compressColIndices(boxes []pdf.TextBox, rowMap map[int]int, compressed int)maxCol := len(cs) - 1cCompressed := make(map[int]map[int]int) // row → (original C → compressed col)- cMaxCol := make(map[int]int) // every row spans the table-wide column countfor ri := 0; ri <= compressed; ri++ {cMap := make(map[int]int)- for _, b := range boxes {- if rowMap[b.R] != ri {+ for i, b := range boxes {+ if rowOf[i] != ri {continue}if strings.TrimSpace(b.Text) == "" && b.H <= 0 && b.SP <= 0 {@@ -334,20 +335,18 @@ func compressColIndices(boxes []pdf.TextBox, rowMap map[int]int, compressed int)}}cCompressed[ri] = cMap- cMaxCol[ri] = maxCol}- return cCompressed, cMaxCol+ return cCompressed, maxCol}// buildGrid builds the final cell grid from the collected and compressed data.-func buildGrid(cmap map[int]map[int]*rb, cCompressed map[int]map[int]int, cMaxCol map[int]int, compressed int) [][]pdf.TSRCell {+func buildGrid(cmap map[int]map[int]*rb, cCompressed map[int]map[int]int, maxCol, compressed int) [][]pdf.TSRCell {rows := make([][]pdf.TSRCell, compressed+1)for ri := 0; ri <= compressed; ri++ {- maxC := cMaxCol[ri]- rows[ri] = make([]pdf.TSRCell, maxC+1)+ rows[ri] = make([]pdf.TSRCell, maxCol+1)for ci, v := range cmap[ri] {cci := cCompressed[ri][ci]- if cci <= maxC {+ if cci <= maxCol {if rows[ri][cci].Text == "" {rows[ri][cci].Text = v.txtrows[ri][cci].X0 = v.x0@@ -381,17 +380,3 @@ func buildGrid(cmap map[int]map[int]*rb, cCompressed map[int]map[int]int, cMaxCo}return rows}--func BoxesHaveAnnotations(boxes []pdf.TextBox) bool {- maxR, maxC := 0, 0- for _, b := range boxes {- if b.R > maxR {- maxR = b.R- }- if b.C > maxC {- maxC = b.C- }- }- // True if at least 2 rows or 2 cols (R/C are 0-based, so maxR>0 means ≥2 rows).- return maxR > 0 || maxC > 0-}diff --git a/internal/deepdoc/parser/pdf/table/group_boxes_test.go b/internal/deepdoc/parser/pdf/table/group_boxes_test.goindex cc69dc1e9b..190c6e4b2e 100644--- a/internal/deepdoc/parser/pdf/table/group_boxes_test.go+++ b/internal/deepdoc/parser/pdf/table/group_boxes_test.go@@ -1,7 +1,7 @@package tableimport (- "sort"+ "reflect""strings""testing"@@ -68,6 +68,58 @@ func TestGroupBoxesByRC_RDiffSplitsRow(t *testing.T) {}}+func TestGroupBoxesByRC_MissingRowStaysInYOrder(t *testing.T) {+ // Mirrors the 01_两行合并表头 PDF: the TSR model missed the 成都公司 row,+ // so its boxes have no R (-1), while the two header rows carry the raw+ // TSR row indices 0 and 1. Python's construct_table keeps missing-R boxes+ // at their Y position, so 成都 must land between 深圳 and 武汉 — not inside+ // the header row.+ boxes := []pdf.TextBox{+ {X0: 10, X1: 90, Top: 0, Bottom: 20, Text: "单位名称", R: 0, C: 0},+ {X0: 110, X1: 190, Top: 0, Bottom: 20, Text: "集团公司采用", R: 0, C: 1},+ {X0: 110, X1: 190, Top: 25, Bottom: 45, Text: "政务", R: 1, C: 1},+ {X0: 210, X1: 290, Top: 25, Bottom: 45, Text: "政工", R: 1, C: 2},+ {X0: 10, X1: 90, Top: 50, Bottom: 70, Text: "深圳公司", R: 5, C: 0},+ {X0: 10, X1: 90, Top: 100, Bottom: 120, Text: "成都公司", R: -1, C: 0},+ {X0: 110, X1: 190, Top: 100, Bottom: 120, Text: "20(1)", R: -1, C: 1},+ {X0: 10, X1: 90, Top: 150, Bottom: 170, Text: "武汉公司", R: 6, C: 0},+ }++ rows := GroupBoxesByRC(boxes)++ if len(rows) != 5 {+ t.Fatalf("expected 5 rows (header0, header1, 深圳, 成都, 武汉), got %d: %v", len(rows), rowTexts(rows))+ }+ want := []string{"单位名称", "政务", "深圳公司", "成都公司", "武汉公司"}+ for i, w := range want {+ got := ""+ for _, c := range rows[i] {+ if c.Text != "" {+ got = c.Text+ break+ }+ }+ if got != w {+ t.Errorf("row %d first text = %q, want %q (rows=%v)", i, got, w, rowTexts(rows))+ }+ }+ if rows[3][0].Text != "成都公司" || rows[3][1].Text != "20(1)" {+ t.Errorf("成都 row cells = %v, want [成都公司 20(1)]", rowTexts(rows[3:4]))+ }+}++func rowTexts(rows [][]pdf.TSRCell) [][]string {+ out := make([][]string, len(rows))+ for i, row := range rows {+ for _, c := range row {+ if c.Text != "" {+ out[i] = append(out[i], c.Text)+ }+ }+ }+ return out+}+func TestFillCellTextFromBoxes_RCOnly(t *testing.T) {cells := []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 50, Label: "table"},@@ -97,6 +149,29 @@ func TestFillCellTextFromBoxes_RCOnly(t *testing.T) {}}+func TestGroupBoxesByRC_ZeroValueBoxesFallBackToYX(t *testing.T) {+ // Zero-value TextBox has R=0, C=0. That must not be mistaken for an+ // annotated first row: these four boxes form a 2x2 grid by geometry.+ boxes := []pdf.TextBox{+ {X0: 0, X1: 50, Top: 0, Bottom: 10, Text: "A"},+ {X0: 60, X1: 110, Top: 0, Bottom: 10, Text: "B"},+ {X0: 0, X1: 50, Top: 20, Bottom: 30, Text: "C"},+ {X0: 60, X1: 110, Top: 20, Bottom: 30, Text: "D"},+ }++ rows := GroupBoxesByRC(boxes)++ if len(rows) != 2 {+ t.Fatalf("expected 2 rows from unannotated boxes, got %d: %v", len(rows), rowTexts(rows))+ }+ if len(rows[0]) < 2 || rows[0][0].Text != "A" || rows[0][1].Text != "B" {+ t.Errorf("row 0 = %v, want [A B]", rowTexts(rows[0:1]))+ }+ if len(rows[1]) < 2 || rows[1][0].Text != "C" || rows[1][1].Text != "D" {+ t.Errorf("row 1 = %v, want [C D]", rowTexts(rows[1:2]))+ }+}+func TestGroupBoxesByRC_FallbackToYXWhenNoAnnotations(t *testing.T) {// When all boxes have R=-1 (Python's case: regex didn't match "table" label),// groupBoxesByRC should fall back to YX coordinate grouping.@@ -107,7 +182,7 @@ func TestGroupBoxesByRC_FallbackToYXWhenNoAnnotations(t *testing.T) {{X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "D", R: -1, C: -1},}rows := GroupBoxesByRC(boxes)- // R=-1 for all → maxR = -1 → grid would be 0 rows. Must fall back to YX.+ // R=-1 for all → no box carries an R annotation → must fall back to YX.if len(rows) == 0 {t.Fatal("groupBoxesByRC returned 0 rows when R=-1 — no YX fallback")}@@ -134,61 +209,75 @@ func TestGroupBoxesByRC_ColspanMissing(t *testing.T) {_ = rows}-func TestCompressRowIndices(t *testing.T) {+func TestAssignRows_RChanges(t *testing.T) {// 6 boxes with 6 different R values → 6 rows.boxes := []pdf.TextBox{- {X0: 10, X1: 90, Top: 0, Bottom: 30, Text: "A", R: 0, C: 0},- {X0: 110, X1: 190, Top: 0, Bottom: 30, Text: "B", R: 1, C: 1},- {X0: 210, X1: 290, Top: 0, Bottom: 30, Text: "C", R: 2, C: 2},- {X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "D", R: 3, C: 0},- {X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "E", R: 4, C: 1},- {X0: 210, X1: 290, Top: 35, Bottom: 65, Text: "F", R: 5, C: 2},- }- // Sort first (as GroupBoxesByRC does)- sort.Slice(boxes, func(i, j int) bool {- if boxes[i].R != boxes[j].R {- return boxes[i].R < boxes[j].R- }- if boxes[i].Top != boxes[j].Top {- return boxes[i].Top < boxes[j].Top+ {X0: 10, X1: 90, Top: 0, Bottom: 30, Text: "A", R: 0, C: 0, RTop: 0, RBott: 30},+ {X0: 110, X1: 190, Top: 0, Bottom: 30, Text: "B", R: 1, C: 1, RTop: 0, RBott: 30},+ {X0: 210, X1: 290, Top: 0, Bottom: 30, Text: "C", R: 2, C: 2, RTop: 0, RBott: 30},+ {X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "D", R: 3, C: 0, RTop: 35, RBott: 65},+ {X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "E", R: 4, C: 1, RTop: 35, RBott: 65},+ {X0: 210, X1: 290, Top: 35, Bottom: 65, Text: "F", R: 5, C: 2, RTop: 35, RBott: 65},+ }+ sortRFirstly(boxes)+ rowOf := assignRows(boxes)+ want := []int{0, 1, 2, 3, 4, 5}+ if len(rowOf) != len(want) {+ t.Fatalf("rowOf len = %d, want %d", len(rowOf), len(want))+ }+ for i := range want {+ if rowOf[i] != want[i] {+ t.Errorf("rowOf = %v, want %v", rowOf, want)+ break}- return boxes[i].X0 < boxes[j].X0- })-- rowMap, compressed := compressRowIndices(boxes)- if compressed != 5 { // 0-based, 6 elements → max index 5- t.Errorf("compressed = %d, want 5", compressed)}- if rowMap[0] != 0 || rowMap[1] != 1 || rowMap[2] != 2 || rowMap[3] != 3 || rowMap[4] != 4 || rowMap[5] != 5 {- t.Errorf("rowMap mapping incorrect: %v", rowMap)+}++func TestAssignRows_SameR(t *testing.T) {+ // Multiple boxes with same R and overlapping Y → same row.+ boxes := []pdf.TextBox{+ {X0: 10, X1: 90, Top: 0, Bottom: 30, Text: "A", R: 0, C: 0, RTop: 0, RBott: 30},+ {X0: 110, X1: 190, Top: 0, Bottom: 30, Text: "B", R: 0, C: 1, RTop: 0, RBott: 30},+ {X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "C", R: 1, C: 0, RTop: 35, RBott: 65},+ {X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "D", R: 1, C: 1, RTop: 35, RBott: 65},+ }+ sortRFirstly(boxes)+ rowOf := assignRows(boxes)+ want := []int{0, 0, 1, 1}+ for i := range want {+ if rowOf[i] != want[i] {+ t.Errorf("rowOf = %v, want %v", rowOf, want)+ break+ }}}-func TestCompressRowIndices_SameR(t *testing.T) {- // Multiple boxes with same R → same compressed row.+func TestAssignRows_PageChangeStartsNewRow(t *testing.T) {+ // Per-page R numbering can repeat; a page boundary must still start a new+ // row even when R and Y are unchanged.boxes := []pdf.TextBox{- {X0: 10, X1: 90, Top: 0, Bottom: 30, Text: "A", R: 0, C: 0},- {X0: 110, X1: 190, Top: 0, Bottom: 30, Text: "B", R: 0, C: 1},- {X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "C", R: 1, C: 0},- {X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "D", R: 1, C: 1},+ {X0: 0, X1: 40, Top: 0, Bottom: 10, Text: "A", R: 0, C: 0, RTop: 0, RBott: 10},+ {X0: 0, X1: 40, Top: 0, Bottom: 10, Text: "B", R: 0, C: 0, PageNumber: 1, RTop: 0, RBott: 10},}- // Sort first- sort.Slice(boxes, func(i, j int) bool {- if boxes[i].R != boxes[j].R {- return boxes[i].R < boxes[j].R- }- if boxes[i].Top != boxes[j].Top {- return boxes[i].Top < boxes[j].Top- }- return boxes[i].X0 < boxes[j].X0- })+ sortRFirstly(boxes)+ rowOf := assignRows(boxes)+ if !reflect.DeepEqual(rowOf, []int{0, 1}) {+ t.Errorf("rowOf = %v, want [0 1]", rowOf)+ }+}- rowMap, compressed := compressRowIndices(boxes)- if compressed != 1 { // 0-based, 2 rows → max index 1- t.Errorf("compressed = %d, want 1", compressed)+func TestAssignRows_SameRMultilineStaysOneRow(t *testing.T) {+ // Two text boxes in the same cell on separate lines share R; Python keeps+ // them in one row because the Y-advance branch only applies when R is+ // missing. Splitting here would break multi-line OCR cells.+ boxes := []pdf.TextBox{+ {X0: 0, X1: 40, Top: 5, Bottom: 10, Text: "R/W", R: 0, C: 0, RTop: 0, RBott: 60},+ {X0: 0, X1: 40, Top: 12, Bottom: 17, Text: "R/W", R: 0, C: 0, RTop: 0, RBott: 60},}- if rowMap[0] != 0 || rowMap[1] != 1 {- t.Errorf("rowMap mapping incorrect: %v", rowMap)+ sortRFirstly(boxes)+ rowOf := assignRows(boxes)+ if !reflect.DeepEqual(rowOf, []int{0, 0}) {+ t.Errorf("rowOf = %v, want [0 0] (same-R lines stay in one row)", rowOf)}}@@ -199,19 +288,9 @@ func TestCollectBoxesPerRow(t *testing.T) {{X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "C", R: 1, C: 0},{X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "D", R: 1, C: 1},}- // Sort first- sort.Slice(boxes, func(i, j int) bool {- if boxes[i].R != boxes[j].R {- return boxes[i].R < boxes[j].R- }- if boxes[i].Top != boxes[j].Top {- return boxes[i].Top < boxes[j].Top- }- return boxes[i].X0 < boxes[j].X0- })-- rowMap, _ := compressRowIndices(boxes)- cmap, maxCols := collectBoxesPerRow(boxes, rowMap)+ sortRFirstly(boxes)+ rowOf := assignRows(boxes)+ cmap := collectBoxesPerRow(boxes, rowOf)if len(cmap) != 2 {t.Errorf("cmap has %d rows, want 2", len(cmap))@@ -222,9 +301,6 @@ func TestCollectBoxesPerRow(t *testing.T) {if cmap[1][0].txt != "C" || cmap[1][1].txt != "D" {t.Errorf("row 1 incorrect: %v, %v", cmap[1][0], cmap[1][1])}- if maxCols[0] != 1 || maxCols[1] != 1 {- t.Errorf("maxCols incorrect: %v", maxCols)- }}func TestCollectBoxesPerRow_MergeSameCell(t *testing.T) {@@ -232,19 +308,9 @@ func TestCollectBoxesPerRow_MergeSameCell(t *testing.T) {{X0: 10, X1: 50, Top: 0, Bottom: 30, Text: "Hello", R: 0, C: 0},{X0: 50, X1: 90, Top: 0, Bottom: 30, Text: "World", R: 0, C: 0},}- // Sort first- sort.Slice(boxes, func(i, j int) bool {- if boxes[i].R != boxes[j].R {- return boxes[i].R < boxes[j].R- }- if boxes[i].Top != boxes[j].Top {- return boxes[i].Top < boxes[j].Top- }- return boxes[i].X0 < boxes[j].X0- })-- rowMap, _ := compressRowIndices(boxes)- cmap, _ := collectBoxesPerRow(boxes, rowMap)+ sortRFirstly(boxes)+ rowOf := assignRows(boxes)+ cmap := collectBoxesPerRow(boxes, rowOf)if cmap[0][0].txt != "Hello World" {t.Errorf("merged text incorrect: %q", cmap[0][0].txt)@@ -258,19 +324,10 @@ func TestCompressColIndices(t *testing.T) {{X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "C", R: 1, C: 0},{X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "D", R: 1, C: 1},}- // Sort first- sort.Slice(boxes, func(i, j int) bool {- if boxes[i].R != boxes[j].R {- return boxes[i].R < boxes[j].R- }- if boxes[i].Top != boxes[j].Top {- return boxes[i].Top < boxes[j].Top- }- return boxes[i].X0 < boxes[j].X0- })-- rowMap, compressed := compressRowIndices(boxes)- cCompressed, cMaxCol := compressColIndices(boxes, rowMap, compressed)+ sortRFirstly(boxes)+ rowOf := assignRows(boxes)+ compressed := rowOf[len(rowOf)-1]+ cCompressed, maxCol := compressColIndices(boxes, rowOf, compressed)if cCompressed[0][0] != 0 || cCompressed[0][1] != 1 {t.Errorf("row 0 compression incorrect: %v", cCompressed[0])@@ -278,8 +335,8 @@ func TestCompressColIndices(t *testing.T) {if cCompressed[1][0] != 0 || cCompressed[1][1] != 1 {t.Errorf("row 1 compression incorrect: %v", cCompressed[1])}- if cMaxCol[0] != 1 || cMaxCol[1] != 1 {- t.Errorf("cMaxCol incorrect: %v", cMaxCol)+ if maxCol != 1 {+ t.Errorf("maxCol = %d, want 1", maxCol)}}@@ -292,27 +349,18 @@ func TestCompressColIndices_AdjacentCAlwaysNewColumn(t *testing.T) {{X0: 10, X1: 100, Top: 0, Bottom: 30, Text: "A", R: 0, C: 0},{X0: 90, X1: 190, Top: 0, Bottom: 30, Text: "B", R: 0, C: 1}, // X-overlaps A but C is adjacent}- // Sort first- sort.Slice(boxes, func(i, j int) bool {- if boxes[i].R != boxes[j].R {- return boxes[i].R < boxes[j].R- }- if boxes[i].Top != boxes[j].Top {- return boxes[i].Top < boxes[j].Top- }- return boxes[i].X0 < boxes[j].X0- })-- rowMap, compressed := compressRowIndices(boxes)- cCompressed, cMaxCol := compressColIndices(boxes, rowMap, compressed)+ sortRFirstly(boxes)+ rowOf := assignRows(boxes)+ compressed := rowOf[len(rowOf)-1]+ cCompressed, maxCol := compressColIndices(boxes, rowOf, compressed)// C=0 → column 0, C=1 → column 1 (distinct C rank, never merged).if cCompressed[0][0] != 0 || cCompressed[0][1] != 1 {t.Errorf("adjacent C must split into separate columns: %v", cCompressed[0])}- // Table-wide column count = 2 → cMaxCol == 1.- if cMaxCol[0] != 1 {- t.Errorf("cMaxCol incorrect: got %d, want 1 (2 distinct C → 2 columns)", cMaxCol[0])+ // Table-wide column count = 2 → maxCol == 1.+ if maxCol != 1 {+ t.Errorf("maxCol = %d, want 1 (2 distinct C → 2 columns)", maxCol)}}@@ -323,22 +371,13 @@ func TestBuildGrid(t *testing.T) {{X0: 10, X1: 90, Top: 35, Bottom: 65, Text: "C", R: 1, C: 0},{X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "D", R: 1, C: 1},}- // Sort first- sort.Slice(boxes, func(i, j int) bool {- if boxes[i].R != boxes[j].R {- return boxes[i].R < boxes[j].R- }- if boxes[i].Top != boxes[j].Top {- return boxes[i].Top < boxes[j].Top- }- return boxes[i].X0 < boxes[j].X0- })-- rowMap, compressed := compressRowIndices(boxes)- cmap, _ := collectBoxesPerRow(boxes, rowMap)- cCompressed, cMaxCol := compressColIndices(boxes, rowMap, compressed)+ sortRFirstly(boxes)+ rowOf := assignRows(boxes)+ compressed := rowOf[len(rowOf)-1]+ cmap := collectBoxesPerRow(boxes, rowOf)+ cCompressed, maxCol := compressColIndices(boxes, rowOf, compressed)- rows := buildGrid(cmap, cCompressed, cMaxCol, compressed)+ rows := buildGrid(cmap, cCompressed, maxCol, compressed)if len(rows) != 2 {t.Fatalf("expected 2 rows, got %d", len(rows))diff --git a/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.go b/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.goindex 88e879a426..10afbf8a6a 100644--- a/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.go+++ b/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.go@@ -34,9 +34,8 @@ func TestGroupBoxesByRCSpanHeaderRowCellCount(t *testing.T) {// (H==0, SP>0) cellPosFromBox must use those bounds, otherwise the span// cell falls back to [0,450] and covers only 5 columns.//- // R starts at 1 (not 0): GroupBoxesByRC falls back to YX grouping when- // maxR<=0, which would skip cellPosFromBox entirely; a real table has- // multiple rows so maxR>0 and the R/C path runs.+ // R starts at 1 (not 0) so the box carries a positive annotation;+ // all-zero R/C boxes are treated as annotation-free and use YX grouping.boxes = append(boxes, pdf.TextBox{X0: 0, X1: 450, Top: 0, Bottom: 20,Text: "HEADER", R: 1, C: 0,diff --git a/internal/deepdoc/parser/pdf/table/table_annotate.go b/internal/deepdoc/parser/pdf/table/table_annotate.goindex eb2bdfed2e..cadfd73ab0 100644--- a/internal/deepdoc/parser/pdf/table/table_annotate.go+++ b/internal/deepdoc/parser/pdf/table/table_annotate.go@@ -505,9 +505,10 @@ func WriteTableAnnotations(boxes []pdf.TextBox, boxIdx []int, cells []pdf.TSRCel}}annotGrid := tb.GroupCells(tableCells)- AnnotateTableBoxes(tblBoxes, annotGrid)+ AnnotateBoxesWithGridAndRows(tblBoxes, annotGrid, AnnotationRowBands(tableCells))for k, idx := range boxIdx {bp := &tblBoxes[k]+ boxes[idx].HasTableAnnotations = bp.HasTableAnnotationsboxes[idx].R = bp.Rboxes[idx].RTop = bp.RTop / scaleboxes[idx].RBott = bp.RBott / scalediff --git a/internal/deepdoc/parser/pdf/table/table_annotate_test.go b/internal/deepdoc/parser/pdf/table/table_annotate_test.goindex 14d9b28815..a702f5ae84 100644--- a/internal/deepdoc/parser/pdf/table/table_annotate_test.go+++ b/internal/deepdoc/parser/pdf/table/table_annotate_test.go@@ -658,10 +658,31 @@ func TestWriteTableAnnotations_EmptyCells(t *testing.T) {tb := &MockTableBuilder{GroupCellsFn: func(cells []pdf.TSRCell) [][]pdf.TSRCell {return nil}}- // Should not panic with empty cells.+ // Should not panic with empty cells; R stays missing and C uses the+ // X-order fallback.WriteTableAnnotations(boxes, BoxIdx, cells, 3.0, 0, 0, tb)- if boxes[0].R != 0 || boxes[0].C != 0 {- t.Errorf("empty cells: R=%d C=%d, want 0,0", boxes[0].R, boxes[0].C)+ if boxes[0].R != -1 || boxes[0].C != 0 {+ t.Errorf("empty cells: R=%d C=%d, want -1,0", boxes[0].R, boxes[0].C)+ }+}++func TestWriteTableAnnotations_EmptyGridKeepsCFallback(t *testing.T) {+ boxes := []pdf.TextBox{+ {X0: 100, X1: 200, Top: 10, Bottom: 30, Text: "B", LayoutType: "table"},+ {X0: 0, X1: 100, Top: 10, Bottom: 30, Text: "A", LayoutType: "table"},+ }+ BoxIdx := []int{0, 1}++ tb := &MockTableBuilder{GroupCellsFn: func(cells []pdf.TSRCell) [][]pdf.TSRCell {+ return nil+ }}+ WriteTableAnnotations(boxes, BoxIdx, nil, 3.0, 0, 0, tb)++ if boxes[1].C != 0 || boxes[0].C != 1 {+ t.Errorf("C fallback = A:%d B:%d, want A:0 B:1", boxes[1].C, boxes[0].C)+ }+ if boxes[0].R != -1 || boxes[1].R != -1 {+ t.Errorf("empty grid R = %d/%d, want -1/-1", boxes[0].R, boxes[1].R)}}diff --git a/internal/deepdoc/parser/pdf/table/table_construct.go b/internal/deepdoc/parser/pdf/table/table_construct.goindex e33272da45..c058219a07 100644--- a/internal/deepdoc/parser/pdf/table/table_construct.go+++ b/internal/deepdoc/parser/pdf/table/table_construct.go@@ -76,25 +76,10 @@ func constructTableCore(cells []pdf.TSRCell, boxes []pdf.TextBox, caption stringrows = GroupTSRCellsToRows(cells)}if len(rows) > 0 && HasText(rows) {- // Clean up orphan columns then orphan rows (Python order: columns at- // construct_table:224-277, then rows at :279-333). Both passes mutate- // the same grid and may drop rows/columns, so item.Grid and item.Rows- // must be re-derived AFTER them (the old code set item.Rows before- // column cleanup, leaving it stale; item.Grid also went stale because- // CleanupOrphanRows returns a re-sliced header).+ // Cleanup may drop rows/columns, so recompute item.Grid and item.Rows+ // after both passes.rows = CleanupOrphanColumns(rows)- // Drop all-empty rows. Python's construct_table groups boxes by their- // R annotation (tbl[i] is the boxes for row i; a TSR row with no- // overlapping box contributes no row). Its HTML emitter also skips- // rows whose rendered HTML stays at "<tr>" (no cell wrote anything).- // Go's grid is a TSR-row cross-product, so an extra "table row" that- // the model emits alongside a "table projected row header" (or any- // other TSR component with no OCR/text overlap) leaks through as a- // 5-cell row of empty strings, inflating item.Rows and the HTML. See- // 13_crosspage_table.pdf page 2: y0=885 "table row" sits on top of- // a "table projected row header" with no box overlap and survives- // every TSR cleanup pass. Drop it here so rows/Rows/HTML all match- // Python.+ // TSR can emit an empty row component; drop it before orphan cleanup.rows = DropAllEmptyRows(rows)rows = CleanupOrphanRows(rows)hdrs := HeaderSetWithBlockType(rows, boxes)@@ -108,8 +93,9 @@ func constructTableCore(cells []pdf.TSRCell, boxes []pdf.TextBox, caption string}return tableutil.RenderTableHTML(ToTableData(rows, caption, hdrs, spanInfo, covered))}- // Fallback: boxes with R/C annotations.- if len(boxes) > 0 && BoxesHaveAnnotations(boxes) {+ // Fallback: rebuild from boxes. GroupBoxesByRC uses R/C when present and+ // falls back to Y/X grouping otherwise.+ if len(boxes) > 0 {rows := GroupBoxesByRC(boxes)if HasText(rows) {if item != nil {@@ -119,18 +105,6 @@ func constructTableCore(cells []pdf.TSRCell, boxes []pdf.TextBox, caption stringreturn tableutil.RenderTableHTML(ToTableData(rows, caption, BoxHeaderSet(rows, boxes), spanInfo, covered))}}- // Test-only: Y/X coordinate grouping (matching Python construct_table).- // Used by table_parity_test.go to verify pipeline with Python boxes.- if len(boxes) > 0 && !BoxesHaveAnnotations(boxes) {- rows := GroupBoxesByYX(boxes)- if HasText(rows) {- if item != nil {- item.Rows = RowsToStrings(rows)- }- spanInfo, covered := CalSpans(rows)- return tableutil.RenderTableHTML(ToTableData(rows, caption, BoxHeaderSet(rows, boxes), spanInfo, covered))- }- }return ""}diff --git a/internal/deepdoc/parser/pdf/table/table_construct_test.go b/internal/deepdoc/parser/pdf/table/table_construct_test.goindex 0d84dac366..c068be7f94 100644--- a/internal/deepdoc/parser/pdf/table/table_construct_test.go+++ b/internal/deepdoc/parser/pdf/table/table_construct_test.go@@ -31,6 +31,31 @@ func constructTableData(t *testing.T, cells []pdf.TSRCell, boxes []pdf.TextBox,return td}+func TestConstructTable_ZeroValueBoxesFallback(t *testing.T) {+ // R=0/C=0 is the Go zero value, not proof of annotation. ConstructTable+ // must keep the Y/X fallback for raw boxes.+ boxes := []pdf.TextBox{+ {X0: 0, X1: 50, Top: 0, Bottom: 10, Text: "A"},+ {X0: 60, X1: 110, Top: 0, Bottom: 10, Text: "B"},+ {X0: 0, X1: 50, Top: 20, Bottom: 30, Text: "C"},+ {X0: 60, X1: 110, Top: 20, Bottom: 30, Text: "D"},+ }++ td := constructTableData(t, nil, boxes, "", nil)+ if td == nil {+ t.Fatal("expected non-nil TableData")+ }+ if len(td.Grid) != 2 {+ t.Fatalf("expected 2 rows, got %d: %v", len(td.Grid), td.Grid)+ }+ if cellTextAt(td, 0, 0) != "A" || cellTextAt(td, 0, 1) != "B" {+ t.Errorf("row 0 = %q/%q, want A/B", cellTextAt(td, 0, 0), cellTextAt(td, 0, 1))+ }+ if cellTextAt(td, 1, 0) != "C" || cellTextAt(td, 1, 1) != "D" {+ t.Errorf("row 1 = %q/%q, want C/D", cellTextAt(td, 1, 0), cellTextAt(td, 1, 1))+ }+}+func TestCellTexts(t *testing.T) {cells := []pdf.TSRCell{{Text: "A"}, {Text: "B"}, {Text: "C"},diff --git a/internal/deepdoc/parser/pdf/table/table_coords.go b/internal/deepdoc/parser/pdf/table/table_coords.goindex 423b048ff6..cc7b55f0d0 100644--- a/internal/deepdoc/parser/pdf/table/table_coords.go+++ b/internal/deepdoc/parser/pdf/table/table_coords.go@@ -43,6 +43,7 @@ func BoxToCropSpace(b pdf.TextBox, scale, cropOffX, cropOffY float64) pdf.TextBo// CopyBoxAnnotations copies the DLA/TSR annotation fields from src to dst.func CopyBoxAnnotations(dst, src *pdf.TextBox) {+ dst.HasTableAnnotations = src.HasTableAnnotationsdst.R = src.Rdst.C = src.Cdst.RTop = src.RTopdiff --git a/internal/deepdoc/parser/pdf/table/table_coords_test.go b/internal/deepdoc/parser/pdf/table/table_coords_test.goindex 4132931d55..3db2286d14 100644--- a/internal/deepdoc/parser/pdf/table/table_coords_test.go+++ b/internal/deepdoc/parser/pdf/table/table_coords_test.go@@ -42,10 +42,13 @@ func TestBoxToCropSpace(t *testing.T) {}func TestCopyBoxAnnotations(t *testing.T) {- src := &pdf.TextBox{R: 1, C: 2, RTop: 10, RBott: 20, H: 3, HTop: 30, HBott: 40,+ src := &pdf.TextBox{HasTableAnnotations: true, R: 1, C: 2, RTop: 10, RBott: 20, H: 3, HTop: 30, HBott: 40,HLeft: 50, HRight: 60, CLeft: 70, CRight: 80, SP: 4}dst := &pdf.TextBox{}CopyBoxAnnotations(dst, src)+ if !dst.HasTableAnnotations {+ t.Error("annotation flag not copied")+ }if dst.R != 1 || dst.C != 2 || dst.RTop != 10 || dst.RBott != 20 {t.Error("R/C fields not copied")}diff --git a/internal/deepdoc/parser/pdf/table/table_layout.go b/internal/deepdoc/parser/pdf/table/table_layout.goindex a8d37e8798..cd1eee489f 100644--- a/internal/deepdoc/parser/pdf/table/table_layout.go+++ b/internal/deepdoc/parser/pdf/table/table_layout.go@@ -174,79 +174,109 @@ func findHorizontallyTightestFit(box pdf.TextBox, clmns []pdf.TSRCell) int {return best}-// AnnotateBoxesWithGrid derives per-box R/C/H/SP annotations in the SAME-// coordinate frame as grid (e.g. a table's crop space), using Python's-// _table_transformer_job semantics. It is the production entry point for-// deriving R/C so the grid can be rebuilt from them (GroupBoxesByRC).-func AnnotateBoxesWithGrid(boxes []pdf.TextBox, grid [][]pdf.TSRCell) {- AnnotateTableBoxes(boxes, grid)+// spanTopRowIndex returns the first physical table row whose centre lies+// inside the span's Y range — the top row of the merged rectangle (Python+// __cal_spans minR). Header components are skipped: they are not rows, and+// using their index would put the span in a different R space from the row+// cells. Returns -1 when no table row centre falls inside the span.+func spanTopRowIndex(rows []pdf.TSRCell, sp pdf.TSRCell) int {+ for i, r := range rows {+ if isHeaderLabel(r.Label) {+ continue+ }+ cy := (r.Y0 + r.Y1) / 2+ if cy >= sp.Y0 && cy <= sp.Y1 {+ return i+ }+ }+ return -1}-// annotateTableBoxes tags table boxes with row/header/column indices using-// TSR cell labels. Matching Python's R/H/C/SP annotation logic.-//-// Python: pdf_parser.py:518-554+// AnnotateBoxesWithGridAndRows derives R/C/H/SP for boxes in the same+// coordinate frame as grid. R is matched against the raw TSR row bands, not+// span-extended grid rows.+func AnnotateBoxesWithGridAndRows(boxes []pdf.TextBox, grid [][]pdf.TSRCell, rows []pdf.TSRCell) {+ annotateTableBoxes(boxes, grid, rows)+}++// AnnotateTableBoxes is the fallback when raw TSR rows are unavailable. It+// matches R against grid rows, which can over-merge a rowspan header;+// production uses AnnotateBoxesWithGridAndRows.func AnnotateTableBoxes(boxes []pdf.TextBox, grid [][]pdf.TSRCell) {- // grid[0] is the header row. Spans are computed by calSpans later.+ annotateTableBoxes(boxes, grid, nil)+}++func annotateTableBoxes(boxes []pdf.TextBox, grid [][]pdf.TSRCell, rows []pdf.TSRCell) {var headers, spans []pdf.TSRCellvar clmns []pdf.TSRCell- // Python t_recognizer.py: headers = gather(r".*header$") — the set of layout- // cells whose label ends in "header", NOT the first grid row. Collect them- // from every grid row so a header that sits on a row other than 0 is still- // matched and tagged with H>0 (fixes the grid[0] approximation).+ // Collect header and spanning cells from every grid row. A header is not+ // necessarily in row 0, and box.SP must be set or CalSpans drops the span.for _, row := range grid {for _, cell := range row {if isHeaderLabel(cell.Label) {headers = append(headers, cell)}- // Collect spanning cells so the SP annotation is propagated to- // overlapping boxes (Python _table_transformer_job appends every- // "SP" cell to its `spans` list and matches boxes against it at- // pdf_parser.py:518-554). Without this, box.SP stays 0, the- // rebuilt grid (GroupBoxesByRC) loses the span, and- // ConstructTable/CalSpans drops the colspan/rowspan — Go emits- // independent empty <th> where Python emits <th colspan=6>- // (e.g. real_pdfs/1.pdf).if strings.Contains(cell.Label, "spanning") {spans = append(spans, cell)}}}if len(grid) > 0 && len(grid[0]) > 0 {- // Python's clmns are the "table column" lines: vertical bboxes spanning- // the whole table height. Derive them from the grid (each column's X- // range from the first row, Y range from the table's top/bottom rows).+ // Recover column edges from non-spanning cells: grid[0] alone can be+ // widened by a span origin and would absorb neighbouring columns.+ colLeft, colRight, _, _ := gridBounds(grid)+ nCols := len(colLeft)tableTop := grid[0][0].Y0tableBot := grid[len(grid)-1][0].Y1- clmns = make([]pdf.TSRCell, len(grid[0]))- for ci := range grid[0] {- clmns[ci] = pdf.TSRCell{X0: grid[0][ci].X0, Y0: tableTop, X1: grid[0][ci].X1, Y1: tableBot}+ if len(rows) > 0 {+ tableTop = rows[0].Y0+ tableBot = rows[len(rows)-1].Y1+ }+ clmns = make([]pdf.TSRCell, nCols)+ for ci := 0; ci < nCols; ci++ {+ if colLeft[ci] == 1e9 {+ // No non-spanning cell in this column: keep the grid bounds.+ colLeft[ci], colRight[ci] = grid[0][ci].X0, grid[0][ci].X1+ }+ clmns[ci] = pdf.TSRCell{X0: colLeft[ci], Y0: tableTop, X1: colRight[ci], Y1: tableBot}}}SortYFirstly(headers, 10)SortXFirstly(clmns, 10)for i := range boxes {- // Python processes only boxes whose layout_type is "table"; callers- // (processOneTable / WriteTableAnnotations) already pass the table- // region's box subset, so an empty LayoutType (e.g. OCR-replay boxes- // that carry no DLA annotation) is treated as table content too.+ // Empty LayoutType is table content too (e.g. OCR replay boxes).if boxes[i].LayoutType != pdf.LayoutTypeTable && boxes[i].LayoutType != "" {continue}- // R: Python find_overlapped_with_threshold(box, rows, 0.3) over the- // WHOLE row line — the grid row's bbox spans the table width (the row- // line's own X range), not individual grid cells.- for ri, row := range grid {- if len(row) == 0 {- continue- }- rowBBox := pdf.TSRCell{X0: row[0].X0, Y0: row[0].Y0, X1: row[len(row)-1].X1, Y1: row[0].Y1}- if findOverlappedWithThreshold(boxes[i], []pdf.TSRCell{rowBBox}, 0.3) >= 0 {+ boxes[i].HasTableAnnotations = true+ // R: Python find_overlapped_with_threshold(box, rows, 0.3). Python+ // leaves R unset when no row line matches; using Go's zero value would+ // silently put that box in row 0, so start from -1 and only write a+ // row when the match succeeds.+ boxes[i].R = -1+ boxes[i].RTop = 0+ boxes[i].RBott = 0+ if len(rows) > 0 {+ if ri := findOverlappedWithThreshold(boxes[i], rows, 0.3); ri >= 0 {boxes[i].R = ri- boxes[i].RTop = row[0].Y0- boxes[i].RBott = row[0].Y1- break+ boxes[i].RTop = rows[ri].Y0+ boxes[i].RBott = rows[ri].Y1+ }+ } else {+ // Fallback: match against grid rows. A span-extended row can+ // over-merge; production passes raw rows to avoid it.+ for ri, row := range grid {+ if len(row) == 0 {+ continue+ }+ rowBBox := pdf.TSRCell{X0: row[0].X0, Y0: row[0].Y0, X1: row[len(row)-1].X1, Y1: row[0].Y1}+ if findOverlappedWithThreshold(boxes[i], []pdf.TSRCell{rowBBox}, 0.3) >= 0 {+ boxes[i].R = ri+ boxes[i].RTop = row[0].Y0+ boxes[i].RBott = row[0].Y1+ break+ }}}if idx := findOverlappedWithThreshold(boxes[i], headers, 0.3); idx >= 0 {@@ -254,11 +284,8 @@ func AnnotateTableBoxes(boxes []pdf.TextBox, grid [][]pdf.TSRCell) {boxes[i].HBott = headers[idx].Y1boxes[i].HLeft = headers[idx].X0boxes[i].HRight = headers[idx].X1- // Offset by 1: store idx+1 so a box matching the FIRST header cell- // (idx == 0) is distinguishable from "no header overlap" (the- // default H == 0). All readers check H > 0, so this keeps the- // boolean semantics while fixing single-column / first-column- // header detection (parity #4, asymmetry 1).+ // Store idx+1: H>0 means matched, and an index of 0 is still+ // distinguishable from "no header".boxes[i].H = idx + 1}// C: Python find_horizontally_tightest_fit(box, clmns).@@ -270,27 +297,28 @@ func AnnotateTableBoxes(boxes []pdf.TextBox, grid [][]pdf.TSRCell) {}}if idx := findOverlappedWithThreshold(boxes[i], spans, 0.3); idx >= 0 {- // Offset by 1 so a box matching the FIRST spanning cell- // (idx == 0) is distinguishable from "no span overlap" (the- // default SP == 0). All readers check SP > 0, matching Python's- // boolean SP semantics (pdf_parser.py:518-554).+ // Store idx+1: SP>0 means matched, including span index 0.boxes[i].SP = idx + 1- // Python _annotate_table_boxes (pdf_parser.py:632-635) copies the- // spanning cell's bbox onto the box as H_top/H_bott/H_left/H_right.- // GroupBoxesByRC then builds the span cell from these full extents- // (cellPosFromBox uses HLeft/HRight when H>0), so CalSpans covers- // every column the span crosses. Without this, the span cell falls- // back to the box's own narrow bounds and Go emits colspan=5 where- // Python emits colspan=6 (real_pdfs/1.pdf).+ // Copy the span's full extent so cellPosFromBox can build a cell+ // that covers every row/column the span crosses.boxes[i].HTop = spans[idx].Y0boxes[i].HBott = spans[idx].Y1boxes[i].HLeft = spans[idx].X0boxes[i].HRight = spans[idx].X1+ // A rowspan cell's text is vertically centred and may sit in the+ // lower row; move R to the span's top row (Python __cal_spans+ // places the merged cell at tbl[minR][minC]).+ if len(rows) > 0 {+ if sr := spanTopRowIndex(rows, spans[idx]); sr >= 0 {+ boxes[i].R = sr+ boxes[i].RTop = rows[sr].Y0+ boxes[i].RBott = rows[sr].Y1+ }+ }}}- // Two-pass C fallback: after all R values are assigned, compute C by X-order within each row.- // This matches Python's behavior when TSR provides few "table column" cells.+ // No usable column structure: assign C by X order within each R group.if len(clmns) <= 1 {// Collect all table boxes grouped by R (LayoutType empty → table content).rBoxes := make(map[int][]int)diff --git a/internal/deepdoc/parser/pdf/table/table_layout_test.go b/internal/deepdoc/parser/pdf/table/table_layout_test.goindex d64fad730d..829587121c 100644--- a/internal/deepdoc/parser/pdf/table/table_layout_test.go+++ b/internal/deepdoc/parser/pdf/table/table_layout_test.go@@ -166,6 +166,226 @@ func TestAnnotateTableBoxes(t *testing.T) {}}+// TestAnnotateTableBoxes_NoRowMatchLeavesRMissing pins Python's missing-R+// state: a box that overlaps no row line must stay unassigned (-1), not fall+// back to Go's zero value (row 0). The old behavior silently folded a row the+// TSR model missed into the header row.+func TestAnnotateTableBoxes_NoRowMatchLeavesRMissing(t *testing.T) {+ grid := [][]pdf.TSRCell{+ {{X0: 0, Y0: 0, X1: 100, Y1: 20, Label: "table row"}},+ }+ boxes := []pdf.TextBox{+ {X0: 0, X1: 100, Top: 100, Bottom: 120, Text: "成都公司", LayoutType: pdf.LayoutTypeTable},+ }++ AnnotateTableBoxes(boxes, grid)++ if boxes[0].R != -1 {+ t.Errorf("unmatched box R = %d, want -1 (missing); a zero value makes GroupBoxesByRC fold it into row 0", boxes[0].R)+ }+}++// TestAnnotateTableBoxes_RawRowBandsOverrideSpanExtendedGrid pins that R is+// matched against the raw TSR row lines, not the span-extended grid cells.+// grid[0] covers both physical rows because a rowspan cell was extended over+// them; a box in the second physical row must still get R=1.+func TestAnnotateTableBoxes_RawRowBandsOverrideSpanExtendedGrid(t *testing.T) {+ grid := [][]pdf.TSRCell{+ // The first row's bbox was extended to y=100 by a rowspan cell, but+ // the cell itself is not a span (so it must not pull R up by span+ // origin relocation either).+ {{X0: 0, Y0: 0, X1: 100, Y1: 100, Label: "table row"}},+ {{X0: 0, Y0: 20, X1: 100, Y1: 40, Label: "table row"}},+ }+ rows := []pdf.TSRCell{+ {X0: 0, Y0: 0, X1: 100, Y1: 20, Label: "table row"},+ {X0: 0, Y0: 20, X1: 100, Y1: 40, Label: "table row"},+ }+ boxes := []pdf.TextBox{+ {X0: 0, X1: 100, Top: 20, Bottom: 40, Text: "第二行", LayoutType: pdf.LayoutTypeTable},+ }++ AnnotateBoxesWithGridAndRows(boxes, grid, rows)++ if boxes[0].R != 1 {+ t.Errorf("second physical row R = %d, want 1; the span-extended grid row 0 must not swallow it", boxes[0].R)+ }+ if boxes[0].RTop != 20 || boxes[0].RBott != 40 {+ t.Errorf("R band = (%v, %v), want (20, 40)", boxes[0].RTop, boxes[0].RBott)+ }+}++// TestAnnotateTableBoxes_ColumnBoundsIgnoreSpanExtension pins that C is+// matched against the real column boundaries, recovered from non-spanning+// cells. Using grid[0] directly would let a wide spanning header cover every+// column and assign all boxes under it to the span's origin column.+func TestAnnotateTableBoxes_ColumnBoundsIgnoreSpanExtension(t *testing.T) {+ grid := [][]pdf.TSRCell{+ {+ {X0: 0, Y0: 0, X1: 300, Y1: 40, Label: "table spanning cell"},+ {X0: 100, Y0: 0, X1: 200, Y1: 20, Label: "table column header"},+ {X0: 200, Y0: 0, X1: 300, Y1: 20, Label: "table column header"},+ },+ {+ {X0: 0, Y0: 20, X1: 100, Y1: 40, Label: "table row"},+ {X0: 100, Y0: 20, X1: 200, Y1: 40, Label: "table row"},+ {X0: 200, Y0: 20, X1: 300, Y1: 40, Label: "table row"},+ },+ }+ boxes := []pdf.TextBox{+ {X0: 250, X1: 300, Top: 0, Bottom: 20, Text: "第三列", LayoutType: pdf.LayoutTypeTable},+ }++ AnnotateTableBoxes(boxes, grid)++ if boxes[0].C != 2 {+ t.Errorf("box in the third column got C=%d, want 2; a span-extended grid[0] column must not absorb it", boxes[0].C)+ }+}++// TestAnnotateTableBoxes_MovesRowspanOriginToTopRow pins that a rowspan cell+// whose text is vertically centred in the lower row is moved to the span's+// top row, matching Python __cal_spans (tbl[minR][minC]).+func TestAnnotateTableBoxes_MovesRowspanOriginToTopRow(t *testing.T) {+ grid := [][]pdf.TSRCell{+ {{X0: 0, Y0: 0, X1: 100, Y1: 40, Label: "table spanning cell"}},+ {{X0: 0, Y0: 20, X1: 100, Y1: 40, Label: "table row"}},+ }+ rows := []pdf.TSRCell{+ {X0: 0, Y0: 0, X1: 100, Y1: 20, Label: "table row"},+ {X0: 0, Y0: 20, X1: 100, Y1: 40, Label: "table row"},+ }+ boxes := []pdf.TextBox{+ // Text is centred in the lower row, but the span starts at row 0.+ {X0: 0, X1: 100, Top: 25, Bottom: 35, Text: "单位名称", LayoutType: pdf.LayoutTypeTable},+ }++ AnnotateBoxesWithGridAndRows(boxes, grid, rows)++ if boxes[0].R != 0 {+ t.Errorf("rowspan origin R = %d, want 0 (top row of the span)", boxes[0].R)+ }+ if boxes[0].SP <= 0 {+ t.Errorf("span not detected: SP=%d", boxes[0].SP)+ }+}++func TestAnnotateBoxesWithGridAndRows_SpanUsesRowNotHeaderIndex(t *testing.T) {+ rows := []pdf.TSRCell{+ {X0: 0, Y0: 0, X1: 100, Y1: 100, Label: "table column header"}, // index 0+ {X0: 0, Y0: 0, X1: 100, Y1: 40, Label: "table row"}, // index 1+ {X0: 0, Y0: 40, X1: 100, Y1: 80, Label: "table row"}, // index 2+ }+ grid := [][]pdf.TSRCell{+ {+ {X0: 0, Y0: 0, X1: 50, Y1: 80, Label: "table spanning cell"},+ {X0: 50, Y0: 0, X1: 100, Y1: 40, Label: "table row"},+ },+ {+ {X0: 0, Y0: 40, X1: 50, Y1: 80, Label: "table row"},+ {X0: 50, Y0: 40, X1: 100, Y1: 80, Label: "table row"},+ },+ }+ boxes := []pdf.TextBox{+ // Span text sits in the lower row but the span starts at row 1.+ {X0: 10, X1: 40, Top: 45, Bottom: 55, Text: "span", LayoutType: pdf.LayoutTypeTable},+ // Normal cell in the first header row.+ {X0: 60, X1: 90, Top: 5, Bottom: 15, Text: "normal", LayoutType: pdf.LayoutTypeTable},+ }++ AnnotateBoxesWithGridAndRows(boxes, grid, rows)++ if boxes[0].SP == 0 {+ t.Fatal("span box was not annotated")+ }+ if boxes[0].R != 1 {+ t.Errorf("span R = %d, want 1 (table row index); the header component is index 0", boxes[0].R)+ }+ if boxes[1].R != 1 {+ t.Errorf("normal header cell R = %d, want 1", boxes[1].R)+ }+}++func TestAnnotateBoxesWithGridAndRows_RealPDFSpanRIndex(t *testing.T) {+ // 1.pdf header coordinates from TestGroupCellsInjectsSpanning:+ // header band 110.9..231.0, row0 111.4..138.9, row1 138.3..231.4,+ // rowspan span #18 135.3..229.7.+ cells := []pdf.TSRCell{+ {X0: 81.1, Y0: 110.4, X1: 212.6, Y1: 290.6, Label: "table column"},+ {X0: 211.5, Y0: 110.4, X1: 249.5, Y1: 290.6, Label: "table column"},+ {X0: 249.4, Y0: 110.4, X1: 286.1, Y1: 290.6, Label: "table column"},+ {X0: 287.2, Y0: 110.4, X1: 331.3, Y1: 290.6, Label: "table column"},+ {X0: 80.3, Y0: 111.4, X1: 503.0, Y1: 138.9, Label: "table row"},+ {X0: 80.3, Y0: 138.3, X1: 502.2, Y1: 231.4, Label: "table row"},+ {X0: 80.3, Y0: 230.3, X1: 501.8, Y1: 290.6, Label: "table row"},+ {X0: 79.1, Y0: 110.9, X1: 503.6, Y1: 231.0, Label: "table column header"},+ {X0: 210.3, Y0: 135.3, X1: 249.7, Y1: 229.7, Label: "table spanning cell"},+ }++ grid := (&DeepDocTableBuilder{}).GroupCells(cells)+ if grid == nil {+ t.Fatal("GroupCells returned nil")+ }++ boxes := []pdf.TextBox{+ {X0: 90, X1: 200, Top: 112, Bottom: 138, Text: "h0c0", LayoutType: pdf.LayoutTypeTable},+ {X0: 250, X1: 280, Top: 112, Bottom: 138, Text: "h0c2", LayoutType: pdf.LayoutTypeTable},+ {X0: 290, X1: 325, Top: 112, Bottom: 138, Text: "h0c3", LayoutType: pdf.LayoutTypeTable},+ {X0: 90, X1: 200, Top: 140, Bottom: 229, Text: "h1c0", LayoutType: pdf.LayoutTypeTable},+ {X0: 250, X1: 280, Top: 140, Bottom: 229, Text: "h1c2", LayoutType: pdf.LayoutTypeTable},+ {X0: 290, X1: 325, Top: 140, Bottom: 229, Text: "h1c3", LayoutType: pdf.LayoutTypeTable},+ {X0: 220, X1: 240, Top: 160, Bottom: 180, Text: "名称", LayoutType: pdf.LayoutTypeTable},+ {X0: 90, X1: 200, Top: 240, Bottom: 280, Text: "d0", LayoutType: pdf.LayoutTypeTable},+ {X0: 250, X1: 280, Top: 240, Bottom: 280, Text: "d2", LayoutType: pdf.LayoutTypeTable},+ {X0: 290, X1: 325, Top: 240, Bottom: 280, Text: "d3", LayoutType: pdf.LayoutTypeTable},+ }++ rowsBands := AnnotationRowBands(cells)+ AnnotateBoxesWithGridAndRows(boxes, grid, rowsBands)++ spanIdx, row1Idx := 6, 3+ if boxes[spanIdx].SP == 0 {+ t.Fatal("span box was not annotated with SP")+ }+ if got := spanTopRowIndex(rowsBands, cells[len(cells)-1]); got != boxes[row1Idx].R {+ t.Errorf("span top row = %d, row1 R = %d; they must use the same annotation index", got, boxes[row1Idx].R)+ }+ if boxes[spanIdx].R != boxes[row1Idx].R {+ t.Errorf("span R = %d, row1 cell R = %d; mixed R index spaces", boxes[spanIdx].R, boxes[row1Idx].R)+ }++ rows := GroupBoxesByRC(boxes)+ if len(rows) != 3 {+ t.Fatalf("expected 3 logical rows (2 header + 1 data), got %d: %v", len(rows), rowTexts(rows))+ }+ if len(rows[1]) != 4 || len(rows[2]) != 4 {+ t.Errorf("header/data row widths = %d/%d, want 4/4", len(rows[1]), len(rows[2]))+ }+}++func TestRowBandsSplitGridAndAnnotationSemantics(t *testing.T) {+ cells := []pdf.TSRCell{+ {X0: 0, Y0: 0, X1: 100, Y1: 100, Label: "table column header"},+ {X0: 0, Y0: 0, X1: 100, Y1: 20, Label: "table row"},+ {X0: 0, Y0: 0, X1: 10, Y1: 100, Label: "table column"},+ {X0: 0, Y0: 20, X1: 100, Y1: 40, Label: "table row"},+ {X0: 0, Y0: 40, X1: 100, Y1: 60, Label: "table projected row header"},+ }++ gridRows := GridRowBands(cells)+ if len(gridRows) != 2 {+ t.Fatalf("GridRowBands returned %d rows, want 2 (table row only): %+v", len(gridRows), gridRows)+ }+ if gridRows[0].Y0 != 0 || gridRows[1].Y0 != 20 {+ t.Errorf("grid row bands = (%v, %v), want (0, 20)", gridRows[0].Y0, gridRows[1].Y0)+ }++ annotationRows := AnnotationRowBands(cells)+ if len(annotationRows) != 4 {+ t.Fatalf("AnnotationRowBands returned %d rows, want 4 (row + header): %+v", len(annotationRows), annotationRows)+ }+}+// ── GroupTSRCellsToRows ─────────────────────────────────────────func TestGroupTSRCellsToRowsLabeled(t *testing.T) {diff --git a/internal/deepdoc/parser/pdf/table/table_spans.go b/internal/deepdoc/parser/pdf/table/table_spans.goindex c2237f2e95..8032f709ce 100644--- a/internal/deepdoc/parser/pdf/table/table_spans.go+++ b/internal/deepdoc/parser/pdf/table/table_spans.go@@ -6,46 +6,31 @@ import (pdf "ragflow/internal/deepdoc/parser/pdf/type")-// calSpans computes colspan and rowspan for spanning cells in the grid.-// Returns spanInfo (row,col → colspan,rowspan) and covered (cells hidden by spans).-// Matches Python's __cal_spans (table_structure_recognizer.py:535).-func CalSpans(rows [][]pdf.TSRCell) (map[[2]int][2]int, map[[2]int]bool) {- spanInfo := make(map[[2]int][2]int)- covered := make(map[[2]int]bool)+// gridBounds returns the per-column and per-row geometry from non-spanning+// cells. Spanning cells are excluded because their bbox is extended over the+// covered region; zero-position padding cells are excluded because they would+// drag boundaries to the origin. Both span detection and R/C annotation use+// these bounds, so the two stages agree on the table's real column edges.+func gridBounds(rows [][]pdf.TSRCell) (colLeft, colRight, rowTop, rowBott []float64) {if len(rows) == 0 || len(rows[0]) == 0 {- return spanInfo, covered+ return nil, nil, nil, nil}-- // Compute column center positions.- nCols := len(rows[0])- colLeft := make([]float64, nCols)- colRight := make([]float64, nCols)+ nCols, nRows := len(rows[0]), len(rows)+ colLeft = make([]float64, nCols)+ colRight = make([]float64, nCols)+ rowTop = make([]float64, nRows)+ rowBott = make([]float64, nRows)for j := 0; j < nCols; j++ {- colLeft[j] = 1e9- colRight[j] = -1e9+ colLeft[j], colRight[j] = 1e9, -1e9}- nRows := len(rows)- rowTop := make([]float64, nRows)- rowBott := make([]float64, nRows)for i := 0; i < nRows; i++ {- rowTop[i] = 1e9- rowBott[i] = -1e9+ rowTop[i], rowBott[i] = 1e9, -1e9}-for i, row := range rows {for j, cell := range row {- if j >= nCols {- continue- }- // Exclude spanning cells from column/row boundary calculations.- // Use label-based detection (O(1), no dependency on column midpoints).- if strings.Contains(cell.Label, "spanning") {+ if j >= nCols || strings.Contains(cell.Label, "spanning") {continue}- // Cells without position data (e.g. the zero-coordinate cells- // padded by MergeTablesAcrossPages to align per-page column- // counts) must not define column/row geometry, or they would drag- // boundaries to the origin and corrupt span detection.if cell.X0 == 0 && cell.X1 == 0 && cell.Y0 == 0 && cell.Y1 == 0 {continue}@@ -63,24 +48,29 @@ func CalSpans(rows [][]pdf.TSRCell) (map[[2]int][2]int, map[[2]int]bool) {}}}+ return colLeft, colRight, rowTop, rowBott+}++// calSpans computes colspan and rowspan for spanning cells in the grid.+// Returns spanInfo (row,col → colspan,rowspan) and covered (cells hidden by spans).+// Matches Python's __cal_spans (table_structure_recognizer.py:535).+func CalSpans(rows [][]pdf.TSRCell) (map[[2]int][2]int, map[[2]int]bool) {+ spanInfo := make(map[[2]int][2]int)+ covered := make(map[[2]int]bool)+ if len(rows) == 0 || len(rows[0]) == 0 {+ return spanInfo, covered+ }++ colLeft, colRight, rowTop, rowBott := gridBounds(rows)+ if colLeft == nil {+ return spanInfo, covered+ }+ nCols := len(colLeft)+ nRows := len(rowTop)- // For each spanning cell, compute how many cols/rows it covers.- // Only cells explicitly recognized as a span participate. Python's- // __cal_spans (table_structure_recognizer.py:500-528) iterates boxes that- // carry an "SP" annotation (`if "SP" not in b: continue`) and never- // treats an ordinary "table header" as a span origin. In Go the TSR- // "table spanning cell" component is the SP equivalent and is labelled- // "spanning" by GroupBoxesByRC/GroupCells, so that is the only branch we- // honour here.- //- // A previous Go-only branch also spanned "table header" cells whose- // HLeft/HRight bbox straddled a neighbour column. That diverges from- // Python and is unsafe: when a cross-page merge widens a header cell's- // bbox (e.g. icbccs "类型" after MergeTablesAcrossPages stacks two pages'- // grids), its X1 can reach past the neighbour column's center and get- // misclassified as a colspan, folding the neighbour's text in and dropping- // a column from the rendered row. Removing the header branch makes Go- // match Python and avoids that regression.+ // Only cells explicitly labelled "spanning" participate (Python's SP).+ // A plain header must never become a span origin: a cross-page merge can+ // widen a header bbox over a neighbouring column.for i, row := range rows {for j, cell := range row {if j >= nCols || covered[[2]int{i, j}] {@@ -165,14 +155,9 @@ func CalSpans(rows [][]pdf.TSRCell) (map[[2]int][2]int, map[[2]int]bool) {}}}- // Fold the covered cells' text into the span origin cell,- // matching Python's __cal_spans (table_structure_recognizer.py:- // 530-577): it walks the covered region row-major and extends- // the span cell's text with every covered cell's text (skipping- // already-folded duplicates via join(arr)). Because Go's- // GroupCells no longer zeroes covered bboxes, those cells hold- // their own box text by fill time; without this fold the span- // cell would render empty and the covered text would be lost.+ // Fold covered-cell text into the span origin, matching Python's+ // __cal_spans. Covered cells keep their bbox and text, so this is+ // where their text reaches the rendered span cell.var merged []stringseen := map[string]bool{}for ri := minR; ri <= maxR && ri < nRows; ri++ {diff --git a/internal/deepdoc/parser/pdf/table_extract.go b/internal/deepdoc/parser/pdf/table_extract.goindex 754484bf6f..3e9ec89741 100644--- a/internal/deepdoc/parser/pdf/table_extract.go+++ b/internal/deepdoc/parser/pdf/table_extract.go@@ -185,9 +185,10 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxes// gather) is used ONLY to derive per-box R/C annotations.annotGrid := tb.GroupCells(cells)if len(annotGrid) > 0 {- // Derive R/C/H/SP with Python's _table_transformer_job semantics- // (whole-row/whole-column line matching), in crop space.- tbl.AnnotateBoxesWithGrid(boxInCrop, annotGrid)+ // Derive R/C/H/SP in crop space. R uses the raw TSR row bands,+ // not the span-extended grid rows, so a rowspan header does not+ // pull the next physical row into its origin row.+ tbl.AnnotateBoxesWithGridAndRows(boxInCrop, annotGrid, tbl.AnnotationRowBands(cells))boxInCrop = dedupSameCellOCRBoxes(boxInCrop)// Rebuild the grid from the derived per-char R/C, exactly like// Python's construct_table groups boxes by their R/C labels —diff --git a/internal/deepdoc/parser/pdf/tool/py_intermediates.go b/internal/deepdoc/parser/pdf/tool/py_intermediates.goindex cf58407e85..5230310a44 100644--- a/internal/deepdoc/parser/pdf/tool/py_intermediates.go+++ b/internal/deepdoc/parser/pdf/tool/py_intermediates.go@@ -213,18 +213,19 @@ func LoadPythonTableBoxes(jsonPath string) ([]pdf.TextBox, error) {boxes := make([]pdf.TextBox, 0, len(dumped))for _, b := range dumped {boxes = append(boxes, pdf.TextBox{- X0: b.X0,- X1: b.X1,- Top: b.Top,- Bottom: b.Bottom,- Text: b.Text,- R: b.R,- C: b.C,- H: b.H,- SP: b.SP,- PageNumber: b.PageNumber,- LayoutNo: b.LayoutNo,- LayoutType: b.LayoutType,+ X0: b.X0,+ X1: b.X1,+ Top: b.Top,+ Bottom: b.Bottom,+ Text: b.Text,+ HasTableAnnotations: true,+ R: b.R,+ C: b.C,+ H: b.H,+ SP: b.SP,+ PageNumber: b.PageNumber,+ LayoutNo: b.LayoutNo,+ LayoutType: b.LayoutType,})}return boxes, nildiff --git a/internal/deepdoc/parser/type/types.go b/internal/deepdoc/parser/type/types.goindex 60ba30e21e..000d4f93cb 100644--- a/internal/deepdoc/parser/type/types.go+++ b/internal/deepdoc/parser/type/types.go@@ -120,7 +120,11 @@ type TextBox struct {// FigureBounds retains DLA geometry in PDF points until figure consolidation.FigureBounds *PositionColID int- R int+ // HasTableAnnotations marks that the post-TSR table annotation pass ran+ // for this box. R/C zero values are otherwise indistinguishable from+ // "not annotated".+ HasTableAnnotations bool+ R int// IsOCR marks a box produced by an OCR pass (ocrDetectAndRecognize /// ocrMergeChars), as opposed to one built from embedded PDF chars// (CharsToBoxes). It scopes OCR-only post-processing (see layout.Dedup*)