Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(pdf): colspan assembly loss + TSR geometry comparison diagnostics (#18727)

xugangqiangd7e3592501cd
diff --git a/internal/deepdoc/parser/pdf/dla_tsr_geom_cmp_test.go b/internal/deepdoc/parser/pdf/dla_tsr_geom_cmp_test.go
new file mode 100644
index 0000000000..42f44ed1a7
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/dla_tsr_geom_cmp_test.go
@@ -0,0 +1,367 @@
+//go:build cgo && integration
+
+package pdf
+
+import (
+ "encoding/json"
+ "math"
+ "os"
+ "path/filepath"
+ "sort"
+ "testing"
+
+ "ragflow/internal/deepdoc/parser/pdf/tool"
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+ util "ragflow/internal/deepdoc/parser/pdf/util"
+)
+
+// cmpBox is a bbox in PDF page-point space (1pt = 1/72in).
+type cmpBox struct{ x0, y0, x1, y1 float64 }
+
+// MarshalJSON emits the corner coordinates explicitly: the struct fields are
+// unexported, so the default json marshaler would skip them and write `{}`.
+func (b cmpBox) MarshalJSON() ([]byte, error) {
+ return json.Marshal(struct {
+ X0, Y0, X1, Y1 float64
+ }{b.x0, b.y0, b.x1, b.y1})
+}
+
+// TestDLATSRGeomCompareReal measures the geometric divergence between Go's
+// PRODUCTION TSR (pdfium render -> DEEPDOC_URL) and Python's production TSR
+// (dumped by dump_py_results.py into tsr_raw/) for ONE real PDF, so we can
+// answer "do the two sides feed the assembly layer different geometry, and by
+// how much?" with measured numbers instead of code-path inference.
+//
+// Usage (needs a live DEEPDOC_URL inference service; see AGENTS.md `integration` tier):
+//
+// TSR_CMP_PDF=/abs/path/to.pdf \
+// TSR_CMP_PY_TSR=/abs/path/to/py_tsr_raw.json \
+// bash build.sh --test-integration ./internal/deepdoc/parser/pdf/ -run TestDLATSRGeomCompareReal
+//
+// Both sides are mapped into PDF page-point space and matched greedily by
+// cell-center proximity. We report cell-count delta, mean/max center offset
+// (points), and mean IoU — the actual geometric gap.
+func TestDLATSRGeomCompareReal(t *testing.T) {
+ pdfPath := os.Getenv("TSR_CMP_PDF")
+ pyTSRPath := os.Getenv("TSR_CMP_PY_TSR")
+ if pdfPath == "" || pyTSRPath == "" {
+ t.Skip("set TSR_CMP_PDF and TSR_CMP_PY_TSR")
+ }
+
+ data, err := os.ReadFile(pdfPath)
+ if err != nil {
+ t.Fatalf("read pdf: %v", err)
+ }
+ eng, err := NewEngine(data)
+ if err != nil {
+ t.Fatalf("open engine: %v", err)
+ }
+ defer eng.Close()
+
+ client := mustConnectInferenceClient(t)
+ ctx := t.Context()
+
+ pageImg, err := RenderPageToImage(eng, 0)
+ if err != nil {
+ t.Fatalf("render: %v", err)
+ }
+ regions, err := client.DLA(ctx, pageImg)
+ if err != nil {
+ t.Fatalf("DLA: %v", err)
+ }
+ var tr *pdf.DLARegion
+ for i := range regions {
+ if regions[i].Label == "table" {
+ tr = &regions[i]
+ break
+ }
+ }
+ if tr == nil {
+ t.Fatal("no table region detected by DLA")
+ }
+ t.Logf("Go DLA table region (full-page px): [%.1f, %.1f, %.1f, %.1f]",
+ tr.X0, tr.Y0, tr.X1, tr.Y1)
+
+ cropped := cropImageRect(pageImg,
+ int(tr.X0), int(tr.Y0), int(tr.X1), int(tr.Y1))
+ goCells, err := client.TSR(ctx, cropped)
+ if err != nil {
+ t.Fatalf("TSR: %v", err)
+ }
+
+ scale := pdf.DlaScale
+ var goBoxes []cmpBox
+ for _, c := range goCells {
+ goBoxes = append(goBoxes, cmpBox{
+ (c.X0 + tr.X0) / scale,
+ (c.Y0 + tr.Y0) / scale,
+ (c.X1 + tr.X0) / scale,
+ (c.Y1 + tr.Y0) / scale,
+ })
+ }
+
+ pyCells, err := tool.LoadPythonTSR(pyTSRPath)
+ if err != nil {
+ t.Fatalf("load py tsr: %v", err)
+ }
+ var pyBoxes []cmpBox
+ for _, c := range pyCells {
+ if c.Page != 0 {
+ continue
+ }
+ pyBoxes = append(pyBoxes, cmpBox{c.X0, c.Y0, c.X1, c.Y1})
+ }
+
+ t.Logf("Go cells (page0, mapped to points) = %d", len(goBoxes))
+ t.Logf("Python cells (page0) = %d", len(pyBoxes))
+
+ // Greedy nearest-center matching: each Python box -> closest unused Go box.
+ pairs := matchNearest(pyBoxes, goBoxes)
+
+ if len(pairs) == 0 {
+ t.Fatalf("no matched cells — impossible to compare geometry")
+ }
+ var sumD, maxD, sumIoU float64
+ for _, p := range pairs {
+ sumD += p.dCenter
+ if p.dCenter > maxD {
+ maxD = p.dCenter
+ }
+ sumIoU += p.iou
+ }
+ meanD := sumD / float64(len(pairs))
+ meanIoU := sumIoU / float64(len(pairs))
+ unmatched := len(pyBoxes) + len(goBoxes) - 2*len(pairs)
+
+ t.Logf("Matched pairs = %d", len(pairs))
+ t.Logf("Unmatched cells = %d", unmatched)
+ t.Logf("Mean center offset = %.2f pt", meanD)
+ t.Logf("Max center offset = %.2f pt", maxD)
+ t.Logf("Mean IoU = %.3f", meanIoU)
+
+ // The raw offset above includes the DLA table-REGION detection gap (Go's
+ // DLA crop may be larger/shifted vs Python's table extent). To isolate the
+ // TSR CELL-GRID geometry itself, align each side's overall table bbox to a
+ // common origin (translation only — both tables are ~same size) and re-measure.
+ goB := bboxOf(goBoxes)
+ pyB := bboxOf(pyBoxes)
+ t.Logf("Go table extent (pt) = %.0f x %.0f", goB.x1-goB.x0, goB.y1-goB.y0)
+ t.Logf("Py table extent (pt) = %.0f x %.0f", pyB.x1-pyB.x0, pyB.y1-pyB.y0)
+ shiftX, shiftY := pyB.x0-goB.x0, pyB.y0-goB.y0
+ var goNorm []cmpBox
+ for _, b := range goBoxes {
+ goNorm = append(goNorm, cmpBox{b.x0 + shiftX, b.y0 + shiftY, b.x1 + shiftX, b.y1 + shiftY})
+ }
+ // Re-match in the aligned frame.
+ pairsN := matchNearest(pyBoxes, goNorm)
+ var sumDN, maxDN, sumIoUN float64
+ for _, p := range pairsN {
+ sumDN += p.dCenter
+ if p.dCenter > maxDN {
+ maxDN = p.dCenter
+ }
+ sumIoUN += p.iou
+ }
+ t.Logf("--- after aligning table bboxes (isolates TSR cell-grid) ---")
+ t.Logf("Aligned mean center offset = %.2f pt", sumDN/float64(len(pairsN)))
+ t.Logf("Aligned max center offset = %.2f pt", maxDN)
+ t.Logf("Aligned mean IoU = %.3f", sumIoUN/float64(len(pairsN)))
+
+ // Surface the worst 5 matched pairs (aligned frame) so the gap is concrete.
+ sort.Slice(pairsN, func(i, j int) bool { return pairsN[i].dCenter > pairsN[j].dCenter })
+ for i := 0; i < len(pairsN) && i < 5; i++ {
+ w := pairsN[i]
+ t.Logf(" worst#%d off=%.1fpt PY=[%.0f,%.0f,%.0f,%.0f] GO=[%.0f,%.0f,%.0f,%.0f]",
+ i+1, w.dCenter,
+ w.py.x0, w.py.y0, w.py.x1, w.py.y1,
+ w.goC.x0, w.goC.y0, w.goC.x1, w.goC.y1)
+ }
+
+ outDir := os.Getenv("TSR_CMP_OUT")
+ if outDir == "" {
+ outDir = filepath.Join("testdata", "output", "render_compare")
+ }
+ if err := os.MkdirAll(outDir, 0755); err != nil {
+ t.Logf("warn: mkdir %s: %v", outDir, err)
+ }
+ if b, err := json.MarshalIndent(goBoxes, "", " "); err != nil {
+ t.Logf("warn: marshal go boxes: %v", err)
+ } else if err := os.WriteFile(filepath.Join(outDir, "go_tsr_real_points.json"), b, 0644); err != nil {
+ t.Logf("warn: write go_tsr_real_points.json: %v", err)
+ }
+}
+
+// TestSameImageTSRConsistency isolates the ONE variable the user asked about:
+// "for the SAME image sent to the inference service, do Go and Python get
+// consistent TSR?" Go's client.TSR internally does util.EncodePNG(cropped)
+// then POSTs multipart(field "request", file "tsr.png") to /predict/tsr. We
+// save that EXACT PNG the Go client would send, then a Python caller POSTs the
+// same bytes to the same endpoint; the bboxes must be identical (the service
+// is stateless + deterministic, caller-agnostic). Any real-PDF Go-vs-Python
+// TSR gap therefore comes from the INPUT IMAGE differing (pdfium vs
+// pdfplumber), not from the service treating callers differently.
+//
+// Usage (needs a live DEEPDOC_URL inference service; see AGENTS.md `integration` tier):
+//
+// TSR_CMP_PDF=/abs/path/to.pdf \
+// bash build.sh --test-integration ./internal/deepdoc/parser/pdf/ -run TestSameImageTSRConsistency
+//
+// then run tools-py/cmp_same_image_tsr.py (reads the saved PNG + go json).
+func TestSameImageTSRConsistency(t *testing.T) {
+ pdfPath := os.Getenv("TSR_CMP_PDF")
+ if pdfPath == "" {
+ t.Skip("set TSR_CMP_PDF")
+ }
+ data, err := os.ReadFile(pdfPath)
+ if err != nil {
+ t.Fatalf("read pdf: %v", err)
+ }
+ eng, err := NewEngine(data)
+ if err != nil {
+ t.Fatalf("open engine: %v", err)
+ }
+ defer eng.Close()
+
+ client := mustConnectInferenceClient(t)
+ ctx := t.Context()
+ pageImg, err := RenderPageToImage(eng, 0)
+ if err != nil {
+ t.Fatalf("render: %v", err)
+ }
+ regions, err := client.DLA(ctx, pageImg)
+ if err != nil {
+ t.Fatalf("DLA: %v", err)
+ }
+ var tr *pdf.DLARegion
+ for i := range regions {
+ if regions[i].Label == "table" {
+ tr = &regions[i]
+ break
+ }
+ }
+ if tr == nil {
+ t.Fatal("no table region detected by DLA")
+ }
+ cropped := cropImageRect(pageImg, int(tr.X0), int(tr.Y0), int(tr.X1), int(tr.Y1))
+
+ // Output dir: override with TSR_CMP_OUT (the default testdata dir may not
+ // be writable under the current user); the Python comparator reads the same.
+ outDir := os.Getenv("TSR_CMP_OUT")
+ if outDir == "" {
+ outDir = filepath.Join("testdata", "output", "render_compare")
+ }
+ if err := os.MkdirAll(outDir, 0755); err != nil {
+ t.Logf("warn: mkdir %s: %v", outDir, err)
+ }
+
+ // The exact bytes Go's client.TSR sends to the service.
+ pngBytes, err := util.EncodePNG(cropped)
+ if err != nil {
+ t.Fatalf("encode png: %v", err)
+ }
+ if err := os.WriteFile(filepath.Join(outDir, "tsr_same_input.png"), pngBytes, 0644); err != nil {
+ t.Fatalf("write png: %v", err)
+ }
+ t.Logf("Saved exact PNG Go sends (%d bytes, %dx%d) → %s/tsr_same_input.png",
+ len(pngBytes), cropped.Bounds().Dx(), cropped.Bounds().Dy(), outDir)
+
+ // Go's TSR result in the PNG's own pixel space (no coordinate transform).
+ goCells, err := client.TSR(ctx, cropped)
+ if err != nil {
+ t.Fatalf("TSR: %v", err)
+ }
+ type rawCell struct {
+ X0, Y0, X1, Y1 float64
+ Label string
+ Score float64
+ }
+ out := make([]rawCell, 0, len(goCells))
+ for _, c := range goCells {
+ out = append(out, rawCell{c.X0, c.Y0, c.X1, c.Y1, c.Label, c.Score})
+ }
+ writeJSON(t, filepath.Join(outDir, "go_tsr_same_px.json"), out)
+ t.Logf("Go TSR cells (PNG px) = %d → %s/go_tsr_same_px.json", len(out), outDir)
+ t.Logf("Now run: python3 internal/deepdoc/parser/pdf/tool-py/cmp_same_image_tsr.py %s", outDir)
+}
+
+// geomPair is one greedily-matched Python↔Go table cell.
+type geomPair struct {
+ dCenter float64
+ iou float64
+ py, goC cmpBox
+}
+
+// matchNearest greedily matches each Python box to the closest unused Go box
+// (by cell-center distance) and returns the matched pairs. Both slices must be
+// expressed in the same coordinate space before calling.
+func matchNearest(pyBoxes, goBoxes []cmpBox) []geomPair {
+ var pairs []geomPair
+ used := make([]bool, len(goBoxes))
+ for _, pb := range pyBoxes {
+ pcx, pcy := boxCenter(pb)
+ best := -1
+ var bestD float64
+ for j, gb := range goBoxes {
+ if used[j] {
+ continue
+ }
+ gcx, gcy := boxCenter(gb)
+ d := math.Hypot(pcx-gcx, pcy-gcy)
+ if best == -1 || d < bestD {
+ best = j
+ bestD = d
+ }
+ }
+ if best == -1 {
+ continue
+ }
+ used[best] = true
+ pairs = append(pairs, geomPair{
+ dCenter: bestD,
+ iou: boxIoU(pb, goBoxes[best]),
+ py: pb,
+ goC: goBoxes[best],
+ })
+ }
+ return pairs
+}
+
+func boxCenter(b cmpBox) (float64, float64) {
+ return (b.x0 + b.x1) / 2, (b.y0 + b.y1) / 2
+}
+
+// bboxOf returns the overall bounding box enclosing all boxes.
+func bboxOf(boxes []cmpBox) cmpBox {
+ b := boxes[0]
+ for _, x := range boxes[1:] {
+ if x.x0 < b.x0 {
+ b.x0 = x.x0
+ }
+ if x.y0 < b.y0 {
+ b.y0 = x.y0
+ }
+ if x.x1 > b.x1 {
+ b.x1 = x.x1
+ }
+ if x.y1 > b.y1 {
+ b.y1 = x.y1
+ }
+ }
+ return b
+}
+
+func boxIoU(a, b cmpBox) float64 {
+ ix0 := math.Max(a.x0, b.x0)
+ iy0 := math.Max(a.y0, b.y0)
+ ix1 := math.Min(a.x1, b.x1)
+ iy1 := math.Min(a.y1, b.y1)
+ iw := ix1 - ix0
+ ih := iy1 - iy0
+ if iw <= 0 || ih <= 0 {
+ return 0
+ }
+ inter := iw * ih
+ areaA := (a.x1 - a.x0) * (a.y1 - a.y0)
+ areaB := (b.x1 - b.x0) * (b.y1 - b.y0)
+ return inter / (areaA + areaB - inter)
+}
diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_test.go
index 18610223e7..f11c7d98c7 100644
--- a/internal/deepdoc/parser/pdf/pipeline_parity_test.go
+++ b/internal/deepdoc/parser/pdf/pipeline_parity_test.go
@@ -35,8 +35,10 @@ import (
// - textSim==100% -> PASS (fully aligned, includes table HTML byte-identical)
// - table PDF, gridSim==100% AND structSim==100% but textSim<100% ->
// NONCELL_TEXT: cell content AND structure match Python, the residual gap
-// is non-cell-text (caption/body paragraph Go omits, and/or HTML tag/format
-// differences). Reported separately (not labeled PASS) and not counted as a
+// is non-cell-text HTML-emitter formatting (e.g. header <th> vs <td>,
+// whitespace/newline) — not cell content. In other PDFs it may also be a
+// caption/body paragraph Go omits. Reported separately (not labeled PASS)
+// and not counted as a
// content failure; classified as go_intentional/go_bug via known_diffs.json.
// - table PDF, gridSim<100% OR structSim<100% (or non-table textSim<100%) ->
// FAIL if not exempted; INTENTIONAL if a go_intentional rule names this
@@ -192,9 +194,10 @@ func TestPipelineParity(t *testing.T) {
// reconstructed grid content (cell text) rather than the rendered HTML.
// Grid content parity (gridSim + structSim) isolates Go's GroupCells +
// FillCellTextFromBoxes assembly. When gridSim AND structSim are 100%
- // but textSim<100%, the residual gap is NON-CELL-TEXT: a <caption> Go
- // omits, an interleaved body paragraph Go drops, and/or HTML tag/format
- // differences — none of which is a table-cell-assembly bug.
+ // but textSim<100%, the residual gap is NON-CELL-TEXT: HTML-emitter
+ // formatting (header <th> vs <td>, whitespace/newline) — none of which is
+ // a table-cell-assembly bug. (A caption/body paragraph Go omits is also
+ // possible in other PDFs.)
pyRows, pyHasTables := loadPythonTables(t, filepath.Join(tablesDir, name+".json"))
goRows := goTableRows(result)
goHasTables := len(goRows) > 0
@@ -220,19 +223,19 @@ func TestPipelineParity(t *testing.T) {
structureSim, shapeDetail := gridStructureSimilarity(goRows, pyRows)
// Full content parity = identical cell text AND identical
// structure. Only then is the residual textSim gap purely the
- // non-cell-text layer (caption/body/format outside table cells),
+ // non-cell-text layer (HTML-emitter formatting outside table cells, e.g.
// reported as NONCELL_TEXT.
contentMatch := gridSim >= 100.0 && structureSim >= 100.0
if contentMatch {
// Cells + structure match Python, but the full text still
- // diverges — the gap is OUTSIDE table cells: caption text
- // Go omits, an interleaved body paragraph Go drops, and/or
- // HTML tag/format differences. Label it noncell-text (not
- // "html-divergent", which wrongly implies content matches)
- // so the divergence is named and not hidden.
+ // diverges — the gap is OUTSIDE table cells: emitter
+ // markup/whitespace OR an omitted caption/body paragraph
+ // (not a table-cell-assembly bug). Label it noncell-text
+ // (not "html-divergent", which wrongly implies content
+ // matches) so the divergence is named and not hidden.
noncellText++
status = "NONCELL_TEXT"
- detail = fmt.Sprintf("gridSim=%.1f%% structSim=%.1f%% (%s) textSim=%.1f%% (non-cell-text divergence: caption/body/format outside table cells)", gridSim, structureSim, shapeDetail, sim)
+ detail = fmt.Sprintf("gridSim=%.1f%% structSim=%.1f%% (%s) textSim=%.1f%% (non-cell-text divergence outside table-cell content: emitter markup/whitespace or omitted caption/body text)", gridSim, structureSim, shapeDetail, sim)
} else if intentionalPDF(name) {
// go_intentional rule covers this PDF: the divergence (cell
// content and/or structure) is a documented, deliberate one
diff --git a/internal/deepdoc/parser/pdf/table/group_boxes.go b/internal/deepdoc/parser/pdf/table/group_boxes.go
index 65ceec9e99..5a794d6c65 100644
--- a/internal/deepdoc/parser/pdf/table/group_boxes.go
+++ b/internal/deepdoc/parser/pdf/table/group_boxes.go
@@ -185,6 +185,26 @@ func cellPosFromBox(b pdf.TextBox) (x0, y0, x1, y1 float64, label string) {
}
} else if b.SP > 0 {
label = "table spanning cell"
+ // A pure-SP box (H==0) still carries the span's full extent in
+ // HLeft/HRight/HTop/HBott, copied together from the TSR spanning cell
+ // by AnnotateTableBoxes (PR #18707). Rebuild the cell from those bounds
+ // so CalSpans covers every column/row the span crosses — matching
+ // Python's _annotate_table_boxes. Without this, the span cell falls
+ // back to the box's own narrow text bounds, under-covers by one
+ // column, and the header row keeps an extra empty cell (real_pdfs/1.pdf
+ // row0 = 4 vs Python's 3; see known_diffs rule
+ // table-1pdf-colspan-assembly-loss, resolved).
+ //
+ // HLeft/HRight (and HTop/HBott) are set together, so treat an axis as
+ // propagated whenever either edge is set. This preserves a real edge at
+ // coordinate 0 (a nonzero check would mistake it for "unset" and fall
+ // back to the box's own bounds, dropping the span's left/top edge).
+ if b.HLeft != 0 || b.HRight != 0 {
+ x0, x1 = b.HLeft, b.HRight
+ }
+ if b.HTop != 0 || b.HBott != 0 {
+ y0, y1 = b.HTop, b.HBott
+ }
}
return
}
diff --git a/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.go b/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.go
new file mode 100644
index 0000000000..88e879a426
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.go
@@ -0,0 +1,104 @@
+package table
+
+// These tests expose the row0 residual of real_pdfs/1.pdf tracked as
+// table-1pdf-colspan-assembly-loss (known_diffs.json, status resolved):
+// Go's header row keeps one extra empty cell (4 vs Python's 3) because
+// cellPosFromBox ignores the span bounds (HLeft/HRight/HTop/HBott) copied onto
+// a pure-SP box (H==0, SP>0) by AnnotateTableBoxes (PR #18707). CalSpans then
+// covers only the columns whose centers fall inside the box's own narrow
+// bounds, leaving the trailing covered column un-marked and alive as a 4th cell.
+//
+// This is the GroupBoxesByRC/CalSpans-level regression test CodeRabbit asked
+// for in PR #18707 (the shipped tests only cover AnnotateTableBoxes field
+// propagation, not the colspan geometry). It is a default unit test (no cgo
+// tag) so it runs in `go test ./...` and CI.
+
+import (
+ "testing"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// TestGroupBoxesByRCSpanHeaderRowCellCount drives the real assembly path
+// (GroupBoxesByRC -> CalSpans -> MarkCoveredCells -> RowsToStrings) with a
+// synthetic 1.pdf-like header: 8 logical columns, a TSR "table spanning cell"
+// covering 6 of them. Python's golden collapses the row to 3 cells (the 6-col
+// span + 2 trailing single columns). Go must match.
+func TestGroupBoxesByRCSpanHeaderRowCellCount(t *testing.T) {
+ const nCols = 8
+ boxes := make([]pdf.TextBox, 0, nCols)
+
+ // Spanning header text box at col 0. Its OCR box is narrow [0,450], but the
+ // TSR span component reaches the full extent [0,600]; AnnotateTableBoxes
+ // copies that onto the box as HLeft/HRight/HTop/HBott. For a pure-SP box
+ // (H==0, SP>0) cellPosFromBox must use those bounds, otherwise the span
+ // cell falls back to [0,450] and covers only 5 columns.
+ //
+ // R starts at 1 (not 0): GroupBoxesByRC falls back to YX grouping when
+ // maxR<=0, which would skip cellPosFromBox entirely; a real table has
+ // multiple rows so maxR>0 and the R/C path runs.
+ boxes = append(boxes, pdf.TextBox{
+ X0: 0, X1: 450, Top: 0, Bottom: 20,
+ Text: "HEADER", R: 1, C: 0,
+ SP: 1, HLeft: 0, HRight: 600, HTop: 0, HBott: 20,
+ })
+
+ // Remaining columns 1..7 as header/content cells, defining column geometry.
+ trailing := []string{"c1", "c2", "c3", "c4", "c5", "c6", "c7"}
+ for c := 1; c < nCols; c++ {
+ boxes = append(boxes, pdf.TextBox{
+ X0: float64(c) * 100, X1: float64(c)*100 + 100, Top: 0, Bottom: 20,
+ Text: trailing[c-1], R: 1, C: c,
+ H: 1,
+ })
+ }
+ // One body row (R=2) so the table has more than one row; its cells are
+ // irrelevant to the header-row assertion.
+ for c := 0; c < nCols; c++ {
+ boxes = append(boxes, pdf.TextBox{
+ X0: float64(c) * 100, X1: float64(c)*100 + 100, Top: 30, Bottom: 50,
+ Text: "b", R: 2, C: c,
+ })
+ }
+
+ rows := GroupBoxesByRC(boxes)
+ if len(rows) == 0 {
+ t.Fatal("GroupBoxesByRC returned an empty grid")
+ }
+ spanInfo, covered := CalSpans(rows)
+ MarkCoveredCells(rows, covered)
+
+ row0 := RowsToStrings(rows)[0]
+ if len(row0) != 3 {
+ t.Errorf("header row should collapse to 3 cells (6-col span + 2 trailing), "+
+ "got %d: %v (spanInfo=%v)", len(row0), row0, spanInfo)
+ }
+ // Sanity: the span must actually be colspan=6, not a narrower span that
+ // happens to leave 3 cells via a different arrangement.
+ if cs, ok := spanInfo[[2]int{0, 0}]; !ok || cs[0] != 6 {
+ t.Errorf("expected colspan=6 on the span origin (0,0), got %v (ok=%v)", cs, ok)
+ }
+}
+
+// TestCellPosFromBoxSpanUsesSpanBounds pins the root cause directly: a pure-SP
+// box (H==0, SP>0) with propagated span bounds must rebuild its cell from those
+// bounds, not from the box's own narrow text bounds. HLeft/HRight (and
+// HTop/HBott) are set together by AnnotateTableBoxes, so an axis with a zero
+// edge is still propagated when the opposite edge is set — a real span edge at
+// coordinate 0 must not be mistaken for "unset". The synthetic box uses a
+// nonzero text box (X0=200) and a zero left/top propagated edge to lock this in.
+func TestCellPosFromBoxSpanUsesSpanBounds(t *testing.T) {
+ b := pdf.TextBox{
+ X0: 200, X1: 300, Top: 0, Bottom: 20,
+ H: 0, SP: 1, HLeft: 0, HRight: 600, HTop: 0, HBott: 25,
+ }
+ x0, y0, x1, y1, _ := cellPosFromBox(b)
+ if x0 != 0 || x1 != 600 {
+ t.Errorf("pure-SP box must use propagated span bounds even when HLeft=0 "+
+ "(span edge at coord 0), got x0=%v x1=%v", x0, x1)
+ }
+ if y0 != 0 || y1 != 25 {
+ t.Errorf("pure-SP box must use propagated span bounds even when HTop=0 "+
+ "(span edge at coord 0), got y0=%v y1=%v", y0, y1)
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
index 723e322055..d7b89473f6 100644
--- a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
+++ b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
@@ -232,9 +232,9 @@
"cellPosFromBox"
],
"owner_fix_side": "go",
- "status": "resolved_partial",
- "resolution": "AnnotateTableBoxes (internal/deepdoc/parser/pdf/table/table_layout.go) now (a) collects TSR 'spanning cell' components into its spans slice and sets box.SP = idx+1 on overlapping boxes (matching Python _table_transformer_job / pdf_parser.py:518-554), and (b) copies the span cell's bbox onto the box as H_top/H_bott/H_left/H_right (matching Python _annotate_table_boxes pdf_parser.py:632-635). GroupBoxesByRC then rebuilds the span cell from these full extents (cellPosFromBox uses HLeft/HRight when set) and CalSpans covers every column the span crosses. Before the fix, box.SP stayed 0 AND HLeft/HRight were not propagated for SP boxes, so the span cell fell back to the box's own narrow bounds and CalSpans covered only 5 columns -> Go emitted colspan=5 where Python emits colspan=6. After the fix Go emits colspan=6 (verified end-to-end via Test1PdfColspanEndToEnd on the live pipeline, and via TestPipelineParity replay: 1.pdf row0 went from 8 independent <th> to 4 cells, textSim 95.7% -> 97.4%). Regression: TestAnnotateTableBoxesSpanCarriesBoxCoords, TestAnnotateTableBoxesPropagatesSpan, TestGroupCellsInjectsSpanning, Test1PdfColspanEndToEnd.",
- "tracking": "Residual: 1.pdf's header row still shows row0 = 4 cells in Go vs 3 cells in Python's golden (structSim 87.5%, the LOCKED grid check still fails). The colspan=6 is correctly emitted, but Go keeps one extra empty spanning cell in row0 (4 vs 3), so the per-row column count is not yet byte-identical to Python. Left as resolved_partial: the colspan-assembly bug is fixed; the 1-cell row0 residual needs a follow-up on how GroupBoxesByRC folds the covered trailing column before 1.pdf can reach structSim=100% / aligned.",
+ "status": "resolved",
+ "resolution": "AnnotateTableBoxes (internal/deepdoc/parser/pdf/table/table_layout.go) (a) collects TSR 'spanning cell' components into its spans slice and sets box.SP = idx+1 on overlapping boxes (matching Python _table_transformer_job / pdf_parser.py:518-554), and (b) copies the span cell's bbox onto the box as H_top/H_bott/H_left/H_right (matching Python _annotate_table_boxes pdf_parser.py:632-635). cellPosFromBox (internal/deepdoc/parser/pdf/table/group_boxes.go) now ALSO applies HLeft/HRight/HTop/HBott for a pure-SP box (H==0, SP>0), so GroupBoxesByRC rebuilds the span cell from the full span extents and CalSpans covers every column the span crosses. Before the fix, box.SP stayed 0 AND the SP-only branch of cellPosFromBox ignored the propagated HLeft/HRight, so the span cell fell back to the box's own narrow bounds and CalSpans covered only 5 columns -> Go emitted colspan=5 where Python emits colspan=6 AND the header row kept 4 cells where Python collapses to 3. After the fix Go emits colspan=6 and row0 collapses to 3 cells (verified end-to-end via Test1PdfColspanEndToEnd on the live pipeline, and via TestPipelineParity replay: 1.pdf row0 went from 8 independent <th> to 3 cells, textSim 95.7% -> 97.4%, structSim 87.5% -> 100%). Regression: TestAnnotateTableBoxesSpanCarriesBoxCoords, TestAnnotateTableBoxesPropagatesSpan, TestGroupCellsInjectsSpanning, Test1PdfColspanEndToEnd, TestGroupBoxesByRCSpanHeaderRowCellCount, TestCellPosFromBoxSpanUsesSpanBounds.",
+ "tracking": "RESOLVED: the row0 = 4-vs-3 residual (structSim 87.5%) was caused by cellPosFromBox's SP-only branch ignoring the span bounds propagated by AnnotateTableBoxes, so the trailing covered column's center fell outside the span cell's narrow X range and was not marked covered. Applying the span bounds in the SP branch makes CalSpans cover all 6 columns, so row0 now collapses to 3 cells (matching Python) and structSim reaches 100% / aligned. The original per-edge `!= 0` sentinel (coordinate 0 mistaken for 'unset') was replaced by a per-axis check (propagate both edges whenever either is set), so a span whose real edge is exactly 0 is now preserved; HLeft/HRight and HTop/HBott are copied together by AnnotateTableBoxes, so the invariant holds.",
"reason": "1.pdf's merged header ('液化石油气储罐(区)(总容积V,m²)') spans 6 of the 8 columns. Python's golden carries colspan=6 on that header cell (its row0 collapses to 3 cells: 1 content + 2 empties around the 6-span). Go USED TO DROP the span: AnnotateTableBoxes only propagated the spanning cell's bbox (H_left/H_right/H_top/H_bott) onto header boxes, not onto 'spanning cell' (SP) boxes, and did not even collect the spans slice, so box.SP stayed 0 and the rebuilt grid (GroupBoxesByRC) lost the span; CalSpans then covered only the box's own narrow bounds -> colspan=5. TSR input is identical on both sides, so this was purely a Go ASSEMBLY bug, the OPPOSITE direction of rule table-span-follow-tsr-geometry (there Go HAS the colspan and Python omits it; here Python HAS it and Go dropped it). Direction matters for classification: this is a go_bug (Go was wrong, now fixed), NOT a go_intentional. Exposed by TestPipelineParity replay (1.pdf: LOCKED gridSim=100% structSim=87.5% regressed below 100%) and the end-to-end Test1PdfColspanEndToEnd."
},
{