RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(pdf): colspan assembly loss + TSR geometry comparison diagnostics (#18727)
xugangqiangd7e3592501cd
diff --git a/internal/deepdoc/parser/pdf/dla_tsr_geom_cmp_test.go b/internal/deepdoc/parser/pdf/dla_tsr_geom_cmp_test.gonew file mode 100644index 0000000000..42f44ed1a7--- /dev/null+++ b/internal/deepdoc/parser/pdf/dla_tsr_geom_cmp_test.go@@ -0,0 +1,367 @@+//go:build cgo && integration++package pdf++import (+ "encoding/json"+ "math"+ "os"+ "path/filepath"+ "sort"+ "testing"++ "ragflow/internal/deepdoc/parser/pdf/tool"+ pdf "ragflow/internal/deepdoc/parser/pdf/type"+ util "ragflow/internal/deepdoc/parser/pdf/util"+)++// cmpBox is a bbox in PDF page-point space (1pt = 1/72in).+type cmpBox struct{ x0, y0, x1, y1 float64 }++// MarshalJSON emits the corner coordinates explicitly: the struct fields are+// unexported, so the default json marshaler would skip them and write `{}`.+func (b cmpBox) MarshalJSON() ([]byte, error) {+ return json.Marshal(struct {+ X0, Y0, X1, Y1 float64+ }{b.x0, b.y0, b.x1, b.y1})+}++// TestDLATSRGeomCompareReal measures the geometric divergence between Go's+// PRODUCTION TSR (pdfium render -> DEEPDOC_URL) and Python's production TSR+// (dumped by dump_py_results.py into tsr_raw/) for ONE real PDF, so we can+// answer "do the two sides feed the assembly layer different geometry, and by+// how much?" with measured numbers instead of code-path inference.+//+// Usage (needs a live DEEPDOC_URL inference service; see AGENTS.md `integration` tier):+//+// TSR_CMP_PDF=/abs/path/to.pdf \+// TSR_CMP_PY_TSR=/abs/path/to/py_tsr_raw.json \+// bash build.sh --test-integration ./internal/deepdoc/parser/pdf/ -run TestDLATSRGeomCompareReal+//+// Both sides are mapped into PDF page-point space and matched greedily by+// cell-center proximity. We report cell-count delta, mean/max center offset+// (points), and mean IoU — the actual geometric gap.+func TestDLATSRGeomCompareReal(t *testing.T) {+ pdfPath := os.Getenv("TSR_CMP_PDF")+ pyTSRPath := os.Getenv("TSR_CMP_PY_TSR")+ if pdfPath == "" || pyTSRPath == "" {+ t.Skip("set TSR_CMP_PDF and TSR_CMP_PY_TSR")+ }++ data, err := os.ReadFile(pdfPath)+ if err != nil {+ t.Fatalf("read pdf: %v", err)+ }+ eng, err := NewEngine(data)+ if err != nil {+ t.Fatalf("open engine: %v", err)+ }+ defer eng.Close()++ client := mustConnectInferenceClient(t)+ ctx := t.Context()++ pageImg, err := RenderPageToImage(eng, 0)+ if err != nil {+ t.Fatalf("render: %v", err)+ }+ regions, err := client.DLA(ctx, pageImg)+ if err != nil {+ t.Fatalf("DLA: %v", err)+ }+ var tr *pdf.DLARegion+ for i := range regions {+ if regions[i].Label == "table" {+ tr = ®ions[i]+ break+ }+ }+ if tr == nil {+ t.Fatal("no table region detected by DLA")+ }+ t.Logf("Go DLA table region (full-page px): [%.1f, %.1f, %.1f, %.1f]",+ tr.X0, tr.Y0, tr.X1, tr.Y1)++ cropped := cropImageRect(pageImg,+ int(tr.X0), int(tr.Y0), int(tr.X1), int(tr.Y1))+ goCells, err := client.TSR(ctx, cropped)+ if err != nil {+ t.Fatalf("TSR: %v", err)+ }++ scale := pdf.DlaScale+ var goBoxes []cmpBox+ for _, c := range goCells {+ goBoxes = append(goBoxes, cmpBox{+ (c.X0 + tr.X0) / scale,+ (c.Y0 + tr.Y0) / scale,+ (c.X1 + tr.X0) / scale,+ (c.Y1 + tr.Y0) / scale,+ })+ }++ pyCells, err := tool.LoadPythonTSR(pyTSRPath)+ if err != nil {+ t.Fatalf("load py tsr: %v", err)+ }+ var pyBoxes []cmpBox+ for _, c := range pyCells {+ if c.Page != 0 {+ continue+ }+ pyBoxes = append(pyBoxes, cmpBox{c.X0, c.Y0, c.X1, c.Y1})+ }++ t.Logf("Go cells (page0, mapped to points) = %d", len(goBoxes))+ t.Logf("Python cells (page0) = %d", len(pyBoxes))++ // Greedy nearest-center matching: each Python box -> closest unused Go box.+ pairs := matchNearest(pyBoxes, goBoxes)++ if len(pairs) == 0 {+ t.Fatalf("no matched cells — impossible to compare geometry")+ }+ var sumD, maxD, sumIoU float64+ for _, p := range pairs {+ sumD += p.dCenter+ if p.dCenter > maxD {+ maxD = p.dCenter+ }+ sumIoU += p.iou+ }+ meanD := sumD / float64(len(pairs))+ meanIoU := sumIoU / float64(len(pairs))+ unmatched := len(pyBoxes) + len(goBoxes) - 2*len(pairs)++ t.Logf("Matched pairs = %d", len(pairs))+ t.Logf("Unmatched cells = %d", unmatched)+ t.Logf("Mean center offset = %.2f pt", meanD)+ t.Logf("Max center offset = %.2f pt", maxD)+ t.Logf("Mean IoU = %.3f", meanIoU)++ // The raw offset above includes the DLA table-REGION detection gap (Go's+ // DLA crop may be larger/shifted vs Python's table extent). To isolate the+ // TSR CELL-GRID geometry itself, align each side's overall table bbox to a+ // common origin (translation only — both tables are ~same size) and re-measure.+ goB := bboxOf(goBoxes)+ pyB := bboxOf(pyBoxes)+ t.Logf("Go table extent (pt) = %.0f x %.0f", goB.x1-goB.x0, goB.y1-goB.y0)+ t.Logf("Py table extent (pt) = %.0f x %.0f", pyB.x1-pyB.x0, pyB.y1-pyB.y0)+ shiftX, shiftY := pyB.x0-goB.x0, pyB.y0-goB.y0+ var goNorm []cmpBox+ for _, b := range goBoxes {+ goNorm = append(goNorm, cmpBox{b.x0 + shiftX, b.y0 + shiftY, b.x1 + shiftX, b.y1 + shiftY})+ }+ // Re-match in the aligned frame.+ pairsN := matchNearest(pyBoxes, goNorm)+ var sumDN, maxDN, sumIoUN float64+ for _, p := range pairsN {+ sumDN += p.dCenter+ if p.dCenter > maxDN {+ maxDN = p.dCenter+ }+ sumIoUN += p.iou+ }+ t.Logf("--- after aligning table bboxes (isolates TSR cell-grid) ---")+ t.Logf("Aligned mean center offset = %.2f pt", sumDN/float64(len(pairsN)))+ t.Logf("Aligned max center offset = %.2f pt", maxDN)+ t.Logf("Aligned mean IoU = %.3f", sumIoUN/float64(len(pairsN)))++ // Surface the worst 5 matched pairs (aligned frame) so the gap is concrete.+ sort.Slice(pairsN, func(i, j int) bool { return pairsN[i].dCenter > pairsN[j].dCenter })+ for i := 0; i < len(pairsN) && i < 5; i++ {+ w := pairsN[i]+ t.Logf(" worst#%d off=%.1fpt PY=[%.0f,%.0f,%.0f,%.0f] GO=[%.0f,%.0f,%.0f,%.0f]",+ i+1, w.dCenter,+ w.py.x0, w.py.y0, w.py.x1, w.py.y1,+ w.goC.x0, w.goC.y0, w.goC.x1, w.goC.y1)+ }++ outDir := os.Getenv("TSR_CMP_OUT")+ if outDir == "" {+ outDir = filepath.Join("testdata", "output", "render_compare")+ }+ if err := os.MkdirAll(outDir, 0755); err != nil {+ t.Logf("warn: mkdir %s: %v", outDir, err)+ }+ if b, err := json.MarshalIndent(goBoxes, "", " "); err != nil {+ t.Logf("warn: marshal go boxes: %v", err)+ } else if err := os.WriteFile(filepath.Join(outDir, "go_tsr_real_points.json"), b, 0644); err != nil {+ t.Logf("warn: write go_tsr_real_points.json: %v", err)+ }+}++// TestSameImageTSRConsistency isolates the ONE variable the user asked about:+// "for the SAME image sent to the inference service, do Go and Python get+// consistent TSR?" Go's client.TSR internally does util.EncodePNG(cropped)+// then POSTs multipart(field "request", file "tsr.png") to /predict/tsr. We+// save that EXACT PNG the Go client would send, then a Python caller POSTs the+// same bytes to the same endpoint; the bboxes must be identical (the service+// is stateless + deterministic, caller-agnostic). Any real-PDF Go-vs-Python+// TSR gap therefore comes from the INPUT IMAGE differing (pdfium vs+// pdfplumber), not from the service treating callers differently.+//+// Usage (needs a live DEEPDOC_URL inference service; see AGENTS.md `integration` tier):+//+// TSR_CMP_PDF=/abs/path/to.pdf \+// bash build.sh --test-integration ./internal/deepdoc/parser/pdf/ -run TestSameImageTSRConsistency+//+// then run tools-py/cmp_same_image_tsr.py (reads the saved PNG + go json).+func TestSameImageTSRConsistency(t *testing.T) {+ pdfPath := os.Getenv("TSR_CMP_PDF")+ if pdfPath == "" {+ t.Skip("set TSR_CMP_PDF")+ }+ data, err := os.ReadFile(pdfPath)+ if err != nil {+ t.Fatalf("read pdf: %v", err)+ }+ eng, err := NewEngine(data)+ if err != nil {+ t.Fatalf("open engine: %v", err)+ }+ defer eng.Close()++ client := mustConnectInferenceClient(t)+ ctx := t.Context()+ pageImg, err := RenderPageToImage(eng, 0)+ if err != nil {+ t.Fatalf("render: %v", err)+ }+ regions, err := client.DLA(ctx, pageImg)+ if err != nil {+ t.Fatalf("DLA: %v", err)+ }+ var tr *pdf.DLARegion+ for i := range regions {+ if regions[i].Label == "table" {+ tr = ®ions[i]+ break+ }+ }+ if tr == nil {+ t.Fatal("no table region detected by DLA")+ }+ cropped := cropImageRect(pageImg, int(tr.X0), int(tr.Y0), int(tr.X1), int(tr.Y1))++ // Output dir: override with TSR_CMP_OUT (the default testdata dir may not+ // be writable under the current user); the Python comparator reads the same.+ outDir := os.Getenv("TSR_CMP_OUT")+ if outDir == "" {+ outDir = filepath.Join("testdata", "output", "render_compare")+ }+ if err := os.MkdirAll(outDir, 0755); err != nil {+ t.Logf("warn: mkdir %s: %v", outDir, err)+ }++ // The exact bytes Go's client.TSR sends to the service.+ pngBytes, err := util.EncodePNG(cropped)+ if err != nil {+ t.Fatalf("encode png: %v", err)+ }+ if err := os.WriteFile(filepath.Join(outDir, "tsr_same_input.png"), pngBytes, 0644); err != nil {+ t.Fatalf("write png: %v", err)+ }+ t.Logf("Saved exact PNG Go sends (%d bytes, %dx%d) → %s/tsr_same_input.png",+ len(pngBytes), cropped.Bounds().Dx(), cropped.Bounds().Dy(), outDir)++ // Go's TSR result in the PNG's own pixel space (no coordinate transform).+ goCells, err := client.TSR(ctx, cropped)+ if err != nil {+ t.Fatalf("TSR: %v", err)+ }+ type rawCell struct {+ X0, Y0, X1, Y1 float64+ Label string+ Score float64+ }+ out := make([]rawCell, 0, len(goCells))+ for _, c := range goCells {+ out = append(out, rawCell{c.X0, c.Y0, c.X1, c.Y1, c.Label, c.Score})+ }+ writeJSON(t, filepath.Join(outDir, "go_tsr_same_px.json"), out)+ t.Logf("Go TSR cells (PNG px) = %d → %s/go_tsr_same_px.json", len(out), outDir)+ t.Logf("Now run: python3 internal/deepdoc/parser/pdf/tool-py/cmp_same_image_tsr.py %s", outDir)+}++// geomPair is one greedily-matched Python↔Go table cell.+type geomPair struct {+ dCenter float64+ iou float64+ py, goC cmpBox+}++// matchNearest greedily matches each Python box to the closest unused Go box+// (by cell-center distance) and returns the matched pairs. Both slices must be+// expressed in the same coordinate space before calling.+func matchNearest(pyBoxes, goBoxes []cmpBox) []geomPair {+ var pairs []geomPair+ used := make([]bool, len(goBoxes))+ for _, pb := range pyBoxes {+ pcx, pcy := boxCenter(pb)+ best := -1+ var bestD float64+ for j, gb := range goBoxes {+ if used[j] {+ continue+ }+ gcx, gcy := boxCenter(gb)+ d := math.Hypot(pcx-gcx, pcy-gcy)+ if best == -1 || d < bestD {+ best = j+ bestD = d+ }+ }+ if best == -1 {+ continue+ }+ used[best] = true+ pairs = append(pairs, geomPair{+ dCenter: bestD,+ iou: boxIoU(pb, goBoxes[best]),+ py: pb,+ goC: goBoxes[best],+ })+ }+ return pairs+}++func boxCenter(b cmpBox) (float64, float64) {+ return (b.x0 + b.x1) / 2, (b.y0 + b.y1) / 2+}++// bboxOf returns the overall bounding box enclosing all boxes.+func bboxOf(boxes []cmpBox) cmpBox {+ b := boxes[0]+ for _, x := range boxes[1:] {+ if x.x0 < b.x0 {+ b.x0 = x.x0+ }+ if x.y0 < b.y0 {+ b.y0 = x.y0+ }+ if x.x1 > b.x1 {+ b.x1 = x.x1+ }+ if x.y1 > b.y1 {+ b.y1 = x.y1+ }+ }+ return b+}++func boxIoU(a, b cmpBox) float64 {+ ix0 := math.Max(a.x0, b.x0)+ iy0 := math.Max(a.y0, b.y0)+ ix1 := math.Min(a.x1, b.x1)+ iy1 := math.Min(a.y1, b.y1)+ iw := ix1 - ix0+ ih := iy1 - iy0+ if iw <= 0 || ih <= 0 {+ return 0+ }+ inter := iw * ih+ areaA := (a.x1 - a.x0) * (a.y1 - a.y0)+ areaB := (b.x1 - b.x0) * (b.y1 - b.y0)+ return inter / (areaA + areaB - inter)+}diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_test.goindex 18610223e7..f11c7d98c7 100644--- a/internal/deepdoc/parser/pdf/pipeline_parity_test.go+++ b/internal/deepdoc/parser/pdf/pipeline_parity_test.go@@ -35,8 +35,10 @@ import (// - textSim==100% -> PASS (fully aligned, includes table HTML byte-identical)// - table PDF, gridSim==100% AND structSim==100% but textSim<100% ->// NONCELL_TEXT: cell content AND structure match Python, the residual gap-// is non-cell-text (caption/body paragraph Go omits, and/or HTML tag/format-// differences). Reported separately (not labeled PASS) and not counted as a+// is non-cell-text HTML-emitter formatting (e.g. header <th> vs <td>,+// whitespace/newline) — not cell content. In other PDFs it may also be a+// caption/body paragraph Go omits. Reported separately (not labeled PASS)+// and not counted as a// content failure; classified as go_intentional/go_bug via known_diffs.json.// - table PDF, gridSim<100% OR structSim<100% (or non-table textSim<100%) ->// FAIL if not exempted; INTENTIONAL if a go_intentional rule names this@@ -192,9 +194,10 @@ func TestPipelineParity(t *testing.T) {// reconstructed grid content (cell text) rather than the rendered HTML.// Grid content parity (gridSim + structSim) isolates Go's GroupCells +// FillCellTextFromBoxes assembly. When gridSim AND structSim are 100%- // but textSim<100%, the residual gap is NON-CELL-TEXT: a <caption> Go- // omits, an interleaved body paragraph Go drops, and/or HTML tag/format- // differences — none of which is a table-cell-assembly bug.+ // but textSim<100%, the residual gap is NON-CELL-TEXT: HTML-emitter+ // formatting (header <th> vs <td>, whitespace/newline) — none of which is+ // a table-cell-assembly bug. (A caption/body paragraph Go omits is also+ // possible in other PDFs.)pyRows, pyHasTables := loadPythonTables(t, filepath.Join(tablesDir, name+".json"))goRows := goTableRows(result)goHasTables := len(goRows) > 0@@ -220,19 +223,19 @@ func TestPipelineParity(t *testing.T) {structureSim, shapeDetail := gridStructureSimilarity(goRows, pyRows)// Full content parity = identical cell text AND identical// structure. Only then is the residual textSim gap purely the- // non-cell-text layer (caption/body/format outside table cells),+ // non-cell-text layer (HTML-emitter formatting outside table cells, e.g.// reported as NONCELL_TEXT.contentMatch := gridSim >= 100.0 && structureSim >= 100.0if contentMatch {// Cells + structure match Python, but the full text still- // diverges — the gap is OUTSIDE table cells: caption text- // Go omits, an interleaved body paragraph Go drops, and/or- // HTML tag/format differences. Label it noncell-text (not- // "html-divergent", which wrongly implies content matches)- // so the divergence is named and not hidden.+ // diverges — the gap is OUTSIDE table cells: emitter+ // markup/whitespace OR an omitted caption/body paragraph+ // (not a table-cell-assembly bug). Label it noncell-text+ // (not "html-divergent", which wrongly implies content+ // matches) so the divergence is named and not hidden.noncellText++status = "NONCELL_TEXT"- detail = fmt.Sprintf("gridSim=%.1f%% structSim=%.1f%% (%s) textSim=%.1f%% (non-cell-text divergence: caption/body/format outside table cells)", gridSim, structureSim, shapeDetail, sim)+ detail = fmt.Sprintf("gridSim=%.1f%% structSim=%.1f%% (%s) textSim=%.1f%% (non-cell-text divergence outside table-cell content: emitter markup/whitespace or omitted caption/body text)", gridSim, structureSim, shapeDetail, sim)} else if intentionalPDF(name) {// go_intentional rule covers this PDF: the divergence (cell// content and/or structure) is a documented, deliberate onediff --git a/internal/deepdoc/parser/pdf/table/group_boxes.go b/internal/deepdoc/parser/pdf/table/group_boxes.goindex 65ceec9e99..5a794d6c65 100644--- a/internal/deepdoc/parser/pdf/table/group_boxes.go+++ b/internal/deepdoc/parser/pdf/table/group_boxes.go@@ -185,6 +185,26 @@ func cellPosFromBox(b pdf.TextBox) (x0, y0, x1, y1 float64, label string) {}} else if b.SP > 0 {label = "table spanning cell"+ // A pure-SP box (H==0) still carries the span's full extent in+ // HLeft/HRight/HTop/HBott, copied together from the TSR spanning cell+ // by AnnotateTableBoxes (PR #18707). Rebuild the cell from those bounds+ // so CalSpans covers every column/row the span crosses — matching+ // Python's _annotate_table_boxes. Without this, the span cell falls+ // back to the box's own narrow text bounds, under-covers by one+ // column, and the header row keeps an extra empty cell (real_pdfs/1.pdf+ // row0 = 4 vs Python's 3; see known_diffs rule+ // table-1pdf-colspan-assembly-loss, resolved).+ //+ // HLeft/HRight (and HTop/HBott) are set together, so treat an axis as+ // propagated whenever either edge is set. This preserves a real edge at+ // coordinate 0 (a nonzero check would mistake it for "unset" and fall+ // back to the box's own bounds, dropping the span's left/top edge).+ if b.HLeft != 0 || b.HRight != 0 {+ x0, x1 = b.HLeft, b.HRight+ }+ if b.HTop != 0 || b.HBott != 0 {+ y0, y1 = b.HTop, b.HBott+ }}return}diff --git a/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.go b/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.gonew file mode 100644index 0000000000..88e879a426--- /dev/null+++ b/internal/deepdoc/parser/pdf/table/span_row0_cellcount_test.go@@ -0,0 +1,104 @@+package table++// These tests expose the row0 residual of real_pdfs/1.pdf tracked as+// table-1pdf-colspan-assembly-loss (known_diffs.json, status resolved):+// Go's header row keeps one extra empty cell (4 vs Python's 3) because+// cellPosFromBox ignores the span bounds (HLeft/HRight/HTop/HBott) copied onto+// a pure-SP box (H==0, SP>0) by AnnotateTableBoxes (PR #18707). CalSpans then+// covers only the columns whose centers fall inside the box's own narrow+// bounds, leaving the trailing covered column un-marked and alive as a 4th cell.+//+// This is the GroupBoxesByRC/CalSpans-level regression test CodeRabbit asked+// for in PR #18707 (the shipped tests only cover AnnotateTableBoxes field+// propagation, not the colspan geometry). It is a default unit test (no cgo+// tag) so it runs in `go test ./...` and CI.++import (+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// TestGroupBoxesByRCSpanHeaderRowCellCount drives the real assembly path+// (GroupBoxesByRC -> CalSpans -> MarkCoveredCells -> RowsToStrings) with a+// synthetic 1.pdf-like header: 8 logical columns, a TSR "table spanning cell"+// covering 6 of them. Python's golden collapses the row to 3 cells (the 6-col+// span + 2 trailing single columns). Go must match.+func TestGroupBoxesByRCSpanHeaderRowCellCount(t *testing.T) {+ const nCols = 8+ boxes := make([]pdf.TextBox, 0, nCols)++ // Spanning header text box at col 0. Its OCR box is narrow [0,450], but the+ // TSR span component reaches the full extent [0,600]; AnnotateTableBoxes+ // copies that onto the box as HLeft/HRight/HTop/HBott. For a pure-SP box+ // (H==0, SP>0) cellPosFromBox must use those bounds, otherwise the span+ // cell falls back to [0,450] and covers only 5 columns.+ //+ // R starts at 1 (not 0): GroupBoxesByRC falls back to YX grouping when+ // maxR<=0, which would skip cellPosFromBox entirely; a real table has+ // multiple rows so maxR>0 and the R/C path runs.+ boxes = append(boxes, pdf.TextBox{+ X0: 0, X1: 450, Top: 0, Bottom: 20,+ Text: "HEADER", R: 1, C: 0,+ SP: 1, HLeft: 0, HRight: 600, HTop: 0, HBott: 20,+ })++ // Remaining columns 1..7 as header/content cells, defining column geometry.+ trailing := []string{"c1", "c2", "c3", "c4", "c5", "c6", "c7"}+ for c := 1; c < nCols; c++ {+ boxes = append(boxes, pdf.TextBox{+ X0: float64(c) * 100, X1: float64(c)*100 + 100, Top: 0, Bottom: 20,+ Text: trailing[c-1], R: 1, C: c,+ H: 1,+ })+ }+ // One body row (R=2) so the table has more than one row; its cells are+ // irrelevant to the header-row assertion.+ for c := 0; c < nCols; c++ {+ boxes = append(boxes, pdf.TextBox{+ X0: float64(c) * 100, X1: float64(c)*100 + 100, Top: 30, Bottom: 50,+ Text: "b", R: 2, C: c,+ })+ }++ rows := GroupBoxesByRC(boxes)+ if len(rows) == 0 {+ t.Fatal("GroupBoxesByRC returned an empty grid")+ }+ spanInfo, covered := CalSpans(rows)+ MarkCoveredCells(rows, covered)++ row0 := RowsToStrings(rows)[0]+ if len(row0) != 3 {+ t.Errorf("header row should collapse to 3 cells (6-col span + 2 trailing), "++ "got %d: %v (spanInfo=%v)", len(row0), row0, spanInfo)+ }+ // Sanity: the span must actually be colspan=6, not a narrower span that+ // happens to leave 3 cells via a different arrangement.+ if cs, ok := spanInfo[[2]int{0, 0}]; !ok || cs[0] != 6 {+ t.Errorf("expected colspan=6 on the span origin (0,0), got %v (ok=%v)", cs, ok)+ }+}++// TestCellPosFromBoxSpanUsesSpanBounds pins the root cause directly: a pure-SP+// box (H==0, SP>0) with propagated span bounds must rebuild its cell from those+// bounds, not from the box's own narrow text bounds. HLeft/HRight (and+// HTop/HBott) are set together by AnnotateTableBoxes, so an axis with a zero+// edge is still propagated when the opposite edge is set — a real span edge at+// coordinate 0 must not be mistaken for "unset". The synthetic box uses a+// nonzero text box (X0=200) and a zero left/top propagated edge to lock this in.+func TestCellPosFromBoxSpanUsesSpanBounds(t *testing.T) {+ b := pdf.TextBox{+ X0: 200, X1: 300, Top: 0, Bottom: 20,+ H: 0, SP: 1, HLeft: 0, HRight: 600, HTop: 0, HBott: 25,+ }+ x0, y0, x1, y1, _ := cellPosFromBox(b)+ if x0 != 0 || x1 != 600 {+ t.Errorf("pure-SP box must use propagated span bounds even when HLeft=0 "++ "(span edge at coord 0), got x0=%v x1=%v", x0, x1)+ }+ if y0 != 0 || y1 != 25 {+ t.Errorf("pure-SP box must use propagated span bounds even when HTop=0 "++ "(span edge at coord 0), got y0=%v y1=%v", y0, y1)+ }+}diff --git a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.jsonindex 723e322055..d7b89473f6 100644--- a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json+++ b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json@@ -232,9 +232,9 @@"cellPosFromBox"],"owner_fix_side": "go",- "status": "resolved_partial",- "resolution": "AnnotateTableBoxes (internal/deepdoc/parser/pdf/table/table_layout.go) now (a) collects TSR 'spanning cell' components into its spans slice and sets box.SP = idx+1 on overlapping boxes (matching Python _table_transformer_job / pdf_parser.py:518-554), and (b) copies the span cell's bbox onto the box as H_top/H_bott/H_left/H_right (matching Python _annotate_table_boxes pdf_parser.py:632-635). GroupBoxesByRC then rebuilds the span cell from these full extents (cellPosFromBox uses HLeft/HRight when set) and CalSpans covers every column the span crosses. Before the fix, box.SP stayed 0 AND HLeft/HRight were not propagated for SP boxes, so the span cell fell back to the box's own narrow bounds and CalSpans covered only 5 columns -> Go emitted colspan=5 where Python emits colspan=6. After the fix Go emits colspan=6 (verified end-to-end via Test1PdfColspanEndToEnd on the live pipeline, and via TestPipelineParity replay: 1.pdf row0 went from 8 independent <th> to 4 cells, textSim 95.7% -> 97.4%). Regression: TestAnnotateTableBoxesSpanCarriesBoxCoords, TestAnnotateTableBoxesPropagatesSpan, TestGroupCellsInjectsSpanning, Test1PdfColspanEndToEnd.",- "tracking": "Residual: 1.pdf's header row still shows row0 = 4 cells in Go vs 3 cells in Python's golden (structSim 87.5%, the LOCKED grid check still fails). The colspan=6 is correctly emitted, but Go keeps one extra empty spanning cell in row0 (4 vs 3), so the per-row column count is not yet byte-identical to Python. Left as resolved_partial: the colspan-assembly bug is fixed; the 1-cell row0 residual needs a follow-up on how GroupBoxesByRC folds the covered trailing column before 1.pdf can reach structSim=100% / aligned.",+ "status": "resolved",+ "resolution": "AnnotateTableBoxes (internal/deepdoc/parser/pdf/table/table_layout.go) (a) collects TSR 'spanning cell' components into its spans slice and sets box.SP = idx+1 on overlapping boxes (matching Python _table_transformer_job / pdf_parser.py:518-554), and (b) copies the span cell's bbox onto the box as H_top/H_bott/H_left/H_right (matching Python _annotate_table_boxes pdf_parser.py:632-635). cellPosFromBox (internal/deepdoc/parser/pdf/table/group_boxes.go) now ALSO applies HLeft/HRight/HTop/HBott for a pure-SP box (H==0, SP>0), so GroupBoxesByRC rebuilds the span cell from the full span extents and CalSpans covers every column the span crosses. Before the fix, box.SP stayed 0 AND the SP-only branch of cellPosFromBox ignored the propagated HLeft/HRight, so the span cell fell back to the box's own narrow bounds and CalSpans covered only 5 columns -> Go emitted colspan=5 where Python emits colspan=6 AND the header row kept 4 cells where Python collapses to 3. After the fix Go emits colspan=6 and row0 collapses to 3 cells (verified end-to-end via Test1PdfColspanEndToEnd on the live pipeline, and via TestPipelineParity replay: 1.pdf row0 went from 8 independent <th> to 3 cells, textSim 95.7% -> 97.4%, structSim 87.5% -> 100%). Regression: TestAnnotateTableBoxesSpanCarriesBoxCoords, TestAnnotateTableBoxesPropagatesSpan, TestGroupCellsInjectsSpanning, Test1PdfColspanEndToEnd, TestGroupBoxesByRCSpanHeaderRowCellCount, TestCellPosFromBoxSpanUsesSpanBounds.",+ "tracking": "RESOLVED: the row0 = 4-vs-3 residual (structSim 87.5%) was caused by cellPosFromBox's SP-only branch ignoring the span bounds propagated by AnnotateTableBoxes, so the trailing covered column's center fell outside the span cell's narrow X range and was not marked covered. Applying the span bounds in the SP branch makes CalSpans cover all 6 columns, so row0 now collapses to 3 cells (matching Python) and structSim reaches 100% / aligned. The original per-edge `!= 0` sentinel (coordinate 0 mistaken for 'unset') was replaced by a per-axis check (propagate both edges whenever either is set), so a span whose real edge is exactly 0 is now preserved; HLeft/HRight and HTop/HBott are copied together by AnnotateTableBoxes, so the invariant holds.","reason": "1.pdf's merged header ('液化石油气储罐(区)(总容积V,m²)') spans 6 of the 8 columns. Python's golden carries colspan=6 on that header cell (its row0 collapses to 3 cells: 1 content + 2 empties around the 6-span). Go USED TO DROP the span: AnnotateTableBoxes only propagated the spanning cell's bbox (H_left/H_right/H_top/H_bott) onto header boxes, not onto 'spanning cell' (SP) boxes, and did not even collect the spans slice, so box.SP stayed 0 and the rebuilt grid (GroupBoxesByRC) lost the span; CalSpans then covered only the box's own narrow bounds -> colspan=5. TSR input is identical on both sides, so this was purely a Go ASSEMBLY bug, the OPPOSITE direction of rule table-span-follow-tsr-geometry (there Go HAS the colspan and Python omits it; here Python HAS it and Go dropped it). Direction matters for classification: this is a go_bug (Go was wrong, now fixed), NOT a go_intentional. Exposed by TestPipelineParity replay (1.pdf: LOCKED gridSim=100% structSim=87.5% regressed below 100%) and the end-to-end Test1PdfColspanEndToEnd."},{