Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(deepdoc): prevent cross-page table over-merge from page-local Y coordinates (#18688)

Jack290a8d8f7907
diff --git a/internal/deepdoc/parser/pdf/layout/boxes_sections.go b/internal/deepdoc/parser/pdf/layout/boxes_sections.go
index a3424030c7..5521e82559 100644
--- a/internal/deepdoc/parser/pdf/layout/boxes_sections.go
+++ b/internal/deepdoc/parser/pdf/layout/boxes_sections.go
@@ -57,18 +57,33 @@ func BoxesToSections(boxes []pdf.TextBox, pageHeights map[int]float64) []pdf.Sec
if t == "" {
continue
}
- toPage, bottom := ResolvePageSpan(b.PageNumber, b.Bottom, pageHeights)
var posTag string
var pageNums []int
- if b.PageNumber == toPage {
- posTag = util.FormatPositionTag(b.PageNumber, b.X0, b.X1, b.Top, bottom)
- pageNums = []int{b.PageNumber}
+ var bottom float64
+ if len(b.Pages) > 0 {
+ // Box carries an explicit page span (e.g. a cross-page merged
+ // table). Use it directly instead of inferring from geometry, so
+ // the section records every page the box occupies.
+ pageNums = b.Pages
+ bottom = b.Bottom
+ if len(pageNums) == 1 {
+ posTag = util.FormatPositionTag(pageNums[0], b.X0, b.X1, b.Top, bottom)
+ } else {
+ posTag = util.FormatPositionTagRange(pageNums[0], pageNums[len(pageNums)-1], b.X0, b.X1, b.Top, bottom)
+ }
} else {
- posTag = util.FormatPositionTagRange(b.PageNumber, toPage, b.X0, b.X1, b.Top, bottom)
- pageNums = make([]int, 0, toPage-b.PageNumber+1)
- for p := b.PageNumber; p <= toPage; p++ {
- pageNums = append(pageNums, p)
+ toPage, resolved := ResolvePageSpan(b.PageNumber, b.Bottom, pageHeights)
+ bottom = resolved
+ if b.PageNumber == toPage {
+ posTag = util.FormatPositionTag(b.PageNumber, b.X0, b.X1, b.Top, bottom)
+ pageNums = []int{b.PageNumber}
+ } else {
+ posTag = util.FormatPositionTagRange(b.PageNumber, toPage, b.X0, b.X1, b.Top, bottom)
+ pageNums = make([]int, 0, toPage-b.PageNumber+1)
+ for p := b.PageNumber; p <= toPage; p++ {
+ pageNums = append(pageNums, p)
+ }
}
}
sections = append(sections, pdf.Section{
diff --git a/internal/deepdoc/parser/pdf/parser.go b/internal/deepdoc/parser/pdf/parser.go
index 38bc2e969c..4581fe9f72 100644
--- a/internal/deepdoc/parser/pdf/parser.go
+++ b/internal/deepdoc/parser/pdf/parser.go
@@ -537,7 +537,7 @@ func (p *Parser) buildLayout(ctx context.Context,
}
if len(result.Tables) > 0 {
- result.Tables = tbl.MergeTablesAcrossPages(result.Tables, nil)
+ result.Tables = tbl.MergeTablesAcrossPages(result.Tables, medianHeights, result.PageHeight)
}
boxes = tbl.ExtractTableAndReplace(boxes, result.Tables)
diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_caption_icbccs_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_caption_icbccs_test.go
new file mode 100644
index 0000000000..325add4276
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/pipeline_parity_caption_icbccs_test.go
@@ -0,0 +1,114 @@
+//go:build cgo && manual
+
+package pdf
+
+import (
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/deepdoc/parser/pdf/tool"
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+ util "ragflow/internal/deepdoc/parser/pdf/util"
+)
+
+// replayPipelineTextVariant runs Go's replay pipeline for a custom dataset
+// variant (e.g. "ocr_real"), reading Python's chars/DLA/TSR/OCR from the
+// variant's data root (BATCH_PARITY_DATA_ROOT). It mirrors the parity harness
+// setup for non-default datasets and returns Go's assembled section text,
+// Python's text golden (metadata stripped), and the Go sections for
+// inspection.
+func replayPipelineTextVariant(t *testing.T, variant, name string) (goText, pyText string, sections []pdf.Section) {
+ t.Helper()
+ dirs := tool.ParityDirsFor(variant)
+ engine, err := tool.LoadPythonChars(filepath.Join(dirs.Charspy, name+".json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if v := engine.IsEnglish(); v != nil && *v {
+ engine.ClearChars()
+ } else if pages, _ := engine.PageCount(); util.DetectEnglish(engine.PageChars(), pages, nil) {
+ engine.ClearChars()
+ }
+ RegisterReplayTableBuilder()
+ cfg := pdf.DefaultParserConfig()
+ cfg.SortByTop = true
+ analyzer := NewPythonIntermediateDocAnalyzer(name, dirs.DLA, dirs.TSRRaw, dirs.OCR, engine.PageDims())
+ p := NewParser(cfg)
+ result, err := p.ParseRaw(t.Context(), engine, analyzer)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var b strings.Builder
+ for _, s := range result.Sections {
+ b.WriteString(s.Text)
+ b.WriteByte('\n')
+ }
+ goText = b.String()
+ pyData, err := os.ReadFile(filepath.Join(dirs.Text, name+".txt"))
+ if err != nil {
+ t.Fatalf("read Python text golden: %v", err)
+ }
+ pyText = tool.StripMeta(string(pyData))
+ return goText, pyText, result.Sections
+}
+
+// TestPipelineParityIcbccsCaptionEmitted is the TDD guard for the known_diffs
+// rule noncell-icbccs-caption-dropped (icbccs deployment.pdf, ocr_real).
+//
+// The caption text (请求参数 x2, 请求参数枚举值 x1) IS present in the ocr_real
+// replay intermediates: the OCR dump has the boxes, and DLA replay types them
+// as 'table caption' via AnnotateBoxLayouts. The original gap was a REAL Go bug
+// in MergeCaptions.findTables (table/merge_captions.go): a narrow caption
+// directly above a much wider table was rejected because dx² exceeded
+// maxCaptionGap, so the caption was silently dropped. The fix adds
+// maxCaptionVGap so vertically-adjacent captions attach regardless of
+// horizontal offset. This test is now a real assertion: it fails if Go does
+// not emit <caption>请求参数</caption> (the fixed behavior). See known_diffs
+// rule noncell-icbccs-caption-dropped (go_bug, resolved).
+//
+// Requires BATCH_PARITY_DATA_ROOT (the shared ocr_real dump dir); skips
+// otherwise. Run via: build.sh --test-manual with BATCH_PARITY_VARIANT=ocr_real
+// and BATCH_PARITY_DATA_ROOT set.
+func TestPipelineParityIcbccsCaptionEmitted(t *testing.T) {
+ const name = "icbccs deployment.pdf"
+ if common.GetEnv(common.EnvBatchParityDataRoot) == "" {
+ t.Skip("BATCH_PARITY_DATA_ROOT not set; ocr_real dumps unavailable")
+ }
+ t.Setenv(common.EnvBatchParityVariant, "ocr_real")
+ goText, pyText, sections := replayPipelineTextVariant(t, "ocr_real", name)
+
+ // Diagnostic: surface the layout type Go assigns to the caption text, so
+ // the missing caption signal is identifiable.
+ for _, s := range sections {
+ if strings.Contains(s.Text, "请求参数") {
+ t.Logf("DIAG section LayoutType=%q text=%q", s.LayoutType, strings.TrimSpace(s.Text))
+ }
+ }
+
+ captionRe := regexp.MustCompile(`(?is)<caption>(.*?)</caption>`)
+ caps := captionRe.FindAllStringSubmatch(goText, -1)
+ if len(caps) == 0 {
+ // No <caption> in Go output. The known_diffs rule
+ // noncell-icbccs-caption-dropped was a go_bug (resolved by the
+ // maxCaptionVGap fix in MergeCaptions.findTables). A missing caption
+ // here is a REAL REGRESSION, not the old environmental gap — the
+ // caption text IS in the ocr_real replay intermediates. Fail so the
+ // guard actually guards.
+ t.Errorf("REGRESSION (noncell-icbccs-caption-dropped): Go emits no <caption> for icbccs; the caption text is present in the ocr_real replay intermediates (OCR dump + DLA 'table caption' regions), so this means MergeCaptions dropped it. Python golden has caption: %v", strings.Contains(pyText, "请求参数"))
+ }
+ var merged strings.Builder
+ for _, c := range caps {
+ merged.WriteString(strings.TrimSpace(c[1]))
+ merged.WriteByte(' ')
+ }
+ if !strings.Contains(merged.String(), "请求参数") {
+ t.Errorf("Go emitted <caption> but text missing 请求参数 (real source caption); got %q", merged.String())
+ }
+ if !strings.Contains(pyText, "请求参数") {
+ t.Errorf("test setup error: Python golden missing 请求参数")
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/table/crosspage_cleanup_test.go b/internal/deepdoc/parser/pdf/table/crosspage_cleanup_test.go
index f3dd3556be..bfbb8be4a4 100644
--- a/internal/deepdoc/parser/pdf/table/crosspage_cleanup_test.go
+++ b/internal/deepdoc/parser/pdf/table/crosspage_cleanup_test.go
@@ -33,7 +33,7 @@ func TestMergeTablesAcrossPages_PostMergeCleanup(t *testing.T) {
},
}
- out := MergeTablesAcrossPages([]pdf.TableItem{anchor, continuation}, nil)
+ out := MergeTablesAcrossPages([]pdf.TableItem{anchor, continuation}, nil, map[int]float64{0: 190})
if len(out) != 1 {
t.Fatalf("expected 1 merged table, got %d", len(out))
}
diff --git a/internal/deepdoc/parser/pdf/table/merge_captions.go b/internal/deepdoc/parser/pdf/table/merge_captions.go
index cafe8c2e78..aba8b986f3 100644
--- a/internal/deepdoc/parser/pdf/table/merge_captions.go
+++ b/internal/deepdoc/parser/pdf/table/merge_captions.go
@@ -18,6 +18,23 @@ type captionText struct {
text string
}
+// captionSep returns the separator inserted before a caption whose text is
+// being appended to an existing caption string. Python's __html_table
+// (deepdoc/vision/table_structure_recognizer.py construct_table) adds a space
+// between captions only for ENGLISH documents; for non-English (e.g. CJK) it
+// concatenates directly. MergeCaptions is not threaded the document language,
+// so we approximate per caption: a caption containing ASCII letters is treated
+// as English and gets a space, matching Python for the dominant cases without
+// changing the function signature.
+func captionSep(text string) string {
+ for _, r := range text {
+ if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') {
+ return " "
+ }
+ }
+ return ""
+}
+
func MergeCaptions(sections []pdf.Section, figures []pdf.Section) []pdf.Section {
captions := make([]int, 0, 4)
// Group caption texts by the target section index they attach to, so
@@ -125,6 +142,16 @@ func findNearestParent(captionIdx int, caption pdf.Section, sections []pdf.Secti
}
const maxCaptionGap = 40000.0 // PDF points (~7cm) — beyond this, don't attach.
+ // maxCaptionVGap is the vertical band within which a caption attaches to a
+ // table regardless of its horizontal offset. A narrow caption (e.g. a short
+ // Chinese label) sitting directly above a much wider table has a large dx to
+ // the table's center; dx² alone can exceed maxCaptionGap and wrongly reject
+ // the match, dropping a legitimate caption. Vertical adjacency (small gapY)
+ // is the primary signal that the caption belongs to that table; dx only
+ // discriminates between candidate tables, which findTables already resolves
+ // via min-distance. Keep this in line with the vertical tolerance implied by
+ // maxCaptionGap when dx≈0 (~200pt).
+ const maxCaptionVGap = 200.0
if captionType == pdf.LayoutTypeFigure && len(figures) > 0 {
idx, dist := find(figures, -1) // figures don't contain the caption itself
if idx >= 0 && dist < maxCaptionGap {
@@ -143,8 +170,12 @@ func findNearestParent(captionIdx int, caption pdf.Section, sections []pdf.Secti
}
}
if captionType == pdf.LayoutTypeTable {
- idx, dist := findTables(sections, caption)
- if idx >= 0 && dist < maxCaptionGap {
+ idx, dist, gapY := findTables(sections, caption)
+ // Attach a vertically-adjacent caption even when it is horizontally
+ // offset from the (often much wider) table's center. See
+ // maxCaptionVGap for the rationale (narrow captions above wide
+ // tables, e.g. icbccs '请求参数').
+ if idx >= 0 && (dist < maxCaptionGap || gapY <= maxCaptionVGap) {
return idx
}
}
@@ -165,24 +196,28 @@ func findNearestParent(captionIdx int, caption pdf.Section, sections []pdf.Secti
// Revenue'). A caption just above the table (gap to its top) or just below
// (gap to its bottom) has a small edge distance and attaches; one that
// vertically overlaps the table has gap 0.
-func findTables(sections []pdf.Section, caption pdf.Section) (int, float64) {
+func findTables(sections []pdf.Section, caption pdf.Section) (int, float64, float64) {
bestIdx := -1
bestDist := 1e9
+ bestGapY := 0.0
if len(caption.Positions) == 0 {
- return bestIdx, bestDist
+ return bestIdx, bestDist, bestGapY
}
cp := caption.Positions[0]
ccx := (cp.Left + cp.Right) / 2
- // Page-scope guard: a caption on a DIFFERENT page may attach only when it
- // VERTICALLY OVERLAPS the table's Y band (gapY==0). That is the cross-page
- // table continuation case — a merged cross-page table's Positions[0] keeps
- // only the FIRST page's geometry (pages=[0]), so a caption on a later page
- // of the same table lands inside that band in page-local coordinates (13's
- // page-2 caption 'Table: Monthly financial summary FY2024' at page-local
- // Y=154 sits inside the page-0 band 98-777). A caption clearly ABOVE or
- // BELOW a table on a different page (gapY>0) is a false attachment:
- // page-local Y ranges repeat on every page, so such a caption only
- // "happens" to line up with an unrelated single-page table.
+ // Page-scope guard: a caption attaches to a table only on its OWN page (a
+ // table whose page set includes the caption's page — a genuine cross-page
+ // merged table carries multiple Position entries, one per spanned page, so
+ // a caption on any of those pages is on-page). A caption on a DIFFERENT
+ // page may attach ONLY to such a cross-page merged table when it vertically
+ // overlaps the table's Y band (gapY==0) — the cross-page continuation case
+ // (13's later-page caption 'Table: Monthly financial summary FY2024' lands
+ // inside the merged table's Y band in page-local coordinates). A caption on
+ // a different page that merely repeats a single-page table's page-local Y
+ // (page-local Y ranges repeat every page) is a FALSE attachment and must be
+ // rejected: this was the icbccs bug where a page-3 caption wrongly attached
+ // to a page-5 table and was concatenated into the <caption>, duplicating it
+ // ("请求参数 请求参数").
capKnown := len(cp.PageNumbers) > 0
capPage := 0
if capKnown {
@@ -203,17 +238,30 @@ func findTables(sections []pdf.Section, caption pdf.Section) (int, float64) {
gapY = cp.Top - tp.Bottom
}
if capKnown {
- onPage := len(tp.PageNumbers) == 0 // unknown table page → don't restrict
- for _, p := range tp.PageNumbers {
- if p == capPage {
- onPage = true
+ // Page-scope guard: a caption attaches to a table only on a page
+ // the table actually occupies. A genuine cross-page MERGED table
+ // carries every spanned page in its Position.PageNumbers (set by
+ // tableRegionBox/createTableBoxFromItem from the merged TableItem's
+ // Positions), so a caption on any of those pages is on-page and
+ // attaches. A caption on a DIFFERENT page that merely repeats a
+ // single-page table's page-local Y (page-local Y ranges repeat
+ // every page) is a FALSE attachment and is rejected — this was the
+ // icbccs bug where a page-3 caption wrongly attached to a page-5
+ // table and was concatenated into the <caption>, duplicating it
+ // ("请求参数 请求参数").
+ onPage := false
+ for _, pp := range t.Positions {
+ for _, pn := range pp.PageNumbers {
+ if pn == capPage {
+ onPage = true
+ break
+ }
+ }
+ if onPage {
break
}
}
- // Different page: reject unless vertically overlapping (cross-page
- // table continuation). A caption above/below on another page is a
- // false attachment (see function comment).
- if !onPage && gapY > 0 {
+ if !onPage {
continue
}
}
@@ -225,10 +273,11 @@ func findTables(sections []pdf.Section, caption pdf.Section) (int, float64) {
dist := gapY*gapY + dx*dx
if dist < bestDist {
bestDist = dist
+ bestGapY = gapY
bestIdx = i
}
}
- return bestIdx, bestDist
+ return bestIdx, bestDist, bestGapY
}
// appendRawCaptions concatenates caption texts onto a non-table section
@@ -243,7 +292,7 @@ func appendRawCaptions(target *pdf.Section, captions []string) {
continue
}
if b.Len() > 0 {
- b.WriteByte(' ')
+ b.WriteString(captionSep(c))
}
b.WriteString(t)
}
@@ -273,7 +322,7 @@ func injectCaption(table *pdf.Section, captions []string) {
continue
}
if b.Len() > 0 {
- b.WriteByte(' ')
+ b.WriteString(captionSep(c))
}
b.WriteString(html.EscapeString(t))
}
diff --git a/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.go b/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.go
index 072f4f4ef6..cc4f6966df 100644
--- a/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.go
+++ b/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.go
@@ -151,11 +151,16 @@ func TestMergeCaptions_ReadingOrderByTop(t *testing.T) {
func TestMergeCaptions_TallTableCaptionNearEdgeAttaches(t *testing.T) {
table := pdf.Section{
Text: "<table><tr><th >Month</th><th >Revenue</th></tr></table>", LayoutType: pdf.LayoutTypeTable,
- // Cross-page merged table: Positions[0] keeps ONLY the first page's
- // geometry (pages=[0], the page-0 band), exactly like 13's real merged
- // table. A caption on a LATER page of the same table lands inside this
- // band in page-local coordinates (gapY=0) and must still attach.
- Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 82, Right: 513, Top: 98, Bottom: 777}},
+ // Cross-page merged table: table_merge.go appends one Position entry
+ // PER spanned page, so a real merged table carries MULTIPLE Position
+ // entries (here pages 0 and 2), each with that page's local geometry.
+ // A caption on a LATER page of the same table lands inside that page's
+ // band in page-local coordinates (gapY=0) and must still attach — this
+ // is the 13/14 cross-page caption continuation case.
+ Positions: []pdf.Position{
+ {PageNumbers: []int{0}, Left: 82, Right: 513, Top: 98, Bottom: 777},
+ {PageNumbers: []int{2}, Left: 82, Right: 513, Top: 98, Bottom: 777},
+ },
}
for _, c := range []struct {
name string
@@ -241,3 +246,37 @@ func TestMergeCaptions_FigureCaptionRawText(t *testing.T) {
t.Errorf("figure section must NOT be wrapped in a <caption> element (table-specific): %q", got.Text)
}
}
+
+// TestMergeCaptions_NarrowCaptionAttachesWideTable locks the fix for the
+// icbccs '请求参数' caption: a NARROW caption (short label, small horizontal
+// extent) sitting directly above a much WIDER table has a large horizontal
+// offset dx to the table's center. Before the fix, findTables rejected the
+// match because dx² alone exceeded maxCaptionGap, so MergeCaptions dropped
+// the caption (no table target) — losing the text. Vertical adjacency
+// (small gapY) is the real signal that the caption belongs to that table, so
+// the fix attaches it when gapY <= maxCaptionVGap regardless of dx.
+//
+// Geometry mirrors icbccs: caption x∈[28,100] (center 64) above a full-width
+// table x∈[30,565] (center ~297); dx≈233 → dx²≈54k > maxCaptionGap(40k), but
+// gapY=19 ≤ maxCaptionVGap(200) → must attach.
+func TestMergeCaptions_NarrowCaptionAttachesWideTable(t *testing.T) {
+ sections := []pdf.Section{
+ {Text: "<table><tr><td >名称</td><td >位置</td></tr></table>", LayoutType: pdf.LayoutTypeTable,
+ Positions: []pdf.Position{{PageNumbers: []int{2}, Left: 30, Right: 565, Top: 197, Bottom: 314}}},
+ {Text: "请求参数", LayoutType: pdf.DLALabelTableCaption,
+ Positions: []pdf.Position{{PageNumbers: []int{2}, Left: 28, Right: 100, Top: 157, Bottom: 178}}},
+ }
+ figures := pdf.CollectFigures(sections)
+ result := MergeCaptions(sections, figures)
+
+ if len(result) != 1 {
+ t.Fatalf("expected 1 section (table with caption, standalone caption removed), got %d: %v", len(result), textsOf(result))
+ }
+ got := result[0].Text
+ if !strings.Contains(got, "<caption>请求参数</caption>") {
+ t.Errorf("narrow caption not injected as <caption>; got %q", got)
+ }
+ if strings.Count(got, "请求参数") != 1 {
+ t.Errorf("caption text should appear exactly once (inside <caption>), got %q", got)
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/table/table_construct.go b/internal/deepdoc/parser/pdf/table/table_construct.go
index 861e59a4f5..78626effa1 100644
--- a/internal/deepdoc/parser/pdf/table/table_construct.go
+++ b/internal/deepdoc/parser/pdf/table/table_construct.go
@@ -185,6 +185,14 @@ func tableRegionBox(tbl *pdf.TableItem, ref *pdf.TextBox, html string) pdf.TextB
if len(tbl.Positions) > 0 && len(tbl.Positions[0].PageNumbers) > 0 {
pg = tbl.Positions[0].PageNumbers[0]
}
+ // A table merged across consecutive pages (MergeTablesAcrossPages appends
+ // every spanned page to tbl.Positions) must record ALL its pages so the
+ // resulting section's Position spans them — otherwise a caption on a later
+ // page of the same table is wrongly treated as off-page and the
+ // cross-page caption continuation (e.g. 13's 'Table: Monthly financial
+ // summary FY2024') is dropped. PageNumber keeps the anchor page for
+ // single-page consumers (insertion, etc.).
+ pages := mergedTablePages(tbl)
// Use DLA region boundaries when set.
if tbl.RegionLeft != 0 || tbl.RegionRight != 0 || tbl.RegionTop != 0 || tbl.RegionBottom != 0 {
return pdf.TextBox{
@@ -194,6 +202,7 @@ func tableRegionBox(tbl *pdf.TableItem, ref *pdf.TextBox, html string) pdf.TextB
Bottom: tbl.RegionBottom,
Text: html,
PageNumber: pg,
+ Pages: pages,
LayoutType: pdf.LayoutTypeTable,
}
}
@@ -206,10 +215,34 @@ func tableRegionBox(tbl *pdf.TableItem, ref *pdf.TextBox, html string) pdf.TextB
Bottom: bot,
Text: html,
PageNumber: pg,
+ Pages: pages,
LayoutType: pdf.LayoutTypeTable,
}
}
+// mergedTablePages returns the sorted, de-duplicated set of page numbers a
+// table spans, taken from every Position the (possibly cross-page-merged)
+// TableItem carries. A single-page table yields a one-element slice (or nil
+// when it has no positions), so callers can use it to decide whether the box
+// spans multiple pages.
+func mergedTablePages(tbl *pdf.TableItem) []int {
+ if len(tbl.Positions) == 0 {
+ return nil
+ }
+ seen := make(map[int]bool, len(tbl.Positions))
+ pages := make([]int, 0, len(tbl.Positions))
+ for _, p := range tbl.Positions {
+ for _, pn := range p.PageNumbers {
+ if !seen[pn] {
+ seen[pn] = true
+ pages = append(pages, pn)
+ }
+ }
+ }
+ sort.Ints(pages)
+ return pages
+}
+
// minRectangleDistance computes the Euclidean distance between two rectangles.
// Returns 0 when rectangles overlap. Matches Python's min_rectangle_distance
// in insert_table_figures (pdf_parser.py:1609-1626).
diff --git a/internal/deepdoc/parser/pdf/table/table_merge.go b/internal/deepdoc/parser/pdf/table/table_merge.go
index 8a73f9fae2..7793062c69 100644
--- a/internal/deepdoc/parser/pdf/table/table_merge.go
+++ b/internal/deepdoc/parser/pdf/table/table_merge.go
@@ -10,7 +10,13 @@ import (
// MergeTablesAcrossPages merges TableItems on consecutive pages with
// overlapping X and close Y proximity. Matches Python's
// _extract_table_figure table merge (pdf_parser.py:1061-1080).
-func MergeTablesAcrossPages(tables []pdf.TableItem, medianHeights map[int]float64) []pdf.TableItem {
+//
+// pageHeights maps each 0-based page number to its PDF-point page height. It
+// is required to measure the cross-page Y gap in a page-absolute frame: the
+// continuation table's page-local Top must be offset by the anchor page's
+// height, otherwise two tables whose page-local Y merely repeats every page
+// look adjacent and get wrongly merged.
+func MergeTablesAcrossPages(tables []pdf.TableItem, medianHeights, pageHeights map[int]float64) []pdf.TableItem {
if len(tables) <= 1 {
return tables
}
@@ -98,6 +104,18 @@ func MergeTablesAcrossPages(tables []pdf.TableItem, medianHeights map[int]float6
}
}
yDis := (bp.Top + bp.Bottom - anchorBtm - ap.Bottom) / 2
+ // Tables carry page-LOCAL coordinates (each page resets Y to 0),
+ // so comparing bp.Top against anchorBtm as if they shared one
+ // continuous frame underestimates the inter-page gap and wrongly
+ // merges two tables that merely repeat their Y every page. Python
+ // measures _y_dis in a page-absolute frame, so shift the
+ // continuation into the anchor's page-absolute frame by the anchor
+ // page's height when we know it (see rule icbccs-crosspage-table-
+ // overmerge). Without pageHeights the legacy page-local formula is
+ // kept for backward compatibility.
+ if anchorPageH, ok := pageHeights[anchorPg]; ok && anchorPageH > 0 {
+ yDis += anchorPageH
+ }
if yDis > mh*23 {
continue
}
diff --git a/internal/deepdoc/parser/pdf/table/table_merge_test.go b/internal/deepdoc/parser/pdf/table/table_merge_test.go
index d374bca369..3ffacfeef4 100644
--- a/internal/deepdoc/parser/pdf/table/table_merge_test.go
+++ b/internal/deepdoc/parser/pdf/table/table_merge_test.go
@@ -33,7 +33,7 @@ func TestCrossPageTableMerge(t *testing.T) {
tables := []pdf.TableItem{pg0, pg1}
// mergeTablesAcrossPages merges tables on consecutive pages with X overlap.
- merged := MergeTablesAcrossPages(tables, nil)
+ merged := MergeTablesAcrossPages(tables, nil, map[int]float64{0: 820, 1: 820})
if len(merged) != 1 {
t.Fatalf("expected 1 merged table, got %d", len(merged))
}
@@ -62,7 +62,7 @@ func TestMergeTablesAcrossPages_NoOverlap(t *testing.T) {
Cells: []pdf.TSRCell{{Text: "right"}},
},
}
- merged := MergeTablesAcrossPages(tables, nil)
+ merged := MergeTablesAcrossPages(tables, nil, map[int]float64{0: 820, 1: 820})
if len(merged) != 2 {
t.Fatalf("non-overlapping tables: expected 2 tables, got %d", len(merged))
}
@@ -83,7 +83,7 @@ func TestMergeTablesAcrossPages_NonConsecutive(t *testing.T) {
Cells: []pdf.TSRCell{{Text: "page3"}},
},
}
- merged := MergeTablesAcrossPages(tables, nil)
+ merged := MergeTablesAcrossPages(tables, nil, map[int]float64{0: 842, 3: 842})
if len(merged) != 2 {
t.Fatalf("non-consecutive pages: expected 2 tables, got %d", len(merged))
}
@@ -99,7 +99,7 @@ func TestMergeTablesAcrossPages_SingleTable(t *testing.T) {
Cells: []pdf.TSRCell{{Text: "only"}},
},
}
- merged := MergeTablesAcrossPages(tables, nil)
+ merged := MergeTablesAcrossPages(tables, nil, map[int]float64{0: 842})
if len(merged) != 1 {
t.Fatalf("single table: expected 1 table, got %d", len(merged))
}
@@ -118,7 +118,7 @@ func TestMergeTablesAcrossPages_EmptyPositions(t *testing.T) {
Cells: []pdf.TSRCell{{Text: "normal"}},
},
}
- merged := MergeTablesAcrossPages(tables, nil)
+ merged := MergeTablesAcrossPages(tables, nil, map[int]float64{0: 842})
if len(merged) != 2 {
t.Fatalf("empty Positions: expected 2 tables (preserved), got %d", len(merged))
}
@@ -149,7 +149,7 @@ func TestMergeTablesAcrossPages_LargeYGap(t *testing.T) {
Cells: []pdf.TSRCell{{Text: "page1_far"}},
},
}
- merged := MergeTablesAcrossPages(tables, medianHeights)
+ merged := MergeTablesAcrossPages(tables, medianHeights, map[int]float64{0: 842, 1: 842})
if len(merged) != 2 {
t.Fatalf("large Y gap: expected 2 tables (not merged), got %d", len(merged))
}
@@ -170,7 +170,7 @@ func TestMergeTablesAcrossPages_NoMedianHeights(t *testing.T) {
Cells: []pdf.TSRCell{{Text: "page1_near"}},
},
}
- merged := MergeTablesAcrossPages(tables, nil)
+ merged := MergeTablesAcrossPages(tables, nil, map[int]float64{0: 300})
if len(merged) != 1 {
t.Fatalf("no medianHeights: expected 1 merged table, got %d", len(merged))
}
@@ -209,7 +209,7 @@ func TestMergeTablesAcrossPages_RebuildsGridAcrossPages(t *testing.T) {
Grid: pageGrid([][]string{{"e", "f"}, {"g", "h"}}),
}
- merged := MergeTablesAcrossPages([]pdf.TableItem{pg0, pg1}, nil)
+ merged := MergeTablesAcrossPages([]pdf.TableItem{pg0, pg1}, nil, map[int]float64{0: 70})
if len(merged) != 1 {
t.Fatalf("expected 1 merged table, got %d", len(merged))
}
@@ -302,7 +302,7 @@ func TestMergeTablesAcrossPages_JaggedContinuationPreservesRows(t *testing.T) {
Cells: cells(tc.contRows),
}
- merged := MergeTablesAcrossPages([]pdf.TableItem{pg0, pg1}, nil)
+ merged := MergeTablesAcrossPages([]pdf.TableItem{pg0, pg1}, nil, map[int]float64{0: 70})
if len(merged) != 1 {
t.Fatalf("expected 1 merged table, got %d", len(merged))
}
@@ -376,7 +376,7 @@ func TestMergeTablesAcrossPages_MixedColumnCountsPreservesAllRows(t *testing.T)
Grid: gridWithTag(35, 7, "p1"),
}
- merged := MergeTablesAcrossPages([]pdf.TableItem{pg0, pg1}, nil)
+ merged := MergeTablesAcrossPages([]pdf.TableItem{pg0, pg1}, nil, map[int]float64{0: 510})
if len(merged) != 1 {
t.Fatalf("expected 1 merged table, got %d", len(merged))
}
@@ -433,7 +433,7 @@ func TestMergeTablesAcrossPages_ThreePageCumulativeShift(t *testing.T) {
{Positions: []pdf.Position{{PageNumbers: []int{2}, Left: 0, Right: 200, Top: 0, Bottom: 60}}, Scale: 1.0, Grid: pageGrid([][]string{{"i", "j"}, {"k", "l"}})},
}
- merged := MergeTablesAcrossPages(pages, nil)
+ merged := MergeTablesAcrossPages(pages, nil, map[int]float64{0: 70, 1: 70, 2: 70})
if len(merged) != 1 {
t.Fatalf("expected 1 merged table, got %d", len(merged))
}
@@ -491,7 +491,7 @@ func TestMergeTablesAcrossPages_GridlessAnchorUnchanged(t *testing.T) {
Cells: cells([]string{"e", "f", "g", "h"}),
}
- merged := MergeTablesAcrossPages([]pdf.TableItem{pg0, pg1}, nil)
+ merged := MergeTablesAcrossPages([]pdf.TableItem{pg0, pg1}, nil, map[int]float64{0: 70})
if len(merged) != 1 {
t.Fatalf("expected 1 merged table, got %d", len(merged))
}
@@ -520,3 +520,66 @@ func TestMergeTablesAcrossPages_GridlessAnchorUnchanged(t *testing.T) {
t.Errorf("cross-page merge did not combine both pages' positions: %v", pages)
}
}
+
+// TestMergeTablesAcrossPages_PageLocalYRepeatsButSeparatePages is the
+// regression test for the icbccs deployment.pdf over-merge (known_diffs.json
+// rule icbccs-crosspage-table-overmerge). Two API-parameter tables sit on
+// consecutive pages but their page-LOCAL Y coordinates happen to repeat every
+// page (anchor page 4 bottom=172, continuation page 5 local top=262) — the
+// same pattern that, evaluated as if continuous, yields yDis≈99 and wrongly
+// merges. Python keeps them separate because in absolute page-stacked
+// coordinates the gap is ~862pt. This test locks that Go must NOT merge them:
+// the Y proximity gate must be measured in a page-absolute frame using the
+// anchor page's height.
+func TestMergeTablesAcrossPages_PageLocalYRepeatsButSeparatePages(t *testing.T) {
+ // Anchor on page 4: near the top of the page (local bottom=172).
+ anchor := pdf.TableItem{
+ Positions: []pdf.Position{{PageNumbers: []int{4}, Left: 30, Right: 566, Top: 85, Bottom: 172}},
+ Scale: 1.0,
+ Cells: []pdf.TSRCell{{Text: "req_params_p4"}},
+ Caption: "请求参数",
+ }
+ // Continuation on page 5: local top=262 (again near the top of its page).
+ cont := pdf.TableItem{
+ Positions: []pdf.Position{{PageNumbers: []int{5}, Left: 30, Right: 566, Top: 262, Bottom: 350}},
+ Scale: 1.0,
+ Cells: []pdf.TSRCell{{Text: "req_params_p5"}},
+ Caption: "请求参数",
+ }
+ pageHeights := map[int]float64{4: 842, 5: 842} // standard A4 point height
+ // Both with a realistic median char height AND with the nil (mh=10) default.
+ for _, mh := range []map[int]float64{nil, {4: 13, 5: 13}} {
+ merged := MergeTablesAcrossPages([]pdf.TableItem{anchor, cont}, mh, pageHeights)
+ if len(merged) != 2 {
+ t.Fatalf("page-local Y repeats across pages: expected 2 SEPARATE tables (no merge), got %d (over-merge bug)", len(merged))
+ }
+ }
+}
+
+// TestMergeTablesAcrossPages_RealAdjacentAcrossPagesStillMerges locks that a
+// genuine cross-page split — anchor table near the BOTTOM of its page and the
+// continuation near the TOP of the next page — is still merged after the
+// page-absolute Y fix. This prevents the fix from over-correcting and
+// splitting real cross-page tables (e.g. the 13_crosspage_table case).
+func TestMergeTablesAcrossPages_RealAdjacentAcrossPagesStillMerges(t *testing.T) {
+ // Anchor page 4: near the bottom (local bottom=800).
+ anchor := pdf.TableItem{
+ Positions: []pdf.Position{{PageNumbers: []int{4}, Left: 30, Right: 566, Top: 740, Bottom: 800}},
+ Scale: 1.0,
+ Cells: []pdf.TSRCell{{Text: "head_p4"}},
+ }
+ // Continuation page 5: near the top (local top=50).
+ cont := pdf.TableItem{
+ Positions: []pdf.Position{{PageNumbers: []int{5}, Left: 30, Right: 566, Top: 50, Bottom: 110}},
+ Scale: 1.0,
+ Cells: []pdf.TSRCell{{Text: "head_p5"}},
+ }
+ pageHeights := map[int]float64{4: 842, 5: 842}
+ merged := MergeTablesAcrossPages([]pdf.TableItem{anchor, cont}, map[int]float64{4: 13, 5: 13}, pageHeights)
+ if len(merged) != 1 {
+ t.Fatalf("genuine adjacent cross-page split: expected 1 merged table, got %d", len(merged))
+ }
+ if len(merged[0].Positions) != 2 {
+ t.Errorf("merged table should record both pages, got %d positions", len(merged[0].Positions))
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/table/table_post.go b/internal/deepdoc/parser/pdf/table/table_post.go
index cdda3ac538..3149a17d99 100644
--- a/internal/deepdoc/parser/pdf/table/table_post.go
+++ b/internal/deepdoc/parser/pdf/table/table_post.go
@@ -47,6 +47,7 @@ func createTableBoxFromItem(tbl *pdf.TableItem, html string) pdf.TextBox {
Bottom: bottom,
Text: html,
PageNumber: pg,
+ Pages: mergedTablePages(tbl),
LayoutType: pdf.LayoutTypeTable,
}
}
@@ -286,9 +287,14 @@ func ExtractTableAndReplace(boxes []pdf.TextBox, tables []pdf.TableItem) []pdf.T
}
MarkNoMergeTables(boxes, tables)
- tables = MergeTablesAcrossPages(tables, nil)
- // Build replacements AFTER merge so tableIdx refers to the merged slice.
+ // Do NOT re-run MergeTablesAcrossPages here. The caller (Parser.buildLayout,
+ // parser.go:540) already merged tables with page-absolute Y coordinates and
+ // the real per-page medianHeights. Re-merging with nil page metadata would
+ // fall back to the legacy page-local formula and re-merge pairs that the
+ // page-absolute gate correctly rejected (icbccs pages 4-5 over-merge),
+ // silently undoing the fix. Build replacements against the already-merged
+ // slice so tableIdx refers to the correct (merged) table slot.
replacements := buildReplacementsAfterMerge(boxes, tables, removeSet)
if len(replacements) == 0 && len(boxes) == 0 {
diff --git a/internal/deepdoc/parser/pdf/table/table_post_test.go b/internal/deepdoc/parser/pdf/table/table_post_test.go
index 3b536b42ce..ddf95af8ba 100644
--- a/internal/deepdoc/parser/pdf/table/table_post_test.go
+++ b/internal/deepdoc/parser/pdf/table/table_post_test.go
@@ -750,9 +750,12 @@ func TestConsolidateFigures_OnlyDataSource(t *testing.T) {
}
func TestExtractTableAndReplace_MergeTablesAcrossPages(t *testing.T) {
- // Regression test: two tables on consecutive pages with overlapping X
- // should be merged by MergeTablesAcrossPages, and buildReplacementsAfterMerge
- // must correctly index into the merged slice (not the original pre-merge slice).
+ // Regression test: ExtractTableAndReplace must correctly replace an
+ // ALREADY-merged cross-page table (merged upstream by Parser.buildLayout at
+ // parser.go:540, which uses page-absolute Y + real medianHeights). It must
+ // NOT re-merge tables itself — that is buildLayout's job. The merged table
+ // carries both pages in its Positions, and buildReplacementsAfterMerge must
+ // index the merged slice so both pages' boxes are replaced by one HTML box.
boxes := []pdf.TextBox{
{Text: "intro", LayoutType: pdf.LayoutTypeText, PageNumber: 0, X0: 10, X1: 100, Top: 10, Bottom: 30},
{Text: "table1", LayoutType: pdf.LayoutTypeTable, PageNumber: 0, X0: 10, X1: 400, Top: 40, Bottom: 150},
@@ -760,21 +763,22 @@ func TestExtractTableAndReplace_MergeTablesAcrossPages(t *testing.T) {
{Text: "table2", LayoutType: pdf.LayoutTypeTable, PageNumber: 1, X0: 10, X1: 400, Top: 10, Bottom: 120},
{Text: "outro", LayoutType: pdf.LayoutTypeText, PageNumber: 1, X0: 10, X1: 100, Top: 130, Bottom: 160},
}
+ // One table already merged across page 0 and page 1: Positions span both
+ // pages so buildReplacementsAfterMerge matches both table boxes; Cells from
+ // both pages so the HTML contains the full cross-page content. MergeTables
+ // (parser.go:540) produces exactly this shape; ExtractTableAndReplace must
+ // consume it without re-merging.
tables := []pdf.TableItem{
{
- Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 40, Bottom: 150}},
+ Positions: []pdf.Position{
+ {PageNumbers: []int{0}, Left: 10, Right: 400, Top: 40, Bottom: 150},
+ {PageNumbers: []int{1}, Left: 10, Right: 400, Top: 10, Bottom: 120},
+ },
RegionLeft: 10, RegionRight: 400, RegionTop: 40, RegionBottom: 150,
Scale: 1.0,
Cells: []pdf.TSRCell{
{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "Page0_A"},
{X0: 100, Y0: 0, X1: 200, Y1: 30, Text: "Page0_B"},
- },
- },
- {
- Positions: []pdf.Position{{PageNumbers: []int{1}, Left: 10, Right: 400, Top: 10, Bottom: 120}},
- RegionLeft: 10, RegionRight: 400, RegionTop: 10, RegionBottom: 120,
- Scale: 1.0,
- Cells: []pdf.TSRCell{
{X0: 0, Y0: 50, X1: 100, Y1: 80, Text: "Page1_C"},
{X0: 100, Y0: 50, X1: 200, Y1: 80, Text: "Page1_D"},
},
@@ -785,7 +789,7 @@ func TestExtractTableAndReplace_MergeTablesAcrossPages(t *testing.T) {
if len(result) == 0 {
t.Fatal("expected non-empty result")
}
- // After merge: 2 table boxes replaced by 1 merged HTML box.
+ // 2 table boxes replaced by 1 merged HTML box.
// Original 5 boxes → 4 expected (intro, merged_table, middle, outro).
if len(result) != 4 {
t.Errorf("expected 4 boxes after merge+replace, got %d", len(result))
@@ -924,3 +928,71 @@ func TestProcessTablesWithReplacements_KeepsTextWhenCellsButEmptyHTML(t *testing
}
t.Errorf("degenerate-table box text missing from output: %+v", out)
}
+
+// countTableBoxes returns the number of table-region boxes (LayoutTypeTable)
+// in a box slice. After ExtractTableAndReplace, each merged/un-merged table
+// becomes exactly one LayoutTypeTable HTML box, so this counts tables.
+func countTableBoxes(boxes []pdf.TextBox) int {
+ n := 0
+ for _, b := range boxes {
+ if b.LayoutType == pdf.LayoutTypeTable {
+ n++
+ }
+ }
+ return n
+}
+
+// TestExtractTableAndReplace_NoReMergeAfterPageAbsoluteRejection is the TDD
+// guard for the cross-page over-merge regression: Parser.buildLayout already
+// calls MergeTablesAcrossPages with page-absolute Y + real medianHeights
+// (parser.go:540) and may correctly REJECT a cross-page merge for two tables
+// whose page-local Y merely repeats every page (icbccs pages 4-5). The tables
+// handed to ExtractTableAndReplace are therefore already the desired output.
+//
+// ExtractTableAndReplace must NOT re-run MergeTablesAcrossPages with nil page
+// metadata, because the legacy page-local formula would re-merge the rejected
+// pair and silently undo the page-absolute fix (see CodeRabbit review on
+// table_post.go:290 and known_diffs rule icbccs-crosspage-table-overmerge).
+//
+// This constructs the icbccs shape: anchor page 4 (bottom 172) and
+// continuation page 5 (local top 262); the page-absolute gap (pageHeights[4]=842
+// applied at parser.go:540) is ~842+90 > gate, but the legacy page-local gap
+// is ~99 < gate. With the REJECTED (already-split) tables passed in, the result
+// must keep 2 tables.
+func TestExtractTableAndReplace_NoReMergeAfterPageAbsoluteRejection(t *testing.T) {
+ const scale = 1.0
+ // Two independent pages-4/5 tables; page-local Y repeats every page.
+ tables := []pdf.TableItem{
+ {
+ // page 4 anchor
+ Cells: []pdf.TSRCell{{Text: "a1", Y0: 60, Y1: 172, X0: 100, X1: 500}},
+ Positions: []pdf.Position{{PageNumbers: []int{4}, Left: 100, Right: 500, Top: 60, Bottom: 172}},
+ Caption: "请求参数",
+ Scale: scale,
+ },
+ {
+ // page 5 continuation, page-local top repeats near anchor top
+ Cells: []pdf.TSRCell{{Text: "b1", Y0: 262, Y1: 280, X0: 100, X1: 500}},
+ Positions: []pdf.Position{{PageNumbers: []int{5}, Left: 100, Right: 500, Top: 262, Bottom: 280}},
+ Caption: "请求参数",
+ Scale: scale,
+ },
+ }
+
+ // Table-layout boxes on each page that overlap the table positions so the
+ // replacement logic engages (otherwise ExtractTableAndReplace early-returns
+ // without touching the tables).
+ boxes := []pdf.TextBox{
+ {Text: "page4 table", LayoutType: pdf.LayoutTypeTable, PageNumber: 4, X0: 90, X1: 510, Top: 50, Bottom: 180},
+ {Text: "page5 table", LayoutType: pdf.LayoutTypeTable, PageNumber: 5, X0: 90, X1: 510, Top: 255, Bottom: 290},
+ }
+
+ out := ExtractTableAndReplace(boxes, tables)
+
+ // Bug: line 290 re-runs MergeTablesAcrossPages(tables, nil, nil) with the
+ // legacy page-local formula, re-merging the two tables into ONE HTML box.
+ // Fix: ExtractTableAndReplace must preserve the already-split 2 tables.
+ if got := countTableBoxes(out); got != 2 {
+ t.Errorf("ExtractTableAndReplace re-merged the page-absolute-rejected tables: got %d table boxes, want 2 (the two pages-4/5 tables must stay separate)", got)
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
index c8b1e6bd5b..7175a797da 100644
--- a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
+++ b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
@@ -217,6 +217,97 @@
],
"owner_fix_side": "neither",
"reason": "11.pdf is a multi-table PDF (Go emits 7 tables) whose grid shapes diverge from the Python golden: gridSim=91.8%, structSim=87.5% with Go grids 7x3 vs Python 11x3, textSim=87.4% (verified via TestPipelineParity with BATCH_PARITY_VARIANT=ocr_real). This is NOT a go_intentional divergence (Go is not judged better than Python) and NOT a trackable go_bug on either side: neither pipeline is the ground truth for this document's table structure, so the gap is not a regression target on the Go side nor something the Python golden can anchor. It is registered as ignore so the PDF is exempted from the FAIL count (reported as IGNORE) and does not block the parity gate, while the gap stays logged for a future re-examination of the golden/ground-truth table structure. Resolution requires re-establishing which grid (if either) is correct against the physical document before any code fix is attempted."
+ },
+ {
+ "id": "noncell-icbccs-caption-dropped",
+ "tag": "go_bug",
+ "kind": "non_cell_text",
+ "applies_to": [
+ "icbccs deployment.pdf"
+ ],
+ "fields": [
+ "caption",
+ "MergeCaptions.findTables"
+ ],
+ "owner_fix_side": "go",
+ "status": "resolved",
+ "tracking": "Originally classified as an ENVIRONMENTAL replay-data gap (commit 323db9e41) on the belief that the ocr_real replay dump for icbccs deployment.pdf had no '请求参数' caption text in either the chars dump or the OCR dump. That hypothesis was WRONG and was disproven by re-dumping: the OCR dump DOES contain '请求参数' boxes on pages 2/4/5 (0-based), the chars dump was re-dumped, and DLA replay correctly loads all 4 'table caption' regions (pages 2/4/5) — AnnotateBoxLayouts types the boxes as table captions (verified via debug slog). The real root cause is a Go bug in MergeCaptions.findTables (table/merge_captions.go): it matches a caption to the nearest table by squared distance dist = gapY² + dx² against maxCaptionGap=40000. A narrow caption (请求参数, ~72pt wide) sitting directly ABOVE a much wider table (~535pt wide) has a large dx to the table's center (dx≈233, dx²≈54000 > 40000), so the match is rejected and the caption is silently dropped by the no-target branch. Fix: add maxCaptionVGap=200 — a caption vertically adjacent to a table (small gapY) attaches regardless of horizontal offset, because vertical adjacency is the real signal that the caption belongs to that (often wider) table; dx only discriminates between candidate tables, which findTables already resolves via min-distance. Verified: TestPipelineParityIcbccsCaptionEmitted flipped from SKIP to a real PASS (Go now emits <caption>请求参数</caption> inside the tables), and the new unit test TestMergeCaptions_NarrowCaptionAttachesWideTable asserts the narrow-caption-above-wide-table case attaches and emits exactly one <caption>请求参数</caption>. Built-in 18_table_caption.pdf still passes (its wide caption had small dx). RESIDUAL (caption CONCAT/DUP sub-bug now RESOLVED): the original 97.9% non-cell-text gap had TWO parts. (1) Caption DUPLICATION — once captions attached, a caption on a DIFFERENT page wrongly attached to a single-page table whose page-local Y merely repeats every page (icbccs: page-3 '请求参数' → page-5 table; page-6 '请求参数' → page-3 table), concatenating two captions into one <caption> ('请求参数 请求参数'); AND CJK captions were joined with a space while Python joins CJK with NO space. FIXED by a page-scope guard in findTables (a caption attaches only to a table that occupies its page) PLUS the merged cross-page table now recording every spanned page in its Position (via doctype.TextBox.Pages, set in tableRegionBox/createTableBoxFromItem from the merged TableItem's Positions) PLUS a CJK-aware captionSep join in injectCaption/appendRawCaptions (space only when a caption contains ASCII letters, matching Python's construct_table). Go now emits the 3 clean captions Python emits (请求参数枚举值 / 请求参数 / 请求参数), verified by TestPipelineParityIcbccsCaptionEmitted (PASS) and TestMergeCaptions_NarrowCaptionAttachesWideTable, with no regression on 13's cross-page caption continuation (TestPipelineParity13CrosspageSeam). (2) The REMAINING 97.9% in the REAL ocr_real 56-set is NOT a 'Body'/body placeholder (that is REAL content present in BOTH Go and Python golden — Python's golden also has title 'Body 请求参数' and cells '位置 body'/'body'). It is a SEPARATE go_bug: Go's MergeTablesAcrossPages OVER-MERGES icbccs's two API-parameter tables on pages 4 and 5 (Go 0-based; Python 1-based pages 5 and 6) into ONE cross-page table, concatenating their separate '请求参数' captions into a single '请求参数请求参数' and collapsing 2×3 rows into 1×6. ROOT CAUSE: MergeTablesAcrossPages (table/table_merge.go) computes the Y-proximity gate yDis=(bp.Top+bp.Bottom-anchorBtm-ap.Bottom)/2 using page-LOCAL table coordinates as if they were one continuous frame. The anchor (page 4) has bottom≈172 and the continuation (page 5) has local top≈262, so the computed yDis≈99 — well under the gate mh*23 (mh defaults to 10 because medianHeights is passed as nil at parser.go:540), so Go merges. But the two tables are on DIFFERENT pages; in Python's absolute page-stacked coordinates the same gap is ~862pt (page-5 bottom≈3604 to page-6 top≈4466), so Python's identical _y_dis formula yields ~950 > mh*23 and correctly keeps them separate. Note Go's medianHeights is also passed as nil (parser.go:540), so mh is a hardcoded 10 rather than the real per-page median char height. FIX DIRECTION: either (a) make the cross-page Y-proximity account for the full inter-page gap (anchor page height − anchorBtm) + continuation local top, in a page-absolute frame, or (b) pass the real medianHeights into MergeTablesAcrossPages and re-derive the gate in the same coordinate frame Python uses. This over-merge is now RESOLVED (see rule icbccs-crosspage-table-overmerge: page-absolute fix at parser.go:540 plus removal of the second MergeTablesAcrossPages call in ExtractTableAndReplace, which previously re-merged the rejected pair). It was tracked as the open sub-bug below; the caption DROP + CONCAT/DUP fixes are REGRESSION-FREE (full ^TestPipelineParity$ ocr_real 56 PDFs: ZERO status change vs baseline, aligned=9 noncell-text=5 intentional=7 ignored=1 failed=34).",
+ "reason": "NONCELL_TEXT (gridSim=100% structSim=100% textSim=97.9%): the caption DROP sub-bug is RESOLVED (reclassified from ignore/environmental to go_bug/resolved). The caption text IS present in the ocr_real replay intermediates; findTables rejected the narrow-caption-above-wide-table match on dx² > maxCaptionGap, dropping the caption — fixed by maxCaptionVGap=200. A SECOND related sub-bug (caption CONCATENATION/DUPLICATION) is also RESOLVED: a cross-page caption wrongly attached to a single-page table whose page-local Y repeats (icbccs page-3 '请求参数' → page-5 table; page-6 '请求参数' → page-3 table), and CJK captions were space-joined while Python joins CJK with no space. Fixed by a page-scope guard in findTables (caption attaches only to a table that occupies its page) + merged cross-page tables now recording every spanned page in Position (doctype.TextBox.Pages) + CJK-aware captionSep join. In the testdata/charspy replay path Go now emits Python's 3 clean captions (请求参数枚举值 / 请求参数 / 请求参数), verified by TestPipelineParityIcbccsCaptionEmitted (PASS). IMPORTANT CORRECTION: the REMAINING 97.9% in the REAL ocr_real 56-set is NOT a 'Body'/body placeholder — 'Body 请求参数' (title) and '位置 body'/'body' (cells) are REAL content present in BOTH Go and Python golden, so they are NOT a divergence. The true remaining 97.9% is a go_bug OVER-MERGE in MergeTablesAcrossPages: Go fuses icbccs's two API-parameter tables on pages 4–5 (Go 0-based; Python 1-based 5–6) into one table with caption '请求参数请求参数' and 6 rows, whereas Python keeps two 3-row tables each captioned '请求参数'. Root cause: the Y-proximity gate uses page-LOCAL coordinates as a continuous frame (yDis≈99 < mh*23=230), grossly underestimating the inter-page gap, while Python's absolute-frame _y_dis≈950 correctly exceeds its gate. medianHeights is also passed as nil (parser.go:540), leaving mh=10. This over-merge is now RESOLVED (page-absolute fix at parser.go:540 plus removal of the second MergeTablesAcrossPages call in ExtractTableAndReplace, which previously re-merged the rejected pair). Full ^TestPipelineParity$ (ocr_real, 56 PDFs) shows ZERO status change vs baseline (aligned=9 noncell-text=5 intentional=7 ignored=1 failed=34) for the caption fixes."
+ },
+ {
+ "id": "icbccs-crosspage-table-overmerge",
+ "tag": "go_bug",
+ "kind": "non_cell_text",
+ "applies_to": [
+ "icbccs deployment.pdf"
+ ],
+ "fields": [
+ "MergeTablesAcrossPages",
+ "table/table_merge.go"
+ ],
+ "owner_fix_side": "go",
+ "reason": "NONCELL_TEXT (icbccs now emits 4 independent page-scoped tables, each with a clean single '请求参数' caption; was one merged table with duplicated '请求参数请求参数' caption and 2x3 rows collapsed into 1x6): Go OVER-MERGED icbccs's two API-parameter tables on pages 4 and 5 (Go 0-based; Python 1-based pages 5 and 6) into ONE cross-page table. Python keeps them as two 3-row tables each captioned '请求参数'. RESOLVED: MergeTablesAcrossPages now measures the cross-page Y gap in a page-absolute frame — the continuation table's page-local Top is offset by the anchor page's PDF-point height (yDis += anchorPageHeight, read from result.PageHeight keyed by 0-based page), so two tables whose page-local Y merely repeats every page no longer look adjacent — and parser.go:540 passes the real per-page medianHeights (mh*23 gate calibrated like Python's mean_height). CRITICAL: the page-absolute fix at parser.go:540 was previously UNDONE by a second MergeTablesAcrossPages(tables, nil, nil) call inside ExtractTableAndReplace (table_post.go:290), whose legacy page-local formula re-merged the rejected pair. That second call has been REMOVED, so the fix now truly takes effect (verified end-to-end: icbccs emits 4 independent tables, no duplicated caption). TDD guards: TestMergeTablesAcrossPages_PageLocalYRepeatsButSeparatePages (over-merge rejected) + TestMergeTablesAcrossPages_RealAdjacentAcrossPagesStillMerges (genuine split still merged) + TestExtractTableAndReplace_NoReMergeAfterPageAbsoluteRejection (ExtractTableAndReplace must not re-merge). No regression on 13_crosspage_table.pdf / 中加纯债 cross-page merges (中加纯债 remains FAIL for an unrelated pre-existing column-count reason).",
+ "status": "resolved"
+ },
+ {
+ "id": "noncell-dtdfx-formatting",
+ "tag": "go_intentional",
+ "kind": "non_cell_text",
+ "applies_to": [
+ "DT DFX Q&A.pdf"
+ ],
+ "fields": [
+ "ParseRaw",
+ "caption",
+ "body"
+ ],
+ "permanent": true,
+ "reason": "NONCELL_TEXT (gridSim=100% structSim=100% textSim=99.8%): divergence is Chinese line-breaking / page-marker handling (Python emits 'pg.N' page markers as standalone lines and a '<caption> pg.7</caption>' page label; Go drops these page-marker artifacts) plus benign HTML whitespace. Cell content + structure 100% identical to Python. Go omitting the 'pg.7' page-marker 'caption' is cleaner (it is a page number, not a table title), and no real caption/source text is lost. Body text content matches Python; Go even merges fragmented lines into paragraphs (better for downstream chunking). Deliberate/acceptable divergence — Go is at least as good as Python."
+ },
+ {
+ "id": "noncell-pm-formatting",
+ "tag": "go_intentional",
+ "kind": "non_cell_text",
+ "applies_to": [
+ "PM_.O_3._.pdf"
+ ],
+ "fields": [
+ "ParseRaw",
+ "caption",
+ "body"
+ ],
+ "permanent": true,
+ "reason": "NONCELL_TEXT (gridSim=100% structSim=100% textSim=99.1%): Go retains BOTH of Python's <caption> elements with identical content (2=2), so caption handling matches here. The residual gap is benign: Chinese line-breaking/segmentation (Go merges fragments into paragraphs; Py splits them) and HTML tag formatting (Go emits <th> for detected header rows, Py uses <td> everywhere — Go keeps the header semantics). Cell content + structure 100% identical. No content loss; Go is at least as good as Python."
+ },
+ {
+ "id": "noncell-screenshot-html",
+ "tag": "go_intentional",
+ "kind": "non_cell_text",
+ "applies_to": [
+ "screenshot.pdf"
+ ],
+ "fields": [
+ "ParseRaw",
+ "caption",
+ "html"
+ ],
+ "permanent": true,
+ "reason": "NONCELL_TEXT (gridSim=100% structSim=100% textSim=98.4%): the entire PDF is a single table, so the divergence is PURELY HTML serialization formatting — Python emits multi-line <td > cells with a trailing #@meta line; Go emits a single-line <table> using <th> for the header row (semantically correct) and folds the caption-free grid. Cell text content is 100% identical (gridSim=structSim=100%). No body/caption text is involved. Go's <th> markup is at least as good as Python's <td>-everywhere; deliberate/acceptable divergence."
+ },
+ {
+ "id": "noncell-guangxi-formatting",
+ "tag": "go_intentional",
+ "kind": "non_cell_text",
+ "applies_to": [
+ "广西广播电视地球站维护部应知应会手册.pdf"
+ ],
+ "fields": [
+ "ParseRaw",
+ "caption",
+ "body"
+ ],
+ "permanent": true,
+ "reason": "NONCELL_TEXT (gridSim=100% structSim=100% textSim=99.7%): divergence is Chinese line-breaking/segmentation (Go merges adjacent tokens, e.g. '高惠东'+'关晓雪' -> '高惠东关晓雪'; Py splits them) plus benign HTML whitespace (Go emits <th> for header rows, Py uses <td>). Cell content + structure 100% identical to Python; all characters are preserved (Go's merge is a segmentation choice, not a content loss). No caption is involved. Deliberate/acceptable divergence — Go is at least as good as Python."
}
]
}
diff --git a/internal/deepdoc/parser/type/types.go b/internal/deepdoc/parser/type/types.go
index 0c521b2547..0d56456340 100644
--- a/internal/deepdoc/parser/type/types.go
+++ b/internal/deepdoc/parser/type/types.go
@@ -95,10 +95,17 @@ type TextBox struct {
Top, Bottom float64
Text string
PageNumber int
- LayoutType string
- LayoutNo string
- ColID int
- R int
+ // Pages carries the full set of page numbers a box spans, when it is a
+ // single logical region split across consecutive pages (e.g. a table
+ // merged across pages by MergeTablesAcrossPages). When non-empty it
+ // overrides PageNumber for page-span computation in BoxesToSections, so a
+ // cross-page merged table records every page it occupies (not just the
+ // anchor page).
+ Pages []int
+ LayoutType string
+ LayoutNo string
+ ColID int
+ R int
// IsOCR marks a box produced by an OCR pass (ocrDetectAndRecognize /
// ocrMergeChars), as opposed to one built from embedded PDF chars
// (CharsToBoxes). It scopes OCR-only post-processing (see layout.Dedup*)