Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(pdf): restore cross-page table merge wrongly rejected by #18688 page-absolute Y gate (#18740)

xugangqiangb6a0a3cc62ea
diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_bond_18688_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_bond_18688_test.go
new file mode 100644
index 0000000000..37b3f593bc
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/pipeline_parity_bond_18688_test.go
@@ -0,0 +1,100 @@
+//go:build cgo && manual
+
+package pdf
+
+import (
+ "path/filepath"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/deepdoc/parser/pdf/tool"
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+ util "ragflow/internal/deepdoc/parser/pdf/util"
+)
+
+// bondName is the ocr_real PDF whose two-page parameter table regressed under
+// PR #18688. The table spans page 0 (table_index 0) and page 1 (table_index 1)
+// and must be merged into ONE 6-column table matching Python's 51-row golden.
+const bondName = "2.中加纯债两年定期开放债券型证券投资基金_参数表(用印版).pdf"
+
+// replayPipelineGridVariant runs Go's replay pipeline for a custom dataset
+// variant (e.g. "ocr_real") and returns the full ParseResult so callers can
+// inspect the reconstructed table grids (result.Tables) and compute grid
+// structure similarity against Python's golden. It mirrors the per-PDF setup
+// of TestPipelineParity Phase 2 for non-default datasets.
+func replayPipelineGridVariant(t *testing.T, variant, name string) *pdf.ParseResult {
+ t.Helper()
+ dirs := tool.ParityDirsFor(variant)
+ engine, err := tool.LoadPythonChars(filepath.Join(dirs.Charspy, name+".json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if v := engine.IsEnglish(); v != nil && *v {
+ engine.ClearChars()
+ } else if pages, _ := engine.PageCount(); util.DetectEnglish(engine.PageChars(), pages, nil) {
+ engine.ClearChars()
+ }
+ RegisterReplayTableBuilder()
+ cfg := pdf.DefaultParserConfig()
+ cfg.SortByTop = true
+ analyzer := NewPythonIntermediateDocAnalyzer(name, dirs.DLA, dirs.TSRRaw, dirs.OCR, engine.PageDims())
+ p := NewParser(cfg)
+ result, err := p.ParseRaw(t.Context(), engine, analyzer)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return result
+}
+
+// TestPipelineParityBondCrossPageMerge18688 is the TDD guard for the
+// regression introduced by PR #18688 (fix(deepdoc): prevent cross-page table
+// over-merge from page-local Y coordinates).
+//
+// Root cause: in table/table_merge.go MergeTablesAcrossPages, #18688 added an
+// unconditional `yDis += anchorPageH` (page-absolute Y shift) to the
+// cross-page proximity test, intending to stop icbccs-style over-merges where
+// two tables merely repeat their page-local Y every page. But for this bond
+// PDF the two pages ARE a genuine continuation, and the page-absolute shift
+// pushes yDis above the `mh*23` threshold so the merge is wrongly skipped
+// (`continue`). Go then emits two separate tables (page 0 = 23x5, page 1 =
+// 23x6) instead of one merged 6-column table, diverging from Python's single
+// 51-row / 6-column table: structSim drops from ~89.4% (pre-#18688) to 44.7%
+// (current HEAD).
+//
+// This test pins the EXPECTED behavior: the two-page table must be merged into
+// a single table whose structure matches Python (structSim >= 89%). It FAILS at
+// current HEAD (44.7%), exposing the regression, and passes once the
+// over-aggressive `yDis += anchorPageH` is corrected so legitimate
+// cross-page continuations still merge.
+//
+// Requires BATCH_PARITY_DATA_ROOT (the shared ocr_real dump dir); skips
+// otherwise. Run via build.sh --test-manual with BATCH_PARITY_VARIANT=ocr_real
+// and BATCH_PARITY_DATA_ROOT set.
+func TestPipelineParityBondCrossPageMerge18688(t *testing.T) {
+ if common.GetEnv(common.EnvBatchParityDataRoot) == "" {
+ t.Skip("BATCH_PARITY_DATA_ROOT not set; ocr_real dumps unavailable")
+ }
+ t.Setenv(common.EnvBatchParityVariant, "ocr_real")
+ dirs := tool.ParityDirsFor("ocr_real")
+
+ result := replayPipelineGridVariant(t, "ocr_real", bondName)
+
+ // Diagnostic: the merged (or split) table count directly shows whether
+ // the cross-page merge fired. The bond PDF is ONE logical table spanning
+ // two pages, so a correct merge yields exactly one TableItem.
+ t.Logf("DIAG result.Tables=%d (expected 1 after cross-page merge)", len(result.Tables))
+
+ pyRows, pyHasTables := loadPythonTables(t, filepath.Join(dirs.Tables, bondName+".json"))
+ goRows := goTableRows(result)
+ gridSim := tool.CharSimilarity(joinGrid(goRows), joinGrid(pyRows))
+ structureSim, shape := gridStructureSimilarity(goRows, pyRows)
+
+ // The regression is a structural (column-count) divergence, not cell-text
+ // loss: gridSim stays ~100% while structSim collapses. Pin structSim.
+ if structureSim < 89.0 {
+ t.Errorf("REGRESSION (#18688 cross-page over-reject): bond PDF structSim=%.1f%% (<89%%); "+
+ "Python merges the two-page parameter table into one 6-column table (51 rows) but Go leaves it split "+
+ "(structSim 44.7%% at HEAD). shape=%s gridSim=%.1f%% pyTables=%v goTables=%d",
+ structureSim, shape, gridSim, pyHasTables, len(result.Tables))
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/table/table_merge.go b/internal/deepdoc/parser/pdf/table/table_merge.go
index 7793062c69..72303ce03c 100644
--- a/internal/deepdoc/parser/pdf/table/table_merge.go
+++ b/internal/deepdoc/parser/pdf/table/table_merge.go
@@ -103,18 +103,20 @@ func MergeTablesAcrossPages(tables []pdf.TableItem, medianHeights, pageHeights m
mh = h
}
}
+ // page-local yDis (Y resets to 0 on each page). A genuine
+ // cross-page continuation sits at the TOP of the next page, which
+ // in page-local coordinates is "above" the anchor, so yDis is
+ // NEGATIVE — merge it as-is. Two separate tables that merely
+ // repeat their page-local Y every page have a POSITIVE yDis; only
+ // then shift into the page-absolute frame (by the anchor page
+ // height) so the over-merge is rejected. This restates the
+ // icbccs-crosspage-table-overmerge guard from #18688 without also
+ // rejecting legitimate continuations.
yDis := (bp.Top + bp.Bottom - anchorBtm - ap.Bottom) / 2
- // Tables carry page-LOCAL coordinates (each page resets Y to 0),
- // so comparing bp.Top against anchorBtm as if they shared one
- // continuous frame underestimates the inter-page gap and wrongly
- // merges two tables that merely repeat their Y every page. Python
- // measures _y_dis in a page-absolute frame, so shift the
- // continuation into the anchor's page-absolute frame by the anchor
- // page's height when we know it (see rule icbccs-crosspage-table-
- // overmerge). Without pageHeights the legacy page-local formula is
- // kept for backward compatibility.
- if anchorPageH, ok := pageHeights[anchorPg]; ok && anchorPageH > 0 {
- yDis += anchorPageH
+ if yDis >= 0 {
+ if anchorPageH, ok := pageHeights[anchorPg]; ok && anchorPageH > 0 {
+ yDis += anchorPageH
+ }
}
if yDis > mh*23 {
continue
diff --git a/internal/deepdoc/parser/pdf/table/table_merge_test.go b/internal/deepdoc/parser/pdf/table/table_merge_test.go
index 3ffacfeef4..ab20ac3896 100644
--- a/internal/deepdoc/parser/pdf/table/table_merge_test.go
+++ b/internal/deepdoc/parser/pdf/table/table_merge_test.go
@@ -583,3 +583,38 @@ func TestMergeTablesAcrossPages_RealAdjacentAcrossPagesStillMerges(t *testing.T)
t.Errorf("merged table should record both pages, got %d positions", len(merged[0].Positions))
}
}
+
+// TestMergeTablesAcrossPages_GenuineContinuationMergesWithoutMedianHeights
+// locks the #18688 bond cross-page regression fix's core insight: the
+// page-absolute Y shift is now gated by the SIGN of the page-local yDis. A
+// genuine continuation sits at the top of the next page, so its page-local
+// yDis is NEGATIVE and must NOT be shifted — it merges even when medianHeights
+// is unavailable (replay char-height is 0, so mh falls back to the default 10).
+//
+// Geometry: anchor page 0 near the bottom (local bottom=800), continuation
+// page 1 near the top (local top=50), page height 842 → page-local
+// yDis = (50+110-800-800)/2 = -720 (< 0) ⇒ no shift ⇒ MERGE. Under the
+// pre-fix #18688 code the shift would make yDis=122 (< mh*23=230) and also
+// merge, so this test mainly guards that the merge no longer depends on
+// medianHeights being populated for a legitimate continuation.
+func TestMergeTablesAcrossPages_GenuineContinuationMergesWithoutMedianHeights(t *testing.T) {
+ anchor := pdf.TableItem{
+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 30, Right: 566, Top: 740, Bottom: 800}},
+ Scale: 1.0,
+ Cells: []pdf.TSRCell{{Text: "anchor_p0"}},
+ }
+ cont := pdf.TableItem{
+ Positions: []pdf.Position{{PageNumbers: []int{1}, Left: 30, Right: 566, Top: 50, Bottom: 110}},
+ Scale: 1.0,
+ Cells: []pdf.TSRCell{{Text: "cont_p1"}},
+ }
+ pageHeights := map[int]float64{0: 842, 1: 842}
+ // medianHeights=nil mimics the replay path (char-height 0).
+ merged := MergeTablesAcrossPages([]pdf.TableItem{anchor, cont}, nil, pageHeights)
+ if len(merged) != 1 {
+ t.Fatalf("genuine cross-page continuation: expected 1 merged table, got %d", len(merged))
+ }
+ if len(merged[0].Positions) != 2 {
+ t.Errorf("merged table should record both pages, got %d positions", len(merged[0].Positions))
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
index d7b89473f6..f15836d123 100644
--- a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
+++ b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
@@ -265,7 +265,7 @@
"table/table_merge.go"
],
"owner_fix_side": "go",
- "reason": "NONCELL_TEXT (icbccs now emits 4 independent page-scoped tables, each with a clean single '请求参数' caption; was one merged table with duplicated '请求参数请求参数' caption and 2x3 rows collapsed into 1x6): Go OVER-MERGED icbccs's two API-parameter tables on pages 4 and 5 (Go 0-based; Python 1-based pages 5 and 6) into ONE cross-page table. Python keeps them as two 3-row tables each captioned '请求参数'. RESOLVED: MergeTablesAcrossPages now measures the cross-page Y gap in a page-absolute frame — the continuation table's page-local Top is offset by the anchor page's PDF-point height (yDis += anchorPageHeight, read from result.PageHeight keyed by 0-based page), so two tables whose page-local Y merely repeats every page no longer look adjacent — and parser.go:540 passes the real per-page medianHeights (mh*23 gate calibrated like Python's mean_height). CRITICAL: the page-absolute fix at parser.go:540 was previously UNDONE by a second MergeTablesAcrossPages(tables, nil, nil) call inside ExtractTableAndReplace (table_post.go:290), whose legacy page-local formula re-merged the rejected pair. That second call has been REMOVED, so the fix now truly takes effect (verified end-to-end: icbccs emits 4 independent tables, no duplicated caption). TDD guards: TestMergeTablesAcrossPages_PageLocalYRepeatsButSeparatePages (over-merge rejected) + TestMergeTablesAcrossPages_RealAdjacentAcrossPagesStillMerges (genuine split still merged) + TestExtractTableAndReplace_NoReMergeAfterPageAbsoluteRejection (ExtractTableAndReplace must not re-merge). No regression on 13_crosspage_table.pdf / 中加纯债 cross-page merges (中加纯债 remains FAIL for an unrelated pre-existing column-count reason).",
+ "reason": "NONCELL_TEXT (icbccs now emits 4 independent page-scoped tables, each with a clean single '请求参数' caption; was one merged table with duplicated '请求参数请求参数' caption and 2x3 rows collapsed into 1x6): Go OVER-MERGED icbccs's two API-parameter tables on pages 4 and 5 (Go 0-based; Python 1-based pages 5 and 6) into ONE cross-page table. Python keeps them as two 3-row tables each captioned '请求参数'. RESOLVED: MergeTablesAcrossPages now measures the cross-page Y gap in a page-absolute frame — the continuation table's page-local Top is offset by the anchor page's PDF-point height (yDis += anchorPageHeight, read from result.PageHeight keyed by 0-based page), so two tables whose page-local Y merely repeats every page no longer look adjacent — and parser.go:540 passes the real per-page medianHeights (mh*23 gate calibrated like Python's mean_height). CRITICAL: the page-absolute fix at parser.go:540 was previously UNDONE by a second MergeTablesAcrossPages(tables, nil, nil) call inside ExtractTableAndReplace (table_post.go:290), whose legacy page-local formula re-merged the rejected pair. That second call has been REMOVED, so the fix now truly takes effect (verified end-to-end: icbccs emits 4 independent tables, no duplicated caption). TDD guards: TestMergeTablesAcrossPages_PageLocalYRepeatsButSeparatePages (over-merge rejected) + TestMergeTablesAcrossPages_RealAdjacentAcrossPagesStillMerges (genuine split still merged) + TestExtractTableAndReplace_NoReMergeAfterPageAbsoluteRejection (ExtractTableAndReplace must not re-merge). No regression on 13_crosspage_table.pdf cross-page merges. NOTE: 中加纯债 IS affected by this same page-absolute Y gate — #18688's yDis+=anchorPageH wrongly rejected its genuine two-page continuation (page-local yDis≈-178 is negative ⇒ should merge), dropping structSim 44.7% → it is NOT a pre-existing column-count issue. That over-rejection was fixed in #18688's follow-up by gating the page-absolute Y shift on the SIGN of the page-local yDis: a genuine continuation sits at the top of the next page, so its page-local yDis is NEGATIVE and is no longer shifted (it merges), while only tables that merely repeat their page-local Y every page (positive yDis) are shifted into the page-absolute frame and rejected. This restores 中加纯债 to structSim 89.4% (one merged 6-column table) without the mh fallback or anchorBtm change that #18688's first attempt needed. The parity verdict for 中加纯债 still FAILs, but only on an unrelated textSim cell-content divergence, not the cross-page merge.",
"status": "resolved"
},
{