RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(pdf): restore cross-page table merge wrongly rejected by #18688 page-absolute Y gate (#18740)
xugangqiangb6a0a3cc62ea
diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_bond_18688_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_bond_18688_test.gonew file mode 100644index 0000000000..37b3f593bc--- /dev/null+++ b/internal/deepdoc/parser/pdf/pipeline_parity_bond_18688_test.go@@ -0,0 +1,100 @@+//go:build cgo && manual++package pdf++import (+ "path/filepath"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/deepdoc/parser/pdf/tool"+ pdf "ragflow/internal/deepdoc/parser/pdf/type"+ util "ragflow/internal/deepdoc/parser/pdf/util"+)++// bondName is the ocr_real PDF whose two-page parameter table regressed under+// PR #18688. The table spans page 0 (table_index 0) and page 1 (table_index 1)+// and must be merged into ONE 6-column table matching Python's 51-row golden.+const bondName = "2.中加纯债两年定期开放债券型证券投资基金_参数表(用印版).pdf"++// replayPipelineGridVariant runs Go's replay pipeline for a custom dataset+// variant (e.g. "ocr_real") and returns the full ParseResult so callers can+// inspect the reconstructed table grids (result.Tables) and compute grid+// structure similarity against Python's golden. It mirrors the per-PDF setup+// of TestPipelineParity Phase 2 for non-default datasets.+func replayPipelineGridVariant(t *testing.T, variant, name string) *pdf.ParseResult {+ t.Helper()+ dirs := tool.ParityDirsFor(variant)+ engine, err := tool.LoadPythonChars(filepath.Join(dirs.Charspy, name+".json"))+ if err != nil {+ t.Fatal(err)+ }+ if v := engine.IsEnglish(); v != nil && *v {+ engine.ClearChars()+ } else if pages, _ := engine.PageCount(); util.DetectEnglish(engine.PageChars(), pages, nil) {+ engine.ClearChars()+ }+ RegisterReplayTableBuilder()+ cfg := pdf.DefaultParserConfig()+ cfg.SortByTop = true+ analyzer := NewPythonIntermediateDocAnalyzer(name, dirs.DLA, dirs.TSRRaw, dirs.OCR, engine.PageDims())+ p := NewParser(cfg)+ result, err := p.ParseRaw(t.Context(), engine, analyzer)+ if err != nil {+ t.Fatal(err)+ }+ return result+}++// TestPipelineParityBondCrossPageMerge18688 is the TDD guard for the+// regression introduced by PR #18688 (fix(deepdoc): prevent cross-page table+// over-merge from page-local Y coordinates).+//+// Root cause: in table/table_merge.go MergeTablesAcrossPages, #18688 added an+// unconditional `yDis += anchorPageH` (page-absolute Y shift) to the+// cross-page proximity test, intending to stop icbccs-style over-merges where+// two tables merely repeat their page-local Y every page. But for this bond+// PDF the two pages ARE a genuine continuation, and the page-absolute shift+// pushes yDis above the `mh*23` threshold so the merge is wrongly skipped+// (`continue`). Go then emits two separate tables (page 0 = 23x5, page 1 =+// 23x6) instead of one merged 6-column table, diverging from Python's single+// 51-row / 6-column table: structSim drops from ~89.4% (pre-#18688) to 44.7%+// (current HEAD).+//+// This test pins the EXPECTED behavior: the two-page table must be merged into+// a single table whose structure matches Python (structSim >= 89%). It FAILS at+// current HEAD (44.7%), exposing the regression, and passes once the+// over-aggressive `yDis += anchorPageH` is corrected so legitimate+// cross-page continuations still merge.+//+// Requires BATCH_PARITY_DATA_ROOT (the shared ocr_real dump dir); skips+// otherwise. Run via build.sh --test-manual with BATCH_PARITY_VARIANT=ocr_real+// and BATCH_PARITY_DATA_ROOT set.+func TestPipelineParityBondCrossPageMerge18688(t *testing.T) {+ if common.GetEnv(common.EnvBatchParityDataRoot) == "" {+ t.Skip("BATCH_PARITY_DATA_ROOT not set; ocr_real dumps unavailable")+ }+ t.Setenv(common.EnvBatchParityVariant, "ocr_real")+ dirs := tool.ParityDirsFor("ocr_real")++ result := replayPipelineGridVariant(t, "ocr_real", bondName)++ // Diagnostic: the merged (or split) table count directly shows whether+ // the cross-page merge fired. The bond PDF is ONE logical table spanning+ // two pages, so a correct merge yields exactly one TableItem.+ t.Logf("DIAG result.Tables=%d (expected 1 after cross-page merge)", len(result.Tables))++ pyRows, pyHasTables := loadPythonTables(t, filepath.Join(dirs.Tables, bondName+".json"))+ goRows := goTableRows(result)+ gridSim := tool.CharSimilarity(joinGrid(goRows), joinGrid(pyRows))+ structureSim, shape := gridStructureSimilarity(goRows, pyRows)++ // The regression is a structural (column-count) divergence, not cell-text+ // loss: gridSim stays ~100% while structSim collapses. Pin structSim.+ if structureSim < 89.0 {+ t.Errorf("REGRESSION (#18688 cross-page over-reject): bond PDF structSim=%.1f%% (<89%%); "++ "Python merges the two-page parameter table into one 6-column table (51 rows) but Go leaves it split "++ "(structSim 44.7%% at HEAD). shape=%s gridSim=%.1f%% pyTables=%v goTables=%d",+ structureSim, shape, gridSim, pyHasTables, len(result.Tables))+ }+}diff --git a/internal/deepdoc/parser/pdf/table/table_merge.go b/internal/deepdoc/parser/pdf/table/table_merge.goindex 7793062c69..72303ce03c 100644--- a/internal/deepdoc/parser/pdf/table/table_merge.go+++ b/internal/deepdoc/parser/pdf/table/table_merge.go@@ -103,18 +103,20 @@ func MergeTablesAcrossPages(tables []pdf.TableItem, medianHeights, pageHeights mmh = h}}+ // page-local yDis (Y resets to 0 on each page). A genuine+ // cross-page continuation sits at the TOP of the next page, which+ // in page-local coordinates is "above" the anchor, so yDis is+ // NEGATIVE — merge it as-is. Two separate tables that merely+ // repeat their page-local Y every page have a POSITIVE yDis; only+ // then shift into the page-absolute frame (by the anchor page+ // height) so the over-merge is rejected. This restates the+ // icbccs-crosspage-table-overmerge guard from #18688 without also+ // rejecting legitimate continuations.yDis := (bp.Top + bp.Bottom - anchorBtm - ap.Bottom) / 2- // Tables carry page-LOCAL coordinates (each page resets Y to 0),- // so comparing bp.Top against anchorBtm as if they shared one- // continuous frame underestimates the inter-page gap and wrongly- // merges two tables that merely repeat their Y every page. Python- // measures _y_dis in a page-absolute frame, so shift the- // continuation into the anchor's page-absolute frame by the anchor- // page's height when we know it (see rule icbccs-crosspage-table-- // overmerge). Without pageHeights the legacy page-local formula is- // kept for backward compatibility.- if anchorPageH, ok := pageHeights[anchorPg]; ok && anchorPageH > 0 {- yDis += anchorPageH+ if yDis >= 0 {+ if anchorPageH, ok := pageHeights[anchorPg]; ok && anchorPageH > 0 {+ yDis += anchorPageH+ }}if yDis > mh*23 {continuediff --git a/internal/deepdoc/parser/pdf/table/table_merge_test.go b/internal/deepdoc/parser/pdf/table/table_merge_test.goindex 3ffacfeef4..ab20ac3896 100644--- a/internal/deepdoc/parser/pdf/table/table_merge_test.go+++ b/internal/deepdoc/parser/pdf/table/table_merge_test.go@@ -583,3 +583,38 @@ func TestMergeTablesAcrossPages_RealAdjacentAcrossPagesStillMerges(t *testing.T)t.Errorf("merged table should record both pages, got %d positions", len(merged[0].Positions))}}++// TestMergeTablesAcrossPages_GenuineContinuationMergesWithoutMedianHeights+// locks the #18688 bond cross-page regression fix's core insight: the+// page-absolute Y shift is now gated by the SIGN of the page-local yDis. A+// genuine continuation sits at the top of the next page, so its page-local+// yDis is NEGATIVE and must NOT be shifted — it merges even when medianHeights+// is unavailable (replay char-height is 0, so mh falls back to the default 10).+//+// Geometry: anchor page 0 near the bottom (local bottom=800), continuation+// page 1 near the top (local top=50), page height 842 → page-local+// yDis = (50+110-800-800)/2 = -720 (< 0) ⇒ no shift ⇒ MERGE. Under the+// pre-fix #18688 code the shift would make yDis=122 (< mh*23=230) and also+// merge, so this test mainly guards that the merge no longer depends on+// medianHeights being populated for a legitimate continuation.+func TestMergeTablesAcrossPages_GenuineContinuationMergesWithoutMedianHeights(t *testing.T) {+ anchor := pdf.TableItem{+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 30, Right: 566, Top: 740, Bottom: 800}},+ Scale: 1.0,+ Cells: []pdf.TSRCell{{Text: "anchor_p0"}},+ }+ cont := pdf.TableItem{+ Positions: []pdf.Position{{PageNumbers: []int{1}, Left: 30, Right: 566, Top: 50, Bottom: 110}},+ Scale: 1.0,+ Cells: []pdf.TSRCell{{Text: "cont_p1"}},+ }+ pageHeights := map[int]float64{0: 842, 1: 842}+ // medianHeights=nil mimics the replay path (char-height 0).+ merged := MergeTablesAcrossPages([]pdf.TableItem{anchor, cont}, nil, pageHeights)+ if len(merged) != 1 {+ t.Fatalf("genuine cross-page continuation: expected 1 merged table, got %d", len(merged))+ }+ if len(merged[0].Positions) != 2 {+ t.Errorf("merged table should record both pages, got %d positions", len(merged[0].Positions))+ }+}diff --git a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.jsonindex d7b89473f6..f15836d123 100644--- a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json+++ b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json@@ -265,7 +265,7 @@"table/table_merge.go"],"owner_fix_side": "go",- "reason": "NONCELL_TEXT (icbccs now emits 4 independent page-scoped tables, each with a clean single '请求参数' caption; was one merged table with duplicated '请求参数请求参数' caption and 2x3 rows collapsed into 1x6): Go OVER-MERGED icbccs's two API-parameter tables on pages 4 and 5 (Go 0-based; Python 1-based pages 5 and 6) into ONE cross-page table. Python keeps them as two 3-row tables each captioned '请求参数'. RESOLVED: MergeTablesAcrossPages now measures the cross-page Y gap in a page-absolute frame — the continuation table's page-local Top is offset by the anchor page's PDF-point height (yDis += anchorPageHeight, read from result.PageHeight keyed by 0-based page), so two tables whose page-local Y merely repeats every page no longer look adjacent — and parser.go:540 passes the real per-page medianHeights (mh*23 gate calibrated like Python's mean_height). CRITICAL: the page-absolute fix at parser.go:540 was previously UNDONE by a second MergeTablesAcrossPages(tables, nil, nil) call inside ExtractTableAndReplace (table_post.go:290), whose legacy page-local formula re-merged the rejected pair. That second call has been REMOVED, so the fix now truly takes effect (verified end-to-end: icbccs emits 4 independent tables, no duplicated caption). TDD guards: TestMergeTablesAcrossPages_PageLocalYRepeatsButSeparatePages (over-merge rejected) + TestMergeTablesAcrossPages_RealAdjacentAcrossPagesStillMerges (genuine split still merged) + TestExtractTableAndReplace_NoReMergeAfterPageAbsoluteRejection (ExtractTableAndReplace must not re-merge). No regression on 13_crosspage_table.pdf / 中加纯债 cross-page merges (中加纯债 remains FAIL for an unrelated pre-existing column-count reason).",+ "reason": "NONCELL_TEXT (icbccs now emits 4 independent page-scoped tables, each with a clean single '请求参数' caption; was one merged table with duplicated '请求参数请求参数' caption and 2x3 rows collapsed into 1x6): Go OVER-MERGED icbccs's two API-parameter tables on pages 4 and 5 (Go 0-based; Python 1-based pages 5 and 6) into ONE cross-page table. Python keeps them as two 3-row tables each captioned '请求参数'. RESOLVED: MergeTablesAcrossPages now measures the cross-page Y gap in a page-absolute frame — the continuation table's page-local Top is offset by the anchor page's PDF-point height (yDis += anchorPageHeight, read from result.PageHeight keyed by 0-based page), so two tables whose page-local Y merely repeats every page no longer look adjacent — and parser.go:540 passes the real per-page medianHeights (mh*23 gate calibrated like Python's mean_height). CRITICAL: the page-absolute fix at parser.go:540 was previously UNDONE by a second MergeTablesAcrossPages(tables, nil, nil) call inside ExtractTableAndReplace (table_post.go:290), whose legacy page-local formula re-merged the rejected pair. That second call has been REMOVED, so the fix now truly takes effect (verified end-to-end: icbccs emits 4 independent tables, no duplicated caption). TDD guards: TestMergeTablesAcrossPages_PageLocalYRepeatsButSeparatePages (over-merge rejected) + TestMergeTablesAcrossPages_RealAdjacentAcrossPagesStillMerges (genuine split still merged) + TestExtractTableAndReplace_NoReMergeAfterPageAbsoluteRejection (ExtractTableAndReplace must not re-merge). No regression on 13_crosspage_table.pdf cross-page merges. NOTE: 中加纯债 IS affected by this same page-absolute Y gate — #18688's yDis+=anchorPageH wrongly rejected its genuine two-page continuation (page-local yDis≈-178 is negative ⇒ should merge), dropping structSim 44.7% → it is NOT a pre-existing column-count issue. That over-rejection was fixed in #18688's follow-up by gating the page-absolute Y shift on the SIGN of the page-local yDis: a genuine continuation sits at the top of the next page, so its page-local yDis is NEGATIVE and is no longer shifted (it merges), while only tables that merely repeat their page-local Y every page (positive yDis) are shifted into the page-absolute frame and rejected. This restores 中加纯债 to structSim 89.4% (one merged 6-column table) without the mh fallback or anchorBtm change that #18688's first attempt needed. The parity verdict for 中加纯债 still FAILs, but only on an unrelated textSim cell-content divergence, not the cross-page merge.","status": "resolved"},{