RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix cross-page merged tables dropping continuation rows (#18362)
Jack68da250f8f23
diff --git a/internal/deepdoc/parser/pdf/table/table_merge.go b/internal/deepdoc/parser/pdf/table/table_merge.goindex 4164ae22a0..0dd3196c66 100644--- a/internal/deepdoc/parser/pdf/table/table_merge.go+++ b/internal/deepdoc/parser/pdf/table/table_merge.go@@ -1,6 +1,7 @@package tableimport (+ "math""sort"pdf "ragflow/internal/deepdoc/parser/pdf/type"@@ -47,6 +48,7 @@ func MergeTablesAcrossPages(tables []pdf.TableItem, medianHeights map[int]float6}anchor := tables[it.idx]merged[it.idx] = true+ var contGrids [][][]pdf.TSRCell// Python nomerge_lout_no: tables whose box is followed by a// caption/title/reference should not be merged cross-page.@@ -102,6 +104,7 @@ func MergeTablesAcrossPages(tables []pdf.TableItem, medianHeights map[int]float6// Merge: combine cells and positions.anchor.Cells = append(anchor.Cells, tables[jt.idx].Cells...)anchor.Positions = append(anchor.Positions, tables[jt.idx].Positions...)+ contGrids = append(contGrids, tables[jt.idx].Grid)if tables[jt.idx].Caption != "" {if anchor.Caption != "" {anchor.Caption += " "@@ -113,6 +116,39 @@ func MergeTablesAcrossPages(tables []pdf.TableItem, medianHeights map[int]float6anchorBtm = bp.Bottomap = anchor.Positions[len(anchor.Positions)-1]}+ // Rebuild the merged Grid from the per-page grids so ConstructTable+ // emits rows from every merged page, not just the stale anchor+ // (page-0) grid. Only when the anchor already had a Grid (the+ // production path); Grid-less tables fall back to the cells path+ // and must be left untouched to avoid regression.+ //+ // Guard: all merged pages must share the anchor's column count. A+ // jagged cross-page stack (continuation page with a different number+ // of columns) would feed ConstructTable a non-uniform grid, causing+ // CalSpans / CleanupOrphanColumns / RowsToHTML to misalign or+ // silently drop continuation columns and possibly delete a+ // legitimate anchor column. In that case we skip the rebuild and+ // keep the anchor-only Grid — the same safe degrade as the+ // len(anchor.Grid)==0 path (continuation rows dropped, but+ // structurally valid HTML).+ if len(anchor.Grid) > 0 && len(contGrids) > 0 {+ anchorCols := len(anchor.Grid[0])+ uniform := true+ for _, cg := range contGrids {+ if len(cg) == 0 || len(cg[0]) != anchorCols {+ uniform = false+ break+ }+ }+ if uniform {+ allGrids := make([][][]pdf.TSRCell, 0, 1+len(contGrids))+ allGrids = append(allGrids, anchor.Grid)+ allGrids = append(allGrids, contGrids...)+ if rebuilt := stackGrids(allGrids...); len(rebuilt) > 0 {+ anchor.Grid = rebuilt+ }+ }+ }result = append(result, anchor)}// Append unprocessed tables (those with empty Positions) so they@@ -124,3 +160,65 @@ func MergeTablesAcrossPages(tables []pdf.TableItem, medianHeights map[int]float6}return result}++// stackGrids concatenates per-page grids (each already built correctly by+// processOneTable) into one grid for a cross-page-merged table. Continuation+// pages are shifted in Y so their rows sit strictly below the anchor rows,+// keeping Y-based downstream logic (span detection, ordering) monotonic.+func stackGrids(grids ...[][]pdf.TSRCell) [][]pdf.TSRCell {+ var out [][]pdf.TSRCell+ prevMaxY := 0.0+ for _, g := range grids {+ if len(g) == 0 {+ continue+ }+ minY, maxY := gridYExtent(g)+ if prevMaxY > 0 {+ // Place this page's rows below everything stacked so far, with a+ // gap of at least one row height to avoid false row grouping.+ shift := prevMaxY - minY + math.Max(maxY-minY, 1)+ g = shiftGridY(g, shift)+ maxY += shift+ }+ out = append(out, g...)+ prevMaxY = maxY+ }+ return out+}++// gridYExtent returns the min/max Y0/Y1 across all cells of a grid.+func gridYExtent(g [][]pdf.TSRCell) (minY, maxY float64) {+ first := true+ for _, row := range g {+ for _, c := range row {+ if first {+ minY, maxY = c.Y0, c.Y1+ first = false+ continue+ }+ if c.Y0 < minY {+ minY = c.Y0+ }+ if c.Y1 > maxY {+ maxY = c.Y1+ }+ }+ }+ return minY, maxY+}++// shiftGridY returns a copy of g with every cell's Y0/Y1 shifted by dy.+func shiftGridY(g [][]pdf.TSRCell, dy float64) [][]pdf.TSRCell {+ out := make([][]pdf.TSRCell, len(g))+ for i, row := range g {+ nr := make([]pdf.TSRCell, len(row))+ for j, c := range row {+ nc := c+ nc.Y0 += dy+ nc.Y1 += dy+ nr[j] = nc+ }+ out[i] = nr+ }+ return out+}diff --git a/internal/deepdoc/parser/pdf/table/table_merge_test.go b/internal/deepdoc/parser/pdf/table/table_merge_test.goindex 28d8e214e9..1072af0f81 100644--- a/internal/deepdoc/parser/pdf/table/table_merge_test.go+++ b/internal/deepdoc/parser/pdf/table/table_merge_test.go@@ -177,3 +177,249 @@ func TestMergeTablesAcrossPages_NoMedianHeights(t *testing.T) {t.Errorf("expected 2 cells after merge, got %d", len(merged[0].Cells))}}++// TestMergeTablesAcrossPages_RebuildsGridAcrossPages verifies that after a+// cross-page merge the merged table's Grid contains rows from BOTH pages,+// not just the anchor (page-0) grid. This catches the regression where+// ConstructTable reads the stale anchor Grid and drops all continuation rows.+func TestMergeTablesAcrossPages_RebuildsGridAcrossPages(t *testing.T) {+ pageGrid := func(rows [][]string) [][]pdf.TSRCell {+ g := make([][]pdf.TSRCell, len(rows))+ for r, row := range rows {+ g[r] = make([]pdf.TSRCell, len(row))+ for c := range row {+ g[r][c] = pdf.TSRCell{+ X0: float64(c) * 100, Y0: float64(r) * 30,+ X1: float64(c)*100 + 100, Y1: float64(r)*30 + 30,+ Text: row[c],+ }+ }+ }+ return g+ }+ pg0 := pdf.TableItem{+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 0, Right: 200, Top: 0, Bottom: 60}},+ Scale: 1.0,+ Grid: pageGrid([][]string{{"a", "b"}, {"c", "d"}}),+ }+ pg1 := pdf.TableItem{+ Positions: []pdf.Position{{PageNumbers: []int{1}, Left: 0, Right: 200, Top: 0, Bottom: 60}},+ Scale: 1.0,+ Grid: pageGrid([][]string{{"e", "f"}, {"g", "h"}}),+ }++ merged := MergeTablesAcrossPages([]pdf.TableItem{pg0, pg1}, nil)+ if len(merged) != 1 {+ t.Fatalf("expected 1 merged table, got %d", len(merged))+ }+ // Anchor has 2 rows, continuation has 2 rows → merged Grid must be 4.+ if len(merged[0].Grid) != 4 {+ t.Fatalf("merged Grid must contain rows from both pages (want 4), got %d", len(merged[0].Grid))+ }+ // Continuation rows must appear after anchor rows, in page order.+ if merged[0].Grid[0][0].Text != "a" || merged[0].Grid[2][0].Text != "e" {+ t.Errorf("row order wrong after stacking: %s / %s", merged[0].Grid[0][0].Text, merged[0].Grid[2][0].Text)+ }+ // Continuation rows must be Y-shifted strictly below the anchor rows so+ // Y-monotonic downstream logic (span detection, ordering) stays correct.+ // Catches a regression where the shift is dropped but stacking is kept:+ // row order would still be correct by page order, so only this assertion+ // would fail.+ if merged[0].Grid[2][0].Y0 <= merged[0].Grid[1][0].Y1 {+ t.Errorf("continuation row was not shifted below the anchor rows (Grid[2][0].Y0=%v <= Grid[1][0].Y1=%v)",+ merged[0].Grid[2][0].Y0, merged[0].Grid[1][0].Y1)+ }+}++// TestMergeTablesAcrossPages_JaggedContinuationFallsBackToAnchorGrid verifies+// that when a continuation page's grid has a different number of columns than+// the anchor (a jagged cross-page stack), MergeTablesAcrossPages does NOT+// rebuild a non-uniform Grid. Instead it keeps the anchor-only Grid, so+// ConstructTable emits a structurally valid (if continuation-dropping) table+// rather than malformed HTML. This is the same safe degrade as the+// len(anchor.Grid)==0 path, and keeps the merge decision (and the appended+// continuation Cells) unchanged.+func TestMergeTablesAcrossPages_JaggedContinuationFallsBackToAnchorGrid(t *testing.T) {+ pageGrid := func(rows [][]string) [][]pdf.TSRCell {+ g := make([][]pdf.TSRCell, len(rows))+ for r, row := range rows {+ g[r] = make([]pdf.TSRCell, len(row))+ for c := range row {+ g[r][c] = pdf.TSRCell{+ X0: float64(c) * 100, Y0: float64(r) * 30,+ X1: float64(c)*100 + 100, Y1: float64(r)*30 + 30,+ Text: row[c],+ }+ }+ }+ return g+ }+ cells := func(rows [][]string) []pdf.TSRCell {+ var cs []pdf.TSRCell+ for r, row := range rows {+ for c := range row {+ cs = append(cs, pdf.TSRCell{+ X0: float64(c) * 100, Y0: float64(r) * 30,+ X1: float64(c)*100 + 100, Y1: float64(r)*30 + 30,+ Text: row[c],+ })+ }+ }+ return cs+ }+ // Anchor: 3 columns. Continuation: 2 columns (jagged).+ pg0 := pdf.TableItem{+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 0, Right: 300, Top: 0, Bottom: 60}},+ Scale: 1.0,+ Grid: pageGrid([][]string{{"a", "b", "c"}, {"d", "e", "f"}}),+ Cells: cells([][]string{{"a", "b", "c"}, {"d", "e", "f"}}),+ }+ pg1 := pdf.TableItem{+ Positions: []pdf.Position{{PageNumbers: []int{1}, Left: 0, Right: 200, Top: 0, Bottom: 60}},+ Scale: 1.0,+ Grid: pageGrid([][]string{{"g", "h"}, {"i", "j"}}),+ Cells: cells([][]string{{"g", "h"}, {"i", "j"}}),+ }++ merged := MergeTablesAcrossPages([]pdf.TableItem{pg0, pg1}, nil)+ if len(merged) != 1 {+ t.Fatalf("expected 1 merged table, got %d", len(merged))+ }+ // Columns differ (3 vs 2) → rebuild must be skipped → Grid stays+ // anchor-only (2 rows), NOT a 4-row jagged grid.+ if len(merged[0].Grid) != 2 {+ t.Fatalf("jagged continuation must fall back to anchor-only Grid (want 2 rows), got %d", len(merged[0].Grid))+ }+ // Anchor rows preserved; continuation NOT stacked into the Grid.+ if merged[0].Grid[0][0].Text != "a" || merged[0].Grid[1][0].Text != "d" {+ t.Errorf("anchor rows corrupted after jagged fallback: %s / %s", merged[0].Grid[0][0].Text, merged[0].Grid[1][0].Text)+ }+ // Continuation Cells are still appended (pre-fix behaviour) — the merge+ // decision is unchanged; only the Grid stays uniform so HTML is valid.+ hasCont := false+ for _, c := range merged[0].Cells {+ if c.Text == "g" {+ hasCont = true+ break+ }+ }+ if !hasCont {+ t.Errorf("continuation Cells should still be appended even when Grid rebuild is skipped")+ }+}++// TestMergeTablesAcrossPages_ThreePageCumulativeShift verifies that with three+// consecutive pages the per-page grids stack cumulatively: each continuation+// page sits strictly below the previous page's last row, and the Y shift+// accumulates (page2 below page1 below page0). Catches a regression where+// stackGrids resets prevMaxY to the anchor instead of carrying the prior+// page's shifted bottom forward.+func TestMergeTablesAcrossPages_ThreePageCumulativeShift(t *testing.T) {+ pageGrid := func(rows [][]string) [][]pdf.TSRCell {+ g := make([][]pdf.TSRCell, len(rows))+ for r, row := range rows {+ g[r] = make([]pdf.TSRCell, len(row))+ for c := range row {+ g[r][c] = pdf.TSRCell{+ X0: float64(c) * 100, Y0: float64(r) * 30,+ X1: float64(c)*100 + 100, Y1: float64(r)*30 + 30,+ Text: row[c],+ }+ }+ }+ return g+ }+ // Three pages, 2 rows × 2 cols each, identical layout.+ pages := []pdf.TableItem{+ {Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 0, Right: 200, Top: 0, Bottom: 60}}, Scale: 1.0, Grid: pageGrid([][]string{{"a", "b"}, {"c", "d"}})},+ {Positions: []pdf.Position{{PageNumbers: []int{1}, Left: 0, Right: 200, Top: 0, Bottom: 60}}, Scale: 1.0, Grid: pageGrid([][]string{{"e", "f"}, {"g", "h"}})},+ {Positions: []pdf.Position{{PageNumbers: []int{2}, Left: 0, Right: 200, Top: 0, Bottom: 60}}, Scale: 1.0, Grid: pageGrid([][]string{{"i", "j"}, {"k", "l"}})},+ }++ merged := MergeTablesAcrossPages(pages, nil)+ if len(merged) != 1 {+ t.Fatalf("expected 1 merged table, got %d", len(merged))+ }+ // 3 pages × 2 rows → 6 rows, in page order.+ if len(merged[0].Grid) != 6 {+ t.Fatalf("3-page merge must stack all rows (want 6), got %d", len(merged[0].Grid))+ }+ if merged[0].Grid[0][0].Text != "a" || merged[0].Grid[2][0].Text != "e" || merged[0].Grid[4][0].Text != "i" {+ t.Errorf("row order wrong after 3-page stacking: %s / %s / %s",+ merged[0].Grid[0][0].Text, merged[0].Grid[2][0].Text, merged[0].Grid[4][0].Text)+ }+ // Cumulative Y shift: page1 strictly below page0, page2 strictly below+ // page1, and page2 below page1 (monotonic accumulation).+ if merged[0].Grid[2][0].Y0 <= merged[0].Grid[1][0].Y1 {+ t.Errorf("page1 not shifted below page0 (Grid[2][0].Y0=%v <= Grid[1][0].Y1=%v)",+ merged[0].Grid[2][0].Y0, merged[0].Grid[1][0].Y1)+ }+ if merged[0].Grid[4][0].Y0 <= merged[0].Grid[3][0].Y1 {+ t.Errorf("page2 not shifted below page1 (Grid[4][0].Y0=%v <= Grid[3][0].Y1=%v)",+ merged[0].Grid[4][0].Y0, merged[0].Grid[3][0].Y1)+ }+ if merged[0].Grid[4][0].Y0 <= merged[0].Grid[2][0].Y0 {+ t.Errorf("Y shift not cumulative: page2 (Y0=%v) must be below page1 (Y0=%v)",+ merged[0].Grid[4][0].Y0, merged[0].Grid[2][0].Y0)+ }+}++// TestMergeTablesAcrossPages_GridlessAnchorUnchanged verifies that when the+// anchor has no Grid, MergeTablesAcrossPages skips the rebuild entirely and+// leaves anchor.Grid empty (nil) so ConstructTable falls back to the Cells+// path — the same pre-fix behaviour. This locks the no-regression promise the+// fix relies on for Grid-less tables (and the 7 pre-existing tests that set no+// Grid). The cross-page merge decision itself is unchanged: continuation+// Cells and Positions are still appended.+func TestMergeTablesAcrossPages_GridlessAnchorUnchanged(t *testing.T) {+ cells := func(texts []string) []pdf.TSRCell {+ cs := make([]pdf.TSRCell, len(texts))+ for i, txt := range texts {+ cs[i] = pdf.TSRCell{+ X0: 0, Y0: float64(i) * 30, X1: 100, Y1: float64(i)*30 + 30,+ Text: txt,+ }+ }+ return cs+ }+ // Anchor has Cells but NO Grid; continuation also Grid-less.+ pg0 := pdf.TableItem{+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 0, Right: 200, Top: 0, Bottom: 60}},+ Scale: 1.0,+ Cells: cells([]string{"a", "b", "c", "d"}),+ }+ pg1 := pdf.TableItem{+ Positions: []pdf.Position{{PageNumbers: []int{1}, Left: 0, Right: 200, Top: 0, Bottom: 60}},+ Scale: 1.0,+ Cells: cells([]string{"e", "f", "g", "h"}),+ }++ merged := MergeTablesAcrossPages([]pdf.TableItem{pg0, pg1}, nil)+ if len(merged) != 1 {+ t.Fatalf("expected 1 merged table, got %d", len(merged))+ }+ // Guard: anchor has no Grid → rebuild skipped entirely → Grid stays empty.+ if len(merged[0].Grid) != 0 {+ t.Fatalf("Grid-less anchor must keep Grid empty (rebuild skipped), got %d rows", len(merged[0].Grid))+ }+ // Merge decision unchanged: all continuation Cells still appended.+ have := map[string]bool{}+ for _, c := range merged[0].Cells {+ have[c.Text] = true+ }+ for _, want := range []string{"a", "b", "c", "d", "e", "f", "g", "h"} {+ if !have[want] {+ t.Errorf("merged Cells missing %q (merge decision changed for Grid-less anchor)", want)+ }+ }+ // Positions from both pages present → the cross-page merge did happen.+ pages := map[int]bool{}+ for _, p := range merged[0].Positions {+ for _, pn := range p.PageNumbers {+ pages[pn] = true+ }+ }+ if !pages[0] || !pages[1] {+ t.Errorf("cross-page merge did not combine both pages' positions: %v", pages)+ }+}