Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(deepdoc): scope table/box overlap to the same page to avoid cross-page expansion (#20089)

xugangqiang8a3d26e78a86
diff --git a/internal/deepdoc/parser/pdf/layout/chars_boxes.go b/internal/deepdoc/parser/pdf/layout/chars_boxes.go
index 4586f734e4..29cdb0fc04 100644
--- a/internal/deepdoc/parser/pdf/layout/chars_boxes.go
+++ b/internal/deepdoc/parser/pdf/layout/chars_boxes.go
@@ -46,6 +46,7 @@ func CharsToBoxes(chars []pdf.TextChar, pageNum int, sortByTop bool) []pdf.TextB
for _, sub := range subLines {
box := LineToTextBox(sub)
box.PageNumber = pageNum
+ box.HasPageNumber = true
boxes = append(boxes, box)
}
}
diff --git a/internal/deepdoc/parser/pdf/parser_ocr.go b/internal/deepdoc/parser/pdf/parser_ocr.go
index 9647be8b16..5b046fbf35 100644
--- a/internal/deepdoc/parser/pdf/parser_ocr.go
+++ b/internal/deepdoc/parser/pdf/parser_ocr.go
@@ -159,12 +159,13 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image,
for _, t := range best {
if strings.TrimSpace(t.Text) != "" {
result = append(result, pdf.TextBox{
- X0: pb.x0,
- X1: pb.x1,
- Top: pb.y0,
- Bottom: pb.y1,
- Text: t.Text,
- PageNumber: pb.pageNum,
+ X0: pb.x0,
+ X1: pb.x1,
+ Top: pb.y0,
+ Bottom: pb.y1,
+ Text: t.Text,
+ PageNumber: pb.pageNum,
+ HasPageNumber: true,
})
}
}
@@ -385,6 +386,7 @@ func (p *Parser) detectBoxes(ctx context.Context, pageImg image.Image, doc pdf.D
}
boxes = append(boxes, ocrDetectBox{box: pdf.TextBox{
X0: x0, X1: x1, Top: y0, Bottom: y1, PageNumber: pageNum,
+ HasPageNumber: true,
}, x0: x0, y0: y0, x1: x1, y1: y1, srcIdx: i})
}
diff --git a/internal/deepdoc/parser/pdf/table/box_overlap_page_test.go b/internal/deepdoc/parser/pdf/table/box_overlap_page_test.go
new file mode 100644
index 0000000000..a7955e212f
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/table/box_overlap_page_test.go
@@ -0,0 +1,134 @@
+package table
+
+import (
+ "testing"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// TestBoxOverlapsPositionPage locks the page-aware behavior of the table/box
+// overlap check. Table positions and table-layout boxes are both stored in
+// page-local coordinates (Y resets near 0 at the top of every page), so a
+// position's Y band is shared by the boxes of every page. The check must
+// therefore reject a box that overlaps only in X/Y but lives on a different
+// page; otherwise a single page-local position wrongly matches the same Y band
+// on all pages (the source of the multi-GB replacement cross-product and of
+// cross-page content mis-attribution).
+func TestBoxOverlapsPositionPage(t *testing.T) {
+ // A position on page 5 with a normal X/Y band.
+ pos := pdf.Position{PageNumbers: []int{5}, Left: 10, Right: 100, Top: 10, Bottom: 50}
+
+ // Same page + X/Y overlap -> true. HasPageNumber marks the page as real
+ // (page numbers are 0-based, so a missing-metadata box must opt out
+ // explicitly rather than relying on PageNumber != 0).
+ samePage := pdf.TextBox{PageNumber: 5, HasPageNumber: true, X0: 10, X1: 100, Top: 10, Bottom: 50}
+ if !boxOverlapsPositionPage(samePage, pos) {
+ t.Errorf("same-page X/Y overlap should be true")
+ }
+
+ // Different page but identical X/Y band -> false. This is the case the old
+ // (page-blind) check got wrong: it would return true and inflate reps.
+ crossPage := pdf.TextBox{PageNumber: 6, HasPageNumber: true, X0: 10, X1: 100, Top: 10, Bottom: 50}
+ if boxOverlapsPositionPage(crossPage, pos) {
+ t.Errorf("cross-page X/Y overlap must be false: page-local Y must not match across pages")
+ }
+
+ // Disjoint X/Y on a different page -> false (sanity, would be false either way).
+ disjoint := pdf.TextBox{PageNumber: 6, HasPageNumber: true, X0: 500, X1: 600, Top: 500, Bottom: 600}
+ if boxOverlapsPositionPage(disjoint, pos) {
+ t.Errorf("disjoint box should be false regardless of page")
+ }
+
+ // Missing page info on the position -> fall back to X/Y-only (overlap true).
+ noPagePos := pdf.Position{PageNumbers: nil, Left: 10, Right: 100, Top: 10, Bottom: 50}
+ if !boxOverlapsPositionPage(samePage, noPagePos) {
+ t.Errorf("fallback to X/Y-only expected when position has no page info")
+ }
+ emptyPagePos := pdf.Position{PageNumbers: []int{}, Left: 10, Right: 100, Top: 10, Bottom: 50}
+ if !boxOverlapsPositionPage(samePage, emptyPagePos) {
+ t.Errorf("fallback to X/Y-only expected when position PageNumbers is empty")
+ }
+
+ // Missing page info on the box (PageNumber == 0) -> fall back to X/Y-only.
+ noBoxPage := pdf.TextBox{PageNumber: 0, X0: 10, X1: 100, Top: 10, Bottom: 50}
+ if !boxOverlapsPositionPage(noBoxPage, pos) {
+ t.Errorf("fallback to X/Y-only expected when box has no page number")
+ }
+}
+
+// TestBoxOverlapsPositionPage_MergedTable locks that a table merged across
+// consecutive pages (MergeTablesAcrossPages appends every spanned page into
+// Position.PageNumbers, see table_merge.go) still claims the per-page
+// table-layout box that sits on one of its continuation pages. The page check
+// must accept the box when its single page is a member of the merged
+// position's multi-page PageNumbers set — not reject it just because the
+// position is no longer single-page. This is the cross-page-merge counterpart
+// of TestBoxOverlapsPositionPage: it guards against the page constraint
+// accidentally breaking legitimate cross-page merges while still rejecting
+// boxes on pages the merged table does not occupy.
+func TestBoxOverlapsPositionPage_MergedTable(t *testing.T) {
+ // A table merged across pages 4 and 5. Both positions share the same
+ // page-local X/Y band (Y resets near 0 on every page), so the only thing
+ // distinguishing them is the page set.
+ merged := pdf.Position{PageNumbers: []int{4, 5}, Left: 10, Right: 400, Top: 60, Bottom: 200}
+
+ // Box on the anchor page (4) -> must match.
+ anchorBox := pdf.TextBox{PageNumber: 4, HasPageNumber: true, X0: 10, X1: 400, Top: 60, Bottom: 200}
+ if !boxOverlapsPositionPage(anchorBox, merged) {
+ t.Errorf("anchor-page box of a merged table should match")
+ }
+
+ // Box on the continuation page (5) -> must still match. This is the case
+ // that proves the page constraint uses membership, not equality with a
+ // single page, so cross-page merges keep their boxes.
+ continuationBox := pdf.TextBox{PageNumber: 5, HasPageNumber: true, X0: 10, X1: 400, Top: 60, Bottom: 200}
+ if !boxOverlapsPositionPage(continuationBox, merged) {
+ t.Errorf("continuation-page box of a merged table should match")
+ }
+
+ // Box on a page the merged table does NOT span (6) -> must be rejected,
+ // even though the page-local X/Y band is identical. Without membership
+ // semantics this would wrongly inflate reps on large documents.
+ outsideBox := pdf.TextBox{PageNumber: 6, HasPageNumber: true, X0: 10, X1: 400, Top: 60, Bottom: 200}
+ if boxOverlapsPositionPage(outsideBox, merged) {
+ t.Errorf("box on a non-spanned page must be rejected even for a merged table")
+ }
+}
+
+// TestBoxOverlapsPositionPage_FirstPageZero is the regression test for the
+// page-0 bug: page numbers are 0-based, so the FIRST page legitimately carries
+// PageNumber == 0. A box on page 0 must be scoped to page 0, not matched
+// against the same page-local Y band on every other page. The bug this guards
+// against is the old `box.PageNumber != 0` guard, which treated PageNumber == 0
+// as "missing metadata" and exempted the entire first page from the page
+// constraint (re-introducing cross-page mis-attribution and the reps blow-up).
+//
+// This test fails against the unfixed code: with no HasPageNumber discrimination
+// the helper skips the page check for PageNumber == 0 and reports a page-5
+// position as overlapping a page-0 box.
+func TestBoxOverlapsPositionPage_FirstPageZero(t *testing.T) {
+ // Box genuinely on the first page (PageNumber == 0, HasPageNumber marks it
+ // as a real page rather than missing metadata).
+ boxP0 := pdf.TextBox{PageNumber: 0, HasPageNumber: true, X0: 10, X1: 100, Top: 10, Bottom: 50}
+
+ // Position on the same page 0 with identical X/Y -> must match.
+ posP0 := pdf.Position{PageNumbers: []int{0}, Left: 10, Right: 100, Top: 10, Bottom: 50}
+ if !boxOverlapsPositionPage(boxP0, posP0) {
+ t.Errorf("page-0 box should match a page-0 position")
+ }
+
+ // Position on a different page (5) but identical page-local X/Y band -> must
+ // NOT match. The old code skipped the page check for PageNumber == 0 and
+ // returned true here, which is the bug.
+ posP5 := pdf.Position{PageNumbers: []int{5}, Left: 10, Right: 100, Top: 10, Bottom: 50}
+ if boxOverlapsPositionPage(boxP0, posP5) {
+ t.Errorf("page-0 box must not match a page-5 position: page-local Y must not cross pages")
+ }
+
+ // A box built without page metadata (HasPageNumber left false) still falls
+ // back to the X/Y-only check so legacy call paths keep working.
+ legacyBox := pdf.TextBox{PageNumber: 0, X0: 10, X1: 100, Top: 10, Bottom: 50}
+ if !boxOverlapsPositionPage(legacyBox, posP5) {
+ t.Errorf("box without page metadata must fall back to X/Y-only")
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/table/table_post.go b/internal/deepdoc/parser/pdf/table/table_post.go
index 18fe5ee448..a65299f8df 100644
--- a/internal/deepdoc/parser/pdf/table/table_post.go
+++ b/internal/deepdoc/parser/pdf/table/table_post.go
@@ -149,7 +149,7 @@ func buildTableHTMLs(boxes []pdf.TextBox, tables []pdf.TableItem) map[int]string
continue
}
for _, tp := range tables[ti].Positions {
- if boxOverlapsPosition(boxes[i], tp) {
+ if boxOverlapsPositionPage(boxes[i], tp) {
tableBoxes = append(tableBoxes, boxes[i])
break
}
@@ -203,26 +203,81 @@ func insertTableBoxes(boxes []pdf.TextBox, tables []pdf.TableItem, removeSet map
// (r"(数据|资料|图表)*来源[:: ]") are removed entirely without replacement —
// matching Python's _extract_table_figure discard behavior.
+// pagePosition pairs a table index with one of its Position indices. It is used
+// to build a per-page index so callers only test a box against table positions
+// on the same page instead of the full cross product.
+type pagePosition struct {
+ tableIdx int
+ posIdx int
+}
+
+// indexTablePositions buckets each table Position by the pages it occupies. A
+// Position with no PageNumbers is page-agnostic — boxOverlapsPositionPage falls
+// back to an X/Y-only test for it — so it is returned separately as noPage and
+// must be tested against every box. all returns every (table, position) pair and
+// is used as the fallback when a box itself carries no page metadata.
+//
+// Correctness note: a box on page P can only match a Position whose PageNumbers
+// contains P (boxOverlapsPositionPage enforces this). A cross-page table whose
+// positions sit on pages 5, 6, 7 is therefore placed in byPage[5], [6] and [7];
+// a box on page 6 only sees the table's page-6 position. This is exactly
+// equivalent to the unindexed cross product and never drops a valid match.
+func indexTablePositions(tables []pdf.TableItem) (byPage map[int][]pagePosition, noPage []pagePosition, all []pagePosition) {
+ byPage = make(map[int][]pagePosition, len(tables))
+ for ti := range tables {
+ for pi := range tables[ti].Positions {
+ pos := tables[ti].Positions[pi]
+ pp := pagePosition{tableIdx: ti, posIdx: pi}
+ all = append(all, pp)
+ if len(pos.PageNumbers) == 0 {
+ noPage = append(noPage, pp)
+ continue
+ }
+ for _, p := range pos.PageNumbers {
+ byPage[p] = append(byPage[p], pp)
+ }
+ }
+ }
+ return byPage, noPage, all
+}
+
// MarkNoMergeTables traverses boxes in page order. When a caption, title, or
// reference immediately follows a table, the preceding table is marked NoMerge
// to prevent cross-page merge. Matches Python's nomerge_lout_no.
func MarkNoMergeTables(boxes []pdf.TextBox, tables []pdf.TableItem) {
+ byPage, noPage, all := indexTablePositions(tables)
var lastTableTI int = -1
for i := range boxes {
lt := boxes[i].LayoutType
if lt == pdf.LayoutTypeTable {
- matched := false
- for ti := range tables {
- for _, tp := range tables[ti].Positions {
- if boxOverlapsPosition(boxes[i], tp) {
- lastTableTI = ti
- matched = true
- break
- }
- }
+ // Restrict candidates to positions on this box's page, plus the
+ // page-agnostic positions. A box without page metadata (HasPageNumber
+ // false) falls back to the full set (boxOverlapsPositionPage skips
+ // the page check for it). Note we key on HasPageNumber, not on
+ // PageNumber == 0: page numbers are 0-based, so the legitimate first
+ // page carries PageNumber == 0 and must still be scoped to its page.
+ // The original cross-product keeps the highest-indexed table a box
+ // overlaps as lastTableTI; candidates are ordered by ascending table
+ // index, so assigning lastTableTI on every match leaves the highest
+ // index in place — matching the original semantics. seen avoids
+ // re-testing a table whose positions span several slots on the page.
+ var cands []pagePosition
+ if !boxes[i].HasPageNumber {
+ cands = all
+ } else {
+ cands = append(cands, byPage[boxes[i].PageNumber]...)
+ cands = append(cands, noPage...)
}
- if !matched {
- lastTableTI = -1
+ lastTableTI = -1
+ seen := make(map[int]bool)
+ for _, c := range cands {
+ if seen[c.tableIdx] {
+ continue
+ }
+ if boxOverlapsPositionPage(boxes[i], tables[c.tableIdx].Positions[c.posIdx]) {
+ seen[c.tableIdx] = true
+ lastTableTI = c.tableIdx
+ }
}
continue
}
@@ -255,18 +310,40 @@ func buildRemoveSet(boxes []pdf.TextBox) map[int]bool {
// buildReplacementsAfterMerge maps each table to overlapping table-layout boxes,
// producing the replacement list. Must be called AFTER MergeTablesAcrossPages so
// that tableIdx in each replacement refers to the correct merged-table slot.
+//
+// The match for a box on page P is restricted to table positions on page P (plus
+// any page-agnostic positions), via the per-page index built by
+// indexTablePositions. This turns the O(tables*boxes*positions) cross product
+// into a page-local scan; it is equivalent to the unindexed version because
+// boxOverlapsPositionPage already requires a box and position to share a page.
func buildReplacementsAfterMerge(boxes []pdf.TextBox, tables []pdf.TableItem, removeSet map[int]bool) []replacement {
+ byPage, noPage, all := indexTablePositions(tables)
var reps []replacement
- for ti := range tables {
- for i := range boxes {
- if boxes[i].LayoutType != pdf.LayoutTypeTable || removeSet[i] {
+ for i := range boxes {
+ if boxes[i].LayoutType != pdf.LayoutTypeTable || removeSet[i] {
+ continue
+ }
+ var cands []pagePosition
+ if !boxes[i].HasPageNumber {
+ cands = all
+ } else {
+ cands = append(cands, byPage[boxes[i].PageNumber]...)
+ cands = append(cands, noPage...)
+ }
+ // Emit one replacement per (table, box) overlap pair. A box can overlap
+ // several tables on its page (plus the page-agnostic positions), and the
+ // original cross-product implementation added a replacement for each such
+ // table — so we must not stop at the first match. seen de-duplicates by
+ // table: a table that spans several positions on the page is still a
+ // single replacement for this box.
+ seen := make(map[int]bool)
+ for _, c := range cands {
+ if seen[c.tableIdx] {
continue
}
- for _, tp := range tables[ti].Positions {
- if boxOverlapsPosition(boxes[i], tp) {
- reps = append(reps, replacement{tableIdx: ti, boxIdx: i})
- break
- }
+ if boxOverlapsPositionPage(boxes[i], tables[c.tableIdx].Positions[c.posIdx]) {
+ seen[c.tableIdx] = true
+ reps = append(reps, replacement{tableIdx: c.tableIdx, boxIdx: i})
}
}
}
@@ -442,8 +519,32 @@ func ConsolidateFigures(boxes []pdf.TextBox) []pdf.TextBox {
return FilterBoxesByRemoveSet(boxes, removeSet)
}
-// boxOverlapsPosition checks if a pdf.TextBox overlaps a pdf.Position with margin.
-func boxOverlapsPosition(box pdf.TextBox, pos pdf.Position) bool {
+// boxOverlapsPositionPage reports whether a pdf.TextBox overlaps a
+// pdf.Position, additionally requiring the box to live on a page the position
+// spans. Table positions and table-layout boxes are both stored in page-local
+// coordinates (Y resets to ~0 at the top of every page), so a position's Y
+// band is shared by the boxes of every page. Without the page constraint a
+// single page-local position matches the same Y band on all pages, which (a)
+// inflates the table/box replacement cross-product into a multi-GB reps slice
+// and (b) makes a table wrongly claim boxes that live on other pages. When page
+// metadata is missing on either side (an empty Position.PageNumbers, or a box
+// whose HasPageNumber is false) we fall back to the X/Y-only check so legacy
+// call paths keep working. HasPageNumber is used instead of `box.PageNumber !=
+// 0` because page numbers are 0-based: the legitimate first page has
+// PageNumber == 0 and must NOT be treated as "missing".
+func boxOverlapsPositionPage(box pdf.TextBox, pos pdf.Position) bool {
+ if len(pos.PageNumbers) > 0 && box.HasPageNumber {
+ onSamePage := false
+ for _, p := range pos.PageNumbers {
+ if p == box.PageNumber {
+ onSamePage = true
+ break
+ }
+ }
+ if !onSamePage {
+ return false
+ }
+ }
const margin = 2.0
return box.X0 <= pos.Right+margin && box.X1 >= pos.Left-margin &&
box.Top <= pos.Bottom+margin && box.Bottom >= pos.Top-margin
diff --git a/internal/deepdoc/parser/pdf/table/table_replace_bench_test.go b/internal/deepdoc/parser/pdf/table/table_replace_bench_test.go
new file mode 100644
index 0000000000..0005b7e07d
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/table/table_replace_bench_test.go
@@ -0,0 +1,77 @@
+package table
+
+import (
+ "math/rand"
+ "testing"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// buildSyntheticDoc constructs a multi-page document with tables that span a few
+// consecutive pages, mirroring a real PDF: the number of (table box, table
+// position) pairs the unindexed scan would enumerate is pages*boxes*tables,
+// while the per-page index only tests boxes against positions on their own page.
+func buildSyntheticDoc(pages, nBoxes, nTables, posPerTab int, seed int64) ([]pdf.TextBox, []pdf.TableItem) {
+ rng := rand.New(rand.NewSource(seed))
+ boxes := make([]pdf.TextBox, nBoxes)
+ for i := 0; i < nBoxes; i++ {
+ p := rng.Intn(pages) + 1
+ x0 := rng.Float64() * 1000
+ y0 := rng.Float64() * 1000
+ boxes[i] = pdf.TextBox{
+ PageNumber: p,
+ HasPageNumber: true,
+ X0: x0,
+ X1: x0 + rng.Float64()*50,
+ Top: y0,
+ Bottom: y0 + rng.Float64()*50,
+ LayoutType: pdf.LayoutTypeTable,
+ }
+ }
+ tables := make([]pdf.TableItem, nTables)
+ for ti := 0; ti < nTables; ti++ {
+ start := rng.Intn(pages) + 1
+ poss := make([]pdf.Position, 0, posPerTab)
+ for k := 0; k < posPerTab; k++ {
+ p := ((start + k - 1) % pages) + 1
+ x0 := rng.Float64() * 1000
+ y0 := rng.Float64() * 1000
+ poss = append(poss, pdf.Position{
+ PageNumbers: []int{p},
+ Left: x0,
+ Right: x0 + rng.Float64()*50,
+ Top: y0,
+ Bottom: y0 + rng.Float64()*50,
+ })
+ }
+ tables[ti] = pdf.TableItem{Positions: poss}
+ }
+ return boxes, tables
+}
+
+func BenchmarkBuildReplacementsAfterMergeIndexed(b *testing.B) {
+ const pages, nBoxes, nTables, posPerTab = 200, 5000, 400, 3
+ boxes, tables := buildSyntheticDoc(pages, nBoxes, nTables, posPerTab, 1)
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ _ = buildReplacementsAfterMerge(boxes, tables, nil)
+ }
+}
+
+func BenchmarkBuildReplacementsAfterMergeBrute(b *testing.B) {
+ const pages, nBoxes, nTables, posPerTab = 200, 5000, 400, 3
+ boxes, tables := buildSyntheticDoc(pages, nBoxes, nTables, posPerTab, 1)
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ _ = referenceBuildReplacementsAfterMerge(boxes, tables, nil)
+ }
+}
+
+func BenchmarkMarkNoMergeTablesIndexed(b *testing.B) {
+ const pages, nBoxes, nTables, posPerTab = 200, 5000, 400, 3
+ boxes, tables := buildSyntheticDoc(pages, nBoxes, nTables, posPerTab, 1)
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ MarkNoMergeTables(boxes, tables)
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/table/table_replace_opt_test.go b/internal/deepdoc/parser/pdf/table/table_replace_opt_test.go
new file mode 100644
index 0000000000..08918b9386
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/table/table_replace_opt_test.go
@@ -0,0 +1,257 @@
+package table
+
+import (
+ "maps"
+ "math/rand"
+ "testing"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// repKey identifies a replacement by (tableIdx, boxIdx) for set comparison,
+// since the per-page index may emit the same pairs in a different order than
+// the brute-force reference.
+type repKey struct{ ti, bi int }
+
+func repSet(reps []replacement) map[repKey]bool {
+ m := make(map[repKey]bool, len(reps))
+ for _, r := range reps {
+ m[repKey{r.tableIdx, r.boxIdx}] = true
+ }
+ return m
+}
+
+// referenceBuildReplacementsAfterMerge is the unoptimized O(tables*boxes*positions)
+// implementation, used as an oracle to prove the per-page index is equivalent.
+func referenceBuildReplacementsAfterMerge(boxes []pdf.TextBox, tables []pdf.TableItem, removeSet map[int]bool) []replacement {
+ var reps []replacement
+ for ti := range tables {
+ for i := range boxes {
+ if boxes[i].LayoutType != pdf.LayoutTypeTable || removeSet[i] {
+ continue
+ }
+ for _, tp := range tables[ti].Positions {
+ if boxOverlapsPositionPage(boxes[i], tp) {
+ reps = append(reps, replacement{tableIdx: ti, boxIdx: i})
+ break
+ }
+ }
+ }
+ }
+ return reps
+}
+
+// referenceMarkNoMergeTables mirrors MarkNoMergeTables with the full cross
+// product, so we can assert the indexed version mutates NoMerge identically.
+func referenceMarkNoMergeTables(boxes []pdf.TextBox, tables []pdf.TableItem) {
+ var lastTableTI int = -1
+ for i := range boxes {
+ lt := boxes[i].LayoutType
+ if lt == pdf.LayoutTypeTable {
+ matched := false
+ for ti := range tables {
+ for _, tp := range tables[ti].Positions {
+ if boxOverlapsPositionPage(boxes[i], tp) {
+ lastTableTI = ti
+ matched = true
+ break
+ }
+ }
+ if matched {
+ break
+ }
+ }
+ if !matched {
+ lastTableTI = -1
+ }
+ continue
+ }
+ if lastTableTI >= 0 && (lt == pdf.LayoutTypeTitle || lt == pdf.DLALabelTableCaption || lt == pdf.DLALabelFigureCaption || lt == pdf.LayoutTypeReference || IsCaptionBox(boxes[i].Text, lt)) {
+ tables[lastTableTI].NoMerge = true
+ }
+ }
+}
+
+// cloneTables deep-copies the NoMerge-relevant part of a table slice so the
+// two implementations can run against independent copies and be compared.
+func cloneTables(in []pdf.TableItem) []pdf.TableItem {
+ out := make([]pdf.TableItem, len(in))
+ for i := range in {
+ out[i] = pdf.TableItem{
+ Positions: in[i].Positions,
+ NoMerge: in[i].NoMerge,
+ }
+ }
+ return out
+}
+
+// TestBuildReplacementsAfterMergeCrossPageEquivalence checks the per-page index
+// against a brute-force oracle over randomly generated multi-page documents,
+// including tables that span several pages. This is the guard that the
+// optimization does not drop valid (cross-page) matches and does not invent
+// invalid ones.
+func TestBuildReplacementsAfterMergeCrossPageEquivalence(t *testing.T) {
+ rng := rand.New(rand.NewSource(1))
+ const (
+ pages = 12
+ nBoxes = 300
+ nTables = 80
+ posPerTab = 3
+ coordRange = 1000.0
+ )
+ boxes := make([]pdf.TextBox, nBoxes)
+ for i := 0; i < nBoxes; i++ {
+ p := rng.Intn(pages) + 1
+ x0 := rng.Float64() * coordRange
+ y0 := rng.Float64() * coordRange
+ boxes[i] = pdf.TextBox{
+ PageNumber: p,
+ HasPageNumber: true,
+ X0: x0,
+ X1: x0 + rng.Float64()*50,
+ Top: y0,
+ Bottom: y0 + rng.Float64()*50,
+ LayoutType: pdf.LayoutTypeTable,
+ }
+ }
+ tables := make([]pdf.TableItem, nTables)
+ for ti := 0; ti < nTables; ti++ {
+ // A table spans 1..posPerTab consecutive pages (a real cross-page table).
+ start := rng.Intn(pages) + 1
+ poss := make([]pdf.Position, 0, posPerTab)
+ for k := 0; k < posPerTab; k++ {
+ p := ((start + k - 1) % pages) + 1
+ x0 := rng.Float64() * coordRange
+ y0 := rng.Float64() * coordRange
+ poss = append(poss, pdf.Position{
+ PageNumbers: []int{p},
+ Left: x0,
+ Right: x0 + rng.Float64()*50,
+ Top: y0,
+ Bottom: y0 + rng.Float64()*50,
+ })
+ }
+ tables[ti] = pdf.TableItem{Positions: poss}
+ }
+ // Add a table whose positions carry NO page metadata: it is page-agnostic
+ // and must still match any box via the X/Y fallback (the noPage bucket).
+ noPageTable := pdf.TableItem{Positions: []pdf.Position{
+ {Left: 0, Right: coordRange, Top: 0, Bottom: coordRange},
+ }}
+ tables = append(tables, noPageTable)
+
+ got := buildReplacementsAfterMerge(boxes, tables, nil)
+ want := referenceBuildReplacementsAfterMerge(boxes, tables, nil)
+ gotSet, wantSet := repSet(got), repSet(want)
+ if !maps.Equal(gotSet, wantSet) {
+ var onlyGot, onlyWant []repKey
+ for k := range gotSet {
+ if !wantSet[k] {
+ onlyGot = append(onlyGot, k)
+ }
+ }
+ for k := range wantSet {
+ if !gotSet[k] {
+ onlyWant = append(onlyWant, k)
+ }
+ }
+ t.Errorf("indexed buildReplacementsAfterMerge diverges from brute-force reference:\n only-in-indexed=%v\n only-in-reference=%v", onlyGot, onlyWant)
+ }
+}
+
+// TestMarkNoMergeTablesCrossPageEquivalence mirrors the above for the NoMerge
+// marking path.
+func TestMarkNoMergeTablesCrossPageEquivalence(t *testing.T) {
+ rng := rand.New(rand.NewSource(7))
+ const (
+ pages = 10
+ nBoxes = 200
+ nTables = 40
+ posPerTab = 2
+ coordRange = 1000.0
+ )
+ boxes := make([]pdf.TextBox, nBoxes)
+ for i := 0; i < nBoxes; i++ {
+ p := rng.Intn(pages) + 1
+ x0 := rng.Float64() * coordRange
+ y0 := rng.Float64() * coordRange
+ lt := pdf.LayoutTypeTable
+ if i%5 == 0 {
+ lt = pdf.LayoutTypeTitle // a caption-like follower
+ }
+ boxes[i] = pdf.TextBox{
+ PageNumber: p,
+ HasPageNumber: true,
+ X0: x0,
+ X1: x0 + rng.Float64()*50,
+ Top: y0,
+ Bottom: y0 + rng.Float64()*50,
+ LayoutType: lt,
+ Text: "caption text",
+ }
+ }
+ tables := make([]pdf.TableItem, nTables)
+ for ti := 0; ti < nTables; ti++ {
+ start := rng.Intn(pages) + 1
+ poss := make([]pdf.Position, 0, posPerTab)
+ for k := 0; k < posPerTab; k++ {
+ p := ((start + k - 1) % pages) + 1
+ x0 := rng.Float64() * coordRange
+ y0 := rng.Float64() * coordRange
+ poss = append(poss, pdf.Position{
+ PageNumbers: []int{p},
+ Left: x0,
+ Right: x0 + rng.Float64()*50,
+ Top: y0,
+ Bottom: y0 + rng.Float64()*50,
+ })
+ }
+ tables[ti] = pdf.TableItem{Positions: poss}
+ }
+
+ indexed := cloneTables(tables)
+ MarkNoMergeTables(boxes, indexed)
+ ref := cloneTables(tables)
+ referenceMarkNoMergeTables(boxes, ref)
+
+ for i := range indexed {
+ if indexed[i].NoMerge != ref[i].NoMerge {
+ t.Errorf("table %d NoMerge mismatch: indexed=%v reference=%v", i, indexed[i].NoMerge, ref[i].NoMerge)
+ }
+ }
+}
+
+// TestMarkNoMergeTablesCrossPageSpanningTable pins the exact behavior the
+// optimization must preserve: a table that occupies pages 5 and 7 must be
+// matchable by boxes on BOTH of those pages, and must NOT be matched by a box
+// on an unrelated page.
+func TestMarkNoMergeTablesCrossPageSpanningTable(t *testing.T) {
+ spanTable := pdf.TableItem{Positions: []pdf.Position{
+ {PageNumbers: []int{5}, Left: 10, Right: 100, Top: 10, Bottom: 50},
+ {PageNumbers: []int{7}, Left: 10, Right: 100, Top: 10, Bottom: 50},
+ }}
+ otherTable := pdf.TableItem{Positions: []pdf.Position{
+ {PageNumbers: []int{6}, Left: 10, Right: 100, Top: 10, Bottom: 50},
+ }}
+
+ // Box on page 5 overlapping the span table's page-5 position.
+ boxP5 := pdf.TextBox{PageNumber: 5, HasPageNumber: true, X0: 10, X1: 100, Top: 10, Bottom: 50, LayoutType: pdf.LayoutTypeTable}
+ // Box on page 7 overlapping the span table's page-7 position.
+ boxP7 := pdf.TextBox{PageNumber: 7, HasPageNumber: true, X0: 10, X1: 100, Top: 10, Bottom: 50, LayoutType: pdf.LayoutTypeTable}
+ // Box on page 6 with identical coordinates but no span-table position there.
+ boxP6 := pdf.TextBox{PageNumber: 6, HasPageNumber: true, X0: 10, X1: 100, Top: 10, Bottom: 50, LayoutType: pdf.LayoutTypeTable}
+ // A caption following on page 5 should mark the span table NoMerge.
+ capP5 := pdf.TextBox{PageNumber: 5, HasPageNumber: true, X0: 10, X1: 100, Top: 60, Bottom: 70, LayoutType: pdf.LayoutTypeTitle, Text: "表 1"}
+
+ boxes := []pdf.TextBox{boxP5, boxP6, boxP7, capP5}
+ tables := []pdf.TableItem{spanTable, otherTable}
+
+ MarkNoMergeTables(boxes, tables)
+
+ if !tables[0].NoMerge {
+ t.Errorf("spanning table (pages 5&7) should be marked NoMerge by the page-5 caption")
+ }
+ if tables[1].NoMerge {
+ t.Errorf("page-6 table must NOT be marked NoMerge: page-6 box did not match it")
+ }
+}
diff --git a/internal/deepdoc/parser/type/types.go b/internal/deepdoc/parser/type/types.go
index 1c175a286b..cd87566e23 100644
--- a/internal/deepdoc/parser/type/types.go
+++ b/internal/deepdoc/parser/type/types.go
@@ -101,6 +101,13 @@ type TextBox struct {
Top, Bottom float64
Text string
PageNumber int
+ // HasPageNumber distinguishes a box whose PageNumber is a real page index
+ // (including page 0, the first page, which is 0-based) from a box that was
+ // built without page metadata. Without it, code that wants to skip the
+ // page check when metadata is absent cannot tell "page 0" apart from
+ // "no page", so the legitimate first page would be treated as missing and
+ // matched against every other page's positions.
+ HasPageNumber bool
// Pages carries the full set of page numbers a box spans, when it is a
// single logical region split across consecutive pages (e.g. a table
// merged across pages by MergeTablesAcrossPages). When non-empty it