RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(deepdoc): scope table/box overlap to the same page to avoid cross-page expansion (#20089)
xugangqiang8a3d26e78a86
diff --git a/internal/deepdoc/parser/pdf/layout/chars_boxes.go b/internal/deepdoc/parser/pdf/layout/chars_boxes.goindex 4586f734e4..29cdb0fc04 100644--- a/internal/deepdoc/parser/pdf/layout/chars_boxes.go+++ b/internal/deepdoc/parser/pdf/layout/chars_boxes.go@@ -46,6 +46,7 @@ func CharsToBoxes(chars []pdf.TextChar, pageNum int, sortByTop bool) []pdf.TextBfor _, sub := range subLines {box := LineToTextBox(sub)box.PageNumber = pageNum+ box.HasPageNumber = trueboxes = append(boxes, box)}}diff --git a/internal/deepdoc/parser/pdf/parser_ocr.go b/internal/deepdoc/parser/pdf/parser_ocr.goindex 9647be8b16..5b046fbf35 100644--- a/internal/deepdoc/parser/pdf/parser_ocr.go+++ b/internal/deepdoc/parser/pdf/parser_ocr.go@@ -159,12 +159,13 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image,for _, t := range best {if strings.TrimSpace(t.Text) != "" {result = append(result, pdf.TextBox{- X0: pb.x0,- X1: pb.x1,- Top: pb.y0,- Bottom: pb.y1,- Text: t.Text,- PageNumber: pb.pageNum,+ X0: pb.x0,+ X1: pb.x1,+ Top: pb.y0,+ Bottom: pb.y1,+ Text: t.Text,+ PageNumber: pb.pageNum,+ HasPageNumber: true,})}}@@ -385,6 +386,7 @@ func (p *Parser) detectBoxes(ctx context.Context, pageImg image.Image, doc pdf.D}boxes = append(boxes, ocrDetectBox{box: pdf.TextBox{X0: x0, X1: x1, Top: y0, Bottom: y1, PageNumber: pageNum,+ HasPageNumber: true,}, x0: x0, y0: y0, x1: x1, y1: y1, srcIdx: i})}diff --git a/internal/deepdoc/parser/pdf/table/box_overlap_page_test.go b/internal/deepdoc/parser/pdf/table/box_overlap_page_test.gonew file mode 100644index 0000000000..a7955e212f--- /dev/null+++ b/internal/deepdoc/parser/pdf/table/box_overlap_page_test.go@@ -0,0 +1,134 @@+package table++import (+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// TestBoxOverlapsPositionPage locks the page-aware behavior of the table/box+// overlap check. Table positions and table-layout boxes are both stored in+// page-local coordinates (Y resets near 0 at the top of every page), so a+// position's Y band is shared by the boxes of every page. The check must+// therefore reject a box that overlaps only in X/Y but lives on a different+// page; otherwise a single page-local position wrongly matches the same Y band+// on all pages (the source of the multi-GB replacement cross-product and of+// cross-page content mis-attribution).+func TestBoxOverlapsPositionPage(t *testing.T) {+ // A position on page 5 with a normal X/Y band.+ pos := pdf.Position{PageNumbers: []int{5}, Left: 10, Right: 100, Top: 10, Bottom: 50}++ // Same page + X/Y overlap -> true. HasPageNumber marks the page as real+ // (page numbers are 0-based, so a missing-metadata box must opt out+ // explicitly rather than relying on PageNumber != 0).+ samePage := pdf.TextBox{PageNumber: 5, HasPageNumber: true, X0: 10, X1: 100, Top: 10, Bottom: 50}+ if !boxOverlapsPositionPage(samePage, pos) {+ t.Errorf("same-page X/Y overlap should be true")+ }++ // Different page but identical X/Y band -> false. This is the case the old+ // (page-blind) check got wrong: it would return true and inflate reps.+ crossPage := pdf.TextBox{PageNumber: 6, HasPageNumber: true, X0: 10, X1: 100, Top: 10, Bottom: 50}+ if boxOverlapsPositionPage(crossPage, pos) {+ t.Errorf("cross-page X/Y overlap must be false: page-local Y must not match across pages")+ }++ // Disjoint X/Y on a different page -> false (sanity, would be false either way).+ disjoint := pdf.TextBox{PageNumber: 6, HasPageNumber: true, X0: 500, X1: 600, Top: 500, Bottom: 600}+ if boxOverlapsPositionPage(disjoint, pos) {+ t.Errorf("disjoint box should be false regardless of page")+ }++ // Missing page info on the position -> fall back to X/Y-only (overlap true).+ noPagePos := pdf.Position{PageNumbers: nil, Left: 10, Right: 100, Top: 10, Bottom: 50}+ if !boxOverlapsPositionPage(samePage, noPagePos) {+ t.Errorf("fallback to X/Y-only expected when position has no page info")+ }+ emptyPagePos := pdf.Position{PageNumbers: []int{}, Left: 10, Right: 100, Top: 10, Bottom: 50}+ if !boxOverlapsPositionPage(samePage, emptyPagePos) {+ t.Errorf("fallback to X/Y-only expected when position PageNumbers is empty")+ }++ // Missing page info on the box (PageNumber == 0) -> fall back to X/Y-only.+ noBoxPage := pdf.TextBox{PageNumber: 0, X0: 10, X1: 100, Top: 10, Bottom: 50}+ if !boxOverlapsPositionPage(noBoxPage, pos) {+ t.Errorf("fallback to X/Y-only expected when box has no page number")+ }+}++// TestBoxOverlapsPositionPage_MergedTable locks that a table merged across+// consecutive pages (MergeTablesAcrossPages appends every spanned page into+// Position.PageNumbers, see table_merge.go) still claims the per-page+// table-layout box that sits on one of its continuation pages. The page check+// must accept the box when its single page is a member of the merged+// position's multi-page PageNumbers set — not reject it just because the+// position is no longer single-page. This is the cross-page-merge counterpart+// of TestBoxOverlapsPositionPage: it guards against the page constraint+// accidentally breaking legitimate cross-page merges while still rejecting+// boxes on pages the merged table does not occupy.+func TestBoxOverlapsPositionPage_MergedTable(t *testing.T) {+ // A table merged across pages 4 and 5. Both positions share the same+ // page-local X/Y band (Y resets near 0 on every page), so the only thing+ // distinguishing them is the page set.+ merged := pdf.Position{PageNumbers: []int{4, 5}, Left: 10, Right: 400, Top: 60, Bottom: 200}++ // Box on the anchor page (4) -> must match.+ anchorBox := pdf.TextBox{PageNumber: 4, HasPageNumber: true, X0: 10, X1: 400, Top: 60, Bottom: 200}+ if !boxOverlapsPositionPage(anchorBox, merged) {+ t.Errorf("anchor-page box of a merged table should match")+ }++ // Box on the continuation page (5) -> must still match. This is the case+ // that proves the page constraint uses membership, not equality with a+ // single page, so cross-page merges keep their boxes.+ continuationBox := pdf.TextBox{PageNumber: 5, HasPageNumber: true, X0: 10, X1: 400, Top: 60, Bottom: 200}+ if !boxOverlapsPositionPage(continuationBox, merged) {+ t.Errorf("continuation-page box of a merged table should match")+ }++ // Box on a page the merged table does NOT span (6) -> must be rejected,+ // even though the page-local X/Y band is identical. Without membership+ // semantics this would wrongly inflate reps on large documents.+ outsideBox := pdf.TextBox{PageNumber: 6, HasPageNumber: true, X0: 10, X1: 400, Top: 60, Bottom: 200}+ if boxOverlapsPositionPage(outsideBox, merged) {+ t.Errorf("box on a non-spanned page must be rejected even for a merged table")+ }+}++// TestBoxOverlapsPositionPage_FirstPageZero is the regression test for the+// page-0 bug: page numbers are 0-based, so the FIRST page legitimately carries+// PageNumber == 0. A box on page 0 must be scoped to page 0, not matched+// against the same page-local Y band on every other page. The bug this guards+// against is the old `box.PageNumber != 0` guard, which treated PageNumber == 0+// as "missing metadata" and exempted the entire first page from the page+// constraint (re-introducing cross-page mis-attribution and the reps blow-up).+//+// This test fails against the unfixed code: with no HasPageNumber discrimination+// the helper skips the page check for PageNumber == 0 and reports a page-5+// position as overlapping a page-0 box.+func TestBoxOverlapsPositionPage_FirstPageZero(t *testing.T) {+ // Box genuinely on the first page (PageNumber == 0, HasPageNumber marks it+ // as a real page rather than missing metadata).+ boxP0 := pdf.TextBox{PageNumber: 0, HasPageNumber: true, X0: 10, X1: 100, Top: 10, Bottom: 50}++ // Position on the same page 0 with identical X/Y -> must match.+ posP0 := pdf.Position{PageNumbers: []int{0}, Left: 10, Right: 100, Top: 10, Bottom: 50}+ if !boxOverlapsPositionPage(boxP0, posP0) {+ t.Errorf("page-0 box should match a page-0 position")+ }++ // Position on a different page (5) but identical page-local X/Y band -> must+ // NOT match. The old code skipped the page check for PageNumber == 0 and+ // returned true here, which is the bug.+ posP5 := pdf.Position{PageNumbers: []int{5}, Left: 10, Right: 100, Top: 10, Bottom: 50}+ if boxOverlapsPositionPage(boxP0, posP5) {+ t.Errorf("page-0 box must not match a page-5 position: page-local Y must not cross pages")+ }++ // A box built without page metadata (HasPageNumber left false) still falls+ // back to the X/Y-only check so legacy call paths keep working.+ legacyBox := pdf.TextBox{PageNumber: 0, X0: 10, X1: 100, Top: 10, Bottom: 50}+ if !boxOverlapsPositionPage(legacyBox, posP5) {+ t.Errorf("box without page metadata must fall back to X/Y-only")+ }+}diff --git a/internal/deepdoc/parser/pdf/table/table_post.go b/internal/deepdoc/parser/pdf/table/table_post.goindex 18fe5ee448..a65299f8df 100644--- a/internal/deepdoc/parser/pdf/table/table_post.go+++ b/internal/deepdoc/parser/pdf/table/table_post.go@@ -149,7 +149,7 @@ func buildTableHTMLs(boxes []pdf.TextBox, tables []pdf.TableItem) map[int]stringcontinue}for _, tp := range tables[ti].Positions {- if boxOverlapsPosition(boxes[i], tp) {+ if boxOverlapsPositionPage(boxes[i], tp) {tableBoxes = append(tableBoxes, boxes[i])break}@@ -203,26 +203,81 @@ func insertTableBoxes(boxes []pdf.TextBox, tables []pdf.TableItem, removeSet map// (r"(数据|资料|图表)*来源[:: ]") are removed entirely without replacement —// matching Python's _extract_table_figure discard behavior.+// pagePosition pairs a table index with one of its Position indices. It is used+// to build a per-page index so callers only test a box against table positions+// on the same page instead of the full cross product.+type pagePosition struct {+ tableIdx int+ posIdx int+}++// indexTablePositions buckets each table Position by the pages it occupies. A+// Position with no PageNumbers is page-agnostic — boxOverlapsPositionPage falls+// back to an X/Y-only test for it — so it is returned separately as noPage and+// must be tested against every box. all returns every (table, position) pair and+// is used as the fallback when a box itself carries no page metadata.+//+// Correctness note: a box on page P can only match a Position whose PageNumbers+// contains P (boxOverlapsPositionPage enforces this). A cross-page table whose+// positions sit on pages 5, 6, 7 is therefore placed in byPage[5], [6] and [7];+// a box on page 6 only sees the table's page-6 position. This is exactly+// equivalent to the unindexed cross product and never drops a valid match.+func indexTablePositions(tables []pdf.TableItem) (byPage map[int][]pagePosition, noPage []pagePosition, all []pagePosition) {+ byPage = make(map[int][]pagePosition, len(tables))+ for ti := range tables {+ for pi := range tables[ti].Positions {+ pos := tables[ti].Positions[pi]+ pp := pagePosition{tableIdx: ti, posIdx: pi}+ all = append(all, pp)+ if len(pos.PageNumbers) == 0 {+ noPage = append(noPage, pp)+ continue+ }+ for _, p := range pos.PageNumbers {+ byPage[p] = append(byPage[p], pp)+ }+ }+ }+ return byPage, noPage, all+}+// MarkNoMergeTables traverses boxes in page order. When a caption, title, or// reference immediately follows a table, the preceding table is marked NoMerge// to prevent cross-page merge. Matches Python's nomerge_lout_no.func MarkNoMergeTables(boxes []pdf.TextBox, tables []pdf.TableItem) {+ byPage, noPage, all := indexTablePositions(tables)var lastTableTI int = -1for i := range boxes {lt := boxes[i].LayoutTypeif lt == pdf.LayoutTypeTable {- matched := false- for ti := range tables {- for _, tp := range tables[ti].Positions {- if boxOverlapsPosition(boxes[i], tp) {- lastTableTI = ti- matched = true- break- }- }+ // Restrict candidates to positions on this box's page, plus the+ // page-agnostic positions. A box without page metadata (HasPageNumber+ // false) falls back to the full set (boxOverlapsPositionPage skips+ // the page check for it). Note we key on HasPageNumber, not on+ // PageNumber == 0: page numbers are 0-based, so the legitimate first+ // page carries PageNumber == 0 and must still be scoped to its page.+ // The original cross-product keeps the highest-indexed table a box+ // overlaps as lastTableTI; candidates are ordered by ascending table+ // index, so assigning lastTableTI on every match leaves the highest+ // index in place — matching the original semantics. seen avoids+ // re-testing a table whose positions span several slots on the page.+ var cands []pagePosition+ if !boxes[i].HasPageNumber {+ cands = all+ } else {+ cands = append(cands, byPage[boxes[i].PageNumber]...)+ cands = append(cands, noPage...)}- if !matched {- lastTableTI = -1+ lastTableTI = -1+ seen := make(map[int]bool)+ for _, c := range cands {+ if seen[c.tableIdx] {+ continue+ }+ if boxOverlapsPositionPage(boxes[i], tables[c.tableIdx].Positions[c.posIdx]) {+ seen[c.tableIdx] = true+ lastTableTI = c.tableIdx+ }}continue}@@ -255,18 +310,40 @@ func buildRemoveSet(boxes []pdf.TextBox) map[int]bool {// buildReplacementsAfterMerge maps each table to overlapping table-layout boxes,// producing the replacement list. Must be called AFTER MergeTablesAcrossPages so// that tableIdx in each replacement refers to the correct merged-table slot.+//+// The match for a box on page P is restricted to table positions on page P (plus+// any page-agnostic positions), via the per-page index built by+// indexTablePositions. This turns the O(tables*boxes*positions) cross product+// into a page-local scan; it is equivalent to the unindexed version because+// boxOverlapsPositionPage already requires a box and position to share a page.func buildReplacementsAfterMerge(boxes []pdf.TextBox, tables []pdf.TableItem, removeSet map[int]bool) []replacement {+ byPage, noPage, all := indexTablePositions(tables)var reps []replacement- for ti := range tables {- for i := range boxes {- if boxes[i].LayoutType != pdf.LayoutTypeTable || removeSet[i] {+ for i := range boxes {+ if boxes[i].LayoutType != pdf.LayoutTypeTable || removeSet[i] {+ continue+ }+ var cands []pagePosition+ if !boxes[i].HasPageNumber {+ cands = all+ } else {+ cands = append(cands, byPage[boxes[i].PageNumber]...)+ cands = append(cands, noPage...)+ }+ // Emit one replacement per (table, box) overlap pair. A box can overlap+ // several tables on its page (plus the page-agnostic positions), and the+ // original cross-product implementation added a replacement for each such+ // table — so we must not stop at the first match. seen de-duplicates by+ // table: a table that spans several positions on the page is still a+ // single replacement for this box.+ seen := make(map[int]bool)+ for _, c := range cands {+ if seen[c.tableIdx] {continue}- for _, tp := range tables[ti].Positions {- if boxOverlapsPosition(boxes[i], tp) {- reps = append(reps, replacement{tableIdx: ti, boxIdx: i})- break- }+ if boxOverlapsPositionPage(boxes[i], tables[c.tableIdx].Positions[c.posIdx]) {+ seen[c.tableIdx] = true+ reps = append(reps, replacement{tableIdx: c.tableIdx, boxIdx: i})}}}@@ -442,8 +519,32 @@ func ConsolidateFigures(boxes []pdf.TextBox) []pdf.TextBox {return FilterBoxesByRemoveSet(boxes, removeSet)}-// boxOverlapsPosition checks if a pdf.TextBox overlaps a pdf.Position with margin.-func boxOverlapsPosition(box pdf.TextBox, pos pdf.Position) bool {+// boxOverlapsPositionPage reports whether a pdf.TextBox overlaps a+// pdf.Position, additionally requiring the box to live on a page the position+// spans. Table positions and table-layout boxes are both stored in page-local+// coordinates (Y resets to ~0 at the top of every page), so a position's Y+// band is shared by the boxes of every page. Without the page constraint a+// single page-local position matches the same Y band on all pages, which (a)+// inflates the table/box replacement cross-product into a multi-GB reps slice+// and (b) makes a table wrongly claim boxes that live on other pages. When page+// metadata is missing on either side (an empty Position.PageNumbers, or a box+// whose HasPageNumber is false) we fall back to the X/Y-only check so legacy+// call paths keep working. HasPageNumber is used instead of `box.PageNumber !=+// 0` because page numbers are 0-based: the legitimate first page has+// PageNumber == 0 and must NOT be treated as "missing".+func boxOverlapsPositionPage(box pdf.TextBox, pos pdf.Position) bool {+ if len(pos.PageNumbers) > 0 && box.HasPageNumber {+ onSamePage := false+ for _, p := range pos.PageNumbers {+ if p == box.PageNumber {+ onSamePage = true+ break+ }+ }+ if !onSamePage {+ return false+ }+ }const margin = 2.0return box.X0 <= pos.Right+margin && box.X1 >= pos.Left-margin &&box.Top <= pos.Bottom+margin && box.Bottom >= pos.Top-margindiff --git a/internal/deepdoc/parser/pdf/table/table_replace_bench_test.go b/internal/deepdoc/parser/pdf/table/table_replace_bench_test.gonew file mode 100644index 0000000000..0005b7e07d--- /dev/null+++ b/internal/deepdoc/parser/pdf/table/table_replace_bench_test.go@@ -0,0 +1,77 @@+package table++import (+ "math/rand"+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// buildSyntheticDoc constructs a multi-page document with tables that span a few+// consecutive pages, mirroring a real PDF: the number of (table box, table+// position) pairs the unindexed scan would enumerate is pages*boxes*tables,+// while the per-page index only tests boxes against positions on their own page.+func buildSyntheticDoc(pages, nBoxes, nTables, posPerTab int, seed int64) ([]pdf.TextBox, []pdf.TableItem) {+ rng := rand.New(rand.NewSource(seed))+ boxes := make([]pdf.TextBox, nBoxes)+ for i := 0; i < nBoxes; i++ {+ p := rng.Intn(pages) + 1+ x0 := rng.Float64() * 1000+ y0 := rng.Float64() * 1000+ boxes[i] = pdf.TextBox{+ PageNumber: p,+ HasPageNumber: true,+ X0: x0,+ X1: x0 + rng.Float64()*50,+ Top: y0,+ Bottom: y0 + rng.Float64()*50,+ LayoutType: pdf.LayoutTypeTable,+ }+ }+ tables := make([]pdf.TableItem, nTables)+ for ti := 0; ti < nTables; ti++ {+ start := rng.Intn(pages) + 1+ poss := make([]pdf.Position, 0, posPerTab)+ for k := 0; k < posPerTab; k++ {+ p := ((start + k - 1) % pages) + 1+ x0 := rng.Float64() * 1000+ y0 := rng.Float64() * 1000+ poss = append(poss, pdf.Position{+ PageNumbers: []int{p},+ Left: x0,+ Right: x0 + rng.Float64()*50,+ Top: y0,+ Bottom: y0 + rng.Float64()*50,+ })+ }+ tables[ti] = pdf.TableItem{Positions: poss}+ }+ return boxes, tables+}++func BenchmarkBuildReplacementsAfterMergeIndexed(b *testing.B) {+ const pages, nBoxes, nTables, posPerTab = 200, 5000, 400, 3+ boxes, tables := buildSyntheticDoc(pages, nBoxes, nTables, posPerTab, 1)+ b.ResetTimer()+ for i := 0; i < b.N; i++ {+ _ = buildReplacementsAfterMerge(boxes, tables, nil)+ }+}++func BenchmarkBuildReplacementsAfterMergeBrute(b *testing.B) {+ const pages, nBoxes, nTables, posPerTab = 200, 5000, 400, 3+ boxes, tables := buildSyntheticDoc(pages, nBoxes, nTables, posPerTab, 1)+ b.ResetTimer()+ for i := 0; i < b.N; i++ {+ _ = referenceBuildReplacementsAfterMerge(boxes, tables, nil)+ }+}++func BenchmarkMarkNoMergeTablesIndexed(b *testing.B) {+ const pages, nBoxes, nTables, posPerTab = 200, 5000, 400, 3+ boxes, tables := buildSyntheticDoc(pages, nBoxes, nTables, posPerTab, 1)+ b.ResetTimer()+ for i := 0; i < b.N; i++ {+ MarkNoMergeTables(boxes, tables)+ }+}diff --git a/internal/deepdoc/parser/pdf/table/table_replace_opt_test.go b/internal/deepdoc/parser/pdf/table/table_replace_opt_test.gonew file mode 100644index 0000000000..08918b9386--- /dev/null+++ b/internal/deepdoc/parser/pdf/table/table_replace_opt_test.go@@ -0,0 +1,257 @@+package table++import (+ "maps"+ "math/rand"+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// repKey identifies a replacement by (tableIdx, boxIdx) for set comparison,+// since the per-page index may emit the same pairs in a different order than+// the brute-force reference.+type repKey struct{ ti, bi int }++func repSet(reps []replacement) map[repKey]bool {+ m := make(map[repKey]bool, len(reps))+ for _, r := range reps {+ m[repKey{r.tableIdx, r.boxIdx}] = true+ }+ return m+}++// referenceBuildReplacementsAfterMerge is the unoptimized O(tables*boxes*positions)+// implementation, used as an oracle to prove the per-page index is equivalent.+func referenceBuildReplacementsAfterMerge(boxes []pdf.TextBox, tables []pdf.TableItem, removeSet map[int]bool) []replacement {+ var reps []replacement+ for ti := range tables {+ for i := range boxes {+ if boxes[i].LayoutType != pdf.LayoutTypeTable || removeSet[i] {+ continue+ }+ for _, tp := range tables[ti].Positions {+ if boxOverlapsPositionPage(boxes[i], tp) {+ reps = append(reps, replacement{tableIdx: ti, boxIdx: i})+ break+ }+ }+ }+ }+ return reps+}++// referenceMarkNoMergeTables mirrors MarkNoMergeTables with the full cross+// product, so we can assert the indexed version mutates NoMerge identically.+func referenceMarkNoMergeTables(boxes []pdf.TextBox, tables []pdf.TableItem) {+ var lastTableTI int = -1+ for i := range boxes {+ lt := boxes[i].LayoutType+ if lt == pdf.LayoutTypeTable {+ matched := false+ for ti := range tables {+ for _, tp := range tables[ti].Positions {+ if boxOverlapsPositionPage(boxes[i], tp) {+ lastTableTI = ti+ matched = true+ break+ }+ }+ if matched {+ break+ }+ }+ if !matched {+ lastTableTI = -1+ }+ continue+ }+ if lastTableTI >= 0 && (lt == pdf.LayoutTypeTitle || lt == pdf.DLALabelTableCaption || lt == pdf.DLALabelFigureCaption || lt == pdf.LayoutTypeReference || IsCaptionBox(boxes[i].Text, lt)) {+ tables[lastTableTI].NoMerge = true+ }+ }+}++// cloneTables deep-copies the NoMerge-relevant part of a table slice so the+// two implementations can run against independent copies and be compared.+func cloneTables(in []pdf.TableItem) []pdf.TableItem {+ out := make([]pdf.TableItem, len(in))+ for i := range in {+ out[i] = pdf.TableItem{+ Positions: in[i].Positions,+ NoMerge: in[i].NoMerge,+ }+ }+ return out+}++// TestBuildReplacementsAfterMergeCrossPageEquivalence checks the per-page index+// against a brute-force oracle over randomly generated multi-page documents,+// including tables that span several pages. This is the guard that the+// optimization does not drop valid (cross-page) matches and does not invent+// invalid ones.+func TestBuildReplacementsAfterMergeCrossPageEquivalence(t *testing.T) {+ rng := rand.New(rand.NewSource(1))+ const (+ pages = 12+ nBoxes = 300+ nTables = 80+ posPerTab = 3+ coordRange = 1000.0+ )+ boxes := make([]pdf.TextBox, nBoxes)+ for i := 0; i < nBoxes; i++ {+ p := rng.Intn(pages) + 1+ x0 := rng.Float64() * coordRange+ y0 := rng.Float64() * coordRange+ boxes[i] = pdf.TextBox{+ PageNumber: p,+ HasPageNumber: true,+ X0: x0,+ X1: x0 + rng.Float64()*50,+ Top: y0,+ Bottom: y0 + rng.Float64()*50,+ LayoutType: pdf.LayoutTypeTable,+ }+ }+ tables := make([]pdf.TableItem, nTables)+ for ti := 0; ti < nTables; ti++ {+ // A table spans 1..posPerTab consecutive pages (a real cross-page table).+ start := rng.Intn(pages) + 1+ poss := make([]pdf.Position, 0, posPerTab)+ for k := 0; k < posPerTab; k++ {+ p := ((start + k - 1) % pages) + 1+ x0 := rng.Float64() * coordRange+ y0 := rng.Float64() * coordRange+ poss = append(poss, pdf.Position{+ PageNumbers: []int{p},+ Left: x0,+ Right: x0 + rng.Float64()*50,+ Top: y0,+ Bottom: y0 + rng.Float64()*50,+ })+ }+ tables[ti] = pdf.TableItem{Positions: poss}+ }+ // Add a table whose positions carry NO page metadata: it is page-agnostic+ // and must still match any box via the X/Y fallback (the noPage bucket).+ noPageTable := pdf.TableItem{Positions: []pdf.Position{+ {Left: 0, Right: coordRange, Top: 0, Bottom: coordRange},+ }}+ tables = append(tables, noPageTable)++ got := buildReplacementsAfterMerge(boxes, tables, nil)+ want := referenceBuildReplacementsAfterMerge(boxes, tables, nil)+ gotSet, wantSet := repSet(got), repSet(want)+ if !maps.Equal(gotSet, wantSet) {+ var onlyGot, onlyWant []repKey+ for k := range gotSet {+ if !wantSet[k] {+ onlyGot = append(onlyGot, k)+ }+ }+ for k := range wantSet {+ if !gotSet[k] {+ onlyWant = append(onlyWant, k)+ }+ }+ t.Errorf("indexed buildReplacementsAfterMerge diverges from brute-force reference:\n only-in-indexed=%v\n only-in-reference=%v", onlyGot, onlyWant)+ }+}++// TestMarkNoMergeTablesCrossPageEquivalence mirrors the above for the NoMerge+// marking path.+func TestMarkNoMergeTablesCrossPageEquivalence(t *testing.T) {+ rng := rand.New(rand.NewSource(7))+ const (+ pages = 10+ nBoxes = 200+ nTables = 40+ posPerTab = 2+ coordRange = 1000.0+ )+ boxes := make([]pdf.TextBox, nBoxes)+ for i := 0; i < nBoxes; i++ {+ p := rng.Intn(pages) + 1+ x0 := rng.Float64() * coordRange+ y0 := rng.Float64() * coordRange+ lt := pdf.LayoutTypeTable+ if i%5 == 0 {+ lt = pdf.LayoutTypeTitle // a caption-like follower+ }+ boxes[i] = pdf.TextBox{+ PageNumber: p,+ HasPageNumber: true,+ X0: x0,+ X1: x0 + rng.Float64()*50,+ Top: y0,+ Bottom: y0 + rng.Float64()*50,+ LayoutType: lt,+ Text: "caption text",+ }+ }+ tables := make([]pdf.TableItem, nTables)+ for ti := 0; ti < nTables; ti++ {+ start := rng.Intn(pages) + 1+ poss := make([]pdf.Position, 0, posPerTab)+ for k := 0; k < posPerTab; k++ {+ p := ((start + k - 1) % pages) + 1+ x0 := rng.Float64() * coordRange+ y0 := rng.Float64() * coordRange+ poss = append(poss, pdf.Position{+ PageNumbers: []int{p},+ Left: x0,+ Right: x0 + rng.Float64()*50,+ Top: y0,+ Bottom: y0 + rng.Float64()*50,+ })+ }+ tables[ti] = pdf.TableItem{Positions: poss}+ }++ indexed := cloneTables(tables)+ MarkNoMergeTables(boxes, indexed)+ ref := cloneTables(tables)+ referenceMarkNoMergeTables(boxes, ref)++ for i := range indexed {+ if indexed[i].NoMerge != ref[i].NoMerge {+ t.Errorf("table %d NoMerge mismatch: indexed=%v reference=%v", i, indexed[i].NoMerge, ref[i].NoMerge)+ }+ }+}++// TestMarkNoMergeTablesCrossPageSpanningTable pins the exact behavior the+// optimization must preserve: a table that occupies pages 5 and 7 must be+// matchable by boxes on BOTH of those pages, and must NOT be matched by a box+// on an unrelated page.+func TestMarkNoMergeTablesCrossPageSpanningTable(t *testing.T) {+ spanTable := pdf.TableItem{Positions: []pdf.Position{+ {PageNumbers: []int{5}, Left: 10, Right: 100, Top: 10, Bottom: 50},+ {PageNumbers: []int{7}, Left: 10, Right: 100, Top: 10, Bottom: 50},+ }}+ otherTable := pdf.TableItem{Positions: []pdf.Position{+ {PageNumbers: []int{6}, Left: 10, Right: 100, Top: 10, Bottom: 50},+ }}++ // Box on page 5 overlapping the span table's page-5 position.+ boxP5 := pdf.TextBox{PageNumber: 5, HasPageNumber: true, X0: 10, X1: 100, Top: 10, Bottom: 50, LayoutType: pdf.LayoutTypeTable}+ // Box on page 7 overlapping the span table's page-7 position.+ boxP7 := pdf.TextBox{PageNumber: 7, HasPageNumber: true, X0: 10, X1: 100, Top: 10, Bottom: 50, LayoutType: pdf.LayoutTypeTable}+ // Box on page 6 with identical coordinates but no span-table position there.+ boxP6 := pdf.TextBox{PageNumber: 6, HasPageNumber: true, X0: 10, X1: 100, Top: 10, Bottom: 50, LayoutType: pdf.LayoutTypeTable}+ // A caption following on page 5 should mark the span table NoMerge.+ capP5 := pdf.TextBox{PageNumber: 5, HasPageNumber: true, X0: 10, X1: 100, Top: 60, Bottom: 70, LayoutType: pdf.LayoutTypeTitle, Text: "表 1"}++ boxes := []pdf.TextBox{boxP5, boxP6, boxP7, capP5}+ tables := []pdf.TableItem{spanTable, otherTable}++ MarkNoMergeTables(boxes, tables)++ if !tables[0].NoMerge {+ t.Errorf("spanning table (pages 5&7) should be marked NoMerge by the page-5 caption")+ }+ if tables[1].NoMerge {+ t.Errorf("page-6 table must NOT be marked NoMerge: page-6 box did not match it")+ }+}diff --git a/internal/deepdoc/parser/type/types.go b/internal/deepdoc/parser/type/types.goindex 1c175a286b..cd87566e23 100644--- a/internal/deepdoc/parser/type/types.go+++ b/internal/deepdoc/parser/type/types.go@@ -101,6 +101,13 @@ type TextBox struct {Top, Bottom float64Text stringPageNumber int+ // HasPageNumber distinguishes a box whose PageNumber is a real page index+ // (including page 0, the first page, which is 0-based) from a box that was+ // built without page metadata. Without it, code that wants to skip the+ // page check when metadata is absent cannot tell "page 0" apart from+ // "no page", so the legitimate first page would be treated as missing and+ // matched against every other page's positions.+ HasPageNumber bool// Pages carries the full set of page numbers a box spans, when it is a// single logical region split across consecutive pages (e.g. a table// merged across pages by MergeTablesAcrossPages). When non-empty it