Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): remove duplicate figures for extracted PDF tables (#21673)

jay77721fc766ce3d756
diff --git a/internal/deepdoc/parser/pdf/table/table_construct.go b/internal/deepdoc/parser/pdf/table/table_construct.go
index 6400a2157e..e56119e2b7 100644
--- a/internal/deepdoc/parser/pdf/table/table_construct.go
+++ b/internal/deepdoc/parser/pdf/table/table_construct.go
@@ -183,7 +183,8 @@ func isDataSourceBox(text string) bool {
// handles test TableItems or bare engines without DLA.
func tableRegionBox(tbl *pdf.TableItem, ref *pdf.TextBox, html string) pdf.TextBox {
pg := 0
- if len(tbl.Positions) > 0 && len(tbl.Positions[0].PageNumbers) > 0 {
+ hasPageNumber := len(tbl.Positions) > 0 && len(tbl.Positions[0].PageNumbers) > 0
+ if hasPageNumber {
pg = tbl.Positions[0].PageNumbers[0]
}
// A table merged across consecutive pages (MergeTablesAcrossPages appends
@@ -197,27 +198,29 @@ func tableRegionBox(tbl *pdf.TableItem, ref *pdf.TextBox, html string) pdf.TextB
// Use DLA region boundaries when set.
if tbl.RegionLeft != 0 || tbl.RegionRight != 0 || tbl.RegionTop != 0 || tbl.RegionBottom != 0 {
return pdf.TextBox{
- X0: tbl.RegionLeft,
- X1: tbl.RegionRight,
- Top: tbl.RegionTop,
- Bottom: tbl.RegionBottom,
- Text: html,
- PageNumber: pg,
- Pages: pages,
- LayoutType: pdf.LayoutTypeTable,
+ X0: tbl.RegionLeft,
+ X1: tbl.RegionRight,
+ Top: tbl.RegionTop,
+ Bottom: tbl.RegionBottom,
+ Text: html,
+ PageNumber: pg,
+ HasPageNumber: hasPageNumber,
+ Pages: pages,
+ LayoutType: pdf.LayoutTypeTable,
}
}
// Fallback: use anchor box coordinates.
x0, x1, top, bot := ref.X0, ref.X1, ref.Top, ref.Bottom
return pdf.TextBox{
- X0: x0,
- X1: x1,
- Top: top,
- Bottom: bot,
- Text: html,
- PageNumber: pg,
- Pages: pages,
- LayoutType: pdf.LayoutTypeTable,
+ X0: x0,
+ X1: x1,
+ Top: top,
+ Bottom: bot,
+ Text: html,
+ PageNumber: pg,
+ HasPageNumber: hasPageNumber,
+ Pages: pages,
+ LayoutType: pdf.LayoutTypeTable,
}
}
diff --git a/internal/deepdoc/parser/pdf/table/table_post.go b/internal/deepdoc/parser/pdf/table/table_post.go
index 36aff2ed2a..c3d1d7bc1d 100644
--- a/internal/deepdoc/parser/pdf/table/table_post.go
+++ b/internal/deepdoc/parser/pdf/table/table_post.go
@@ -3,6 +3,7 @@ package table
import (
"math"
"sort"
+ "strings"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
)
@@ -40,14 +41,15 @@ func createTableBoxFromItem(tbl *pdf.TableItem, html string) pdf.TextBox {
x0, x1, top, bottom = p.Left, p.Right, p.Top, p.Bottom
}
return pdf.TextBox{
- X0: x0,
- X1: x1,
- Top: top,
- Bottom: bottom,
- Text: html,
- PageNumber: pg,
- Pages: mergedTablePages(tbl),
- LayoutType: pdf.LayoutTypeTable,
+ X0: x0,
+ X1: x1,
+ Top: top,
+ Bottom: bottom,
+ Text: html,
+ PageNumber: pg,
+ HasPageNumber: len(tbl.Positions) > 0 && len(tbl.Positions[0].PageNumbers) > 0,
+ Pages: mergedTablePages(tbl),
+ LayoutType: pdf.LayoutTypeTable,
}
}
@@ -625,9 +627,36 @@ func ConsolidateFigures(boxes []pdf.TextBox) []pdf.TextBox {
}
applyFigureBounds(boxes)
+ markDuplicateTableFigures(boxes, removeSet)
return FilterBoxesByRemoveSet(boxes, removeSet)
}
+// Remove only empty figure regions duplicating a successfully extracted table.
+// Nested regions and captioned figures retain their independent content.
+func markDuplicateTableFigures(boxes []pdf.TextBox, removeSet map[int]bool) {
+ tables := make(map[int][]pdf.TextBox)
+ for i, b := range boxes {
+ if removeSet[i] || b.LayoutType != pdf.LayoutTypeTable || !b.HasPageNumber || len(b.Pages) > 1 ||
+ !strings.HasPrefix(strings.TrimSpace(b.Text), "<table>") || b.X1 <= b.X0 || b.Bottom <= b.Top {
+ continue
+ }
+ tables[b.PageNumber] = append(tables[b.PageNumber], b)
+ }
+ const tolerance = 1e-6 // PDF-point rounding noise, not a geometric overlap threshold.
+ for i, b := range boxes {
+ if removeSet[i] || b.LayoutType != pdf.LayoutTypeFigure || !b.HasPageNumber || len(b.Pages) > 1 || strings.TrimSpace(b.Text) != "" {
+ continue
+ }
+ for _, table := range tables[b.PageNumber] {
+ if math.Abs(b.X0-table.X0) <= tolerance && math.Abs(b.X1-table.X1) <= tolerance &&
+ math.Abs(b.Top-table.Top) <= tolerance && math.Abs(b.Bottom-table.Bottom) <= tolerance {
+ removeSet[i] = true
+ break
+ }
+ }
+ }
+}
+
// applyFigureBounds pins every figure box to its matched DLA region.
//
// The DLA region is the crop contract: a figure section's geometry must be
diff --git a/internal/deepdoc/parser/pdf/table/table_post_test.go b/internal/deepdoc/parser/pdf/table/table_post_test.go
index 9fc8d74e51..07a40a69ef 100644
--- a/internal/deepdoc/parser/pdf/table/table_post_test.go
+++ b/internal/deepdoc/parser/pdf/table/table_post_test.go
@@ -1038,3 +1038,74 @@ func TestExtractTableAndReplace_NoReMergeAfterPageAbsoluteRejection(t *testing.T
t.Errorf("ExtractTableAndReplace re-merged the page-absolute-rejected tables: got %d table boxes, want 2 (the two pages-4/5 tables must stay separate)", got)
}
}
+
+func TestConsolidateFiguresRemovesOnlyDuplicateTableGeometry(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ change func(*pdf.TextBox, *pdf.TextBox)
+ want int
+ }{
+ {"same region", func(*pdf.TextBox, *pdf.TextBox) {}, 1},
+ {"floating point noise", func(_ *pdf.TextBox, f *pdf.TextBox) { f.FigureBounds.Right += 1e-8 }, 1},
+ {"different page", func(_ *pdf.TextBox, f *pdf.TextBox) { f.PageNumber++ }, 2},
+ {"missing page", func(t *pdf.TextBox, _ *pdf.TextBox) { t.HasPageNumber = false }, 2},
+ {"missing figure page", func(_ *pdf.TextBox, f *pdf.TextBox) { f.HasPageNumber = false }, 2},
+ {"nested image", func(_ *pdf.TextBox, f *pdf.TextBox) { f.FigureBounds.Right = 50 }, 2},
+ {"containing image", func(_ *pdf.TextBox, f *pdf.TextBox) { f.FigureBounds.Right = 200 }, 2},
+ {"caption", func(_ *pdf.TextBox, f *pdf.TextBox) { f.Text = "independent caption" }, 2},
+ {"unsuccessful table", func(t *pdf.TextBox, _ *pdf.TextBox) { t.Text = "unprocessed text" }, 2},
+ {"cross page table", func(t *pdf.TextBox, _ *pdf.TextBox) { t.Pages = []int{0, 1} }, 2},
+ {"cross page figure", func(_ *pdf.TextBox, f *pdf.TextBox) { f.Pages = []int{0, 1} }, 2},
+ {"different region", func(_ *pdf.TextBox, f *pdf.TextBox) { f.FigureBounds.Right += 1 }, 2},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ table := tableRegionBox(&pdf.TableItem{Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 100, Top: 20, Bottom: 80}}, RegionLeft: 10, RegionRight: 100, RegionTop: 20, RegionBottom: 80}, &pdf.TextBox{}, "<table><tr><td>value</td></tr></table>")
+ figure := pdf.TextBox{LayoutType: pdf.LayoutTypeFigure, LayoutNo: "figure-0", HasPageNumber: true, PageNumber: 0, FigureBounds: &pdf.Position{Left: 10, Right: 100, Top: 20, Bottom: 80}}
+ tc.change(&table, &figure)
+ got := ConsolidateFigures([]pdf.TextBox{table, figure})
+ if len(got) != tc.want {
+ t.Fatalf("boxes=%d want=%d: %+v", len(got), tc.want, got)
+ }
+ if got[0].Text != table.Text {
+ t.Fatal("table text changed")
+ }
+ })
+ }
+}
+
+func TestConsolidateFiguresRequiresKnownTableAnchorPage(t *testing.T) {
+ const html = "<table><tr><td>value</td></tr></table>"
+ for _, builder := range []struct {
+ name string
+ build func(*pdf.TableItem) pdf.TextBox
+ }{
+ {"region", func(tbl *pdf.TableItem) pdf.TextBox { return tableRegionBox(tbl, &pdf.TextBox{}, html) }},
+ {"anchor bounds", func(tbl *pdf.TableItem) pdf.TextBox {
+ copy := *tbl
+ copy.RegionLeft, copy.RegionRight, copy.RegionTop, copy.RegionBottom = 0, 0, 0, 0
+ return tableRegionBox(&copy, &pdf.TextBox{X0: 10, X1: 100, Top: 20, Bottom: 80}, html)
+ }},
+ {"image only", func(tbl *pdf.TableItem) pdf.TextBox { return createTableBoxFromItem(tbl, html) }},
+ } {
+ for _, tc := range []struct {
+ name string
+ positions []pdf.Position
+ page int
+ want int
+ }{
+ {"first page", []pdf.Position{{PageNumbers: []int{0}}}, 0, 1},
+ {"later page", []pdf.Position{{PageNumbers: []int{5}}}, 5, 1},
+ {"missing anchor page", []pdf.Position{{}, {PageNumbers: []int{5}}}, 0, 2},
+ {"missing pages", nil, 0, 2},
+ } {
+ t.Run(builder.name+"/"+tc.name, func(t *testing.T) {
+ table := builder.build(&pdf.TableItem{Positions: tc.positions, RegionLeft: 10, RegionRight: 100, RegionTop: 20, RegionBottom: 80})
+ figure := pdf.TextBox{LayoutType: pdf.LayoutTypeFigure, HasPageNumber: true, PageNumber: tc.page, FigureBounds: &pdf.Position{Left: 10, Right: 100, Top: 20, Bottom: 80}}
+ got := ConsolidateFigures([]pdf.TextBox{table, figure})
+ if len(got) != tc.want {
+ t.Fatalf("boxes=%d want=%d: %+v", len(got), tc.want, got)
+ }
+ })
+ }
+ }
+}