RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): remove duplicate figures for extracted PDF tables (#21673)
jay77721fc766ce3d756
diff --git a/internal/deepdoc/parser/pdf/table/table_construct.go b/internal/deepdoc/parser/pdf/table/table_construct.goindex 6400a2157e..e56119e2b7 100644--- a/internal/deepdoc/parser/pdf/table/table_construct.go+++ b/internal/deepdoc/parser/pdf/table/table_construct.go@@ -183,7 +183,8 @@ func isDataSourceBox(text string) bool {// handles test TableItems or bare engines without DLA.func tableRegionBox(tbl *pdf.TableItem, ref *pdf.TextBox, html string) pdf.TextBox {pg := 0- if len(tbl.Positions) > 0 && len(tbl.Positions[0].PageNumbers) > 0 {+ hasPageNumber := len(tbl.Positions) > 0 && len(tbl.Positions[0].PageNumbers) > 0+ if hasPageNumber {pg = tbl.Positions[0].PageNumbers[0]}// A table merged across consecutive pages (MergeTablesAcrossPages appends@@ -197,27 +198,29 @@ func tableRegionBox(tbl *pdf.TableItem, ref *pdf.TextBox, html string) pdf.TextB// Use DLA region boundaries when set.if tbl.RegionLeft != 0 || tbl.RegionRight != 0 || tbl.RegionTop != 0 || tbl.RegionBottom != 0 {return pdf.TextBox{- X0: tbl.RegionLeft,- X1: tbl.RegionRight,- Top: tbl.RegionTop,- Bottom: tbl.RegionBottom,- Text: html,- PageNumber: pg,- Pages: pages,- LayoutType: pdf.LayoutTypeTable,+ X0: tbl.RegionLeft,+ X1: tbl.RegionRight,+ Top: tbl.RegionTop,+ Bottom: tbl.RegionBottom,+ Text: html,+ PageNumber: pg,+ HasPageNumber: hasPageNumber,+ Pages: pages,+ LayoutType: pdf.LayoutTypeTable,}}// Fallback: use anchor box coordinates.x0, x1, top, bot := ref.X0, ref.X1, ref.Top, ref.Bottomreturn pdf.TextBox{- X0: x0,- X1: x1,- Top: top,- Bottom: bot,- Text: html,- PageNumber: pg,- Pages: pages,- LayoutType: pdf.LayoutTypeTable,+ X0: x0,+ X1: x1,+ Top: top,+ Bottom: bot,+ Text: html,+ PageNumber: pg,+ HasPageNumber: hasPageNumber,+ Pages: pages,+ LayoutType: pdf.LayoutTypeTable,}}diff --git a/internal/deepdoc/parser/pdf/table/table_post.go b/internal/deepdoc/parser/pdf/table/table_post.goindex 36aff2ed2a..c3d1d7bc1d 100644--- a/internal/deepdoc/parser/pdf/table/table_post.go+++ b/internal/deepdoc/parser/pdf/table/table_post.go@@ -3,6 +3,7 @@ package tableimport ("math""sort"+ "strings"pdf "ragflow/internal/deepdoc/parser/pdf/type")@@ -40,14 +41,15 @@ func createTableBoxFromItem(tbl *pdf.TableItem, html string) pdf.TextBox {x0, x1, top, bottom = p.Left, p.Right, p.Top, p.Bottom}return pdf.TextBox{- X0: x0,- X1: x1,- Top: top,- Bottom: bottom,- Text: html,- PageNumber: pg,- Pages: mergedTablePages(tbl),- LayoutType: pdf.LayoutTypeTable,+ X0: x0,+ X1: x1,+ Top: top,+ Bottom: bottom,+ Text: html,+ PageNumber: pg,+ HasPageNumber: len(tbl.Positions) > 0 && len(tbl.Positions[0].PageNumbers) > 0,+ Pages: mergedTablePages(tbl),+ LayoutType: pdf.LayoutTypeTable,}}@@ -625,9 +627,36 @@ func ConsolidateFigures(boxes []pdf.TextBox) []pdf.TextBox {}applyFigureBounds(boxes)+ markDuplicateTableFigures(boxes, removeSet)return FilterBoxesByRemoveSet(boxes, removeSet)}+// Remove only empty figure regions duplicating a successfully extracted table.+// Nested regions and captioned figures retain their independent content.+func markDuplicateTableFigures(boxes []pdf.TextBox, removeSet map[int]bool) {+ tables := make(map[int][]pdf.TextBox)+ for i, b := range boxes {+ if removeSet[i] || b.LayoutType != pdf.LayoutTypeTable || !b.HasPageNumber || len(b.Pages) > 1 ||+ !strings.HasPrefix(strings.TrimSpace(b.Text), "<table>") || b.X1 <= b.X0 || b.Bottom <= b.Top {+ continue+ }+ tables[b.PageNumber] = append(tables[b.PageNumber], b)+ }+ const tolerance = 1e-6 // PDF-point rounding noise, not a geometric overlap threshold.+ for i, b := range boxes {+ if removeSet[i] || b.LayoutType != pdf.LayoutTypeFigure || !b.HasPageNumber || len(b.Pages) > 1 || strings.TrimSpace(b.Text) != "" {+ continue+ }+ for _, table := range tables[b.PageNumber] {+ if math.Abs(b.X0-table.X0) <= tolerance && math.Abs(b.X1-table.X1) <= tolerance &&+ math.Abs(b.Top-table.Top) <= tolerance && math.Abs(b.Bottom-table.Bottom) <= tolerance {+ removeSet[i] = true+ break+ }+ }+ }+}+// applyFigureBounds pins every figure box to its matched DLA region.//// The DLA region is the crop contract: a figure section's geometry must bediff --git a/internal/deepdoc/parser/pdf/table/table_post_test.go b/internal/deepdoc/parser/pdf/table/table_post_test.goindex 9fc8d74e51..07a40a69ef 100644--- a/internal/deepdoc/parser/pdf/table/table_post_test.go+++ b/internal/deepdoc/parser/pdf/table/table_post_test.go@@ -1038,3 +1038,74 @@ func TestExtractTableAndReplace_NoReMergeAfterPageAbsoluteRejection(t *testing.Tt.Errorf("ExtractTableAndReplace re-merged the page-absolute-rejected tables: got %d table boxes, want 2 (the two pages-4/5 tables must stay separate)", got)}}++func TestConsolidateFiguresRemovesOnlyDuplicateTableGeometry(t *testing.T) {+ for _, tc := range []struct {+ name string+ change func(*pdf.TextBox, *pdf.TextBox)+ want int+ }{+ {"same region", func(*pdf.TextBox, *pdf.TextBox) {}, 1},+ {"floating point noise", func(_ *pdf.TextBox, f *pdf.TextBox) { f.FigureBounds.Right += 1e-8 }, 1},+ {"different page", func(_ *pdf.TextBox, f *pdf.TextBox) { f.PageNumber++ }, 2},+ {"missing page", func(t *pdf.TextBox, _ *pdf.TextBox) { t.HasPageNumber = false }, 2},+ {"missing figure page", func(_ *pdf.TextBox, f *pdf.TextBox) { f.HasPageNumber = false }, 2},+ {"nested image", func(_ *pdf.TextBox, f *pdf.TextBox) { f.FigureBounds.Right = 50 }, 2},+ {"containing image", func(_ *pdf.TextBox, f *pdf.TextBox) { f.FigureBounds.Right = 200 }, 2},+ {"caption", func(_ *pdf.TextBox, f *pdf.TextBox) { f.Text = "independent caption" }, 2},+ {"unsuccessful table", func(t *pdf.TextBox, _ *pdf.TextBox) { t.Text = "unprocessed text" }, 2},+ {"cross page table", func(t *pdf.TextBox, _ *pdf.TextBox) { t.Pages = []int{0, 1} }, 2},+ {"cross page figure", func(_ *pdf.TextBox, f *pdf.TextBox) { f.Pages = []int{0, 1} }, 2},+ {"different region", func(_ *pdf.TextBox, f *pdf.TextBox) { f.FigureBounds.Right += 1 }, 2},+ } {+ t.Run(tc.name, func(t *testing.T) {+ table := tableRegionBox(&pdf.TableItem{Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 100, Top: 20, Bottom: 80}}, RegionLeft: 10, RegionRight: 100, RegionTop: 20, RegionBottom: 80}, &pdf.TextBox{}, "<table><tr><td>value</td></tr></table>")+ figure := pdf.TextBox{LayoutType: pdf.LayoutTypeFigure, LayoutNo: "figure-0", HasPageNumber: true, PageNumber: 0, FigureBounds: &pdf.Position{Left: 10, Right: 100, Top: 20, Bottom: 80}}+ tc.change(&table, &figure)+ got := ConsolidateFigures([]pdf.TextBox{table, figure})+ if len(got) != tc.want {+ t.Fatalf("boxes=%d want=%d: %+v", len(got), tc.want, got)+ }+ if got[0].Text != table.Text {+ t.Fatal("table text changed")+ }+ })+ }+}++func TestConsolidateFiguresRequiresKnownTableAnchorPage(t *testing.T) {+ const html = "<table><tr><td>value</td></tr></table>"+ for _, builder := range []struct {+ name string+ build func(*pdf.TableItem) pdf.TextBox+ }{+ {"region", func(tbl *pdf.TableItem) pdf.TextBox { return tableRegionBox(tbl, &pdf.TextBox{}, html) }},+ {"anchor bounds", func(tbl *pdf.TableItem) pdf.TextBox {+ copy := *tbl+ copy.RegionLeft, copy.RegionRight, copy.RegionTop, copy.RegionBottom = 0, 0, 0, 0+ return tableRegionBox(©, &pdf.TextBox{X0: 10, X1: 100, Top: 20, Bottom: 80}, html)+ }},+ {"image only", func(tbl *pdf.TableItem) pdf.TextBox { return createTableBoxFromItem(tbl, html) }},+ } {+ for _, tc := range []struct {+ name string+ positions []pdf.Position+ page int+ want int+ }{+ {"first page", []pdf.Position{{PageNumbers: []int{0}}}, 0, 1},+ {"later page", []pdf.Position{{PageNumbers: []int{5}}}, 5, 1},+ {"missing anchor page", []pdf.Position{{}, {PageNumbers: []int{5}}}, 0, 2},+ {"missing pages", nil, 0, 2},+ } {+ t.Run(builder.name+"/"+tc.name, func(t *testing.T) {+ table := builder.build(&pdf.TableItem{Positions: tc.positions, RegionLeft: 10, RegionRight: 100, RegionTop: 20, RegionBottom: 80})+ figure := pdf.TextBox{LayoutType: pdf.LayoutTypeFigure, HasPageNumber: true, PageNumber: tc.page, FigureBounds: &pdf.Position{Left: 10, Right: 100, Top: 20, Bottom: 80}}+ got := ConsolidateFigures([]pdf.TextBox{table, figure})+ if len(got) != tc.want {+ t.Fatalf("boxes=%d want=%d: %+v", len(got), tc.want, got)+ }+ })+ }+ }+}