Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(pdf): drop text boxes only when replaced by a table HTML (#18363)

Jackbd01c78d2ba7
diff --git a/internal/deepdoc/parser/pdf/table/table_post.go b/internal/deepdoc/parser/pdf/table/table_post.go
index 0ce7066554..cdda3ac538 100644
--- a/internal/deepdoc/parser/pdf/table/table_post.go
+++ b/internal/deepdoc/parser/pdf/table/table_post.go
@@ -318,11 +318,19 @@ func processTablesWithReplacements(
removeSet map[int]bool,
replacements []replacement,
) []pdf.TextBox {
- for _, r := range replacements {
- removeSet[r.boxIdx] = true
- }
anchors := findTableAnchorsWithReplacements(boxes, tables, replacements)
htmls := buildTableHTMLs(boxes, tables)
+ // A box is removed only when it is actually replaced by a table HTML box.
+ // A DLA-tagged region that produced no table HTML (no cells, or a
+ // degenerate table whose ConstructTable returned "") keeps its original
+ // text as prose instead of being silently dropped. Marking per replacement
+ // (rather than un-marking empties) also avoids duplicating content when one
+ // box is covered by both an empty and a real table.
+ for _, r := range replacements {
+ if html, ok := htmls[r.tableIdx]; ok && html != "" {
+ removeSet[r.boxIdx] = true
+ }
+ }
anchorList := buildAndSortAnchors(anchors)
return insertTableBoxes(boxes, tables, removeSet, anchorList, htmls)
}
diff --git a/internal/deepdoc/parser/pdf/table/table_post_test.go b/internal/deepdoc/parser/pdf/table/table_post_test.go
index ec7846c197..3b536b42ce 100644
--- a/internal/deepdoc/parser/pdf/table/table_post_test.go
+++ b/internal/deepdoc/parser/pdf/table/table_post_test.go
@@ -800,3 +800,127 @@ func TestExtractTableAndReplace_MergeTablesAcrossPages(t *testing.T) {
t.Error("non-table boxes should be preserved in original order")
}
}
+
+// TestProcessTablesWithReplacements_KeepsTextWhenNoCells pins the contract that
+// a text box mislabeled LayoutTypeTable by DLA is kept as prose when TSR yields
+// no cells (HTML missing), instead of being silently dropped.
+func TestProcessTablesWithReplacements_KeepsTextWhenNoCells(t *testing.T) {
+ const body = "REFERENCE [1] body text that must not vanish"
+ boxes := []pdf.TextBox{
+ {Text: "para one", PageNumber: 1, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 0, Bottom: 10},
+ {Text: body, PageNumber: 1, LayoutType: pdf.LayoutTypeTable, X0: 0, X1: 100, Top: 12, Bottom: 22},
+ {Text: "para three", PageNumber: 1, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 24, Bottom: 34},
+ }
+ // DLA table region but no TSR cells → buildTableHTMLs skips, htmls[0] unset.
+ tables := []pdf.TableItem{{}}
+ removeSet := map[int]bool{}
+ replacements := []replacement{{tableIdx: 0, boxIdx: 1}}
+
+ out := processTablesWithReplacements(boxes, tables, removeSet, replacements)
+
+ expectedTexts := []string{"para one", body, "para three"}
+ if len(out) != len(expectedTexts) {
+ t.Fatalf("over-labeled 'table' box was dropped: want %d boxes, got %d: %+v", len(expectedTexts), len(out), out)
+ }
+ for i, want := range expectedTexts {
+ if out[i].Text != want {
+ t.Errorf("output box %d: want text %q, got %q (%+v)", i, want, out[i].Text, out)
+ }
+ }
+}
+
+// TestProcessTablesWithReplacements_NoDupWhenCoveredByEmptyAndRealTable pins the
+// boundary where a single box is covered by TWO replacements — one whose table
+// has no cells (empty HTML) and one whose table has real cells (non-empty HTML)
+// — so the box must NOT be kept while the real table's HTML is also inserted
+// (that would duplicate the content). The box must be removed and only the real
+// table's HTML inserted.
+func TestProcessTablesWithReplacements_NoDupWhenCoveredByEmptyAndRealTable(t *testing.T) {
+ const body = "SHARED BODY spanning two table regions"
+ boxes := []pdf.TextBox{
+ {Text: "para one", PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 0, Bottom: 10},
+ {Text: body, PageNumber: 0, LayoutType: pdf.LayoutTypeTable, X0: 0, X1: 400, Top: 12, Bottom: 100},
+ {Text: "para three", PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 110, Bottom: 120},
+ }
+ // Both tables occupy the same region as box 1.
+ region := pdf.Position{PageNumbers: []int{0}, Left: 0, Right: 400, Top: 12, Bottom: 100}
+ tables := []pdf.TableItem{
+ {
+ // DLA region but no TSR cells → buildTableHTMLs skips, htmls[0] unset.
+ Positions: []pdf.Position{region},
+ RegionLeft: 0, RegionRight: 400, RegionTop: 12, RegionBottom: 100,
+ },
+ {
+ // Real table with cells → htmls[1] non-empty.
+ Positions: []pdf.Position{region},
+ RegionLeft: 0, RegionRight: 400, RegionTop: 12, RegionBottom: 100,
+ Scale: 1.0,
+ Cells: []pdf.TSRCell{{Text: "cell1"}},
+ },
+ }
+ removeSet := map[int]bool{}
+ // box 1 is the replacement target for BOTH tables.
+ replacements := []replacement{{tableIdx: 0, boxIdx: 1}, {tableIdx: 1, boxIdx: 1}}
+
+ out := processTablesWithReplacements(boxes, tables, removeSet, replacements)
+
+ // The box must be removed (replaced by the real table's HTML), not kept
+ // alongside it — otherwise the content is duplicated.
+ for _, b := range out {
+ if b.Text == body {
+ t.Fatalf("box covered by a real table was kept, duplicating content: %+v", out)
+ }
+ }
+ // Exactly one table HTML box (table 1) should be present.
+ var htmlCount int
+ for _, b := range out {
+ if b.LayoutType == pdf.LayoutTypeTable && strings.Contains(b.Text, "cell1") {
+ htmlCount++
+ }
+ }
+ if htmlCount != 1 {
+ t.Errorf("expected exactly one table HTML box, got %d: %+v", htmlCount, out)
+ }
+ if len(out) != 3 {
+ t.Errorf("expected 3 boxes (2 text + 1 HTML), got %d: %+v", len(out), out)
+ }
+}
+
+// TestProcessTablesWithReplacements_KeepsTextWhenCellsButEmptyHTML guards the
+// defensive case where a table has cells but ConstructTable returns an empty
+// string (e.g. degenerate/orphaned cells). The box must be kept as prose rather
+// than silently dropped. This is reachable when the target box is NOT collected
+// as a table box (so the Y/X fallback in ConstructTable cannot recover text),
+// e.g. a DLA-overlabeled text box whose layout type was not upgraded to Table.
+func TestProcessTablesWithReplacements_KeepsTextWhenCellsButEmptyHTML(t *testing.T) {
+ const body = "PROSE that must survive a degenerate table"
+ boxes := []pdf.TextBox{
+ {Text: "para one", PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 0, Bottom: 10},
+ {Text: body, PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 400, Top: 12, Bottom: 100},
+ {Text: "para three", PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 110, Bottom: 120},
+ }
+ tables := []pdf.TableItem{
+ {
+ // Cells present but without text, and the target box is LayoutTypeText
+ // (not collected as a table box) → ConstructTable yields "" → htmls[0]="".
+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 0, Right: 400, Top: 12, Bottom: 100}},
+ RegionLeft: 0, RegionRight: 400, RegionTop: 12, RegionBottom: 100,
+ Scale: 1.0,
+ Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30}},
+ },
+ }
+ removeSet := map[int]bool{}
+ replacements := []replacement{{tableIdx: 0, boxIdx: 1}}
+
+ out := processTablesWithReplacements(boxes, tables, removeSet, replacements)
+
+ if len(out) != 3 {
+ t.Fatalf("degenerate-table box was dropped: want 3 boxes, got %d: %+v", len(out), out)
+ }
+ for _, b := range out {
+ if b.Text == body {
+ return // original text preserved
+ }
+ }
+ t.Errorf("degenerate-table box text missing from output: %+v", out)
+}