RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(pdf): drop text boxes only when replaced by a table HTML (#18363)
Jackbd01c78d2ba7
diff --git a/internal/deepdoc/parser/pdf/table/table_post.go b/internal/deepdoc/parser/pdf/table/table_post.goindex 0ce7066554..cdda3ac538 100644--- a/internal/deepdoc/parser/pdf/table/table_post.go+++ b/internal/deepdoc/parser/pdf/table/table_post.go@@ -318,11 +318,19 @@ func processTablesWithReplacements(removeSet map[int]bool,replacements []replacement,) []pdf.TextBox {- for _, r := range replacements {- removeSet[r.boxIdx] = true- }anchors := findTableAnchorsWithReplacements(boxes, tables, replacements)htmls := buildTableHTMLs(boxes, tables)+ // A box is removed only when it is actually replaced by a table HTML box.+ // A DLA-tagged region that produced no table HTML (no cells, or a+ // degenerate table whose ConstructTable returned "") keeps its original+ // text as prose instead of being silently dropped. Marking per replacement+ // (rather than un-marking empties) also avoids duplicating content when one+ // box is covered by both an empty and a real table.+ for _, r := range replacements {+ if html, ok := htmls[r.tableIdx]; ok && html != "" {+ removeSet[r.boxIdx] = true+ }+ }anchorList := buildAndSortAnchors(anchors)return insertTableBoxes(boxes, tables, removeSet, anchorList, htmls)}diff --git a/internal/deepdoc/parser/pdf/table/table_post_test.go b/internal/deepdoc/parser/pdf/table/table_post_test.goindex ec7846c197..3b536b42ce 100644--- a/internal/deepdoc/parser/pdf/table/table_post_test.go+++ b/internal/deepdoc/parser/pdf/table/table_post_test.go@@ -800,3 +800,127 @@ func TestExtractTableAndReplace_MergeTablesAcrossPages(t *testing.T) {t.Error("non-table boxes should be preserved in original order")}}++// TestProcessTablesWithReplacements_KeepsTextWhenNoCells pins the contract that+// a text box mislabeled LayoutTypeTable by DLA is kept as prose when TSR yields+// no cells (HTML missing), instead of being silently dropped.+func TestProcessTablesWithReplacements_KeepsTextWhenNoCells(t *testing.T) {+ const body = "REFERENCE [1] body text that must not vanish"+ boxes := []pdf.TextBox{+ {Text: "para one", PageNumber: 1, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 0, Bottom: 10},+ {Text: body, PageNumber: 1, LayoutType: pdf.LayoutTypeTable, X0: 0, X1: 100, Top: 12, Bottom: 22},+ {Text: "para three", PageNumber: 1, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 24, Bottom: 34},+ }+ // DLA table region but no TSR cells → buildTableHTMLs skips, htmls[0] unset.+ tables := []pdf.TableItem{{}}+ removeSet := map[int]bool{}+ replacements := []replacement{{tableIdx: 0, boxIdx: 1}}++ out := processTablesWithReplacements(boxes, tables, removeSet, replacements)++ expectedTexts := []string{"para one", body, "para three"}+ if len(out) != len(expectedTexts) {+ t.Fatalf("over-labeled 'table' box was dropped: want %d boxes, got %d: %+v", len(expectedTexts), len(out), out)+ }+ for i, want := range expectedTexts {+ if out[i].Text != want {+ t.Errorf("output box %d: want text %q, got %q (%+v)", i, want, out[i].Text, out)+ }+ }+}++// TestProcessTablesWithReplacements_NoDupWhenCoveredByEmptyAndRealTable pins the+// boundary where a single box is covered by TWO replacements — one whose table+// has no cells (empty HTML) and one whose table has real cells (non-empty HTML)+// — so the box must NOT be kept while the real table's HTML is also inserted+// (that would duplicate the content). The box must be removed and only the real+// table's HTML inserted.+func TestProcessTablesWithReplacements_NoDupWhenCoveredByEmptyAndRealTable(t *testing.T) {+ const body = "SHARED BODY spanning two table regions"+ boxes := []pdf.TextBox{+ {Text: "para one", PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 0, Bottom: 10},+ {Text: body, PageNumber: 0, LayoutType: pdf.LayoutTypeTable, X0: 0, X1: 400, Top: 12, Bottom: 100},+ {Text: "para three", PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 110, Bottom: 120},+ }+ // Both tables occupy the same region as box 1.+ region := pdf.Position{PageNumbers: []int{0}, Left: 0, Right: 400, Top: 12, Bottom: 100}+ tables := []pdf.TableItem{+ {+ // DLA region but no TSR cells → buildTableHTMLs skips, htmls[0] unset.+ Positions: []pdf.Position{region},+ RegionLeft: 0, RegionRight: 400, RegionTop: 12, RegionBottom: 100,+ },+ {+ // Real table with cells → htmls[1] non-empty.+ Positions: []pdf.Position{region},+ RegionLeft: 0, RegionRight: 400, RegionTop: 12, RegionBottom: 100,+ Scale: 1.0,+ Cells: []pdf.TSRCell{{Text: "cell1"}},+ },+ }+ removeSet := map[int]bool{}+ // box 1 is the replacement target for BOTH tables.+ replacements := []replacement{{tableIdx: 0, boxIdx: 1}, {tableIdx: 1, boxIdx: 1}}++ out := processTablesWithReplacements(boxes, tables, removeSet, replacements)++ // The box must be removed (replaced by the real table's HTML), not kept+ // alongside it — otherwise the content is duplicated.+ for _, b := range out {+ if b.Text == body {+ t.Fatalf("box covered by a real table was kept, duplicating content: %+v", out)+ }+ }+ // Exactly one table HTML box (table 1) should be present.+ var htmlCount int+ for _, b := range out {+ if b.LayoutType == pdf.LayoutTypeTable && strings.Contains(b.Text, "cell1") {+ htmlCount+++ }+ }+ if htmlCount != 1 {+ t.Errorf("expected exactly one table HTML box, got %d: %+v", htmlCount, out)+ }+ if len(out) != 3 {+ t.Errorf("expected 3 boxes (2 text + 1 HTML), got %d: %+v", len(out), out)+ }+}++// TestProcessTablesWithReplacements_KeepsTextWhenCellsButEmptyHTML guards the+// defensive case where a table has cells but ConstructTable returns an empty+// string (e.g. degenerate/orphaned cells). The box must be kept as prose rather+// than silently dropped. This is reachable when the target box is NOT collected+// as a table box (so the Y/X fallback in ConstructTable cannot recover text),+// e.g. a DLA-overlabeled text box whose layout type was not upgraded to Table.+func TestProcessTablesWithReplacements_KeepsTextWhenCellsButEmptyHTML(t *testing.T) {+ const body = "PROSE that must survive a degenerate table"+ boxes := []pdf.TextBox{+ {Text: "para one", PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 0, Bottom: 10},+ {Text: body, PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 400, Top: 12, Bottom: 100},+ {Text: "para three", PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 110, Bottom: 120},+ }+ tables := []pdf.TableItem{+ {+ // Cells present but without text, and the target box is LayoutTypeText+ // (not collected as a table box) → ConstructTable yields "" → htmls[0]="".+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 0, Right: 400, Top: 12, Bottom: 100}},+ RegionLeft: 0, RegionRight: 400, RegionTop: 12, RegionBottom: 100,+ Scale: 1.0,+ Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30}},+ },+ }+ removeSet := map[int]bool{}+ replacements := []replacement{{tableIdx: 0, boxIdx: 1}}++ out := processTablesWithReplacements(boxes, tables, removeSet, replacements)++ if len(out) != 3 {+ t.Fatalf("degenerate-table box was dropped: want 3 boxes, got %d: %+v", len(out), out)+ }+ for _, b := range out {+ if b.Text == body {+ return // original text preserved+ }+ }+ t.Errorf("degenerate-table box text missing from output: %+v", out)+}