Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(deepdoc): preserve table rows before cell assignment (#21489)

jay777211a6a382a2faa
diff --git a/internal/deepdoc/parser/pdf/parser_pipeline_integration_test.go b/internal/deepdoc/parser/pdf/parser_pipeline_integration_test.go
index aba74972f7..cd9b39a1d2 100644
--- a/internal/deepdoc/parser/pdf/parser_pipeline_integration_test.go
+++ b/internal/deepdoc/parser/pdf/parser_pipeline_integration_test.go
@@ -11,6 +11,7 @@ import (
"testing"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
+ "ragflow/internal/parser/tableutil"
)
// ── golden-file helpers ────────────────────────────────────────────────────
@@ -676,3 +677,73 @@ func TestE2E_ParseAndPostProcess(t *testing.T) {
figs := result.Figures()
t.Logf("figures: %d", len(figs))
}
+
+func TestManualAN0590TableRows(t *testing.T) {
+ path := os.Getenv("AN0590_PDF")
+ if path == "" {
+ t.Skip("AN0590_PDF required")
+ }
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ engine, err := NewEngine(data)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer engine.Close()
+ cfg := pdf.DefaultParserConfig()
+ result, err := NewParser(cfg).ParseRaw(t.Context(), engine, mustConnectInProcessAnalyzer(t))
+ if err != nil {
+ t.Fatal(err)
+ }
+ functions, registers := 0, 0
+ for _, s := range result.Sections {
+ if s.LayoutType == pdf.LayoutTypeTable && strings.Contains(s.Text, ">SIMA6<") {
+ registers++
+ td, err := tableutil.ParseTableHTML(s.Text)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(td.Grid) != 4 {
+ t.Fatalf("register rows = %d, want 4", len(td.Grid))
+ }
+ for _, row := range td.Grid {
+ if len(row) != 9 {
+ t.Fatalf("register columns = %d, want 9", len(row))
+ }
+ }
+ }
+ if s.LayoutType != pdf.LayoutTypeTable || !strings.Contains(s.Text, "函数名") {
+ continue
+ }
+ functions++
+ td, err := tableutil.ParseTableHTML(s.Text)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(td.Grid) != 6 {
+ t.Fatalf("function rows = %d, want 6", len(td.Grid))
+ }
+ want := []string{"函数名", "功能", "Buffer_data", "WriteAddr", "NumByteToWrite", "输出"}
+ if strings.Contains(s.Text, "I2C_EEPROM_BufferRead") {
+ want[2], want[3], want[4] = "Receive_buffer", "ReadAddr", "NumByteToRead"
+ }
+ for r, row := range td.Grid {
+ var text strings.Builder
+ for _, cell := range row {
+ text.WriteString(cell.Text)
+ }
+ if !strings.Contains(text.String(), want[r]) {
+ t.Errorf("row %d = %q, want %q", r, text.String(), want[r])
+ }
+ }
+ t.Log(s.Text)
+ }
+ if functions != 3 {
+ t.Fatalf("function tables = %d, want 3", functions)
+ }
+ if registers != 1 {
+ t.Fatalf("register tables = %d, want 1", registers)
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_14_interleaved_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_14_interleaved_test.go
index 0b0842a8a8..2a4050cff4 100644
--- a/internal/deepdoc/parser/pdf/pipeline_parity_14_interleaved_test.go
+++ b/internal/deepdoc/parser/pdf/pipeline_parity_14_interleaved_test.go
@@ -22,7 +22,7 @@ import (
// NaiveVerticalMerge before cell fill, and a strictly-NESTED re-detected box
// ('Hardware' fully inside 'Software Hardware') was CONCATENATED by
// mergeTwoBoxes into 'Software Hardware Hardware'; Python keeps the longer
-// box only. Fixed by dedupNestedBoxes (table_extract.go), which drops a box
+// box only. Fixed by dedupTableBoxes (table_extract.go), which drops a box
// strictly nested inside another box whose trimmed text contains it, before
// the merge.
//
@@ -136,7 +136,7 @@ func TestPipelineParity14TextTableInterleaved(t *testing.T) {
ex = fmt.Sprintf("e.g. %q vs Python %q", dup[0].goVal, dup[0].pyVal)
}
t.Errorf("table-cell-fill-pre-vs-post-merge-order regression: %d Go cells contain a duplicated token from overlapping-box concatenation, gridSim=%.1f%% (%s). "+
- "Go must merge overlapping OCR boxes before cell fill, matching Python's construct_table order.",
+ "Duplicate OCR fragments must be removed before cell assignment.",
len(dup), gridSim, ex)
}
}
diff --git a/internal/deepdoc/parser/pdf/snapshot_test.go b/internal/deepdoc/parser/pdf/snapshot_test.go
index 01002ca87b..197a73f883 100644
--- a/internal/deepdoc/parser/pdf/snapshot_test.go
+++ b/internal/deepdoc/parser/pdf/snapshot_test.go
@@ -8,6 +8,7 @@ import (
"math"
"os"
"path/filepath"
+ "ragflow/internal/common"
lyt "ragflow/internal/deepdoc/parser/pdf/layout"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
"sort"
diff --git a/internal/deepdoc/parser/pdf/table_extract.go b/internal/deepdoc/parser/pdf/table_extract.go
index c3fe0cb014..e818aa2ce2 100644
--- a/internal/deepdoc/parser/pdf/table_extract.go
+++ b/internal/deepdoc/parser/pdf/table_extract.go
@@ -6,7 +6,6 @@ import (
"math"
"strings"
- lyt "ragflow/internal/deepdoc/parser/pdf/layout"
tbl "ragflow/internal/deepdoc/parser/pdf/table"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
util "ragflow/internal/deepdoc/parser/pdf/util"
@@ -153,14 +152,9 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxes
if firstCellTop == 1e9 {
firstCellTop = cells[0].Y0
}
- // Collapse overlapping/adjacent OCR boxes before cell-fill, mirroring
- // Python's pipeline order — _naive_vertical_merge runs before
- // construct_table, so overlapping boxes are merged before they reach
- // cell assignment. Go runs table cell-fill per-page (here), before the
- // document-wide vertical merge in buildLayout, so it must run its own
- // collapse on this table's box subset first or overlapping OCR boxes
- // duplicate text across cells (e.g. 13_crosspage_table page 2:
- // '2024-43 2024-44' y=(1014,1045) + nested '2024-44' y=(1032,1045)).
+ // Preserve each text box until its cell has been assigned. Paragraph
+ // merging here would join adjacent table rows before their R/C labels
+ // exist. GroupBoxesByRC joins text within the assigned cell below.
tableBoxes := make([]pdf.TextBox, 0, len(tm.BoxIdx))
for _, idx := range tm.BoxIdx {
b := boxes[idx]
@@ -169,17 +163,12 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxes
}
tableBoxes = append(tableBoxes, b)
}
- // Drop OCR boxes that are strictly nested inside another box whose
- // text contains theirs (e.g. a re-detected "Hardware" box fully
- // inside "Software Hardware"). Without this, NaiveVerticalMerge
- // concatenates the two into "Software Hardware Hardware" while
- // Python's construct_table keeps the longer box only. Mirrors
- // Python's effective behavior: the nested duplicate is not merged.
- tableBoxes = dedupNestedBoxes(tableBoxes)
- tableBoxes = lyt.NaiveVerticalMerge(tableBoxes, nil, nil, nil)
+ tableBoxes = dedupTableBoxes(tableBoxes)
boxInCrop = make([]pdf.TextBox, 0, len(tableBoxes))
for _, b := range tableBoxes {
- boxInCrop = append(boxInCrop, tbl.BoxToCropSpace(b, scale, cropOffX, cropOffY))
+ cropBox := tbl.BoxToCropSpace(b, scale, cropOffX, cropOffY)
+ cropBox.IsOCR = b.IsOCR
+ boxInCrop = append(boxInCrop, cropBox)
}
}
var positions []pdf.Position
@@ -199,6 +188,7 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxes
// Derive R/C/H/SP with Python's _table_transformer_job semantics
// (whole-row/whole-column line matching), in crop space.
tbl.AnnotateBoxesWithGrid(boxInCrop, annotGrid)
+ boxInCrop = dedupSameCellOCRBoxes(boxInCrop)
// Rebuild the grid from the derived per-char R/C, exactly like
// Python's construct_table groups boxes by their R/C labels —
// rows are produced only for R values that carry boxes. Fall back
@@ -221,18 +211,9 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxes
return item
}
-// dedupNestedBoxes drops OCR boxes that are strictly nested inside another
-// box whose trimmed text contains the nested box's trimmed text. This removes
-// re-detected duplicate boxes (e.g. a "Hardware" detection fully inside a
-// "Software Hardware" box) so the subsequent NaiveVerticalMerge does not
-// concatenate them into "Software Hardware Hardware". Python's
-// construct_table keeps the longer box only, so this matches Python's output.
-//
-// Containment requires the nested box's bbox to lie entirely within the other
-// box's bbox (same or narrower X, same or shorter Y). Side-by-side or
-// half-overlapping boxes are never nested, so they are preserved and merged
-// normally.
-func dedupNestedBoxes(boxes []pdf.TextBox) []pdf.TextBox {
+// dedupTableBoxes drops fully contained text fragments. Partial overlaps are
+// preserved because the boxes may belong to different cells.
+func dedupTableBoxes(boxes []pdf.TextBox) []pdf.TextBox {
keep := make([]bool, len(boxes))
for i := range boxes {
keep[i] = strings.TrimSpace(boxes[i].Text) != ""
@@ -250,10 +231,11 @@ func dedupNestedBoxes(boxes []pdf.TextBox) []pdf.TextBox {
if bt == "" || !strings.Contains(at, bt) {
continue
}
- // Drop the nested (shorter) box when it sits fully inside the
- // outer box's bbox.
- if boxes[j].X0 >= boxes[i].X0 && boxes[j].X1 <= boxes[i].X1 &&
- boxes[j].Top >= boxes[i].Top && boxes[j].Bottom <= boxes[i].Bottom {
+ outer, inner := boxes[i], boxes[j]
+ if inner.X0 < outer.X0 || inner.X1 > outer.X1 {
+ continue
+ }
+ if inner.Top >= outer.Top && inner.Bottom <= outer.Bottom {
keep[j] = false
}
}
@@ -266,3 +248,32 @@ func dedupNestedBoxes(boxes []pdf.TextBox) []pdf.TextBox {
}
return out
}
+
+// dedupSameCellOCRBoxes removes overlapping copies of the same OCR text after
+// cell assignment, comparing original boxes rather than accumulated cell text.
+func dedupSameCellOCRBoxes(boxes []pdf.TextBox) []pdf.TextBox {
+ out := boxes[:0]
+ for _, b := range boxes {
+ duplicate := false
+ if b.IsOCR && b.RBott > b.RTop {
+ for _, kept := range out {
+ if !kept.IsOCR || kept.RBott <= kept.RTop || b.R != kept.R || b.C != kept.C ||
+ strings.TrimSpace(b.Text) != strings.TrimSpace(kept.Text) {
+ continue
+ }
+ width := math.Min(b.X1-b.X0, kept.X1-kept.X0)
+ height := math.Min(b.Bottom-b.Top, kept.Bottom-kept.Top)
+ overlapX := math.Min(b.X1, kept.X1) - math.Max(b.X0, kept.X0)
+ overlapY := math.Min(b.Bottom, kept.Bottom) - math.Max(b.Top, kept.Top)
+ if width > 0 && height > 0 && overlapX >= width*0.6 && overlapY >= height*0.6 {
+ duplicate = true
+ break
+ }
+ }
+ }
+ if !duplicate {
+ out = append(out, b)
+ }
+ }
+ return out
+}
diff --git a/internal/deepdoc/parser/pdf/table_extract_test.go b/internal/deepdoc/parser/pdf/table_extract_test.go
index ee47f60206..7e00b7977b 100644
--- a/internal/deepdoc/parser/pdf/table_extract_test.go
+++ b/internal/deepdoc/parser/pdf/table_extract_test.go
@@ -5,6 +5,7 @@ import (
"fmt"
"image"
"math"
+ "reflect"
"strings"
"testing"
@@ -248,66 +249,46 @@ func (b *twoRowTableBuilder) GroupCells(cells []pdf.TSRCell) [][]pdf.TSRCell {
return rows
}
-// TestProcessOneTable_CollapsesOverlappingBoxesBeforeCellFill verifies that
-// two overlapping OCR boxes over the same table region (one text box and a
-// nested duplicate detection covering part of the same text, e.g. "Alpha
-// Beta" and "Beta") are collapsed before cell assignment, matching Python's
-// pipeline order (_naive_vertical_merge runs before construct_table).
-// Without the collapse, the two boxes independently pick their own
-// best-matching row, spreading the duplicated word across two different
-// cells instead of landing once in a single row.
-func TestProcessOneTable_CollapsesOverlappingBoxesBeforeCellFill(t *testing.T) {
- cfg := pdf.DefaultParserConfig()
- p := NewParser(cfg)
-
- pageImg := image.NewRGBA(image.Rect(0, 0, 600, 600))
- // PDF-point space. Box2 ("Beta") is a nested duplicate OCR detection
- // overlapping the tail of box1 ("Alpha Beta"): by itself it best-matches
- // the second TSR row, while box1 alone best-matches the first.
- boxes := []pdf.TextBox{
- {X0: 0, X1: 50, Top: 5, Bottom: 25, Text: "Alpha Beta"},
- {X0: 20, X1: 50, Top: 18.33, Bottom: 28.33, Text: "Beta"},
- }
- match := tbl.TableMatch{
- Region: pdf.DLARegion{X0: 0, Y0: 0, X1: 600, Y1: 600, Label: pdf.LayoutTypeTable},
- BoxIdx: []int{0, 1},
- }
- // TSR cells in crop-pixel space (scale = DlaScale = 3): row0 y=[0,60],
- // row1 y=[60,120], single column x=[0,150].
- builder := &twoRowTableBuilder{
- cells: []pdf.TSRCell{
- {X0: 0, Y0: 0, X1: 150, Y1: 60, Label: "table row"},
- {X0: 0, Y0: 60, X1: 150, Y1: 120, Label: "table row"},
- },
- }
-
- item := p.processOneTable(t.Context(), pageImg, boxes, 0, &orientationScoringDoc{}, builder, match, pdf.DlaScale)
- // Python's construct_table groups boxes by their R/C labels, producing a
- // row only for an R that carries a box. Both boxes land in the SAME row
- // (they overlap vertically), so the grid is 1x1 — not 2x1.
- if len(item.Grid) != 1 || len(item.Grid[0]) != 1 {
- t.Fatalf("grid shape = %v, want 1x1 (R/C grouping emits a row only for the R that carries the box)", item.Grid)
- }
-
- row0 := item.Grid[0][0].Text
- if row0 == "" {
- t.Fatalf("row0 = %q, want non-empty: overlapping boxes must merge into a single cell assignment instead of spreading duplicated text across rows", row0)
+func TestProcessOneTable_HandlesTextFragments(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ bottom float64
+ want [][]string
+ }{
+ {"contained fragment", 24, [][]string{{"Alpha Beta"}}},
+ {"partial fragment across rows", 28.33, [][]string{{"Alpha Beta"}, {"Beta"}}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ boxes := []pdf.TextBox{
+ {X0: 0, X1: 50, Top: 5, Bottom: 25, Text: "Alpha Beta", IsOCR: true},
+ {X0: 20, X1: 50, Top: 18.33, Bottom: tc.bottom, Text: "Beta", IsOCR: true},
+ }
+ builder := &twoRowTableBuilder{cells: []pdf.TSRCell{
+ {X0: 0, Y0: 0, X1: 150, Y1: 60}, {X0: 0, Y0: 60, X1: 150, Y1: 120},
+ }}
+ match := tbl.TableMatch{Region: pdf.DLARegion{X0: 0, Y0: 0, X1: 150, Y1: 120}, BoxIdx: []int{0, 1}}
+ item := NewParser(pdf.DefaultParserConfig()).processOneTable(t.Context(), image.NewGray(image.Rect(0, 0, 180, 150)), boxes, 0, &orientationScoringDoc{}, builder, match, pdf.DlaScale)
+ tbl.ConstructTable(item.Cells, nil, "", &item)
+ if !reflect.DeepEqual(item.Rows, tc.want) {
+ t.Fatalf("rows = %v, want %v", item.Rows, tc.want)
+ }
+ })
}
}
-// TestDedupNestedBoxes_IdenticalBBoxKeepsContent locks that two boxes with
+// TestDedupTableBoxes_IdenticalBBoxKeepsContent locks that two boxes with
// IDENTICAL bbox AND identical text are not BOTH dropped. The containment
// check drops the inner box when its bbox lies fully inside the outer's and
// the outer text contains the inner's — but for identical bboxes each box is
// "inside" the other, so both get marked for removal and the cell content
// vanishes. Only STRICT containment may drop; identical-bbox boxes must keep
// at least one copy.
-func TestDedupNestedBoxes_IdenticalBBoxKeepsContent(t *testing.T) {
+func TestDedupTableBoxes_IdenticalBBoxKeepsContent(t *testing.T) {
boxes := []pdf.TextBox{
{X0: 100, X1: 300, Top: 100, Bottom: 120, Text: "Revenue", IsOCR: true},
{X0: 100, X1: 300, Top: 100, Bottom: 120, Text: "Revenue", IsOCR: true},
}
- got := dedupNestedBoxes(boxes)
+ got := dedupTableBoxes(boxes)
if len(got) == 0 {
t.Fatal("identical bbox+text boxes were BOTH dropped — content loss; at least one copy must survive")
}
@@ -317,3 +298,129 @@ func TestDedupNestedBoxes_IdenticalBBoxKeepsContent(t *testing.T) {
}
}
}
+
+func TestDedupTableBoxes_TableFragments(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ boxes []pdf.TextBox
+ want string
+ }{
+ {"overlapping fragment", []pdf.TextBox{
+ {X0: 0, X1: 50, Top: 5, Bottom: 25, Text: "Alpha Beta", IsOCR: true},
+ {X0: 20, X1: 50, Top: 18.33, Bottom: 28.33, Text: "Beta", IsOCR: true},
+ }, "Alpha Beta|Beta"},
+ {"adjacent identical rows", []pdf.TextBox{
+ {X0: 0, X1: 50, Top: 5, Bottom: 15, Text: "R/W", IsOCR: true},
+ {X0: 0, X1: 50, Top: 14, Bottom: 24, Text: "R/W", IsOCR: true},
+ }, "R/W|R/W"},
+ {"adjacent columns", []pdf.TextBox{
+ {X0: 0, X1: 20, Top: 5, Bottom: 15, Text: "R/W", IsOCR: true},
+ {X0: 21, X1: 41, Top: 5, Bottom: 15, Text: "R/W", IsOCR: true},
+ }, "R/W|R/W"},
+ {"taller fragment", []pdf.TextBox{
+ {X0: 0, X1: 50, Top: 5, Bottom: 15, Text: "Alpha Beta", IsOCR: true},
+ {X0: 20, X1: 50, Top: 5, Bottom: 25, Text: "Beta", IsOCR: true},
+ }, "Alpha Beta|Beta"},
+ {"overlap with distinct text", []pdf.TextBox{
+ {X0: 0, X1: 50, Top: 5, Bottom: 25, Text: "Alpha", IsOCR: true},
+ {X0: 20, X1: 50, Top: 18.33, Bottom: 28.33, Text: "Beta", IsOCR: true},
+ }, "Alpha|Beta"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ var texts []string
+ for _, b := range dedupTableBoxes(tc.boxes) {
+ texts = append(texts, b.Text)
+ }
+ if got := strings.Join(texts, "|"); got != tc.want {
+ t.Fatalf("text = %q, want %q", got, tc.want)
+ }
+ })
+ }
+}
+
+func TestProcessOneTable_PreservesAdjacentRows(t *testing.T) {
+ boxes := []pdf.TextBox{
+ {X0: 10, X1: 90, Top: 5, Bottom: 15, Text: "功能"},
+ {X0: 10, X1: 90, Top: 17, Bottom: 27, Text: "输入"},
+ {X0: 10, X1: 90, Top: 29, Bottom: 39, Text: "输出"},
+ }
+ match := tbl.TableMatch{Region: pdf.DLARegion{X0: 0, Y0: 0, X1: 300, Y1: 150}, BoxIdx: []int{0, 1, 2}}
+ builder := &twoRowTableBuilder{cells: []pdf.TSRCell{
+ {X0: 0, X1: 300, Y0: 0, Y1: 48},
+ {X0: 0, X1: 300, Y0: 48, Y1: 84},
+ {X0: 0, X1: 300, Y0: 84, Y1: 126},
+ }}
+ item := NewParser(pdf.DefaultParserConfig()).processOneTable(t.Context(), image.NewGray(image.Rect(0, 0, 300, 150)), boxes, 0, &orientationScoringDoc{}, builder, match, pdf.DlaScale)
+ tbl.ConstructTable(item.Cells, nil, "", &item)
+ if want := [][]string{{"功能"}, {"输入"}, {"输出"}}; !reflect.DeepEqual(item.Rows, want) {
+ t.Fatalf("rows = %v, want %v", item.Rows, want)
+ }
+}
+
+func TestProcessOneTable_PreservesOverlappingRows(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ boxes []pdf.TextBox
+ want [][]string
+ }{
+ {"identical text", []pdf.TextBox{
+ {X0: 0, X1: 50, Top: 5, Bottom: 15, Text: "R/W", IsOCR: true},
+ {X0: 0, X1: 50, Top: 9, Bottom: 19, Text: "R/W", IsOCR: true},
+ }, [][]string{{"R/W"}, {"R/W"}}},
+ {"substring text", []pdf.TextBox{
+ {X0: 0, X1: 50, Top: 5, Bottom: 15, Text: "100", IsOCR: true},
+ {X0: 0, X1: 50, Top: 9, Bottom: 19, Text: "10", IsOCR: true},
+ }, [][]string{{"100"}, {"10"}}},
+ {"shorter substring text", []pdf.TextBox{
+ {X0: 0, X1: 50, Top: 5, Bottom: 15, Text: "100", IsOCR: true},
+ {X0: 0, X1: 50, Top: 9, Bottom: 18, Text: "10", IsOCR: true},
+ }, [][]string{{"100"}, {"10"}}},
+ {"identical text with different heights", []pdf.TextBox{
+ {X0: 0, X1: 50, Top: 3, Bottom: 15, Text: "R/W", IsOCR: true},
+ {X0: 0, X1: 50, Top: 9, Bottom: 19, Text: "R/W", IsOCR: true},
+ }, [][]string{{"R/W"}, {"R/W"}}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ builder := &twoRowTableBuilder{cells: []pdf.TSRCell{
+ {X0: 0, X1: 150, Y0: 0, Y1: 30},
+ {X0: 0, X1: 150, Y0: 30, Y1: 60},
+ }}
+ match := tbl.TableMatch{Region: pdf.DLARegion{X0: 0, Y0: 0, X1: 150, Y1: 60}, BoxIdx: []int{0, 1}}
+ item := NewParser(pdf.DefaultParserConfig()).processOneTable(t.Context(), image.NewGray(image.Rect(0, 0, 180, 90)), tc.boxes, 0, &orientationScoringDoc{}, builder, match, pdf.DlaScale)
+ tbl.ConstructTable(item.Cells, nil, "", &item)
+ if !reflect.DeepEqual(item.Rows, tc.want) {
+ t.Fatalf("rows = %v, want %v", item.Rows, tc.want)
+ }
+ })
+ }
+}
+
+func TestProcessOneTable_DeduplicatesSameCellOCR(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ first, second pdf.TextBox
+ want [][]string
+ }{
+ {"duplicate OCR", pdf.TextBox{X0: 0, X1: 40, Top: 5, Bottom: 15, Text: "R/W", IsOCR: true}, pdf.TextBox{X0: 0, X1: 40, Top: 9, Bottom: 19, Text: " R/W ", IsOCR: true}, [][]string{{"R/W", ""}, {"", "bottom"}}},
+ {"separate words", pdf.TextBox{X0: 0, X1: 15, Top: 5, Bottom: 15, Text: "R/W", IsOCR: true}, pdf.TextBox{X0: 20, X1: 35, Top: 5, Bottom: 15, Text: "R/W", IsOCR: true}, [][]string{{"R/W R/W", ""}, {"", "bottom"}}},
+ {"separate lines", pdf.TextBox{X0: 0, X1: 40, Top: 5, Bottom: 10, Text: "R/W", IsOCR: true}, pdf.TextBox{X0: 0, X1: 40, Top: 12, Bottom: 17, Text: "R/W", IsOCR: true}, [][]string{{"R/W R/W", ""}, {"", "bottom"}}},
+ {"different columns", pdf.TextBox{X0: 0, X1: 80, Top: 5, Bottom: 15, Text: "R/W", IsOCR: true}, pdf.TextBox{X0: 20, X1: 100, Top: 5, Bottom: 15, Text: "R/W", IsOCR: true}, [][]string{{"R/W", "R/W"}, {"", "bottom"}}},
+ {"native text", pdf.TextBox{X0: 0, X1: 40, Top: 5, Bottom: 15, Text: "R/W"}, pdf.TextBox{X0: 0, X1: 40, Top: 9, Bottom: 19, Text: "R/W"}, [][]string{{"R/W R/W", ""}, {"", "bottom"}}},
+ {"native then OCR", pdf.TextBox{X0: 0, X1: 40, Top: 5, Bottom: 15, Text: "R/W"}, pdf.TextBox{X0: 0, X1: 40, Top: 9, Bottom: 19, Text: "R/W", IsOCR: true}, [][]string{{"R/W R/W", ""}, {"", "bottom"}}},
+ {"OCR then native", pdf.TextBox{X0: 0, X1: 40, Top: 5, Bottom: 15, Text: "R/W", IsOCR: true}, pdf.TextBox{X0: 0, X1: 40, Top: 9, Bottom: 19, Text: "R/W"}, [][]string{{"R/W R/W", ""}, {"", "bottom"}}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ boxes := []pdf.TextBox{tc.first, tc.second, {X0: 60, X1: 90, Top: 25, Bottom: 35, Text: "bottom", IsOCR: true}}
+ doc := &MockDocAnalyzer{Healthy: true, TSRCells: []pdf.TSRCell{
+ {X0: 0, X1: 300, Y0: 0, Y1: 60, Label: "table row"}, {X0: 0, X1: 300, Y0: 60, Y1: 120, Label: "table row"},
+ {X0: 0, X1: 150, Y0: 0, Y1: 120, Label: "table column"}, {X0: 150, X1: 300, Y0: 0, Y1: 120, Label: "table column"},
+ }}
+ match := tbl.TableMatch{Region: pdf.DLARegion{X0: 0, Y0: 0, X1: 300, Y1: 120}, BoxIdx: []int{0, 1, 2}}
+ item := NewParser(pdf.DefaultParserConfig()).processOneTable(t.Context(), image.NewGray(image.Rect(0, 0, 330, 150)), boxes, 0, doc, tbl.NewDeepDocTableBuilder(doc), match, pdf.DlaScale)
+ tbl.ConstructTable(item.Cells, nil, "", &item)
+ if !reflect.DeepEqual(item.Rows, tc.want) {
+ t.Fatalf("rows = %v, want %v", item.Rows, tc.want)
+ }
+ })
+ }
+}