Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

chore(table): remove stale manual parity-issues test (#18403)

Jack97db1b4c38b0
diff --git a/internal/deepdoc/parser/pdf/table/table_parity_issues_test.go b/internal/deepdoc/parser/pdf/table/table_parity_issues_test.go
deleted file mode 100644
index 59e2df867e..0000000000
--- a/internal/deepdoc/parser/pdf/table/table_parity_issues_test.go
+++ /dev/null
@@ -1,885 +0,0 @@
-//go:build manual
-
-package table
-
-import (
- "bytes"
- "context"
- "encoding/base64"
- "image"
- pdf "ragflow/internal/deepdoc/parser/pdf/type"
- "regexp"
- "strings"
- "testing"
-)
-
-// =============================================================================
-// Issue 1: Figure insertion strategy
-// Python's insert_table_figures(figs, "figure") inserts figure boxes back into
-// self.boxes. Go's extractTableAndReplace only handles LayoutType=="table",
-// leaving figure boxes in the list. This test documents the current behavior.
-// =============================================================================
-
-// TestExtractTableAndReplace_IgnoresFigures documents that extractTableAndReplace
-// does NOT pop or replace figure boxes. In Python's _extract_table_figure,
-// figure boxes are popped and re-inserted via insert_table_figures with cropped
-// images. Go leaves them in the box list for downstream boxesToSections.
-func TestExtractTableAndReplace_IgnoresFigures(t *testing.T) {
- boxes := []pdf.TextBox{
- {X0: 10, X1: 200, Top: 0, Bottom: 50, Text: "Figure text", LayoutType: "figure", PageNumber: 0},
- {X0: 10, X1: 200, Top: 60, Bottom: 80, Text: "表1:标题", LayoutType: "table", PageNumber: 0},
- }
-
- // Table with cells so extractTableAndReplace generates HTML.
- tables := []pdf.TableItem{{
- Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "A", Label: "table row"}},
- Positions: []pdf.Position{{Left: 0, Right: 300, Top: 0, Bottom: 100}},
- Scale: 1.0,
- }}
-
- result := ExtractTableAndReplace(boxes, tables)
-
- // BUG: Figure box is still present — it was not popped or replaced.
- // Python's _extract_table_figure pops figure boxes and re-inserts them
- // via insert_table_figures with cropped images.
- hasFigure := false
- for _, b := range result {
- if b.LayoutType == "figure" {
- hasFigure = true
- // Figure text is still raw text, not a consolidated image+text block
- // like Python's insert_table_figures would produce.
- if b.Text != "Figure text" {
- t.Errorf("figure text should be unchanged, got %q", b.Text)
- }
- }
- }
- if !hasFigure {
- t.Error("BUG EXPOSED: extractTableAndReplace removed figure box (unexpected)")
- }
- t.Log("NOTE: Figure box remains in list as raw text. Python inserts figures back with cropped images via insert_table_figures. Go collects figures separately via pdf.CollectFigures without re-inserting.")
-}
-
-// TestBoxesToSections_FiguresNotReinserted documents that boxesToSections converts
-// figure boxes to sections but without the consolidated image that Python's
-// insert_table_figures would attach.
-func TestBoxesToSections_FiguresNotReinserted(t *testing.T) {
- // Simulate post-extractTableAndReplace boxes with figures still present.
- boxes := []pdf.TextBox{
- {X0: 10, X1: 200, Top: 0, Bottom: 50, Text: "Some text", LayoutType: "text", PageNumber: 0},
- {X0: 10, X1: 200, Top: 60, Bottom: 100, Text: "Figure description", LayoutType: "figure", PageNumber: 0},
- }
-
- sections := boxesToSections(boxes, nil)
- figures := pdf.CollectFigures(sections)
-
- // BUG: figures are collected separately but NOT re-inserted into sections
- // after image processing. In Python, insert_table_figures(figs, "figure")
- // creates new boxes with layout_type="figure", image=cropped_img, and
- // inserts them at the nearest position among text boxes.
- if len(figures) != 1 {
- t.Fatalf("expected 1 figure, got %d", len(figures))
- }
- if figures[0].LayoutType != "figure" {
- t.Errorf("expected LayoutType 'figure', got %q", figures[0].LayoutType)
- }
- // Figure image is empty at this stage (cropSectionImage runs later in pipeline).
- if figures[0].Image != "" {
- t.Log("figure has image (cropSectionImage already ran)")
- } else {
- t.Log("NOTE: Figure section has no Image yet. Python's cropout creates a consolidated cropped image for the entire figure region before insert_table_figures.")
- }
-
- t.Logf("Sections count: %d (figure present as raw text section)", len(sections))
- t.Logf("Figures count: %d (collected separately, Python re-inserts them)", len(figures))
-}
-
-// =============================================================================
-// Issue 2a: blockType classification missing
-// Python's construct_table classifies each cell into 9 types (Dt/Nu/Ca/En/NE/
-// Sg/Tx/Lx/Nr/Ot). The dominant type drives header detection: if max_type is
-// "Nu" (numeric), numeric cells don't count as headers. Go's headerSet only
-// checks TSR labels — no cell content type analysis.
-// =============================================================================
-
-// TestConstructTable_HeaderDetection_NoBlockType documents that Go's header
-// detection is purely TSR-label-based. Python would use blockType to skip
-// numeric cells when the dominant type is "Nu".
-func TestConstructTable_HeaderDetection_NoBlockType(t *testing.T) {
- // A table where the "header" row has numeric content (like years, amounts).
- // With blockType: "2020","2021" → Nu, "100","200" → Nu — maxType=Nu.
- // block-type-aware detection skips Nu cells → 0 headers.
- // Falls back to TSR label-based detection → still gets 2 <th >.
- cells := []pdf.TSRCell{
- {X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "2020", Label: "table column header"},
- {X0: 101, Y0: 0, X1: 200, Y1: 30, Text: "2021", Label: "table column header"},
- {X0: 0, Y0: 35, X1: 100, Y1: 65, Text: "100", Label: "table row"},
- {X0: 101, Y0: 35, X1: 200, Y1: 65, Text: "200", Label: "table row"},
- }
-
- item := &pdf.TableItem{}
- html := ConstructTable(cells, nil, "", item)
-
- // FIX VERIFIED: headerSetWithBlockType computes block types (all "Nu"),
- // skips Nu headers when maxType=Nu, then falls back to TSR label detection.
- // Header row still gets <th > because TSR labels contain "header".
- thCount := strings.Count(html, "<th ")
- if thCount != 2 {
- t.Errorf("expected 2 <th >, got %d. HTML: %s", thCount, html)
- }
-
- t.Log("FIX: blockType classification added. maxType=Nu skips Nu headers in primary pass.")
- t.Log("TSR label fallback still marks header rows with 'header' in label.")
-}
-
-// TestConstructTable_BlockType_DominantTypeMissing documents that Go has no
-// concept of a "dominant cell type" that Python uses for header detection.
-func TestConstructTable_BlockType_DominantTypeMissing(t *testing.T) {
- // Mixed table with numeric-dominant data, testing blockType header detection.
- // "年份"/"金额" → Tx (short text), "2020"/"1000"/etc → Nu. maxType=Nu.
- // Header cells are non-Nu → count as headers even under Nu-dominant logic.
- // FIX: blockType now classifies cells and drives header detection.
- cells := []pdf.TSRCell{
- {X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "年份", Label: "table column header"},
- {X0: 101, Y0: 0, X1: 200, Y1: 30, Text: "金额", Label: "table column header"},
- {X0: 0, Y0: 35, X1: 100, Y1: 65, Text: "2020", Label: "table row"},
- {X0: 101, Y0: 35, X1: 200, Y1: 65, Text: "1000", Label: "table row"},
- {X0: 0, Y0: 70, X1: 100, Y1: 100, Text: "2021", Label: "table row"},
- {X0: 101, Y0: 70, X1: 200, Y1: 100, Text: "2000", Label: "table row"},
- {X0: 0, Y0: 105, X1: 100, Y1: 135, Text: "2022", Label: "table row"},
- {X0: 101, Y0: 105, X1: 200, Y1: 135, Text: "3000", Label: "table row"},
- }
-
- item := &pdf.TableItem{}
- html := ConstructTable(cells, nil, "", item)
-
- thCount := strings.Count(html, "<th ")
- if thCount != 2 {
- t.Errorf("expected 2 <th > for non-numeric headers under Nu-dominant table, got %d. HTML: %s", thCount, html)
- }
-
- t.Log("FIX: blockType classifies '年份'/'金额' as non-Nu headers, '2020'/'1000' as Nu data.")
- t.Logf("BlockType('年份')=%q BlockType('2020')=%q", BlockType("年份"), BlockType("2020"))
-}
-
-// TestConstructTable_BlockTypeChangesHeaderDetection verifies blockType
-// changes header detection for a table WITHOUT TSR header labels.
-// This is the case where pure label-based detection would fail.
-func TestConstructTable_BlockTypeChangesHeaderDetection(t *testing.T) {
- // Table with NO "header" labels — label-based detection gives 0 headers.
- // blockType: "姓名"/"年龄" → Tx, "张三"/"25" → Ot/En/? — maxType varies.
- // With Nu-dominant data, non-Nu top row cells count as possible headers.
- cells := []pdf.TSRCell{
- {X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "姓名", Label: "table row"},
- {X0: 101, Y0: 0, X1: 200, Y1: 30, Text: "年龄", Label: "table row"},
- {X0: 0, Y0: 35, X1: 100, Y1: 65, Text: "张三", Label: "table row"},
- {X0: 101, Y0: 35, X1: 200, Y1: 65, Text: "25", Label: "table row"},
- {X0: 0, Y0: 70, X1: 100, Y1: 100, Text: "李四", Label: "table row"},
- {X0: 101, Y0: 70, X1: 200, Y1: 100, Text: "30", Label: "table row"},
- {X0: 0, Y0: 105, X1: 100, Y1: 135, Text: "王五", Label: "table row"},
- {X0: 101, Y0: 105, X1: 200, Y1: 135, Text: "28", Label: "table row"},
- }
-
- html := ConstructTable(cells, nil, "", &pdf.TableItem{Grid: GroupTSRCellsToRows(cells)})
-
- // blockType analysis:
- // "姓名"(Tx), "年龄"(Tx), "张三"(Ot), "25"(Nu), "李四"(Ot), "30"(Nu), "王五"(Ot), "28"(Nu)
- // maxType could be Ot(3), Nu(3), or Tx(2).
- // Fallback catches the case where no headers detected by block-type path.
- t.Logf("HTML:\n%s", html)
- t.Log("FIX: blockType+fallback header detection works for tables without TSR header labels")
-}
-
-// =============================================================================
-// Issue 2b: colspan/rowspan missing
-// Python's __cal_spans computes colspan/rowspan from spanning cells by
-// clustering column centers and row centers. Go's rowsToHTML produces
-// a flat grid with no spanning attributes.
-// =============================================================================
-
-// TestRowsToHTML_NoColspanRowspan documents that rowsToHTML never produces
-// colspan or rowspan attributes, even for spanning cells.
-func TestRowsToHTML_NoColspanRowspan(t *testing.T) {
- // Two rows with a spanning cell in row 0.
- // In Python, a "table spanning cell" covering columns 0-1 would get colspan=2.
- rows := [][]pdf.TSRCell{
- {
- {Text: "跨列标题", Label: "table spanning cell"},
- {Text: "", Label: ""}, // padded cell
- },
- {
- {Text: "数据A", Label: "table row"},
- {Text: "数据B", Label: "table row"},
- },
- }
-
- html := RowsToHTML(rows, "", nil, nil, nil)
-
- // BUG: No colspan or rowspan attributes in output.
- if strings.Contains(html, "colspan") {
- t.Error("unexpected: colspan found in output (should not be present without __cal_spans)")
- }
- if strings.Contains(html, "rowspan") {
- t.Error("unexpected: rowspan found in output (should not be present without __cal_spans)")
- }
-
- // The spanning cell is rendered as a plain <td > with text, and the padded
- // empty cell is also rendered as an empty <td >. Python would merge them.
- tdCount := strings.Count(html, "<td ")
- if tdCount == 4 {
- t.Logf("Got %d <td > cells (flat grid, spanning cell + padded empty cell both rendered)", tdCount)
- } else {
- t.Logf("Got %d <td > cells. HTML:\n%s", tdCount, html)
- }
-
- t.Log("NOTE: Python's __cal_spans clusters column centers within spanning cells")
- t.Log("to compute colspan/rowspan. Go outputs a flat grid without spanning attributes.")
-}
-
-// TestConstructTable_SpannedTable_NoMerge documents the full constructTable
-// path with spanning cells — no colspan/rowspan in output.
-func TestConstructTable_SpannedTable_NoMerge(t *testing.T) {
- // Spanning cell at same Y as row cells so GroupTSRCellsToRows
- // puts them in the same row group. The spanning cell covers X=0-200
- // (both columns); Python's __cal_spans would give it colspan=2.
- cells := []pdf.TSRCell{
- // Row 0: a spanning cell that covers both columns + one regular cell.
- {X0: 0, Y0: 0, X1: 200, Y1: 30, Text: "部门开支汇总", Label: "table spanning cell"},
- {X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "Q1", Label: "table row"},
- {X0: 101, Y0: 0, X1: 200, Y1: 30, Text: "Q2", Label: "table row"},
- // Row 1: data row
- {X0: 0, Y0: 35, X1: 100, Y1: 65, Text: "100", Label: "table row"},
- {X0: 101, Y0: 35, X1: 200, Y1: 65, Text: "200", Label: "table row"},
- }
-
- item := &pdf.TableItem{}
- html := ConstructTable(cells, nil, "", item)
-
- // Verify colspan IS now detected (calSpans aligned with Python's __cal_spans).
- if !strings.Contains(html, "colspan") {
- t.Error("expected colspan on spanning cell, calSpans should detect it")
- }
-
- // Verify the HTML structure — spanning cell exists WITH colspan.
- if !strings.Contains(html, "部门开支汇总") {
- t.Error("spanning cell text missing")
- }
- if !strings.Contains(html, "Q1") {
- t.Error("Q1 cell should still be present (covered by span)")
- }
- t.Logf("HTML:\n%s", html)
-}
-
-// =============================================================================
-// Issue 2c: Single column/row cleanup missing
-// Python's construct_table removes orphan columns (only one non-empty cell)
-// when ≥4 rows, and orphan rows when ≥4 columns. Go has no such cleanup.
-// =============================================================================
-
-// TestConstructTable_OrphanColumn_NotCleanedUp documents that Go does NOT
-// remove columns that have only one non-empty cell.
-func TestConstructTable_OrphanColumn_NotCleanedUp(t *testing.T) {
- // 4 rows × 3 columns. Column index 1 has only ONE non-empty cell.
- // Python would relocate/merge that orphan column.
- cells := []pdf.TSRCell{
- {X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "姓名", Label: "table column header"},
- {X0: 101, Y0: 0, X1: 200, Y1: 30, Text: "备注", Label: "table row"}, // orphan col
- {X0: 201, Y0: 0, X1: 300, Y1: 30, Text: "年龄", Label: "table column header"},
- {X0: 0, Y0: 35, X1: 100, Y1: 65, Text: "张三", Label: "table row"},
- {X0: 101, Y0: 35, X1: 200, Y1: 65, Text: "", Label: "table row"}, // col 1 empty
- {X0: 201, Y0: 35, X1: 300, Y1: 65, Text: "25", Label: "table row"},
- {X0: 0, Y0: 70, X1: 100, Y1: 100, Text: "李四", Label: "table row"},
- {X0: 101, Y0: 70, X1: 200, Y1: 100, Text: "", Label: "table row"}, // col 1 empty
- {X0: 201, Y0: 70, X1: 300, Y1: 100, Text: "30", Label: "table row"},
- {X0: 0, Y0: 105, X1: 100, Y1: 135, Text: "王五", Label: "table row"},
- {X0: 101, Y0: 105, X1: 200, Y1: 135, Text: "", Label: "table row"}, // col 1 empty
- {X0: 201, Y0: 105, X1: 300, Y1: 135, Text: "28", Label: "table row"},
- }
-
- item := &pdf.TableItem{}
- html := ConstructTable(cells, nil, "", item)
-
- // BUG: All 4 rows have 3 cells each (orphan column preserved).
- // Python's construct_table pops single-cell columns when ≥4 rows.
- trCount := strings.Count(html, "<tr>")
- totalTdTh := strings.Count(html, "<td ") + strings.Count(html, "<th ")
-
- t.Logf("Rows: %d, Total cells: %d (Python would cleanup orphan columns)", trCount, totalTdTh)
- t.Log("NOTE: Python's construct_table removes columns with only one non-empty cell")
- t.Log("when there are ≥4 rows, and removes rows with only one non-empty cell")
- t.Log("when there are ≥4 columns. Go has no equivalent cleanup.")
- t.Logf("HTML:\n%s", html)
-}
-
-// =============================================================================
-// Issue 2d: is_caption pattern matching in mergeCaptions
-// Python's is_caption detects captions by text patterns (图表, Fig., Table, etc.)
-// AND layout_type. Go's mergeCaptions only checks LayoutType. If DLA labels a
-// caption as "text", Go misses it.
-// =============================================================================
-
-// TestMergeCaptions_NoIsCaptionPatternMatch documents that mergeCaptions only
-// uses LayoutType, NOT text patterns, for caption detection.
-func TestMergeCaptions_NoIsCaptionPatternMatch(t *testing.T) {
- // A caption-like text labeled as "text" by DLA (happens with imperfect DLA).
- // Python's is_caption would match "表1:测试数据" pattern regardless of layout_type.
- // FIX: mergeCaptions now calls captionKind → isCaptionBox to detect these.
- sections := []pdf.Section{
- {Text: "T", LayoutType: "table", Positions: []pdf.Position{
- {PageNumbers: []int{0, 0}, Left: 10, Right: 100, Top: 0, Bottom: 30},
- }},
- // This is clearly a table caption by text pattern, but DLA labeled it as "text".
- {Text: "表1:测试数据", LayoutType: "text", Positions: []pdf.Position{
- {PageNumbers: []int{0, 0}, Left: 10, Right: 100, Top: 40, Bottom: 55},
- }},
- }
-
- figures := pdf.CollectFigures(sections)
- result := mergeCaptions(sections, figures)
-
- // FIX VERIFIED: "表1:测试数据" should be detected as caption via isCaptionBox
- // and merged into the table section.
- merged := false
- for _, s := range result {
- if s.LayoutType == "table" && strings.Contains(s.Text, "表1:测试数据") {
- merged = true
- t.Log("FIX VERIFIED: caption with LayoutType='text' detected via isCaptionBox and merged into table")
- }
- }
- if !merged {
- t.Error("FIX FAILED: caption '表1:测试数据' should be merged into table via isCaptionBox pattern matching")
- }
-
- // Caption section should be removed.
- for _, s := range result {
- if s.LayoutType == "text" && s.Text == "表1:测试数据" {
- t.Error("FIX FAILED: caption section should be removed after merge")
- }
- }
-}
-
-// TestIsCaptionBox_MatchesChinesePattern verifies the existing isCaptionBox
-// function works correctly (it exists but is only used in fillCellTextFromBoxes,
-// not in mergeCaptions or caption detection pipeline).
-func TestIsCaptionBox_MatchesChinesePattern(t *testing.T) {
- tests := []struct {
- text string
- layoutType string
- want bool
- }{
- {"表1:交通工具等级", "", true},
- {"表 1:测试数据", "", true},
- {"图1:系统架构", "", true},
- {"图表 3: 实验结果", "", true},
- {"Fig. 1: Architecture", "", true},
- {"Figure 2: Pipeline", "", true},
- {"Table 3: Results", "", true},
- {"普通文本", "", false},
- {"", "", false},
- {"第一章 概述", "", false},
- // LayoutType-based detection
- {"anything", "figure caption", true},
- {"anything", "table caption", true},
- }
-
- for _, tt := range tests {
- got := IsCaptionBox(tt.text, tt.layoutType)
- if got != tt.want {
- t.Errorf("IsCaptionBox(%q, %q) = %v, want %v", tt.text, tt.layoutType, got, tt.want)
- }
- }
-
- t.Log("NOTE: isCaptionBox is now called by mergeCaptions via captionKind for DLA-mislabeled captions.")
-}
-
-// TestFigureInsertion_EndToEnd runs the full Parse pipeline on a PDF with
-// a figure DLA region containing TWO text boxes far enough apart that
-// NaiveVerticalMerge won't merge them. Python's _extract_table_figure +
-// insert_table_figures pops ALL figure boxes and re-inserts ONE unified
-// figure block regardless of text box positions. Go leaves the individual
-// text boxes as separate sections — this test FAILS to expose that.
-func TestFigureInsertion_EndToEnd(t *testing.T) {
- eng := &mockEngine{
- pageCount: 1,
- renderW: 1800, renderH: 2400,
- chars: map[int][]pdf.TextChar{0: {
- // Two text boxes in the SAME figure DLA region, but far apart.
- // DLA pixel: X=100-500 Y=80-600 → PDF 33-167 x 27-200.
- // Box 1 near top, box 2 near bottom.
- {X0: 50, X1: 150, Top: 40, Bottom: 55, Text: "架构图"},
- {X0: 50, X1: 150, Top: 170, Bottom: 185, Text: "系统模块"},
- }},
- }
- mock := &MockDocAnalyzer{
- Healthy: true,
- DLARegions: []pdf.DLARegion{
- // Large figure region covering both text boxes.
- {X0: 100, Y0: 80, X1: 500, Y1: 600, Label: "figure", Confidence: 0.9},
- },
- }
- p := NewParser(pdf.DefaultParserConfig(), mock)
-
- result, err := p.Parse(context.Background(), eng)
- if err != nil {
- t.Fatalf("Parse: %v", err)
- }
-
- // ── Python behavior: _extract_table_figure + insert_table_figures ──
- // Pops ALL figure boxes regardless of position, cropout creates ONE
- // consolidated image covering the entire DLA figure region, and
- // insert_table_figures re-inserts ONE figure block.
- // Expected: 1 figure section with combined text + cropped image.
-
- // ── Go current behavior ──
- // Figure boxes stay in list. NaiveVerticalMerge may NOT merge them
- // if the gap is too large (> 1.5 × median_height ≈ 15pt).
- // Each figure text box → separate section in result.Sections.
- // pdf.CollectFigures collects them into result.Figures() but doesn't re-insert.
-
- var figureSections []pdf.Section
- for _, s := range result.Sections {
- if s.LayoutType == "figure" {
- figureSections = append(figureSections, s)
- }
- }
-
- // Assert 1: Python expects exactly 1 consolidated figure section.
- // Go currently produces 2 (one per unmerged text box) — this FAILS.
- if len(figureSections) != 1 {
- t.Errorf("FIGURE INSERTION BUG: expected 1 consolidated figure section (Python insert_table_figures), got %d. Go does not consolidate figure text boxes into a single block.", len(figureSections))
- }
-
- // Assert 2: The single figure section must contain BOTH text fragments.
- if len(figureSections) == 1 {
- combined := figureSections[0].Text
- if !strings.Contains(combined, "架构图") || !strings.Contains(combined, "系统模块") {
- t.Errorf("FIGURE INSERTION BUG: figure section text=%q should contain both fragments. Python merges all figure-region text.", combined)
- }
- }
-
- t.Logf("figure sections in Sections: %d", len(figureSections))
- t.Logf("result.Figures() count: %d", len(result.Figures()))
- t.Logf("result.Sections total: %d", len(result.Sections))
- for i, s := range result.Sections {
- t.Logf(" section[%d] layout=%q text=%q", i, s.LayoutType, s.Text)
- }
-}
-
-// =============================================================================
-// Issue 3: Multi-page table merging
-// Python's _extract_table_figure merges tables with same layoutno across
-// consecutive pages (gap ≤ 1 page, Y-dis ≤ 23× median height).
-// Go's extractTableAndReplace does NOT merge tables across pages.
-// =============================================================================
-
-// TestExtractTableAndReplace_NoCrossPageMerge exposes that extractTableAndReplace
-// does not merge tables from consecutive pages even with the same layoutno.
-func TestExtractTableAndReplace_NoCrossPageMerge(t *testing.T) {
- // Simulate a table spanning pages 0 and 1.
- // Python would merge these because: same layoutno, consecutive pages,
- // Y-distance ≤ 23× median_height.
- boxes := []pdf.TextBox{
- {X0: 10, X1: 200, Top: 500, Bottom: 530, Text: "续表内容", LayoutType: "table", PageNumber: 0, LayoutNo: "0"},
- {X0: 10, X1: 200, Top: 50, Bottom: 80, Text: "表尾内容", LayoutType: "table", PageNumber: 1, LayoutNo: "0"},
- }
-
- // Two separate TableItems — one per page. Python would merge these
- // before insert_table_figures.
- tables := []pdf.TableItem{
- {
- Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "Page0", Label: "table row"}},
- Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 0, Right: 300, Top: 500, Bottom: 530}},
- Scale: 1.0,
- },
- {
- Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "Page1", Label: "table row"}},
- Positions: []pdf.Position{{PageNumbers: []int{1}, Left: 0, Right: 300, Top: 50, Bottom: 80}},
- Scale: 1.0,
- },
- }
-
- result := ExtractTableAndReplace(boxes, tables)
-
- // Go produces 2 separate HTML table boxes (one per page).
- // Python would produce 1 merged table with cells from both pages.
- tableCount := 0
- for _, b := range result {
- if strings.Contains(b.Text, "<table>") {
- tableCount++
- }
- }
- if tableCount == 2 {
- t.Errorf("CROSS-PAGE TABLE MERGE BUG: got %d separate HTML tables across pages. Python would merge same-layoutno tables on consecutive pages into 1 consolidated table.", tableCount)
- }
- t.Logf("table HTML boxes: %d (Python would merge into 1)", tableCount)
-}
-
-// =============================================================================
-// Issue 3a: nomerge_lout_no — don't merge tables separated by captions
-// Python's _extract_table_figure tracks nomerge_lout_no: when a table box
-// is followed by a caption/title/reference, the table's key is added to
-// nomerge_lout_no. Later, cross-page merge skips tables in nomerge_lout_no.
-//
-// Example:
-// Page 0: table "0-table-3" → caption "表1:..." → table "0-table-4"
-// Page 1: table "1-table-3" (same layoutNo)
-// → Page 0's table-3 should NOT merge with Page 1's table-3,
-// because the caption on page 0 indicates the table ended.
-// → Go's mergeTablesAcrossPages has no nomerge_lout_no check.
-// =============================================================================
-
-// TestMergeTablesAcrossPages_NomergeAfterCaption_Missing exposes that
-// mergeTablesAcrossPages unconditionally merges consecutive-page tables,
-// even when Python's nomerge_lout_no would prevent it.
-func TestMergeTablesAcrossPages_NomergeAfterCaption_Missing(t *testing.T) {
- // Simulate: page 0 has table at top, followed by a caption,
- // then another table. Page 1 has the same-layoutNo table continuing.
- // In Python, page 0's first table goes into nomerge_lout_no because
- // the next box is a caption → no cross-page merge for that table group.
- tables := []pdf.TableItem{
- {
- Cells: []pdf.TSRCell{{Text: "Page0-first", Label: "table row"}},
- Positions: []pdf.Position{{
- PageNumbers: []int{0},
- Left: 0, Right: 300,
- Top: 0, Bottom: 50,
- }},
- NoMerge: true, // Set when caption follows this table on the page
- },
- {
- Cells: []pdf.TSRCell{{Text: "Page1-cont", Label: "table row"}},
- Positions: []pdf.Position{{
- PageNumbers: []int{1},
- Left: 0, Right: 300,
- Top: 0, Bottom: 50,
- }},
- },
- }
-
- result := MergeTablesAcrossPages(tables, nil)
-
- // Verify NoMerge prevents cross-page merging.
- if len(result) != 2 {
- t.Errorf("NOMERGE BUG: expected 2 separate table groups, got %d.", len(result))
- }
- t.Log("NoMerge flag correctly prevents cross-page merge.")
-}
-
-// =============================================================================
-// Issue 3b: insert position — min_rectangle_distance vs anchor
-// Python's insert_table_figures uses min_rectangle_distance to find the
-// spatially nearest text box and inserts the table/figure next to it.
-// Go's extractTableAndReplace uses the first replaced table box index as
-// the anchor (insert position).
-//
-// When the DLA table region extends beyond the anchor box's bottom and
-// overlaps a text box below the table, Python puts the table next to that
-// overlapping text box (distance=0); Go puts it at the anchor position.
-// =============================================================================
-
-// TestExtractTableAndReplace_InsertionPosition_DistanceBug exposes that
-// extractTableAndReplace uses the first table box as anchor, rather than
-// finding the spatially nearest text box like Python.
-func TestExtractTableAndReplace_InsertionPosition_DistanceBug(t *testing.T) {
- // Two text boxes above the table: L0 (left, near table) and R0 (right, far).
- // Python: nearest to table is L0 (dx=0, dy=70). L0 bottom=30 < table top=100
- // → insert AFTER L0. Result: [L0, table, R0, R1, L2].
- // Go: anchor = first table box (L1 at index 2). Result: [L0, R0, table, R1, L2].
- // The table is one position off.
- boxes := []pdf.TextBox{
- {X0: 10, X1: 100, Top: 10, Bottom: 30, Text: "L0", LayoutType: "text", PageNumber: 0},
- {X0: 300, X1: 400, Top: 10, Bottom: 30, Text: "R0", LayoutType: "text", PageNumber: 0},
- {X0: 10, X1: 100, Top: 100, Bottom: 130, Text: "table", LayoutType: "table", PageNumber: 0},
- {X0: 300, X1: 400, Top: 100, Bottom: 130, Text: "R1", LayoutType: "text", PageNumber: 0},
- {X0: 10, X1: 100, Top: 250, Bottom: 270, Text: "L2", LayoutType: "text", PageNumber: 0},
- }
-
- tables := []pdf.TableItem{{
- Cells: []pdf.TSRCell{{Text: "cell", Label: "table row"}},
- Positions: []pdf.Position{{Left: 10, Right: 100, Top: 100, Bottom: 130, PageNumbers: []int{0}}},
- Scale: 1.0,
- RegionLeft: 10, RegionRight: 100, RegionTop: 100, RegionBottom: 130,
- }}
-
- result := ExtractTableAndReplace(boxes, tables)
-
- // Find L0 and table positions.
- l0Idx, tableIdx := -1, -1
- for i, b := range result {
- if strings.TrimSpace(b.Text) == "L0" {
- l0Idx = i
- }
- if b.LayoutType == "table" {
- tableIdx = i
- }
- }
-
- // BUG: table should immediately follow L0 (nearest neighbor, insert_after).
- // Python: min_rectangle_distance → L0 nearest (dx=0, dy=70), L0 below table
- // → insert_at+1 → table right after L0.
- // Go: anchor = first table box index → table at original table box position.
- if tableIdx != l0Idx+1 {
- t.Errorf("INSERTION POSITION BUG: table (idx=%d) should immediately follow L0 (idx=%d). "+
- "Python's min_rectangle_distance finds L0 as nearest text box and inserts table after it. "+
- "Go anchors at first table box position (between R0 and R1).", tableIdx, l0Idx)
- }
- t.Logf("L0 at idx=%d, table at idx=%d", l0Idx, tableIdx)
- t.Log("Fix: replace first-replaced-box anchor with min_rectangle_distance nearest-neighbor (Python pdf_parser.py:1608-1655).")
-}
-
-// =============================================================================
-// Issue 4: page_cum_height coordinate system
-// Python tracks cumulative page image heights for cross-page position tags
-// and image cropping. Go uses per-page coordinates only.
-// =============================================================================
-
-// TestBoxesToSections_PerPageCoordinates confirms position tags use
-// page-relative coordinates. Python's _line_tag also produces local
-// coordinates (subtracts page_cum_height). The page number differentiates
-// pages; page_cum_height is an internal implementation detail.
-func TestBoxesToSections_PerPageCoordinates(t *testing.T) {
- boxes := []pdf.TextBox{
- {X0: 10, X1: 100, Top: 40, Bottom: 60, Text: "Page 0 text", LayoutType: "text", PageNumber: 0},
- {X0: 10, X1: 100, Top: 40, Bottom: 60, Text: "Page 1 text", LayoutType: "text", PageNumber: 1},
- }
- sections := boxesToSections(boxes, nil)
- if len(sections) != 2 {
- t.Fatalf("expected 2 sections, got %d", len(sections))
- }
- s0, s1 := sections[0], sections[1]
- if len(s0.Positions) > 0 && len(s1.Positions) > 0 {
- p0, p1 := s0.Positions[0], s1.Positions[0]
- // Both Python and Go use local (page-relative) coordinates.
- // Python's _line_tag: top = bx["top"] - page_cum_height[pn-1]
- // gives local coordinate. Same as Go.
- if p0.Top != p1.Top || p0.Bottom != p1.Bottom {
- t.Errorf("expected same local coords, got Top=(%.0f,%.0f) Bottom=(%.0f,%.0f)", p0.Top, p1.Top, p0.Bottom, p1.Bottom)
- }
- t.Logf("page 0: Page=%v Top=%.0f Bottom=%.0f", p0.PageNumbers, p0.Top, p0.Bottom)
- t.Logf("page 1: Page=%v Top=%.0f Bottom=%.0f", p1.PageNumbers, p1.Top, p1.Bottom)
- t.Log("OK: position tags use page-relative coordinates in both Go and Python.")
- }
-}
-
-// =============================================================================
-// Issue 6: cropSectionImage padding logic
-// Python's self.crop adds 120px context above first segment, 120px context
-// below last segment, 6px gap between pages, and overlay transparency.
-// Go has simpler crop logic.
-// =============================================================================
-
-// TestCropSectionImage_PaddingVsPython documents that Go's cropSectionImage
-// adds context padding differently from Python's self.crop.
-func TestCropSectionImage_PaddingVsPython(t *testing.T) {
- // Create a page image and position tag for a small text region.
- img := image.NewRGBA(image.Rect(0, 0, 300, 800)) // 300×800 page at zoom=3 → PDF 100×267
- pageImages := map[int]image.Image{0: img}
-
- // pdf.Position tag for a small text box near the top of the page.
- posTag := FormatPositionTag(0, 50.0, 100.0, 10.0, 30.0)
-
- result := util.CropSectionImage(posTag, pageImages, 3.0)
-
- if result == "" {
- t.Error("cropSectionImage returned empty string for valid position")
- }
- // Decode result to check image dimensions.
- data, err := base64.StdEncoding.DecodeString(result)
- if err != nil {
- t.Fatalf("failed to decode base64: %v", err)
- }
- cropped, _, err := image.Decode(bytes.NewReader(data))
- if err != nil {
- t.Fatalf("failed to decode PNG: %v", err)
- }
- croppedH := cropped.Bounds().Dy()
- // Original text region: Top=10, Bottom=30 → height=20 at PDF points.
- // zoom=3 → 60px text height.
- // Python adds 120px context above + 120px below + 6px gap → ~306px.
- // Go adds contextPad=120 points above/below at PDF scale → with zoom=3: 360+60+360=780px.
- // Python uses pixel-space padding (120px literally), Go uses PDF-point padding (120pt).
- expectedMin := 60 // bare minimum: text region itself
- if croppedH <= expectedMin {
- t.Errorf("CROP PADDING BUG: cropped image height=%dpx, expected >%dpx with context padding. Python adds 120px above and below for context.", croppedH, expectedMin)
- }
- t.Logf("cropped image: %dx%d (text region 60px, expecting padding)", cropped.Bounds().Dx(), croppedH)
- t.Log("NOTE: Python's self.crop adds 120px context padding in pixel space, multi-page stitching, and overlay transparency. Go's cropSectionImage uses PDF-point padding and simpler stitching.")
-}
-
-// =============================================================================
-// Issue 7: Data-source filter missing
-// Python's _extract_table_figure pops table/figure boxes matching
-// r"(数据|资料|图表)*来源[:: ]" (pdf_parser.py:1040-1042, 1050-1052).
-// These boxes are discarded — not extracted, not inserted back.
-// Go has no equivalent filter in extractTableAndReplace or consolidateFigures.
-// =============================================================================
-
-// dataSourcePattern is a Go translation of Python's
-// r"(数据|资料|图表)*来源[:: ]" used with re.match (anchored at start).
-var dataSourcePattern = `^(数据|资料|图表)*来源[:: ]`
-
-// TestDataSourcePattern_RegexCoverage validates the Python regex behavior
-// that should be adopted. Documents which strings match and which don't.
-func TestDataSourcePattern_RegexCoverage(t *testing.T) {
- tests := []struct {
- text string
- want bool // Python re.match truthiness
- }{
- // ── Matching patterns (should be filtered) ──
- {"数据来源:国家统计局", true}, // 数据 + 来源 + fullwidth colon
- {"资料来源: 某报告", true}, // 资料 + 来源 + halfwidth colon
- {"图表来源:某数据库", true}, // 图表 + 来源 + fullwidth colon
- {"来源:权威机构", true}, // zero prefix + 来源 + fullwidth colon
- {"来源: 参考数据", true}, // zero prefix + 来源 + halfwidth colon
- {"数据来源 说明", true}, // 数据 + 来源 + space
-
- // ── Non-matching patterns (should NOT be filtered) ──
- {"数据来源明细", false}, // 来源 followed by 明, not ::space
- {"普通来源说明", false}, // doesn't start with keyword
- {"数据", false}, // too short
- {"来源", false}, // 来源 but no ::space after
- {"资料来源说明", false}, // 来源 followed by 说, not ::space
- {"", false}, // empty
- {"TABLE 1: 数据来源统计", false}, // doesn't start with keyword
- }
-
- for _, tt := range tests {
- matched := regexp.MustCompile(dataSourcePattern).MatchString(tt.text)
- if matched != tt.want {
- t.Errorf("dataSourcePattern.MatchString(%q) = %v, want %v", tt.text, matched, tt.want)
- }
- }
- t.Log("NOTE: Python re.match(r\"(数据|资料|图表)*来源[:: ]\", text) — anchored at start.")
- t.Log("Go regexp.MatchString equivalent with ^ prefix.")
-}
-
-// TestExtractTableAndReplace_DataSourceFilter_Missing exposes that Go does NOT
-// filter out table boxes whose text matches r"(数据|资料|图表)*来源[:: ]".
-// Python's _extract_table_figure pops these boxes from self.boxes without
-// adding them to the tables dict (pdf_parser.py:1040-1042).
-func TestExtractTableAndReplace_DataSourceFilter_Missing(t *testing.T) {
- // A table box with data-source text and a normal table box.
- // Both overlap a pdf.TableItem position, so both would be replaced with HTML.
- boxes := []pdf.TextBox{
- {X0: 10, X1: 200, Top: 0, Bottom: 50, Text: "数据来源:国家统计局", LayoutType: "table", PageNumber: 0},
- {X0: 10, X1: 200, Top: 60, Bottom: 80, Text: "表1:正常数据", LayoutType: "table", PageNumber: 0},
- }
-
- // Two TableItems — one per table box — so each would independently produce HTML.
- tables := []pdf.TableItem{
- {
- Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "来源", Label: "table row"}},
- Positions: []pdf.Position{{Left: 0, Right: 300, Top: 0, Bottom: 50}},
- Scale: 1.0,
- },
- {
- Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "正常", Label: "table row"}},
- Positions: []pdf.Position{{Left: 0, Right: 300, Top: 60, Bottom: 80}},
- Scale: 1.0,
- },
- }
-
- result := ExtractTableAndReplace(boxes, tables)
-
- // Python behavior: "数据来源:国家统计局" is popped from self.boxes,
- // NOT added to tables dict, NOT replaced with HTML. Gone entirely.
- // "表1:正常数据" is replaced with HTML as usual.
- // Expected result: exactly 1 HTML table box for the normal table.
- //
- // BUG: Go replaces both boxes with HTML tables. The data-source box
- // produces an HTML table with cell text "来源" — this should NOT exist.
- htmlTableCount := 0
- hasDataSourceTable := false
- for _, b := range result {
- if strings.Contains(b.Text, "<table>") {
- htmlTableCount++
- // The data-source table's cell text "来源" ends up in the HTML.
- // c.f. constructTable which uses pdf.TSRCell text, not box text.
- if strings.Contains(b.Text, ">来源<") {
- hasDataSourceTable = true
- }
- }
- }
- if htmlTableCount != 1 {
- t.Errorf("DATA SOURCE FILTER BUG: expected 1 HTML table (normal only), got %d. Python pops data-source table box entirely in _extract_table_figure (pdf_parser.py:1040-1042). Go replaces it with an HTML table.", htmlTableCount)
- }
- if hasDataSourceTable {
- t.Errorf("DATA SOURCE FILTER BUG: data-source table should NOT produce HTML output. Cell '来源' appears in HTML: Python discards these boxes, Go incorrectly constructs a table for them.")
- }
-
- t.Log("NOTE: Python filters table boxes matching r\"(数据|资料|图表)*来源[:: ]\" in _extract_table_figure.")
- t.Log("Go's extractTableAndReplace has no equivalent filter — data-source boxes get replaced with HTML instead of being discarded.")
-}
-
-// TestExtractTableAndReplace_DataSourceVariants tests multiple variants of
-// the data-source pattern that should all be filtered.
-func TestExtractTableAndReplace_DataSourceVariants(t *testing.T) {
- variants := []string{
- "数据来源:国家统计局",
- "资料来源: 某报告",
- "图表来源:某数据库",
- "来源:权威机构",
- "来源: 参考数据",
- }
-
- for _, variant := range variants {
- t.Run(variant, func(t *testing.T) {
- boxes := []pdf.TextBox{
- {X0: 10, X1: 200, Top: 0, Bottom: 50, Text: variant, LayoutType: "table", PageNumber: 0},
- }
-
- tables := []pdf.TableItem{{
- Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "A", Label: "table row"}},
- Positions: []pdf.Position{{Left: 0, Right: 300, Top: 0, Bottom: 50}},
- Scale: 1.0,
- }}
-
- result := ExtractTableAndReplace(boxes, tables)
-
- // BUG: box with data-source text should be REMOVED entirely —
- // zero HTML output. Python pops these boxes without replacement.
- for _, b := range result {
- if strings.Contains(b.Text, "<table>") {
- t.Errorf("DATA SOURCE FILTER BUG: variant %q should be removed without HTML replacement. Python pops data-source table boxes entirely.", variant)
- }
- }
- })
- }
- t.Log("NOTE: All variants of r\"(数据|资料|图表)*来源[:: ]\" should be filtered by extractTableAndReplace.")
-}
-
-// TestConsolidateFigures_DataSourceFilter_Missing exposes that Go does NOT
-// filter out figure boxes whose text matches r"(数据|资料|图表)*来源[:: ]".
-// Python's _extract_table_figure pops these boxes from self.boxes without
-// adding them to the figures dict (pdf_parser.py:1050-1052).
-func TestConsolidateFigures_DataSourceFilter_Missing(t *testing.T) {
- boxes := []pdf.TextBox{
- {X0: 10, X1: 200, Top: 0, Bottom: 50, Text: "数据来源:某机构", LayoutType: "figure", PageNumber: 0, LayoutNo: "figure-0"},
- {X0: 10, X1: 200, Top: 60, Bottom: 80, Text: "架构图", LayoutType: "figure", PageNumber: 0, LayoutNo: "figure-0"},
- }
-
- result := ConsolidateFigures(boxes)
-
- // Python behavior: "数据来源:某机构" is popped from self.boxes,
- // NOT added to figures dict → gone entirely.
- // "架构图" is extracted normally.
- // Expected result: exactly 1 figure box with "架构图" text only.
- for _, b := range result {
- if strings.Contains(b.Text, "数据来源") || strings.Contains(b.Text, "某机构") {
- t.Errorf("DATA SOURCE FIGURE FILTER BUG: '数据来源:某机构' figure box should be removed entirely. Python pops data-source figure boxes in _extract_table_figure (pdf_parser.py:1050-1052). Go still includes it.")
- }
- }
-
- // Verify the normal figure box IS still present.
- foundFigure := false
- for _, b := range result {
- if strings.Contains(b.Text, "架构图") {
- foundFigure = true
- }
- }
- if !foundFigure {
- t.Error("normal figure box '架构图' should still be present")
- }
-
- t.Log("NOTE: Python filters figure boxes matching r\"(数据|资料|图表)*来源[:: ]\" in _extract_table_figure.")
- t.Log("Go's consolidateFigures has no equivalent filter.")
-}