RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
chore(table): remove stale manual parity-issues test (#18403)
Jack97db1b4c38b0
diff --git a/internal/deepdoc/parser/pdf/table/table_parity_issues_test.go b/internal/deepdoc/parser/pdf/table/table_parity_issues_test.godeleted file mode 100644index 59e2df867e..0000000000--- a/internal/deepdoc/parser/pdf/table/table_parity_issues_test.go+++ /dev/null@@ -1,885 +0,0 @@-//go:build manual--package table--import (- "bytes"- "context"- "encoding/base64"- "image"- pdf "ragflow/internal/deepdoc/parser/pdf/type"- "regexp"- "strings"- "testing"-)--// =============================================================================-// Issue 1: Figure insertion strategy-// Python's insert_table_figures(figs, "figure") inserts figure boxes back into-// self.boxes. Go's extractTableAndReplace only handles LayoutType=="table",-// leaving figure boxes in the list. This test documents the current behavior.-// =============================================================================--// TestExtractTableAndReplace_IgnoresFigures documents that extractTableAndReplace-// does NOT pop or replace figure boxes. In Python's _extract_table_figure,-// figure boxes are popped and re-inserted via insert_table_figures with cropped-// images. Go leaves them in the box list for downstream boxesToSections.-func TestExtractTableAndReplace_IgnoresFigures(t *testing.T) {- boxes := []pdf.TextBox{- {X0: 10, X1: 200, Top: 0, Bottom: 50, Text: "Figure text", LayoutType: "figure", PageNumber: 0},- {X0: 10, X1: 200, Top: 60, Bottom: 80, Text: "表1:标题", LayoutType: "table", PageNumber: 0},- }-- // Table with cells so extractTableAndReplace generates HTML.- tables := []pdf.TableItem{{- Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "A", Label: "table row"}},- Positions: []pdf.Position{{Left: 0, Right: 300, Top: 0, Bottom: 100}},- Scale: 1.0,- }}-- result := ExtractTableAndReplace(boxes, tables)-- // BUG: Figure box is still present — it was not popped or replaced.- // Python's _extract_table_figure pops figure boxes and re-inserts them- // via insert_table_figures with cropped images.- hasFigure := false- for _, b := range result {- if b.LayoutType == "figure" {- hasFigure = true- // Figure text is still raw text, not a consolidated image+text block- // like Python's insert_table_figures would produce.- if b.Text != "Figure text" {- t.Errorf("figure text should be unchanged, got %q", b.Text)- }- }- }- if !hasFigure {- t.Error("BUG EXPOSED: extractTableAndReplace removed figure box (unexpected)")- }- t.Log("NOTE: Figure box remains in list as raw text. Python inserts figures back with cropped images via insert_table_figures. Go collects figures separately via pdf.CollectFigures without re-inserting.")-}--// TestBoxesToSections_FiguresNotReinserted documents that boxesToSections converts-// figure boxes to sections but without the consolidated image that Python's-// insert_table_figures would attach.-func TestBoxesToSections_FiguresNotReinserted(t *testing.T) {- // Simulate post-extractTableAndReplace boxes with figures still present.- boxes := []pdf.TextBox{- {X0: 10, X1: 200, Top: 0, Bottom: 50, Text: "Some text", LayoutType: "text", PageNumber: 0},- {X0: 10, X1: 200, Top: 60, Bottom: 100, Text: "Figure description", LayoutType: "figure", PageNumber: 0},- }-- sections := boxesToSections(boxes, nil)- figures := pdf.CollectFigures(sections)-- // BUG: figures are collected separately but NOT re-inserted into sections- // after image processing. In Python, insert_table_figures(figs, "figure")- // creates new boxes with layout_type="figure", image=cropped_img, and- // inserts them at the nearest position among text boxes.- if len(figures) != 1 {- t.Fatalf("expected 1 figure, got %d", len(figures))- }- if figures[0].LayoutType != "figure" {- t.Errorf("expected LayoutType 'figure', got %q", figures[0].LayoutType)- }- // Figure image is empty at this stage (cropSectionImage runs later in pipeline).- if figures[0].Image != "" {- t.Log("figure has image (cropSectionImage already ran)")- } else {- t.Log("NOTE: Figure section has no Image yet. Python's cropout creates a consolidated cropped image for the entire figure region before insert_table_figures.")- }-- t.Logf("Sections count: %d (figure present as raw text section)", len(sections))- t.Logf("Figures count: %d (collected separately, Python re-inserts them)", len(figures))-}--// =============================================================================-// Issue 2a: blockType classification missing-// Python's construct_table classifies each cell into 9 types (Dt/Nu/Ca/En/NE/-// Sg/Tx/Lx/Nr/Ot). The dominant type drives header detection: if max_type is-// "Nu" (numeric), numeric cells don't count as headers. Go's headerSet only-// checks TSR labels — no cell content type analysis.-// =============================================================================--// TestConstructTable_HeaderDetection_NoBlockType documents that Go's header-// detection is purely TSR-label-based. Python would use blockType to skip-// numeric cells when the dominant type is "Nu".-func TestConstructTable_HeaderDetection_NoBlockType(t *testing.T) {- // A table where the "header" row has numeric content (like years, amounts).- // With blockType: "2020","2021" → Nu, "100","200" → Nu — maxType=Nu.- // block-type-aware detection skips Nu cells → 0 headers.- // Falls back to TSR label-based detection → still gets 2 <th >.- cells := []pdf.TSRCell{- {X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "2020", Label: "table column header"},- {X0: 101, Y0: 0, X1: 200, Y1: 30, Text: "2021", Label: "table column header"},- {X0: 0, Y0: 35, X1: 100, Y1: 65, Text: "100", Label: "table row"},- {X0: 101, Y0: 35, X1: 200, Y1: 65, Text: "200", Label: "table row"},- }-- item := &pdf.TableItem{}- html := ConstructTable(cells, nil, "", item)-- // FIX VERIFIED: headerSetWithBlockType computes block types (all "Nu"),- // skips Nu headers when maxType=Nu, then falls back to TSR label detection.- // Header row still gets <th > because TSR labels contain "header".- thCount := strings.Count(html, "<th ")- if thCount != 2 {- t.Errorf("expected 2 <th >, got %d. HTML: %s", thCount, html)- }-- t.Log("FIX: blockType classification added. maxType=Nu skips Nu headers in primary pass.")- t.Log("TSR label fallback still marks header rows with 'header' in label.")-}--// TestConstructTable_BlockType_DominantTypeMissing documents that Go has no-// concept of a "dominant cell type" that Python uses for header detection.-func TestConstructTable_BlockType_DominantTypeMissing(t *testing.T) {- // Mixed table with numeric-dominant data, testing blockType header detection.- // "年份"/"金额" → Tx (short text), "2020"/"1000"/etc → Nu. maxType=Nu.- // Header cells are non-Nu → count as headers even under Nu-dominant logic.- // FIX: blockType now classifies cells and drives header detection.- cells := []pdf.TSRCell{- {X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "年份", Label: "table column header"},- {X0: 101, Y0: 0, X1: 200, Y1: 30, Text: "金额", Label: "table column header"},- {X0: 0, Y0: 35, X1: 100, Y1: 65, Text: "2020", Label: "table row"},- {X0: 101, Y0: 35, X1: 200, Y1: 65, Text: "1000", Label: "table row"},- {X0: 0, Y0: 70, X1: 100, Y1: 100, Text: "2021", Label: "table row"},- {X0: 101, Y0: 70, X1: 200, Y1: 100, Text: "2000", Label: "table row"},- {X0: 0, Y0: 105, X1: 100, Y1: 135, Text: "2022", Label: "table row"},- {X0: 101, Y0: 105, X1: 200, Y1: 135, Text: "3000", Label: "table row"},- }-- item := &pdf.TableItem{}- html := ConstructTable(cells, nil, "", item)-- thCount := strings.Count(html, "<th ")- if thCount != 2 {- t.Errorf("expected 2 <th > for non-numeric headers under Nu-dominant table, got %d. HTML: %s", thCount, html)- }-- t.Log("FIX: blockType classifies '年份'/'金额' as non-Nu headers, '2020'/'1000' as Nu data.")- t.Logf("BlockType('年份')=%q BlockType('2020')=%q", BlockType("年份"), BlockType("2020"))-}--// TestConstructTable_BlockTypeChangesHeaderDetection verifies blockType-// changes header detection for a table WITHOUT TSR header labels.-// This is the case where pure label-based detection would fail.-func TestConstructTable_BlockTypeChangesHeaderDetection(t *testing.T) {- // Table with NO "header" labels — label-based detection gives 0 headers.- // blockType: "姓名"/"年龄" → Tx, "张三"/"25" → Ot/En/? — maxType varies.- // With Nu-dominant data, non-Nu top row cells count as possible headers.- cells := []pdf.TSRCell{- {X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "姓名", Label: "table row"},- {X0: 101, Y0: 0, X1: 200, Y1: 30, Text: "年龄", Label: "table row"},- {X0: 0, Y0: 35, X1: 100, Y1: 65, Text: "张三", Label: "table row"},- {X0: 101, Y0: 35, X1: 200, Y1: 65, Text: "25", Label: "table row"},- {X0: 0, Y0: 70, X1: 100, Y1: 100, Text: "李四", Label: "table row"},- {X0: 101, Y0: 70, X1: 200, Y1: 100, Text: "30", Label: "table row"},- {X0: 0, Y0: 105, X1: 100, Y1: 135, Text: "王五", Label: "table row"},- {X0: 101, Y0: 105, X1: 200, Y1: 135, Text: "28", Label: "table row"},- }-- html := ConstructTable(cells, nil, "", &pdf.TableItem{Grid: GroupTSRCellsToRows(cells)})-- // blockType analysis:- // "姓名"(Tx), "年龄"(Tx), "张三"(Ot), "25"(Nu), "李四"(Ot), "30"(Nu), "王五"(Ot), "28"(Nu)- // maxType could be Ot(3), Nu(3), or Tx(2).- // Fallback catches the case where no headers detected by block-type path.- t.Logf("HTML:\n%s", html)- t.Log("FIX: blockType+fallback header detection works for tables without TSR header labels")-}--// =============================================================================-// Issue 2b: colspan/rowspan missing-// Python's __cal_spans computes colspan/rowspan from spanning cells by-// clustering column centers and row centers. Go's rowsToHTML produces-// a flat grid with no spanning attributes.-// =============================================================================--// TestRowsToHTML_NoColspanRowspan documents that rowsToHTML never produces-// colspan or rowspan attributes, even for spanning cells.-func TestRowsToHTML_NoColspanRowspan(t *testing.T) {- // Two rows with a spanning cell in row 0.- // In Python, a "table spanning cell" covering columns 0-1 would get colspan=2.- rows := [][]pdf.TSRCell{- {- {Text: "跨列标题", Label: "table spanning cell"},- {Text: "", Label: ""}, // padded cell- },- {- {Text: "数据A", Label: "table row"},- {Text: "数据B", Label: "table row"},- },- }-- html := RowsToHTML(rows, "", nil, nil, nil)-- // BUG: No colspan or rowspan attributes in output.- if strings.Contains(html, "colspan") {- t.Error("unexpected: colspan found in output (should not be present without __cal_spans)")- }- if strings.Contains(html, "rowspan") {- t.Error("unexpected: rowspan found in output (should not be present without __cal_spans)")- }-- // The spanning cell is rendered as a plain <td > with text, and the padded- // empty cell is also rendered as an empty <td >. Python would merge them.- tdCount := strings.Count(html, "<td ")- if tdCount == 4 {- t.Logf("Got %d <td > cells (flat grid, spanning cell + padded empty cell both rendered)", tdCount)- } else {- t.Logf("Got %d <td > cells. HTML:\n%s", tdCount, html)- }-- t.Log("NOTE: Python's __cal_spans clusters column centers within spanning cells")- t.Log("to compute colspan/rowspan. Go outputs a flat grid without spanning attributes.")-}--// TestConstructTable_SpannedTable_NoMerge documents the full constructTable-// path with spanning cells — no colspan/rowspan in output.-func TestConstructTable_SpannedTable_NoMerge(t *testing.T) {- // Spanning cell at same Y as row cells so GroupTSRCellsToRows- // puts them in the same row group. The spanning cell covers X=0-200- // (both columns); Python's __cal_spans would give it colspan=2.- cells := []pdf.TSRCell{- // Row 0: a spanning cell that covers both columns + one regular cell.- {X0: 0, Y0: 0, X1: 200, Y1: 30, Text: "部门开支汇总", Label: "table spanning cell"},- {X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "Q1", Label: "table row"},- {X0: 101, Y0: 0, X1: 200, Y1: 30, Text: "Q2", Label: "table row"},- // Row 1: data row- {X0: 0, Y0: 35, X1: 100, Y1: 65, Text: "100", Label: "table row"},- {X0: 101, Y0: 35, X1: 200, Y1: 65, Text: "200", Label: "table row"},- }-- item := &pdf.TableItem{}- html := ConstructTable(cells, nil, "", item)-- // Verify colspan IS now detected (calSpans aligned with Python's __cal_spans).- if !strings.Contains(html, "colspan") {- t.Error("expected colspan on spanning cell, calSpans should detect it")- }-- // Verify the HTML structure — spanning cell exists WITH colspan.- if !strings.Contains(html, "部门开支汇总") {- t.Error("spanning cell text missing")- }- if !strings.Contains(html, "Q1") {- t.Error("Q1 cell should still be present (covered by span)")- }- t.Logf("HTML:\n%s", html)-}--// =============================================================================-// Issue 2c: Single column/row cleanup missing-// Python's construct_table removes orphan columns (only one non-empty cell)-// when ≥4 rows, and orphan rows when ≥4 columns. Go has no such cleanup.-// =============================================================================--// TestConstructTable_OrphanColumn_NotCleanedUp documents that Go does NOT-// remove columns that have only one non-empty cell.-func TestConstructTable_OrphanColumn_NotCleanedUp(t *testing.T) {- // 4 rows × 3 columns. Column index 1 has only ONE non-empty cell.- // Python would relocate/merge that orphan column.- cells := []pdf.TSRCell{- {X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "姓名", Label: "table column header"},- {X0: 101, Y0: 0, X1: 200, Y1: 30, Text: "备注", Label: "table row"}, // orphan col- {X0: 201, Y0: 0, X1: 300, Y1: 30, Text: "年龄", Label: "table column header"},- {X0: 0, Y0: 35, X1: 100, Y1: 65, Text: "张三", Label: "table row"},- {X0: 101, Y0: 35, X1: 200, Y1: 65, Text: "", Label: "table row"}, // col 1 empty- {X0: 201, Y0: 35, X1: 300, Y1: 65, Text: "25", Label: "table row"},- {X0: 0, Y0: 70, X1: 100, Y1: 100, Text: "李四", Label: "table row"},- {X0: 101, Y0: 70, X1: 200, Y1: 100, Text: "", Label: "table row"}, // col 1 empty- {X0: 201, Y0: 70, X1: 300, Y1: 100, Text: "30", Label: "table row"},- {X0: 0, Y0: 105, X1: 100, Y1: 135, Text: "王五", Label: "table row"},- {X0: 101, Y0: 105, X1: 200, Y1: 135, Text: "", Label: "table row"}, // col 1 empty- {X0: 201, Y0: 105, X1: 300, Y1: 135, Text: "28", Label: "table row"},- }-- item := &pdf.TableItem{}- html := ConstructTable(cells, nil, "", item)-- // BUG: All 4 rows have 3 cells each (orphan column preserved).- // Python's construct_table pops single-cell columns when ≥4 rows.- trCount := strings.Count(html, "<tr>")- totalTdTh := strings.Count(html, "<td ") + strings.Count(html, "<th ")-- t.Logf("Rows: %d, Total cells: %d (Python would cleanup orphan columns)", trCount, totalTdTh)- t.Log("NOTE: Python's construct_table removes columns with only one non-empty cell")- t.Log("when there are ≥4 rows, and removes rows with only one non-empty cell")- t.Log("when there are ≥4 columns. Go has no equivalent cleanup.")- t.Logf("HTML:\n%s", html)-}--// =============================================================================-// Issue 2d: is_caption pattern matching in mergeCaptions-// Python's is_caption detects captions by text patterns (图表, Fig., Table, etc.)-// AND layout_type. Go's mergeCaptions only checks LayoutType. If DLA labels a-// caption as "text", Go misses it.-// =============================================================================--// TestMergeCaptions_NoIsCaptionPatternMatch documents that mergeCaptions only-// uses LayoutType, NOT text patterns, for caption detection.-func TestMergeCaptions_NoIsCaptionPatternMatch(t *testing.T) {- // A caption-like text labeled as "text" by DLA (happens with imperfect DLA).- // Python's is_caption would match "表1:测试数据" pattern regardless of layout_type.- // FIX: mergeCaptions now calls captionKind → isCaptionBox to detect these.- sections := []pdf.Section{- {Text: "T", LayoutType: "table", Positions: []pdf.Position{- {PageNumbers: []int{0, 0}, Left: 10, Right: 100, Top: 0, Bottom: 30},- }},- // This is clearly a table caption by text pattern, but DLA labeled it as "text".- {Text: "表1:测试数据", LayoutType: "text", Positions: []pdf.Position{- {PageNumbers: []int{0, 0}, Left: 10, Right: 100, Top: 40, Bottom: 55},- }},- }-- figures := pdf.CollectFigures(sections)- result := mergeCaptions(sections, figures)-- // FIX VERIFIED: "表1:测试数据" should be detected as caption via isCaptionBox- // and merged into the table section.- merged := false- for _, s := range result {- if s.LayoutType == "table" && strings.Contains(s.Text, "表1:测试数据") {- merged = true- t.Log("FIX VERIFIED: caption with LayoutType='text' detected via isCaptionBox and merged into table")- }- }- if !merged {- t.Error("FIX FAILED: caption '表1:测试数据' should be merged into table via isCaptionBox pattern matching")- }-- // Caption section should be removed.- for _, s := range result {- if s.LayoutType == "text" && s.Text == "表1:测试数据" {- t.Error("FIX FAILED: caption section should be removed after merge")- }- }-}--// TestIsCaptionBox_MatchesChinesePattern verifies the existing isCaptionBox-// function works correctly (it exists but is only used in fillCellTextFromBoxes,-// not in mergeCaptions or caption detection pipeline).-func TestIsCaptionBox_MatchesChinesePattern(t *testing.T) {- tests := []struct {- text string- layoutType string- want bool- }{- {"表1:交通工具等级", "", true},- {"表 1:测试数据", "", true},- {"图1:系统架构", "", true},- {"图表 3: 实验结果", "", true},- {"Fig. 1: Architecture", "", true},- {"Figure 2: Pipeline", "", true},- {"Table 3: Results", "", true},- {"普通文本", "", false},- {"", "", false},- {"第一章 概述", "", false},- // LayoutType-based detection- {"anything", "figure caption", true},- {"anything", "table caption", true},- }-- for _, tt := range tests {- got := IsCaptionBox(tt.text, tt.layoutType)- if got != tt.want {- t.Errorf("IsCaptionBox(%q, %q) = %v, want %v", tt.text, tt.layoutType, got, tt.want)- }- }-- t.Log("NOTE: isCaptionBox is now called by mergeCaptions via captionKind for DLA-mislabeled captions.")-}--// TestFigureInsertion_EndToEnd runs the full Parse pipeline on a PDF with-// a figure DLA region containing TWO text boxes far enough apart that-// NaiveVerticalMerge won't merge them. Python's _extract_table_figure +-// insert_table_figures pops ALL figure boxes and re-inserts ONE unified-// figure block regardless of text box positions. Go leaves the individual-// text boxes as separate sections — this test FAILS to expose that.-func TestFigureInsertion_EndToEnd(t *testing.T) {- eng := &mockEngine{- pageCount: 1,- renderW: 1800, renderH: 2400,- chars: map[int][]pdf.TextChar{0: {- // Two text boxes in the SAME figure DLA region, but far apart.- // DLA pixel: X=100-500 Y=80-600 → PDF 33-167 x 27-200.- // Box 1 near top, box 2 near bottom.- {X0: 50, X1: 150, Top: 40, Bottom: 55, Text: "架构图"},- {X0: 50, X1: 150, Top: 170, Bottom: 185, Text: "系统模块"},- }},- }- mock := &MockDocAnalyzer{- Healthy: true,- DLARegions: []pdf.DLARegion{- // Large figure region covering both text boxes.- {X0: 100, Y0: 80, X1: 500, Y1: 600, Label: "figure", Confidence: 0.9},- },- }- p := NewParser(pdf.DefaultParserConfig(), mock)-- result, err := p.Parse(context.Background(), eng)- if err != nil {- t.Fatalf("Parse: %v", err)- }-- // ── Python behavior: _extract_table_figure + insert_table_figures ──- // Pops ALL figure boxes regardless of position, cropout creates ONE- // consolidated image covering the entire DLA figure region, and- // insert_table_figures re-inserts ONE figure block.- // Expected: 1 figure section with combined text + cropped image.-- // ── Go current behavior ──- // Figure boxes stay in list. NaiveVerticalMerge may NOT merge them- // if the gap is too large (> 1.5 × median_height ≈ 15pt).- // Each figure text box → separate section in result.Sections.- // pdf.CollectFigures collects them into result.Figures() but doesn't re-insert.-- var figureSections []pdf.Section- for _, s := range result.Sections {- if s.LayoutType == "figure" {- figureSections = append(figureSections, s)- }- }-- // Assert 1: Python expects exactly 1 consolidated figure section.- // Go currently produces 2 (one per unmerged text box) — this FAILS.- if len(figureSections) != 1 {- t.Errorf("FIGURE INSERTION BUG: expected 1 consolidated figure section (Python insert_table_figures), got %d. Go does not consolidate figure text boxes into a single block.", len(figureSections))- }-- // Assert 2: The single figure section must contain BOTH text fragments.- if len(figureSections) == 1 {- combined := figureSections[0].Text- if !strings.Contains(combined, "架构图") || !strings.Contains(combined, "系统模块") {- t.Errorf("FIGURE INSERTION BUG: figure section text=%q should contain both fragments. Python merges all figure-region text.", combined)- }- }-- t.Logf("figure sections in Sections: %d", len(figureSections))- t.Logf("result.Figures() count: %d", len(result.Figures()))- t.Logf("result.Sections total: %d", len(result.Sections))- for i, s := range result.Sections {- t.Logf(" section[%d] layout=%q text=%q", i, s.LayoutType, s.Text)- }-}--// =============================================================================-// Issue 3: Multi-page table merging-// Python's _extract_table_figure merges tables with same layoutno across-// consecutive pages (gap ≤ 1 page, Y-dis ≤ 23× median height).-// Go's extractTableAndReplace does NOT merge tables across pages.-// =============================================================================--// TestExtractTableAndReplace_NoCrossPageMerge exposes that extractTableAndReplace-// does not merge tables from consecutive pages even with the same layoutno.-func TestExtractTableAndReplace_NoCrossPageMerge(t *testing.T) {- // Simulate a table spanning pages 0 and 1.- // Python would merge these because: same layoutno, consecutive pages,- // Y-distance ≤ 23× median_height.- boxes := []pdf.TextBox{- {X0: 10, X1: 200, Top: 500, Bottom: 530, Text: "续表内容", LayoutType: "table", PageNumber: 0, LayoutNo: "0"},- {X0: 10, X1: 200, Top: 50, Bottom: 80, Text: "表尾内容", LayoutType: "table", PageNumber: 1, LayoutNo: "0"},- }-- // Two separate TableItems — one per page. Python would merge these- // before insert_table_figures.- tables := []pdf.TableItem{- {- Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "Page0", Label: "table row"}},- Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 0, Right: 300, Top: 500, Bottom: 530}},- Scale: 1.0,- },- {- Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "Page1", Label: "table row"}},- Positions: []pdf.Position{{PageNumbers: []int{1}, Left: 0, Right: 300, Top: 50, Bottom: 80}},- Scale: 1.0,- },- }-- result := ExtractTableAndReplace(boxes, tables)-- // Go produces 2 separate HTML table boxes (one per page).- // Python would produce 1 merged table with cells from both pages.- tableCount := 0- for _, b := range result {- if strings.Contains(b.Text, "<table>") {- tableCount++- }- }- if tableCount == 2 {- t.Errorf("CROSS-PAGE TABLE MERGE BUG: got %d separate HTML tables across pages. Python would merge same-layoutno tables on consecutive pages into 1 consolidated table.", tableCount)- }- t.Logf("table HTML boxes: %d (Python would merge into 1)", tableCount)-}--// =============================================================================-// Issue 3a: nomerge_lout_no — don't merge tables separated by captions-// Python's _extract_table_figure tracks nomerge_lout_no: when a table box-// is followed by a caption/title/reference, the table's key is added to-// nomerge_lout_no. Later, cross-page merge skips tables in nomerge_lout_no.-//-// Example:-// Page 0: table "0-table-3" → caption "表1:..." → table "0-table-4"-// Page 1: table "1-table-3" (same layoutNo)-// → Page 0's table-3 should NOT merge with Page 1's table-3,-// because the caption on page 0 indicates the table ended.-// → Go's mergeTablesAcrossPages has no nomerge_lout_no check.-// =============================================================================--// TestMergeTablesAcrossPages_NomergeAfterCaption_Missing exposes that-// mergeTablesAcrossPages unconditionally merges consecutive-page tables,-// even when Python's nomerge_lout_no would prevent it.-func TestMergeTablesAcrossPages_NomergeAfterCaption_Missing(t *testing.T) {- // Simulate: page 0 has table at top, followed by a caption,- // then another table. Page 1 has the same-layoutNo table continuing.- // In Python, page 0's first table goes into nomerge_lout_no because- // the next box is a caption → no cross-page merge for that table group.- tables := []pdf.TableItem{- {- Cells: []pdf.TSRCell{{Text: "Page0-first", Label: "table row"}},- Positions: []pdf.Position{{- PageNumbers: []int{0},- Left: 0, Right: 300,- Top: 0, Bottom: 50,- }},- NoMerge: true, // Set when caption follows this table on the page- },- {- Cells: []pdf.TSRCell{{Text: "Page1-cont", Label: "table row"}},- Positions: []pdf.Position{{- PageNumbers: []int{1},- Left: 0, Right: 300,- Top: 0, Bottom: 50,- }},- },- }-- result := MergeTablesAcrossPages(tables, nil)-- // Verify NoMerge prevents cross-page merging.- if len(result) != 2 {- t.Errorf("NOMERGE BUG: expected 2 separate table groups, got %d.", len(result))- }- t.Log("NoMerge flag correctly prevents cross-page merge.")-}--// =============================================================================-// Issue 3b: insert position — min_rectangle_distance vs anchor-// Python's insert_table_figures uses min_rectangle_distance to find the-// spatially nearest text box and inserts the table/figure next to it.-// Go's extractTableAndReplace uses the first replaced table box index as-// the anchor (insert position).-//-// When the DLA table region extends beyond the anchor box's bottom and-// overlaps a text box below the table, Python puts the table next to that-// overlapping text box (distance=0); Go puts it at the anchor position.-// =============================================================================--// TestExtractTableAndReplace_InsertionPosition_DistanceBug exposes that-// extractTableAndReplace uses the first table box as anchor, rather than-// finding the spatially nearest text box like Python.-func TestExtractTableAndReplace_InsertionPosition_DistanceBug(t *testing.T) {- // Two text boxes above the table: L0 (left, near table) and R0 (right, far).- // Python: nearest to table is L0 (dx=0, dy=70). L0 bottom=30 < table top=100- // → insert AFTER L0. Result: [L0, table, R0, R1, L2].- // Go: anchor = first table box (L1 at index 2). Result: [L0, R0, table, R1, L2].- // The table is one position off.- boxes := []pdf.TextBox{- {X0: 10, X1: 100, Top: 10, Bottom: 30, Text: "L0", LayoutType: "text", PageNumber: 0},- {X0: 300, X1: 400, Top: 10, Bottom: 30, Text: "R0", LayoutType: "text", PageNumber: 0},- {X0: 10, X1: 100, Top: 100, Bottom: 130, Text: "table", LayoutType: "table", PageNumber: 0},- {X0: 300, X1: 400, Top: 100, Bottom: 130, Text: "R1", LayoutType: "text", PageNumber: 0},- {X0: 10, X1: 100, Top: 250, Bottom: 270, Text: "L2", LayoutType: "text", PageNumber: 0},- }-- tables := []pdf.TableItem{{- Cells: []pdf.TSRCell{{Text: "cell", Label: "table row"}},- Positions: []pdf.Position{{Left: 10, Right: 100, Top: 100, Bottom: 130, PageNumbers: []int{0}}},- Scale: 1.0,- RegionLeft: 10, RegionRight: 100, RegionTop: 100, RegionBottom: 130,- }}-- result := ExtractTableAndReplace(boxes, tables)-- // Find L0 and table positions.- l0Idx, tableIdx := -1, -1- for i, b := range result {- if strings.TrimSpace(b.Text) == "L0" {- l0Idx = i- }- if b.LayoutType == "table" {- tableIdx = i- }- }-- // BUG: table should immediately follow L0 (nearest neighbor, insert_after).- // Python: min_rectangle_distance → L0 nearest (dx=0, dy=70), L0 below table- // → insert_at+1 → table right after L0.- // Go: anchor = first table box index → table at original table box position.- if tableIdx != l0Idx+1 {- t.Errorf("INSERTION POSITION BUG: table (idx=%d) should immediately follow L0 (idx=%d). "+- "Python's min_rectangle_distance finds L0 as nearest text box and inserts table after it. "+- "Go anchors at first table box position (between R0 and R1).", tableIdx, l0Idx)- }- t.Logf("L0 at idx=%d, table at idx=%d", l0Idx, tableIdx)- t.Log("Fix: replace first-replaced-box anchor with min_rectangle_distance nearest-neighbor (Python pdf_parser.py:1608-1655).")-}--// =============================================================================-// Issue 4: page_cum_height coordinate system-// Python tracks cumulative page image heights for cross-page position tags-// and image cropping. Go uses per-page coordinates only.-// =============================================================================--// TestBoxesToSections_PerPageCoordinates confirms position tags use-// page-relative coordinates. Python's _line_tag also produces local-// coordinates (subtracts page_cum_height). The page number differentiates-// pages; page_cum_height is an internal implementation detail.-func TestBoxesToSections_PerPageCoordinates(t *testing.T) {- boxes := []pdf.TextBox{- {X0: 10, X1: 100, Top: 40, Bottom: 60, Text: "Page 0 text", LayoutType: "text", PageNumber: 0},- {X0: 10, X1: 100, Top: 40, Bottom: 60, Text: "Page 1 text", LayoutType: "text", PageNumber: 1},- }- sections := boxesToSections(boxes, nil)- if len(sections) != 2 {- t.Fatalf("expected 2 sections, got %d", len(sections))- }- s0, s1 := sections[0], sections[1]- if len(s0.Positions) > 0 && len(s1.Positions) > 0 {- p0, p1 := s0.Positions[0], s1.Positions[0]- // Both Python and Go use local (page-relative) coordinates.- // Python's _line_tag: top = bx["top"] - page_cum_height[pn-1]- // gives local coordinate. Same as Go.- if p0.Top != p1.Top || p0.Bottom != p1.Bottom {- t.Errorf("expected same local coords, got Top=(%.0f,%.0f) Bottom=(%.0f,%.0f)", p0.Top, p1.Top, p0.Bottom, p1.Bottom)- }- t.Logf("page 0: Page=%v Top=%.0f Bottom=%.0f", p0.PageNumbers, p0.Top, p0.Bottom)- t.Logf("page 1: Page=%v Top=%.0f Bottom=%.0f", p1.PageNumbers, p1.Top, p1.Bottom)- t.Log("OK: position tags use page-relative coordinates in both Go and Python.")- }-}--// =============================================================================-// Issue 6: cropSectionImage padding logic-// Python's self.crop adds 120px context above first segment, 120px context-// below last segment, 6px gap between pages, and overlay transparency.-// Go has simpler crop logic.-// =============================================================================--// TestCropSectionImage_PaddingVsPython documents that Go's cropSectionImage-// adds context padding differently from Python's self.crop.-func TestCropSectionImage_PaddingVsPython(t *testing.T) {- // Create a page image and position tag for a small text region.- img := image.NewRGBA(image.Rect(0, 0, 300, 800)) // 300×800 page at zoom=3 → PDF 100×267- pageImages := map[int]image.Image{0: img}-- // pdf.Position tag for a small text box near the top of the page.- posTag := FormatPositionTag(0, 50.0, 100.0, 10.0, 30.0)-- result := util.CropSectionImage(posTag, pageImages, 3.0)-- if result == "" {- t.Error("cropSectionImage returned empty string for valid position")- }- // Decode result to check image dimensions.- data, err := base64.StdEncoding.DecodeString(result)- if err != nil {- t.Fatalf("failed to decode base64: %v", err)- }- cropped, _, err := image.Decode(bytes.NewReader(data))- if err != nil {- t.Fatalf("failed to decode PNG: %v", err)- }- croppedH := cropped.Bounds().Dy()- // Original text region: Top=10, Bottom=30 → height=20 at PDF points.- // zoom=3 → 60px text height.- // Python adds 120px context above + 120px below + 6px gap → ~306px.- // Go adds contextPad=120 points above/below at PDF scale → with zoom=3: 360+60+360=780px.- // Python uses pixel-space padding (120px literally), Go uses PDF-point padding (120pt).- expectedMin := 60 // bare minimum: text region itself- if croppedH <= expectedMin {- t.Errorf("CROP PADDING BUG: cropped image height=%dpx, expected >%dpx with context padding. Python adds 120px above and below for context.", croppedH, expectedMin)- }- t.Logf("cropped image: %dx%d (text region 60px, expecting padding)", cropped.Bounds().Dx(), croppedH)- t.Log("NOTE: Python's self.crop adds 120px context padding in pixel space, multi-page stitching, and overlay transparency. Go's cropSectionImage uses PDF-point padding and simpler stitching.")-}--// =============================================================================-// Issue 7: Data-source filter missing-// Python's _extract_table_figure pops table/figure boxes matching-// r"(数据|资料|图表)*来源[:: ]" (pdf_parser.py:1040-1042, 1050-1052).-// These boxes are discarded — not extracted, not inserted back.-// Go has no equivalent filter in extractTableAndReplace or consolidateFigures.-// =============================================================================--// dataSourcePattern is a Go translation of Python's-// r"(数据|资料|图表)*来源[:: ]" used with re.match (anchored at start).-var dataSourcePattern = `^(数据|资料|图表)*来源[:: ]`--// TestDataSourcePattern_RegexCoverage validates the Python regex behavior-// that should be adopted. Documents which strings match and which don't.-func TestDataSourcePattern_RegexCoverage(t *testing.T) {- tests := []struct {- text string- want bool // Python re.match truthiness- }{- // ── Matching patterns (should be filtered) ──- {"数据来源:国家统计局", true}, // 数据 + 来源 + fullwidth colon- {"资料来源: 某报告", true}, // 资料 + 来源 + halfwidth colon- {"图表来源:某数据库", true}, // 图表 + 来源 + fullwidth colon- {"来源:权威机构", true}, // zero prefix + 来源 + fullwidth colon- {"来源: 参考数据", true}, // zero prefix + 来源 + halfwidth colon- {"数据来源 说明", true}, // 数据 + 来源 + space-- // ── Non-matching patterns (should NOT be filtered) ──- {"数据来源明细", false}, // 来源 followed by 明, not ::space- {"普通来源说明", false}, // doesn't start with keyword- {"数据", false}, // too short- {"来源", false}, // 来源 but no ::space after- {"资料来源说明", false}, // 来源 followed by 说, not ::space- {"", false}, // empty- {"TABLE 1: 数据来源统计", false}, // doesn't start with keyword- }-- for _, tt := range tests {- matched := regexp.MustCompile(dataSourcePattern).MatchString(tt.text)- if matched != tt.want {- t.Errorf("dataSourcePattern.MatchString(%q) = %v, want %v", tt.text, matched, tt.want)- }- }- t.Log("NOTE: Python re.match(r\"(数据|资料|图表)*来源[:: ]\", text) — anchored at start.")- t.Log("Go regexp.MatchString equivalent with ^ prefix.")-}--// TestExtractTableAndReplace_DataSourceFilter_Missing exposes that Go does NOT-// filter out table boxes whose text matches r"(数据|资料|图表)*来源[:: ]".-// Python's _extract_table_figure pops these boxes from self.boxes without-// adding them to the tables dict (pdf_parser.py:1040-1042).-func TestExtractTableAndReplace_DataSourceFilter_Missing(t *testing.T) {- // A table box with data-source text and a normal table box.- // Both overlap a pdf.TableItem position, so both would be replaced with HTML.- boxes := []pdf.TextBox{- {X0: 10, X1: 200, Top: 0, Bottom: 50, Text: "数据来源:国家统计局", LayoutType: "table", PageNumber: 0},- {X0: 10, X1: 200, Top: 60, Bottom: 80, Text: "表1:正常数据", LayoutType: "table", PageNumber: 0},- }-- // Two TableItems — one per table box — so each would independently produce HTML.- tables := []pdf.TableItem{- {- Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "来源", Label: "table row"}},- Positions: []pdf.Position{{Left: 0, Right: 300, Top: 0, Bottom: 50}},- Scale: 1.0,- },- {- Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "正常", Label: "table row"}},- Positions: []pdf.Position{{Left: 0, Right: 300, Top: 60, Bottom: 80}},- Scale: 1.0,- },- }-- result := ExtractTableAndReplace(boxes, tables)-- // Python behavior: "数据来源:国家统计局" is popped from self.boxes,- // NOT added to tables dict, NOT replaced with HTML. Gone entirely.- // "表1:正常数据" is replaced with HTML as usual.- // Expected result: exactly 1 HTML table box for the normal table.- //- // BUG: Go replaces both boxes with HTML tables. The data-source box- // produces an HTML table with cell text "来源" — this should NOT exist.- htmlTableCount := 0- hasDataSourceTable := false- for _, b := range result {- if strings.Contains(b.Text, "<table>") {- htmlTableCount++- // The data-source table's cell text "来源" ends up in the HTML.- // c.f. constructTable which uses pdf.TSRCell text, not box text.- if strings.Contains(b.Text, ">来源<") {- hasDataSourceTable = true- }- }- }- if htmlTableCount != 1 {- t.Errorf("DATA SOURCE FILTER BUG: expected 1 HTML table (normal only), got %d. Python pops data-source table box entirely in _extract_table_figure (pdf_parser.py:1040-1042). Go replaces it with an HTML table.", htmlTableCount)- }- if hasDataSourceTable {- t.Errorf("DATA SOURCE FILTER BUG: data-source table should NOT produce HTML output. Cell '来源' appears in HTML: Python discards these boxes, Go incorrectly constructs a table for them.")- }-- t.Log("NOTE: Python filters table boxes matching r\"(数据|资料|图表)*来源[:: ]\" in _extract_table_figure.")- t.Log("Go's extractTableAndReplace has no equivalent filter — data-source boxes get replaced with HTML instead of being discarded.")-}--// TestExtractTableAndReplace_DataSourceVariants tests multiple variants of-// the data-source pattern that should all be filtered.-func TestExtractTableAndReplace_DataSourceVariants(t *testing.T) {- variants := []string{- "数据来源:国家统计局",- "资料来源: 某报告",- "图表来源:某数据库",- "来源:权威机构",- "来源: 参考数据",- }-- for _, variant := range variants {- t.Run(variant, func(t *testing.T) {- boxes := []pdf.TextBox{- {X0: 10, X1: 200, Top: 0, Bottom: 50, Text: variant, LayoutType: "table", PageNumber: 0},- }-- tables := []pdf.TableItem{{- Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "A", Label: "table row"}},- Positions: []pdf.Position{{Left: 0, Right: 300, Top: 0, Bottom: 50}},- Scale: 1.0,- }}-- result := ExtractTableAndReplace(boxes, tables)-- // BUG: box with data-source text should be REMOVED entirely —- // zero HTML output. Python pops these boxes without replacement.- for _, b := range result {- if strings.Contains(b.Text, "<table>") {- t.Errorf("DATA SOURCE FILTER BUG: variant %q should be removed without HTML replacement. Python pops data-source table boxes entirely.", variant)- }- }- })- }- t.Log("NOTE: All variants of r\"(数据|资料|图表)*来源[:: ]\" should be filtered by extractTableAndReplace.")-}--// TestConsolidateFigures_DataSourceFilter_Missing exposes that Go does NOT-// filter out figure boxes whose text matches r"(数据|资料|图表)*来源[:: ]".-// Python's _extract_table_figure pops these boxes from self.boxes without-// adding them to the figures dict (pdf_parser.py:1050-1052).-func TestConsolidateFigures_DataSourceFilter_Missing(t *testing.T) {- boxes := []pdf.TextBox{- {X0: 10, X1: 200, Top: 0, Bottom: 50, Text: "数据来源:某机构", LayoutType: "figure", PageNumber: 0, LayoutNo: "figure-0"},- {X0: 10, X1: 200, Top: 60, Bottom: 80, Text: "架构图", LayoutType: "figure", PageNumber: 0, LayoutNo: "figure-0"},- }-- result := ConsolidateFigures(boxes)-- // Python behavior: "数据来源:某机构" is popped from self.boxes,- // NOT added to figures dict → gone entirely.- // "架构图" is extracted normally.- // Expected result: exactly 1 figure box with "架构图" text only.- for _, b := range result {- if strings.Contains(b.Text, "数据来源") || strings.Contains(b.Text, "某机构") {- t.Errorf("DATA SOURCE FIGURE FILTER BUG: '数据来源:某机构' figure box should be removed entirely. Python pops data-source figure boxes in _extract_table_figure (pdf_parser.py:1050-1052). Go still includes it.")- }- }-- // Verify the normal figure box IS still present.- foundFigure := false- for _, b := range result {- if strings.Contains(b.Text, "架构图") {- foundFigure = true- }- }- if !foundFigure {- t.Error("normal figure box '架构图' should still be present")- }-- t.Log("NOTE: Python filters figure boxes matching r\"(数据|资料|图表)*来源[:: ]\" in _extract_table_figure.")- t.Log("Go's consolidateFigures has no equivalent filter.")-}