Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Remove RowsToHTML; render PDF tables via canonical RenderTableHTML (#21083)

xugangqiang5cb4abb40a45
diff --git a/internal/deepdoc/parser/pdf/table/table_construct.go b/internal/deepdoc/parser/pdf/table/table_construct.go
index 78626effa1..6400a2157e 100644
--- a/internal/deepdoc/parser/pdf/table/table_construct.go
+++ b/internal/deepdoc/parser/pdf/table/table_construct.go
@@ -7,6 +7,7 @@ import (
"strings"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
+ "ragflow/internal/parser/tableutil"
)
// constructTable produces an HTML table string from TSR cells and text boxes.
@@ -82,7 +83,7 @@ func ConstructTable(cells []pdf.TSRCell, boxes []pdf.TextBox, caption string, it
if item != nil {
MarkCoveredCells(item.Grid, covered)
}
- return RowsToHTML(rows, caption, hdrs, spanInfo, covered)
+ return tableutil.RenderTableHTML(ToTableData(rows, caption, hdrs, spanInfo, covered))
}
// Fallback: boxes with R/C annotations.
if len(boxes) > 0 && BoxesHaveAnnotations(boxes) {
@@ -92,7 +93,7 @@ func ConstructTable(cells []pdf.TSRCell, boxes []pdf.TextBox, caption string, it
item.Rows = RowsToStrings(rows)
}
spanInfo, covered := CalSpans(rows)
- return RowsToHTML(rows, caption, BoxHeaderSet(rows, boxes), spanInfo, covered)
+ return tableutil.RenderTableHTML(ToTableData(rows, caption, BoxHeaderSet(rows, boxes), spanInfo, covered))
}
}
// Test-only: Y/X coordinate grouping (matching Python construct_table).
@@ -104,7 +105,7 @@ func ConstructTable(cells []pdf.TSRCell, boxes []pdf.TextBox, caption string, it
item.Rows = RowsToStrings(rows)
}
spanInfo, covered := CalSpans(rows)
- return RowsToHTML(rows, caption, BoxHeaderSet(rows, boxes), spanInfo, covered)
+ return tableutil.RenderTableHTML(ToTableData(rows, caption, BoxHeaderSet(rows, boxes), spanInfo, covered))
}
}
return ""
diff --git a/internal/deepdoc/parser/pdf/table/table_construct_test.go b/internal/deepdoc/parser/pdf/table/table_construct_test.go
index 8dcb82dfc0..0d84dac366 100644
--- a/internal/deepdoc/parser/pdf/table/table_construct_test.go
+++ b/internal/deepdoc/parser/pdf/table/table_construct_test.go
@@ -5,8 +5,32 @@ import (
"testing"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/tableutil"
)
+// cellTextAt returns the cell text at (r,c), or "" when out of range.
+func cellTextAt(td *entity.TableData, r, c int) string {
+ if td == nil || r < 0 || r >= len(td.Grid) || c < 0 || c >= len(td.Grid[r]) {
+ return ""
+ }
+ return td.Grid[r][c].Text
+}
+
+// constructTableData runs ConstructTable and parses its canonical HTML back to
+// TableData — exactly what the production pipeline (projectPDFTable) does — so
+// the assembly can be asserted on the structured contract rather than on HTML
+// serialization details.
+func constructTableData(t *testing.T, cells []pdf.TSRCell, boxes []pdf.TextBox, caption string, item *pdf.TableItem) *entity.TableData {
+ t.Helper()
+ html := ConstructTable(cells, boxes, caption, item)
+ td, err := tableutil.ParseTableHTML(html)
+ if err != nil {
+ t.Fatalf("ParseTableHTML(%q): %v", html, err)
+ }
+ return td
+}
+
func TestCellTexts(t *testing.T) {
cells := []pdf.TSRCell{
{Text: "A"}, {Text: "B"}, {Text: "C"},
@@ -29,23 +53,27 @@ func TestConstructTable_Simple3x2(t *testing.T) {
{X0: 201, Y0: 51, X1: 300, Y1: 100, Text: "F", Label: "table row"},
}
boxes := []pdf.TextBox{}
- html := ConstructTable(cells, boxes, "", nil)
- if !strings.Contains(html, "<table>") {
- t.Error("expected <table> tag")
- }
- if !strings.Contains(html, "A") || !strings.Contains(html, "B") || !strings.Contains(html, "C") {
- t.Error("expected cell texts A, B, C in HTML")
- }
- // Should have 2 <tr> elements
- trCount := strings.Count(html, "<tr>")
- if trCount != 2 {
- t.Errorf("expected 2 <tr> rows, got %d", trCount)
- }
- tdCount := strings.Count(html, "<td ")
- if tdCount != 6 {
- t.Errorf("expected 6 <td > cells, got %d", tdCount)
+ td := constructTableData(t, cells, boxes, "", nil)
+ if td == nil {
+ t.Fatal("expected non-nil TableData")
+ }
+ if len(td.Grid) != 2 {
+ t.Fatalf("rows=%d want 2", len(td.Grid))
+ }
+ // No cell carries a header block type, so the canonical render/parse round
+ // trip must still default HeaderRows to 1 (leading row assumed header) —
+ // the same default the removed RowsToHTML path produced.
+ if td.HeaderRows != 1 {
+ t.Errorf("HeaderRows=%d want 1 (default when no header detected)", td.HeaderRows)
+ }
+ for _, w := range []struct {
+ r, c int
+ s string
+ }{{0, 0, "A"}, {0, 1, "B"}, {0, 2, "C"}, {1, 0, "D"}, {1, 1, "E"}, {1, 2, "F"}} {
+ if got := cellTextAt(td, w.r, w.c); got != w.s {
+ t.Errorf("cell[%d][%d]=%q want %q", w.r, w.c, got, w.s)
+ }
}
- t.Logf("HTML:\n%s", html)
}
// TestConstructTable_DropsAllEmptyRow pins Python parity: when the
@@ -91,30 +119,33 @@ func TestConstructTable_DropsAllEmptyRow(t *testing.T) {
}
func TestConstructTable_EmptyCells(t *testing.T) {
- html := ConstructTable(nil, nil, "", nil)
- if html != "" {
+ if html := ConstructTable(nil, nil, "", nil); html != "" {
t.Errorf("expected empty string for empty cells, got %q", html)
}
- html = ConstructTable([]pdf.TSRCell{}, []pdf.TextBox{}, "", nil)
- if html != "" {
+ if html := ConstructTable([]pdf.TSRCell{}, []pdf.TextBox{}, "", nil); html != "" {
t.Errorf("expected empty string for empty cells slice, got %q", html)
}
}
func TestConstructTable_NoMatchingBox(t *testing.T) {
- // Cell has no overlapping text box → empty <td >
+ // Cell has no overlapping text box → empty cell
cells := []pdf.TSRCell{
{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "Has text", Label: "table row"},
{X0: 101, Y0: 0, X1: 200, Y1: 50, Label: "table row"},
}
boxes := []pdf.TextBox{}
- html := ConstructTable(cells, boxes, "", nil)
- if !strings.Contains(html, "Has text") {
- t.Error("expected first cell text")
+ td := constructTableData(t, cells, boxes, "", nil)
+ if td == nil {
+ t.Fatal("expected non-nil TableData")
+ }
+ if len(td.Grid) != 1 || len(td.Grid[0]) != 2 {
+ t.Fatalf("expected 1x2 grid, got %v", td.Grid)
}
- // Should still have 2 <td > cells
- if strings.Count(html, "<td ") != 2 {
- t.Errorf("expected 2 <td > cells, got %d. HTML:\n%s", strings.Count(html, "<td "), html)
+ if cellTextAt(td, 0, 0) != "Has text" {
+ t.Errorf("expected first cell text 'Has text', got %q", cellTextAt(td, 0, 0))
+ }
+ if cellTextAt(td, 0, 1) != "" {
+ t.Errorf("expected empty second cell, got %q", cellTextAt(td, 0, 1))
}
}
@@ -122,11 +153,13 @@ func TestConstructTable_WithCaption(t *testing.T) {
cells := []pdf.TSRCell{
{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "X", Label: "table row"},
}
- html := ConstructTable(cells, nil, "表1:测试标题", nil)
- if !strings.Contains(html, "<caption>表1:测试标题</caption>") {
- t.Errorf("expected caption, got:\n%s", html)
+ td := constructTableData(t, cells, nil, "表1:测试标题", nil)
+ if td == nil {
+ t.Fatal("expected non-nil TableData")
+ }
+ if td.Caption != "表1:测试标题" {
+ t.Errorf("expected caption, got %q", td.Caption)
}
- t.Logf("HTML:\n%s", html)
}
func TestConstructTable_SingleRow(t *testing.T) {
@@ -134,12 +167,15 @@ func TestConstructTable_SingleRow(t *testing.T) {
{X0: 0, Y0: 0, X1: 50, Y1: 40, Text: "Col1", Label: "table row"},
{X0: 51, Y0: 0, X1: 100, Y1: 40, Text: "Col2", Label: "table row"},
}
- html := ConstructTable(cells, nil, "", nil)
- if strings.Count(html, "<tr>") != 1 {
- t.Errorf("expected 1 row, got %d", strings.Count(html, "<tr>"))
+ td := constructTableData(t, cells, nil, "", nil)
+ if td == nil {
+ t.Fatal("expected non-nil TableData")
}
- if strings.Count(html, "<td ") != 2 {
- t.Errorf("expected 2 cells, got %d", strings.Count(html, "<td "))
+ if len(td.Grid) != 1 {
+ t.Errorf("expected 1 row, got %d", len(td.Grid))
+ }
+ if len(td.Grid[0]) != 2 {
+ t.Errorf("expected 2 cells, got %d", len(td.Grid[0]))
}
}
@@ -172,12 +208,16 @@ func TestConstructTable_YBasedFallback(t *testing.T) {
{X0: 51, Y0: 0, X1: 100, Y1: 30, Text: "R1C2", Label: "table"},
{X0: 0, Y0: 31, X1: 50, Y1: 60, Text: "R2C1", Label: "table"},
}
- html := ConstructTable(cells, nil, "", nil)
- if strings.Count(html, "<tr>") != 2 {
- t.Errorf("expected 2 rows from Y-fallback, got %d", strings.Count(html, "<tr>"))
+ td := constructTableData(t, cells, nil, "", nil)
+ if td == nil {
+ t.Fatal("expected non-nil TableData")
+ }
+ if len(td.Grid) != 2 {
+ t.Fatalf("expected 2 rows from Y-fallback, got %d", len(td.Grid))
}
- if strings.Count(html, "<td ") != 3 { // 2 in row0, 1 in row1 (no padding in basic grouping)
- t.Errorf("expected 3 cells, got %d", strings.Count(html, "<td "))
+ // 2 cells in row0, 1 cell in row1 (no padding in basic grouping).
+ if len(td.Grid[0]) != 2 || len(td.Grid[1]) != 1 {
+ t.Errorf("expected 2+1 cells, got %d+%d", len(td.Grid[0]), len(td.Grid[1]))
}
}
@@ -259,26 +299,25 @@ func TestConstructTable_FromBoxesRC(t *testing.T) {
// constructTable should build HTML directly from boxes by R/C grouping,
// ignoring cell text (matching Python's construct_table).
item := &pdf.TableItem{}
- html := ConstructTable(nil, boxes, "", item)
+ td := constructTableData(t, nil, boxes, "", item)
- if !strings.Contains(html, "姓名") || !strings.Contains(html, "张三") {
- t.Errorf("HTML missing box text: %s", html)
+ if cellTextAt(td, 0, 0) != "姓名" || cellTextAt(td, 1, 0) != "张三" {
+ t.Errorf("TableData missing box text: 姓名/张三, got %v", td)
}
// 2 rows, 2 cols
- if strings.Count(html, "<tr>") != 2 {
- t.Errorf("expected 2 rows, got %d. HTML: %s", strings.Count(html, "<tr>"), html)
+ if len(td.Grid) != 2 {
+ t.Errorf("expected 2 rows, got %d", len(td.Grid))
}
- if strings.Count(html, "<td ") != 4 {
- t.Errorf("expected 4 cells, got %d. HTML: %s", strings.Count(html, "<td "), html)
+ if len(td.Grid[0]) != 2 {
+ t.Errorf("expected 2 cols, got %d", len(td.Grid[0]))
}
- // Verify Rows output
+ // Verify Rows output (side effect)
if len(item.Rows) != 2 || len(item.Rows[0]) != 2 {
t.Errorf("Rows: expected 2x2, got %dx%d", len(item.Rows), len(item.Rows[0]))
}
if item.Rows[0][0] != "姓名" {
t.Errorf("Rows[0][0] = %q, want %q", item.Rows[0][0], "姓名")
}
- t.Logf("HTML: %s", html)
}
func TestFillCellTextFromBoxes_RCAnnotations(t *testing.T) {
@@ -349,9 +388,9 @@ func TestConstructTable_SingleRowMultiCol(t *testing.T) {
{X0: 201, X1: 300, Top: 0, Bottom: 30, Text: "性别", R: 0, C: 2},
}
item := &pdf.TableItem{}
- html := ConstructTable(nil, boxes, "", item)
- if strings.Count(html, "<td ") != 3 {
- t.Errorf("expected 3 cells, got %d. HTML: %s", strings.Count(html, "<td "), html)
+ td := constructTableData(t, nil, boxes, "", item)
+ if len(td.Grid[0]) != 3 {
+ t.Errorf("expected 3 cells, got %d", len(td.Grid[0]))
}
if item.Rows[0][0] != "姓名" || item.Rows[0][1] != "年龄" || item.Rows[0][2] != "性别" {
t.Errorf("wrong row text: %v", item.Rows[0])
@@ -365,9 +404,9 @@ func TestConstructTable_MultiRowSingleCol(t *testing.T) {
{X0: 0, X1: 100, Top: 70, Bottom: 100, Text: "第三行", R: 2, C: 0},
}
item := &pdf.TableItem{}
- html := ConstructTable(nil, boxes, "", item)
- if strings.Count(html, "<tr>") != 3 {
- t.Errorf("expected 3 rows, got %d. HTML: %s", strings.Count(html, "<tr>"), html)
+ td := constructTableData(t, nil, boxes, "", item)
+ if len(td.Grid) != 3 {
+ t.Errorf("expected 3 rows, got %d", len(td.Grid))
}
if item.Rows[0][0] != "第一行" || item.Rows[1][0] != "第二行" || item.Rows[2][0] != "第三行" {
t.Errorf("wrong text: row0=%q row1=%q row2=%q", item.Rows[0][0], item.Rows[1][0], item.Rows[2][0])
@@ -384,12 +423,12 @@ func TestConstructTable_RCAfterMerge(t *testing.T) {
{X0: 355, X1: 500, Top: 35, Bottom: 65, Text: "经济舱位", R: 1, C: 1},
}
item := &pdf.TableItem{}
- html := ConstructTable(nil, postMerge, "", item)
- if !strings.Contains(html, "公司级领导") {
- t.Errorf("missing merged text: %s", html)
+ td := constructTableData(t, nil, postMerge, "", item)
+ if !strings.Contains(cellTextAt(td, 0, 0), "公司级领导") {
+ t.Errorf("missing merged text: %q", cellTextAt(td, 0, 0))
}
- if strings.Count(html, "<tr>") != 2 {
- t.Errorf("expected 2 rows, got %d", strings.Count(html, "<tr>"))
+ if len(td.Grid) != 2 {
+ t.Errorf("expected 2 rows, got %d", len(td.Grid))
}
if item.Rows[0][0] != "公司级领导人员(含公司董事、总监)" {
t.Errorf("row 0 col 0 = %q", item.Rows[0][0])
@@ -606,16 +645,19 @@ func TestConstructTable_StripsCaptionFromCells(t *testing.T) {
{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "表1:标题"},
{X0: 100, Y0: 0, X1: 200, Y1: 50, Text: "数据"},
}
- html := ConstructTable(cells, nil, "", nil)
- // "表1:标题" should NOT appear in the HTML (stripped as caption).
- if strings.Contains(html, "表1") {
- t.Errorf("caption text '表1:标题' should be stripped: %s", html)
+ td := constructTableData(t, cells, nil, "", nil)
+ // "表1:标题" should NOT survive as a cell (stripped as caption).
+ for r := range td.Grid {
+ for c := range td.Grid[r] {
+ if strings.Contains(td.Grid[r][c].Text, "表1") {
+ t.Errorf("caption text '表1:标题' should be stripped, found in cell[%d][%d]: %q", r, c, td.Grid[r][c].Text)
+ }
+ }
}
// "数据" should still be there.
- if !strings.Contains(html, "数据") {
- t.Errorf("data text '数据' should be preserved: %s", html)
+ if cellTextAt(td, 0, 1) != "数据" {
+ t.Errorf("data text '数据' should be preserved, got %q", cellTextAt(td, 0, 1))
}
- t.Logf("HTML: %s", html)
}
func TestExtractTableAndReplace(t *testing.T) {
diff --git a/internal/deepdoc/parser/pdf/table/table_html.go b/internal/deepdoc/parser/pdf/table/table_html.go
index 62b735d81c..df05a46637 100644
--- a/internal/deepdoc/parser/pdf/table/table_html.go
+++ b/internal/deepdoc/parser/pdf/table/table_html.go
@@ -1,68 +1,12 @@
package table
import (
- "fmt"
"html"
"strings"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
)
-// RowsToHTML renders a table grid to HTML. The output is intentionally
-// compact (no inter-tag whitespace) and the caption is html-escaped. This
-// diverges from Python's __html_table, which emits newlines between rows and
-// leaves the caption unescaped; the difference is HTML serialization only and
-// never affects cell content (gridSim=100% in the parity harness). Registered
-// as go_intentional in table/testdata/parity/known_diffs.json
-// (rule "table-html-emission-format").
-func RowsToHTML(rows [][]pdf.TSRCell, caption string, headerRows map[int]bool, spanInfo map[[2]int][2]int, covered map[[2]int]bool) string {
- var b strings.Builder
- b.WriteString("<table>")
- if caption != "" {
- b.WriteString("<caption>")
- b.WriteString(html.EscapeString(caption))
- b.WriteString("</caption>")
- }
- for ri, row := range rows {
- b.WriteString("<tr>")
- for ci, cell := range row {
- if covered[[2]int{ri, ci}] {
- continue
- }
- tag := "td"
- if headerRows[ri] {
- tag = "th"
- }
- b.WriteString("<")
- b.WriteString(tag)
- sp := ""
- if s, ok := spanInfo[[2]int{ri, ci}]; ok {
- if s[0] > 1 {
- sp = fmt.Sprintf("colspan=%d", s[0])
- }
- if s[1] > 1 {
- if sp != "" {
- sp += " "
- }
- sp += fmt.Sprintf("rowspan=%d", s[1])
- }
- }
- if sp != "" {
- b.WriteString(" ")
- b.WriteString(sp)
- }
- b.WriteString(" >")
- b.WriteString(html.EscapeString(cell.Text))
- b.WriteString("</")
- b.WriteString(tag)
- b.WriteString(">")
- }
- b.WriteString("</tr>")
- }
- b.WriteString("</table>")
- return b.String()
-}
-
// SimpleRowsToHTML converts plain string-based table data to an HTML table.
// The first row is treated as a header (<th>). Used by DOCX, XLSX, PPTX,
// and HTML parsers that produce [][]string directly.
diff --git a/internal/deepdoc/parser/pdf/table/table_html_test.go b/internal/deepdoc/parser/pdf/table/table_html_test.go
index 68f4cf2bec..6df3f12c8b 100644
--- a/internal/deepdoc/parser/pdf/table/table_html_test.go
+++ b/internal/deepdoc/parser/pdf/table/table_html_test.go
@@ -1,101 +1,37 @@
package table
import (
- "strings"
"testing"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
)
-func TestRowsToHTML(t *testing.T) {
- // rowsToHTML takes [][]pdf.TSRCell instead of [][]string (tableToHTML removed).
- toCells := func(rows [][]string) [][]pdf.TSRCell {
- out := make([][]pdf.TSRCell, len(rows))
- for ri, row := range rows {
- out[ri] = make([]pdf.TSRCell, len(row))
- for ci, s := range row {
- out[ri][ci] = pdf.TSRCell{Text: s}
- }
- }
- return out
- }
-
- t.Run("simple 2x2 table", func(t *testing.T) {
- rows := toCells([][]string{
- {"姓名", "年龄"},
- {"张三", "25"},
- })
- html := RowsToHTML(rows, "", nil, nil, nil)
- expected := "<table><tr><td >姓名</td><td >年龄</td></tr><tr><td >张三</td><td >25</td></tr></table>"
- if html != expected {
- t.Errorf("got %q\nwant %q", html, expected)
- }
- })
-
- t.Run("empty table", func(t *testing.T) {
- html := RowsToHTML(nil, "", nil, nil, nil)
- if html != "<table></table>" {
- t.Errorf("expected '<table></table>', got %q", html)
- }
- })
-
- t.Run("single cell", func(t *testing.T) {
- rows := toCells([][]string{{"X"}})
- html := RowsToHTML(rows, "", nil, nil, nil)
- expected := "<table><tr><td >X</td></tr></table>"
- if html != expected {
- t.Errorf("got %q\nwant %q", html, expected)
- }
- })
-
- t.Run("matches Python format for 公司差旅费", func(t *testing.T) {
- rows := toCells([][]string{
- {"标职务", "飞机", "火车", "轮船", "其他交通工具(不含的士)"},
- {"公司级领导人员", "经济舱位", "火车软席", "二等舱位", "按实报销"},
- {"其他工作人员", "经济舱位", "火车硬席", "三等舱位", "按实报销"},
- })
- html := RowsToHTML(rows, "", nil, nil, nil)
- if !strings.HasPrefix(html, "<table>") || !strings.HasSuffix(html, "</table>") {
- t.Errorf("not valid HTML: %s", html)
- }
- if !strings.Contains(html, "<td >标职务</td>") {
- t.Errorf("missing cell '标职务': %s", html)
- }
- if strings.Count(html, "<tr>") != 3 {
- t.Errorf("expected 3 rows, got %d", strings.Count(html, "<tr>"))
- }
- })
-}
-
-func TestRowsToHTML_HeaderRows(t *testing.T) {
+// TestConstructTable_HeaderRowDetected pins that a TSR row whose cells carry
+// the "table column header" block type is counted as the header row
+// (HeaderRows) in the produced TableData.
+func TestConstructTable_HeaderRowDetected(t *testing.T) {
cells := []pdf.TSRCell{
{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "Name", Label: "table column header"},
{X0: 101, Y0: 0, X1: 200, Y1: 30, Text: "Age", Label: "table column header"},
{X0: 0, Y0: 35, X1: 100, Y1: 65, Text: "John", Label: "table row"},
{X0: 101, Y0: 35, X1: 200, Y1: 65, Text: "30", Label: "table row"},
}
- // constructTable should produce <th > for header row.
- item := &pdf.TableItem{}
- html := ConstructTable(cells, nil, "", item)
- // Header row should use <th >, data row <td >.
- if !strings.Contains(html, "<th >") {
- t.Errorf("expected <th > for header row. HTML: %s", html)
+ td := constructTableData(t, cells, nil, "", &pdf.TableItem{})
+ if td == nil {
+ t.Fatal("expected non-nil TableData")
}
- if strings.Count(html, "<th ") != 2 {
- t.Errorf("expected 2 <th > cells, got %d. HTML: %s", strings.Count(html, "<th "), html)
+ if td.HeaderRows != 1 {
+ t.Errorf("expected 1 header row, got %d", td.HeaderRows)
}
- if strings.Count(html, "<td ") != 2 {
- t.Errorf("expected 2 <td > cells (data row), got %d", strings.Count(html, "<td "))
+ if cellTextAt(td, 0, 0) != "Name" || cellTextAt(td, 1, 0) != "John" {
+ t.Errorf("unexpected cell text: %v", td.Grid)
}
}
-func TestRowsToHTML_Colspan(t *testing.T) {
- // Python's __cal_spans only spans boxes carrying an "SP" annotation
- // (TSR "table spanning cell"). Here the SP box at column 1 has an X range
- // (X0=10, X1=190) that covers column 0's center, so CalSpans must fold the
- // covered "Name" cell into a colspan=2. This mirrors the real pipeline
- // where GroupBoxesByRC labels an SP box "table spanning cell"; a plain
- // "table header" must NOT trigger a span (see icbccs cross-page merge).
+// TestToTableData_ColspanFromCalSpans verifies that a span derived from CalSpans
+// (an "SP"-annotated box covering two columns) produces a colspan origin in the
+// produced TableData — the same geometry the removed RowsToHTML emitted.
+func TestToTableData_ColspanFromCalSpans(t *testing.T) {
boxes := []pdf.TextBox{
{X0: 10, X1: 90, Top: 0, Bottom: 30, Text: "Name", R: 0, C: 0},
{X0: 10, X1: 190, Top: 0, Bottom: 30, Text: "", R: 0, C: 1, SP: 1},
@@ -103,10 +39,15 @@ func TestRowsToHTML_Colspan(t *testing.T) {
{X0: 110, X1: 190, Top: 35, Bottom: 65, Text: "30", R: 1, C: 1},
}
rows := GroupBoxesByRC(boxes)
- spans, covered := CalSpans(rows)
- html := RowsToHTML(rows, "", nil, spans, covered)
- if !strings.Contains(html, "colspan") {
- t.Errorf("expected colspan attribute, got: %s", html)
+ spanInfo, covered := CalSpans(rows)
+ td := ToTableData(rows, "", nil, spanInfo, covered)
+ if td == nil {
+ t.Fatal("expected non-nil TableData")
+ }
+ if td.Grid[0][0].ColSpan != 2 {
+ t.Errorf("expected colspan=2 origin, got %d", td.Grid[0][0].ColSpan)
+ }
+ if !td.Grid[0][1].Covered {
+ t.Errorf("expected covered cell at [0][1]")
}
- t.Logf("HTML: %s", html)
}
diff --git a/internal/deepdoc/parser/pdf/table/table_merge.go b/internal/deepdoc/parser/pdf/table/table_merge.go
index 6ce1e7596f..def1bbe9f0 100644
--- a/internal/deepdoc/parser/pdf/table/table_merge.go
+++ b/internal/deepdoc/parser/pdf/table/table_merge.go
@@ -172,7 +172,7 @@ func MergeTablesAcrossPages(tables []pdf.TableItem, medianHeights, pageHeights m
// shorter rows by index. Column i of a continuation page maps to
// column i of the anchor because they are the same logical column of
// one cross-page table, so padding keeps the grid uniform
- // (CalSpans / CleanupOrphanColumns / RowsToHTML never see a jagged
+ // (CalSpans / CleanupOrphanColumns / ToTableData never see a jagged
// grid) while preserving every row.
if len(anchor.Grid) > 0 && len(contGrids) > 0 {
allGrids := append([][][]pdf.TSRCell{anchor.Grid}, contGrids...)
diff --git a/internal/deepdoc/parser/pdf/table/table_todata.go b/internal/deepdoc/parser/pdf/table/table_todata.go
new file mode 100644
index 0000000000..e4d0905f93
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/table/table_todata.go
@@ -0,0 +1,49 @@
+package table
+
+import (
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/tableutil"
+)
+
+// ToTableData builds the canonical entity.TableData directly from a TSR cell
+// grid. ConstructTable pairs it with tableutil.RenderTableHTML so a table is
+// serialized by the single canonical renderer (RenderTableHTML) instead of the
+// former second, compact RowsToHTML emitter. ToTableData reuses
+// tableutil.BuildTableData's normalization (header count, span clamp,
+// ragged-no-pad, trailing-empty trim, Covered reconstruction), so the grid it
+// produces is identical to what the legacy RowsToHTML -> ParseTableHTML round
+// trip produced.
+//
+// Covered cells are skipped here (their engulfed positions are reconstructed as
+// Covered markers from the origin's span), mirroring how the legacy emitter
+// never produced a cell for a covered slot. headerRows may flag non-leading rows
+// as headers; only the leading consecutive run is counted (the same
+// leading-prefix semantics ParseTableHTML applies to <th>).
+func ToTableData(rows [][]pdf.TSRCell, caption string, headerRows map[int]bool, spanInfo map[[2]int][2]int, covered map[[2]int]bool) *entity.TableData {
+ inputs := make([][]tableutil.CellInput, len(rows))
+ headerFlags := make([]bool, len(rows))
+ for ri, row := range rows {
+ headerFlags[ri] = headerRows[ri]
+ for ci, cell := range row {
+ if covered[[2]int{ri, ci}] {
+ continue
+ }
+ cs, rs := 1, 1
+ if s, ok := spanInfo[[2]int{ri, ci}]; ok {
+ if s[0] > 1 {
+ cs = s[0]
+ }
+ if s[1] > 1 {
+ rs = s[1]
+ }
+ }
+ inputs[ri] = append(inputs[ri], tableutil.CellInput{
+ Text: cell.Text,
+ ColSpan: cs,
+ RowSpan: rs,
+ })
+ }
+ }
+ return tableutil.BuildTableData(inputs, headerFlags, caption)
+}
diff --git a/internal/deepdoc/parser/pdf/table/table_todata_test.go b/internal/deepdoc/parser/pdf/table/table_todata_test.go
new file mode 100644
index 0000000000..7e3c45cf08
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/table/table_todata_test.go
@@ -0,0 +1,159 @@
+package table
+
+import (
+ "testing"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// cellsFromStrings builds a [][]pdf.TSRCell grid from string rows for tests.
+func cellsFromStrings(rows [][]string) [][]pdf.TSRCell {
+ out := make([][]pdf.TSRCell, len(rows))
+ for ri, row := range rows {
+ out[ri] = make([]pdf.TSRCell, len(row))
+ for ci, s := range row {
+ out[ri][ci] = pdf.TSRCell{Text: s}
+ }
+ }
+ return out
+}
+
+// TestToTableData_Simple2x2 verifies a plain 2x2 grid with no span/caption
+// produces the canonical Grid and the default single header row.
+func TestToTableData_Simple2x2(t *testing.T) {
+ g := cellsFromStrings([][]string{{"姓名", "年龄"}, {"张三", "25"}})
+ td := ToTableData(g, "", nil, nil, nil)
+ if td == nil {
+ t.Fatal("expected non-nil TableData")
+ }
+ if len(td.Grid) != 2 {
+ t.Fatalf("rows=%d want 2", len(td.Grid))
+ }
+ if len(td.Grid[0]) != 2 {
+ t.Fatalf("cols=%d want 2", len(td.Grid[0]))
+ }
+ if td.Grid[0][0].Text != "姓名" || td.Grid[1][1].Text != "25" {
+ t.Errorf("cell text mismatch: %q/%q", td.Grid[0][0].Text, td.Grid[1][1].Text)
+ }
+ if td.HeaderRows != 1 {
+ t.Errorf("HeaderRows=%d want 1 (default)", td.HeaderRows)
+ }
+ for _, row := range td.Grid {
+ for _, c := range row {
+ if c.ColSpan != 1 || c.RowSpan != 1 || c.Covered {
+ t.Errorf("unexpected span/covered: %+v", c)
+ }
+ }
+ }
+}
+
+// TestToTableData_Empty verifies an empty / nil grid yields a nil Grid rather
+// than a panic or a ragged zero-width slice.
+func TestToTableData_Empty(t *testing.T) {
+ td := ToTableData(nil, "", nil, nil, nil)
+ if td == nil {
+ t.Fatal("expected non-nil TableData wrapper")
+ }
+ if td.Grid != nil {
+ t.Errorf("expected nil Grid, got %v", td.Grid)
+ }
+}
+
+// TestToTableData_Colspan verifies a colspan origin keeps its ColSpan and the
+// engulfed position becomes a Covered marker pointing back at the origin.
+func TestToTableData_Colspan(t *testing.T) {
+ rows := cellsFromStrings([][]string{{"Name", ""}, {"John", "30"}})
+ spanInfo := map[[2]int][2]int{{0, 0}: {2, 1}}
+ covered := map[[2]int]bool{{0, 1}: true}
+ td := ToTableData(rows, "", nil, spanInfo, covered)
+ if len(td.Grid) != 2 {
+ t.Fatalf("rows=%d want 2", len(td.Grid))
+ }
+ if td.Grid[0][0].ColSpan != 2 {
+ t.Errorf("origin ColSpan=%d want 2", td.Grid[0][0].ColSpan)
+ }
+ if !td.Grid[0][1].Covered {
+ t.Errorf("expected Covered at [0][1]")
+ }
+ if td.Grid[0][1].OriginRow != 0 || td.Grid[0][1].OriginCol != 0 {
+ t.Errorf("origin back-pointer wrong: %+v", td.Grid[0][1])
+ }
+}
+
+// TestToTableData_Rowspan verifies a rowspan origin keeps its RowSpan and the
+// continuation row stays (not trimmed) as a Covered marker.
+func TestToTableData_Rowspan(t *testing.T) {
+ rows := cellsFromStrings([][]string{{"H", "x"}, {"", "y"}})
+ spanInfo := map[[2]int][2]int{{0, 0}: {1, 2}}
+ covered := map[[2]int]bool{{1, 0}: true}
+ td := ToTableData(rows, "", nil, spanInfo, covered)
+ if td.Grid[0][0].RowSpan != 2 {
+ t.Errorf("RowSpan=%d want 2", td.Grid[0][0].RowSpan)
+ }
+ if !td.Grid[1][0].Covered {
+ t.Errorf("expected Covered continuation at [1][0]")
+ }
+ if td.Grid[1][0].OriginRow != 0 || td.Grid[1][0].OriginCol != 0 {
+ t.Errorf("origin back-pointer wrong: %+v", td.Grid[1][0])
+ }
+}
+
+// TestToTableData_RaggedNoPad verifies a short row keeps its actual (ragged)
+// length and is NOT padded to the widest row — matching the legacy Rows text
+// view produced by ParseTableHTML.
+func TestToTableData_RaggedNoPad(t *testing.T) {
+ rows := cellsFromStrings([][]string{{"a", "b", "c"}, {"d"}})
+ td := ToTableData(rows, "", nil, nil, nil)
+ if len(td.Grid[0]) != 3 {
+ t.Errorf("row0 cols=%d want 3", len(td.Grid[0]))
+ }
+ if len(td.Grid[1]) != 1 {
+ t.Errorf("row1 cols=%d want 1 (no padding)", len(td.Grid[1]))
+ }
+}
+
+// TestToTableData_CaptionEscape verifies the caption is stored verbatim. The
+// legacy round trip escaped then un-escaped it; ToTableData stores it directly
+// and must reach the same decoded value.
+func TestToTableData_CaptionEscape(t *testing.T) {
+ rows := cellsFromStrings([][]string{{"x"}})
+ td := ToTableData(rows, "a<b&c", nil, nil, nil)
+ if td.Caption != "a<b&c" {
+ t.Errorf("Caption=%q want %q", td.Caption, "a<b&c")
+ }
+}
+
+// TestToTableData_HeaderLeadingOnly locks the equivalence with the legacy
+// ParseTableHTML behaviour: headerRows may flag non-leading rows as headers,
+// but only the leading consecutive run counts (this matches ParseTableHTML's
+// leading-consecutive <th> scan, NOT a per-row flag).
+func TestToTableData_HeaderLeadingOnly(t *testing.T) {
+ rows := cellsFromStrings([][]string{{"h0"}, {"d0"}, {"h2"}})
+ headerRows := map[int]bool{0: true, 2: true}
+ td := ToTableData(rows, "", headerRows, nil, nil)
+ if td.HeaderRows != 1 {
+ t.Errorf("HeaderRows=%d want 1 (leading only)", td.HeaderRows)
+ }
+}
+
+// TestToTableData_TrailingEmptyTrimmed verifies a trailing row that carries NO
+// cells at all (zero cells, len==0) is trimmed. A row holding an empty-text
+// cell is NOT trimmed — it is a real row, matching buildGrid's legacy behaviour.
+func TestToTableData_TrailingEmptyTrimmed(t *testing.T) {
+ rows := cellsFromStrings([][]string{{"a"}, {}})
+ td := ToTableData(rows, "", nil, nil, nil)
+ if len(td.Grid) != 1 {
+ t.Errorf("rows=%d want 1 (trailing zero-cell row trimmed)", len(td.Grid))
+ }
+}
+
+// TestToTableData_EmptyTextCellKept verifies a trailing row that holds an
+// empty-text cell is a real row and is NOT trimmed (adversarial guard against
+// over-eager trim).
+func TestToTableData_EmptyTextCellKept(t *testing.T) {
+ rows := cellsFromStrings([][]string{{"a"}, {""}})
+ td := ToTableData(rows, "", nil, nil, nil)
+ if len(td.Grid) != 2 {
+ t.Errorf("rows=%d want 2 (empty-text cell row kept)", len(td.Grid))
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
index f15836d123..bf2d3ba525 100644
--- a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
+++ b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
@@ -74,10 +74,10 @@
"13_crosspage_table.pdf"
],
"fields": [
- "RowsToHTML"
+ "RenderTableHTML"
],
"permanent": true,
- "reason": "This rule's caption/content-loss half is FULLY RESOLVED; the residual is the HTML serialization format only. (1) RESOLVED — caption content loss: MergeCaptions used to DROP the standalone 'table caption' section (findNearestParent returned -1 and the no-target branch removed it), then multiple caption boxes for one table emitted MULTIPLE <caption> elements in one <table> (invalid HTML — consumers keep only the first), and captions of cross-page tables were rejected because the distance was measured to the merged table's CENTER (far from a caption near the edge). All three are fixed: caption text is injected as a single <caption> per table (injectCaption); caption sentences are concatenated in READING order (top->bottom, matching Python's construct_table order); and the table match uses edge distance (findTables: vertical gap to the table's nearest top/bottom edge + horizontal offset), so 13/14's captions near a tall cross-page table now attach. Result: 06/13/14/18 all retain every caption sentence and their <caption> text now equals Python's (verified: 06 'The following table summarizes... Table 1: Quarterly sales by product category (in USD)' and 13 'Extended Financial Report Table: Monthly financial summary FY2024' match byte-for-byte; 14's 'Table 1: Revenue' is restored). 14 and 18 are now ALIGNED (textSim=100%). Regression-guarded by TestMergeCaptions_EmitsCaptionTag, TestMergeCaptions_LeftMarginCaptionAttaches, TestMergeCaptions_SingleCaptionPerTable, TestMergeCaptions_ReadingOrderByTop, TestMergeCaptions_TallTableCaptionNearEdgeAttaches, and the end-to-end TestPipelineParityCaptionEmitted (at most one <caption> per <table>, every sentence retained + real). (2) RESIDUAL — serialization format (go_intentional, this rule): the remaining textSim<100% on 06/13 comes ONLY from HTML tag/whitespace formatting: Go emits <th> for detected header rows (Python emits <td> everywhere — Go deliberately keeps the header semantics, see the <th> analysis) and compact single-space tags (<td >) vs Python's double-space <td > + leading/trailing cell spaces. Cell text and row/col structure match exactly (gridSim=100%, structSim=100%); the gap is NON-CELL-TEXT only and deliberate."
+ "reason": "This rule's caption/content-loss half is FULLY RESOLVED; the residual is the HTML serialization format only. (1) RESOLVED — caption content loss: MergeCaptions used to DROP the standalone 'table caption' section (findNearestParent returned -1 and the no-target branch removed it), then multiple caption boxes for one table emitted MULTIPLE <caption> elements in one <table> (invalid HTML — consumers keep only the first), and captions of cross-page tables were rejected because the distance was measured to the merged table's CENTER (far from a caption near the edge). All three are fixed: caption text is injected as a single <caption> per table (injectCaption); caption sentences are concatenated in READING order (top->bottom, matching Python's construct_table order); and the table match uses edge distance (findTables: vertical gap to the table's nearest top/bottom edge + horizontal offset), so 13/14's captions near a tall cross-page table now attach. Result: 06/13/14/18 all retain every caption sentence and their <caption> text now equals Python's (verified: 06 'The following table summarizes... Table 1: Quarterly sales by product category (in USD)' and 13 'Extended Financial Report Table: Monthly financial summary FY2024' match byte-for-byte; 14's 'Table 1: Revenue' is restored). 14 and 18 are now ALIGNED (textSim=100%). Regression-guarded by TestMergeCaptions_EmitsCaptionTag, TestMergeCaptions_LeftMarginCaptionAttaches, TestMergeCaptions_SingleCaptionPerTable, TestMergeCaptions_ReadingOrderByTop, TestMergeCaptions_TallTableCaptionNearEdgeAttaches, and the end-to-end TestPipelineParityCaptionEmitted (at most one <caption> per <table>, every sentence retained + real). (2) RESIDUAL — serialization format (go_intentional, this rule): the remaining textSim<100% on 06/13 comes ONLY from HTML tag/whitespace formatting, now emitted by the single canonical renderer tableutil.RenderTableHTML (the former second, compact RowsToHTML emitter was removed): Go emits <th> for detected header rows (Python emits <td> everywhere — Go deliberately keeps the header semantics), quoted span attributes (colspan=2 / rowspan=2), newlines between rows, and an html-escaped <caption>, whereas Python's __html_table uses its own spacing and leaves the caption unescaped. Cell text and row/col structure match exactly (gridSim=100%, structSim=100%); the gap is NON-CELL-TEXT only and deliberate."
},
{
"id": "table-text-interleaved-paragraph-dropped",
diff --git a/internal/parser/tableutil/table_html.go b/internal/parser/tableutil/table_html.go
index b257061fff..c618ef3e57 100644
--- a/internal/parser/tableutil/table_html.go
+++ b/internal/parser/tableutil/table_html.go
@@ -49,7 +49,7 @@ func ParseTableHTML(s string) (*entity.TableData, error) {
if err != nil {
return nil, err
}
- var rows [][]cellInput
+ var rows [][]CellInput
var headerFlags []bool
var caption string
var walk func(*html.Node)
@@ -62,14 +62,14 @@ func ParseTableHTML(s string) (*entity.TableData, error) {
return
}
if isHTMLElement(n, "tr") {
- var cells []cellInput
+ var cells []CellInput
anyTh := false
for c := n.FirstChild; c != nil; c = c.NextSibling {
if isHTMLElement(c, "td") || isHTMLElement(c, "th") {
if isHTMLElement(c, "th") {
anyTh = true
}
- cells = append(cells, cellInput{
+ cells = append(cells, CellInput{
Text: cellText(c),
ColSpan: attrInt(c, "colspan", maxColSpan),
RowSpan: attrInt(c, "rowspan", maxRowSpan),
@@ -88,6 +88,20 @@ func ParseTableHTML(s string) (*entity.TableData, error) {
}
}
walk(doc)
+ return BuildTableData(rows, headerFlags, caption), nil
+}
+
+// BuildTableData assembles the canonical TableData from collected cell rows,
+// per-row header flags and a caption. It owns the header-row count (leading
+// consecutive run, defaulting to 1 when the table has any rows) and the shared
+// grid normalization (buildGrid). Both ParseTableHTML and the deepdoc direct
+// converter (ToTableData) route through it, so the two produce identical grids
+// by construction rather than by a runtime comparison.
+//
+// ColSpan/RowSpan inputs are clamped to [1, max] to match ParseTableHTML's
+// attribute parser and to keep a hostile or malformed span from driving a
+// multi-gigabyte allocation inside buildGrid's width pre-pass.
+func BuildTableData(rows [][]CellInput, headerFlags []bool, caption string) *entity.TableData {
headerCount := 0
for headerCount < len(headerFlags) && headerFlags[headerCount] {
headerCount++
@@ -95,15 +109,32 @@ func ParseTableHTML(s string) (*entity.TableData, error) {
if len(rows) > 0 && headerCount == 0 {
headerCount = 1
}
+ for r := range rows {
+ for i := range rows[r] {
+ rows[r][i].ColSpan = clampSpan(rows[r][i].ColSpan, maxColSpan)
+ rows[r][i].RowSpan = clampSpan(rows[r][i].RowSpan, maxRowSpan)
+ }
+ }
return &entity.TableData{
Grid: buildGrid(rows),
HeaderRows: headerCount,
Caption: caption,
- }, nil
+ }
+}
+
+// clampSpan bounds a span extent to [1, max].
+func clampSpan(v, max int) int {
+ if v < 1 {
+ return 1
+ }
+ if v > max {
+ return max
+ }
+ return v
}
-// cellInput is one cell as collected from a <tr> before grid normalization.
-type cellInput struct {
+// CellInput is one cell as collected from a <tr> before grid normalization.
+type CellInput struct {
Text string
ColSpan int
RowSpan int
@@ -132,7 +163,7 @@ type origin struct {
// (the legacy computation), an ancestor rowspan would push a later row's cells
// past the width and its trailing real cells would be silently dropped by the
// column guard — both losing data and regressing the legacy Rows text view.
-func buildGrid(rows [][]cellInput) [][]entity.TableCell {
+func buildGrid(rows [][]CellInput) [][]entity.TableCell {
// Pre-pass: discover the columns that ancestor rowspans occupy in each
// later row (recorded in `covered`) and the true grid width. The placement
// here mirrors the main loop below exactly so the width matches what the