Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): bake merged-cell geometry into the structured TableData grid (#20832)

xugangqiang5d35f968b07b
diff --git a/internal/entity/table.go b/internal/entity/table.go
index 328038480d..1d4e2d5ad3 100644
--- a/internal/entity/table.go
+++ b/internal/entity/table.go
@@ -16,40 +16,74 @@ package entity
// TableData is the canonical structured table contract shared by every parser
// (office / markdown / html / pdf) and consumed by the ingestion chunkers.
+//
+// The table is stored as a dense, normalized grid (BuildGrid in
+// internal/parser/tableutil enforces this): every logical position appears
+// exactly once. A merged cell keeps its text on the origin slot and sets
+// ColSpan/RowSpan > 1; the positions it covers are Covered slots that carry no
+// text of their own. This replaces the previous parallel Spans list, which was
+// never populated, so merge geometry is now a single source of truth instead
+// of two fields that could drift apart.
+//
// Coordinates and merge geometry are optional so coordinate-free tables
// (spreadsheet / markdown / html) and coordinate-bearing tables (pdf) share
-// one type. The cell text in Rows is already normalized the same way the
+// one type. The cell text in Grid is already normalized the same way the
// legacy chunker normalized HTML <td>/<th> text (TrimSpace, <br> -> newline,
// inert markup skipped, nested table text folded), so consumers can render
-// chunk text directly from Rows without re-parsing any markup.
+// chunk text directly from Grid without re-parsing any markup.
type TableData struct {
- // Rows holds the cell text, row-major. rows[0:HeaderRows] are header rows.
- Rows [][]string `json:"rows"`
+ // Grid holds the cell text, row-major and dense (every row has the same
+ // width). Grid[0:HeaderRows] are header rows. A Covered slot contributes
+ // no text; its content lives at the origin slot named by OriginRow/
+ // OriginCol.
+ Grid [][]TableCell `json:"grid"`
// HeaderRows is the number of leading header rows. The legacy rule is:
// rows carrying a <th> cell count as headers; when no row uses <th> at
// all the first row is treated as the header (HeaderRows == 1).
HeaderRows int `json:"header_rows"`
// Caption is the table caption (pdf emits it; office uses the sheet name).
Caption string `json:"caption,omitempty"`
- // Spans records merged cells. Recovered information: pdf already emits
- // colspan/rowspan in its HTML and office loses merged-cell geometry today;
- // consumers do not use Spans yet (preserved for a later follow-up).
- Spans []CellSpan `json:"spans,omitempty"`
// Page is the pdf 0-based page number, used for layout retrieval.
Page int `json:"page,omitempty"`
- // Cells carries optional per-cell coordinates (pdf only); currently lost
- // in the legacy HTML serialization and preserved here for later use.
- Cells []CellCoord `json:"cells,omitempty"`
}
-// CellSpan identifies a merged cell: the top-left anchor at (Row, Col) spans
-// RowSpan rows and ColSpan columns.
-type CellSpan struct {
- Row, Col, RowSpan, ColSpan int
+// TableCell is one position in TableData.Grid.
+type TableCell struct {
+ // Text is the cell's content (already normalized like the legacy <td>
+ // text: TrimSpace, <br> -> newline, inert markup skipped).
+ Text string `json:"text"`
+ // ColSpan / RowSpan are the extents a merged cell covers. 0 or 1 means the
+ // cell is not merged. Only meaningful on non-Covered origin slots.
+ ColSpan int `json:"col_span,omitempty"`
+ RowSpan int `json:"row_span,omitempty"`
+ // Covered marks a position swallowed by an origin cell's span. Its Text is
+ // empty; readers must skip it (the real content is at the origin slot).
+ Covered bool `json:"covered,omitempty"`
+ // OriginRow / OriginCol name the slot whose span produced this Covered
+ // marker. They are diagnostic only and are not serialized.
+ OriginRow int `json:"-"`
+ OriginCol int `json:"-"`
}
-// CellCoord is the bounding box of a single cell in pdf page coordinates.
-type CellCoord struct {
- Row, Col int
- X0, Y0, X1, Y1 float64
+// CellTexts returns the row-major cell texts, skipping Covered positions
+// (their content lives at the origin). For a grid with no merges this is
+// exactly the legacy Rows: same strings, same row lengths. Consumers that only
+// care about the textual content use this view; consumers that need merge
+// geometry read Grid directly.
+func (td *TableData) CellTexts() [][]string {
+ if td == nil {
+ return nil
+ }
+ out := make([][]string, 0, len(td.Grid))
+ for _, row := range td.Grid {
+ cells := make([]string, 0, len(row))
+ for _, c := range row {
+ if c.Covered {
+ continue
+ }
+ cells = append(cells, c.Text)
+ }
+ out = append(out, cells)
+ }
+ return out
}
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index 51816d2c6a..ddf85ccd0b 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -553,7 +553,7 @@ func generalContextSourceText(doc schema.ChunkDoc) (string, bool) {
if td == nil {
return "", false
}
- rows, headerCount := td.Rows, td.HeaderRows
+ rows, headerCount := td.CellTexts(), td.HeaderRows
lines := make([]string, 0, len(rows)-headerCount)
for _, row := range rows[headerCount:] {
cells := make([]string, 0, len(row))
diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.go
index cbd9152d61..e0196e274f 100644
--- a/internal/ingestion/component/chunker/general_test.go
+++ b/internal/ingestion/component/chunker/general_test.go
@@ -61,10 +61,18 @@ func spreadsheetSegmentItem(sheet string, header []string, rows [][]string, shee
for i := range rows {
matrix = append(matrix, tuple(firstRow+i))
}
- td := &entity.TableData{Rows: allRows, HeaderRows: 1, Caption: sheet}
+ tdGrid := make([][]entity.TableCell, 0, len(allRows))
+ for _, r := range allRows {
+ cells := make([]entity.TableCell, 0, len(r))
+ for _, c := range r {
+ cells = append(cells, entity.TableCell{Text: c})
+ }
+ tdGrid = append(tdGrid, cells)
+ }
+ td := &entity.TableData{Grid: tdGrid, HeaderRows: 1, Caption: sheet}
return map[string]any{
"table": map[string]any{
- "rows": allRows,
+ "grid": tdGrid,
"header_rows": 1,
"caption": sheet,
},
@@ -1085,9 +1093,9 @@ func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testing
if !ok {
t.Fatalf("chunk %d missing structured TableData", i)
}
- tblRows, _ := tbl["rows"].([]any)
- if len(tblRows) < 2 {
- t.Fatalf("chunk %d has too few rows: %v", i, tblRows)
+ tblGrid, _ := tbl["grid"].([]any)
+ if len(tblGrid) < 2 {
+ t.Fatalf("chunk %d has too few rows: %v", i, tblGrid)
}
if !strings.Contains(chunk["text"].(string), "Name") {
t.Errorf("chunk %d text lost the header cell: %q", i, chunk["text"])
@@ -1105,7 +1113,7 @@ func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testing
if matrix[0][1] != 1 {
t.Errorf("chunk %d header tuple = %v, want sheet 1 row 1", i, matrix[0])
}
- dataRows := len(tblRows) - 1
+ dataRows := len(tblGrid) - 1
if dataRows != len(matrix)-1 {
t.Errorf("chunk %d data rows %d != position tuples %d", i, dataRows, len(matrix)-1)
}
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go
index 1fc8c30b8e..f381120aa4 100644
--- a/internal/ingestion/component/chunker/qa.go
+++ b/internal/ingestion/component/chunker/qa.go
@@ -432,7 +432,7 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {
// wire must be re-parsed rather than re-chunked.
var rows [][]string
if td != nil {
- rows = td.Rows
+ rows = td.CellTexts()
}
if len(rows) > 0 {
tmp = qaPairsFromRows(rows, strictCSV, item.Positions, item.SheetIndex != nil || item.Sheet != "")
@@ -468,7 +468,7 @@ func extractQATable(htmlStr string, strictPairs bool) []qaPair {
if err != nil || td == nil {
return nil
}
- return qaPairsFromRows(td.Rows, strictPairs, nil, false)
+ return qaPairsFromRows(td.CellTexts(), strictPairs, nil, false)
}
// qaPairsFromRows builds the pairs of one table: the first two non-empty
diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.go
index 4947595e34..787f3900d8 100644
--- a/internal/ingestion/component/chunker/table.go
+++ b/internal/ingestion/component/chunker/table.go
@@ -137,6 +137,85 @@ func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {
}
// expandHTMLTableRows turns one table payload into one chunk per data row.
+// tableChunkRows projects a structured TableData to row-major cell texts while
+// preserving one value per logical column. Unlike CellTexts it does NOT drop
+// Covered slots: a slot engulfed by a span is resolved back to the spanning
+// origin's text so each projected row keeps the same column count as the header
+// and stays aligned with it. CellTexts would drop a spanned continuation column
+// and shift the surviving cells under the wrong headers — exactly the mis-match
+// a merged HTML table hits when it is expanded per row.
+//
+// Resolution works from the grid alone. As rows are scanned top-to-bottom a
+// per-column "active rowspan" remembers the text each ancestor rowspan still
+// contributes, and within a row a running colspan remembers the text of a just
+// seen wide origin. This deliberately avoids OriginRow/OriginCol, which are
+// json:"-" and therefore absent after the TableData is serialized to JSON and
+// read back — the path expandHTMLTableRows actually consumes.
+func tableChunkRows(td *entity.TableData) [][]string {
+ if td == nil {
+ return nil
+ }
+ out := make([][]string, 0, len(td.Grid))
+ rowspanText := []string{}
+ rowspanLeft := []int{}
+ for _, row := range td.Grid {
+ texts := make([]string, 0, len(row))
+ colspanLeft := 0
+ colspanText := ""
+ for c, cell := range row {
+ for len(rowspanText) <= c {
+ rowspanText = append(rowspanText, "")
+ rowspanLeft = append(rowspanLeft, 0)
+ }
+ if cell.Covered {
+ if rowspanLeft[c] > 0 {
+ texts = append(texts, rowspanText[c])
+ } else {
+ texts = append(texts, colspanText)
+ }
+ } else {
+ texts = append(texts, cell.Text)
+ // A merged cell that spans rows AND columns must register its
+ // active rowspan across the full [c, c+ColSpan) span, not just
+ // the origin column. Its continuation rows own every covered
+ // column, so each must resolve back to the origin text; had we
+ // only registered the start column, the extra covered columns
+ // would read as blank and the row would lose those fields.
+ // An unset ColSpan (0) normalizes to 1 so a row-only span still
+ // registers its single start column.
+ if cell.RowSpan > 1 {
+ colSpan := cell.ColSpan
+ if colSpan < 1 {
+ colSpan = 1
+ }
+ for cc := c; cc < c+colSpan; cc++ {
+ for len(rowspanText) <= cc {
+ rowspanText = append(rowspanText, "")
+ rowspanLeft = append(rowspanLeft, 0)
+ }
+ rowspanText[cc] = cell.Text
+ rowspanLeft[cc] = cell.RowSpan
+ }
+ }
+ if cell.ColSpan > 1 {
+ colspanLeft = cell.ColSpan
+ colspanText = cell.Text
+ }
+ }
+ if colspanLeft > 0 {
+ colspanLeft--
+ }
+ }
+ for c := range rowspanLeft {
+ if rowspanLeft[c] > 0 {
+ rowspanLeft[c]--
+ }
+ }
+ out = append(out, texts)
+ }
+ return out
+}
+
// Non-table payloads pass through unchanged. It reads the structured
// TableData contract that the parser emits on ChunkDoc.TableData; items that
// do not carry TableData (e.g. pre-upgrade HTML markup or non-table records)
@@ -149,7 +228,7 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {
if td == nil {
return []schema.ChunkDoc{item}
}
- rows, headerCount := td.Rows, td.HeaderRows
+ rows, headerCount := tableChunkRows(td), td.HeaderRows
if len(rows) <= headerCount {
out := item
out.Text = tableutil.RenderTableText(td)
diff --git a/internal/ingestion/component/chunker/table_split.go b/internal/ingestion/component/chunker/table_split.go
index 3b086feea8..aeef29cb3e 100644
--- a/internal/ingestion/component/chunker/table_split.go
+++ b/internal/ingestion/component/chunker/table_split.go
@@ -29,6 +29,12 @@ import (
// so the per-row chunk produced downstream keeps full row semantics. The token
// budget is counted on the real cell text of each row, not on any HTML markup.
//
+// Splitting is also span-safe: a cut never falls inside a merged cell's
+// rowspan, so an origin and every row it covers stay in the same part, and a
+// header cell whose rowspan reached into data rows is re-derived per part so
+// it never claims rows the part does not contain. Every part therefore
+// satisfies the same grid contract as a freshly parsed table.
+//
// Alongside the parts it returns dataRanges and headerRows: for part i, the
// half-open range [start, end) of DATA rows (rows below the header) the part
// contains, and the number of leading header rows every part replicates. A
@@ -37,22 +43,55 @@ import (
// aligned with the sub-table. headerRows and ranges are 0/nil when the table
// fits in a single chunk (the caller then emits the whole table unchanged).
func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(string) int) (parts []entity.TableData, dataRanges [][2]int, headerRows int) {
- if td == nil || len(td.Rows) == 0 {
+ if td == nil || len(td.Grid) == 0 {
return nil, nil, 0
}
headerRows = td.HeaderRows
- if headerRows > len(td.Rows) {
- headerRows = len(td.Rows)
+ if headerRows > len(td.Grid) {
+ headerRows = len(td.Grid)
}
- dataRows := td.Rows[headerRows:]
+ dataRows := td.Grid[headerRows:]
if len(dataRows) == 0 {
return nil, nil, 0
}
+ headerGrid := append([][]entity.TableCell{}, td.Grid[:headerRows]...)
+
// Base tokens: header rows + caption, replicated into every sub-table.
- base := &entity.TableData{Rows: td.Rows[:headerRows], Caption: td.Caption}
+ base := &entity.TableData{Grid: headerGrid, Caption: td.Caption}
baseTokens := countTokens(tableutil.RenderTableText(base))
+ // spanThrough[i] is the last data-row index any rowspan originating at or
+ // before row i still covers (-1 while no span is active). Header-origin
+ // spans need no entry here: every part carries the whole header block.
+ spanThrough := make([]int, len(dataRows))
+ maxEnd := -1
+ for i, row := range dataRows {
+ for _, cell := range row {
+ if !cell.Covered && cell.RowSpan > 1 {
+ if end := i + cell.RowSpan - 1; end > maxEnd {
+ maxEnd = end
+ }
+ }
+ }
+ spanThrough[i] = maxEnd
+ }
+ // safeCut moves a desired cut at j right to the end of the span group it
+ // would otherwise split. Span-group integrity wins over the budget, the
+ // same way single-row integrity already does.
+ safeCut := func(j int) int {
+ if j <= 0 || j >= len(dataRows) {
+ return j
+ }
+ if end := spanThrough[j-1]; end >= j {
+ if end+1 > len(dataRows) {
+ return len(dataRows)
+ }
+ return end + 1
+ }
+ return j
+ }
+
var start int
var currentTokens int
flush := func(end int) {
@@ -62,7 +101,8 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strin
part := entity.TableData{
Caption: td.Caption,
HeaderRows: headerRows,
- Rows: append(append([][]string{}, td.Rows[:headerRows]...), dataRows[start:end]...),
+ Grid: append(cloneHeaderGridForPart(headerGrid, dataRows, start, end, headerRows),
+ partDataRows(dataRows, start, end, headerRows)...),
}
parts = append(parts, part)
dataRanges = append(dataRanges, [2]int{start, end})
@@ -72,12 +112,19 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strin
currentTokens = baseTokens
for i, row := range dataRows {
- rowTokens := countTokens(strings.Join(row, " "))
+ if i < start {
+ continue // already belongs to a flushed part
+ }
+ rowTokens := countTokens(rowTokenText(row))
// Flush the running batch before this row if adding it would exceed the
// budget. A single row that alone exceeds the budget still becomes its
// own chunk (row integrity wins over the budget).
if i > start && currentTokens+rowTokens > maxTokens {
- flush(i)
+ end := safeCut(i)
+ flush(end)
+ if i < end {
+ continue // this row is inside the flushed part
+ }
}
currentTokens += rowTokens
}
@@ -88,3 +135,75 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strin
}
return parts, dataRanges, headerRows
}
+
+// cloneHeaderGridForPart copies the header grid for one split part and
+// re-derives each header origin's RowSpan from the data rows the part
+// actually contains. A header cell whose rowspan reached into data rows must
+// not claim rows outside the part: the part keeps the full span when all
+// spanned rows travel with it, shrinks it when only some do, and degrades it
+// to a plain cell when none do. ColSpan is row-local and never changes.
+func cloneHeaderGridForPart(headerGrid, dataRows [][]entity.TableCell, start, end, headerRows int) [][]entity.TableCell {
+ cloned := make([][]entity.TableCell, len(headerGrid))
+ for r, row := range headerGrid {
+ cloned[r] = append([]entity.TableCell{}, row...)
+ for c := range cloned[r] {
+ cell := &cloned[r][c]
+ if cell.Covered || cell.RowSpan <= 1 {
+ continue
+ }
+ // Rows below the origin inside the header block are in every
+ // part; count those first, then the part's own covered rows.
+ below := headerRows - 1 - r
+ if below > cell.RowSpan-1 {
+ below = cell.RowSpan - 1
+ }
+ span := 1 + below
+ for i := start; i < end && span < cell.RowSpan; i++ {
+ if c >= len(dataRows[i]) {
+ break
+ }
+ covered := dataRows[i][c]
+ if !covered.Covered || covered.OriginRow != r || covered.OriginCol != c {
+ break
+ }
+ span++
+ }
+ cell.RowSpan = span
+ }
+ }
+ return cloned
+}
+
+// partDataRows clones the data rows [start,end) for one part and shifts the
+// OriginRow back-pointer of every Covered slot whose origin is a data-row
+// cell (OriginRow >= headerRows) so it names the origin's row inside the
+// part; data rows move down by start. Header-origin markers keep their index
+// because the header block is identical in every part. The source grid is
+// never mutated.
+func partDataRows(dataRows [][]entity.TableCell, start, end, headerRows int) [][]entity.TableCell {
+ out := make([][]entity.TableCell, 0, end-start)
+ for i := start; i < end; i++ {
+ row := make([]entity.TableCell, len(dataRows[i]))
+ copy(row, dataRows[i])
+ for c := range row {
+ if row[c].Covered && row[c].OriginRow >= headerRows {
+ row[c].OriginRow -= start
+ }
+ }
+ out = append(out, row)
+ }
+ return out
+}
+
+// rowTokenText joins the non-covered cell texts of a grid row for token
+// counting, mirroring how the row is later rendered.
+func rowTokenText(row []entity.TableCell) string {
+ texts := make([]string, 0, len(row))
+ for _, c := range row {
+ if c.Covered {
+ continue
+ }
+ texts = append(texts, c.Text)
+ }
+ return strings.Join(texts, " ")
+}
diff --git a/internal/ingestion/component/chunker/table_split_test.go b/internal/ingestion/component/chunker/table_split_test.go
index f2b29f5242..7ae56f10cb 100644
--- a/internal/ingestion/component/chunker/table_split_test.go
+++ b/internal/ingestion/component/chunker/table_split_test.go
@@ -15,20 +15,22 @@
package chunker
import (
+ "reflect"
"testing"
"ragflow/internal/entity"
+ "ragflow/internal/parser/tableutil"
)
// charTokens counts runes, a cheap deterministic stand-in for a real tokenizer.
func charTokens(s string) int { return len([]rune(s)) }
func makeTable(dataRows int) *entity.TableData {
- rows := [][]string{{"Name", "Age"}} // header
+ grid := [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}} // header
for i := 0; i < dataRows; i++ {
- rows = append(rows, []string{"person", "30"})
+ grid = append(grid, []entity.TableCell{{Text: "person"}, {Text: "30"}})
}
- return &entity.TableData{Rows: rows, HeaderRows: 1}
+ return &entity.TableData{Grid: grid, HeaderRows: 1}
}
func TestSplitLargeTableFitsWhole(t *testing.T) {
@@ -62,10 +64,10 @@ func TestSplitLargeTableRowGranular(t *testing.T) {
if part.HeaderRows != 1 {
t.Fatalf("part %d headerRows = %d", i, part.HeaderRows)
}
- if len(part.Rows) < 2 {
- t.Fatalf("part %d has <1 data row: %v", i, part.Rows)
+ if len(part.CellTexts()) < 2 {
+ t.Fatalf("part %d has <1 data row: %v", i, part.CellTexts())
}
- if part.Rows[0][0] != "Name" {
+ if part.CellTexts()[0][0] != "Name" {
t.Fatalf("part %d missing header row", i)
}
r := ranges[i]
@@ -80,9 +82,9 @@ func TestSplitLargeTableRowGranular(t *testing.T) {
// The concatenation of every part's data rows must equal the original rows.
var rebuilt [][]string
for _, part := range parts {
- rebuilt = append(rebuilt, part.Rows[1:]...)
+ rebuilt = append(rebuilt, part.CellTexts()[1:]...)
}
- for i, row := range td.Rows[1:] {
+ for i, row := range td.CellTexts()[1:] {
if len(rebuilt[i]) != len(row) || rebuilt[i][0] != row[0] {
t.Fatalf("row %d mismatch: got %v want %v", i, rebuilt[i], row)
}
@@ -94,7 +96,7 @@ func TestSplitLargeTableSingleRowOverBudgetKeepsRow(t *testing.T) {
// (row integrity wins over the budget). A single part means "not split,
// emit whole" — the over-budget row becomes one chunk rather than being cut.
td := &entity.TableData{
- Rows: [][]string{{"H"}, {"x" + string(make([]byte, 200))}},
+ Grid: [][]entity.TableCell{{{Text: "H"}}, {{Text: "x" + string(make([]byte, 200))}}},
HeaderRows: 1,
}
parts, ranges, _ := splitLargeTable(td, 5, charTokens)
@@ -104,9 +106,151 @@ func TestSplitLargeTableSingleRowOverBudgetKeepsRow(t *testing.T) {
}
func TestSplitLargeTableHeaderOnly(t *testing.T) {
- td := &entity.TableData{Rows: [][]string{{"Name", "Age"}}, HeaderRows: 1}
+ td := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}}, HeaderRows: 1}
parts, ranges, _ := splitLargeTable(td, 10, charTokens)
if ranges != nil || len(parts) != 0 {
t.Fatalf("header-only table should not split")
}
}
+
+// assertPartGridContract checks that a split part is a self-contained grid:
+// every Covered slot names an uncovered origin inside the part that actually
+// spans over it, and every origin's stored spans stay within the rows and
+// columns it owns in the part.
+func assertPartGridContract(t *testing.T, part *entity.TableData, idx int) {
+ t.Helper()
+ for r, row := range part.Grid {
+ for c, cell := range row {
+ if cell.Covered {
+ if cell.OriginRow < 0 || cell.OriginRow >= len(part.Grid) ||
+ cell.OriginCol < 0 || cell.OriginCol >= len(part.Grid[cell.OriginRow]) {
+ t.Fatalf("part %d: covered slot (%d,%d) back-pointer out of bounds: %+v", idx, r, c, cell)
+ }
+ origin := part.Grid[cell.OriginRow][cell.OriginCol]
+ if origin.Covered || origin.RowSpan <= r-cell.OriginRow || origin.ColSpan <= c-cell.OriginCol {
+ t.Fatalf("part %d: covered slot (%d,%d) not owned by origin (%d,%d): %+v", idx, r, c, cell.OriginRow, cell.OriginCol, origin)
+ }
+ continue
+ }
+ if r+cell.RowSpan > len(part.Grid) {
+ t.Fatalf("part %d: cell (%d,%d) RowSpan=%d exceeds part rows %d", idx, r, c, cell.RowSpan, len(part.Grid))
+ }
+ if c+cell.ColSpan > len(row) {
+ t.Fatalf("part %d: cell (%d,%d) ColSpan=%d exceeds row width %d", idx, r, c, cell.ColSpan, len(row))
+ }
+ }
+ }
+}
+
+// partDataTexts concatenates the data-row (below-header) cell texts of every
+// part; splitting must preserve the source's data rows exactly once each.
+func partDataTexts(parts []entity.TableData, headerRows int) [][]string {
+ var out [][]string
+ for _, part := range parts {
+ out = append(out, part.CellTexts()[headerRows:]...)
+ }
+ return out
+}
+
+// A header origin whose rowspan reaches into the data rows is replicated
+// into every part. Each part must re-derive that span from the data rows it
+// actually contains: keep it when all spanned rows travel with the part,
+// shrink it when only some do, and degrade it to a plain header cell when
+// none do — otherwise a later part's header claims rows it does not have.
+func TestSplitLargeTableHeaderRowSpanClippedPerPart(t *testing.T) {
+ td, err := tableutil.ParseTableHTML(`<table>
+<tr><th rowspan="3">A</th><th>x</th></tr>
+<tr><td>y</td></tr>
+<tr><td>z</td></tr>
+<tr><td>p</td><td>q</td></tr>
+<tr><td>r</td><td>s</td></tr>
+<tr><td>t</td><td>u</td></tr>
+</table>`)
+ if err != nil || td == nil {
+ t.Fatalf("parse: %v", err)
+ }
+ // Budget 6: parts hold data rows [0,2), [2,3), [3,4), [4,5). The first
+ // part carries both rows A spans, so A keeps RowSpan 3; the later parts
+ // carry none of them, so A degrades to a plain header cell.
+ parts, ranges, headerRows := splitLargeTable(td, 6, charTokens)
+ if headerRows != 1 || len(parts) != 4 {
+ t.Fatalf("parts=%d ranges=%v headerRows=%d", len(parts), ranges, headerRows)
+ }
+ wantRanges := [][2]int{{0, 2}, {2, 3}, {3, 4}, {4, 5}}
+ if !reflect.DeepEqual(ranges, wantRanges) {
+ t.Fatalf("ranges=%v want %v", ranges, wantRanges)
+ }
+ wantSpans := []int{3, 1, 1, 1}
+ for i, part := range parts {
+ assertPartGridContract(t, &part, i)
+ if got := part.Grid[0][0].RowSpan; got != wantSpans[i] {
+ t.Fatalf("part %d header A RowSpan=%d want %d", i, got, wantSpans[i])
+ }
+ }
+ // Budget 4: every part holds a single data row, so A keeps exactly the
+ // covered rows that travel with the part.
+ parts, _, _ = splitLargeTable(td, 4, charTokens)
+ wantSpans = []int{2, 2, 1, 1, 1}
+ if len(parts) != len(wantSpans) {
+ t.Fatalf("parts=%d want %d", len(parts), len(wantSpans))
+ }
+ for i, part := range parts {
+ assertPartGridContract(t, &part, i)
+ if got := part.Grid[0][0].RowSpan; got != wantSpans[i] {
+ t.Fatalf("part %d header A RowSpan=%d want %d", i, got, wantSpans[i])
+ }
+ }
+ wantTexts := [][]string{{"y"}, {"z"}, {"p", "q"}, {"r", "s"}, {"t", "u"}}
+ if got := partDataTexts(parts, headerRows); !reflect.DeepEqual(got, wantTexts) {
+ t.Fatalf("data texts=%v want %v", got, wantTexts)
+ }
+}
+
+// A cut must never fall inside a data row's rowspan: the origin and every
+// row it covers belong to the same part. Otherwise the first part's origin
+// claims rows outside the part and the next part starts with a Covered slot
+// whose origin is gone.
+func TestSplitLargeTableNeverSplitsInsideRowSpan(t *testing.T) {
+ td, err := tableutil.ParseTableHTML(`<table>
+<tr><th>H</th><th>I</th></tr>
+<tr><td rowspan="3">B</td><td>b1</td></tr>
+<tr><td>b2</td></tr>
+<tr><td>b3</td></tr>
+<tr><td>p</td><td>q</td></tr>
+<tr><td>r</td><td>s</td></tr>
+</table>`)
+ if err != nil || td == nil {
+ t.Fatalf("parse: %v", err)
+ }
+ // The budget alone would cut after data row 2, inside B's span (B covers
+ // data rows 0..2); the cut must move to after row 2, i.e. to 3.
+ parts, ranges, headerRows := splitLargeTable(td, 11, charTokens)
+ if headerRows != 1 {
+ t.Fatalf("headerRows=%d want 1", headerRows)
+ }
+ wantRanges := [][2]int{{0, 3}, {3, 5}}
+ if !reflect.DeepEqual(ranges, wantRanges) {
+ t.Fatalf("ranges=%v want %v (cut must not fall inside B's rowspan)", ranges, wantRanges)
+ }
+ for i, part := range parts {
+ assertPartGridContract(t, &part, i)
+ }
+ wantTexts := [][]string{{"B", "b1"}, {"b2"}, {"b3"}, {"p", "q"}, {"r", "s"}}
+ if got := partDataTexts(parts, headerRows); !reflect.DeepEqual(got, wantTexts) {
+ t.Fatalf("data texts=%v want %v", got, wantTexts)
+ }
+ // B's merged value survives in exactly one part.
+ seen := 0
+ for _, part := range parts {
+ for _, row := range part.CellTexts()[headerRows:] {
+ for _, text := range row {
+ if text == "B" {
+ seen++
+ }
+ }
+ }
+ }
+ if seen != 1 {
+ t.Fatalf("merged value B appears %d times across parts, want 1", seen)
+ }
+}
diff --git a/internal/ingestion/component/chunker/table_struct_test.go b/internal/ingestion/component/chunker/table_struct_test.go
index a8a6cb16ed..06f9d3b9cb 100644
--- a/internal/ingestion/component/chunker/table_struct_test.go
+++ b/internal/ingestion/component/chunker/table_struct_test.go
@@ -23,7 +23,7 @@ import (
func TestExpandHTMLTableRowsFromTableData(t *testing.T) {
td := &entity.TableData{
- Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}, {"Bob", "25"}},
+ Grid: [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}, {{Text: "Alice"}, {Text: "30"}}, {{Text: "Bob"}, {Text: "25"}}},
HeaderRows: 1,
}
out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})
@@ -44,7 +44,7 @@ func TestExpandHTMLTableRowsFromTableData(t *testing.T) {
}
func TestExpandHTMLTableRowsHeaderOnlySingleChunk(t *testing.T) {
- td := &entity.TableData{Rows: [][]string{{"Name", "Age"}}, HeaderRows: 1}
+ td := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}}, HeaderRows: 1}
out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})
if len(out) != 1 {
t.Fatalf("got %d chunks, want 1 (header-only)", len(out))
@@ -61,3 +61,105 @@ func TestExpandHTMLTableRowsNonTablePassesThrough(t *testing.T) {
t.Fatalf("non-table not passed through: %+v", out)
}
}
+
+// A merged table where the first column carries a rowspan across two data rows
+// must keep both data rows column-aligned with the header after expansion. The
+// continuation row must not lose the spanned column (CellTexts drops Covered
+// slots, which would shift its value under the wrong header). The projection
+// must recover the spanned value from the grid alone, without relying on
+// OriginRow/OriginCol — those fields are json:"-" and absent after a JSON
+// round-trip, which is the path expandHTMLTableRows actually consumes.
+func TestExpandHTMLTableRowsKeepsRowspanColumns(t *testing.T) {
+ td := &entity.TableData{
+ HeaderRows: 1,
+ Grid: [][]entity.TableCell{
+ {{Text: "Name"}, {Text: "Age"}},
+ {{Text: "Alice", RowSpan: 2}, {Text: "30"}},
+ {{Covered: true, OriginRow: 0, OriginCol: 0}, {Text: "25"}},
+ },
+ }
+ // Simulate a JSON round-trip that strips the json:"-" OriginRow/OriginCol.
+ for r := range td.Grid {
+ for c := range td.Grid[r] {
+ td.Grid[r][c].OriginRow = 0
+ td.Grid[r][c].OriginCol = 0
+ }
+ }
+ out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})
+ if len(out) != 2 {
+ t.Fatalf("got %d chunks, want 2", len(out))
+ }
+ if out[0].Text != "- Name: Alice\n- Age: 30" {
+ t.Fatalf("row0 text = %q", out[0].Text)
+ }
+ // The continuation row must keep the spanned "Name" column resolved to the
+ // origin value and stay aligned: "Name: Alice" + "Age: 25", not a shifted
+ // "Name: 25" that drops the Age column.
+ if out[1].Text != "- Name: Alice\n- Age: 25" {
+ t.Fatalf("row1 text = %q (rowspan column must be preserved, not dropped/misaligned)", out[1].Text)
+ }
+}
+
+// A cell that spans BOTH rows and columns (RowSpan>1 && ColSpan>1) must keep
+// every column it covers on the continuation rows, not just the first. The
+// active-rowspan state machine in tableChunkRows used to register the span
+// only at the origin's start column, so a 2x2 block left its second covered
+// column blank on the row below — dropping that field's value and misaligning
+// the row under the header.
+func TestExpandHTMLTableRowsKeepsRowColSpanBlock(t *testing.T) {
+ td := &entity.TableData{
+ HeaderRows: 1,
+ Grid: [][]entity.TableCell{
+ {{Text: "Name"}, {Text: "B"}, {Text: "C"}},
+ {{Text: "Wide", RowSpan: 2, ColSpan: 2}, {Covered: true, OriginRow: 1, OriginCol: 0}, {Text: "x"}},
+ {{Covered: true, OriginRow: 1, OriginCol: 0}, {Covered: true, OriginRow: 1, OriginCol: 0}, {Text: "y"}},
+ },
+ }
+ // Simulate a JSON round-trip that strips the json:"-" OriginRow/OriginCol;
+ // the projection must recover the spanned value from the grid alone.
+ for r := range td.Grid {
+ for c := range td.Grid[r] {
+ td.Grid[r][c].OriginRow = 0
+ td.Grid[r][c].OriginCol = 0
+ }
+ }
+ out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})
+ if len(out) != 2 {
+ t.Fatalf("got %d chunks, want 2 (one per data row)", len(out))
+ }
+ if out[0].Text != "- Name: Wide\n- B: Wide\n- C: x" {
+ t.Fatalf("row0 text = %q", out[0].Text)
+ }
+ // The continuation row must keep BOTH covered columns resolved to "Wide",
+ // not drop the second one to a blank and misalign under the header.
+ if out[1].Text != "- Name: Wide\n- B: Wide\n- C: y" {
+ t.Fatalf("row1 text = %q (both spanned columns must be preserved)", out[1].Text)
+ }
+}
+
+// A data row whose first cell spans two columns must keep both columns after
+// expansion so the row stays aligned with the two-column header. CellTexts
+// would drop the engulfed slot and leave only one value, misaligning it.
+func TestExpandHTMLTableRowsKeepsColspanColumns(t *testing.T) {
+ td := &entity.TableData{
+ HeaderRows: 1,
+ Grid: [][]entity.TableCell{
+ {{Text: "Name"}, {Text: "Age"}},
+ {{Text: "Info", ColSpan: 2}, {Covered: true, OriginRow: 1, OriginCol: 0}},
+ },
+ }
+ // Simulate a JSON round-trip that strips the json:"-" OriginRow/OriginCol.
+ for r := range td.Grid {
+ for c := range td.Grid[r] {
+ td.Grid[r][c].OriginRow = 0
+ td.Grid[r][c].OriginCol = 0
+ }
+ }
+ out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})
+ if len(out) != 1 {
+ t.Fatalf("got %d chunks, want 1", len(out))
+ }
+ if out[0].Text != "- Name: Info\n- Age: Info" {
+ t.Fatalf("row text = %q (colspan must fill both columns)", out[0].Text)
+ }
+}
diff --git a/internal/ingestion/component/schema/chunk_table_test.go b/internal/ingestion/component/schema/chunk_table_test.go
index 400b4c5c42..0b2bb07016 100644
--- a/internal/ingestion/component/schema/chunk_table_test.go
+++ b/internal/ingestion/component/schema/chunk_table_test.go
@@ -24,7 +24,7 @@ import (
func TestChunkDocUnmarshalTableField(t *testing.T) {
src := `{
"doc_type_kwd": "table",
- "table": {"rows": [["Name", "Age"], ["Alice", "30"]], "header_rows": 1, "caption": "People"},
+ "table": {"grid": [[{"text":"Name"},{"text":"Age"}],[{"text":"Alice"},{"text":"30"}]], "header_rows": 1, "caption": "People"},
"text": "legacy-html-should-be-ignored"
}`
var cd ChunkDoc
@@ -40,8 +40,8 @@ func TestChunkDocUnmarshalTableField(t *testing.T) {
if cd.TableData.Caption != "People" {
t.Fatalf("Caption = %q, want %q", cd.TableData.Caption, "People")
}
- if len(cd.TableData.Rows) != 2 || cd.TableData.Rows[1][0] != "Alice" {
- t.Fatalf("Rows = %v, unexpected", cd.TableData.Rows)
+ if len(cd.TableData.CellTexts()) != 2 || cd.TableData.CellTexts()[1][0] != "Alice" {
+ t.Fatalf("CellTexts = %v, unexpected", cd.TableData.CellTexts())
}
// The "table" key must NOT also land in Extra (no double storage).
if _, ok := cd.Extra["table"]; ok {
@@ -53,7 +53,7 @@ func TestChunkDocMarshalRoundTripTable(t *testing.T) {
cd := ChunkDoc{
DocType: "table",
TableData: &entity.TableData{
- Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}},
+ Grid: [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}, {{Text: "Alice"}, {Text: "30"}}},
HeaderRows: 1,
},
}
@@ -65,7 +65,7 @@ func TestChunkDocMarshalRoundTripTable(t *testing.T) {
if err := json.Unmarshal(b, &back); err != nil {
t.Fatalf("Unmarshal error = %v", err)
}
- if back.TableData == nil || len(back.TableData.Rows) != 2 {
+ if back.TableData == nil || len(back.TableData.Grid) != 2 {
t.Fatalf("round-trip lost TableData: %v", back.TableData)
}
}
diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.go
index 20a82e92c4..29792c98f8 100644
--- a/internal/parser/parser/docx_ir_test.go
+++ b/internal/parser/parser/docx_ir_test.go
@@ -32,7 +32,7 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {
{"text": "Title", "image": nil, "doc_type_kwd": "text", "ck_type": "heading"},
{"text": "Hello", "image": nil, "doc_type_kwd": "text"},
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
- {"image": nil, "doc_type_kwd": "table", "table": &entity.TableData{Rows: [][]string{{"cell"}}, HeaderRows: 1}},
+ {"image": nil, "doc_type_kwd": "table", "table": &entity.TableData{Grid: [][]entity.TableCell{{{Text: "cell", ColSpan: 1, RowSpan: 1}}}, HeaderRows: 1}},
{"text": "item1", "image": nil, "doc_type_kwd": "text"},
}
if !reflect.DeepEqual(got, want) {
diff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.go
index b222854a41..0e35118c95 100644
--- a/internal/parser/parser/docx_parser_test.go
+++ b/internal/parser/parser/docx_parser_test.go
@@ -102,7 +102,7 @@ func TestBuildDOCXJSONSections_Tables(t *testing.T) {
t.Fatal("table field missing or not *entity.TableData")
}
var all strings.Builder
- for _, row := range td.Rows {
+ for _, row := range td.CellTexts() {
for _, c := range row {
all.WriteString(c)
}
@@ -110,10 +110,10 @@ func TestBuildDOCXJSONSections_Tables(t *testing.T) {
s := all.String()
if !strings.Contains(s, "A1") || !strings.Contains(s, "B1") ||
!strings.Contains(s, "A2") || !strings.Contains(s, "B2") {
- t.Errorf("table rows = %#v, missing cell content", td.Rows)
+ t.Errorf("table rows = %#v, missing cell content", td.CellTexts())
}
- if len(td.Rows) != 2 {
- t.Errorf("table rows = %#v, want 2 rows", td.Rows)
+ if len(td.CellTexts()) != 2 {
+ t.Errorf("table rows = %#v, want 2 rows", td.CellTexts())
}
if got, ok := item["doc_type_kwd"].(string); !ok || got != "table" {
t.Errorf("doc_type_kwd = %q, want %q", got, "table")
diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go
index 3365889472..c4bb08dc40 100644
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -281,7 +281,7 @@ func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {
t.Fatalf("table item missing *entity.TableData: %#v", item)
}
if !tableRowsContain(td, "Check item") {
- t.Errorf("table item rows missing header label: %+v", td.Rows)
+ t.Errorf("table item rows missing header label: %+v", td.CellTexts())
}
case "text":
if strings.Contains(text, "<table") {
@@ -965,7 +965,7 @@ func TestMarkdownParser_AlignmentGolden(t *testing.T) {
// the substring want, so table-contract tests can assert cell content without
// depending on the legacy HTML markup shape.
func tableRowsContain(td *entity.TableData, want string) bool {
- for _, row := range td.Rows {
+ for _, row := range td.CellTexts() {
for _, cell := range row {
if strings.Contains(cell, want) {
return true
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index 4c074d5e97..059279b3a1 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -172,7 +172,15 @@ func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow int
rowsOut = append(rowsOut, trimCells(r.cells))
matrix = append(matrix, r.tuple)
}
- td := &entity.TableData{Rows: rowsOut, HeaderRows: 1, Caption: sheet}
+ grid := make([][]entity.TableCell, 0, len(rowsOut))
+ for _, r := range rowsOut {
+ cells := make([]entity.TableCell, 0, len(r))
+ for _, c := range r {
+ cells = append(cells, entity.TableCell{Text: c})
+ }
+ grid = append(grid, cells)
+ }
+ td := &entity.TableData{Grid: grid, HeaderRows: 1, Caption: sheet}
item := NewTableDataJSONItem(td, sheet, matrix)
item["sheet_index"] = sheetIndex
return item
diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.go
index 380f4bf0b7..f10dfad3c1 100644
--- a/internal/parser/parser/parse_result_test.go
+++ b/internal/parser/parser/parse_result_test.go
@@ -25,7 +25,7 @@ import (
func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {
td := &entity.TableData{
- Rows: [][]string{{"col"}, {"value"}},
+ Grid: [][]entity.TableCell{{{Text: "col"}}, {{Text: "value"}}},
HeaderRows: 1,
Caption: "Sheet1",
}
@@ -41,7 +41,7 @@ func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {
if !ok {
t.Fatalf("item[%q] = %T, want *entity.TableData", "table", item["table"])
}
- if got.Caption != "Sheet1" || len(got.Rows) != 2 {
+ if got.Caption != "Sheet1" || len(got.Grid) != 2 {
t.Fatalf("item table = %+v, want caption Sheet1 and 2 rows", got)
}
if _, ok := item["positions"].([][]float64); !ok {
diff --git a/internal/parser/parser/pdf_parser_opendataloader_test.go b/internal/parser/parser/pdf_parser_opendataloader_test.go
index b20301ca03..ea84fe3b68 100644
--- a/internal/parser/parser/pdf_parser_opendataloader_test.go
+++ b/internal/parser/parser/pdf_parser_opendataloader_test.go
@@ -143,8 +143,9 @@ func TestOpenDataLoaderItems_TableCellsFallback(t *testing.T) {
if items[0]["text"] != nil {
t.Fatalf("table item must not carry legacy HTML text: %#v", items[0])
}
- if len(td.Rows) != 1 || len(td.Rows[0]) != 2 || td.Rows[0][0] != "a" || td.Rows[0][1] != "b" {
- t.Fatalf("table rows = %#v, want [[a b]]", td.Rows)
+ rows := td.CellTexts()
+ if len(rows) != 1 || len(rows[0]) != 2 || rows[0][0] != "a" || rows[0][1] != "b" {
+ t.Fatalf("table rows = %#v, want [[a b]]", rows)
}
if td.HeaderRows != 1 {
t.Fatalf("HeaderRows = %d, want 1", td.HeaderRows)
diff --git a/internal/parser/parser/pdf_parser_table_contract_test.go b/internal/parser/parser/pdf_parser_table_contract_test.go
index e6e6dcf0fe..c804d46a47 100644
--- a/internal/parser/parser/pdf_parser_table_contract_test.go
+++ b/internal/parser/parser/pdf_parser_table_contract_test.go
@@ -88,7 +88,7 @@ func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {
if !ok || td == nil {
t.Fatalf("doc_type_kwd table item missing structured *entity.TableData: %#v", item)
}
- if len(td.Rows) == 0 {
+ if len(td.CellTexts()) == 0 {
t.Fatalf("structured table has no rows: %#v", td)
}
if got := tableutil.RenderTableText(td); !strings.Contains(got, "What is RAGFlow?") {
diff --git a/internal/parser/parser/pdf_table_no_html_text_test.go b/internal/parser/parser/pdf_table_no_html_text_test.go
index e741e91a81..091f4b0e39 100644
--- a/internal/parser/parser/pdf_table_no_html_text_test.go
+++ b/internal/parser/parser/pdf_table_no_html_text_test.go
@@ -58,7 +58,7 @@ func TestPDFTableItemsOmitHTMLText(t *testing.T) {
t.Fatalf("table item still carries legacy HTML text %q", v)
}
td, ok := tableItem["table"].(*entity.TableData)
- if !ok || td == nil || len(td.Rows) == 0 {
+ if !ok || td == nil || len(td.CellTexts()) == 0 {
t.Fatalf("table item missing structured TableData: %#v", tableItem)
}
if got := tableutil.RenderTableText(td); !strings.Contains(got, "What is RAGFlow?") && !strings.Contains(got, "cell") {
@@ -76,7 +76,7 @@ func TestPDFTableItemsOmitHTMLText(t *testing.T) {
// ParseTableHTML/RenderTableHTML round-trip).
func TestPDFItemsToMarkdownRendersTableFromTableField(t *testing.T) {
td := &entity.TableData{
- Rows: [][]string{{"What is RAGFlow?", "A RAG engine."}, {"Is it open source?", "Yes."}},
+ Grid: [][]entity.TableCell{{{Text: "What is RAGFlow?"}, {Text: "A RAG engine."}}, {{Text: "Is it open source?"}, {Text: "Yes."}}},
HeaderRows: 1,
}
// Producer-after-cleanup shape: `table` present, `text` absent.
diff --git a/internal/parser/tableutil/table_grid_test.go b/internal/parser/tableutil/table_grid_test.go
new file mode 100644
index 0000000000..f8326950de
--- /dev/null
+++ b/internal/parser/tableutil/table_grid_test.go
@@ -0,0 +1,441 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tableutil
+
+import (
+ "reflect"
+ "strings"
+ "testing"
+
+ "ragflow/internal/entity"
+)
+
+// gridTexts is the consumer view: row-major cell texts with covered positions
+// skipped. For a 1x1 grid it must equal the legacy Rows exactly.
+func gridTexts(td *entity.TableData) [][]string {
+ return td.CellTexts()
+}
+
+// happy path: a plain table with no spans becomes a dense grid with no covered
+// slots, and its text view is identical to the legacy Rows.
+func TestGridNoSpans(t *testing.T) {
+ td, err := ParseTableHTML(`<table>
+<tr><th>Name</th><th>Age</th></tr>
+<tr><td>Alice</td><td>30</td></tr>
+<tr><td>Bob</td><td>25</td></tr>
+</table>`)
+ if err != nil || td == nil {
+ t.Fatalf("parse: %v", err)
+ }
+ if td.HeaderRows != 1 {
+ t.Fatalf("HeaderRows=%d want 1", td.HeaderRows)
+ }
+ if len(td.Grid) != 3 {
+ t.Fatalf("grid rows=%d want 3", len(td.Grid))
+ }
+ for _, row := range td.Grid {
+ if len(row) != 2 {
+ t.Fatalf("row width=%d want 2: %+v", len(row), row)
+ }
+ for _, c := range row {
+ if c.Covered || c.ColSpan != 1 || c.RowSpan != 1 {
+ t.Fatalf("non-merged cell must be 1x1 uncovered: %+v", c)
+ }
+ }
+ }
+ want := [][]string{{"Name", "Age"}, {"Alice", "30"}, {"Bob", "25"}}
+ if got := gridTexts(td); !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%v want %v", got, want)
+ }
+}
+
+// colspan: the wide cell keeps its text on the origin slot with ColSpan=2, the
+// engulfed position becomes a Covered marker, and the text view still yields
+// the two real cell texts (unchanged vs legacy, which ignored colspan).
+func TestGridColspan(t *testing.T) {
+ td, _ := ParseTableHTML(`<table><tr><td colspan="2">wide</td><td>narrow</td></tr></table>`)
+ if len(td.Grid) != 1 {
+ t.Fatalf("rows=%d", len(td.Grid))
+ }
+ row := td.Grid[0]
+ if len(row) != 3 {
+ t.Fatalf("grid width=%d want 3 (origin + covered + narrow)", len(row))
+ }
+ if row[0].Text != "wide" || row[0].ColSpan != 2 || row[0].RowSpan != 1 {
+ t.Fatalf("origin slot wrong: %+v", row[0])
+ }
+ if !row[1].Covered {
+ t.Fatalf("engulfed slot must be Covered: %+v", row[1])
+ }
+ if row[1].OriginCol != 0 {
+ t.Fatalf("covered slot should point at origin col 0: %+v", row[1])
+ }
+ if row[2].Text != "narrow" || row[2].ColSpan != 1 {
+ t.Fatalf("narrow slot wrong: %+v", row[2])
+ }
+ if got := gridTexts(td); !reflect.DeepEqual(got, [][]string{{"wide", "narrow"}}) {
+ t.Fatalf("texts=%v want [[wide narrow]]", got)
+ }
+}
+
+// rowspan: a cell spanning two rows leaves a Covered marker directly below it;
+// the text of the second row is the cell that actually lives there.
+func TestGridRowspan(t *testing.T) {
+ td, _ := ParseTableHTML(`<table>
+<tr><td rowspan="2">A</td><td>x</td></tr>
+<tr><td>y</td></tr>
+</table>`)
+ if len(td.Grid) != 2 {
+ t.Fatalf("rows=%d want 2", len(td.Grid))
+ }
+ top := td.Grid[0]
+ bottom := td.Grid[1]
+ if top[0].Text != "A" || top[0].RowSpan != 2 {
+ t.Fatalf("origin wrong: %+v", top[0])
+ }
+ if !bottom[0].Covered || bottom[0].OriginRow != 0 || bottom[0].OriginCol != 0 {
+ t.Fatalf("covered below wrong: %+v", bottom[0])
+ }
+ // text view: each row keeps only its own (non-covered) cells.
+ want := [][]string{{"A", "x"}, {"y"}}
+ if got := gridTexts(td); !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%v want %v", got, want)
+ }
+}
+
+// A rowspan that extends into a trailing row whose own cells do not reach the
+// spanned column must still keep the continuation Covered marker, so the grid
+// stays dense and the origin's RowSpan never exceeds the available rows. This
+// guards the invariant that every claimed position is backed by a real row.
+func TestGridRowspanKeepsContinuationRow(t *testing.T) {
+ td, _ := ParseTableHTML(`<table>
+<tr><td rowspan="3">A</td><td>B</td></tr>
+<tr><td>C</td></tr>
+<tr></tr>
+</table>`)
+ // r0: A(rs3) + B ; r1: C (with a covered continuation of A at c0) ;
+ // r2: an empty <tr> must become a covered continuation of A, not be
+ // dropped, otherwise the grid is no longer dense and A.RowSpan (3) would
+ // exceed the available rows.
+ if len(td.Grid) != 3 {
+ t.Fatalf("grid rows=%d want 3 (rowspan continuation must survive)", len(td.Grid))
+ }
+ if td.Grid[0][0].RowSpan != 3 {
+ t.Fatalf("origin RowSpan=%d want 3", td.Grid[0][0].RowSpan)
+ }
+ if !td.Grid[2][0].Covered || td.Grid[2][0].OriginRow != 0 || td.Grid[2][0].OriginCol != 0 {
+ t.Fatalf("r2c0 must be a covered continuation of A: %+v", td.Grid[2][0])
+ }
+ // Density invariant: every origin's claimed rows exist.
+ for r, row := range td.Grid {
+ for c, cell := range row {
+ if cell.Covered {
+ continue
+ }
+ if r+cell.RowSpan > len(td.Grid) {
+ t.Fatalf("cell (%d,%d) RowSpan=%d exceeds grid rows %d", r, c, cell.RowSpan, len(td.Grid))
+ }
+ }
+ }
+}
+
+// An ancestor rowspan shifts a later row's cells to the right. The grid width
+// must count the columns an ancestor rowspan occupies, otherwise the later
+// row's trailing real cells are dropped (silent data loss that also regresses
+// the legacy Rows text). This guards buildGrid's width computation.
+func TestGridRowspanShiftsTrailingCells(t *testing.T) {
+ td, _ := ParseTableHTML(`<table>
+<tr><td rowspan="2">A</td><td>x</td></tr>
+<tr><td>y</td><td>z</td></tr>
+</table>`)
+ // r0: A(rs2) + x ; r1: covered at c0, then y at c1, z at c2 -> width 3.
+ if len(td.Grid) != 2 {
+ t.Fatalf("rows=%d want 2", len(td.Grid))
+ }
+ if len(td.Grid[1]) != 3 {
+ t.Fatalf("row1 width=%d want 3 (y and z both present, not dropped)", len(td.Grid[1]))
+ }
+ if !td.Grid[1][0].Covered || td.Grid[1][0].OriginCol != 0 {
+ t.Fatalf("r1c0 must be a covered continuation of A: %+v", td.Grid[1][0])
+ }
+ if td.Grid[1][1].Text != "y" || td.Grid[1][2].Text != "z" {
+ t.Fatalf("r1 must keep both y and z: %+v", td.Grid[1])
+ }
+ // Text view must reproduce the legacy Rows exactly (no cell dropped).
+ want := [][]string{{"A", "x"}, {"y", "z"}}
+ if got := gridTexts(td); !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%v want %v", got, want)
+ }
+}
+
+// Realistic case: a single-column ID cell spans two rows while the second data
+// row carries more cells than the first. The extra cells must not be dropped.
+func TestGridRowspanKeepsExtraCells(t *testing.T) {
+ td, _ := ParseTableHTML(`<table>
+<tr><td rowspan="2">ID</td><td>a</td><td>b</td></tr>
+<tr><td>c</td><td>d</td><td>e</td></tr>
+</table>`)
+ if len(td.Grid[1]) != 4 {
+ t.Fatalf("row1 width=%d want 4 (covered ID + c,d,e)", len(td.Grid[1]))
+ }
+ want := [][]string{{"ID", "a", "b"}, {"c", "d", "e"}}
+ if got := gridTexts(td); !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%v want %v", got, want)
+ }
+}
+
+// combined colspan+rowspan: the big cell covers (r0c0,r0c1,r1c0,r1c1) while c
+// and d are real cells, so 2 rows survive with r1's first two slots Covered.
+func TestGridSpanAndTrailingTrim(t *testing.T) {
+ td, _ := ParseTableHTML(`<table>
+<tr><td colspan="2" rowspan="2">big</td><td>c</td></tr>
+<tr><td>d</td></tr>
+</table>`)
+ // The big cell covers (r0c0,r0c1,r1c0,r1c1); only c and d are real in r0/r1.
+ // No fully-covered trailing row here (r1 has d at c2), so 2 rows survive.
+ if len(td.Grid) != 2 {
+ t.Fatalf("rows=%d want 2 (trailing covered row trimmed if any)", len(td.Grid))
+ }
+ if td.Grid[0][0].Text != "big" || td.Grid[0][0].ColSpan != 2 || td.Grid[0][0].RowSpan != 2 {
+ t.Fatalf("big origin wrong: %+v", td.Grid[0][0])
+ }
+ // r1 should be all covered in its first two slots and carry d at c2.
+ if !td.Grid[1][0].Covered || !td.Grid[1][1].Covered {
+ t.Fatalf("r1 covered slots wrong: %+v", td.Grid[1])
+ }
+ if td.Grid[1][2].Text != "d" {
+ t.Fatalf("r1 c2 should be d: %+v", td.Grid[1][2])
+ }
+}
+
+// overlapping spans must clamp instead of double-counting: a later origin that
+// would land on an already-covered position is pushed past it and keeps a span
+// consistent with what it owns (mirrors AnyDoc's GridBuilder clamp).
+func TestGridOverlapClamps(t *testing.T) {
+ td, _ := ParseTableHTML(`<table>
+<tr><td rowspan="3">tall</td><td>a</td></tr>
+<tr><td colspan="2">wide</td></tr>
+</table>`)
+ // r0: tall(rspan3) + a ; r1: covered + wide(cspan2) -> width 3
+ // The wide cell at r1 would cover c1(r1) and c2(r1); that is fine (c1 is
+ // covered by tall only at r0/r1/r2, c2 has nothing). Assert consistency:
+ // every Covered slot points back to an origin within bounds.
+ for r, row := range td.Grid {
+ for c, cell := range row {
+ if cell.Covered {
+ // The back-pointer must name a slot that exists in the
+ // grid: OriginRow is above (rowspan) or the same row
+ // (colspan), and OriginCol is a real column of that row.
+ if cell.OriginRow < 0 || cell.OriginRow >= len(td.Grid) ||
+ cell.OriginCol < 0 || cell.OriginCol >= len(row) {
+ t.Fatalf("covered slot (%d,%d) back-pointer out of bounds: %+v", r, c, cell)
+ }
+ }
+ }
+ }
+}
+
+// malformed span attributes (0, negative, non-numeric) are clamped to 1 rather
+// than producing huge or negative grids.
+func TestGridBadSpanAttrsClamped(t *testing.T) {
+ td, _ := ParseTableHTML(`<table>
+<tr><td colspan="0">a</td><td colspan="-3">b</td><td rowspan="abc">c</td></tr>
+</table>`)
+ row := td.Grid[0]
+ if len(row) != 3 {
+ t.Fatalf("width=%d want 3 (all spans clamped to 1)", len(row))
+ }
+ for _, c := range row {
+ if c.ColSpan != 1 || c.RowSpan != 1 || c.Covered {
+ t.Fatalf("clamped cell must be 1x1 uncovered: %+v", c)
+ }
+ }
+ if got := gridTexts(td); !reflect.DeepEqual(got, [][]string{{"a", "b", "c"}}) {
+ t.Fatalf("texts=%v", got)
+ }
+}
+
+// A colspan origin's engulfed slot becomes a Covered marker, so the row stays
+// dense (no empty cell) and its width matches rows that have no spans. Ragged
+// (genuinely shorter) source rows are NOT padded with empty cells.
+func TestGridColspanFillsCoveredSlot(t *testing.T) {
+ td, _ := ParseTableHTML(`<table>
+<tr><td colspan="2">wide</td></tr>
+<tr><td>p</td><td>q</td></tr>
+</table>`)
+ if len(td.Grid[0]) != len(td.Grid[1]) {
+ t.Fatalf("rows not uniform width: %d vs %d", len(td.Grid[0]), len(td.Grid[1]))
+ }
+ if len(td.Grid[0]) != 2 {
+ t.Fatalf("width=%d want 2", len(td.Grid[0]))
+ }
+ if !td.Grid[0][1].Covered {
+ t.Fatalf("r0c1 should be covered: %+v", td.Grid[0][1])
+ }
+}
+
+// RenderTableHTML for a merged table emits colspan/rowspan on the origin and
+// skips covered slots; the markup must round-trip back to the same grid.
+func TestRenderMergedEmitsSpanAttrs(t *testing.T) {
+ td, _ := ParseTableHTML(`<table><tr><td colspan="2">wide</td><td>narrow</td></tr></table>`)
+ h := RenderTableHTML(td)
+ if !strings.Contains(h, `colspan="2"`) {
+ t.Fatalf("merged render missing colspan: %s", h)
+ }
+ if strings.Contains(h, `colspan="1"`) {
+ t.Fatalf("1x1 cell must not emit colspan attr: %s", h)
+ }
+ // round-trip
+ back, err := ParseTableHTML(h)
+ if err != nil || back == nil {
+ t.Fatalf("round-trip: %v", err)
+ }
+ if !reflect.DeepEqual(gridTexts(back), gridTexts(td)) {
+ t.Fatalf("round-trip texts differ: %v vs %v", gridTexts(back), gridTexts(td))
+ }
+ if back.Grid[0][0].ColSpan != 2 {
+ t.Fatalf("round-trip lost colspan: %+v", back.Grid[0][0])
+ }
+}
+
+// RenderTableHTML -> ParseTableHTML round-trip must preserve both colspan and
+// rowspan geometry (not just the text view), so a downstream consumer that
+// renders or indexes the grid from the re-serialized markup sees the same
+// merge geometry.
+func TestRenderRoundTripPreservesSpans(t *testing.T) {
+ td, _ := ParseTableHTML(`<table>
+<tr><td colspan="2" rowspan="2">big</td><td>c</td></tr>
+<tr><td>d</td></tr>
+</table>`)
+ h := RenderTableHTML(td)
+ back, err := ParseTableHTML(h)
+ if err != nil || back == nil {
+ t.Fatalf("round-trip: %v", err)
+ }
+ if back.Grid[0][0].ColSpan != 2 || back.Grid[0][0].RowSpan != 2 {
+ t.Fatalf("round-trip lost spans: %+v", back.Grid[0][0])
+ }
+ if !back.Grid[1][0].Covered || !back.Grid[1][1].Covered {
+ t.Fatalf("round-trip lost covered continuation: %+v", back.Grid[1])
+ }
+ if got := gridTexts(back); !reflect.DeepEqual(got, gridTexts(td)) {
+ t.Fatalf("round-trip texts differ: %v vs %v", gridTexts(back), gridTexts(td))
+ }
+}
+
+// A non-merged table renders byte-for-byte identical to the legacy markup so
+// existing downstream consumers see no change.
+func TestRenderNonMergedByteIdentical(t *testing.T) {
+ td := &entity.TableData{
+ Grid: [][]entity.TableCell{
+ {{Text: "Name"}, {Text: "Age"}},
+ {{Text: "Alice"}, {Text: "30"}},
+ },
+ HeaderRows: 1,
+ }
+ got := RenderTableHTML(td)
+ want := "<table>\n<tr><th>Name</th><th>Age</th></tr>\n" +
+ "<tr><td>Alice</td><td>30</td></tr>\n</table>\n"
+ if got != want {
+ t.Fatalf("render mismatch:\n got=%q\nwant=%q", got, want)
+ }
+}
+
+// RenderTableText strips tags; a merged table yields the same stripped text as
+// the equivalent non-merged table (text-level output is unchanged by spans).
+func TestRenderTextMergedSameAsNonMerged(t *testing.T) {
+ merged, _ := ParseTableHTML(`<table><tr><td colspan="2">wide</td><td>narrow</td></tr></table>`)
+ plain, _ := ParseTableHTML(`<table><tr><td>wide</td><td>narrow</td></tr></table>`)
+ if RenderTableText(merged) != RenderTableText(plain) {
+ t.Fatalf("merged text %q != plain text %q", RenderTableText(merged), RenderTableText(plain))
+ }
+}
+
+// A colspan far beyond any real table must be clamped to the HTML-standard
+// maximum (1000, the limit browsers enforce) before buildGrid sees it, so a
+// malformed or hostile attribute can never drive a multi-gigabyte allocation
+// or hang the parser. The grid still stays dense: the origin claims 1000
+// columns, the engulfed positions become Covered markers, and the text view
+// is unchanged.
+func TestGridHugeColSpanClampedToSpecMax(t *testing.T) {
+ td, _ := ParseTableHTML(`<table><tr><td colspan="100000000">wide</td><td>narrow</td></tr></table>`)
+ if len(td.Grid) != 1 {
+ t.Fatalf("rows=%d want 1", len(td.Grid))
+ }
+ row := td.Grid[0]
+ if row[0].Text != "wide" || row[0].ColSpan != 1000 {
+ t.Fatalf("origin ColSpan=%d want 1000 (HTML standard max): %+v", row[0].ColSpan, row[0])
+ }
+ if len(row) != 1001 {
+ t.Fatalf("row width=%d want 1001 (origin + 999 covered + narrow)", len(row))
+ }
+ for c := 1; c <= 999; c++ {
+ if !row[c].Covered || row[c].OriginCol != 0 {
+ t.Fatalf("engulfed slot %d must be Covered pointing at origin col 0: %+v", c, row[c])
+ }
+ }
+ if row[1000].Text != "narrow" || row[1000].ColSpan != 1 {
+ t.Fatalf("trailing cell wrong: %+v", row[1000])
+ }
+ if got := gridTexts(td); !reflect.DeepEqual(got, [][]string{{"wide", "narrow"}}) {
+ t.Fatalf("texts=%v want [[wide narrow]]", got)
+ }
+}
+
+// A rowspan beyond the HTML-standard maximum (65534) must be clamped too; in a
+// small table the surviving extent is additionally bounded by the number of
+// rows below the origin, so the stored span always names rows that exist.
+func TestGridHugeRowSpanClamped(t *testing.T) {
+ td, _ := ParseTableHTML(`<table>
+<tr><td rowspan="999999">A</td><td>x</td></tr>
+<tr><td>y</td></tr>
+<tr><td>z</td></tr>
+</table>`)
+ if len(td.Grid) != 3 {
+ t.Fatalf("rows=%d want 3", len(td.Grid))
+ }
+ if td.Grid[0][0].RowSpan != 3 {
+ t.Fatalf("RowSpan=%d want 3 (clamped to 65534, then to grid rows)", td.Grid[0][0].RowSpan)
+ }
+ if !td.Grid[1][0].Covered || !td.Grid[2][0].Covered {
+ t.Fatalf("continuation rows must stay Covered: %+v / %+v", td.Grid[1][0], td.Grid[2][0])
+ }
+ if got := gridTexts(td); !reflect.DeepEqual(got, [][]string{{"A", "x"}, {"y"}, {"z"}}) {
+ t.Fatalf("texts=%v", got)
+ }
+}
+
+// The renderer must emit the clamped span, never the raw attribute value, so a
+// pathological table round-trips without leaking a huge span attribute.
+func TestRenderHugeSpanRoundTripBounded(t *testing.T) {
+ td, _ := ParseTableHTML(`<table><tr><td colspan="100000000">wide</td><td>narrow</td></tr></table>`)
+ h := RenderTableHTML(td)
+ if strings.Contains(h, "100000000") {
+ t.Fatalf("render leaked the raw span attribute: %s", h)
+ }
+ if !strings.Contains(h, `colspan="1000"`) {
+ t.Fatalf("render must emit the clamped colspan: %s", h)
+ }
+ back, err := ParseTableHTML(h)
+ if err != nil || back == nil {
+ t.Fatalf("round-trip parse: %v", err)
+ }
+ if back.Grid[0][0].ColSpan != 1000 {
+ t.Fatalf("round-trip lost the clamped span: %+v", back.Grid[0][0])
+ }
+ if !reflect.DeepEqual(gridTexts(back), gridTexts(td)) {
+ t.Fatalf("round-trip texts differ: %v vs %v", gridTexts(back), gridTexts(td))
+ }
+}
diff --git a/internal/parser/tableutil/table_html.go b/internal/parser/tableutil/table_html.go
index 1ac54b5726..3006714fe4 100644
--- a/internal/parser/tableutil/table_html.go
+++ b/internal/parser/tableutil/table_html.go
@@ -21,6 +21,7 @@ package tableutil
import (
"regexp"
+ "strconv"
"strings"
"golang.org/x/net/html"
@@ -33,6 +34,13 @@ import (
// is a header row, <br> becomes a newline, inert elements
// (script/style/noscript/template) are skipped, and a nested table's text is
// folded into the enclosing cell (the nested <tr> does NOT become a row).
+//
+// Unlike the legacy extractor, ParseTableHTML now honours colspan/rowspan:
+// a merged cell keeps its text on the origin slot (ColSpan/RowSpan > 1) and
+// the engulfed positions become Covered markers. This recovers merge geometry
+// that the legacy code discarded. For tables without merges the resulting grid
+// is identical to the legacy Rows, so CellTexts (the consumer view) is
+// unchanged.
func ParseTableHTML(s string) (*entity.TableData, error) {
lower := strings.ToLower(s)
if strings.Contains(lower, "<tr") && !strings.Contains(lower, "<table") {
@@ -42,7 +50,7 @@ func ParseTableHTML(s string) (*entity.TableData, error) {
if err != nil {
return nil, err
}
- var rows [][]string
+ var rows [][]cellInput
var headerFlags []bool
var caption string
var walk func(*html.Node)
@@ -55,18 +63,25 @@ func ParseTableHTML(s string) (*entity.TableData, error) {
return
}
if isHTMLElement(n, "tr") {
- var cells []string
+ var cells []cellInput
anyTh := false
for c := n.FirstChild; c != nil; c = c.NextSibling {
if isHTMLElement(c, "td") || isHTMLElement(c, "th") {
if isHTMLElement(c, "th") {
anyTh = true
}
- cells = append(cells, cellText(c))
+ cells = append(cells, cellInput{
+ Text: cellText(c),
+ ColSpan: attrInt(c, "colspan", maxColSpan),
+ RowSpan: attrInt(c, "rowspan", maxRowSpan),
+ })
}
}
rows = append(rows, cells)
headerFlags = append(headerFlags, anyTh)
+ // Do not recurse into the row's cells: a nested <table> inside a
+ // cell is folded into that cell's text by cellText, not parsed as
+ // a separate row.
return
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
@@ -81,16 +96,249 @@ func ParseTableHTML(s string) (*entity.TableData, error) {
if len(rows) > 0 && headerCount == 0 {
headerCount = 1
}
- return &entity.TableData{Rows: rows, HeaderRows: headerCount, Caption: caption}, nil
+ return &entity.TableData{
+ Grid: buildGrid(rows),
+ HeaderRows: headerCount,
+ Caption: caption,
+ }, nil
+}
+
+// cellInput is one cell as collected from a <tr> before grid normalization.
+type cellInput struct {
+ Text string
+ ColSpan int
+ RowSpan int
+}
+
+// origin names the slot whose span produced a Covered marker.
+type origin struct {
+ row, col int
+}
+
+// buildGrid normalizes ragged cell rows into a dense grid where every logical
+// position appears exactly once. A merged cell (ColSpan/RowSpan > 1) occupies
+// its origin slot and the engulfed positions become Covered markers pointing
+// back at the origin; overlapping spans clamp rather than double-count. A row
+// that is genuinely shorter than the widest row is NOT padded with empty cells
+// (a real gap stays a shorter row), but a row can be extended by Covered
+// markers produced by an ancestor's rowspan so the grid stays dense. Trailing
+// rows that carry no cell at all (neither a real cell nor a Covered marker)
+// are trimmed. Spans are clamped to the surviving grid so a stored
+// ColSpan/RowSpan never exceeds the positions it actually owns (mirrors
+// AnyDoc's GridBuilder).
+//
+// The grid width is computed by a pre-pass that walks the same placement rules
+// the main grid build uses, so it counts columns an ancestor rowspan still
+// occupies in a later row. If the width only summed each row's own colspan
+// (the legacy computation), an ancestor rowspan would push a later row's cells
+// past the width and its trailing real cells would be silently dropped by the
+// column guard — both losing data and regressing the legacy Rows text view.
+func buildGrid(rows [][]cellInput) [][]entity.TableCell {
+ // Pre-pass: discover the columns that ancestor rowspans occupy in each
+ // later row (recorded in `covered`) and the true grid width. The placement
+ // here mirrors the main loop below exactly so the width matches what the
+ // main loop will actually reach.
+ covered := make([]map[int]origin, len(rows))
+ width := 0
+ for r, row := range rows {
+ col := 0
+ ci := 0
+ for {
+ if _, ok := covered[r][col]; ok {
+ // Ancestor rowspan owns this column here; advance past it.
+ col++
+ } else if ci < len(row) {
+ c := row[ci]
+ ci++
+ cs := spanOf(c.ColSpan)
+ rs := spanOf(c.RowSpan)
+ col += cs
+ if rs > 1 {
+ for dr := 1; dr < rs; dr++ {
+ rr := r + dr
+ if rr >= len(covered) {
+ break
+ }
+ if covered[rr] == nil {
+ covered[rr] = map[int]origin{}
+ }
+ for dc := 0; dc < cs; dc++ {
+ covered[rr][col-1-dc] = origin{r, col - cs}
+ }
+ }
+ }
+ } else {
+ // No real cell at this column. An ancestor may still own
+ // columns further right, so jump to the next covered column
+ // instead of stopping; otherwise the row is exhausted.
+ next := nextCoveredCol(covered[r], col)
+ if next < 0 {
+ break
+ }
+ col = next
+ }
+ if col > width {
+ width = col
+ }
+ }
+ }
+ if width == 0 {
+ return nil
+ }
+ grid := make([][]entity.TableCell, len(rows))
+ for r, row := range rows {
+ // Size this row to its OWN column count, not the whole-table width: a
+ // wide row elsewhere must not make every short row allocate megabytes
+ // of unused backing capacity. The exact cell count for the row is the
+ // sum of each real cell's colspan plus the ancestor rowspan markers
+ // already recorded in covered[r] — distinct positions, so the total is
+ // exact and append never over-grows the slice.
+ realCols := 0
+ for _, c := range row {
+ realCols += spanOf(c.ColSpan)
+ }
+ grid[r] = make([]entity.TableCell, 0, realCols+len(covered[r]))
+ col := 0
+ ci := 0
+ for col < width {
+ if o, ok := covered[r][col]; ok {
+ grid[r] = append(grid[r], entity.TableCell{
+ Covered: true,
+ OriginRow: o.row,
+ OriginCol: o.col,
+ })
+ col++
+ continue
+ }
+ if ci >= len(row) {
+ // No more real cells in this source row, but an ancestor's
+ // rowspan may still own columns further right: keep advancing
+ // so those Covered markers are emitted instead of stopping
+ // early (which would drop a rowspan continuation and break the
+ // dense-grid invariant).
+ col++
+ continue
+ }
+ c := row[ci]
+ ci++
+ cs := spanOf(c.ColSpan)
+ rs := spanOf(c.RowSpan)
+ grid[r] = append(grid[r], entity.TableCell{
+ Text: c.Text,
+ ColSpan: cs,
+ RowSpan: rs,
+ })
+ // Mark the same-row positions this origin engulfs as Covered
+ // markers so the grid stays dense and exactly-once (mirrors
+ // AnyDoc's GridBuilder, which fills every covered slot). The
+ // back-pointer names the origin slot (col), not the engulfed
+ // position (col+dc). Ancestor rowspan continuation markers for
+ // later rows were already registered in the pre-pass above.
+ for dc := 1; dc < cs; dc++ {
+ grid[r] = append(grid[r], entity.TableCell{
+ Covered: true,
+ OriginRow: r,
+ OriginCol: col,
+ })
+ }
+ col += cs
+ }
+ // Rows are NOT padded to a uniform width: a row that genuinely has
+ // fewer cells than the widest row keeps its actual (ragged) length so
+ // CellTexts() reproduces the legacy row-major Rows exactly. Only
+ // merged cells (colspan/rowspan) introduce Covered markers; a real
+ // gap is simply a shorter row, which both the consumer text view and
+ // HTML render handle without padding artifacts.
+ }
+ // Clamp stored spans to the grid so every claimed position is backed.
+ for r := range grid {
+ w := len(grid[r])
+ for c := range grid[r] {
+ cell := &grid[r][c]
+ if cell.Covered {
+ continue
+ }
+ if cell.RowSpan > len(grid)-r {
+ cell.RowSpan = len(grid) - r
+ }
+ if cell.ColSpan > w-c {
+ cell.ColSpan = w - c
+ }
+ }
+ }
+ // Trim trailing rows that are completely empty: a row with no real cell
+ // and no Covered marker carries no information. A row that only carries
+ // Covered markers is a genuine rowspan continuation and MUST stay so the
+ // grid stays dense and every origin's RowSpan remains backed by a real
+ // row; dropping it would make the claimed span exceed the available rows.
+ for len(grid) > 0 && len(grid[len(grid)-1]) == 0 {
+ grid = grid[:len(grid)-1]
+ }
+ return grid
+}
+
+// spanOf clamps a span extent to a minimum of 1.
+func spanOf(v int) int {
+ if v < 1 {
+ return 1
+ }
+ return v
+}
+
+// nextCoveredCol returns the smallest column key in m strictly greater than col,
+// or -1 if no such key exists. It is used by buildGrid's width pre-pass to jump
+// past empty columns to a column still owned by an ancestor rowspan.
+func nextCoveredCol(m map[int]origin, col int) int {
+ next := -1
+ for k := range m {
+ if k > col && (next < 0 || k < next) {
+ next = k
+ }
+ }
+ return next
+}
+
+// maxColSpan and maxRowSpan are the HTML standard's own limits: browsers
+// clamp a colspan above 1000 and a rowspan above 65534, so the grid must
+// apply the same limits. Without an upper bound a malformed or hostile
+// attribute (e.g. colspan="100000000") would reach buildGrid and drive a
+// multi-gigabyte allocation (row capacity, covered-slot insertion) inside
+// the ingestion worker.
+const (
+ maxColSpan = 1000
+ maxRowSpan = 65534
+)
+
+// attrInt reads a non-negative integer HTML attribute, clamping 0, negative,
+// non-numeric, and above-max values (to 1 and max respectively) so a
+// malformed colspan/rowspan never produces a huge or negative grid.
+func attrInt(n *html.Node, name string, max int) int {
+ for _, a := range n.Attr {
+ if strings.EqualFold(a.Key, name) {
+ v, err := strconv.Atoi(strings.TrimSpace(a.Val))
+ if err != nil || v < 1 {
+ return 1
+ }
+ if v > max {
+ return max
+ }
+ return v
+ }
+ }
+ return 1
}
// RenderTableHTML emits a deterministic <table> markup for a TableData. It is
// used only as a test oracle / round-trip helper and is NOT part of the
// persisted contract (producers emit TableData, not HTML). The emitted shape
// matches the legacy spreadsheet producer's renderSpreadsheetTable exactly
-// (including the inter-row newlines) so that RenderTableText reproduces the
-// legacy htmlTableRE-stripped text byte for byte.
+// (including the inter-row newlines) for 1x1 grids, so RenderTableText
+// reproduces the legacy htmlTableRE-stripped text byte for byte. For merged
+// tables it emits colspan/rowspan on the origin slot and skips Covered slots.
func RenderTableHTML(td *entity.TableData) string {
+ if td == nil {
+ return ""
+ }
var b strings.Builder
b.WriteString("<table>")
if td.Caption != "" {
@@ -99,18 +347,31 @@ func RenderTableHTML(td *entity.TableData) string {
b.WriteString("</caption>")
}
b.WriteString("\n")
- for i, row := range td.Rows {
- b.WriteString("<tr>")
+ for i, row := range td.Grid {
isHeader := i < td.HeaderRows
+ b.WriteString("<tr>")
for _, cell := range row {
+ if cell.Covered {
+ continue
+ }
tag := "td"
if isHeader {
tag = "th"
}
b.WriteString("<")
b.WriteString(tag)
+ if cell.ColSpan > 1 {
+ b.WriteString(" colspan=\"")
+ b.WriteString(strconv.Itoa(cell.ColSpan))
+ b.WriteString("\"")
+ }
+ if cell.RowSpan > 1 {
+ b.WriteString(" rowspan=\"")
+ b.WriteString(strconv.Itoa(cell.RowSpan))
+ b.WriteString("\"")
+ }
b.WriteString(">")
- b.WriteString(html.EscapeString(cell))
+ b.WriteString(html.EscapeString(cell.Text))
b.WriteString("</")
b.WriteString(tag)
b.WriteString(">")
@@ -123,11 +384,12 @@ func RenderTableHTML(td *entity.TableData) string {
return b.String()
}
-// tableTagRE mirrors the legacy tokenizer stripping pattern
-// (htmlTableRE in internal/ingestion/component/tokenizer.go) so that the
-// plaintext a chunker writes for an embed-only path matches the legacy
-// tag-stripped output exactly.
-var tableTagRE = regexp.MustCompile(`</?(table|td|caption|tr|th)( [^<>]{0,12})?>`)
+// tableTagRE strips table markup tags (with any attributes) so the plaintext a
+// chunker writes for an embed-only path matches the legacy tag-stripped output.
+// The pattern tolerates arbitrary attribute lists (including colspan/rowspan)
+// so merged-table markup is stripped completely rather than left partly in the
+// text.
+var tableTagRE = regexp.MustCompile(`</?(?:table|td|caption|tr|th)(?:\s[^<>]*)?>`)
// RenderTableText renders a TableData to plain text with no markup tags, for
// the rare embed path that receives a table without a downstream TableChunker.
diff --git a/internal/parser/tableutil/table_html_test.go b/internal/parser/tableutil/table_html_test.go
index abfea73ad6..f8c0060d8e 100644
--- a/internal/parser/tableutil/table_html_test.go
+++ b/internal/parser/tableutil/table_html_test.go
@@ -44,7 +44,7 @@ func TestParseTableHTMLHappyPath(t *testing.T) {
{"Alice", "30"},
{"Bob", "25"},
}
- assertRowsEqual(t, td.Rows, want)
+ assertRowsEqual(t, td.CellTexts(), want)
}
func TestParseTableHTMLMultipleHeaderRows(t *testing.T) {
@@ -67,8 +67,8 @@ func TestParseTableHTMLNoThTreatsFirstRowAsHeader(t *testing.T) {
func TestParseTableHTMLEmpty(t *testing.T) {
td := mustParse(t, `<table></table>`)
- if len(td.Rows) != 0 {
- t.Fatalf("Rows = %v, want empty", td.Rows)
+ if len(td.CellTexts()) != 0 {
+ t.Fatalf("Rows = %v, want empty", td.CellTexts())
}
if td.HeaderRows != 0 {
t.Fatalf("HeaderRows = %d, want 0", td.HeaderRows)
@@ -78,7 +78,7 @@ func TestParseTableHTMLEmpty(t *testing.T) {
func TestParseTableHTMLTrimsCellWhitespace(t *testing.T) {
html := `<table><tr><td> spaced </td></tr></table>`
td := mustParse(t, html)
- if got := td.Rows[0][0]; got != "spaced" {
+ if got := td.CellTexts()[0][0]; got != "spaced" {
t.Fatalf("cell = %q, want %q", got, "spaced")
}
}
@@ -86,7 +86,7 @@ func TestParseTableHTMLTrimsCellWhitespace(t *testing.T) {
func TestParseTableHTMLBrBecomesNewline(t *testing.T) {
html := `<table><tr><td>line1<br>line2</td></tr></table>`
td := mustParse(t, html)
- if got := td.Rows[0][0]; got != "line1\nline2" {
+ if got := td.CellTexts()[0][0]; got != "line1\nline2" {
t.Fatalf("cell = %q, want %q", got, "line1\nline2")
}
}
@@ -96,7 +96,7 @@ func TestParseTableHTMLInertElementsIgnored(t *testing.T) {
html := `<table><tr><td>keep<script>alert(1)</script>middle` +
`<style>.x{}</style>end</td></tr></table>`
td := mustParse(t, html)
- got := td.Rows[0][0]
+ got := td.CellTexts()[0][0]
if strings.Contains(got, "alert") || strings.Contains(got, ".x{}") {
t.Fatalf("inert content leaked into cell: %q", got)
}
@@ -110,10 +110,10 @@ func TestParseTableHTMLNestedTableFoldedIntoCell(t *testing.T) {
td := mustParse(t, html)
// Outer <tr> yields exactly one row with one cell; the nested <tr> must
// NOT become a separate row.
- if len(td.Rows) != 1 {
- t.Fatalf("len(Rows) = %d, want 1 (nested tr not a row)", len(td.Rows))
+ if len(td.CellTexts()) != 1 {
+ t.Fatalf("len(Rows) = %d, want 1 (nested tr not a row)", len(td.CellTexts()))
}
- got := td.Rows[0][0]
+ got := td.CellTexts()[0][0]
if !strings.Contains(got, "outer") || !strings.Contains(got, "inner") {
t.Fatalf("nested table text not folded: %q", got)
}
@@ -126,26 +126,64 @@ func TestParseTableHTMLBareTrWrapped(t *testing.T) {
// Legacy behaviour: a bare <tr> fragment without <table> still parses.
html := `<tr><td>a</td><td>b</td></tr>`
td := mustParse(t, html)
- assertRowsEqual(t, td.Rows, [][]string{{"a", "b"}})
+ assertRowsEqual(t, td.CellTexts(), [][]string{{"a", "b"}})
}
-func TestParseTableHTMLColspanIgnoredLikeLegacy(t *testing.T) {
- // Legacy tableRowsWithHeader ignores colspan/rowspan and just collects
- // each <td>/<th> as one cell; a spanned cell keeps its own text.
+// A huge colspan on one row drives the grid width up, but each row's backing
+// slice must be sized to its OWN column count, not the whole-table width.
+// buildGrid used to pre-allocate every row to the full table width, so a small
+// HTML input with one wide row could allocate megabytes for every short row
+// that followed. The dense grid stays ragged (short rows are not padded), so a
+// short row's capacity must equal its actual length.
+func TestParseTableHTMLRowCapacityNotFullWidth(t *testing.T) {
+ td := mustParse(t, "<table><tr><td colspan=\"1000\">WIDE</td></tr><tr><td>a</td><td>b</td></tr></table>")
+ if len(td.Grid) != 2 {
+ t.Fatalf("got %d rows, want 2", len(td.Grid))
+ }
+ // Row 1 is short (two 1x1 cells, no ancestor rowspan): its length is 2 and
+ // its backing capacity must equal that, not the full table width (1000).
+ if len(td.Grid[1]) != 2 {
+ t.Fatalf("row1 len = %d, want 2", len(td.Grid[1]))
+ }
+ if cap(td.Grid[1]) != len(td.Grid[1]) {
+ t.Fatalf("row1 cap = %d, want %d (no full-width pre-allocation)", cap(td.Grid[1]), len(td.Grid[1]))
+ }
+}
+
+func TestParseTableHTMLCapturesColspan(t *testing.T) {
+ // A colspan is captured, not discarded: the wide cell keeps its text on the
+ // origin slot (ColSpan=2) and the engulfed position becomes a Covered
+ // marker, but the text view still yields the two real cell texts.
html := `<table><tr><td colspan="2">wide</td><td>narrow</td></tr></table>`
td := mustParse(t, html)
- assertRowsEqual(t, td.Rows, [][]string{{"wide", "narrow"}})
+ row := td.Grid[0]
+ if len(row) != 3 {
+ t.Fatalf("grid width = %d, want 3 (origin + covered + narrow)", len(row))
+ }
+ if row[0].Text != "wide" || row[0].ColSpan != 2 || row[0].RowSpan != 1 {
+ t.Fatalf("origin slot wrong: %+v", row[0])
+ }
+ if !row[1].Covered || row[1].OriginCol != 0 {
+ t.Fatalf("engulfed slot must be Covered pointing at col 0: %+v", row[1])
+ }
+ if row[2].Text != "narrow" || row[2].ColSpan != 1 {
+ t.Fatalf("narrow slot wrong: %+v", row[2])
+ }
+ assertRowsEqual(t, td.CellTexts(), [][]string{{"wide", "narrow"}})
}
func TestRenderTableHTMLRoundTrip(t *testing.T) {
td := &entity.TableData{
- Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}},
+ Grid: [][]entity.TableCell{
+ {{Text: "Name"}, {Text: "Age"}},
+ {{Text: "Alice"}, {Text: "30"}},
+ },
HeaderRows: 1,
Caption: "People",
}
rendered := RenderTableHTML(td)
back := mustParse(t, rendered)
- assertRowsEqual(t, back.Rows, td.Rows)
+ assertRowsEqual(t, back.CellTexts(), td.CellTexts())
if back.HeaderRows != td.HeaderRows {
t.Fatalf("HeaderRows round-trip = %d, want %d", back.HeaderRows, td.HeaderRows)
}
@@ -156,7 +194,10 @@ func TestRenderTableHTMLRoundTrip(t *testing.T) {
func TestRenderTableTextHasNoTags(t *testing.T) {
td := &entity.TableData{
- Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}},
+ Grid: [][]entity.TableCell{
+ {{Text: "Name"}, {Text: "Age"}},
+ {{Text: "Alice"}, {Text: "30"}},
+ },
HeaderRows: 1,
Caption: "People",
}