RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(parser): bake merged-cell geometry into the structured TableData grid (#20832)
xugangqiang5d35f968b07b
diff --git a/internal/entity/table.go b/internal/entity/table.goindex 328038480d..1d4e2d5ad3 100644--- a/internal/entity/table.go+++ b/internal/entity/table.go@@ -16,40 +16,74 @@ package entity// TableData is the canonical structured table contract shared by every parser// (office / markdown / html / pdf) and consumed by the ingestion chunkers.+//+// The table is stored as a dense, normalized grid (BuildGrid in+// internal/parser/tableutil enforces this): every logical position appears+// exactly once. A merged cell keeps its text on the origin slot and sets+// ColSpan/RowSpan > 1; the positions it covers are Covered slots that carry no+// text of their own. This replaces the previous parallel Spans list, which was+// never populated, so merge geometry is now a single source of truth instead+// of two fields that could drift apart.+//// Coordinates and merge geometry are optional so coordinate-free tables// (spreadsheet / markdown / html) and coordinate-bearing tables (pdf) share-// one type. The cell text in Rows is already normalized the same way the+// one type. The cell text in Grid is already normalized the same way the// legacy chunker normalized HTML <td>/<th> text (TrimSpace, <br> -> newline,// inert markup skipped, nested table text folded), so consumers can render-// chunk text directly from Rows without re-parsing any markup.+// chunk text directly from Grid without re-parsing any markup.type TableData struct {- // Rows holds the cell text, row-major. rows[0:HeaderRows] are header rows.- Rows [][]string `json:"rows"`+ // Grid holds the cell text, row-major and dense (every row has the same+ // width). Grid[0:HeaderRows] are header rows. A Covered slot contributes+ // no text; its content lives at the origin slot named by OriginRow/+ // OriginCol.+ Grid [][]TableCell `json:"grid"`// HeaderRows is the number of leading header rows. The legacy rule is:// rows carrying a <th> cell count as headers; when no row uses <th> at// all the first row is treated as the header (HeaderRows == 1).HeaderRows int `json:"header_rows"`// Caption is the table caption (pdf emits it; office uses the sheet name).Caption string `json:"caption,omitempty"`- // Spans records merged cells. Recovered information: pdf already emits- // colspan/rowspan in its HTML and office loses merged-cell geometry today;- // consumers do not use Spans yet (preserved for a later follow-up).- Spans []CellSpan `json:"spans,omitempty"`// Page is the pdf 0-based page number, used for layout retrieval.Page int `json:"page,omitempty"`- // Cells carries optional per-cell coordinates (pdf only); currently lost- // in the legacy HTML serialization and preserved here for later use.- Cells []CellCoord `json:"cells,omitempty"`}-// CellSpan identifies a merged cell: the top-left anchor at (Row, Col) spans-// RowSpan rows and ColSpan columns.-type CellSpan struct {- Row, Col, RowSpan, ColSpan int+// TableCell is one position in TableData.Grid.+type TableCell struct {+ // Text is the cell's content (already normalized like the legacy <td>+ // text: TrimSpace, <br> -> newline, inert markup skipped).+ Text string `json:"text"`+ // ColSpan / RowSpan are the extents a merged cell covers. 0 or 1 means the+ // cell is not merged. Only meaningful on non-Covered origin slots.+ ColSpan int `json:"col_span,omitempty"`+ RowSpan int `json:"row_span,omitempty"`+ // Covered marks a position swallowed by an origin cell's span. Its Text is+ // empty; readers must skip it (the real content is at the origin slot).+ Covered bool `json:"covered,omitempty"`+ // OriginRow / OriginCol name the slot whose span produced this Covered+ // marker. They are diagnostic only and are not serialized.+ OriginRow int `json:"-"`+ OriginCol int `json:"-"`}-// CellCoord is the bounding box of a single cell in pdf page coordinates.-type CellCoord struct {- Row, Col int- X0, Y0, X1, Y1 float64+// CellTexts returns the row-major cell texts, skipping Covered positions+// (their content lives at the origin). For a grid with no merges this is+// exactly the legacy Rows: same strings, same row lengths. Consumers that only+// care about the textual content use this view; consumers that need merge+// geometry read Grid directly.+func (td *TableData) CellTexts() [][]string {+ if td == nil {+ return nil+ }+ out := make([][]string, 0, len(td.Grid))+ for _, row := range td.Grid {+ cells := make([]string, 0, len(row))+ for _, c := range row {+ if c.Covered {+ continue+ }+ cells = append(cells, c.Text)+ }+ out = append(out, cells)+ }+ return out}diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex 51816d2c6a..ddf85ccd0b 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -553,7 +553,7 @@ func generalContextSourceText(doc schema.ChunkDoc) (string, bool) {if td == nil {return "", false}- rows, headerCount := td.Rows, td.HeaderRows+ rows, headerCount := td.CellTexts(), td.HeaderRowslines := make([]string, 0, len(rows)-headerCount)for _, row := range rows[headerCount:] {cells := make([]string, 0, len(row))diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.goindex cbd9152d61..e0196e274f 100644--- a/internal/ingestion/component/chunker/general_test.go+++ b/internal/ingestion/component/chunker/general_test.go@@ -61,10 +61,18 @@ func spreadsheetSegmentItem(sheet string, header []string, rows [][]string, sheefor i := range rows {matrix = append(matrix, tuple(firstRow+i))}- td := &entity.TableData{Rows: allRows, HeaderRows: 1, Caption: sheet}+ tdGrid := make([][]entity.TableCell, 0, len(allRows))+ for _, r := range allRows {+ cells := make([]entity.TableCell, 0, len(r))+ for _, c := range r {+ cells = append(cells, entity.TableCell{Text: c})+ }+ tdGrid = append(tdGrid, cells)+ }+ td := &entity.TableData{Grid: tdGrid, HeaderRows: 1, Caption: sheet}return map[string]any{"table": map[string]any{- "rows": allRows,+ "grid": tdGrid,"header_rows": 1,"caption": sheet,},@@ -1085,9 +1093,9 @@ func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testingif !ok {t.Fatalf("chunk %d missing structured TableData", i)}- tblRows, _ := tbl["rows"].([]any)- if len(tblRows) < 2 {- t.Fatalf("chunk %d has too few rows: %v", i, tblRows)+ tblGrid, _ := tbl["grid"].([]any)+ if len(tblGrid) < 2 {+ t.Fatalf("chunk %d has too few rows: %v", i, tblGrid)}if !strings.Contains(chunk["text"].(string), "Name") {t.Errorf("chunk %d text lost the header cell: %q", i, chunk["text"])@@ -1105,7 +1113,7 @@ func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testingif matrix[0][1] != 1 {t.Errorf("chunk %d header tuple = %v, want sheet 1 row 1", i, matrix[0])}- dataRows := len(tblRows) - 1+ dataRows := len(tblGrid) - 1if dataRows != len(matrix)-1 {t.Errorf("chunk %d data rows %d != position tuples %d", i, dataRows, len(matrix)-1)}diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.goindex 1fc8c30b8e..f381120aa4 100644--- a/internal/ingestion/component/chunker/qa.go+++ b/internal/ingestion/component/chunker/qa.go@@ -432,7 +432,7 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {// wire must be re-parsed rather than re-chunked.var rows [][]stringif td != nil {- rows = td.Rows+ rows = td.CellTexts()}if len(rows) > 0 {tmp = qaPairsFromRows(rows, strictCSV, item.Positions, item.SheetIndex != nil || item.Sheet != "")@@ -468,7 +468,7 @@ func extractQATable(htmlStr string, strictPairs bool) []qaPair {if err != nil || td == nil {return nil}- return qaPairsFromRows(td.Rows, strictPairs, nil, false)+ return qaPairsFromRows(td.CellTexts(), strictPairs, nil, false)}// qaPairsFromRows builds the pairs of one table: the first two non-emptydiff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.goindex 4947595e34..787f3900d8 100644--- a/internal/ingestion/component/chunker/table.go+++ b/internal/ingestion/component/chunker/table.go@@ -137,6 +137,85 @@ func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {}// expandHTMLTableRows turns one table payload into one chunk per data row.+// tableChunkRows projects a structured TableData to row-major cell texts while+// preserving one value per logical column. Unlike CellTexts it does NOT drop+// Covered slots: a slot engulfed by a span is resolved back to the spanning+// origin's text so each projected row keeps the same column count as the header+// and stays aligned with it. CellTexts would drop a spanned continuation column+// and shift the surviving cells under the wrong headers — exactly the mis-match+// a merged HTML table hits when it is expanded per row.+//+// Resolution works from the grid alone. As rows are scanned top-to-bottom a+// per-column "active rowspan" remembers the text each ancestor rowspan still+// contributes, and within a row a running colspan remembers the text of a just+// seen wide origin. This deliberately avoids OriginRow/OriginCol, which are+// json:"-" and therefore absent after the TableData is serialized to JSON and+// read back — the path expandHTMLTableRows actually consumes.+func tableChunkRows(td *entity.TableData) [][]string {+ if td == nil {+ return nil+ }+ out := make([][]string, 0, len(td.Grid))+ rowspanText := []string{}+ rowspanLeft := []int{}+ for _, row := range td.Grid {+ texts := make([]string, 0, len(row))+ colspanLeft := 0+ colspanText := ""+ for c, cell := range row {+ for len(rowspanText) <= c {+ rowspanText = append(rowspanText, "")+ rowspanLeft = append(rowspanLeft, 0)+ }+ if cell.Covered {+ if rowspanLeft[c] > 0 {+ texts = append(texts, rowspanText[c])+ } else {+ texts = append(texts, colspanText)+ }+ } else {+ texts = append(texts, cell.Text)+ // A merged cell that spans rows AND columns must register its+ // active rowspan across the full [c, c+ColSpan) span, not just+ // the origin column. Its continuation rows own every covered+ // column, so each must resolve back to the origin text; had we+ // only registered the start column, the extra covered columns+ // would read as blank and the row would lose those fields.+ // An unset ColSpan (0) normalizes to 1 so a row-only span still+ // registers its single start column.+ if cell.RowSpan > 1 {+ colSpan := cell.ColSpan+ if colSpan < 1 {+ colSpan = 1+ }+ for cc := c; cc < c+colSpan; cc++ {+ for len(rowspanText) <= cc {+ rowspanText = append(rowspanText, "")+ rowspanLeft = append(rowspanLeft, 0)+ }+ rowspanText[cc] = cell.Text+ rowspanLeft[cc] = cell.RowSpan+ }+ }+ if cell.ColSpan > 1 {+ colspanLeft = cell.ColSpan+ colspanText = cell.Text+ }+ }+ if colspanLeft > 0 {+ colspanLeft--+ }+ }+ for c := range rowspanLeft {+ if rowspanLeft[c] > 0 {+ rowspanLeft[c]--+ }+ }+ out = append(out, texts)+ }+ return out+}+// Non-table payloads pass through unchanged. It reads the structured// TableData contract that the parser emits on ChunkDoc.TableData; items that// do not carry TableData (e.g. pre-upgrade HTML markup or non-table records)@@ -149,7 +228,7 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {if td == nil {return []schema.ChunkDoc{item}}- rows, headerCount := td.Rows, td.HeaderRows+ rows, headerCount := tableChunkRows(td), td.HeaderRowsif len(rows) <= headerCount {out := itemout.Text = tableutil.RenderTableText(td)diff --git a/internal/ingestion/component/chunker/table_split.go b/internal/ingestion/component/chunker/table_split.goindex 3b086feea8..aeef29cb3e 100644--- a/internal/ingestion/component/chunker/table_split.go+++ b/internal/ingestion/component/chunker/table_split.go@@ -29,6 +29,12 @@ import (// so the per-row chunk produced downstream keeps full row semantics. The token// budget is counted on the real cell text of each row, not on any HTML markup.//+// Splitting is also span-safe: a cut never falls inside a merged cell's+// rowspan, so an origin and every row it covers stay in the same part, and a+// header cell whose rowspan reached into data rows is re-derived per part so+// it never claims rows the part does not contain. Every part therefore+// satisfies the same grid contract as a freshly parsed table.+//// Alongside the parts it returns dataRanges and headerRows: for part i, the// half-open range [start, end) of DATA rows (rows below the header) the part// contains, and the number of leading header rows every part replicates. A@@ -37,22 +43,55 @@ import (// aligned with the sub-table. headerRows and ranges are 0/nil when the table// fits in a single chunk (the caller then emits the whole table unchanged).func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(string) int) (parts []entity.TableData, dataRanges [][2]int, headerRows int) {- if td == nil || len(td.Rows) == 0 {+ if td == nil || len(td.Grid) == 0 {return nil, nil, 0}headerRows = td.HeaderRows- if headerRows > len(td.Rows) {- headerRows = len(td.Rows)+ if headerRows > len(td.Grid) {+ headerRows = len(td.Grid)}- dataRows := td.Rows[headerRows:]+ dataRows := td.Grid[headerRows:]if len(dataRows) == 0 {return nil, nil, 0}+ headerGrid := append([][]entity.TableCell{}, td.Grid[:headerRows]...)+// Base tokens: header rows + caption, replicated into every sub-table.- base := &entity.TableData{Rows: td.Rows[:headerRows], Caption: td.Caption}+ base := &entity.TableData{Grid: headerGrid, Caption: td.Caption}baseTokens := countTokens(tableutil.RenderTableText(base))+ // spanThrough[i] is the last data-row index any rowspan originating at or+ // before row i still covers (-1 while no span is active). Header-origin+ // spans need no entry here: every part carries the whole header block.+ spanThrough := make([]int, len(dataRows))+ maxEnd := -1+ for i, row := range dataRows {+ for _, cell := range row {+ if !cell.Covered && cell.RowSpan > 1 {+ if end := i + cell.RowSpan - 1; end > maxEnd {+ maxEnd = end+ }+ }+ }+ spanThrough[i] = maxEnd+ }+ // safeCut moves a desired cut at j right to the end of the span group it+ // would otherwise split. Span-group integrity wins over the budget, the+ // same way single-row integrity already does.+ safeCut := func(j int) int {+ if j <= 0 || j >= len(dataRows) {+ return j+ }+ if end := spanThrough[j-1]; end >= j {+ if end+1 > len(dataRows) {+ return len(dataRows)+ }+ return end + 1+ }+ return j+ }+var start intvar currentTokens intflush := func(end int) {@@ -62,7 +101,8 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strinpart := entity.TableData{Caption: td.Caption,HeaderRows: headerRows,- Rows: append(append([][]string{}, td.Rows[:headerRows]...), dataRows[start:end]...),+ Grid: append(cloneHeaderGridForPart(headerGrid, dataRows, start, end, headerRows),+ partDataRows(dataRows, start, end, headerRows)...),}parts = append(parts, part)dataRanges = append(dataRanges, [2]int{start, end})@@ -72,12 +112,19 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strincurrentTokens = baseTokensfor i, row := range dataRows {- rowTokens := countTokens(strings.Join(row, " "))+ if i < start {+ continue // already belongs to a flushed part+ }+ rowTokens := countTokens(rowTokenText(row))// Flush the running batch before this row if adding it would exceed the// budget. A single row that alone exceeds the budget still becomes its// own chunk (row integrity wins over the budget).if i > start && currentTokens+rowTokens > maxTokens {- flush(i)+ end := safeCut(i)+ flush(end)+ if i < end {+ continue // this row is inside the flushed part+ }}currentTokens += rowTokens}@@ -88,3 +135,75 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strin}return parts, dataRanges, headerRows}++// cloneHeaderGridForPart copies the header grid for one split part and+// re-derives each header origin's RowSpan from the data rows the part+// actually contains. A header cell whose rowspan reached into data rows must+// not claim rows outside the part: the part keeps the full span when all+// spanned rows travel with it, shrinks it when only some do, and degrades it+// to a plain cell when none do. ColSpan is row-local and never changes.+func cloneHeaderGridForPart(headerGrid, dataRows [][]entity.TableCell, start, end, headerRows int) [][]entity.TableCell {+ cloned := make([][]entity.TableCell, len(headerGrid))+ for r, row := range headerGrid {+ cloned[r] = append([]entity.TableCell{}, row...)+ for c := range cloned[r] {+ cell := &cloned[r][c]+ if cell.Covered || cell.RowSpan <= 1 {+ continue+ }+ // Rows below the origin inside the header block are in every+ // part; count those first, then the part's own covered rows.+ below := headerRows - 1 - r+ if below > cell.RowSpan-1 {+ below = cell.RowSpan - 1+ }+ span := 1 + below+ for i := start; i < end && span < cell.RowSpan; i++ {+ if c >= len(dataRows[i]) {+ break+ }+ covered := dataRows[i][c]+ if !covered.Covered || covered.OriginRow != r || covered.OriginCol != c {+ break+ }+ span+++ }+ cell.RowSpan = span+ }+ }+ return cloned+}++// partDataRows clones the data rows [start,end) for one part and shifts the+// OriginRow back-pointer of every Covered slot whose origin is a data-row+// cell (OriginRow >= headerRows) so it names the origin's row inside the+// part; data rows move down by start. Header-origin markers keep their index+// because the header block is identical in every part. The source grid is+// never mutated.+func partDataRows(dataRows [][]entity.TableCell, start, end, headerRows int) [][]entity.TableCell {+ out := make([][]entity.TableCell, 0, end-start)+ for i := start; i < end; i++ {+ row := make([]entity.TableCell, len(dataRows[i]))+ copy(row, dataRows[i])+ for c := range row {+ if row[c].Covered && row[c].OriginRow >= headerRows {+ row[c].OriginRow -= start+ }+ }+ out = append(out, row)+ }+ return out+}++// rowTokenText joins the non-covered cell texts of a grid row for token+// counting, mirroring how the row is later rendered.+func rowTokenText(row []entity.TableCell) string {+ texts := make([]string, 0, len(row))+ for _, c := range row {+ if c.Covered {+ continue+ }+ texts = append(texts, c.Text)+ }+ return strings.Join(texts, " ")+}diff --git a/internal/ingestion/component/chunker/table_split_test.go b/internal/ingestion/component/chunker/table_split_test.goindex f2b29f5242..7ae56f10cb 100644--- a/internal/ingestion/component/chunker/table_split_test.go+++ b/internal/ingestion/component/chunker/table_split_test.go@@ -15,20 +15,22 @@package chunkerimport (+ "reflect""testing""ragflow/internal/entity"+ "ragflow/internal/parser/tableutil")// charTokens counts runes, a cheap deterministic stand-in for a real tokenizer.func charTokens(s string) int { return len([]rune(s)) }func makeTable(dataRows int) *entity.TableData {- rows := [][]string{{"Name", "Age"}} // header+ grid := [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}} // headerfor i := 0; i < dataRows; i++ {- rows = append(rows, []string{"person", "30"})+ grid = append(grid, []entity.TableCell{{Text: "person"}, {Text: "30"}})}- return &entity.TableData{Rows: rows, HeaderRows: 1}+ return &entity.TableData{Grid: grid, HeaderRows: 1}}func TestSplitLargeTableFitsWhole(t *testing.T) {@@ -62,10 +64,10 @@ func TestSplitLargeTableRowGranular(t *testing.T) {if part.HeaderRows != 1 {t.Fatalf("part %d headerRows = %d", i, part.HeaderRows)}- if len(part.Rows) < 2 {- t.Fatalf("part %d has <1 data row: %v", i, part.Rows)+ if len(part.CellTexts()) < 2 {+ t.Fatalf("part %d has <1 data row: %v", i, part.CellTexts())}- if part.Rows[0][0] != "Name" {+ if part.CellTexts()[0][0] != "Name" {t.Fatalf("part %d missing header row", i)}r := ranges[i]@@ -80,9 +82,9 @@ func TestSplitLargeTableRowGranular(t *testing.T) {// The concatenation of every part's data rows must equal the original rows.var rebuilt [][]stringfor _, part := range parts {- rebuilt = append(rebuilt, part.Rows[1:]...)+ rebuilt = append(rebuilt, part.CellTexts()[1:]...)}- for i, row := range td.Rows[1:] {+ for i, row := range td.CellTexts()[1:] {if len(rebuilt[i]) != len(row) || rebuilt[i][0] != row[0] {t.Fatalf("row %d mismatch: got %v want %v", i, rebuilt[i], row)}@@ -94,7 +96,7 @@ func TestSplitLargeTableSingleRowOverBudgetKeepsRow(t *testing.T) {// (row integrity wins over the budget). A single part means "not split,// emit whole" — the over-budget row becomes one chunk rather than being cut.td := &entity.TableData{- Rows: [][]string{{"H"}, {"x" + string(make([]byte, 200))}},+ Grid: [][]entity.TableCell{{{Text: "H"}}, {{Text: "x" + string(make([]byte, 200))}}},HeaderRows: 1,}parts, ranges, _ := splitLargeTable(td, 5, charTokens)@@ -104,9 +106,151 @@ func TestSplitLargeTableSingleRowOverBudgetKeepsRow(t *testing.T) {}func TestSplitLargeTableHeaderOnly(t *testing.T) {- td := &entity.TableData{Rows: [][]string{{"Name", "Age"}}, HeaderRows: 1}+ td := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}}, HeaderRows: 1}parts, ranges, _ := splitLargeTable(td, 10, charTokens)if ranges != nil || len(parts) != 0 {t.Fatalf("header-only table should not split")}}++// assertPartGridContract checks that a split part is a self-contained grid:+// every Covered slot names an uncovered origin inside the part that actually+// spans over it, and every origin's stored spans stay within the rows and+// columns it owns in the part.+func assertPartGridContract(t *testing.T, part *entity.TableData, idx int) {+ t.Helper()+ for r, row := range part.Grid {+ for c, cell := range row {+ if cell.Covered {+ if cell.OriginRow < 0 || cell.OriginRow >= len(part.Grid) ||+ cell.OriginCol < 0 || cell.OriginCol >= len(part.Grid[cell.OriginRow]) {+ t.Fatalf("part %d: covered slot (%d,%d) back-pointer out of bounds: %+v", idx, r, c, cell)+ }+ origin := part.Grid[cell.OriginRow][cell.OriginCol]+ if origin.Covered || origin.RowSpan <= r-cell.OriginRow || origin.ColSpan <= c-cell.OriginCol {+ t.Fatalf("part %d: covered slot (%d,%d) not owned by origin (%d,%d): %+v", idx, r, c, cell.OriginRow, cell.OriginCol, origin)+ }+ continue+ }+ if r+cell.RowSpan > len(part.Grid) {+ t.Fatalf("part %d: cell (%d,%d) RowSpan=%d exceeds part rows %d", idx, r, c, cell.RowSpan, len(part.Grid))+ }+ if c+cell.ColSpan > len(row) {+ t.Fatalf("part %d: cell (%d,%d) ColSpan=%d exceeds row width %d", idx, r, c, cell.ColSpan, len(row))+ }+ }+ }+}++// partDataTexts concatenates the data-row (below-header) cell texts of every+// part; splitting must preserve the source's data rows exactly once each.+func partDataTexts(parts []entity.TableData, headerRows int) [][]string {+ var out [][]string+ for _, part := range parts {+ out = append(out, part.CellTexts()[headerRows:]...)+ }+ return out+}++// A header origin whose rowspan reaches into the data rows is replicated+// into every part. Each part must re-derive that span from the data rows it+// actually contains: keep it when all spanned rows travel with the part,+// shrink it when only some do, and degrade it to a plain header cell when+// none do — otherwise a later part's header claims rows it does not have.+func TestSplitLargeTableHeaderRowSpanClippedPerPart(t *testing.T) {+ td, err := tableutil.ParseTableHTML(`<table>+<tr><th rowspan="3">A</th><th>x</th></tr>+<tr><td>y</td></tr>+<tr><td>z</td></tr>+<tr><td>p</td><td>q</td></tr>+<tr><td>r</td><td>s</td></tr>+<tr><td>t</td><td>u</td></tr>+</table>`)+ if err != nil || td == nil {+ t.Fatalf("parse: %v", err)+ }+ // Budget 6: parts hold data rows [0,2), [2,3), [3,4), [4,5). The first+ // part carries both rows A spans, so A keeps RowSpan 3; the later parts+ // carry none of them, so A degrades to a plain header cell.+ parts, ranges, headerRows := splitLargeTable(td, 6, charTokens)+ if headerRows != 1 || len(parts) != 4 {+ t.Fatalf("parts=%d ranges=%v headerRows=%d", len(parts), ranges, headerRows)+ }+ wantRanges := [][2]int{{0, 2}, {2, 3}, {3, 4}, {4, 5}}+ if !reflect.DeepEqual(ranges, wantRanges) {+ t.Fatalf("ranges=%v want %v", ranges, wantRanges)+ }+ wantSpans := []int{3, 1, 1, 1}+ for i, part := range parts {+ assertPartGridContract(t, &part, i)+ if got := part.Grid[0][0].RowSpan; got != wantSpans[i] {+ t.Fatalf("part %d header A RowSpan=%d want %d", i, got, wantSpans[i])+ }+ }+ // Budget 4: every part holds a single data row, so A keeps exactly the+ // covered rows that travel with the part.+ parts, _, _ = splitLargeTable(td, 4, charTokens)+ wantSpans = []int{2, 2, 1, 1, 1}+ if len(parts) != len(wantSpans) {+ t.Fatalf("parts=%d want %d", len(parts), len(wantSpans))+ }+ for i, part := range parts {+ assertPartGridContract(t, &part, i)+ if got := part.Grid[0][0].RowSpan; got != wantSpans[i] {+ t.Fatalf("part %d header A RowSpan=%d want %d", i, got, wantSpans[i])+ }+ }+ wantTexts := [][]string{{"y"}, {"z"}, {"p", "q"}, {"r", "s"}, {"t", "u"}}+ if got := partDataTexts(parts, headerRows); !reflect.DeepEqual(got, wantTexts) {+ t.Fatalf("data texts=%v want %v", got, wantTexts)+ }+}++// A cut must never fall inside a data row's rowspan: the origin and every+// row it covers belong to the same part. Otherwise the first part's origin+// claims rows outside the part and the next part starts with a Covered slot+// whose origin is gone.+func TestSplitLargeTableNeverSplitsInsideRowSpan(t *testing.T) {+ td, err := tableutil.ParseTableHTML(`<table>+<tr><th>H</th><th>I</th></tr>+<tr><td rowspan="3">B</td><td>b1</td></tr>+<tr><td>b2</td></tr>+<tr><td>b3</td></tr>+<tr><td>p</td><td>q</td></tr>+<tr><td>r</td><td>s</td></tr>+</table>`)+ if err != nil || td == nil {+ t.Fatalf("parse: %v", err)+ }+ // The budget alone would cut after data row 2, inside B's span (B covers+ // data rows 0..2); the cut must move to after row 2, i.e. to 3.+ parts, ranges, headerRows := splitLargeTable(td, 11, charTokens)+ if headerRows != 1 {+ t.Fatalf("headerRows=%d want 1", headerRows)+ }+ wantRanges := [][2]int{{0, 3}, {3, 5}}+ if !reflect.DeepEqual(ranges, wantRanges) {+ t.Fatalf("ranges=%v want %v (cut must not fall inside B's rowspan)", ranges, wantRanges)+ }+ for i, part := range parts {+ assertPartGridContract(t, &part, i)+ }+ wantTexts := [][]string{{"B", "b1"}, {"b2"}, {"b3"}, {"p", "q"}, {"r", "s"}}+ if got := partDataTexts(parts, headerRows); !reflect.DeepEqual(got, wantTexts) {+ t.Fatalf("data texts=%v want %v", got, wantTexts)+ }+ // B's merged value survives in exactly one part.+ seen := 0+ for _, part := range parts {+ for _, row := range part.CellTexts()[headerRows:] {+ for _, text := range row {+ if text == "B" {+ seen+++ }+ }+ }+ }+ if seen != 1 {+ t.Fatalf("merged value B appears %d times across parts, want 1", seen)+ }+}diff --git a/internal/ingestion/component/chunker/table_struct_test.go b/internal/ingestion/component/chunker/table_struct_test.goindex a8a6cb16ed..06f9d3b9cb 100644--- a/internal/ingestion/component/chunker/table_struct_test.go+++ b/internal/ingestion/component/chunker/table_struct_test.go@@ -23,7 +23,7 @@ import (func TestExpandHTMLTableRowsFromTableData(t *testing.T) {td := &entity.TableData{- Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}, {"Bob", "25"}},+ Grid: [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}, {{Text: "Alice"}, {Text: "30"}}, {{Text: "Bob"}, {Text: "25"}}},HeaderRows: 1,}out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})@@ -44,7 +44,7 @@ func TestExpandHTMLTableRowsFromTableData(t *testing.T) {}func TestExpandHTMLTableRowsHeaderOnlySingleChunk(t *testing.T) {- td := &entity.TableData{Rows: [][]string{{"Name", "Age"}}, HeaderRows: 1}+ td := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}}, HeaderRows: 1}out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})if len(out) != 1 {t.Fatalf("got %d chunks, want 1 (header-only)", len(out))@@ -61,3 +61,105 @@ func TestExpandHTMLTableRowsNonTablePassesThrough(t *testing.T) {t.Fatalf("non-table not passed through: %+v", out)}}++// A merged table where the first column carries a rowspan across two data rows+// must keep both data rows column-aligned with the header after expansion. The+// continuation row must not lose the spanned column (CellTexts drops Covered+// slots, which would shift its value under the wrong header). The projection+// must recover the spanned value from the grid alone, without relying on+// OriginRow/OriginCol — those fields are json:"-" and absent after a JSON+// round-trip, which is the path expandHTMLTableRows actually consumes.+func TestExpandHTMLTableRowsKeepsRowspanColumns(t *testing.T) {+ td := &entity.TableData{+ HeaderRows: 1,+ Grid: [][]entity.TableCell{+ {{Text: "Name"}, {Text: "Age"}},+ {{Text: "Alice", RowSpan: 2}, {Text: "30"}},+ {{Covered: true, OriginRow: 0, OriginCol: 0}, {Text: "25"}},+ },+ }+ // Simulate a JSON round-trip that strips the json:"-" OriginRow/OriginCol.+ for r := range td.Grid {+ for c := range td.Grid[r] {+ td.Grid[r][c].OriginRow = 0+ td.Grid[r][c].OriginCol = 0+ }+ }+ out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})+ if len(out) != 2 {+ t.Fatalf("got %d chunks, want 2", len(out))+ }+ if out[0].Text != "- Name: Alice\n- Age: 30" {+ t.Fatalf("row0 text = %q", out[0].Text)+ }+ // The continuation row must keep the spanned "Name" column resolved to the+ // origin value and stay aligned: "Name: Alice" + "Age: 25", not a shifted+ // "Name: 25" that drops the Age column.+ if out[1].Text != "- Name: Alice\n- Age: 25" {+ t.Fatalf("row1 text = %q (rowspan column must be preserved, not dropped/misaligned)", out[1].Text)+ }+}++// A cell that spans BOTH rows and columns (RowSpan>1 && ColSpan>1) must keep+// every column it covers on the continuation rows, not just the first. The+// active-rowspan state machine in tableChunkRows used to register the span+// only at the origin's start column, so a 2x2 block left its second covered+// column blank on the row below — dropping that field's value and misaligning+// the row under the header.+func TestExpandHTMLTableRowsKeepsRowColSpanBlock(t *testing.T) {+ td := &entity.TableData{+ HeaderRows: 1,+ Grid: [][]entity.TableCell{+ {{Text: "Name"}, {Text: "B"}, {Text: "C"}},+ {{Text: "Wide", RowSpan: 2, ColSpan: 2}, {Covered: true, OriginRow: 1, OriginCol: 0}, {Text: "x"}},+ {{Covered: true, OriginRow: 1, OriginCol: 0}, {Covered: true, OriginRow: 1, OriginCol: 0}, {Text: "y"}},+ },+ }+ // Simulate a JSON round-trip that strips the json:"-" OriginRow/OriginCol;+ // the projection must recover the spanned value from the grid alone.+ for r := range td.Grid {+ for c := range td.Grid[r] {+ td.Grid[r][c].OriginRow = 0+ td.Grid[r][c].OriginCol = 0+ }+ }+ out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})+ if len(out) != 2 {+ t.Fatalf("got %d chunks, want 2 (one per data row)", len(out))+ }+ if out[0].Text != "- Name: Wide\n- B: Wide\n- C: x" {+ t.Fatalf("row0 text = %q", out[0].Text)+ }+ // The continuation row must keep BOTH covered columns resolved to "Wide",+ // not drop the second one to a blank and misalign under the header.+ if out[1].Text != "- Name: Wide\n- B: Wide\n- C: y" {+ t.Fatalf("row1 text = %q (both spanned columns must be preserved)", out[1].Text)+ }+}++// A data row whose first cell spans two columns must keep both columns after+// expansion so the row stays aligned with the two-column header. CellTexts+// would drop the engulfed slot and leave only one value, misaligning it.+func TestExpandHTMLTableRowsKeepsColspanColumns(t *testing.T) {+ td := &entity.TableData{+ HeaderRows: 1,+ Grid: [][]entity.TableCell{+ {{Text: "Name"}, {Text: "Age"}},+ {{Text: "Info", ColSpan: 2}, {Covered: true, OriginRow: 1, OriginCol: 0}},+ },+ }+ // Simulate a JSON round-trip that strips the json:"-" OriginRow/OriginCol.+ for r := range td.Grid {+ for c := range td.Grid[r] {+ td.Grid[r][c].OriginRow = 0+ td.Grid[r][c].OriginCol = 0+ }+ }+ out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})+ if len(out) != 1 {+ t.Fatalf("got %d chunks, want 1", len(out))+ }+ if out[0].Text != "- Name: Info\n- Age: Info" {+ t.Fatalf("row text = %q (colspan must fill both columns)", out[0].Text)+ }+}diff --git a/internal/ingestion/component/schema/chunk_table_test.go b/internal/ingestion/component/schema/chunk_table_test.goindex 400b4c5c42..0b2bb07016 100644--- a/internal/ingestion/component/schema/chunk_table_test.go+++ b/internal/ingestion/component/schema/chunk_table_test.go@@ -24,7 +24,7 @@ import (func TestChunkDocUnmarshalTableField(t *testing.T) {src := `{"doc_type_kwd": "table",- "table": {"rows": [["Name", "Age"], ["Alice", "30"]], "header_rows": 1, "caption": "People"},+ "table": {"grid": [[{"text":"Name"},{"text":"Age"}],[{"text":"Alice"},{"text":"30"}]], "header_rows": 1, "caption": "People"},"text": "legacy-html-should-be-ignored"}`var cd ChunkDoc@@ -40,8 +40,8 @@ func TestChunkDocUnmarshalTableField(t *testing.T) {if cd.TableData.Caption != "People" {t.Fatalf("Caption = %q, want %q", cd.TableData.Caption, "People")}- if len(cd.TableData.Rows) != 2 || cd.TableData.Rows[1][0] != "Alice" {- t.Fatalf("Rows = %v, unexpected", cd.TableData.Rows)+ if len(cd.TableData.CellTexts()) != 2 || cd.TableData.CellTexts()[1][0] != "Alice" {+ t.Fatalf("CellTexts = %v, unexpected", cd.TableData.CellTexts())}// The "table" key must NOT also land in Extra (no double storage).if _, ok := cd.Extra["table"]; ok {@@ -53,7 +53,7 @@ func TestChunkDocMarshalRoundTripTable(t *testing.T) {cd := ChunkDoc{DocType: "table",TableData: &entity.TableData{- Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}},+ Grid: [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}, {{Text: "Alice"}, {Text: "30"}}},HeaderRows: 1,},}@@ -65,7 +65,7 @@ func TestChunkDocMarshalRoundTripTable(t *testing.T) {if err := json.Unmarshal(b, &back); err != nil {t.Fatalf("Unmarshal error = %v", err)}- if back.TableData == nil || len(back.TableData.Rows) != 2 {+ if back.TableData == nil || len(back.TableData.Grid) != 2 {t.Fatalf("round-trip lost TableData: %v", back.TableData)}}diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.goindex 20a82e92c4..29792c98f8 100644--- a/internal/parser/parser/docx_ir_test.go+++ b/internal/parser/parser/docx_ir_test.go@@ -32,7 +32,7 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {{"text": "Title", "image": nil, "doc_type_kwd": "text", "ck_type": "heading"},{"text": "Hello", "image": nil, "doc_type_kwd": "text"},{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},- {"image": nil, "doc_type_kwd": "table", "table": &entity.TableData{Rows: [][]string{{"cell"}}, HeaderRows: 1}},+ {"image": nil, "doc_type_kwd": "table", "table": &entity.TableData{Grid: [][]entity.TableCell{{{Text: "cell", ColSpan: 1, RowSpan: 1}}}, HeaderRows: 1}},{"text": "item1", "image": nil, "doc_type_kwd": "text"},}if !reflect.DeepEqual(got, want) {diff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.goindex b222854a41..0e35118c95 100644--- a/internal/parser/parser/docx_parser_test.go+++ b/internal/parser/parser/docx_parser_test.go@@ -102,7 +102,7 @@ func TestBuildDOCXJSONSections_Tables(t *testing.T) {t.Fatal("table field missing or not *entity.TableData")}var all strings.Builder- for _, row := range td.Rows {+ for _, row := range td.CellTexts() {for _, c := range row {all.WriteString(c)}@@ -110,10 +110,10 @@ func TestBuildDOCXJSONSections_Tables(t *testing.T) {s := all.String()if !strings.Contains(s, "A1") || !strings.Contains(s, "B1") ||!strings.Contains(s, "A2") || !strings.Contains(s, "B2") {- t.Errorf("table rows = %#v, missing cell content", td.Rows)+ t.Errorf("table rows = %#v, missing cell content", td.CellTexts())}- if len(td.Rows) != 2 {- t.Errorf("table rows = %#v, want 2 rows", td.Rows)+ if len(td.CellTexts()) != 2 {+ t.Errorf("table rows = %#v, want 2 rows", td.CellTexts())}if got, ok := item["doc_type_kwd"].(string); !ok || got != "table" {t.Errorf("doc_type_kwd = %q, want %q", got, "table")diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.goindex 3365889472..c4bb08dc40 100644--- a/internal/parser/parser/markdown_parser_test.go+++ b/internal/parser/parser/markdown_parser_test.go@@ -281,7 +281,7 @@ func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {t.Fatalf("table item missing *entity.TableData: %#v", item)}if !tableRowsContain(td, "Check item") {- t.Errorf("table item rows missing header label: %+v", td.Rows)+ t.Errorf("table item rows missing header label: %+v", td.CellTexts())}case "text":if strings.Contains(text, "<table") {@@ -965,7 +965,7 @@ func TestMarkdownParser_AlignmentGolden(t *testing.T) {// the substring want, so table-contract tests can assert cell content without// depending on the legacy HTML markup shape.func tableRowsContain(td *entity.TableData, want string) bool {- for _, row := range td.Rows {+ for _, row := range td.CellTexts() {for _, cell := range row {if strings.Contains(cell, want) {return truediff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex 4c074d5e97..059279b3a1 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -172,7 +172,15 @@ func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow introwsOut = append(rowsOut, trimCells(r.cells))matrix = append(matrix, r.tuple)}- td := &entity.TableData{Rows: rowsOut, HeaderRows: 1, Caption: sheet}+ grid := make([][]entity.TableCell, 0, len(rowsOut))+ for _, r := range rowsOut {+ cells := make([]entity.TableCell, 0, len(r))+ for _, c := range r {+ cells = append(cells, entity.TableCell{Text: c})+ }+ grid = append(grid, cells)+ }+ td := &entity.TableData{Grid: grid, HeaderRows: 1, Caption: sheet}item := NewTableDataJSONItem(td, sheet, matrix)item["sheet_index"] = sheetIndexreturn itemdiff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.goindex 380f4bf0b7..f10dfad3c1 100644--- a/internal/parser/parser/parse_result_test.go+++ b/internal/parser/parser/parse_result_test.go@@ -25,7 +25,7 @@ import (func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {td := &entity.TableData{- Rows: [][]string{{"col"}, {"value"}},+ Grid: [][]entity.TableCell{{{Text: "col"}}, {{Text: "value"}}},HeaderRows: 1,Caption: "Sheet1",}@@ -41,7 +41,7 @@ func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {if !ok {t.Fatalf("item[%q] = %T, want *entity.TableData", "table", item["table"])}- if got.Caption != "Sheet1" || len(got.Rows) != 2 {+ if got.Caption != "Sheet1" || len(got.Grid) != 2 {t.Fatalf("item table = %+v, want caption Sheet1 and 2 rows", got)}if _, ok := item["positions"].([][]float64); !ok {diff --git a/internal/parser/parser/pdf_parser_opendataloader_test.go b/internal/parser/parser/pdf_parser_opendataloader_test.goindex b20301ca03..ea84fe3b68 100644--- a/internal/parser/parser/pdf_parser_opendataloader_test.go+++ b/internal/parser/parser/pdf_parser_opendataloader_test.go@@ -143,8 +143,9 @@ func TestOpenDataLoaderItems_TableCellsFallback(t *testing.T) {if items[0]["text"] != nil {t.Fatalf("table item must not carry legacy HTML text: %#v", items[0])}- if len(td.Rows) != 1 || len(td.Rows[0]) != 2 || td.Rows[0][0] != "a" || td.Rows[0][1] != "b" {- t.Fatalf("table rows = %#v, want [[a b]]", td.Rows)+ rows := td.CellTexts()+ if len(rows) != 1 || len(rows[0]) != 2 || rows[0][0] != "a" || rows[0][1] != "b" {+ t.Fatalf("table rows = %#v, want [[a b]]", rows)}if td.HeaderRows != 1 {t.Fatalf("HeaderRows = %d, want 1", td.HeaderRows)diff --git a/internal/parser/parser/pdf_parser_table_contract_test.go b/internal/parser/parser/pdf_parser_table_contract_test.goindex e6e6dcf0fe..c804d46a47 100644--- a/internal/parser/parser/pdf_parser_table_contract_test.go+++ b/internal/parser/parser/pdf_parser_table_contract_test.go@@ -88,7 +88,7 @@ func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {if !ok || td == nil {t.Fatalf("doc_type_kwd table item missing structured *entity.TableData: %#v", item)}- if len(td.Rows) == 0 {+ if len(td.CellTexts()) == 0 {t.Fatalf("structured table has no rows: %#v", td)}if got := tableutil.RenderTableText(td); !strings.Contains(got, "What is RAGFlow?") {diff --git a/internal/parser/parser/pdf_table_no_html_text_test.go b/internal/parser/parser/pdf_table_no_html_text_test.goindex e741e91a81..091f4b0e39 100644--- a/internal/parser/parser/pdf_table_no_html_text_test.go+++ b/internal/parser/parser/pdf_table_no_html_text_test.go@@ -58,7 +58,7 @@ func TestPDFTableItemsOmitHTMLText(t *testing.T) {t.Fatalf("table item still carries legacy HTML text %q", v)}td, ok := tableItem["table"].(*entity.TableData)- if !ok || td == nil || len(td.Rows) == 0 {+ if !ok || td == nil || len(td.CellTexts()) == 0 {t.Fatalf("table item missing structured TableData: %#v", tableItem)}if got := tableutil.RenderTableText(td); !strings.Contains(got, "What is RAGFlow?") && !strings.Contains(got, "cell") {@@ -76,7 +76,7 @@ func TestPDFTableItemsOmitHTMLText(t *testing.T) {// ParseTableHTML/RenderTableHTML round-trip).func TestPDFItemsToMarkdownRendersTableFromTableField(t *testing.T) {td := &entity.TableData{- Rows: [][]string{{"What is RAGFlow?", "A RAG engine."}, {"Is it open source?", "Yes."}},+ Grid: [][]entity.TableCell{{{Text: "What is RAGFlow?"}, {Text: "A RAG engine."}}, {{Text: "Is it open source?"}, {Text: "Yes."}}},HeaderRows: 1,}// Producer-after-cleanup shape: `table` present, `text` absent.diff --git a/internal/parser/tableutil/table_grid_test.go b/internal/parser/tableutil/table_grid_test.gonew file mode 100644index 0000000000..f8326950de--- /dev/null+++ b/internal/parser/tableutil/table_grid_test.go@@ -0,0 +1,441 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package tableutil++import (+ "reflect"+ "strings"+ "testing"++ "ragflow/internal/entity"+)++// gridTexts is the consumer view: row-major cell texts with covered positions+// skipped. For a 1x1 grid it must equal the legacy Rows exactly.+func gridTexts(td *entity.TableData) [][]string {+ return td.CellTexts()+}++// happy path: a plain table with no spans becomes a dense grid with no covered+// slots, and its text view is identical to the legacy Rows.+func TestGridNoSpans(t *testing.T) {+ td, err := ParseTableHTML(`<table>+<tr><th>Name</th><th>Age</th></tr>+<tr><td>Alice</td><td>30</td></tr>+<tr><td>Bob</td><td>25</td></tr>+</table>`)+ if err != nil || td == nil {+ t.Fatalf("parse: %v", err)+ }+ if td.HeaderRows != 1 {+ t.Fatalf("HeaderRows=%d want 1", td.HeaderRows)+ }+ if len(td.Grid) != 3 {+ t.Fatalf("grid rows=%d want 3", len(td.Grid))+ }+ for _, row := range td.Grid {+ if len(row) != 2 {+ t.Fatalf("row width=%d want 2: %+v", len(row), row)+ }+ for _, c := range row {+ if c.Covered || c.ColSpan != 1 || c.RowSpan != 1 {+ t.Fatalf("non-merged cell must be 1x1 uncovered: %+v", c)+ }+ }+ }+ want := [][]string{{"Name", "Age"}, {"Alice", "30"}, {"Bob", "25"}}+ if got := gridTexts(td); !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%v want %v", got, want)+ }+}++// colspan: the wide cell keeps its text on the origin slot with ColSpan=2, the+// engulfed position becomes a Covered marker, and the text view still yields+// the two real cell texts (unchanged vs legacy, which ignored colspan).+func TestGridColspan(t *testing.T) {+ td, _ := ParseTableHTML(`<table><tr><td colspan="2">wide</td><td>narrow</td></tr></table>`)+ if len(td.Grid) != 1 {+ t.Fatalf("rows=%d", len(td.Grid))+ }+ row := td.Grid[0]+ if len(row) != 3 {+ t.Fatalf("grid width=%d want 3 (origin + covered + narrow)", len(row))+ }+ if row[0].Text != "wide" || row[0].ColSpan != 2 || row[0].RowSpan != 1 {+ t.Fatalf("origin slot wrong: %+v", row[0])+ }+ if !row[1].Covered {+ t.Fatalf("engulfed slot must be Covered: %+v", row[1])+ }+ if row[1].OriginCol != 0 {+ t.Fatalf("covered slot should point at origin col 0: %+v", row[1])+ }+ if row[2].Text != "narrow" || row[2].ColSpan != 1 {+ t.Fatalf("narrow slot wrong: %+v", row[2])+ }+ if got := gridTexts(td); !reflect.DeepEqual(got, [][]string{{"wide", "narrow"}}) {+ t.Fatalf("texts=%v want [[wide narrow]]", got)+ }+}++// rowspan: a cell spanning two rows leaves a Covered marker directly below it;+// the text of the second row is the cell that actually lives there.+func TestGridRowspan(t *testing.T) {+ td, _ := ParseTableHTML(`<table>+<tr><td rowspan="2">A</td><td>x</td></tr>+<tr><td>y</td></tr>+</table>`)+ if len(td.Grid) != 2 {+ t.Fatalf("rows=%d want 2", len(td.Grid))+ }+ top := td.Grid[0]+ bottom := td.Grid[1]+ if top[0].Text != "A" || top[0].RowSpan != 2 {+ t.Fatalf("origin wrong: %+v", top[0])+ }+ if !bottom[0].Covered || bottom[0].OriginRow != 0 || bottom[0].OriginCol != 0 {+ t.Fatalf("covered below wrong: %+v", bottom[0])+ }+ // text view: each row keeps only its own (non-covered) cells.+ want := [][]string{{"A", "x"}, {"y"}}+ if got := gridTexts(td); !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%v want %v", got, want)+ }+}++// A rowspan that extends into a trailing row whose own cells do not reach the+// spanned column must still keep the continuation Covered marker, so the grid+// stays dense and the origin's RowSpan never exceeds the available rows. This+// guards the invariant that every claimed position is backed by a real row.+func TestGridRowspanKeepsContinuationRow(t *testing.T) {+ td, _ := ParseTableHTML(`<table>+<tr><td rowspan="3">A</td><td>B</td></tr>+<tr><td>C</td></tr>+<tr></tr>+</table>`)+ // r0: A(rs3) + B ; r1: C (with a covered continuation of A at c0) ;+ // r2: an empty <tr> must become a covered continuation of A, not be+ // dropped, otherwise the grid is no longer dense and A.RowSpan (3) would+ // exceed the available rows.+ if len(td.Grid) != 3 {+ t.Fatalf("grid rows=%d want 3 (rowspan continuation must survive)", len(td.Grid))+ }+ if td.Grid[0][0].RowSpan != 3 {+ t.Fatalf("origin RowSpan=%d want 3", td.Grid[0][0].RowSpan)+ }+ if !td.Grid[2][0].Covered || td.Grid[2][0].OriginRow != 0 || td.Grid[2][0].OriginCol != 0 {+ t.Fatalf("r2c0 must be a covered continuation of A: %+v", td.Grid[2][0])+ }+ // Density invariant: every origin's claimed rows exist.+ for r, row := range td.Grid {+ for c, cell := range row {+ if cell.Covered {+ continue+ }+ if r+cell.RowSpan > len(td.Grid) {+ t.Fatalf("cell (%d,%d) RowSpan=%d exceeds grid rows %d", r, c, cell.RowSpan, len(td.Grid))+ }+ }+ }+}++// An ancestor rowspan shifts a later row's cells to the right. The grid width+// must count the columns an ancestor rowspan occupies, otherwise the later+// row's trailing real cells are dropped (silent data loss that also regresses+// the legacy Rows text). This guards buildGrid's width computation.+func TestGridRowspanShiftsTrailingCells(t *testing.T) {+ td, _ := ParseTableHTML(`<table>+<tr><td rowspan="2">A</td><td>x</td></tr>+<tr><td>y</td><td>z</td></tr>+</table>`)+ // r0: A(rs2) + x ; r1: covered at c0, then y at c1, z at c2 -> width 3.+ if len(td.Grid) != 2 {+ t.Fatalf("rows=%d want 2", len(td.Grid))+ }+ if len(td.Grid[1]) != 3 {+ t.Fatalf("row1 width=%d want 3 (y and z both present, not dropped)", len(td.Grid[1]))+ }+ if !td.Grid[1][0].Covered || td.Grid[1][0].OriginCol != 0 {+ t.Fatalf("r1c0 must be a covered continuation of A: %+v", td.Grid[1][0])+ }+ if td.Grid[1][1].Text != "y" || td.Grid[1][2].Text != "z" {+ t.Fatalf("r1 must keep both y and z: %+v", td.Grid[1])+ }+ // Text view must reproduce the legacy Rows exactly (no cell dropped).+ want := [][]string{{"A", "x"}, {"y", "z"}}+ if got := gridTexts(td); !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%v want %v", got, want)+ }+}++// Realistic case: a single-column ID cell spans two rows while the second data+// row carries more cells than the first. The extra cells must not be dropped.+func TestGridRowspanKeepsExtraCells(t *testing.T) {+ td, _ := ParseTableHTML(`<table>+<tr><td rowspan="2">ID</td><td>a</td><td>b</td></tr>+<tr><td>c</td><td>d</td><td>e</td></tr>+</table>`)+ if len(td.Grid[1]) != 4 {+ t.Fatalf("row1 width=%d want 4 (covered ID + c,d,e)", len(td.Grid[1]))+ }+ want := [][]string{{"ID", "a", "b"}, {"c", "d", "e"}}+ if got := gridTexts(td); !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%v want %v", got, want)+ }+}++// combined colspan+rowspan: the big cell covers (r0c0,r0c1,r1c0,r1c1) while c+// and d are real cells, so 2 rows survive with r1's first two slots Covered.+func TestGridSpanAndTrailingTrim(t *testing.T) {+ td, _ := ParseTableHTML(`<table>+<tr><td colspan="2" rowspan="2">big</td><td>c</td></tr>+<tr><td>d</td></tr>+</table>`)+ // The big cell covers (r0c0,r0c1,r1c0,r1c1); only c and d are real in r0/r1.+ // No fully-covered trailing row here (r1 has d at c2), so 2 rows survive.+ if len(td.Grid) != 2 {+ t.Fatalf("rows=%d want 2 (trailing covered row trimmed if any)", len(td.Grid))+ }+ if td.Grid[0][0].Text != "big" || td.Grid[0][0].ColSpan != 2 || td.Grid[0][0].RowSpan != 2 {+ t.Fatalf("big origin wrong: %+v", td.Grid[0][0])+ }+ // r1 should be all covered in its first two slots and carry d at c2.+ if !td.Grid[1][0].Covered || !td.Grid[1][1].Covered {+ t.Fatalf("r1 covered slots wrong: %+v", td.Grid[1])+ }+ if td.Grid[1][2].Text != "d" {+ t.Fatalf("r1 c2 should be d: %+v", td.Grid[1][2])+ }+}++// overlapping spans must clamp instead of double-counting: a later origin that+// would land on an already-covered position is pushed past it and keeps a span+// consistent with what it owns (mirrors AnyDoc's GridBuilder clamp).+func TestGridOverlapClamps(t *testing.T) {+ td, _ := ParseTableHTML(`<table>+<tr><td rowspan="3">tall</td><td>a</td></tr>+<tr><td colspan="2">wide</td></tr>+</table>`)+ // r0: tall(rspan3) + a ; r1: covered + wide(cspan2) -> width 3+ // The wide cell at r1 would cover c1(r1) and c2(r1); that is fine (c1 is+ // covered by tall only at r0/r1/r2, c2 has nothing). Assert consistency:+ // every Covered slot points back to an origin within bounds.+ for r, row := range td.Grid {+ for c, cell := range row {+ if cell.Covered {+ // The back-pointer must name a slot that exists in the+ // grid: OriginRow is above (rowspan) or the same row+ // (colspan), and OriginCol is a real column of that row.+ if cell.OriginRow < 0 || cell.OriginRow >= len(td.Grid) ||+ cell.OriginCol < 0 || cell.OriginCol >= len(row) {+ t.Fatalf("covered slot (%d,%d) back-pointer out of bounds: %+v", r, c, cell)+ }+ }+ }+ }+}++// malformed span attributes (0, negative, non-numeric) are clamped to 1 rather+// than producing huge or negative grids.+func TestGridBadSpanAttrsClamped(t *testing.T) {+ td, _ := ParseTableHTML(`<table>+<tr><td colspan="0">a</td><td colspan="-3">b</td><td rowspan="abc">c</td></tr>+</table>`)+ row := td.Grid[0]+ if len(row) != 3 {+ t.Fatalf("width=%d want 3 (all spans clamped to 1)", len(row))+ }+ for _, c := range row {+ if c.ColSpan != 1 || c.RowSpan != 1 || c.Covered {+ t.Fatalf("clamped cell must be 1x1 uncovered: %+v", c)+ }+ }+ if got := gridTexts(td); !reflect.DeepEqual(got, [][]string{{"a", "b", "c"}}) {+ t.Fatalf("texts=%v", got)+ }+}++// A colspan origin's engulfed slot becomes a Covered marker, so the row stays+// dense (no empty cell) and its width matches rows that have no spans. Ragged+// (genuinely shorter) source rows are NOT padded with empty cells.+func TestGridColspanFillsCoveredSlot(t *testing.T) {+ td, _ := ParseTableHTML(`<table>+<tr><td colspan="2">wide</td></tr>+<tr><td>p</td><td>q</td></tr>+</table>`)+ if len(td.Grid[0]) != len(td.Grid[1]) {+ t.Fatalf("rows not uniform width: %d vs %d", len(td.Grid[0]), len(td.Grid[1]))+ }+ if len(td.Grid[0]) != 2 {+ t.Fatalf("width=%d want 2", len(td.Grid[0]))+ }+ if !td.Grid[0][1].Covered {+ t.Fatalf("r0c1 should be covered: %+v", td.Grid[0][1])+ }+}++// RenderTableHTML for a merged table emits colspan/rowspan on the origin and+// skips covered slots; the markup must round-trip back to the same grid.+func TestRenderMergedEmitsSpanAttrs(t *testing.T) {+ td, _ := ParseTableHTML(`<table><tr><td colspan="2">wide</td><td>narrow</td></tr></table>`)+ h := RenderTableHTML(td)+ if !strings.Contains(h, `colspan="2"`) {+ t.Fatalf("merged render missing colspan: %s", h)+ }+ if strings.Contains(h, `colspan="1"`) {+ t.Fatalf("1x1 cell must not emit colspan attr: %s", h)+ }+ // round-trip+ back, err := ParseTableHTML(h)+ if err != nil || back == nil {+ t.Fatalf("round-trip: %v", err)+ }+ if !reflect.DeepEqual(gridTexts(back), gridTexts(td)) {+ t.Fatalf("round-trip texts differ: %v vs %v", gridTexts(back), gridTexts(td))+ }+ if back.Grid[0][0].ColSpan != 2 {+ t.Fatalf("round-trip lost colspan: %+v", back.Grid[0][0])+ }+}++// RenderTableHTML -> ParseTableHTML round-trip must preserve both colspan and+// rowspan geometry (not just the text view), so a downstream consumer that+// renders or indexes the grid from the re-serialized markup sees the same+// merge geometry.+func TestRenderRoundTripPreservesSpans(t *testing.T) {+ td, _ := ParseTableHTML(`<table>+<tr><td colspan="2" rowspan="2">big</td><td>c</td></tr>+<tr><td>d</td></tr>+</table>`)+ h := RenderTableHTML(td)+ back, err := ParseTableHTML(h)+ if err != nil || back == nil {+ t.Fatalf("round-trip: %v", err)+ }+ if back.Grid[0][0].ColSpan != 2 || back.Grid[0][0].RowSpan != 2 {+ t.Fatalf("round-trip lost spans: %+v", back.Grid[0][0])+ }+ if !back.Grid[1][0].Covered || !back.Grid[1][1].Covered {+ t.Fatalf("round-trip lost covered continuation: %+v", back.Grid[1])+ }+ if got := gridTexts(back); !reflect.DeepEqual(got, gridTexts(td)) {+ t.Fatalf("round-trip texts differ: %v vs %v", gridTexts(back), gridTexts(td))+ }+}++// A non-merged table renders byte-for-byte identical to the legacy markup so+// existing downstream consumers see no change.+func TestRenderNonMergedByteIdentical(t *testing.T) {+ td := &entity.TableData{+ Grid: [][]entity.TableCell{+ {{Text: "Name"}, {Text: "Age"}},+ {{Text: "Alice"}, {Text: "30"}},+ },+ HeaderRows: 1,+ }+ got := RenderTableHTML(td)+ want := "<table>\n<tr><th>Name</th><th>Age</th></tr>\n" ++ "<tr><td>Alice</td><td>30</td></tr>\n</table>\n"+ if got != want {+ t.Fatalf("render mismatch:\n got=%q\nwant=%q", got, want)+ }+}++// RenderTableText strips tags; a merged table yields the same stripped text as+// the equivalent non-merged table (text-level output is unchanged by spans).+func TestRenderTextMergedSameAsNonMerged(t *testing.T) {+ merged, _ := ParseTableHTML(`<table><tr><td colspan="2">wide</td><td>narrow</td></tr></table>`)+ plain, _ := ParseTableHTML(`<table><tr><td>wide</td><td>narrow</td></tr></table>`)+ if RenderTableText(merged) != RenderTableText(plain) {+ t.Fatalf("merged text %q != plain text %q", RenderTableText(merged), RenderTableText(plain))+ }+}++// A colspan far beyond any real table must be clamped to the HTML-standard+// maximum (1000, the limit browsers enforce) before buildGrid sees it, so a+// malformed or hostile attribute can never drive a multi-gigabyte allocation+// or hang the parser. The grid still stays dense: the origin claims 1000+// columns, the engulfed positions become Covered markers, and the text view+// is unchanged.+func TestGridHugeColSpanClampedToSpecMax(t *testing.T) {+ td, _ := ParseTableHTML(`<table><tr><td colspan="100000000">wide</td><td>narrow</td></tr></table>`)+ if len(td.Grid) != 1 {+ t.Fatalf("rows=%d want 1", len(td.Grid))+ }+ row := td.Grid[0]+ if row[0].Text != "wide" || row[0].ColSpan != 1000 {+ t.Fatalf("origin ColSpan=%d want 1000 (HTML standard max): %+v", row[0].ColSpan, row[0])+ }+ if len(row) != 1001 {+ t.Fatalf("row width=%d want 1001 (origin + 999 covered + narrow)", len(row))+ }+ for c := 1; c <= 999; c++ {+ if !row[c].Covered || row[c].OriginCol != 0 {+ t.Fatalf("engulfed slot %d must be Covered pointing at origin col 0: %+v", c, row[c])+ }+ }+ if row[1000].Text != "narrow" || row[1000].ColSpan != 1 {+ t.Fatalf("trailing cell wrong: %+v", row[1000])+ }+ if got := gridTexts(td); !reflect.DeepEqual(got, [][]string{{"wide", "narrow"}}) {+ t.Fatalf("texts=%v want [[wide narrow]]", got)+ }+}++// A rowspan beyond the HTML-standard maximum (65534) must be clamped too; in a+// small table the surviving extent is additionally bounded by the number of+// rows below the origin, so the stored span always names rows that exist.+func TestGridHugeRowSpanClamped(t *testing.T) {+ td, _ := ParseTableHTML(`<table>+<tr><td rowspan="999999">A</td><td>x</td></tr>+<tr><td>y</td></tr>+<tr><td>z</td></tr>+</table>`)+ if len(td.Grid) != 3 {+ t.Fatalf("rows=%d want 3", len(td.Grid))+ }+ if td.Grid[0][0].RowSpan != 3 {+ t.Fatalf("RowSpan=%d want 3 (clamped to 65534, then to grid rows)", td.Grid[0][0].RowSpan)+ }+ if !td.Grid[1][0].Covered || !td.Grid[2][0].Covered {+ t.Fatalf("continuation rows must stay Covered: %+v / %+v", td.Grid[1][0], td.Grid[2][0])+ }+ if got := gridTexts(td); !reflect.DeepEqual(got, [][]string{{"A", "x"}, {"y"}, {"z"}}) {+ t.Fatalf("texts=%v", got)+ }+}++// The renderer must emit the clamped span, never the raw attribute value, so a+// pathological table round-trips without leaking a huge span attribute.+func TestRenderHugeSpanRoundTripBounded(t *testing.T) {+ td, _ := ParseTableHTML(`<table><tr><td colspan="100000000">wide</td><td>narrow</td></tr></table>`)+ h := RenderTableHTML(td)+ if strings.Contains(h, "100000000") {+ t.Fatalf("render leaked the raw span attribute: %s", h)+ }+ if !strings.Contains(h, `colspan="1000"`) {+ t.Fatalf("render must emit the clamped colspan: %s", h)+ }+ back, err := ParseTableHTML(h)+ if err != nil || back == nil {+ t.Fatalf("round-trip parse: %v", err)+ }+ if back.Grid[0][0].ColSpan != 1000 {+ t.Fatalf("round-trip lost the clamped span: %+v", back.Grid[0][0])+ }+ if !reflect.DeepEqual(gridTexts(back), gridTexts(td)) {+ t.Fatalf("round-trip texts differ: %v vs %v", gridTexts(back), gridTexts(td))+ }+}diff --git a/internal/parser/tableutil/table_html.go b/internal/parser/tableutil/table_html.goindex 1ac54b5726..3006714fe4 100644--- a/internal/parser/tableutil/table_html.go+++ b/internal/parser/tableutil/table_html.go@@ -21,6 +21,7 @@ package tableutilimport ("regexp"+ "strconv""strings""golang.org/x/net/html"@@ -33,6 +34,13 @@ import (// is a header row, <br> becomes a newline, inert elements// (script/style/noscript/template) are skipped, and a nested table's text is// folded into the enclosing cell (the nested <tr> does NOT become a row).+//+// Unlike the legacy extractor, ParseTableHTML now honours colspan/rowspan:+// a merged cell keeps its text on the origin slot (ColSpan/RowSpan > 1) and+// the engulfed positions become Covered markers. This recovers merge geometry+// that the legacy code discarded. For tables without merges the resulting grid+// is identical to the legacy Rows, so CellTexts (the consumer view) is+// unchanged.func ParseTableHTML(s string) (*entity.TableData, error) {lower := strings.ToLower(s)if strings.Contains(lower, "<tr") && !strings.Contains(lower, "<table") {@@ -42,7 +50,7 @@ func ParseTableHTML(s string) (*entity.TableData, error) {if err != nil {return nil, err}- var rows [][]string+ var rows [][]cellInputvar headerFlags []boolvar caption stringvar walk func(*html.Node)@@ -55,18 +63,25 @@ func ParseTableHTML(s string) (*entity.TableData, error) {return}if isHTMLElement(n, "tr") {- var cells []string+ var cells []cellInputanyTh := falsefor c := n.FirstChild; c != nil; c = c.NextSibling {if isHTMLElement(c, "td") || isHTMLElement(c, "th") {if isHTMLElement(c, "th") {anyTh = true}- cells = append(cells, cellText(c))+ cells = append(cells, cellInput{+ Text: cellText(c),+ ColSpan: attrInt(c, "colspan", maxColSpan),+ RowSpan: attrInt(c, "rowspan", maxRowSpan),+ })}}rows = append(rows, cells)headerFlags = append(headerFlags, anyTh)+ // Do not recurse into the row's cells: a nested <table> inside a+ // cell is folded into that cell's text by cellText, not parsed as+ // a separate row.return}for c := n.FirstChild; c != nil; c = c.NextSibling {@@ -81,16 +96,249 @@ func ParseTableHTML(s string) (*entity.TableData, error) {if len(rows) > 0 && headerCount == 0 {headerCount = 1}- return &entity.TableData{Rows: rows, HeaderRows: headerCount, Caption: caption}, nil+ return &entity.TableData{+ Grid: buildGrid(rows),+ HeaderRows: headerCount,+ Caption: caption,+ }, nil+}++// cellInput is one cell as collected from a <tr> before grid normalization.+type cellInput struct {+ Text string+ ColSpan int+ RowSpan int+}++// origin names the slot whose span produced a Covered marker.+type origin struct {+ row, col int+}++// buildGrid normalizes ragged cell rows into a dense grid where every logical+// position appears exactly once. A merged cell (ColSpan/RowSpan > 1) occupies+// its origin slot and the engulfed positions become Covered markers pointing+// back at the origin; overlapping spans clamp rather than double-count. A row+// that is genuinely shorter than the widest row is NOT padded with empty cells+// (a real gap stays a shorter row), but a row can be extended by Covered+// markers produced by an ancestor's rowspan so the grid stays dense. Trailing+// rows that carry no cell at all (neither a real cell nor a Covered marker)+// are trimmed. Spans are clamped to the surviving grid so a stored+// ColSpan/RowSpan never exceeds the positions it actually owns (mirrors+// AnyDoc's GridBuilder).+//+// The grid width is computed by a pre-pass that walks the same placement rules+// the main grid build uses, so it counts columns an ancestor rowspan still+// occupies in a later row. If the width only summed each row's own colspan+// (the legacy computation), an ancestor rowspan would push a later row's cells+// past the width and its trailing real cells would be silently dropped by the+// column guard — both losing data and regressing the legacy Rows text view.+func buildGrid(rows [][]cellInput) [][]entity.TableCell {+ // Pre-pass: discover the columns that ancestor rowspans occupy in each+ // later row (recorded in `covered`) and the true grid width. The placement+ // here mirrors the main loop below exactly so the width matches what the+ // main loop will actually reach.+ covered := make([]map[int]origin, len(rows))+ width := 0+ for r, row := range rows {+ col := 0+ ci := 0+ for {+ if _, ok := covered[r][col]; ok {+ // Ancestor rowspan owns this column here; advance past it.+ col+++ } else if ci < len(row) {+ c := row[ci]+ ci+++ cs := spanOf(c.ColSpan)+ rs := spanOf(c.RowSpan)+ col += cs+ if rs > 1 {+ for dr := 1; dr < rs; dr++ {+ rr := r + dr+ if rr >= len(covered) {+ break+ }+ if covered[rr] == nil {+ covered[rr] = map[int]origin{}+ }+ for dc := 0; dc < cs; dc++ {+ covered[rr][col-1-dc] = origin{r, col - cs}+ }+ }+ }+ } else {+ // No real cell at this column. An ancestor may still own+ // columns further right, so jump to the next covered column+ // instead of stopping; otherwise the row is exhausted.+ next := nextCoveredCol(covered[r], col)+ if next < 0 {+ break+ }+ col = next+ }+ if col > width {+ width = col+ }+ }+ }+ if width == 0 {+ return nil+ }+ grid := make([][]entity.TableCell, len(rows))+ for r, row := range rows {+ // Size this row to its OWN column count, not the whole-table width: a+ // wide row elsewhere must not make every short row allocate megabytes+ // of unused backing capacity. The exact cell count for the row is the+ // sum of each real cell's colspan plus the ancestor rowspan markers+ // already recorded in covered[r] — distinct positions, so the total is+ // exact and append never over-grows the slice.+ realCols := 0+ for _, c := range row {+ realCols += spanOf(c.ColSpan)+ }+ grid[r] = make([]entity.TableCell, 0, realCols+len(covered[r]))+ col := 0+ ci := 0+ for col < width {+ if o, ok := covered[r][col]; ok {+ grid[r] = append(grid[r], entity.TableCell{+ Covered: true,+ OriginRow: o.row,+ OriginCol: o.col,+ })+ col+++ continue+ }+ if ci >= len(row) {+ // No more real cells in this source row, but an ancestor's+ // rowspan may still own columns further right: keep advancing+ // so those Covered markers are emitted instead of stopping+ // early (which would drop a rowspan continuation and break the+ // dense-grid invariant).+ col+++ continue+ }+ c := row[ci]+ ci+++ cs := spanOf(c.ColSpan)+ rs := spanOf(c.RowSpan)+ grid[r] = append(grid[r], entity.TableCell{+ Text: c.Text,+ ColSpan: cs,+ RowSpan: rs,+ })+ // Mark the same-row positions this origin engulfs as Covered+ // markers so the grid stays dense and exactly-once (mirrors+ // AnyDoc's GridBuilder, which fills every covered slot). The+ // back-pointer names the origin slot (col), not the engulfed+ // position (col+dc). Ancestor rowspan continuation markers for+ // later rows were already registered in the pre-pass above.+ for dc := 1; dc < cs; dc++ {+ grid[r] = append(grid[r], entity.TableCell{+ Covered: true,+ OriginRow: r,+ OriginCol: col,+ })+ }+ col += cs+ }+ // Rows are NOT padded to a uniform width: a row that genuinely has+ // fewer cells than the widest row keeps its actual (ragged) length so+ // CellTexts() reproduces the legacy row-major Rows exactly. Only+ // merged cells (colspan/rowspan) introduce Covered markers; a real+ // gap is simply a shorter row, which both the consumer text view and+ // HTML render handle without padding artifacts.+ }+ // Clamp stored spans to the grid so every claimed position is backed.+ for r := range grid {+ w := len(grid[r])+ for c := range grid[r] {+ cell := &grid[r][c]+ if cell.Covered {+ continue+ }+ if cell.RowSpan > len(grid)-r {+ cell.RowSpan = len(grid) - r+ }+ if cell.ColSpan > w-c {+ cell.ColSpan = w - c+ }+ }+ }+ // Trim trailing rows that are completely empty: a row with no real cell+ // and no Covered marker carries no information. A row that only carries+ // Covered markers is a genuine rowspan continuation and MUST stay so the+ // grid stays dense and every origin's RowSpan remains backed by a real+ // row; dropping it would make the claimed span exceed the available rows.+ for len(grid) > 0 && len(grid[len(grid)-1]) == 0 {+ grid = grid[:len(grid)-1]+ }+ return grid+}++// spanOf clamps a span extent to a minimum of 1.+func spanOf(v int) int {+ if v < 1 {+ return 1+ }+ return v+}++// nextCoveredCol returns the smallest column key in m strictly greater than col,+// or -1 if no such key exists. It is used by buildGrid's width pre-pass to jump+// past empty columns to a column still owned by an ancestor rowspan.+func nextCoveredCol(m map[int]origin, col int) int {+ next := -1+ for k := range m {+ if k > col && (next < 0 || k < next) {+ next = k+ }+ }+ return next+}++// maxColSpan and maxRowSpan are the HTML standard's own limits: browsers+// clamp a colspan above 1000 and a rowspan above 65534, so the grid must+// apply the same limits. Without an upper bound a malformed or hostile+// attribute (e.g. colspan="100000000") would reach buildGrid and drive a+// multi-gigabyte allocation (row capacity, covered-slot insertion) inside+// the ingestion worker.+const (+ maxColSpan = 1000+ maxRowSpan = 65534+)++// attrInt reads a non-negative integer HTML attribute, clamping 0, negative,+// non-numeric, and above-max values (to 1 and max respectively) so a+// malformed colspan/rowspan never produces a huge or negative grid.+func attrInt(n *html.Node, name string, max int) int {+ for _, a := range n.Attr {+ if strings.EqualFold(a.Key, name) {+ v, err := strconv.Atoi(strings.TrimSpace(a.Val))+ if err != nil || v < 1 {+ return 1+ }+ if v > max {+ return max+ }+ return v+ }+ }+ return 1}// RenderTableHTML emits a deterministic <table> markup for a TableData. It is// used only as a test oracle / round-trip helper and is NOT part of the// persisted contract (producers emit TableData, not HTML). The emitted shape// matches the legacy spreadsheet producer's renderSpreadsheetTable exactly-// (including the inter-row newlines) so that RenderTableText reproduces the-// legacy htmlTableRE-stripped text byte for byte.+// (including the inter-row newlines) for 1x1 grids, so RenderTableText+// reproduces the legacy htmlTableRE-stripped text byte for byte. For merged+// tables it emits colspan/rowspan on the origin slot and skips Covered slots.func RenderTableHTML(td *entity.TableData) string {+ if td == nil {+ return ""+ }var b strings.Builderb.WriteString("<table>")if td.Caption != "" {@@ -99,18 +347,31 @@ func RenderTableHTML(td *entity.TableData) string {b.WriteString("</caption>")}b.WriteString("\n")- for i, row := range td.Rows {- b.WriteString("<tr>")+ for i, row := range td.Grid {isHeader := i < td.HeaderRows+ b.WriteString("<tr>")for _, cell := range row {+ if cell.Covered {+ continue+ }tag := "td"if isHeader {tag = "th"}b.WriteString("<")b.WriteString(tag)+ if cell.ColSpan > 1 {+ b.WriteString(" colspan=\"")+ b.WriteString(strconv.Itoa(cell.ColSpan))+ b.WriteString("\"")+ }+ if cell.RowSpan > 1 {+ b.WriteString(" rowspan=\"")+ b.WriteString(strconv.Itoa(cell.RowSpan))+ b.WriteString("\"")+ }b.WriteString(">")- b.WriteString(html.EscapeString(cell))+ b.WriteString(html.EscapeString(cell.Text))b.WriteString("</")b.WriteString(tag)b.WriteString(">")@@ -123,11 +384,12 @@ func RenderTableHTML(td *entity.TableData) string {return b.String()}-// tableTagRE mirrors the legacy tokenizer stripping pattern-// (htmlTableRE in internal/ingestion/component/tokenizer.go) so that the-// plaintext a chunker writes for an embed-only path matches the legacy-// tag-stripped output exactly.-var tableTagRE = regexp.MustCompile(`</?(table|td|caption|tr|th)( [^<>]{0,12})?>`)+// tableTagRE strips table markup tags (with any attributes) so the plaintext a+// chunker writes for an embed-only path matches the legacy tag-stripped output.+// The pattern tolerates arbitrary attribute lists (including colspan/rowspan)+// so merged-table markup is stripped completely rather than left partly in the+// text.+var tableTagRE = regexp.MustCompile(`</?(?:table|td|caption|tr|th)(?:\s[^<>]*)?>`)// RenderTableText renders a TableData to plain text with no markup tags, for// the rare embed path that receives a table without a downstream TableChunker.diff --git a/internal/parser/tableutil/table_html_test.go b/internal/parser/tableutil/table_html_test.goindex abfea73ad6..f8c0060d8e 100644--- a/internal/parser/tableutil/table_html_test.go+++ b/internal/parser/tableutil/table_html_test.go@@ -44,7 +44,7 @@ func TestParseTableHTMLHappyPath(t *testing.T) {{"Alice", "30"},{"Bob", "25"},}- assertRowsEqual(t, td.Rows, want)+ assertRowsEqual(t, td.CellTexts(), want)}func TestParseTableHTMLMultipleHeaderRows(t *testing.T) {@@ -67,8 +67,8 @@ func TestParseTableHTMLNoThTreatsFirstRowAsHeader(t *testing.T) {func TestParseTableHTMLEmpty(t *testing.T) {td := mustParse(t, `<table></table>`)- if len(td.Rows) != 0 {- t.Fatalf("Rows = %v, want empty", td.Rows)+ if len(td.CellTexts()) != 0 {+ t.Fatalf("Rows = %v, want empty", td.CellTexts())}if td.HeaderRows != 0 {t.Fatalf("HeaderRows = %d, want 0", td.HeaderRows)@@ -78,7 +78,7 @@ func TestParseTableHTMLEmpty(t *testing.T) {func TestParseTableHTMLTrimsCellWhitespace(t *testing.T) {html := `<table><tr><td> spaced </td></tr></table>`td := mustParse(t, html)- if got := td.Rows[0][0]; got != "spaced" {+ if got := td.CellTexts()[0][0]; got != "spaced" {t.Fatalf("cell = %q, want %q", got, "spaced")}}@@ -86,7 +86,7 @@ func TestParseTableHTMLTrimsCellWhitespace(t *testing.T) {func TestParseTableHTMLBrBecomesNewline(t *testing.T) {html := `<table><tr><td>line1<br>line2</td></tr></table>`td := mustParse(t, html)- if got := td.Rows[0][0]; got != "line1\nline2" {+ if got := td.CellTexts()[0][0]; got != "line1\nline2" {t.Fatalf("cell = %q, want %q", got, "line1\nline2")}}@@ -96,7 +96,7 @@ func TestParseTableHTMLInertElementsIgnored(t *testing.T) {html := `<table><tr><td>keep<script>alert(1)</script>middle` +`<style>.x{}</style>end</td></tr></table>`td := mustParse(t, html)- got := td.Rows[0][0]+ got := td.CellTexts()[0][0]if strings.Contains(got, "alert") || strings.Contains(got, ".x{}") {t.Fatalf("inert content leaked into cell: %q", got)}@@ -110,10 +110,10 @@ func TestParseTableHTMLNestedTableFoldedIntoCell(t *testing.T) {td := mustParse(t, html)// Outer <tr> yields exactly one row with one cell; the nested <tr> must// NOT become a separate row.- if len(td.Rows) != 1 {- t.Fatalf("len(Rows) = %d, want 1 (nested tr not a row)", len(td.Rows))+ if len(td.CellTexts()) != 1 {+ t.Fatalf("len(Rows) = %d, want 1 (nested tr not a row)", len(td.CellTexts()))}- got := td.Rows[0][0]+ got := td.CellTexts()[0][0]if !strings.Contains(got, "outer") || !strings.Contains(got, "inner") {t.Fatalf("nested table text not folded: %q", got)}@@ -126,26 +126,64 @@ func TestParseTableHTMLBareTrWrapped(t *testing.T) {// Legacy behaviour: a bare <tr> fragment without <table> still parses.html := `<tr><td>a</td><td>b</td></tr>`td := mustParse(t, html)- assertRowsEqual(t, td.Rows, [][]string{{"a", "b"}})+ assertRowsEqual(t, td.CellTexts(), [][]string{{"a", "b"}})}-func TestParseTableHTMLColspanIgnoredLikeLegacy(t *testing.T) {- // Legacy tableRowsWithHeader ignores colspan/rowspan and just collects- // each <td>/<th> as one cell; a spanned cell keeps its own text.+// A huge colspan on one row drives the grid width up, but each row's backing+// slice must be sized to its OWN column count, not the whole-table width.+// buildGrid used to pre-allocate every row to the full table width, so a small+// HTML input with one wide row could allocate megabytes for every short row+// that followed. The dense grid stays ragged (short rows are not padded), so a+// short row's capacity must equal its actual length.+func TestParseTableHTMLRowCapacityNotFullWidth(t *testing.T) {+ td := mustParse(t, "<table><tr><td colspan=\"1000\">WIDE</td></tr><tr><td>a</td><td>b</td></tr></table>")+ if len(td.Grid) != 2 {+ t.Fatalf("got %d rows, want 2", len(td.Grid))+ }+ // Row 1 is short (two 1x1 cells, no ancestor rowspan): its length is 2 and+ // its backing capacity must equal that, not the full table width (1000).+ if len(td.Grid[1]) != 2 {+ t.Fatalf("row1 len = %d, want 2", len(td.Grid[1]))+ }+ if cap(td.Grid[1]) != len(td.Grid[1]) {+ t.Fatalf("row1 cap = %d, want %d (no full-width pre-allocation)", cap(td.Grid[1]), len(td.Grid[1]))+ }+}++func TestParseTableHTMLCapturesColspan(t *testing.T) {+ // A colspan is captured, not discarded: the wide cell keeps its text on the+ // origin slot (ColSpan=2) and the engulfed position becomes a Covered+ // marker, but the text view still yields the two real cell texts.html := `<table><tr><td colspan="2">wide</td><td>narrow</td></tr></table>`td := mustParse(t, html)- assertRowsEqual(t, td.Rows, [][]string{{"wide", "narrow"}})+ row := td.Grid[0]+ if len(row) != 3 {+ t.Fatalf("grid width = %d, want 3 (origin + covered + narrow)", len(row))+ }+ if row[0].Text != "wide" || row[0].ColSpan != 2 || row[0].RowSpan != 1 {+ t.Fatalf("origin slot wrong: %+v", row[0])+ }+ if !row[1].Covered || row[1].OriginCol != 0 {+ t.Fatalf("engulfed slot must be Covered pointing at col 0: %+v", row[1])+ }+ if row[2].Text != "narrow" || row[2].ColSpan != 1 {+ t.Fatalf("narrow slot wrong: %+v", row[2])+ }+ assertRowsEqual(t, td.CellTexts(), [][]string{{"wide", "narrow"}})}func TestRenderTableHTMLRoundTrip(t *testing.T) {td := &entity.TableData{- Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}},+ Grid: [][]entity.TableCell{+ {{Text: "Name"}, {Text: "Age"}},+ {{Text: "Alice"}, {Text: "30"}},+ },HeaderRows: 1,Caption: "People",}rendered := RenderTableHTML(td)back := mustParse(t, rendered)- assertRowsEqual(t, back.Rows, td.Rows)+ assertRowsEqual(t, back.CellTexts(), td.CellTexts())if back.HeaderRows != td.HeaderRows {t.Fatalf("HeaderRows round-trip = %d, want %d", back.HeaderRows, td.HeaderRows)}@@ -156,7 +194,10 @@ func TestRenderTableHTMLRoundTrip(t *testing.T) {func TestRenderTableTextHasNoTags(t *testing.T) {td := &entity.TableData{- Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}},+ Grid: [][]entity.TableCell{+ {{Text: "Name"}, {Text: "Age"}},+ {{Text: "Alice"}, {Text: "30"}},+ },HeaderRows: 1,Caption: "People",}