Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Render table chunks as HTML on Text at the chunker (#20918)

xugangqiang9fe7fec2c606
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index ddf85ccd0b..4dcb080274 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -994,7 +994,13 @@ func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []
}
parts, ranges, headerRows := splitLargeTable(td, c.param.ChunkTokenSize, tokenizeStr)
if ranges == nil {
- return []schema.ChunkDoc{cloneChunkDoc(unit)}
+ // The whole table fits one chunk: render it as HTML on Text so the
+ // persisted content and the id (hash of Text) stay byte-consistent and
+ // the UI can display a real table. TableData is dropped, never indexed.
+ out := cloneChunkDoc(unit)
+ out.Text = tableutil.RenderTableHTML(td)
+ out.TableData = nil
+ return []schema.ChunkDoc{out}
}
var matrix [][]float64
// Spreadsheet items only: PDF items write layout boxes into the same
@@ -1021,11 +1027,12 @@ func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []
out := make([]schema.ChunkDoc, 0, len(parts))
for i, part := range parts {
piece := cloneChunkDoc(unit)
- // Text carries a plain-text rendering (no markup) for the rare embed
- // path without a downstream TableChunker; the structured TableData is
- // the authoritative content for chunkers that read it.
- piece.Text = tableutil.RenderTableText(&part)
- piece.TableData = &part
+ // Text carries the sub-table rendered as HTML so the persisted content
+ // and the chunk id (hash of Text) stay byte-consistent and the UI can
+ // display a real table. The structured TableData is dropped here and is
+ // never serialized to the index.
+ piece.Text = tableutil.RenderTableHTML(&part)
+ piece.TableData = nil
piece.Positions = nil
lo := headerRows + ranges[i][0]
hi := headerRows + ranges[i][1]
diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.go
index e0196e274f..7a10f0c81a 100644
--- a/internal/ingestion/component/chunker/general_test.go
+++ b/internal/ingestion/component/chunker/general_test.go
@@ -76,9 +76,9 @@ func spreadsheetSegmentItem(sheet string, header []string, rows [][]string, shee
"header_rows": 1,
"caption": sheet,
},
- // Mirror what the chunker derives from TableData so text-bound
- // assertions (e.g. TokenChunker boundary tests) stay valid.
- "text": tableutil.RenderTableText(td),
+ // Mirror what the chunker derives from TableData (its HTML rendering)
+ // so text-bound assertions (e.g. TokenChunker boundary tests) stay valid.
+ "text": tableutil.RenderTableHTML(td),
"doc_type_kwd": "table",
"ck_type": "table",
"sheet_index": sheetIndex,
@@ -1086,19 +1086,21 @@ func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testing
}
covered := 0
for i, chunk := range chunks {
- // New contract: Text is the plain rendering (no markup); the
- // replicated header lives in the structured TableData, and positions
- // are sliced to this sub-table when the matrix is row-aligned.
- tbl, ok := chunk["table"].(map[string]any)
- if !ok {
- t.Fatalf("chunk %d missing structured TableData", i)
+ // New contract: Text is the sub-table rendered as HTML (replicated
+ // header + this part's data rows); the structured TableData is dropped
+ // and never indexed; positions are sliced to this sub-table when the
+ // matrix is row-aligned.
+ if _, ok := chunk["table"]; ok {
+ t.Fatalf("chunk %d still carries structured table", i)
}
- tblGrid, _ := tbl["grid"].([]any)
- if len(tblGrid) < 2 {
- t.Fatalf("chunk %d has too few rows: %v", i, tblGrid)
+ text, _ := chunk["text"].(string)
+ if !strings.Contains(text, "<table>") || !strings.Contains(text, "<th>Name</th>") {
+ t.Fatalf("chunk %d text lost the header cell: %q", i, text)
}
- if !strings.Contains(chunk["text"].(string), "Name") {
- t.Errorf("chunk %d text lost the header cell: %q", i, chunk["text"])
+ // Single-column table: each data row is one <td>value-…</td> cell.
+ dataRows := strings.Count(text, "<td>value-")
+ if dataRows == 0 {
+ t.Fatalf("chunk %d has no data rows: %q", i, text)
}
var matrix [][]float64
raw, _ := json.Marshal(chunk["positions"])
@@ -1113,7 +1115,6 @@ func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testing
if matrix[0][1] != 1 {
t.Errorf("chunk %d header tuple = %v, want sheet 1 row 1", i, matrix[0])
}
- dataRows := len(tblGrid) - 1
if dataRows != len(matrix)-1 {
t.Errorf("chunk %d data rows %d != position tuples %d", i, dataRows, len(matrix)-1)
}
diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.go
index 787f3900d8..1e674a1289 100644
--- a/internal/ingestion/component/chunker/table.go
+++ b/internal/ingestion/component/chunker/table.go
@@ -216,13 +216,16 @@ func tableChunkRows(td *entity.TableData) [][]string {
return out
}
-// Non-table payloads pass through unchanged. It reads the structured
-// TableData contract that the parser emits on ChunkDoc.TableData; items that
-// do not carry TableData (e.g. pre-upgrade HTML markup or non-table records)
-// pass through as a single chunk. A table whose only row is the header keeps
-// the whole item as its single chunk: the header line is then the only
-// searchable representation. The structured TableData is cleared on every
-// emitted chunk so the whole table is not duplicated into the index.
+// expandHTMLTableRows turns one table payload into one chunk per data row.
+// It reads the structured TableData contract the parser emits on
+// ChunkDoc.TableData; items without TableData (pre-upgrade HTML markup or
+// non-table records) pass through unchanged as a single chunk. Every emitted
+// chunk carries its content rendered as an HTML <table> on Text, so the
+// persisted content and the chunk id (a hash of Text) describe the same bytes
+// and the UI/index can display a real table; the structured TableData is
+// dropped here and never serialized to the index. Header-only and fallback
+// chunks keep the whole table's HTML (merge geometry intact); per-row chunks
+// carry a flat header+row sub-table (merge-free, see cellGrid).
func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {
td := item.TableData
if td == nil {
@@ -231,7 +234,7 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {
rows, headerCount := tableChunkRows(td), td.HeaderRows
if len(rows) <= headerCount {
out := item
- out.Text = tableutil.RenderTableText(td)
+ out.Text = tableutil.RenderTableHTML(td)
out.TableData = nil
return []schema.ChunkDoc{out}
}
@@ -257,12 +260,19 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {
}
out := make([]schema.ChunkDoc, 0, len(rows)-headerCount)
for i, row := range rows[headerCount:] {
- text := tableRowRecordText(names, row)
- if text == "" {
+ // Skip empty rows (no cell content); the HTML render of an empty row
+ // would be a header-only table that duplicates the header-only chunk.
+ if tableRowRecordText(names, row) == "" {
continue
}
doc := item
- doc.Text = text
+ // Render this row as its own HTML table: every projected header row
+ // plus this data row, so the displayed table keeps the full header.
+ doc.Text = tableutil.RenderTableHTML(&entity.TableData{
+ Grid: cellGrid(append(append([][]string{}, rows[:headerCount]...), row)),
+ HeaderRows: headerCount,
+ Caption: td.Caption,
+ })
doc.TKNums = nil
doc.Positions = nil
doc.TableData = nil
@@ -275,13 +285,28 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {
}
if len(out) == 0 {
single := item
- single.Text = tableutil.RenderTableText(td)
+ single.Text = tableutil.RenderTableHTML(td)
single.TableData = nil
return []schema.ChunkDoc{single}
}
return out
}
+// cellGrid builds a merge-free dense Grid from row-major cell strings. It is
+// used to render a per-row sub-table (the projected header rows plus one data
+// row) as HTML; merged-cell geometry is intentionally not reconstructed here,
+// so the sub-table shows each cell plainly rather than with colspan/rowspan.
+func cellGrid(rows [][]string) [][]entity.TableCell {
+ grid := make([][]entity.TableCell, len(rows))
+ for i, r := range rows {
+ grid[i] = make([]entity.TableCell, len(r))
+ for j, c := range r {
+ grid[i][j] = entity.TableCell{Text: c}
+ }
+ }
+ return grid
+}
+
// tableDataOf returns the structured TableData for an item. The parser emits
// it on ChunkDoc.TableData; items without it (pre-upgrade HTML markup or
// non-table records) return nil and pass through unchanged. Re-chunk always
diff --git a/internal/ingestion/component/chunker/table_html_coverage_test.go b/internal/ingestion/component/chunker/table_html_coverage_test.go
new file mode 100644
index 0000000000..93fef6dc1d
--- /dev/null
+++ b/internal/ingestion/component/chunker/table_html_coverage_test.go
@@ -0,0 +1,238 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import (
+ "strings"
+ "testing"
+
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/tableutil"
+)
+
+// gridOf builds a TableData Grid from row-major cell strings (header + data).
+func gridOf(header []string, data ...[]string) [][]entity.TableCell {
+ rows := make([][]entity.TableCell, 0, 1+len(data))
+ rows = append(rows, cellsOf(header))
+ for _, d := range data {
+ rows = append(rows, cellsOf(d))
+ }
+ return rows
+}
+
+func cellsOf(vals []string) []entity.TableCell {
+ cells := make([]entity.TableCell, len(vals))
+ for i, v := range vals {
+ cells[i] = entity.TableCell{Text: v}
+ }
+ return cells
+}
+
+// --- A: the shared decode fallback renders structured tables as HTML ---
+
+// A structured table that arrives with only the TableData contract (no Text —
+// the office/xlsx producer path) must get an HTML <table> on Text from the
+// shared decode fallback, so every chunker that consumes it (token/group/
+// hierarchy/one/page/qa) stores and displays a real table, not plain text.
+func TestDecodeChunkerFromUpstreamRendersTableHTML(t *testing.T) {
+ up, err := decodeChunkerFromUpstream(map[string]any{
+ "name": "sheet.xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ tableItem(t, "<table><caption>Q1</caption><tr><th>Name</th><th>Age</th></tr><tr><td>Alice</td><td>30</td></tr></table>"),
+ },
+ })
+ if err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+ if len(up.JSONResult) != 1 {
+ t.Fatalf("got %d items, want 1", len(up.JSONResult))
+ }
+ text := up.JSONResult[0].Text
+ if !strings.Contains(text, "<table>") || !strings.Contains(text, "<td>Alice</td>") {
+ t.Fatalf("structured table not rendered to HTML on Text: %q", text)
+ }
+ if !strings.Contains(text, "<caption>Q1</caption>") {
+ t.Fatalf("caption not carried into HTML: %q", text)
+ }
+ // The fallback fires on the "chunks" payload too (re-chunk path).
+ upChunks, err := decodeChunkerFromUpstream(map[string]any{
+ "name": "sheet.xlsx",
+ "output_format": "chunks",
+ "chunks": []map[string]any{
+ tableItem(t, "<table><tr><th>k</th></tr><tr><td>v</td></tr></table>"),
+ },
+ })
+ if err != nil {
+ t.Fatalf("decode chunks: %v", err)
+ }
+ if len(upChunks.Chunks) != 1 || !strings.Contains(upChunks.Chunks[0].Text, "<table>") {
+ t.Fatalf("chunks payload table not rendered to HTML: %+v", upChunks.Chunks)
+ }
+}
+
+// End-to-end confirmation through a real non-specialized chunker: a structured
+// table (no Text) fed to the TokenChunker must surface an HTML table chunk.
+func TestTokenChunkerEmitsHTMLForStructuredTable(t *testing.T) {
+ component, err := NewTokenChunker(nil)
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "sheet.xlsx",
+ "file_type": "xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ tableItem(t, "<table><tr><th>Name</th><th>Age</th></tr><tr><td>Alice</td><td>30</td></tr></table>"),
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ found := false
+ for _, c := range outputChunks(t, out) {
+ if s, ok := c["text"].(string); ok && strings.Contains(s, "<table>") && strings.Contains(s, "<td>Alice</td>") {
+ found = true
+ }
+ }
+ if !found {
+ t.Fatalf("no chunk carries the HTML table text: %+v", out)
+ }
+}
+
+// --- B: the split budget is measured on the HTML that is actually stored ---
+
+// B1: the split budget is measured on the HTML that is actually stored. A
+// budget below the whole table's HTML cost forces a split, and every emitted
+// part's HTML must then fit the budget — which only holds when the budget is
+// counted on the HTML rendering, not the smaller clean projection (under a
+// clean-projection budget each part's HTML would overshoot).
+func TestSplitLargeTableBudgetsOnHTML(t *testing.T) {
+ data := make([][]string, 0, 10)
+ for i := 0; i < 10; i++ {
+ data = append(data, []string{"row-" + string(rune('a'+i)), "a moderately long padded value number " + string(rune('a'+i))})
+ }
+ td := &entity.TableData{Grid: gridOf([]string{"id", "value"}, data...), HeaderRows: 1}
+
+ whole := tokenizeStr(tableutil.RenderTableHTML(td))
+ budget := whole / 2
+
+ parts, ranges, headerRows := splitLargeTable(td, budget, tokenizeStr)
+ if len(parts) < 2 {
+ t.Fatalf("table did not split under the HTML budget: whole=%d budget=%d parts=%d", whole, budget, len(parts))
+ }
+ if headerRows != 1 {
+ t.Fatalf("headerRows = %d, want 1", headerRows)
+ }
+ covered := 0
+ for i, p := range parts {
+ if n := tokenizeStr(tableutil.RenderTableHTML(&p)); n > budget {
+ t.Fatalf("part %d HTML tokens %d exceed budget %d (budget must cover the stored HTML)", i, n, budget)
+ }
+ covered += ranges[i][1] - ranges[i][0]
+ }
+ if covered != len(data) {
+ t.Fatalf("parts cover %d data rows, want %d (rows lost or duplicated)", covered, len(data))
+ }
+}
+
+// B2: row integrity is preserved under the HTML budget — a single row whose
+// HTML alone exceeds the budget becomes its own whole chunk, never cut.
+func TestSplitLargeTableHTMLBudgetKeepsSingleOverBudgetRowWhole(t *testing.T) {
+ huge := strings.Repeat("x", 400)
+ td := &entity.TableData{Grid: gridOf([]string{"id", "blob"},
+ []string{"r1", "small"},
+ []string{"r2", huge},
+ []string{"r3", "small"},
+ ), HeaderRows: 1}
+
+ // Budget below the huge row's HTML but above the header + a small row.
+ small := tokenizeStr(tableutil.RenderTableHTML(&entity.TableData{Grid: gridOf([]string{"id", "blob"}, []string{"r1", "small"}), HeaderRows: 1}))
+ hugeRow := tokenizeStr(tableutil.RenderTableHTML(&entity.TableData{Grid: gridOf([]string{"id", "blob"}, []string{"r2", huge}), HeaderRows: 1}))
+ budget := (small + hugeRow) / 2
+ if budget >= hugeRow {
+ t.Fatalf("test setup: budget %d should be below the huge row HTML %d", budget, hugeRow)
+ }
+
+ parts, _, _ := splitLargeTable(td, budget, tokenizeStr)
+ if len(parts) < 2 {
+ t.Fatalf("expected the huge row to isolate into its own part, got %d parts", len(parts))
+ }
+ // The part carrying the huge row must contain it whole (its blob intact).
+ found := false
+ for _, p := range parts {
+ if n := tokenizeStr(tableutil.RenderTableHTML(&p)); n > budget {
+ // An over-budget part is only acceptable when it is the single
+ // huge row standing whole (row integrity wins over the budget).
+ if len(p.Grid) != 2 { // header + the one data row
+ t.Fatalf("over-budget part has %d rows, want the single huge row whole", len(p.Grid))
+ }
+ if p.Grid[1][1].Text != huge {
+ t.Fatalf("over-budget part lost the huge row's content")
+ }
+ found = true
+ }
+ }
+ if !found {
+ t.Fatalf("the huge row was not emitted as its own over-budget part")
+ }
+}
+
+// B3: span-safe cut is preserved under the HTML budget — a data-row cell whose
+// rowspan covers the next data row keeps that group in one part (never split
+// between the spanned rows).
+func TestSplitLargeTableHTMLBudgetKeepsRowspanGroup(t *testing.T) {
+ td := &entity.TableData{
+ HeaderRows: 1,
+ Grid: [][]entity.TableCell{
+ {{Text: "h1"}, {Text: "h2"}}, // header
+ {{Text: "grp", RowSpan: 2}, {Text: "a"}}, // data 0: grp spans data 0,1
+ {{Text: "", Covered: true, OriginRow: 1, OriginCol: 0}, {Text: "b"}}, // data 1: covered by grp
+ {{Text: "solo"}, {Text: "c"}}, // data 2
+ },
+ }
+ // Budget = the HTML of just the header + the first (spanning) data row, so
+ // the naive cut falls right after data 0; span-safe cut must pull data 1 in.
+ budget := tokenizeStr(tableutil.RenderTableHTML(&entity.TableData{
+ Grid: [][]entity.TableCell{td.Grid[0], td.Grid[1]},
+ HeaderRows: 1,
+ }))
+ parts, _, _ := splitLargeTable(td, budget, tokenizeStr)
+ if len(parts) < 2 {
+ t.Fatalf("test setup: expected a split, got %d parts", len(parts))
+ }
+ // Whatever the split, the spanned rows "a" and "b" must share a part.
+ partOf := map[string]int{}
+ for i, p := range parts {
+ html := tableutil.RenderTableHTML(&p)
+ if strings.Contains(html, "<td>a</td>") {
+ partOf["a"] = i
+ }
+ if strings.Contains(html, "<td>b</td>") {
+ partOf["b"] = i
+ }
+ }
+ if _, okA := partOf["a"]; !okA {
+ t.Fatalf("spanned row 'a' missing from all parts: %+v", parts)
+ }
+ if _, okB := partOf["b"]; !okB {
+ t.Fatalf("spanned row 'b' missing from all parts: %+v", parts)
+ }
+ if partOf["a"] != partOf["b"] {
+ t.Fatalf("rowspan group split across parts: a in %d, b in %d", partOf["a"], partOf["b"])
+ }
+}
diff --git a/internal/ingestion/component/chunker/table_html_text_test.go b/internal/ingestion/component/chunker/table_html_text_test.go
new file mode 100644
index 0000000000..917374dea1
--- /dev/null
+++ b/internal/ingestion/component/chunker/table_html_text_test.go
@@ -0,0 +1,339 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+ "testing"
+
+ "ragflow/internal/entity"
+)
+
+// generalTableChunksOf drives GeneralChunker.Invoke over structured TableData
+// items and returns the emitted chunk maps.
+func generalTableChunksOf(t *testing.T, params map[string]any, items ...map[string]any) []map[string]any {
+ t.Helper()
+ component, err := NewGeneralChunker(params)
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "sheet.csv",
+ "output_format": "json",
+ "json": items,
+ })
+ if err != nil {
+ t.Fatalf("GeneralChunker.Invoke: %v", err)
+ }
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok {
+ t.Fatalf("output chunks type = %T, want []map[string]any", out["chunks"])
+ }
+ return chunks
+}
+
+// --- Table chunker: structured TableData renders to HTML text ---
+
+func TestTableChunkerEmitsHTMLTextPerRow(t *testing.T) {
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "sheet.csv",
+ "output_format": "json",
+ "json": []map[string]any{
+ tableItem(t, "<table><tr><th>Name</th><th>Age</th></tr><tr><td>Alice</td><td>30</td></tr><tr><td>Bob</td><td>25</td></tr></table>"),
+ },
+ })
+ if len(chunks) != 2 {
+ t.Fatalf("got %d chunks, want 2 (one per data row)", len(chunks))
+ }
+ for i, want := range []string{"Alice", "Bob"} {
+ text, ok := chunks[i]["text"].(string)
+ if !ok {
+ t.Fatalf("chunk %d text missing", i)
+ }
+ if !strings.Contains(text, "<table>") || !strings.Contains(text, "<td>"+want+"</td>") {
+ t.Fatalf("chunk %d text not HTML with its row: %q", i, text)
+ }
+ // The header must ride along so the rendered row is a real table.
+ if !strings.Contains(text, "<th>Name</th>") || !strings.Contains(text, "<th>Age</th>") {
+ t.Fatalf("chunk %d HTML missing header cells: %q", i, text)
+ }
+ // The structured payload must not survive onto the persisted chunk.
+ if _, exists := chunks[i]["table"]; exists {
+ t.Fatalf("chunk %d still carries structured table: %#v", i, chunks[i]["table"])
+ }
+ }
+}
+
+func TestTableChunkerEmitsHTMLTextHeaderOnly(t *testing.T) {
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "sheet.csv",
+ "output_format": "json",
+ "json": []map[string]any{
+ tableItem(t, "<table><tr><th>Name</th><th>Age</th></tr></table>"),
+ },
+ })
+ if len(chunks) != 1 {
+ t.Fatalf("got %d chunks, want 1 (header-only)", len(chunks))
+ }
+ text, _ := chunks[0]["text"].(string)
+ if !strings.Contains(text, "<table>") || !strings.Contains(text, "<th>Name</th>") {
+ t.Fatalf("header-only chunk text not HTML: %q", text)
+ }
+ if _, exists := chunks[0]["table"]; exists {
+ t.Fatalf("header-only chunk still carries structured table")
+ }
+}
+
+func TestTableChunkerEmitsHTMLTextKeepsAllHeaderRows(t *testing.T) {
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "sheet.csv",
+ "output_format": "json",
+ "json": []map[string]any{
+ tableItem(t, "<table><tr><th>Group</th><th>Metric</th></tr><tr><th>A</th><th>B</th></tr><tr><td>x</td><td>1</td></tr></table>"),
+ },
+ })
+ if len(chunks) != 1 {
+ t.Fatalf("got %d chunks, want 1", len(chunks))
+ }
+ text, _ := chunks[0]["text"].(string)
+ // Both projected header rows must survive into the rendered HTML.
+ for _, want := range []string{"<th>Group</th>", "<th>Metric</th>", "<th>A</th>", "<th>B</th>", "<td>x</td>"} {
+ if !strings.Contains(text, want) {
+ t.Fatalf("multi-header HTML missing %q: %q", want, text)
+ }
+ }
+}
+
+func TestTableChunkerEmitsHTMLTextEscapesSpecialChars(t *testing.T) {
+ // Build the structured TableData directly so the cell bodies literally
+ // contain <, >, & and " — characters the HTML parser would otherwise treat
+ // as markup. The rendered chunk text must escape every one of them so the
+ // persisted HTML stays well-formed and no cell body leaks as a live tag.
+ td := &entity.TableData{
+ Grid: [][]entity.TableCell{
+ {{Text: "k<>&\""}},
+ {{Text: "a<b>&\"c\""}},
+ },
+ HeaderRows: 1,
+ }
+ b, err := json.Marshal(td)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ var tbl map[string]any
+ if err := json.Unmarshal(b, &tbl); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "sheet.csv",
+ "output_format": "json",
+ "json": []map[string]any{{"table": tbl, "doc_type_kwd": "table"}},
+ })
+ if len(chunks) != 1 {
+ t.Fatalf("got %d chunks, want 1", len(chunks))
+ }
+ text, _ := chunks[0]["text"].(string)
+ for _, want := range []string{"&lt;", "&gt;", "&amp;", "&#34;"} {
+ if !strings.Contains(text, want) {
+ t.Fatalf("cell body not HTML-escaped (missing %q): %q", want, text)
+ }
+ }
+ if strings.Contains(text, "<b>") {
+ t.Fatalf("cell markup leaked unescaped into HTML: %q", text)
+ }
+}
+
+func TestTableChunkerEmitsHTMLTextCaption(t *testing.T) {
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "sheet.csv",
+ "output_format": "json",
+ "json": []map[string]any{
+ tableItem(t, "<table><caption>Q1 Sales</caption><tr><th>k</th></tr><tr><td>v</td></tr></table>"),
+ },
+ })
+ if len(chunks) != 1 {
+ t.Fatalf("got %d chunks, want 1", len(chunks))
+ }
+ text, _ := chunks[0]["text"].(string)
+ if !strings.Contains(text, "<caption>Q1 Sales</caption>") {
+ t.Fatalf("caption not rendered into HTML: %q", text)
+ }
+}
+
+func TestTableChunkerEmitsHTMLTextNilTableDataPassthrough(t *testing.T) {
+ // An item without structured TableData is not a table payload: it must
+ // pass through untouched (text preserved), never rendered.
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "sheet.csv",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "plain body", "doc_type_kwd": "text"},
+ },
+ })
+ if len(chunks) != 1 {
+ t.Fatalf("got %d chunks, want 1", len(chunks))
+ }
+ if got := chunks[0]["text"]; got != "plain body" {
+ t.Fatalf("non-table text mutated: %q", got)
+ }
+}
+
+func TestTableChunkerEmitsHTMLTextWideRowStaysWhole(t *testing.T) {
+ // A single very wide row must become exactly one chunk whose HTML carries
+ // every cell — the row is never cut across chunks.
+ var b strings.Builder
+ b.WriteString("<table><tr>")
+ for i := 0; i < 40; i++ {
+ b.WriteString("<th>h")
+ b.WriteString(strings.Repeat("x", i%5))
+ b.WriteString("</th>")
+ }
+ b.WriteString("</tr><tr>")
+ for i := 0; i < 40; i++ {
+ b.WriteString("<td>cell")
+ b.WriteByte(byte('a' + i%26))
+ b.WriteString("</td>")
+ }
+ b.WriteString("</tr></table>")
+
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "sheet.csv",
+ "output_format": "json",
+ "json": []map[string]any{tableItem(t, b.String())},
+ })
+ if len(chunks) != 1 {
+ t.Fatalf("wide row produced %d chunks, want 1 (row never cut)", len(chunks))
+ }
+ text, _ := chunks[0]["text"].(string)
+ for _, want := range []string{"<td>cella</td>", "<td>cells</td>", "<td>cellz</td>"} {
+ if !strings.Contains(text, want) {
+ t.Fatalf("wide-row HTML missing %q", want)
+ }
+ }
+}
+
+// --- General chunker: structured spreadsheet table renders to HTML text ---
+
+func TestGeneralChunkerEmitsHTMLTextSingleChunk(t *testing.T) {
+ // A small table that fits the budget becomes a single chunk whose text is
+ // the whole table rendered as HTML (not the plain-text projection).
+ chunks := generalTableChunksOf(t, nil, tableItem(t,
+ "<table><caption>People</caption><tr><th>Name</th><th>Age</th></tr><tr><td>Alice</td><td>30</td></tr></table>"))
+ if len(chunks) != 1 {
+ t.Fatalf("got %d chunks, want 1", len(chunks))
+ }
+ text, ok := chunks[0]["text"].(string)
+ if !ok {
+ t.Fatalf("text missing")
+ }
+ if !strings.Contains(text, "<table>") || !strings.Contains(text, "<td>Alice</td>") {
+ t.Fatalf("single-chunk table text not HTML: %q", text)
+ }
+ if !strings.Contains(text, "<caption>People</caption>") {
+ t.Fatalf("caption not rendered: %q", text)
+ }
+ if _, exists := chunks[0]["table"]; exists {
+ t.Fatalf("structured table leaked onto chunk: %#v", chunks[0]["table"])
+ }
+}
+
+func TestGeneralChunkerEmitsHTMLTextLargeTableRowGranular(t *testing.T) {
+ // Force a split with a tiny budget. The table must break on row boundaries
+ // (never mid-row), and every emitted part must be HTML carrying the header.
+ const nRows = 30
+ var b strings.Builder
+ b.WriteString("<table><tr><th>id</th><th>val</th></tr>")
+ for i := 0; i < nRows; i++ {
+ b.WriteString(fmt.Sprintf("<tr><td>r%02d</td><td>%s</td></tr>", i, strings.Repeat("v", 8)))
+ }
+ b.WriteString("</table>")
+
+ chunks := generalTableChunksOf(t, map[string]any{"chunk_token_size": 40}, tableItem(t, b.String()))
+ if len(chunks) < 2 {
+ t.Fatalf("got %d chunks, want >=2 (budget forces a split)", len(chunks))
+ }
+ seen := map[string]bool{}
+ for i, ck := range chunks {
+ text, ok := ck["text"].(string)
+ if !ok || !strings.Contains(text, "<table>") {
+ t.Fatalf("part %d not HTML: %q", i, text)
+ }
+ if !strings.Contains(text, "<th>id</th>") || !strings.Contains(text, "<th>val</th>") {
+ t.Fatalf("part %d missing replicated header: %q", i, text)
+ }
+ // Each part is whole rows only: every <tr> opens and closes.
+ if strings.Count(text, "<tr>") != strings.Count(text, "</tr>") {
+ t.Fatalf("part %d has unbalanced rows (a row was cut): %q", i, text)
+ }
+ // Collect the unique data-row ids that landed in this part; a row id
+ // appearing in two parts would mean it was split or duplicated.
+ rest := text
+ for {
+ idx := strings.Index(rest, "<td>r")
+ if idx < 0 {
+ break
+ }
+ id := rest[idx+4 : idx+7] // "r%02d"
+ rest = rest[idx+7:]
+ if seen[id] {
+ t.Fatalf("row id %q appears in two parts (split/duplicated)", id)
+ }
+ seen[id] = true
+ }
+ }
+ if len(seen) != nRows {
+ t.Fatalf("saw %d distinct rows across parts, want %d (rows lost or merged)", len(seen), nRows)
+ }
+}
+
+// --- Row-span group integrity across the HTML rendering path ---
+
+func TestGeneralChunkerEmitsHTMLTextRowspanGroupNotSplit(t *testing.T) {
+ // A header cell spanning two data rows must keep its group in one part; the
+ // rendered HTML must show the colspan/rowspan, and no part may cut the group.
+ td := &entity.TableData{
+ Grid: [][]entity.TableCell{
+ {{Text: "g", RowSpan: 2}, {Text: "m"}},
+ {{Text: "", Covered: true, OriginRow: 0, OriginCol: 0}, {Text: "a"}},
+ {{Text: "", Covered: true, OriginRow: 0, OriginCol: 0}, {Text: "b"}},
+ },
+ HeaderRows: 1,
+ }
+ b, err := json.Marshal(td)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ var tbl map[string]any
+ if err := json.Unmarshal(b, &tbl); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ chunks := generalTableChunksOf(t, map[string]any{"chunk_token_size": 40},
+ map[string]any{"table": tbl, "doc_type_kwd": "table"})
+ for i, ck := range chunks {
+ text, _ := ck["text"].(string)
+ // The origin cell and every row it covers must be in the same part:
+ // if a part contains "b" (the last spanned data row) it must also
+ // contain the origin "g" with its rowspan.
+ if strings.Contains(text, "<td>b</td>") {
+ if !strings.Contains(text, "rowspan=\"2\"") {
+ t.Fatalf("part %d split the rowspan group: %q", i, text)
+ }
+ }
+ }
+}
diff --git a/internal/ingestion/component/chunker/table_split.go b/internal/ingestion/component/chunker/table_split.go
index aeef29cb3e..ce3c0efa6d 100644
--- a/internal/ingestion/component/chunker/table_split.go
+++ b/internal/ingestion/component/chunker/table_split.go
@@ -17,8 +17,6 @@
package chunker
import (
- "strings"
-
"ragflow/internal/entity"
"ragflow/internal/parser/tableutil"
)
@@ -27,7 +25,8 @@ import (
// splitting it into sub-tables that fit. Splitting is row-granular: every
// emitted sub-table is a whole number of complete rows (never a partial row),
// so the per-row chunk produced downstream keeps full row semantics. The token
-// budget is counted on the real cell text of each row, not on any HTML markup.
+// budget is counted on the HTML rendering of each row — the text actually
+// stored on the chunk — so a rendered part never exceeds the budget.
//
// Splitting is also span-safe: a cut never falls inside a merged cell's
// rowspan, so an origin and every row it covers stay in the same part, and a
@@ -57,9 +56,14 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strin
headerGrid := append([][]entity.TableCell{}, td.Grid[:headerRows]...)
- // Base tokens: header rows + caption, replicated into every sub-table.
+ // Base tokens: header rows + caption, replicated into every sub-table,
+ // counted on the HTML rendering — the text actually stored on the chunk —
+ // so a rendered part never exceeds the budget. The <table> wrapper + caption
+ // appear once per part; emptyWrapper isolates them so each data row below is
+ // charged only its own <tr> contribution.
base := &entity.TableData{Grid: headerGrid, Caption: td.Caption}
- baseTokens := countTokens(tableutil.RenderTableText(base))
+ baseTokens := countTokens(tableutil.RenderTableHTML(base))
+ emptyWrapper := countTokens(tableutil.RenderTableHTML(&entity.TableData{Caption: td.Caption}))
// spanThrough[i] is the last data-row index any rowspan originating at or
// before row i still covers (-1 while no span is active). Header-origin
@@ -115,7 +119,7 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strin
if i < start {
continue // already belongs to a flushed part
}
- rowTokens := countTokens(rowTokenText(row))
+ rowTokens := countTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: [][]entity.TableCell{row}, Caption: td.Caption})) - emptyWrapper
// Flush the running batch before this row if adding it would exceed the
// budget. A single row that alone exceeds the budget still becomes its
// own chunk (row integrity wins over the budget).
@@ -194,16 +198,3 @@ func partDataRows(dataRows [][]entity.TableCell, start, end, headerRows int) [][
}
return out
}
-
-// rowTokenText joins the non-covered cell texts of a grid row for token
-// counting, mirroring how the row is later rendered.
-func rowTokenText(row []entity.TableCell) string {
- texts := make([]string, 0, len(row))
- for _, c := range row {
- if c.Covered {
- continue
- }
- texts = append(texts, c.Text)
- }
- return strings.Join(texts, " ")
-}
diff --git a/internal/ingestion/component/chunker/table_split_test.go b/internal/ingestion/component/chunker/table_split_test.go
index 7ae56f10cb..4f54154e78 100644
--- a/internal/ingestion/component/chunker/table_split_test.go
+++ b/internal/ingestion/component/chunker/table_split_test.go
@@ -169,40 +169,51 @@ func TestSplitLargeTableHeaderRowSpanClippedPerPart(t *testing.T) {
if err != nil || td == nil {
t.Fatalf("parse: %v", err)
}
- // Budget 6: parts hold data rows [0,2), [2,3), [3,4), [4,5). The first
- // part carries both rows A spans, so A keeps RowSpan 3; the later parts
- // carry none of them, so A degrades to a plain header cell.
- parts, ranges, headerRows := splitLargeTable(td, 6, charTokens)
- if headerRows != 1 || len(parts) != 4 {
+ // Budget = the HTML cost of the header + the two rows A spans (y, z), so
+ // the first part is exactly those rows and keeps A's RowSpan 3; later parts
+ // carry none of them, so A degrades to a plain header cell (RowSpan 1).
+ budget := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:3], HeaderRows: 1}))
+ parts, ranges, headerRows := splitLargeTable(td, budget, charTokens)
+ if headerRows != 1 || len(parts) < 2 {
t.Fatalf("parts=%d ranges=%v headerRows=%d", len(parts), ranges, headerRows)
}
- wantRanges := [][2]int{{0, 2}, {2, 3}, {3, 4}, {4, 5}}
- if !reflect.DeepEqual(ranges, wantRanges) {
- t.Fatalf("ranges=%v want %v", ranges, wantRanges)
+ if ranges[0] != [2]int{0, 2} {
+ t.Fatalf("first range = %v, want [0 2] (A's two spanned rows)", ranges[0])
}
- wantSpans := []int{3, 1, 1, 1}
- for i, part := range parts {
- assertPartGridContract(t, &part, i)
- if got := part.Grid[0][0].RowSpan; got != wantSpans[i] {
- t.Fatalf("part %d header A RowSpan=%d want %d", i, got, wantSpans[i])
+ if got := parts[0].Grid[0][0].RowSpan; got != 3 {
+ t.Fatalf("part 0 A RowSpan = %d, want 3 (carries both spanned rows)", got)
+ }
+ for i := 1; i < len(parts); i++ {
+ assertPartGridContract(t, &parts[i], i)
+ if got := parts[i].Grid[0][0].RowSpan; got != 1 {
+ t.Fatalf("part %d A RowSpan = %d, want 1 (carries none of A's spanned rows)", i, got)
}
}
- // Budget 4: every part holds a single data row, so A keeps exactly the
- // covered rows that travel with the part.
- parts, _, _ = splitLargeTable(td, 4, charTokens)
- wantSpans = []int{2, 2, 1, 1, 1}
- if len(parts) != len(wantSpans) {
- t.Fatalf("parts=%d want %d", len(parts), len(wantSpans))
+ covered := 0
+ for _, r := range ranges {
+ covered += r[1] - r[0]
}
- for i, part := range parts {
- assertPartGridContract(t, &part, i)
- if got := part.Grid[0][0].RowSpan; got != wantSpans[i] {
- t.Fatalf("part %d header A RowSpan=%d want %d", i, got, wantSpans[i])
- }
+ if covered != 5 {
+ t.Fatalf("parts cover %d data rows, want 5", covered)
}
- wantTexts := [][]string{{"y"}, {"z"}, {"p", "q"}, {"r", "s"}, {"t", "u"}}
- if got := partDataTexts(parts, headerRows); !reflect.DeepEqual(got, wantTexts) {
- t.Fatalf("data texts=%v want %v", got, wantTexts)
+
+ // Budget = the HTML cost of the header + only the first spanned row (y), so
+ // the first part carries exactly one of A's spanned rows and A keeps
+ // RowSpan 2 (header + that one row), not the full 3.
+ budget = charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:2], HeaderRows: 1}))
+ parts, ranges, _ = splitLargeTable(td, budget, charTokens)
+ if len(parts) < 2 || ranges[0] != [2]int{0, 1} {
+ t.Fatalf("parts=%d ranges=%v, want first part [0 1]", len(parts), ranges)
+ }
+ if got := parts[0].Grid[0][0].RowSpan; got != 2 {
+ t.Fatalf("part 0 A RowSpan = %d, want 2 (carries exactly one spanned row)", got)
+ }
+ covered = 0
+ for _, r := range ranges {
+ covered += r[1] - r[0]
+ }
+ if covered != 5 {
+ t.Fatalf("parts cover %d data rows, want 5", covered)
}
}
@@ -222,15 +233,19 @@ func TestSplitLargeTableNeverSplitsInsideRowSpan(t *testing.T) {
if err != nil || td == nil {
t.Fatalf("parse: %v", err)
}
- // The budget alone would cut after data row 2, inside B's span (B covers
- // data rows 0..2); the cut must move to after row 2, i.e. to 3.
- parts, ranges, headerRows := splitLargeTable(td, 11, charTokens)
+ // Budget = the HTML cost of the header + B's own row only. The budget alone
+ // would cut right after data row 0, inside B's span (B covers data rows
+ // 0..2); span-safe cut must move the boundary to after row 2, i.e. to 3.
+ budget := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:2], HeaderRows: 1}))
+ parts, ranges, headerRows := splitLargeTable(td, budget, charTokens)
if headerRows != 1 {
t.Fatalf("headerRows=%d want 1", headerRows)
}
- wantRanges := [][2]int{{0, 3}, {3, 5}}
- if !reflect.DeepEqual(ranges, wantRanges) {
- t.Fatalf("ranges=%v want %v (cut must not fall inside B's rowspan)", ranges, wantRanges)
+ // The first boundary must land after B's span (row 3), never inside it —
+ // that is the span-safe guarantee. (How the remaining rows chunk is purely a
+ // budget artifact and not what this test pins.)
+ if ranges[0] != [2]int{0, 3} {
+ t.Fatalf("first range = %v, want [0 3] (cut must not fall inside B's rowspan)", ranges[0])
}
for i, part := range parts {
assertPartGridContract(t, &part, i)
diff --git a/internal/ingestion/component/chunker/table_struct_test.go b/internal/ingestion/component/chunker/table_struct_test.go
index 06f9d3b9cb..5e3988a00d 100644
--- a/internal/ingestion/component/chunker/table_struct_test.go
+++ b/internal/ingestion/component/chunker/table_struct_test.go
@@ -15,12 +15,33 @@
package chunker
import (
+ "strings"
"testing"
"ragflow/internal/entity"
"ragflow/internal/ingestion/component/schema"
)
+// wantHTMLCells asserts the chunk text is an HTML table carrying exactly the
+// given header and data cells (in order). The row chunks render header+row as
+// HTML, so the projected row's cells appear as <td> values.
+func wantHTMLCells(t *testing.T, text string, header, data []string) {
+ t.Helper()
+ if !strings.Contains(text, "<table>") {
+ t.Fatalf("not HTML: %q", text)
+ }
+ for _, h := range header {
+ if !strings.Contains(text, "<th>"+h+"</th>") {
+ t.Fatalf("missing header cell %q in %q", h, text)
+ }
+ }
+ for _, d := range data {
+ if !strings.Contains(text, "<td>"+d+"</td>") {
+ t.Fatalf("missing data cell %q in %q", d, text)
+ }
+ }
+}
+
func TestExpandHTMLTableRowsFromTableData(t *testing.T) {
td := &entity.TableData{
Grid: [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}, {{Text: "Alice"}, {Text: "30"}}, {{Text: "Bob"}, {Text: "25"}}},
@@ -30,12 +51,8 @@ func TestExpandHTMLTableRowsFromTableData(t *testing.T) {
if len(out) != 2 {
t.Fatalf("got %d chunks, want 2", len(out))
}
- if out[0].Text != "- Name: Alice\n- Age: 30" {
- t.Fatalf("row0 text = %q", out[0].Text)
- }
- if out[1].Text != "- Name: Bob\n- Age: 25" {
- t.Fatalf("row1 text = %q", out[1].Text)
- }
+ wantHTMLCells(t, out[0].Text, []string{"Name", "Age"}, []string{"Alice", "30"})
+ wantHTMLCells(t, out[1].Text, []string{"Name", "Age"}, []string{"Bob", "25"})
for _, c := range out {
if c.TableData != nil {
t.Fatalf("TableData must be cleared on emitted row chunk, got %+v", c.TableData)
@@ -89,15 +106,11 @@ func TestExpandHTMLTableRowsKeepsRowspanColumns(t *testing.T) {
if len(out) != 2 {
t.Fatalf("got %d chunks, want 2", len(out))
}
- if out[0].Text != "- Name: Alice\n- Age: 30" {
- t.Fatalf("row0 text = %q", out[0].Text)
- }
+ wantHTMLCells(t, out[0].Text, []string{"Name", "Age"}, []string{"Alice", "30"})
// The continuation row must keep the spanned "Name" column resolved to the
- // origin value and stay aligned: "Name: Alice" + "Age: 25", not a shifted
- // "Name: 25" that drops the Age column.
- if out[1].Text != "- Name: Alice\n- Age: 25" {
- t.Fatalf("row1 text = %q (rowspan column must be preserved, not dropped/misaligned)", out[1].Text)
- }
+ // origin value and stay aligned: "Alice" + "25", not a shifted "25" that
+ // drops the Age column.
+ wantHTMLCells(t, out[1].Text, []string{"Name", "Age"}, []string{"Alice", "25"})
}
// A cell that spans BOTH rows and columns (RowSpan>1 && ColSpan>1) must keep
@@ -127,14 +140,11 @@ func TestExpandHTMLTableRowsKeepsRowColSpanBlock(t *testing.T) {
if len(out) != 2 {
t.Fatalf("got %d chunks, want 2 (one per data row)", len(out))
}
- if out[0].Text != "- Name: Wide\n- B: Wide\n- C: x" {
- t.Fatalf("row0 text = %q", out[0].Text)
- }
+ // The origin row keeps both covered columns resolved to "Wide".
+ wantHTMLCells(t, out[0].Text, []string{"Name", "B", "C"}, []string{"Wide", "Wide", "x"})
// The continuation row must keep BOTH covered columns resolved to "Wide",
- // not drop the second one to a blank and misalign under the header.
- if out[1].Text != "- Name: Wide\n- B: Wide\n- C: y" {
- t.Fatalf("row1 text = %q (both spanned columns must be preserved)", out[1].Text)
- }
+ // not drop the second one and misalign under the header.
+ wantHTMLCells(t, out[1].Text, []string{"Name", "B", "C"}, []string{"Wide", "Wide", "y"})
}
// A data row whose first cell spans two columns must keep both columns after
@@ -159,7 +169,6 @@ func TestExpandHTMLTableRowsKeepsColspanColumns(t *testing.T) {
if len(out) != 1 {
t.Fatalf("got %d chunks, want 1", len(out))
}
- if out[0].Text != "- Name: Info\n- Age: Info" {
- t.Fatalf("row text = %q (colspan must fill both columns)", out[0].Text)
- }
+ // colspan must fill both columns with the spanned value "Info".
+ wantHTMLCells(t, out[0].Text, []string{"Name", "Age"}, []string{"Info", "Info"})
}
diff --git a/internal/ingestion/component/chunker/table_test.go b/internal/ingestion/component/chunker/table_test.go
index 596766c3f3..86f7b39c3c 100644
--- a/internal/ingestion/component/chunker/table_test.go
+++ b/internal/ingestion/component/chunker/table_test.go
@@ -148,8 +148,10 @@ func TestTableChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {
t.Fatalf("got %d chunks, want data row plus header-only segment", len(chunks))
}
// The header row of the data segment never becomes a chunk of its own;
- // its column names live in the row record text instead.
- if chunks[0]["text"] != "- ID: A-1\n- Name: paid" {
+ // its column names ride along as the HTML header of the row chunk.
+ row0, _ := chunks[0]["text"].(string)
+ if !strings.Contains(row0, "<table>") || !strings.Contains(row0, "<td>A-1</td>") ||
+ !strings.Contains(row0, "<td>paid</td>") || !strings.Contains(row0, "<th>ID</th>") {
t.Fatalf("chunk0 = %#v", chunks[0])
}
if chunks[1]["ck_type"] != "table" || chunks[1]["table"] != nil {
@@ -181,11 +183,14 @@ func TestTableChunker_ExpandsHTMLRowsWithAlignedPositions(t *testing.T) {
if len(chunks) != 2 {
t.Fatalf("got %d chunks, want one per data row", len(chunks))
}
- if chunks[0]["text"] != "- ID: A-1\n- Status: paid" {
+ c0, _ := chunks[0]["text"].(string)
+ if !strings.Contains(c0, "<td>A-1</td>") || !strings.Contains(c0, "<td>paid</td>") || !strings.Contains(c0, "<th>ID</th>") {
t.Errorf("chunk0 text = %v", chunks[0]["text"])
}
- // The empty cell is skipped, matching the Python table chunker's per-cell skip.
- if chunks[1]["text"] != "- ID: A-2" {
+ // The empty cell is skipped in the row record; the HTML row keeps the row's
+ // own id and renders the empty cell plainly.
+ c1, _ := chunks[1]["text"].(string)
+ if !strings.Contains(c1, "<td>A-2</td>") {
t.Errorf("chunk1 text = %v", chunks[1]["text"])
}
if got := fmt.Sprintf("%v", chunks[0]["positions"]); !strings.Contains(got, "2") || strings.Contains(got, "3") {
@@ -249,15 +254,15 @@ func TestTableChunker_NonSpreadsheetPositionsNotAttached(t *testing.T) {
}
// TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload: a table whose only
-// row is the header has nothing to expand; the whole markup stays one chunk
-// carrying the plain-text rendering of the header.
+// row is the header has nothing to expand; the whole table stays one chunk
+// carrying its HTML rendering.
func TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload(t *testing.T) {
table := "<table><caption>t</caption>\n<tr><th>A</th><th>B</th></tr>\n</table>\n"
td, err := tableutil.ParseTableHTML(table)
if err != nil {
t.Fatalf("ParseTableHTML: %v", err)
}
- want := tableutil.RenderTableText(td)
+ want := tableutil.RenderTableHTML(td)
chunks := tableChunksOf(t, map[string]any{
"name": "template.xlsx",
"output_format": "json",
@@ -287,7 +292,9 @@ func TestTableChunker_HeaderlessHTMLFirstRowIsHeader(t *testing.T) {
if len(chunks) != 1 {
t.Fatalf("got %d chunks, want the single data row", len(chunks))
}
- if chunks[0]["text"] != "- k: a\n- v: b" {
+ txt, _ := chunks[0]["text"].(string)
+ if !strings.Contains(txt, "<th>k</th>") || !strings.Contains(txt, "<th>v</th>") ||
+ !strings.Contains(txt, "<td>a</td>") || !strings.Contains(txt, "<td>b</td>") {
t.Errorf("text = %v", chunks[0]["text"])
}
}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 40c39a6194..1b8cd2fc72 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -257,18 +257,18 @@ func decodeChunkerFromUpstream(inputs map[string]any) (schema.ChunkerFromUpstrea
if err := out.Validate(); err != nil {
return out, err
}
- // A structured table carries no markup text of its own. Derive a plain
- // (tag-free) text from its cells so consumers that read Text — TokenChunker
- // and the embed path — still receive a searchable body when the producer
- // emitted only the TableData contract (decision 1-B: no HTML in Text).
+ // A structured table (e.g. the office/xlsx producer) carries no Text of its
+ // own. Derive its HTML <table> rendering on Text so every chunker that
+ // consumes it (token/group/hierarchy/one/page/qa) stores and displays a real
+ // table, and the chunk id (a hash of Text) matches the stored content.
for i := range out.JSONResult {
if out.JSONResult[i].TableData != nil && out.JSONResult[i].Text == "" {
- out.JSONResult[i].Text = tableutil.RenderTableText(out.JSONResult[i].TableData)
+ out.JSONResult[i].Text = tableutil.RenderTableHTML(out.JSONResult[i].TableData)
}
}
for i := range out.Chunks {
if out.Chunks[i].TableData != nil && out.Chunks[i].Text == "" {
- out.Chunks[i].Text = tableutil.RenderTableText(out.Chunks[i].TableData)
+ out.Chunks[i].Text = tableutil.RenderTableHTML(out.Chunks[i].TableData)
}
}
return out, nil
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index f19a2dbb05..ab41cfb45b 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -105,9 +105,9 @@ const ComponentNameTokenizer = "Tokenizer"
// `re.sub(r"\.[a-zA-Z]+$", "", name)` in tokenizer.py:137.
var titleExtRE = regexp.MustCompile(`\.[a-zA-Z]+$`)
-// Table content is no longer carried as HTML markup (the parser emits the
-// structured entity.TableData contract), so there is no table markup to strip
-// from the embedded text. The legacy htmlTableRE was removed.
+// Table chunks carry their row rendered as an HTML <table> on Text (produced
+// by the chunker), so the embedding input intentionally includes the table
+// markup — there is no markup to strip. The legacy htmlTableRE stays removed.
// EmbeddingResult carries a vector plus the model-reported token usage
// for that input batch entry.
diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.go
index ce770512a1..97ea6d8ca8 100644
--- a/internal/ingestion/task/indexdoc/process_test.go
+++ b/internal/ingestion/task/indexdoc/process_test.go
@@ -4,6 +4,8 @@ import (
"reflect"
"testing"
"time"
+
+ "ragflow/internal/common"
)
func TestProcessChunksForPipeline_PreservesTypedMetadata(t *testing.T) {
@@ -189,6 +191,34 @@ func TestProcessChunksForPipeline_PreservesExistingID(t *testing.T) {
}
}
+// TestProcessChunksForPipeline_TableHTMLIDMatchesContent pins the table-HTML
+// redesign's core invariant: a table chunk arrives from the chunker with its
+// row rendered as HTML on "text" and an id already stamped (by the chunker's
+// decorator) from that HTML text. The persist boundary must keep the id
+// byte-consistent with the stored content_with_weight — both hash the same
+// HTML — so re-indexing stays stable and the displayed table never diverges
+// from the chunk id. (The old design stamped the id from the clean text while
+// storing HTML, which broke this.)
+func TestProcessChunksForPipeline_TableHTMLIDMatchesContent(t *testing.T) {
+ html := "<table>\n<tr><th>Name</th></tr>\n<tr><td>Alice</td></tr>\n</table>\n"
+ chunks := []map[string]any{{
+ "doc_type": "table",
+ "text": html,
+ "id": common.ChunkID("doc-1", html),
+ }}
+ if _, err := ProcessChunksForPipeline(chunks, "doc-1", "report.pdf", time.Now()); err != nil {
+ t.Fatalf("ProcessChunksForPipeline: %v", err)
+ }
+ ck := chunks[0]
+ content, ok := ck["content_with_weight"].(string)
+ if !ok || content != html {
+ t.Fatalf("content_with_weight = %q, want the HTML text unchanged", content)
+ }
+ if want := common.ChunkID("doc-1", content); ck["id"] != want {
+ t.Fatalf("id = %v, want ChunkID(docID, content_with_weight) = %v", ck["id"], want)
+ }
+}
+
func TestProcessChunksForPipeline_QuestionsProcessing(t *testing.T) {
chunks := []map[string]any{{"text": "hello", "questions": "Q1\nQ2\nQ3"}}
_, err := ProcessChunksForPipeline(chunks, "doc-1", "test-doc.pdf", time.Now())
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index 059279b3a1..229e9e2aa7 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -71,7 +71,8 @@ func trimCells(row []string) []string {
// loses column names) and carries a row-aligned position matrix — one tuple
// per row in strict order, header included — so row-level consumers index
// positions by row number with no side channel. The chunker reads the rows
-// directly; no HTML markup is emitted.
+// directly and renders each emitted chunk's Text as an HTML <table> (see
+// tableutil.RenderTableHTML); no HTML markup is emitted here at the parser.
//
// An image lands after the last row whose (row, colStart) sorts at or before
// its anchor; rows before the header's anchor keep the header in front of
diff --git a/internal/parser/parser/pdf_parser_table_contract_test.go b/internal/parser/parser/pdf_parser_table_contract_test.go
index c804d46a47..5e7e114498 100644
--- a/internal/parser/parser/pdf_parser_table_contract_test.go
+++ b/internal/parser/parser/pdf_parser_table_contract_test.go
@@ -5,9 +5,20 @@ import (
"testing"
"ragflow/internal/entity"
- "ragflow/internal/parser/tableutil"
)
+// cellTextsContain reports whether any cell of the structured table contains sub.
+func cellTextsContain(td *entity.TableData, sub string) bool {
+ for _, row := range td.CellTexts() {
+ for _, cell := range row {
+ if strings.Contains(cell, sub) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
// Consumers of doc_type_kwd "table" parse the item text as <table> markup
// (the QA chunker reads its <tr> rows), so a table item that carries other
// text loses its rows downstream. The structured TableData contract must also
@@ -91,8 +102,8 @@ func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {
if len(td.CellTexts()) == 0 {
t.Fatalf("structured table has no rows: %#v", td)
}
- if got := tableutil.RenderTableText(td); !strings.Contains(got, "What is RAGFlow?") {
- t.Fatalf("structured table missing cell text %q: %#v", got, td)
+ if !cellTextsContain(td, "What is RAGFlow?") {
+ t.Fatalf("structured table missing cell text %q: %#v", "What is RAGFlow?", td)
}
}
})
diff --git a/internal/parser/parser/pdf_table_no_html_text_test.go b/internal/parser/parser/pdf_table_no_html_text_test.go
index 091f4b0e39..305965af88 100644
--- a/internal/parser/parser/pdf_table_no_html_text_test.go
+++ b/internal/parser/parser/pdf_table_no_html_text_test.go
@@ -61,8 +61,8 @@ func TestPDFTableItemsOmitHTMLText(t *testing.T) {
if !ok || td == nil || len(td.CellTexts()) == 0 {
t.Fatalf("table item missing structured TableData: %#v", tableItem)
}
- if got := tableutil.RenderTableText(td); !strings.Contains(got, "What is RAGFlow?") && !strings.Contains(got, "cell") {
- t.Fatalf("structured table missing cell text %q: %#v", got, td)
+ if !cellTextsContain(td, "What is RAGFlow?") && !cellTextsContain(td, "cell") {
+ t.Fatalf("structured table missing cell text %q: %#v", "What is RAGFlow?", td)
}
})
}
diff --git a/internal/parser/tableutil/table_grid_test.go b/internal/parser/tableutil/table_grid_test.go
index f8326950de..0dfbf4e8b2 100644
--- a/internal/parser/tableutil/table_grid_test.go
+++ b/internal/parser/tableutil/table_grid_test.go
@@ -353,13 +353,13 @@ func TestRenderNonMergedByteIdentical(t *testing.T) {
}
}
-// RenderTableText strips tags; a merged table yields the same stripped text as
-// the equivalent non-merged table (text-level output is unchanged by spans).
-func TestRenderTextMergedSameAsNonMerged(t *testing.T) {
+// CellTexts is span-agnostic: a merged table yields the same cell texts as the
+// equivalent non-merged table (the Covered slot contributes no text).
+func TestCellTextsMergedSameAsNonMerged(t *testing.T) {
merged, _ := ParseTableHTML(`<table><tr><td colspan="2">wide</td><td>narrow</td></tr></table>`)
plain, _ := ParseTableHTML(`<table><tr><td>wide</td><td>narrow</td></tr></table>`)
- if RenderTableText(merged) != RenderTableText(plain) {
- t.Fatalf("merged text %q != plain text %q", RenderTableText(merged), RenderTableText(plain))
+ if !reflect.DeepEqual(merged.CellTexts(), plain.CellTexts()) {
+ t.Fatalf("merged %v != plain %v", merged.CellTexts(), plain.CellTexts())
}
}
diff --git a/internal/parser/tableutil/table_html.go b/internal/parser/tableutil/table_html.go
index 3006714fe4..b257061fff 100644
--- a/internal/parser/tableutil/table_html.go
+++ b/internal/parser/tableutil/table_html.go
@@ -20,7 +20,6 @@
package tableutil
import (
- "regexp"
"strconv"
"strings"
@@ -329,12 +328,10 @@ func attrInt(n *html.Node, name string, max int) int {
}
// RenderTableHTML emits a deterministic <table> markup for a TableData. It is
-// used only as a test oracle / round-trip helper and is NOT part of the
-// persisted contract (producers emit TableData, not HTML). The emitted shape
-// matches the legacy spreadsheet producer's renderSpreadsheetTable exactly
-// (including the inter-row newlines) for 1x1 grids, so RenderTableText
-// reproduces the legacy htmlTableRE-stripped text byte for byte. For merged
-// tables it emits colspan/rowspan on the origin slot and skips Covered slots.
+// the persisted display form of a table chunk: the chunker renders each table
+// chunk's Text with it, and the chunk id is a hash of that rendered text. For
+// merged tables it emits colspan/rowspan on the origin slot and skips Covered
+// slots.
func RenderTableHTML(td *entity.TableData) string {
if td == nil {
return ""
@@ -384,22 +381,6 @@ func RenderTableHTML(td *entity.TableData) string {
return b.String()
}
-// tableTagRE strips table markup tags (with any attributes) so the plaintext a
-// chunker writes for an embed-only path matches the legacy tag-stripped output.
-// The pattern tolerates arbitrary attribute lists (including colspan/rowspan)
-// so merged-table markup is stripped completely rather than left partly in the
-// text.
-var tableTagRE = regexp.MustCompile(`</?(?:table|td|caption|tr|th)(?:\s[^<>]*)?>`)
-
-// RenderTableText renders a TableData to plain text with no markup tags, for
-// the rare embed path that receives a table without a downstream TableChunker.
-// It reproduces the legacy htmlTableRE stripping of RenderTableHTML(td).
-func RenderTableText(td *entity.TableData) string {
- h := RenderTableHTML(td)
- h = tableTagRE.ReplaceAllString(h, " ")
- return strings.TrimSpace(h)
-}
-
// cellText returns the visible text of a table cell: text nodes contribute
// verbatim, <br> becomes a newline, inert elements are skipped, and other
// elements are descended. The result is TrimSpace'd.
diff --git a/internal/parser/tableutil/table_html_test.go b/internal/parser/tableutil/table_html_test.go
index f8c0060d8e..80b365043a 100644
--- a/internal/parser/tableutil/table_html_test.go
+++ b/internal/parser/tableutil/table_html_test.go
@@ -192,24 +192,6 @@ func TestRenderTableHTMLRoundTrip(t *testing.T) {
}
}
-func TestRenderTableTextHasNoTags(t *testing.T) {
- td := &entity.TableData{
- Grid: [][]entity.TableCell{
- {{Text: "Name"}, {Text: "Age"}},
- {{Text: "Alice"}, {Text: "30"}},
- },
- HeaderRows: 1,
- Caption: "People",
- }
- got := RenderTableText(td)
- if strings.ContainsAny(got, "<>") {
- t.Fatalf("RenderTableText contains tags: %q", got)
- }
- if !strings.Contains(got, "Name") || !strings.Contains(got, "Alice") {
- t.Fatalf("RenderTableText lost cell text: %q", got)
- }
-}
-
func assertRowsEqual(t *testing.T, got, want [][]string) {
t.Helper()
if len(got) != len(want) {