RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Render table chunks as HTML on Text at the chunker (#20918)
xugangqiang9fe7fec2c606
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex ddf85ccd0b..4dcb080274 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -994,7 +994,13 @@ func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []}parts, ranges, headerRows := splitLargeTable(td, c.param.ChunkTokenSize, tokenizeStr)if ranges == nil {- return []schema.ChunkDoc{cloneChunkDoc(unit)}+ // The whole table fits one chunk: render it as HTML on Text so the+ // persisted content and the id (hash of Text) stay byte-consistent and+ // the UI can display a real table. TableData is dropped, never indexed.+ out := cloneChunkDoc(unit)+ out.Text = tableutil.RenderTableHTML(td)+ out.TableData = nil+ return []schema.ChunkDoc{out}}var matrix [][]float64// Spreadsheet items only: PDF items write layout boxes into the same@@ -1021,11 +1027,12 @@ func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []out := make([]schema.ChunkDoc, 0, len(parts))for i, part := range parts {piece := cloneChunkDoc(unit)- // Text carries a plain-text rendering (no markup) for the rare embed- // path without a downstream TableChunker; the structured TableData is- // the authoritative content for chunkers that read it.- piece.Text = tableutil.RenderTableText(&part)- piece.TableData = &part+ // Text carries the sub-table rendered as HTML so the persisted content+ // and the chunk id (hash of Text) stay byte-consistent and the UI can+ // display a real table. The structured TableData is dropped here and is+ // never serialized to the index.+ piece.Text = tableutil.RenderTableHTML(&part)+ piece.TableData = nilpiece.Positions = nillo := headerRows + ranges[i][0]hi := headerRows + ranges[i][1]diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.goindex e0196e274f..7a10f0c81a 100644--- a/internal/ingestion/component/chunker/general_test.go+++ b/internal/ingestion/component/chunker/general_test.go@@ -76,9 +76,9 @@ func spreadsheetSegmentItem(sheet string, header []string, rows [][]string, shee"header_rows": 1,"caption": sheet,},- // Mirror what the chunker derives from TableData so text-bound- // assertions (e.g. TokenChunker boundary tests) stay valid.- "text": tableutil.RenderTableText(td),+ // Mirror what the chunker derives from TableData (its HTML rendering)+ // so text-bound assertions (e.g. TokenChunker boundary tests) stay valid.+ "text": tableutil.RenderTableHTML(td),"doc_type_kwd": "table","ck_type": "table","sheet_index": sheetIndex,@@ -1086,19 +1086,21 @@ func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testing}covered := 0for i, chunk := range chunks {- // New contract: Text is the plain rendering (no markup); the- // replicated header lives in the structured TableData, and positions- // are sliced to this sub-table when the matrix is row-aligned.- tbl, ok := chunk["table"].(map[string]any)- if !ok {- t.Fatalf("chunk %d missing structured TableData", i)+ // New contract: Text is the sub-table rendered as HTML (replicated+ // header + this part's data rows); the structured TableData is dropped+ // and never indexed; positions are sliced to this sub-table when the+ // matrix is row-aligned.+ if _, ok := chunk["table"]; ok {+ t.Fatalf("chunk %d still carries structured table", i)}- tblGrid, _ := tbl["grid"].([]any)- if len(tblGrid) < 2 {- t.Fatalf("chunk %d has too few rows: %v", i, tblGrid)+ text, _ := chunk["text"].(string)+ if !strings.Contains(text, "<table>") || !strings.Contains(text, "<th>Name</th>") {+ t.Fatalf("chunk %d text lost the header cell: %q", i, text)}- if !strings.Contains(chunk["text"].(string), "Name") {- t.Errorf("chunk %d text lost the header cell: %q", i, chunk["text"])+ // Single-column table: each data row is one <td>value-…</td> cell.+ dataRows := strings.Count(text, "<td>value-")+ if dataRows == 0 {+ t.Fatalf("chunk %d has no data rows: %q", i, text)}var matrix [][]float64raw, _ := json.Marshal(chunk["positions"])@@ -1113,7 +1115,6 @@ func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testingif matrix[0][1] != 1 {t.Errorf("chunk %d header tuple = %v, want sheet 1 row 1", i, matrix[0])}- dataRows := len(tblGrid) - 1if dataRows != len(matrix)-1 {t.Errorf("chunk %d data rows %d != position tuples %d", i, dataRows, len(matrix)-1)}diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.goindex 787f3900d8..1e674a1289 100644--- a/internal/ingestion/component/chunker/table.go+++ b/internal/ingestion/component/chunker/table.go@@ -216,13 +216,16 @@ func tableChunkRows(td *entity.TableData) [][]string {return out}-// Non-table payloads pass through unchanged. It reads the structured-// TableData contract that the parser emits on ChunkDoc.TableData; items that-// do not carry TableData (e.g. pre-upgrade HTML markup or non-table records)-// pass through as a single chunk. A table whose only row is the header keeps-// the whole item as its single chunk: the header line is then the only-// searchable representation. The structured TableData is cleared on every-// emitted chunk so the whole table is not duplicated into the index.+// expandHTMLTableRows turns one table payload into one chunk per data row.+// It reads the structured TableData contract the parser emits on+// ChunkDoc.TableData; items without TableData (pre-upgrade HTML markup or+// non-table records) pass through unchanged as a single chunk. Every emitted+// chunk carries its content rendered as an HTML <table> on Text, so the+// persisted content and the chunk id (a hash of Text) describe the same bytes+// and the UI/index can display a real table; the structured TableData is+// dropped here and never serialized to the index. Header-only and fallback+// chunks keep the whole table's HTML (merge geometry intact); per-row chunks+// carry a flat header+row sub-table (merge-free, see cellGrid).func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {td := item.TableDataif td == nil {@@ -231,7 +234,7 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {rows, headerCount := tableChunkRows(td), td.HeaderRowsif len(rows) <= headerCount {out := item- out.Text = tableutil.RenderTableText(td)+ out.Text = tableutil.RenderTableHTML(td)out.TableData = nilreturn []schema.ChunkDoc{out}}@@ -257,12 +260,19 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {}out := make([]schema.ChunkDoc, 0, len(rows)-headerCount)for i, row := range rows[headerCount:] {- text := tableRowRecordText(names, row)- if text == "" {+ // Skip empty rows (no cell content); the HTML render of an empty row+ // would be a header-only table that duplicates the header-only chunk.+ if tableRowRecordText(names, row) == "" {continue}doc := item- doc.Text = text+ // Render this row as its own HTML table: every projected header row+ // plus this data row, so the displayed table keeps the full header.+ doc.Text = tableutil.RenderTableHTML(&entity.TableData{+ Grid: cellGrid(append(append([][]string{}, rows[:headerCount]...), row)),+ HeaderRows: headerCount,+ Caption: td.Caption,+ })doc.TKNums = nildoc.Positions = nildoc.TableData = nil@@ -275,13 +285,28 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {}if len(out) == 0 {single := item- single.Text = tableutil.RenderTableText(td)+ single.Text = tableutil.RenderTableHTML(td)single.TableData = nilreturn []schema.ChunkDoc{single}}return out}+// cellGrid builds a merge-free dense Grid from row-major cell strings. It is+// used to render a per-row sub-table (the projected header rows plus one data+// row) as HTML; merged-cell geometry is intentionally not reconstructed here,+// so the sub-table shows each cell plainly rather than with colspan/rowspan.+func cellGrid(rows [][]string) [][]entity.TableCell {+ grid := make([][]entity.TableCell, len(rows))+ for i, r := range rows {+ grid[i] = make([]entity.TableCell, len(r))+ for j, c := range r {+ grid[i][j] = entity.TableCell{Text: c}+ }+ }+ return grid+}+// tableDataOf returns the structured TableData for an item. The parser emits// it on ChunkDoc.TableData; items without it (pre-upgrade HTML markup or// non-table records) return nil and pass through unchanged. Re-chunk alwaysdiff --git a/internal/ingestion/component/chunker/table_html_coverage_test.go b/internal/ingestion/component/chunker/table_html_coverage_test.gonew file mode 100644index 0000000000..93fef6dc1d--- /dev/null+++ b/internal/ingestion/component/chunker/table_html_coverage_test.go@@ -0,0 +1,238 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "strings"+ "testing"++ "ragflow/internal/entity"+ "ragflow/internal/parser/tableutil"+)++// gridOf builds a TableData Grid from row-major cell strings (header + data).+func gridOf(header []string, data ...[]string) [][]entity.TableCell {+ rows := make([][]entity.TableCell, 0, 1+len(data))+ rows = append(rows, cellsOf(header))+ for _, d := range data {+ rows = append(rows, cellsOf(d))+ }+ return rows+}++func cellsOf(vals []string) []entity.TableCell {+ cells := make([]entity.TableCell, len(vals))+ for i, v := range vals {+ cells[i] = entity.TableCell{Text: v}+ }+ return cells+}++// --- A: the shared decode fallback renders structured tables as HTML ---++// A structured table that arrives with only the TableData contract (no Text —+// the office/xlsx producer path) must get an HTML <table> on Text from the+// shared decode fallback, so every chunker that consumes it (token/group/+// hierarchy/one/page/qa) stores and displays a real table, not plain text.+func TestDecodeChunkerFromUpstreamRendersTableHTML(t *testing.T) {+ up, err := decodeChunkerFromUpstream(map[string]any{+ "name": "sheet.xlsx",+ "output_format": "json",+ "json": []map[string]any{+ tableItem(t, "<table><caption>Q1</caption><tr><th>Name</th><th>Age</th></tr><tr><td>Alice</td><td>30</td></tr></table>"),+ },+ })+ if err != nil {+ t.Fatalf("decode: %v", err)+ }+ if len(up.JSONResult) != 1 {+ t.Fatalf("got %d items, want 1", len(up.JSONResult))+ }+ text := up.JSONResult[0].Text+ if !strings.Contains(text, "<table>") || !strings.Contains(text, "<td>Alice</td>") {+ t.Fatalf("structured table not rendered to HTML on Text: %q", text)+ }+ if !strings.Contains(text, "<caption>Q1</caption>") {+ t.Fatalf("caption not carried into HTML: %q", text)+ }+ // The fallback fires on the "chunks" payload too (re-chunk path).+ upChunks, err := decodeChunkerFromUpstream(map[string]any{+ "name": "sheet.xlsx",+ "output_format": "chunks",+ "chunks": []map[string]any{+ tableItem(t, "<table><tr><th>k</th></tr><tr><td>v</td></tr></table>"),+ },+ })+ if err != nil {+ t.Fatalf("decode chunks: %v", err)+ }+ if len(upChunks.Chunks) != 1 || !strings.Contains(upChunks.Chunks[0].Text, "<table>") {+ t.Fatalf("chunks payload table not rendered to HTML: %+v", upChunks.Chunks)+ }+}++// End-to-end confirmation through a real non-specialized chunker: a structured+// table (no Text) fed to the TokenChunker must surface an HTML table chunk.+func TestTokenChunkerEmitsHTMLForStructuredTable(t *testing.T) {+ component, err := NewTokenChunker(nil)+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "sheet.xlsx",+ "file_type": "xlsx",+ "output_format": "json",+ "json": []map[string]any{+ tableItem(t, "<table><tr><th>Name</th><th>Age</th></tr><tr><td>Alice</td><td>30</td></tr></table>"),+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ found := false+ for _, c := range outputChunks(t, out) {+ if s, ok := c["text"].(string); ok && strings.Contains(s, "<table>") && strings.Contains(s, "<td>Alice</td>") {+ found = true+ }+ }+ if !found {+ t.Fatalf("no chunk carries the HTML table text: %+v", out)+ }+}++// --- B: the split budget is measured on the HTML that is actually stored ---++// B1: the split budget is measured on the HTML that is actually stored. A+// budget below the whole table's HTML cost forces a split, and every emitted+// part's HTML must then fit the budget — which only holds when the budget is+// counted on the HTML rendering, not the smaller clean projection (under a+// clean-projection budget each part's HTML would overshoot).+func TestSplitLargeTableBudgetsOnHTML(t *testing.T) {+ data := make([][]string, 0, 10)+ for i := 0; i < 10; i++ {+ data = append(data, []string{"row-" + string(rune('a'+i)), "a moderately long padded value number " + string(rune('a'+i))})+ }+ td := &entity.TableData{Grid: gridOf([]string{"id", "value"}, data...), HeaderRows: 1}++ whole := tokenizeStr(tableutil.RenderTableHTML(td))+ budget := whole / 2++ parts, ranges, headerRows := splitLargeTable(td, budget, tokenizeStr)+ if len(parts) < 2 {+ t.Fatalf("table did not split under the HTML budget: whole=%d budget=%d parts=%d", whole, budget, len(parts))+ }+ if headerRows != 1 {+ t.Fatalf("headerRows = %d, want 1", headerRows)+ }+ covered := 0+ for i, p := range parts {+ if n := tokenizeStr(tableutil.RenderTableHTML(&p)); n > budget {+ t.Fatalf("part %d HTML tokens %d exceed budget %d (budget must cover the stored HTML)", i, n, budget)+ }+ covered += ranges[i][1] - ranges[i][0]+ }+ if covered != len(data) {+ t.Fatalf("parts cover %d data rows, want %d (rows lost or duplicated)", covered, len(data))+ }+}++// B2: row integrity is preserved under the HTML budget — a single row whose+// HTML alone exceeds the budget becomes its own whole chunk, never cut.+func TestSplitLargeTableHTMLBudgetKeepsSingleOverBudgetRowWhole(t *testing.T) {+ huge := strings.Repeat("x", 400)+ td := &entity.TableData{Grid: gridOf([]string{"id", "blob"},+ []string{"r1", "small"},+ []string{"r2", huge},+ []string{"r3", "small"},+ ), HeaderRows: 1}++ // Budget below the huge row's HTML but above the header + a small row.+ small := tokenizeStr(tableutil.RenderTableHTML(&entity.TableData{Grid: gridOf([]string{"id", "blob"}, []string{"r1", "small"}), HeaderRows: 1}))+ hugeRow := tokenizeStr(tableutil.RenderTableHTML(&entity.TableData{Grid: gridOf([]string{"id", "blob"}, []string{"r2", huge}), HeaderRows: 1}))+ budget := (small + hugeRow) / 2+ if budget >= hugeRow {+ t.Fatalf("test setup: budget %d should be below the huge row HTML %d", budget, hugeRow)+ }++ parts, _, _ := splitLargeTable(td, budget, tokenizeStr)+ if len(parts) < 2 {+ t.Fatalf("expected the huge row to isolate into its own part, got %d parts", len(parts))+ }+ // The part carrying the huge row must contain it whole (its blob intact).+ found := false+ for _, p := range parts {+ if n := tokenizeStr(tableutil.RenderTableHTML(&p)); n > budget {+ // An over-budget part is only acceptable when it is the single+ // huge row standing whole (row integrity wins over the budget).+ if len(p.Grid) != 2 { // header + the one data row+ t.Fatalf("over-budget part has %d rows, want the single huge row whole", len(p.Grid))+ }+ if p.Grid[1][1].Text != huge {+ t.Fatalf("over-budget part lost the huge row's content")+ }+ found = true+ }+ }+ if !found {+ t.Fatalf("the huge row was not emitted as its own over-budget part")+ }+}++// B3: span-safe cut is preserved under the HTML budget — a data-row cell whose+// rowspan covers the next data row keeps that group in one part (never split+// between the spanned rows).+func TestSplitLargeTableHTMLBudgetKeepsRowspanGroup(t *testing.T) {+ td := &entity.TableData{+ HeaderRows: 1,+ Grid: [][]entity.TableCell{+ {{Text: "h1"}, {Text: "h2"}}, // header+ {{Text: "grp", RowSpan: 2}, {Text: "a"}}, // data 0: grp spans data 0,1+ {{Text: "", Covered: true, OriginRow: 1, OriginCol: 0}, {Text: "b"}}, // data 1: covered by grp+ {{Text: "solo"}, {Text: "c"}}, // data 2+ },+ }+ // Budget = the HTML of just the header + the first (spanning) data row, so+ // the naive cut falls right after data 0; span-safe cut must pull data 1 in.+ budget := tokenizeStr(tableutil.RenderTableHTML(&entity.TableData{+ Grid: [][]entity.TableCell{td.Grid[0], td.Grid[1]},+ HeaderRows: 1,+ }))+ parts, _, _ := splitLargeTable(td, budget, tokenizeStr)+ if len(parts) < 2 {+ t.Fatalf("test setup: expected a split, got %d parts", len(parts))+ }+ // Whatever the split, the spanned rows "a" and "b" must share a part.+ partOf := map[string]int{}+ for i, p := range parts {+ html := tableutil.RenderTableHTML(&p)+ if strings.Contains(html, "<td>a</td>") {+ partOf["a"] = i+ }+ if strings.Contains(html, "<td>b</td>") {+ partOf["b"] = i+ }+ }+ if _, okA := partOf["a"]; !okA {+ t.Fatalf("spanned row 'a' missing from all parts: %+v", parts)+ }+ if _, okB := partOf["b"]; !okB {+ t.Fatalf("spanned row 'b' missing from all parts: %+v", parts)+ }+ if partOf["a"] != partOf["b"] {+ t.Fatalf("rowspan group split across parts: a in %d, b in %d", partOf["a"], partOf["b"])+ }+}diff --git a/internal/ingestion/component/chunker/table_html_text_test.go b/internal/ingestion/component/chunker/table_html_text_test.gonew file mode 100644index 0000000000..917374dea1--- /dev/null+++ b/internal/ingestion/component/chunker/table_html_text_test.go@@ -0,0 +1,339 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "encoding/json"+ "fmt"+ "strings"+ "testing"++ "ragflow/internal/entity"+)++// generalTableChunksOf drives GeneralChunker.Invoke over structured TableData+// items and returns the emitted chunk maps.+func generalTableChunksOf(t *testing.T, params map[string]any, items ...map[string]any) []map[string]any {+ t.Helper()+ component, err := NewGeneralChunker(params)+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "sheet.csv",+ "output_format": "json",+ "json": items,+ })+ if err != nil {+ t.Fatalf("GeneralChunker.Invoke: %v", err)+ }+ chunks, ok := out["chunks"].([]map[string]any)+ if !ok {+ t.Fatalf("output chunks type = %T, want []map[string]any", out["chunks"])+ }+ return chunks+}++// --- Table chunker: structured TableData renders to HTML text ---++func TestTableChunkerEmitsHTMLTextPerRow(t *testing.T) {+ chunks := tableChunksOf(t, map[string]any{+ "name": "sheet.csv",+ "output_format": "json",+ "json": []map[string]any{+ tableItem(t, "<table><tr><th>Name</th><th>Age</th></tr><tr><td>Alice</td><td>30</td></tr><tr><td>Bob</td><td>25</td></tr></table>"),+ },+ })+ if len(chunks) != 2 {+ t.Fatalf("got %d chunks, want 2 (one per data row)", len(chunks))+ }+ for i, want := range []string{"Alice", "Bob"} {+ text, ok := chunks[i]["text"].(string)+ if !ok {+ t.Fatalf("chunk %d text missing", i)+ }+ if !strings.Contains(text, "<table>") || !strings.Contains(text, "<td>"+want+"</td>") {+ t.Fatalf("chunk %d text not HTML with its row: %q", i, text)+ }+ // The header must ride along so the rendered row is a real table.+ if !strings.Contains(text, "<th>Name</th>") || !strings.Contains(text, "<th>Age</th>") {+ t.Fatalf("chunk %d HTML missing header cells: %q", i, text)+ }+ // The structured payload must not survive onto the persisted chunk.+ if _, exists := chunks[i]["table"]; exists {+ t.Fatalf("chunk %d still carries structured table: %#v", i, chunks[i]["table"])+ }+ }+}++func TestTableChunkerEmitsHTMLTextHeaderOnly(t *testing.T) {+ chunks := tableChunksOf(t, map[string]any{+ "name": "sheet.csv",+ "output_format": "json",+ "json": []map[string]any{+ tableItem(t, "<table><tr><th>Name</th><th>Age</th></tr></table>"),+ },+ })+ if len(chunks) != 1 {+ t.Fatalf("got %d chunks, want 1 (header-only)", len(chunks))+ }+ text, _ := chunks[0]["text"].(string)+ if !strings.Contains(text, "<table>") || !strings.Contains(text, "<th>Name</th>") {+ t.Fatalf("header-only chunk text not HTML: %q", text)+ }+ if _, exists := chunks[0]["table"]; exists {+ t.Fatalf("header-only chunk still carries structured table")+ }+}++func TestTableChunkerEmitsHTMLTextKeepsAllHeaderRows(t *testing.T) {+ chunks := tableChunksOf(t, map[string]any{+ "name": "sheet.csv",+ "output_format": "json",+ "json": []map[string]any{+ tableItem(t, "<table><tr><th>Group</th><th>Metric</th></tr><tr><th>A</th><th>B</th></tr><tr><td>x</td><td>1</td></tr></table>"),+ },+ })+ if len(chunks) != 1 {+ t.Fatalf("got %d chunks, want 1", len(chunks))+ }+ text, _ := chunks[0]["text"].(string)+ // Both projected header rows must survive into the rendered HTML.+ for _, want := range []string{"<th>Group</th>", "<th>Metric</th>", "<th>A</th>", "<th>B</th>", "<td>x</td>"} {+ if !strings.Contains(text, want) {+ t.Fatalf("multi-header HTML missing %q: %q", want, text)+ }+ }+}++func TestTableChunkerEmitsHTMLTextEscapesSpecialChars(t *testing.T) {+ // Build the structured TableData directly so the cell bodies literally+ // contain <, >, & and " — characters the HTML parser would otherwise treat+ // as markup. The rendered chunk text must escape every one of them so the+ // persisted HTML stays well-formed and no cell body leaks as a live tag.+ td := &entity.TableData{+ Grid: [][]entity.TableCell{+ {{Text: "k<>&\""}},+ {{Text: "a<b>&\"c\""}},+ },+ HeaderRows: 1,+ }+ b, err := json.Marshal(td)+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ var tbl map[string]any+ if err := json.Unmarshal(b, &tbl); err != nil {+ t.Fatalf("unmarshal: %v", err)+ }+ chunks := tableChunksOf(t, map[string]any{+ "name": "sheet.csv",+ "output_format": "json",+ "json": []map[string]any{{"table": tbl, "doc_type_kwd": "table"}},+ })+ if len(chunks) != 1 {+ t.Fatalf("got %d chunks, want 1", len(chunks))+ }+ text, _ := chunks[0]["text"].(string)+ for _, want := range []string{"<", ">", "&", """} {+ if !strings.Contains(text, want) {+ t.Fatalf("cell body not HTML-escaped (missing %q): %q", want, text)+ }+ }+ if strings.Contains(text, "<b>") {+ t.Fatalf("cell markup leaked unescaped into HTML: %q", text)+ }+}++func TestTableChunkerEmitsHTMLTextCaption(t *testing.T) {+ chunks := tableChunksOf(t, map[string]any{+ "name": "sheet.csv",+ "output_format": "json",+ "json": []map[string]any{+ tableItem(t, "<table><caption>Q1 Sales</caption><tr><th>k</th></tr><tr><td>v</td></tr></table>"),+ },+ })+ if len(chunks) != 1 {+ t.Fatalf("got %d chunks, want 1", len(chunks))+ }+ text, _ := chunks[0]["text"].(string)+ if !strings.Contains(text, "<caption>Q1 Sales</caption>") {+ t.Fatalf("caption not rendered into HTML: %q", text)+ }+}++func TestTableChunkerEmitsHTMLTextNilTableDataPassthrough(t *testing.T) {+ // An item without structured TableData is not a table payload: it must+ // pass through untouched (text preserved), never rendered.+ chunks := tableChunksOf(t, map[string]any{+ "name": "sheet.csv",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "plain body", "doc_type_kwd": "text"},+ },+ })+ if len(chunks) != 1 {+ t.Fatalf("got %d chunks, want 1", len(chunks))+ }+ if got := chunks[0]["text"]; got != "plain body" {+ t.Fatalf("non-table text mutated: %q", got)+ }+}++func TestTableChunkerEmitsHTMLTextWideRowStaysWhole(t *testing.T) {+ // A single very wide row must become exactly one chunk whose HTML carries+ // every cell — the row is never cut across chunks.+ var b strings.Builder+ b.WriteString("<table><tr>")+ for i := 0; i < 40; i++ {+ b.WriteString("<th>h")+ b.WriteString(strings.Repeat("x", i%5))+ b.WriteString("</th>")+ }+ b.WriteString("</tr><tr>")+ for i := 0; i < 40; i++ {+ b.WriteString("<td>cell")+ b.WriteByte(byte('a' + i%26))+ b.WriteString("</td>")+ }+ b.WriteString("</tr></table>")++ chunks := tableChunksOf(t, map[string]any{+ "name": "sheet.csv",+ "output_format": "json",+ "json": []map[string]any{tableItem(t, b.String())},+ })+ if len(chunks) != 1 {+ t.Fatalf("wide row produced %d chunks, want 1 (row never cut)", len(chunks))+ }+ text, _ := chunks[0]["text"].(string)+ for _, want := range []string{"<td>cella</td>", "<td>cells</td>", "<td>cellz</td>"} {+ if !strings.Contains(text, want) {+ t.Fatalf("wide-row HTML missing %q", want)+ }+ }+}++// --- General chunker: structured spreadsheet table renders to HTML text ---++func TestGeneralChunkerEmitsHTMLTextSingleChunk(t *testing.T) {+ // A small table that fits the budget becomes a single chunk whose text is+ // the whole table rendered as HTML (not the plain-text projection).+ chunks := generalTableChunksOf(t, nil, tableItem(t,+ "<table><caption>People</caption><tr><th>Name</th><th>Age</th></tr><tr><td>Alice</td><td>30</td></tr></table>"))+ if len(chunks) != 1 {+ t.Fatalf("got %d chunks, want 1", len(chunks))+ }+ text, ok := chunks[0]["text"].(string)+ if !ok {+ t.Fatalf("text missing")+ }+ if !strings.Contains(text, "<table>") || !strings.Contains(text, "<td>Alice</td>") {+ t.Fatalf("single-chunk table text not HTML: %q", text)+ }+ if !strings.Contains(text, "<caption>People</caption>") {+ t.Fatalf("caption not rendered: %q", text)+ }+ if _, exists := chunks[0]["table"]; exists {+ t.Fatalf("structured table leaked onto chunk: %#v", chunks[0]["table"])+ }+}++func TestGeneralChunkerEmitsHTMLTextLargeTableRowGranular(t *testing.T) {+ // Force a split with a tiny budget. The table must break on row boundaries+ // (never mid-row), and every emitted part must be HTML carrying the header.+ const nRows = 30+ var b strings.Builder+ b.WriteString("<table><tr><th>id</th><th>val</th></tr>")+ for i := 0; i < nRows; i++ {+ b.WriteString(fmt.Sprintf("<tr><td>r%02d</td><td>%s</td></tr>", i, strings.Repeat("v", 8)))+ }+ b.WriteString("</table>")++ chunks := generalTableChunksOf(t, map[string]any{"chunk_token_size": 40}, tableItem(t, b.String()))+ if len(chunks) < 2 {+ t.Fatalf("got %d chunks, want >=2 (budget forces a split)", len(chunks))+ }+ seen := map[string]bool{}+ for i, ck := range chunks {+ text, ok := ck["text"].(string)+ if !ok || !strings.Contains(text, "<table>") {+ t.Fatalf("part %d not HTML: %q", i, text)+ }+ if !strings.Contains(text, "<th>id</th>") || !strings.Contains(text, "<th>val</th>") {+ t.Fatalf("part %d missing replicated header: %q", i, text)+ }+ // Each part is whole rows only: every <tr> opens and closes.+ if strings.Count(text, "<tr>") != strings.Count(text, "</tr>") {+ t.Fatalf("part %d has unbalanced rows (a row was cut): %q", i, text)+ }+ // Collect the unique data-row ids that landed in this part; a row id+ // appearing in two parts would mean it was split or duplicated.+ rest := text+ for {+ idx := strings.Index(rest, "<td>r")+ if idx < 0 {+ break+ }+ id := rest[idx+4 : idx+7] // "r%02d"+ rest = rest[idx+7:]+ if seen[id] {+ t.Fatalf("row id %q appears in two parts (split/duplicated)", id)+ }+ seen[id] = true+ }+ }+ if len(seen) != nRows {+ t.Fatalf("saw %d distinct rows across parts, want %d (rows lost or merged)", len(seen), nRows)+ }+}++// --- Row-span group integrity across the HTML rendering path ---++func TestGeneralChunkerEmitsHTMLTextRowspanGroupNotSplit(t *testing.T) {+ // A header cell spanning two data rows must keep its group in one part; the+ // rendered HTML must show the colspan/rowspan, and no part may cut the group.+ td := &entity.TableData{+ Grid: [][]entity.TableCell{+ {{Text: "g", RowSpan: 2}, {Text: "m"}},+ {{Text: "", Covered: true, OriginRow: 0, OriginCol: 0}, {Text: "a"}},+ {{Text: "", Covered: true, OriginRow: 0, OriginCol: 0}, {Text: "b"}},+ },+ HeaderRows: 1,+ }+ b, err := json.Marshal(td)+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ var tbl map[string]any+ if err := json.Unmarshal(b, &tbl); err != nil {+ t.Fatalf("unmarshal: %v", err)+ }+ chunks := generalTableChunksOf(t, map[string]any{"chunk_token_size": 40},+ map[string]any{"table": tbl, "doc_type_kwd": "table"})+ for i, ck := range chunks {+ text, _ := ck["text"].(string)+ // The origin cell and every row it covers must be in the same part:+ // if a part contains "b" (the last spanned data row) it must also+ // contain the origin "g" with its rowspan.+ if strings.Contains(text, "<td>b</td>") {+ if !strings.Contains(text, "rowspan=\"2\"") {+ t.Fatalf("part %d split the rowspan group: %q", i, text)+ }+ }+ }+}diff --git a/internal/ingestion/component/chunker/table_split.go b/internal/ingestion/component/chunker/table_split.goindex aeef29cb3e..ce3c0efa6d 100644--- a/internal/ingestion/component/chunker/table_split.go+++ b/internal/ingestion/component/chunker/table_split.go@@ -17,8 +17,6 @@package chunkerimport (- "strings"-"ragflow/internal/entity""ragflow/internal/parser/tableutil")@@ -27,7 +25,8 @@ import (// splitting it into sub-tables that fit. Splitting is row-granular: every// emitted sub-table is a whole number of complete rows (never a partial row),// so the per-row chunk produced downstream keeps full row semantics. The token-// budget is counted on the real cell text of each row, not on any HTML markup.+// budget is counted on the HTML rendering of each row — the text actually+// stored on the chunk — so a rendered part never exceeds the budget.//// Splitting is also span-safe: a cut never falls inside a merged cell's// rowspan, so an origin and every row it covers stay in the same part, and a@@ -57,9 +56,14 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strinheaderGrid := append([][]entity.TableCell{}, td.Grid[:headerRows]...)- // Base tokens: header rows + caption, replicated into every sub-table.+ // Base tokens: header rows + caption, replicated into every sub-table,+ // counted on the HTML rendering — the text actually stored on the chunk —+ // so a rendered part never exceeds the budget. The <table> wrapper + caption+ // appear once per part; emptyWrapper isolates them so each data row below is+ // charged only its own <tr> contribution.base := &entity.TableData{Grid: headerGrid, Caption: td.Caption}- baseTokens := countTokens(tableutil.RenderTableText(base))+ baseTokens := countTokens(tableutil.RenderTableHTML(base))+ emptyWrapper := countTokens(tableutil.RenderTableHTML(&entity.TableData{Caption: td.Caption}))// spanThrough[i] is the last data-row index any rowspan originating at or// before row i still covers (-1 while no span is active). Header-origin@@ -115,7 +119,7 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strinif i < start {continue // already belongs to a flushed part}- rowTokens := countTokens(rowTokenText(row))+ rowTokens := countTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: [][]entity.TableCell{row}, Caption: td.Caption})) - emptyWrapper// Flush the running batch before this row if adding it would exceed the// budget. A single row that alone exceeds the budget still becomes its// own chunk (row integrity wins over the budget).@@ -194,16 +198,3 @@ func partDataRows(dataRows [][]entity.TableCell, start, end, headerRows int) [][}return out}--// rowTokenText joins the non-covered cell texts of a grid row for token-// counting, mirroring how the row is later rendered.-func rowTokenText(row []entity.TableCell) string {- texts := make([]string, 0, len(row))- for _, c := range row {- if c.Covered {- continue- }- texts = append(texts, c.Text)- }- return strings.Join(texts, " ")-}diff --git a/internal/ingestion/component/chunker/table_split_test.go b/internal/ingestion/component/chunker/table_split_test.goindex 7ae56f10cb..4f54154e78 100644--- a/internal/ingestion/component/chunker/table_split_test.go+++ b/internal/ingestion/component/chunker/table_split_test.go@@ -169,40 +169,51 @@ func TestSplitLargeTableHeaderRowSpanClippedPerPart(t *testing.T) {if err != nil || td == nil {t.Fatalf("parse: %v", err)}- // Budget 6: parts hold data rows [0,2), [2,3), [3,4), [4,5). The first- // part carries both rows A spans, so A keeps RowSpan 3; the later parts- // carry none of them, so A degrades to a plain header cell.- parts, ranges, headerRows := splitLargeTable(td, 6, charTokens)- if headerRows != 1 || len(parts) != 4 {+ // Budget = the HTML cost of the header + the two rows A spans (y, z), so+ // the first part is exactly those rows and keeps A's RowSpan 3; later parts+ // carry none of them, so A degrades to a plain header cell (RowSpan 1).+ budget := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:3], HeaderRows: 1}))+ parts, ranges, headerRows := splitLargeTable(td, budget, charTokens)+ if headerRows != 1 || len(parts) < 2 {t.Fatalf("parts=%d ranges=%v headerRows=%d", len(parts), ranges, headerRows)}- wantRanges := [][2]int{{0, 2}, {2, 3}, {3, 4}, {4, 5}}- if !reflect.DeepEqual(ranges, wantRanges) {- t.Fatalf("ranges=%v want %v", ranges, wantRanges)+ if ranges[0] != [2]int{0, 2} {+ t.Fatalf("first range = %v, want [0 2] (A's two spanned rows)", ranges[0])}- wantSpans := []int{3, 1, 1, 1}- for i, part := range parts {- assertPartGridContract(t, &part, i)- if got := part.Grid[0][0].RowSpan; got != wantSpans[i] {- t.Fatalf("part %d header A RowSpan=%d want %d", i, got, wantSpans[i])+ if got := parts[0].Grid[0][0].RowSpan; got != 3 {+ t.Fatalf("part 0 A RowSpan = %d, want 3 (carries both spanned rows)", got)+ }+ for i := 1; i < len(parts); i++ {+ assertPartGridContract(t, &parts[i], i)+ if got := parts[i].Grid[0][0].RowSpan; got != 1 {+ t.Fatalf("part %d A RowSpan = %d, want 1 (carries none of A's spanned rows)", i, got)}}- // Budget 4: every part holds a single data row, so A keeps exactly the- // covered rows that travel with the part.- parts, _, _ = splitLargeTable(td, 4, charTokens)- wantSpans = []int{2, 2, 1, 1, 1}- if len(parts) != len(wantSpans) {- t.Fatalf("parts=%d want %d", len(parts), len(wantSpans))+ covered := 0+ for _, r := range ranges {+ covered += r[1] - r[0]}- for i, part := range parts {- assertPartGridContract(t, &part, i)- if got := part.Grid[0][0].RowSpan; got != wantSpans[i] {- t.Fatalf("part %d header A RowSpan=%d want %d", i, got, wantSpans[i])- }+ if covered != 5 {+ t.Fatalf("parts cover %d data rows, want 5", covered)}- wantTexts := [][]string{{"y"}, {"z"}, {"p", "q"}, {"r", "s"}, {"t", "u"}}- if got := partDataTexts(parts, headerRows); !reflect.DeepEqual(got, wantTexts) {- t.Fatalf("data texts=%v want %v", got, wantTexts)++ // Budget = the HTML cost of the header + only the first spanned row (y), so+ // the first part carries exactly one of A's spanned rows and A keeps+ // RowSpan 2 (header + that one row), not the full 3.+ budget = charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:2], HeaderRows: 1}))+ parts, ranges, _ = splitLargeTable(td, budget, charTokens)+ if len(parts) < 2 || ranges[0] != [2]int{0, 1} {+ t.Fatalf("parts=%d ranges=%v, want first part [0 1]", len(parts), ranges)+ }+ if got := parts[0].Grid[0][0].RowSpan; got != 2 {+ t.Fatalf("part 0 A RowSpan = %d, want 2 (carries exactly one spanned row)", got)+ }+ covered = 0+ for _, r := range ranges {+ covered += r[1] - r[0]+ }+ if covered != 5 {+ t.Fatalf("parts cover %d data rows, want 5", covered)}}@@ -222,15 +233,19 @@ func TestSplitLargeTableNeverSplitsInsideRowSpan(t *testing.T) {if err != nil || td == nil {t.Fatalf("parse: %v", err)}- // The budget alone would cut after data row 2, inside B's span (B covers- // data rows 0..2); the cut must move to after row 2, i.e. to 3.- parts, ranges, headerRows := splitLargeTable(td, 11, charTokens)+ // Budget = the HTML cost of the header + B's own row only. The budget alone+ // would cut right after data row 0, inside B's span (B covers data rows+ // 0..2); span-safe cut must move the boundary to after row 2, i.e. to 3.+ budget := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:2], HeaderRows: 1}))+ parts, ranges, headerRows := splitLargeTable(td, budget, charTokens)if headerRows != 1 {t.Fatalf("headerRows=%d want 1", headerRows)}- wantRanges := [][2]int{{0, 3}, {3, 5}}- if !reflect.DeepEqual(ranges, wantRanges) {- t.Fatalf("ranges=%v want %v (cut must not fall inside B's rowspan)", ranges, wantRanges)+ // The first boundary must land after B's span (row 3), never inside it —+ // that is the span-safe guarantee. (How the remaining rows chunk is purely a+ // budget artifact and not what this test pins.)+ if ranges[0] != [2]int{0, 3} {+ t.Fatalf("first range = %v, want [0 3] (cut must not fall inside B's rowspan)", ranges[0])}for i, part := range parts {assertPartGridContract(t, &part, i)diff --git a/internal/ingestion/component/chunker/table_struct_test.go b/internal/ingestion/component/chunker/table_struct_test.goindex 06f9d3b9cb..5e3988a00d 100644--- a/internal/ingestion/component/chunker/table_struct_test.go+++ b/internal/ingestion/component/chunker/table_struct_test.go@@ -15,12 +15,33 @@package chunkerimport (+ "strings""testing""ragflow/internal/entity""ragflow/internal/ingestion/component/schema")+// wantHTMLCells asserts the chunk text is an HTML table carrying exactly the+// given header and data cells (in order). The row chunks render header+row as+// HTML, so the projected row's cells appear as <td> values.+func wantHTMLCells(t *testing.T, text string, header, data []string) {+ t.Helper()+ if !strings.Contains(text, "<table>") {+ t.Fatalf("not HTML: %q", text)+ }+ for _, h := range header {+ if !strings.Contains(text, "<th>"+h+"</th>") {+ t.Fatalf("missing header cell %q in %q", h, text)+ }+ }+ for _, d := range data {+ if !strings.Contains(text, "<td>"+d+"</td>") {+ t.Fatalf("missing data cell %q in %q", d, text)+ }+ }+}+func TestExpandHTMLTableRowsFromTableData(t *testing.T) {td := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}, {{Text: "Alice"}, {Text: "30"}}, {{Text: "Bob"}, {Text: "25"}}},@@ -30,12 +51,8 @@ func TestExpandHTMLTableRowsFromTableData(t *testing.T) {if len(out) != 2 {t.Fatalf("got %d chunks, want 2", len(out))}- if out[0].Text != "- Name: Alice\n- Age: 30" {- t.Fatalf("row0 text = %q", out[0].Text)- }- if out[1].Text != "- Name: Bob\n- Age: 25" {- t.Fatalf("row1 text = %q", out[1].Text)- }+ wantHTMLCells(t, out[0].Text, []string{"Name", "Age"}, []string{"Alice", "30"})+ wantHTMLCells(t, out[1].Text, []string{"Name", "Age"}, []string{"Bob", "25"})for _, c := range out {if c.TableData != nil {t.Fatalf("TableData must be cleared on emitted row chunk, got %+v", c.TableData)@@ -89,15 +106,11 @@ func TestExpandHTMLTableRowsKeepsRowspanColumns(t *testing.T) {if len(out) != 2 {t.Fatalf("got %d chunks, want 2", len(out))}- if out[0].Text != "- Name: Alice\n- Age: 30" {- t.Fatalf("row0 text = %q", out[0].Text)- }+ wantHTMLCells(t, out[0].Text, []string{"Name", "Age"}, []string{"Alice", "30"})// The continuation row must keep the spanned "Name" column resolved to the- // origin value and stay aligned: "Name: Alice" + "Age: 25", not a shifted- // "Name: 25" that drops the Age column.- if out[1].Text != "- Name: Alice\n- Age: 25" {- t.Fatalf("row1 text = %q (rowspan column must be preserved, not dropped/misaligned)", out[1].Text)- }+ // origin value and stay aligned: "Alice" + "25", not a shifted "25" that+ // drops the Age column.+ wantHTMLCells(t, out[1].Text, []string{"Name", "Age"}, []string{"Alice", "25"})}// A cell that spans BOTH rows and columns (RowSpan>1 && ColSpan>1) must keep@@ -127,14 +140,11 @@ func TestExpandHTMLTableRowsKeepsRowColSpanBlock(t *testing.T) {if len(out) != 2 {t.Fatalf("got %d chunks, want 2 (one per data row)", len(out))}- if out[0].Text != "- Name: Wide\n- B: Wide\n- C: x" {- t.Fatalf("row0 text = %q", out[0].Text)- }+ // The origin row keeps both covered columns resolved to "Wide".+ wantHTMLCells(t, out[0].Text, []string{"Name", "B", "C"}, []string{"Wide", "Wide", "x"})// The continuation row must keep BOTH covered columns resolved to "Wide",- // not drop the second one to a blank and misalign under the header.- if out[1].Text != "- Name: Wide\n- B: Wide\n- C: y" {- t.Fatalf("row1 text = %q (both spanned columns must be preserved)", out[1].Text)- }+ // not drop the second one and misalign under the header.+ wantHTMLCells(t, out[1].Text, []string{"Name", "B", "C"}, []string{"Wide", "Wide", "y"})}// A data row whose first cell spans two columns must keep both columns after@@ -159,7 +169,6 @@ func TestExpandHTMLTableRowsKeepsColspanColumns(t *testing.T) {if len(out) != 1 {t.Fatalf("got %d chunks, want 1", len(out))}- if out[0].Text != "- Name: Info\n- Age: Info" {- t.Fatalf("row text = %q (colspan must fill both columns)", out[0].Text)- }+ // colspan must fill both columns with the spanned value "Info".+ wantHTMLCells(t, out[0].Text, []string{"Name", "Age"}, []string{"Info", "Info"})}diff --git a/internal/ingestion/component/chunker/table_test.go b/internal/ingestion/component/chunker/table_test.goindex 596766c3f3..86f7b39c3c 100644--- a/internal/ingestion/component/chunker/table_test.go+++ b/internal/ingestion/component/chunker/table_test.go@@ -148,8 +148,10 @@ func TestTableChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {t.Fatalf("got %d chunks, want data row plus header-only segment", len(chunks))}// The header row of the data segment never becomes a chunk of its own;- // its column names live in the row record text instead.- if chunks[0]["text"] != "- ID: A-1\n- Name: paid" {+ // its column names ride along as the HTML header of the row chunk.+ row0, _ := chunks[0]["text"].(string)+ if !strings.Contains(row0, "<table>") || !strings.Contains(row0, "<td>A-1</td>") ||+ !strings.Contains(row0, "<td>paid</td>") || !strings.Contains(row0, "<th>ID</th>") {t.Fatalf("chunk0 = %#v", chunks[0])}if chunks[1]["ck_type"] != "table" || chunks[1]["table"] != nil {@@ -181,11 +183,14 @@ func TestTableChunker_ExpandsHTMLRowsWithAlignedPositions(t *testing.T) {if len(chunks) != 2 {t.Fatalf("got %d chunks, want one per data row", len(chunks))}- if chunks[0]["text"] != "- ID: A-1\n- Status: paid" {+ c0, _ := chunks[0]["text"].(string)+ if !strings.Contains(c0, "<td>A-1</td>") || !strings.Contains(c0, "<td>paid</td>") || !strings.Contains(c0, "<th>ID</th>") {t.Errorf("chunk0 text = %v", chunks[0]["text"])}- // The empty cell is skipped, matching the Python table chunker's per-cell skip.- if chunks[1]["text"] != "- ID: A-2" {+ // The empty cell is skipped in the row record; the HTML row keeps the row's+ // own id and renders the empty cell plainly.+ c1, _ := chunks[1]["text"].(string)+ if !strings.Contains(c1, "<td>A-2</td>") {t.Errorf("chunk1 text = %v", chunks[1]["text"])}if got := fmt.Sprintf("%v", chunks[0]["positions"]); !strings.Contains(got, "2") || strings.Contains(got, "3") {@@ -249,15 +254,15 @@ func TestTableChunker_NonSpreadsheetPositionsNotAttached(t *testing.T) {}// TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload: a table whose only-// row is the header has nothing to expand; the whole markup stays one chunk-// carrying the plain-text rendering of the header.+// row is the header has nothing to expand; the whole table stays one chunk+// carrying its HTML rendering.func TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload(t *testing.T) {table := "<table><caption>t</caption>\n<tr><th>A</th><th>B</th></tr>\n</table>\n"td, err := tableutil.ParseTableHTML(table)if err != nil {t.Fatalf("ParseTableHTML: %v", err)}- want := tableutil.RenderTableText(td)+ want := tableutil.RenderTableHTML(td)chunks := tableChunksOf(t, map[string]any{"name": "template.xlsx","output_format": "json",@@ -287,7 +292,9 @@ func TestTableChunker_HeaderlessHTMLFirstRowIsHeader(t *testing.T) {if len(chunks) != 1 {t.Fatalf("got %d chunks, want the single data row", len(chunks))}- if chunks[0]["text"] != "- k: a\n- v: b" {+ txt, _ := chunks[0]["text"].(string)+ if !strings.Contains(txt, "<th>k</th>") || !strings.Contains(txt, "<th>v</th>") ||+ !strings.Contains(txt, "<td>a</td>") || !strings.Contains(txt, "<td>b</td>") {t.Errorf("text = %v", chunks[0]["text"])}}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 40c39a6194..1b8cd2fc72 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -257,18 +257,18 @@ func decodeChunkerFromUpstream(inputs map[string]any) (schema.ChunkerFromUpstreaif err := out.Validate(); err != nil {return out, err}- // A structured table carries no markup text of its own. Derive a plain- // (tag-free) text from its cells so consumers that read Text — TokenChunker- // and the embed path — still receive a searchable body when the producer- // emitted only the TableData contract (decision 1-B: no HTML in Text).+ // A structured table (e.g. the office/xlsx producer) carries no Text of its+ // own. Derive its HTML <table> rendering on Text so every chunker that+ // consumes it (token/group/hierarchy/one/page/qa) stores and displays a real+ // table, and the chunk id (a hash of Text) matches the stored content.for i := range out.JSONResult {if out.JSONResult[i].TableData != nil && out.JSONResult[i].Text == "" {- out.JSONResult[i].Text = tableutil.RenderTableText(out.JSONResult[i].TableData)+ out.JSONResult[i].Text = tableutil.RenderTableHTML(out.JSONResult[i].TableData)}}for i := range out.Chunks {if out.Chunks[i].TableData != nil && out.Chunks[i].Text == "" {- out.Chunks[i].Text = tableutil.RenderTableText(out.Chunks[i].TableData)+ out.Chunks[i].Text = tableutil.RenderTableHTML(out.Chunks[i].TableData)}}return out, nildiff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex f19a2dbb05..ab41cfb45b 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -105,9 +105,9 @@ const ComponentNameTokenizer = "Tokenizer"// `re.sub(r"\.[a-zA-Z]+$", "", name)` in tokenizer.py:137.var titleExtRE = regexp.MustCompile(`\.[a-zA-Z]+$`)-// Table content is no longer carried as HTML markup (the parser emits the-// structured entity.TableData contract), so there is no table markup to strip-// from the embedded text. The legacy htmlTableRE was removed.+// Table chunks carry their row rendered as an HTML <table> on Text (produced+// by the chunker), so the embedding input intentionally includes the table+// markup — there is no markup to strip. The legacy htmlTableRE stays removed.// EmbeddingResult carries a vector plus the model-reported token usage// for that input batch entry.diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.goindex ce770512a1..97ea6d8ca8 100644--- a/internal/ingestion/task/indexdoc/process_test.go+++ b/internal/ingestion/task/indexdoc/process_test.go@@ -4,6 +4,8 @@ import ("reflect""testing""time"++ "ragflow/internal/common")func TestProcessChunksForPipeline_PreservesTypedMetadata(t *testing.T) {@@ -189,6 +191,34 @@ func TestProcessChunksForPipeline_PreservesExistingID(t *testing.T) {}}+// TestProcessChunksForPipeline_TableHTMLIDMatchesContent pins the table-HTML+// redesign's core invariant: a table chunk arrives from the chunker with its+// row rendered as HTML on "text" and an id already stamped (by the chunker's+// decorator) from that HTML text. The persist boundary must keep the id+// byte-consistent with the stored content_with_weight — both hash the same+// HTML — so re-indexing stays stable and the displayed table never diverges+// from the chunk id. (The old design stamped the id from the clean text while+// storing HTML, which broke this.)+func TestProcessChunksForPipeline_TableHTMLIDMatchesContent(t *testing.T) {+ html := "<table>\n<tr><th>Name</th></tr>\n<tr><td>Alice</td></tr>\n</table>\n"+ chunks := []map[string]any{{+ "doc_type": "table",+ "text": html,+ "id": common.ChunkID("doc-1", html),+ }}+ if _, err := ProcessChunksForPipeline(chunks, "doc-1", "report.pdf", time.Now()); err != nil {+ t.Fatalf("ProcessChunksForPipeline: %v", err)+ }+ ck := chunks[0]+ content, ok := ck["content_with_weight"].(string)+ if !ok || content != html {+ t.Fatalf("content_with_weight = %q, want the HTML text unchanged", content)+ }+ if want := common.ChunkID("doc-1", content); ck["id"] != want {+ t.Fatalf("id = %v, want ChunkID(docID, content_with_weight) = %v", ck["id"], want)+ }+}+func TestProcessChunksForPipeline_QuestionsProcessing(t *testing.T) {chunks := []map[string]any{{"text": "hello", "questions": "Q1\nQ2\nQ3"}}_, err := ProcessChunksForPipeline(chunks, "doc-1", "test-doc.pdf", time.Now())diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex 059279b3a1..229e9e2aa7 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -71,7 +71,8 @@ func trimCells(row []string) []string {// loses column names) and carries a row-aligned position matrix — one tuple// per row in strict order, header included — so row-level consumers index// positions by row number with no side channel. The chunker reads the rows-// directly; no HTML markup is emitted.+// directly and renders each emitted chunk's Text as an HTML <table> (see+// tableutil.RenderTableHTML); no HTML markup is emitted here at the parser.//// An image lands after the last row whose (row, colStart) sorts at or before// its anchor; rows before the header's anchor keep the header in front ofdiff --git a/internal/parser/parser/pdf_parser_table_contract_test.go b/internal/parser/parser/pdf_parser_table_contract_test.goindex c804d46a47..5e7e114498 100644--- a/internal/parser/parser/pdf_parser_table_contract_test.go+++ b/internal/parser/parser/pdf_parser_table_contract_test.go@@ -5,9 +5,20 @@ import ("testing""ragflow/internal/entity"- "ragflow/internal/parser/tableutil")+// cellTextsContain reports whether any cell of the structured table contains sub.+func cellTextsContain(td *entity.TableData, sub string) bool {+ for _, row := range td.CellTexts() {+ for _, cell := range row {+ if strings.Contains(cell, sub) {+ return true+ }+ }+ }+ return false+}+// Consumers of doc_type_kwd "table" parse the item text as <table> markup// (the QA chunker reads its <tr> rows), so a table item that carries other// text loses its rows downstream. The structured TableData contract must also@@ -91,8 +102,8 @@ func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {if len(td.CellTexts()) == 0 {t.Fatalf("structured table has no rows: %#v", td)}- if got := tableutil.RenderTableText(td); !strings.Contains(got, "What is RAGFlow?") {- t.Fatalf("structured table missing cell text %q: %#v", got, td)+ if !cellTextsContain(td, "What is RAGFlow?") {+ t.Fatalf("structured table missing cell text %q: %#v", "What is RAGFlow?", td)}}})diff --git a/internal/parser/parser/pdf_table_no_html_text_test.go b/internal/parser/parser/pdf_table_no_html_text_test.goindex 091f4b0e39..305965af88 100644--- a/internal/parser/parser/pdf_table_no_html_text_test.go+++ b/internal/parser/parser/pdf_table_no_html_text_test.go@@ -61,8 +61,8 @@ func TestPDFTableItemsOmitHTMLText(t *testing.T) {if !ok || td == nil || len(td.CellTexts()) == 0 {t.Fatalf("table item missing structured TableData: %#v", tableItem)}- if got := tableutil.RenderTableText(td); !strings.Contains(got, "What is RAGFlow?") && !strings.Contains(got, "cell") {- t.Fatalf("structured table missing cell text %q: %#v", got, td)+ if !cellTextsContain(td, "What is RAGFlow?") && !cellTextsContain(td, "cell") {+ t.Fatalf("structured table missing cell text %q: %#v", "What is RAGFlow?", td)}})}diff --git a/internal/parser/tableutil/table_grid_test.go b/internal/parser/tableutil/table_grid_test.goindex f8326950de..0dfbf4e8b2 100644--- a/internal/parser/tableutil/table_grid_test.go+++ b/internal/parser/tableutil/table_grid_test.go@@ -353,13 +353,13 @@ func TestRenderNonMergedByteIdentical(t *testing.T) {}}-// RenderTableText strips tags; a merged table yields the same stripped text as-// the equivalent non-merged table (text-level output is unchanged by spans).-func TestRenderTextMergedSameAsNonMerged(t *testing.T) {+// CellTexts is span-agnostic: a merged table yields the same cell texts as the+// equivalent non-merged table (the Covered slot contributes no text).+func TestCellTextsMergedSameAsNonMerged(t *testing.T) {merged, _ := ParseTableHTML(`<table><tr><td colspan="2">wide</td><td>narrow</td></tr></table>`)plain, _ := ParseTableHTML(`<table><tr><td>wide</td><td>narrow</td></tr></table>`)- if RenderTableText(merged) != RenderTableText(plain) {- t.Fatalf("merged text %q != plain text %q", RenderTableText(merged), RenderTableText(plain))+ if !reflect.DeepEqual(merged.CellTexts(), plain.CellTexts()) {+ t.Fatalf("merged %v != plain %v", merged.CellTexts(), plain.CellTexts())}}diff --git a/internal/parser/tableutil/table_html.go b/internal/parser/tableutil/table_html.goindex 3006714fe4..b257061fff 100644--- a/internal/parser/tableutil/table_html.go+++ b/internal/parser/tableutil/table_html.go@@ -20,7 +20,6 @@package tableutilimport (- "regexp""strconv""strings"@@ -329,12 +328,10 @@ func attrInt(n *html.Node, name string, max int) int {}// RenderTableHTML emits a deterministic <table> markup for a TableData. It is-// used only as a test oracle / round-trip helper and is NOT part of the-// persisted contract (producers emit TableData, not HTML). The emitted shape-// matches the legacy spreadsheet producer's renderSpreadsheetTable exactly-// (including the inter-row newlines) for 1x1 grids, so RenderTableText-// reproduces the legacy htmlTableRE-stripped text byte for byte. For merged-// tables it emits colspan/rowspan on the origin slot and skips Covered slots.+// the persisted display form of a table chunk: the chunker renders each table+// chunk's Text with it, and the chunk id is a hash of that rendered text. For+// merged tables it emits colspan/rowspan on the origin slot and skips Covered+// slots.func RenderTableHTML(td *entity.TableData) string {if td == nil {return ""@@ -384,22 +381,6 @@ func RenderTableHTML(td *entity.TableData) string {return b.String()}-// tableTagRE strips table markup tags (with any attributes) so the plaintext a-// chunker writes for an embed-only path matches the legacy tag-stripped output.-// The pattern tolerates arbitrary attribute lists (including colspan/rowspan)-// so merged-table markup is stripped completely rather than left partly in the-// text.-var tableTagRE = regexp.MustCompile(`</?(?:table|td|caption|tr|th)(?:\s[^<>]*)?>`)--// RenderTableText renders a TableData to plain text with no markup tags, for-// the rare embed path that receives a table without a downstream TableChunker.-// It reproduces the legacy htmlTableRE stripping of RenderTableHTML(td).-func RenderTableText(td *entity.TableData) string {- h := RenderTableHTML(td)- h = tableTagRE.ReplaceAllString(h, " ")- return strings.TrimSpace(h)-}-// cellText returns the visible text of a table cell: text nodes contribute// verbatim, <br> becomes a newline, inert elements are skipped, and other// elements are descended. The result is TrimSpace'd.diff --git a/internal/parser/tableutil/table_html_test.go b/internal/parser/tableutil/table_html_test.goindex f8c0060d8e..80b365043a 100644--- a/internal/parser/tableutil/table_html_test.go+++ b/internal/parser/tableutil/table_html_test.go@@ -192,24 +192,6 @@ func TestRenderTableHTMLRoundTrip(t *testing.T) {}}-func TestRenderTableTextHasNoTags(t *testing.T) {- td := &entity.TableData{- Grid: [][]entity.TableCell{- {{Text: "Name"}, {Text: "Age"}},- {{Text: "Alice"}, {Text: "30"}},- },- HeaderRows: 1,- Caption: "People",- }- got := RenderTableText(td)- if strings.ContainsAny(got, "<>") {- t.Fatalf("RenderTableText contains tags: %q", got)- }- if !strings.Contains(got, "Name") || !strings.Contains(got, "Alice") {- t.Fatalf("RenderTableText lost cell text: %q", got)- }-}-func assertRowsEqual(t *testing.T, got, want [][]string) {t.Helper()if len(got) != len(want) {