RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(parser): emit structured TableData instead of HTML for tables (#20741)
xugangqiang97b648fd32ff
diff --git a/internal/entity/table.go b/internal/entity/table.gonew file mode 100644index 0000000000..328038480d--- /dev/null+++ b/internal/entity/table.go@@ -0,0 +1,55 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package entity++// TableData is the canonical structured table contract shared by every parser+// (office / markdown / html / pdf) and consumed by the ingestion chunkers.+// Coordinates and merge geometry are optional so coordinate-free tables+// (spreadsheet / markdown / html) and coordinate-bearing tables (pdf) share+// one type. The cell text in Rows is already normalized the same way the+// legacy chunker normalized HTML <td>/<th> text (TrimSpace, <br> -> newline,+// inert markup skipped, nested table text folded), so consumers can render+// chunk text directly from Rows without re-parsing any markup.+type TableData struct {+ // Rows holds the cell text, row-major. rows[0:HeaderRows] are header rows.+ Rows [][]string `json:"rows"`+ // HeaderRows is the number of leading header rows. The legacy rule is:+ // rows carrying a <th> cell count as headers; when no row uses <th> at+ // all the first row is treated as the header (HeaderRows == 1).+ HeaderRows int `json:"header_rows"`+ // Caption is the table caption (pdf emits it; office uses the sheet name).+ Caption string `json:"caption,omitempty"`+ // Spans records merged cells. Recovered information: pdf already emits+ // colspan/rowspan in its HTML and office loses merged-cell geometry today;+ // consumers do not use Spans yet (preserved for a later follow-up).+ Spans []CellSpan `json:"spans,omitempty"`+ // Page is the pdf 0-based page number, used for layout retrieval.+ Page int `json:"page,omitempty"`+ // Cells carries optional per-cell coordinates (pdf only); currently lost+ // in the legacy HTML serialization and preserved here for later use.+ Cells []CellCoord `json:"cells,omitempty"`+}++// CellSpan identifies a merged cell: the top-left anchor at (Row, Col) spans+// RowSpan rows and ColSpan columns.+type CellSpan struct {+ Row, Col, RowSpan, ColSpan int+}++// CellCoord is the bounding box of a single cell in pdf page coordinates.+type CellCoord struct {+ Row, Col int+ X0, Y0, X1, Y1 float64+}diff --git a/internal/ingestion/component/chunker/coverage_blind_spots_test.go b/internal/ingestion/component/chunker/coverage_blind_spots_test.goindex c1c9d4023f..0841533af7 100644--- a/internal/ingestion/component/chunker/coverage_blind_spots_test.go+++ b/internal/ingestion/component/chunker/coverage_blind_spots_test.go@@ -175,7 +175,12 @@ func TestGeneralChunkerKeepsHeaderOnlySheetsSeparate(t *testing.T) {if len(chunks) != 2 {t.Fatalf("chunks = %#v, want one header-only chunk per sheet", chunks)}- if chunks[0]["text"] != segOne["text"] || chunks[1]["text"] != segTwo["text"] {- t.Fatalf("header-only chunks = %#v", chunks)+ // Header-only segments carry no data rows; the chunk keeps a plain-text+ // body derived from the structured TableData (caption + header cells).+ if got, _ := chunks[0]["text"].(string); got == "" || !strings.Contains(got, "Name") {+ t.Fatalf("header-only chunk 0 lost content: %#v", chunks[0])+ }+ if got, _ := chunks[1]["text"].(string); got == "" || !strings.Contains(got, "Amount") {+ t.Fatalf("header-only chunk 1 lost content: %#v", chunks[1])}}diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex f533038218..b53a204cb0 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -39,6 +39,7 @@ import ("ragflow/internal/common""ragflow/internal/ingestion/component/schema""ragflow/internal/parser/chunk"+ "ragflow/internal/parser/tableutil")const ComponentNameGeneralChunker = "GeneralChunker"@@ -500,10 +501,14 @@ func generalContextSourceText(doc schema.ChunkDoc) (string, bool) {case "text":return doc.Text, truecase "table":- if !hasSpreadsheetIdentity(doc) || !isTableStrictHTML(doc.Text) {+ if !hasSpreadsheetIdentity(doc) {return "", false}- rows, headerCount := tableRowsWithHeader(doc.Text)+ td := tableDataOf(doc)+ if td == nil {+ return "", false+ }+ rows, headerCount := td.Rows, td.HeaderRowslines := make([]string, 0, len(rows)-headerCount)for _, row := range rows[headerCount:] {cells := make([]string, 0, len(row))@@ -897,7 +902,7 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstreamattachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)chunks := make([]schema.ChunkDoc, 0, len(units))for _, unit := range units {- if itemDocType(unit) == "table" && isTableStrictHTML(unit.Text) {+ if itemDocType(unit) == "table" && tableDataOf(unit) != nil {chunks = append(chunks, c.splitSpreadsheetTable(unit)...)continue}@@ -922,7 +927,11 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream// and deferred the header; a segment now passes through whole unless the// budget forces a cut, and the header repeats in every part.func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []schema.ChunkDoc {- parts, ranges, headerRows := splitLargeHTMLTable(unit.Text, c.param.ChunkTokenSize, tokenizeStr)+ td := tableDataOf(unit)+ if td == nil {+ return []schema.ChunkDoc{cloneChunkDoc(unit)}+ }+ parts, ranges, headerRows := splitLargeTable(td, c.param.ChunkTokenSize, tokenizeStr)if ranges == nil {return []schema.ChunkDoc{cloneChunkDoc(unit)}}@@ -934,9 +943,8 @@ func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []matrix = nil}}- // The matrix carries one five-field tuple per <tr>; the splitter cut those- // same rows, so its last range end is the total data-row count — no second- // full parse of the markup is needed to compare lengths.+ // The matrix carries one five-field tuple per row; the splitter cut those+ // same rows, so its last range end is the total data-row count.aligned := falseif len(ranges) > 0 {if expected := headerRows + ranges[len(ranges)-1][1]; expected > 0 && len(matrix) == expected {@@ -952,7 +960,11 @@ func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []out := make([]schema.ChunkDoc, 0, len(parts))for i, part := range parts {piece := cloneChunkDoc(unit)- piece.Text = part+ // Text carries a plain-text rendering (no markup) for the rare embed+ // path without a downstream TableChunker; the structured TableData is+ // the authoritative content for chunkers that read it.+ piece.Text = tableutil.RenderTableText(&part)+ piece.TableData = &partpiece.Positions = nillo := headerRows + ranges[i][0]hi := headerRows + ranges[i][1]diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.goindex c384ab8d59..daeea3f92a 100644--- a/internal/ingestion/component/chunker/general_test.go+++ b/internal/ingestion/component/chunker/general_test.go@@ -34,30 +34,25 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common"+ "ragflow/internal/entity""ragflow/internal/ingestion/component/schema""ragflow/internal/ingestion/task/indexdoc"+ "ragflow/internal/parser/tableutil")// spreadsheetSegmentItem builds one wire-format spreadsheet table segment:// the markup the parser renders (captioned <table>, <th> header, <td> rows)// plus the row-aligned positions matrix (one tuple per <tr>, header first).// firstRow is the sheet row of the first DATA row; the header tuple covers-// firstRow-1.+// firstRow-1. It emits the structured TableData contract a post-migration+// producer writes (caption = sheet name, header rows, data rows), not HTML+// markup.func spreadsheetSegmentItem(sheet string, header []string, rows [][]string, sheetIndex, firstRow int) map[string]any {- var markup strings.Builder- markup.WriteString("<table><caption>" + sheet + "</caption>\n<tr>")- for _, cell := range header {- markup.WriteString("<th>" + cell + "</th>")- }- markup.WriteString("</tr>\n")- for _, row := range rows {- markup.WriteString("<tr>")- for _, cell := range row {- markup.WriteString("<td>" + cell + "</td>")- }- markup.WriteString("</tr>\n")+ allRows := make([][]string, 0, len(rows)+1)+ if len(header) > 0 {+ allRows = append(allRows, header)}- markup.WriteString("</table>\n")+ allRows = append(allRows, rows...)tuple := func(row int) []float64 {return []float64{float64(sheetIndex), float64(row), float64(row), 1, float64(len(header))}@@ -66,8 +61,16 @@ func spreadsheetSegmentItem(sheet string, header []string, rows [][]string, sheefor i := range rows {matrix = append(matrix, tuple(firstRow+i))}+ td := &entity.TableData{Rows: allRows, HeaderRows: 1, Caption: sheet}return map[string]any{- "text": markup.String(),+ "table": map[string]any{+ "rows": allRows,+ "header_rows": 1,+ "caption": sheet,+ },+ // Mirror what the chunker derives from TableData so text-bound+ // assertions (e.g. TokenChunker boundary tests) stay valid.+ "text": tableutil.RenderTableText(td),"doc_type_kwd": "table","ck_type": "table","sheet_index": sheetIndex,@@ -983,7 +986,8 @@ func TestGeneralChunkerSpreadsheetHeaderOnlyPreservesHeaderChunk(t *testing.T) {if len(chunks) != 1 {t.Fatalf("chunks = %#v, want one header-only chunk", chunks)}- if chunks[0]["text"] != item["text"] || chunks[0]["ck_type"] != "table" {+ text, _ := chunks[0]["text"].(string)+ if chunks[0]["ck_type"] != "table" || text == "" || !strings.Contains(text, "Name") || !strings.Contains(text, "Amount") {t.Fatalf("header-only chunk = %#v", chunks[0])}}@@ -1074,9 +1078,19 @@ func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testing}covered := 0for i, chunk := range chunks {- text, _ := chunk["text"].(string)- if !strings.Contains(text, "<th>Name</th>") {- t.Errorf("chunk %d lost the replicated header", i)+ // New contract: Text is the plain rendering (no markup); the+ // replicated header lives in the structured TableData, and positions+ // are sliced to this sub-table when the matrix is row-aligned.+ tbl, ok := chunk["table"].(map[string]any)+ if !ok {+ t.Fatalf("chunk %d missing structured TableData", i)+ }+ tblRows, _ := tbl["rows"].([]any)+ if len(tblRows) < 2 {+ t.Fatalf("chunk %d has too few rows: %v", i, tblRows)+ }+ if !strings.Contains(chunk["text"].(string), "Name") {+ t.Errorf("chunk %d text lost the header cell: %q", i, chunk["text"])}var matrix [][]float64raw, _ := json.Marshal(chunk["positions"])@@ -1091,11 +1105,11 @@ func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testingif matrix[0][1] != 1 {t.Errorf("chunk %d header tuple = %v, want sheet 1 row 1", i, matrix[0])}- trCount := strings.Count(text, "<tr>")- if trCount != len(matrix) {- t.Errorf("chunk %d has %d rows but %d position tuples", i, trCount, len(matrix))+ dataRows := len(tblRows) - 1+ if dataRows != len(matrix)-1 {+ t.Errorf("chunk %d data rows %d != position tuples %d", i, dataRows, len(matrix)-1)}- covered += len(matrix) - 1+ covered += dataRows}if covered != len(rows) {t.Errorf("sub-tables cover %d data rows, want %d", covered, len(rows))diff --git a/internal/ingestion/component/chunker/html_rows.go b/internal/ingestion/component/chunker/html_rows.godeleted file mode 100644index a1874deac0..0000000000--- a/internal/ingestion/component/chunker/html_rows.go+++ /dev/null@@ -1,168 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--package chunker--import (- "strings"-- "golang.org/x/net/html"-)--// ----------------------------------------------------------------------------// HTML table row extraction (shared by the Table, QA and General chunkers)-// -----------------------------------------------------------------------------// tableRows returns every row's cell text, header rows included.-func tableRows(htmlStr string) [][]string {- rows, _ := tableRowsWithHeader(htmlStr)- return rows-}--// tableRowsWithHeader returns every row's cell text in document order, plus-// the number of leading header rows. A row counts as a header row when any of-// its cells is a <th>. When no row uses <th> at all the first row is treated-// as the header — the convention spreadsheet and markup tables are read with.-//-// The markup is parsed into a tree rather than matched with a regex because-// this input is not guaranteed to be well formed: seven parsers render table-// items (xlsx, csv, docx, html, pdf, …) and some of that markup originates-// from user-supplied documents. A tree also settles the cases a tag-level-// scan gets wrong: a nested <table> no longer terminates its enclosing row-// early — that row keeps its own cells, with the nested table's text folded-// into the cell holding it — and a row or cell missing its closing tag is-// recovered rather than dropped.-func tableRowsWithHeader(htmlStr string) (rows [][]string, headerCount int) {- // A <tr> outside a <table> is discarded by the HTML5 "in body" insertion- // mode, so a bare row fragment would yield nothing. Give the parser the- // table context it needs instead of dropping the rows silently.- lower := strings.ToLower(htmlStr)- if strings.Contains(lower, "<tr") && !strings.Contains(lower, "<table") {- htmlStr = "<table>" + htmlStr + "</table>"- }- doc, err := html.Parse(strings.NewReader(htmlStr))- if err != nil {- return nil, 0- }- var headerFlags []bool- var walk func(*html.Node)- walk = func(n *html.Node) {- // An inert subtree is parsed but never rendered. <template> puts its- // content straight into the ordinary child list (the parser has no- // separate template-contents field), so without this its rows would- // be read as rows of the enclosing table.- if n.Type == html.ElementNode && isInertElement(n.Data) {- return- }- if isHTMLElement(n, "tr") {- var cells []string- anyTh := false- for c := n.FirstChild; c != nil; c = c.NextSibling {- if isHTMLElement(c, "td") || isHTMLElement(c, "th") {- if isHTMLElement(c, "th") {- anyTh = true- }- cells = append(cells, cellText(c))- }- }- // Return without descending: the cells above already collected- // the nested table's text, so its rows must not be reported a- // second time as rows of the enclosing table.- rows = append(rows, cells)- headerFlags = append(headerFlags, anyTh)- return- }- for c := n.FirstChild; c != nil; c = c.NextSibling {- walk(c)- }- }- walk(doc)- for len(headerFlags) > headerCount && headerFlags[headerCount] {- headerCount++- }- if len(rows) > 0 && headerCount == 0 {- headerCount = 1- }- return rows, headerCount-}--// isTableStrictHTML reports whether block text is an outer <table> element-// (the inlined GFM/HTML table). Only such blocks are emitted as structured-// table items; other raw HTML (e.g. <div>, <style>) is plain text. The-// spreadsheet wire is the markup this reports on (see the parser's-// renderSpreadsheetTable), so callers that slice a positions matrix use-// this strict form rather than the candidate filter.-func isTableStrictHTML(s string) bool {- return strings.HasPrefix(strings.TrimSpace(strings.ToLower(s)), "<table")-}--// isTableHTML is the cheap candidate filter for table-vs-prose routing: text-// that opens an outer <table> element, or a bare-row fragment holding <tr>.-// It only narrows the candidates — the caller decides on tableRows' result —-// so a block that merely opens with "<table" text and holds no row stays on-// the prose path instead of silently pairing nothing.-func isTableHTML(s string) bool {- lower := strings.ToLower(s)- return strings.HasPrefix(strings.TrimSpace(lower), "<table") || strings.Contains(lower, "<tr")-}--// cellText returns the visible text of a table cell. The parser hands text-// nodes over already unescaped, nested markup contributes its text without-// its tags (a nested table's cells are concatenated, not separated), and a-// <br> becomes a newline instead of silently gluing the two halves of the-// cell together.-func cellText(cell *html.Node) string {- var sb strings.Builder- var walk func(*html.Node)- walk = func(n *html.Node) {- switch {- case n.Type == html.TextNode:- sb.WriteString(n.Data)- case isHTMLElement(n, "br"):- sb.WriteByte('\n')- case n.Type == html.ElementNode && isInertElement(n.Data):- // Stop here rather than descending: the content is parsed but- // never rendered, so it is not text a reader of the cell sees.- return- }- for c := n.FirstChild; c != nil; c = c.NextSibling {- walk(c)- }- }- walk(cell)- return strings.TrimSpace(sb.String())-}--// isHTMLElement reports whether n is an element with the given tag name in-// the HTML namespace. Foreign content reuses HTML tag names for unrelated-// elements — an <svg><tr> is not a table row — so matching on the tag name-// alone would read markup that carries no table semantics.-func isHTMLElement(n *html.Node, tag string) bool {- return n.Type == html.ElementNode && n.Namespace == "" && n.Data == tag-}--// isInertElement reports whether an element's content is inert — parsed, but-// never rendered as visible text. An inline <script> or <style> inside a-// table cell of a user-supplied document would otherwise be embedded into the-// Q&A pair as if it were part of the sentence, and a <template>'s placeholder-// rows would be read as real ones.-func isInertElement(tag string) bool {- switch tag {- case "script", "style", "noscript", "template":- return true- }- return false-}diff --git a/internal/ingestion/component/chunker/html_rows_test.go b/internal/ingestion/component/chunker/html_rows_test.godeleted file mode 100644index 76690052bb..0000000000--- a/internal/ingestion/component/chunker/html_rows_test.go+++ /dev/null@@ -1,70 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--package chunker--import "testing"--// TestIsTableStrictHTML pins the predicate's prefix semantics: it reports-// whether the block opens with the "<table" text, exactly as the markdown-// parser has always classified its HTML blocks. It is deliberately loose (a-// "<tableau>" token passes), which is why routing never treats it as the final-// word — the row walker decides.-func TestIsTableStrictHTML(t *testing.T) {- cases := []struct {- text string- want bool- }{- {"<table>", true},- {" <TABLE class=\"x\">", true},- {"<table-x>", true}, // historical prefix semantics: "<table" prefix only- {"<tableau>", true},- {"<div><table>", false},- {"plain text", false},- {"", false},- }- for _, tc := range cases {- if got := isTableStrictHTML(tc.text); got != tc.want {- t.Errorf("isTableStrictHTML(%q) = %v, want %v", tc.text, got, tc.want)- }- }-}--// TestIsTableHTMLCoversWhatTheWalkerReads guards the routing invariant: the-// candidate filter must never deny a payload the row walker can read, or a-// readable table would silently fall through to the prose extractor.-func TestIsTableHTMLCoversWhatTheWalkerReads(t *testing.T) {- texts := []string{- "<table><tr><th>q</th><th>a</th></tr></table>",- "<table><caption>c</caption>\n<tr><td>q</td><td>a</td></tr>\n</table>\n",- "<tr><td>q</td><td>a</td></tr>", // bare fragment- "<tr><td>q</td><td>a</td></tr><tr><td>q2</td><td>a2</td></tr>", // bare rows- "<tbody><tr><td>q</td><td>a</td></tr></tbody>", // rows wrapped in tbody- "<table><tr><td>q<table><tr><td>inner</td></tr></table></td></tr></table>", // nested- "<table><template><tr><td>x</td></tr></template></table>", // template rows are inert- "<table></table>", // nothing to read- "plain text",- "",- }- for _, text := range texts {- if len(tableRows(text)) == 0 {- continue- }- if !isTableHTML(text) {- t.Errorf("tableRows can read %q but isTableHTML denies it", text)- }- }-}diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.goindex 9e9d36d785..5ef0f1a953 100644--- a/internal/ingestion/component/chunker/qa.go+++ b/internal/ingestion/component/chunker/qa.go@@ -42,6 +42,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/tableutil""ragflow/internal/tokenizer")@@ -403,8 +404,11 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {strictCSV := strings.EqualFold(fileType, "csv")for _, item := range items {var tmp []qaPair+ td := tableDataOf(item)txt, _ := itemText(item)- if txt == "" {+ // A structured table with no markup text still carries rows; only a+ // truly empty item (no text and no TableData) is skipped.+ if txt == "" && td == nil {continue}// Route on what the row walker can read, not on the type label.@@ -423,8 +427,8 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {// fall through to the text extractor, so documents from before this// wire must be re-parsed rather than re-chunked.var rows [][]string- if isTableHTML(txt) {- rows = tableRows(txt)+ if td != nil {+ rows = td.Rows}if len(rows) > 0 {tmp = qaPairsFromRows(rows, strictCSV, item.Positions, item.SheetIndex != nil)@@ -454,7 +458,11 @@ func extractQATable(htmlStr string, strictPairs bool) []qaPair {if htmlStr == "" {return nil}- return qaPairsFromRows(tableRows(htmlStr), strictPairs, nil, false)+ td, err := tableutil.ParseTableHTML(htmlStr)+ if err != nil || td == nil {+ return nil+ }+ return qaPairsFromRows(td.Rows, strictPairs, nil, false)}// qaPairsFromRows builds the pairs of one table: the first two non-emptydiff --git a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.goindex becbdc7934..70b139eca8 100644--- a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go+++ b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go@@ -72,11 +72,10 @@ func TestCSVJSONHTMLTableUsesStrictContinuationRules(t *testing.T) {"name": "qa.csv","file_type": "csv","output_format": "json",- "json": []map[string]any{{- "doc_type_kwd": "table",- "text": "<table><tr><td>q1</td><td>a1</td></tr>" +- "<tr><td>q2</td><td>a2</td><td>trailing</td></tr></table>",- }},+ "json": []map[string]any{+ tableItem(t, "<table><tr><td>q1</td><td>a1</td></tr>"++ "<tr><td>q2</td><td>a2</td><td>trailing</td></tr></table>"),+ },})if err != nil {t.Fatal(err)diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.goindex 468edcf9c3..9a1e8785a4 100644--- a/internal/ingestion/component/chunker/qa_test.go+++ b/internal/ingestion/component/chunker/qa_test.go@@ -22,9 +22,22 @@ import ("testing""ragflow/internal/agent/runtime"+ "ragflow/internal/entity""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/tableutil")+// mustTableData parses markup into the structured TableData contract a+// post-migration producer emits, failing the test on error.+func mustTableData(t *testing.T, html string) *entity.TableData {+ t.Helper()+ td, err := tableutil.ParseTableHTML(html)+ if err != nil {+ t.Fatalf("ParseTableHTML(%q): %v", html, err)+ }+ return td+}+func TestQAChunker_Registered(t *testing.T) {factory, _, _, ok := runtime.DefaultRegistry.Lookup("QAChunker")if !ok {@@ -394,7 +407,7 @@ func TestQAChunker_XLSXJSONRegression(t *testing.T) {"output_format": "json","json": []map[string]any{{- "text": "<table><caption>Sheet1</caption><tr><th>question</th><th>answer</th></tr><tr><td>What is RAGFlow?</td><td>A RAG engine.</td></tr><tr><td>Where are the docs?</td><td>On the website.</td></tr></table>",+ "table": mustTableData(t, "<table><caption>Sheet1</caption><tr><th>question</th><th>answer</th></tr><tr><td>What is RAGFlow?</td><td>A RAG engine.</td></tr><tr><td>Where are the docs?</td><td>On the website.</td></tr></table>"),"doc_type_kwd": "table",},},@@ -493,7 +506,7 @@ func TestQAChunker_PlainTablePayloadNotSwallowed(t *testing.T) {// genuinely holds table markup is now read as a table.func TestQAChunker_TextLabelledHTMLTableReadAsTable(t *testing.T) {items := []schema.ChunkDoc{- {Text: "<table><tr><td>q1</td><td>a1</td></tr><tr><td>q2</td><td>a2</td></tr></table>", DocType: "text"},+ {TableData: mustTableData(t, "<table><tr><td>q1</td><td>a1</td></tr><tr><td>q2</td><td>a2</td></tr></table>"), DocType: "text"},}pairs := extractQAJSON(items, "")if len(pairs) != 2 {@@ -545,7 +558,7 @@ func TestQAChunkerPositionsRequireSpreadsheetIdentityAndFiveFields(t *testing.T)for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {item := schema.ChunkDoc{- Text: table,+ TableData: mustTableData(t, table),DocType: "table",Positions: json.RawMessage(tc.positions),SheetIndex: tc.sheetIndex,diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.goindex 986733f3c6..4947595e34 100644--- a/internal/ingestion/component/chunker/table.go+++ b/internal/ingestion/component/chunker/table.go@@ -21,11 +21,11 @@// Unlike TokenChunker (which token-shreds a row's text into multiple// pieces) or OneChunker (which merges many rows into a single chunk),// TableChunker keeps the row as the unit of chunking. Every table arrives as-// rendered HTML — the spreadsheet parsers emit captioned table segments — and-// each data row becomes one chunk whose text repeats the column names in the-// Python "- field: value" line format, carrying its own position tuple when-// the item is a spreadsheet item with a row-aligned matrix. A table whose only-// row is the header stays one whole chunk.+// structured TableData on ChunkDoc.TableData — the parsers emit captioned+// table segments — and each data row becomes one chunk whose text repeats the+// column names in the Python "- field: value" line format, carrying its own+// position tuple when the item is a spreadsheet item with a row-aligned+// matrix. A table whose only row is the header stays one whole chunk.package chunkerimport (@@ -35,7 +35,9 @@ import ("strings""ragflow/internal/agent/runtime"+ "ragflow/internal/entity""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/tableutil""gorm.io/gorm")@@ -114,11 +116,11 @@ func (c *TableChunkerComponent) invoke(_ context.Context, inputs map[string]any)}// tableItems returns the per-row records, preferring JSONResult and-// falling back to Chunks. Each HTML table row becomes exactly one chunk;-// every other payload record passes through as one chunk — including-// pre-upgrade row-IR records (ck_type: table_row/table_header with cells),-// which hold no markup and therefore keep no per-row positions; documents-// from before this wire must be re-parsed rather than re-chunked.+// falling back to Chunks. Each table row becomes exactly one chunk; every+// other payload record passes through as one chunk — including pre-upgrade+// row-IR records (ck_type: table_row/table_header with cells), which hold no+// markup and therefore keep no per-row positions; documents from before this+// wire must be re-parsed rather than re-chunked.func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {source := itemsif len(source) == 0 {@@ -134,22 +136,30 @@ func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {return filtered}-// expandHTMLTableRows turns one HTML <table> payload into one chunk per data-// row. Non-table payloads pass through unchanged. A table whose only row is-// the header keeps the whole markup as its single chunk: the header line is-// then the only searchable representation.+// expandHTMLTableRows turns one table payload into one chunk per data row.+// Non-table payloads pass through unchanged. It reads the structured+// TableData contract that the parser emits on ChunkDoc.TableData; items that+// do not carry TableData (e.g. pre-upgrade HTML markup or non-table records)+// pass through as a single chunk. A table whose only row is the header keeps+// the whole item as its single chunk: the header line is then the only+// searchable representation. The structured TableData is cleared on every+// emitted chunk so the whole table is not duplicated into the index.func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {- if !isTableHTML(item.Text) {+ td := item.TableData+ if td == nil {return []schema.ChunkDoc{item}}- rows, headerCount := tableRowsWithHeader(item.Text)+ rows, headerCount := td.Rows, td.HeaderRowsif len(rows) <= headerCount {- return []schema.ChunkDoc{item}+ out := item+ out.Text = tableutil.RenderTableText(td)+ out.TableData = nil+ return []schema.ChunkDoc{out}}names := rows[0]// R1: a row chunk must carry only its own tuple. That is only sound when// the item carries spreadsheet identity and its matrix was built- // row-aligned (one five-field tuple per <tr>, header included); a+ // row-aligned (one five-field tuple per row, header included); a// whole-table tuple must not be copied onto every row, and PDF items write// layout boxes into the same field, so misaligned or non-spreadsheet// payloads get no positions at all.@@ -176,6 +186,7 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {doc.Text = textdoc.TKNums = nildoc.Positions = nil+ doc.TableData = nilif aligned {if tuple, err := json.Marshal([][]float64{matrix[headerCount+i]}); err == nil {doc.Positions = tuple@@ -184,11 +195,23 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {out = append(out, doc)}if len(out) == 0 {- return []schema.ChunkDoc{item}+ single := item+ single.Text = tableutil.RenderTableText(td)+ single.TableData = nil+ return []schema.ChunkDoc{single}}return out}+// tableDataOf returns the structured TableData for an item. The parser emits+// it on ChunkDoc.TableData; items without it (pre-upgrade HTML markup or+// non-table records) return nil and pass through unchanged. Re-chunk always+// re-parses from the source document, so a payload never arrives with HTML+// but no TableData, and no HTML fallback is performed (decision §9.1).+func tableDataOf(item schema.ChunkDoc) *entity.TableData {+ return item.TableData+}+// tableRowRecordText renders one row in the Python table chunker's line format:// "- {field}: {value}" per non-empty cell, newline joined. A cell whose// column has no header name keeps its value alone rather than being dropped.diff --git a/internal/ingestion/component/chunker/table_split.go b/internal/ingestion/component/chunker/table_split.goindex 0957114cef..3b086feea8 100644--- a/internal/ingestion/component/chunker/table_split.go+++ b/internal/ingestion/component/chunker/table_split.go@@ -17,173 +17,74 @@package chunkerimport (- "regexp""strings"-)--var trRegex = regexp.MustCompile(`(?is)<tr\b[^>]*>.*?</tr>`)-// tableOpenTagEnd returns the index of the ">" that closes the open tag whose-// name s begins after, skipping ">" inside quoted attribute values. It returns-// -1 when the tag never closes before the next "<": a malformed open tag must-// not be cut at the next ">", which may belong to a closing tag — slicing-// "<table </table>" there would invert the slice bounds.-func tableOpenTagEnd(s string) int {- quote := byte(0)- for i := 0; i < len(s); i++ {- switch c := s[i]; {- case quote != 0:- if c == quote {- quote = 0- }- case c == '"' || c == '\'':- quote = c- case c == '>':- return i- case c == '<':- return -1- }- }- return -1-}+ "ragflow/internal/entity"+ "ragflow/internal/parser/tableutil"+)-// splitLargeHTMLTable splits an HTML table into sub-tables that fit within-// maxTokens, replicating the caption and header row(s) in each sub-table.-// Token counting is injected (countTokens) so the caller's tokenizer decides-// what "size" means. This is the chunker-side size budget: Python's flow path-// never splits a spreadsheet sheet, but honouring chunk_token_size keeps a-// wide or long table from losing its tail rows to embedding truncation.-//-// The text is expected to hold one outer <table> with flat rows — the shape-// the spreadsheet parsers render. Markup this function cannot cut safely (an-// open tag that never closes, a ">" inside an attribute value, a nested or-// second table) is returned unchanged, so callers fall back to one whole-// chunk instead of cutting at a guessed boundary.+// splitLargeTable enforces the ChunkTokenSize budget on a structured table,+// splitting it into sub-tables that fit. Splitting is row-granular: every+// emitted sub-table is a whole number of complete rows (never a partial row),+// so the per-row chunk produced downstream keeps full row semantics. The token+// budget is counted on the real cell text of each row, not on any HTML markup.//// Alongside the parts it returns dataRanges and headerRows: for part i, the-// half-open range [start, end) of DATA rows (rows below the header in the-// original table) the part contains, and the number of leading header rows-// every part replicates. A caller carrying a per-row position matrix slices+// half-open range [start, end) of DATA rows (rows below the header) the part+// contains, and the number of leading header rows every part replicates. A+// caller carrying a per-row position matrix slices// matrix[:headerRows] + matrix[headerRows+start : headerRows+end] to stay-// aligned with the sub-table markup. headerRows and ranges are 0/nil when-// the text is returned unchanged.-func splitLargeHTMLTable(text string, maxTokens int, countTokens func(string) int) (parts []string, dataRanges [][2]int, headerRows int) {- lower := strings.ToLower(text)- startIdx := strings.Index(lower, "<table")- endIdx := strings.LastIndex(lower, "</table>")- if startIdx < 0 || endIdx < 0 || endIdx <= startIdx {- return []string{text}, nil, 0- }-- prefix := text[:startIdx]- afterOpen := text[startIdx:]- const openTag = "<table"- // The match must be a tag name, not the prefix of a longer one such as- // "<tableau>".- if len(afterOpen) <= len(openTag) || !strings.ContainsRune(" \t\n\f\r/>", rune(afterOpen[len(openTag)])) {- return []string{text}, nil, 0- }- tagRel := tableOpenTagEnd(afterOpen[len(openTag):])- if tagRel < 0 {- return []string{text}, nil, 0- }- tagEnd := len(openTag) + tagRel- innerStart := startIdx + tagEnd + 1- if innerStart > endIdx {- return []string{text}, nil, 0+// aligned with the sub-table. headerRows and ranges are 0/nil when the table+// fits in a single chunk (the caller then emits the whole table unchanged).+func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(string) int) (parts []entity.TableData, dataRanges [][2]int, headerRows int) {+ if td == nil || len(td.Rows) == 0 {+ return nil, nil, 0}- tableOpenTag := afterOpen[:tagEnd+1]- inner := text[innerStart:endIdx]- // A second table (or a nested one) has no modelled boundary here; leaving- // the text alone keeps the caller on the single-chunk path.- if innerLower := strings.ToLower(inner); strings.Contains(innerLower, "<table") || strings.Contains(innerLower, "</table>") {- return []string{text}, nil, 0+ headerRows = td.HeaderRows+ if headerRows > len(td.Rows) {+ headerRows = len(td.Rows)}- suffix := text[endIdx+len("</table>"):]-- // Extract caption if present.- lowerInner := strings.ToLower(inner)- capStart := strings.Index(lowerInner, "<caption")- var captionHTML string- if capStart >= 0 {- capEndRel := strings.Index(lowerInner[capStart:], "</caption>")- if capEndRel >= 0 {- capEnd := capStart + capEndRel + len("</caption>")- captionHTML = inner[capStart:capEnd]- }+ dataRows := td.Rows[headerRows:]+ if len(dataRows) == 0 {+ return nil, nil, 0}- // Extract all rows.- rows := trRegex.FindAllString(inner, -1)- if len(rows) <= 1 {- return []string{text}, nil, 0- }-- // Determine header rows vs data rows.- headerEnd := 0- for i, r := range rows {- if strings.Contains(strings.ToLower(r), "<th") {- headerEnd = i + 1- } else {- break- }- }- if headerEnd == 0 || headerEnd >= len(rows) {- headerEnd = 1- }- headerRowHTML := rows[:headerEnd]- dataRows := rows[headerEnd:]-- baseHeader := tableOpenTag + captionHTML + strings.Join(headerRowHTML, "")- prefixTokens := countTokens(prefix)- baseTokens := countTokens(baseHeader + "</table>")-- // prefix (text before the table, e.g. a short Markdown heading) belongs to- // the table's position in the document: repeat only the table wrapper,- // caption and header rows in every chunk, and prepend prefix to the first.- var chunks []string- var ranges [][2]int- var currentDataRows []string- currentTokens := baseTokens + prefixTokens- currentStart := 0- firstChunk := true+ // Base tokens: header rows + caption, replicated into every sub-table.+ base := &entity.TableData{Rows: td.Rows[:headerRows], Caption: td.Caption}+ baseTokens := countTokens(tableutil.RenderTableText(base))+ var start int+ var currentTokens intflush := func(end int) {- if len(currentDataRows) == 0 {+ if end <= start {return}- var b strings.Builder- if firstChunk {- b.WriteString(prefix)- firstChunk = false+ part := entity.TableData{+ Caption: td.Caption,+ HeaderRows: headerRows,+ Rows: append(append([][]string{}, td.Rows[:headerRows]...), dataRows[start:end]...),}- b.WriteString(baseHeader)- for _, dr := range currentDataRows {- b.WriteString(dr)- }- b.WriteString("</table>")- chunks = append(chunks, b.String())- ranges = append(ranges, [2]int{currentStart, end})- currentDataRows = nil+ parts = append(parts, part)+ dataRanges = append(dataRanges, [2]int{start, end})+ start = endcurrentTokens = baseTokens- currentStart = end}+ currentTokens = baseTokensfor i, row := range dataRows {- rowTokens := countTokens(row)- if len(currentDataRows) > 0 && currentTokens+rowTokens > maxTokens {+ rowTokens := countTokens(strings.Join(row, " "))+ // Flush the running batch before this row if adding it would exceed the+ // budget. A single row that alone exceeds the budget still becomes its+ // own chunk (row integrity wins over the budget).+ if i > start && currentTokens+rowTokens > maxTokens {flush(i)}- currentDataRows = append(currentDataRows, row)currentTokens += rowTokens}flush(len(dataRows))- if len(chunks) <= 1 {- return []string{text}, nil, 0- }- if suffix != "" {- chunks[len(chunks)-1] += suffix+ if len(parts) <= 1 {+ return nil, nil, 0}- return chunks, ranges, headerEnd+ return parts, dataRanges, headerRows}diff --git a/internal/ingestion/component/chunker/table_split_test.go b/internal/ingestion/component/chunker/table_split_test.goindex 652bdba8d4..f2b29f5242 100644--- a/internal/ingestion/component/chunker/table_split_test.go+++ b/internal/ingestion/component/chunker/table_split_test.go@@ -1,165 +1,112 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.+// http://www.apache.org/licenses/LICENSE-2.0//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.package chunkerimport (- "strings""testing"++ "ragflow/internal/entity")-// charTokens approximates a tokenizer for tests: one token per byte.-func charTokens(s string) int { return len(s) }+// charTokens counts runes, a cheap deterministic stand-in for a real tokenizer.+func charTokens(s string) int { return len([]rune(s)) }-func buildTable(header string, data []string) string {- var b strings.Builder- b.WriteString("<table><caption>cap</caption>\n")- b.WriteString("<tr>" + header + "</tr>\n")- for _, row := range data {- b.WriteString("<tr>" + row + "</tr>\n")+func makeTable(dataRows int) *entity.TableData {+ rows := [][]string{{"Name", "Age"}} // header+ for i := 0; i < dataRows; i++ {+ rows = append(rows, []string{"person", "30"})}- b.WriteString("</table>\n")- return b.String()+ return &entity.TableData{Rows: rows, HeaderRows: 1}}-func TestSplitLargeHTMLTableNoSplit(t *testing.T) {- small := buildTable("<th>a</th><th>b</th>", []string{"<td>1</td><td>2</td>"})- parts, ranges, headerRows := splitLargeHTMLTable(small, len(small)+100, charTokens)- if len(parts) != 1 || parts[0] != small || ranges != nil || headerRows != 0 {- t.Fatalf("small table must return unchanged with nil ranges, got %d parts ranges=%v headerRows=%d", len(parts), ranges, headerRows)+func TestSplitLargeTableFitsWhole(t *testing.T) {+ td := makeTable(3)+ parts, ranges, headerRows := splitLargeTable(td, 1_000_000, charTokens)+ if ranges != nil || len(parts) != 0 {+ t.Fatalf("small table should not split: parts=%v ranges=%v", parts, ranges)}- parts, ranges, headerRows = splitLargeHTMLTable("no table here", 1, charTokens)- if len(parts) != 1 || parts[0] != "no table here" || ranges != nil || headerRows != 0 {- t.Fatalf("non-table text must pass through unchanged")+ if headerRows != 0 {+ t.Fatalf("headerRows = %d, want 0 when unsplit", headerRows)}}-func TestSplitLargeHTMLTableReplicatesHeaderAndSlicesRanges(t *testing.T) {- data := make([]string, 6)- for i := range data {- data[i] = "<td>d" + string(rune('0'+i)) + "</td><td>x</td>"- }- text := buildTable("<th>h1</th><th>h2</th>", data)- // Budget that holds header + 2 data rows at most.- rowLen := len("<td>d0</td><td>x</td>")- budget := len("<table><caption>cap</caption><tr><th>h1</th><th>h2</th></tr></table>") + 2*rowLen- parts, ranges, headerRows := splitLargeHTMLTable(text, budget, charTokens)+func TestSplitLargeTableRowGranular(t *testing.T) {+ td := makeTable(20)+ // Budget bounded so a part holds ~3 data rows.+ parts, ranges, headerRows := splitLargeTable(td, 22, charTokens)if headerRows != 1 {- t.Fatalf("headerRows = %d, want 1 (the single <th> row)", headerRows)+ t.Fatalf("headerRows = %d, want 1", headerRows)}- if len(parts) < 3 {- t.Fatalf("expected several sub-tables, got %d", len(parts))+ if len(parts) <= 1 {+ t.Fatalf("expected multiple parts, got %d", len(parts))}- if len(ranges) != len(parts) {- t.Fatalf("ranges count %d != parts count %d", len(ranges), len(parts))+ if len(parts) != len(ranges) {+ t.Fatalf("parts(%d) != ranges(%d)", len(parts), len(ranges))}+ // Every part replicates the header and holds a whole number of rows; the+ // data ranges are contiguous and cover all 20 data rows exactly once.seen := 0- for i, p := range parts {- if !strings.HasPrefix(p, "<table><caption>cap</caption>") {- t.Errorf("part %d lost its caption: %q", i, p)- }- if !strings.Contains(p, "<th>h1</th>") {- t.Errorf("part %d lost the header row", i)+ for i, part := range parts {+ if part.HeaderRows != 1 {+ t.Fatalf("part %d headerRows = %d", i, part.HeaderRows)}- lo, hi := ranges[i][0], ranges[i][1]- if hi <= lo || hi > len(data) {- t.Fatalf("part %d range [%d,%d) out of bounds", i, lo, hi)+ if len(part.Rows) < 2 {+ t.Fatalf("part %d has <1 data row: %v", i, part.Rows)}- for r := lo; r < hi; r++ {- if !strings.Contains(p, data[r]) {- t.Errorf("part %d missing row %d", i, r)- }- seen+++ if part.Rows[0][0] != "Name" {+ t.Fatalf("part %d missing header row", i)}- // Rows outside the range must not leak into this part.- for r := 0; r < len(data); r++ {- if (r < lo || r >= hi) && strings.Contains(p, data[r]) {- t.Errorf("part %d leaked row %d outside [%d,%d)", i, r, lo, hi)- }+ r := ranges[i]+ if r[0] != seen {+ t.Fatalf("part %d range start %d != running %d", i, r[0], seen)}+ seen = r[1]}- if seen != len(data) {- t.Errorf("rows covered %d, want %d", seen, len(data))- }-}--func TestSplitLargeHTMLTableHeaderlessReplicatesFirstRow(t *testing.T) {- // Original behaviour: with no <th> row the first row is treated as the- // repeating header.- var b strings.Builder- b.WriteString("<table>")- b.WriteString("<tr><td>r0</td></tr>")- for i := 1; i < 5; i++ {- b.WriteString("<tr><td>row" + string(rune('0'+i)) + "</td></tr>")- }- b.WriteString("</table>")- parts, ranges, headerRows := splitLargeHTMLTable(b.String(), len("<table><tr><td>r0</td></tr><tr><td>row1</td></tr></table>")+10, charTokens)- if headerRows != 1 {- t.Fatalf("headerRows = %d, want 1 (first row treated as header)", headerRows)+ if seen != 20 {+ t.Fatalf("data rows covered = %d, want 20", seen)}- if len(parts) < 2 {- t.Fatalf("expected split, got %d parts", len(parts))+ // The concatenation of every part's data rows must equal the original rows.+ var rebuilt [][]string+ for _, part := range parts {+ rebuilt = append(rebuilt, part.Rows[1:]...)}- for i, p := range parts {- if !strings.Contains(p, "<td>r0</td>") {- t.Errorf("part %d lost replicated first row", i)+ for i, row := range td.Rows[1:] {+ if len(rebuilt[i]) != len(row) || rebuilt[i][0] != row[0] {+ t.Fatalf("row %d mismatch: got %v want %v", i, rebuilt[i], row)}}- if ranges[0][0] != 0 {- t.Errorf("ranges index DATA rows below the treated header, want first range to start at 0, got %v", ranges[0])- }}-// TestSplitLargeHTMLTableRefusesUncuttableMarkup: an open tag that never-// closes, a longer tag name, or a nested table is markup this splitter must-// not cut at a guessed boundary — the text comes back unchanged. The-// "<table </table>" case used to invert the slice bounds.-func TestSplitLargeHTMLTableRefusesUncuttableMarkup(t *testing.T) {- unchanged := []string{- "<table </table>",- "<tableau><tr><td>a</td></tr><tr><td>b</td></tr></table>",- "<table><tr><td>a</td></tr><table><tr><td>b</td></tr></table></table>",- }- for _, text := range unchanged {- parts, ranges, headerRows := splitLargeHTMLTable(text, 1, charTokens)- if len(parts) != 1 || parts[0] != text || ranges != nil || headerRows != 0 {- t.Errorf("text %q must pass through unchanged, got %d parts ranges=%v headerRows=%d", text, len(parts), ranges, headerRows)- }+func TestSplitLargeTableSingleRowOverBudgetKeepsRow(t *testing.T) {+ // One data row alone exceeds the budget; it must still form its own chunk+ // (row integrity wins over the budget). A single part means "not split,+ // emit whole" — the over-budget row becomes one chunk rather than being cut.+ td := &entity.TableData{+ Rows: [][]string{{"H"}, {"x" + string(make([]byte, 200))}},+ HeaderRows: 1,+ }+ parts, ranges, _ := splitLargeTable(td, 5, charTokens)+ if ranges != nil || len(parts) != 0 {+ t.Fatalf("over-budget single row must emit whole (ranges nil): parts=%v ranges=%v", parts, ranges)}}-// TestSplitLargeHTMLTableQuotedAngleBracketStaysInOpenTag: a ">" inside a-// quoted attribute value must not end the open tag — every part repeats the-// complete tag instead of being cut at the attribute.-func TestSplitLargeHTMLTableQuotedAngleBracketStaysInOpenTag(t *testing.T) {- text := "<table data-meta=\"a>b\">" +- "<tr><td>r0</td></tr><tr><td>r1</td></tr><tr><td>r2</td></tr></table>"- parts, ranges, headerRows := splitLargeHTMLTable(text, 60, charTokens)- if headerRows != 1 {- t.Fatalf("headerRows = %d, want 1 (first row treated as header)", headerRows)- }- if len(parts) < 2 {- t.Fatalf("expected a split, got %d parts", len(parts))- }- if len(ranges) != len(parts) {- t.Fatalf("ranges count %d != parts count %d", len(ranges), len(parts))- }- for i, p := range parts {- if !strings.HasPrefix(p, "<table data-meta=\"a>b\">") {- t.Errorf("part %d lost the complete open tag: %q", i, p)- }+func TestSplitLargeTableHeaderOnly(t *testing.T) {+ td := &entity.TableData{Rows: [][]string{{"Name", "Age"}}, HeaderRows: 1}+ parts, ranges, _ := splitLargeTable(td, 10, charTokens)+ if ranges != nil || len(parts) != 0 {+ t.Fatalf("header-only table should not split")}}diff --git a/internal/ingestion/component/chunker/table_struct_test.go b/internal/ingestion/component/chunker/table_struct_test.gonew file mode 100644index 0000000000..a8a6cb16ed--- /dev/null+++ b/internal/ingestion/component/chunker/table_struct_test.go@@ -0,0 +1,63 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package chunker++import (+ "testing"++ "ragflow/internal/entity"+ "ragflow/internal/ingestion/component/schema"+)++func TestExpandHTMLTableRowsFromTableData(t *testing.T) {+ td := &entity.TableData{+ Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}, {"Bob", "25"}},+ HeaderRows: 1,+ }+ out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})+ if len(out) != 2 {+ t.Fatalf("got %d chunks, want 2", len(out))+ }+ if out[0].Text != "- Name: Alice\n- Age: 30" {+ t.Fatalf("row0 text = %q", out[0].Text)+ }+ if out[1].Text != "- Name: Bob\n- Age: 25" {+ t.Fatalf("row1 text = %q", out[1].Text)+ }+ for _, c := range out {+ if c.TableData != nil {+ t.Fatalf("TableData must be cleared on emitted row chunk, got %+v", c.TableData)+ }+ }+}++func TestExpandHTMLTableRowsHeaderOnlySingleChunk(t *testing.T) {+ td := &entity.TableData{Rows: [][]string{{"Name", "Age"}}, HeaderRows: 1}+ out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})+ if len(out) != 1 {+ t.Fatalf("got %d chunks, want 1 (header-only)", len(out))+ }+ if out[0].TableData != nil {+ t.Fatalf("TableData must be cleared on the single chunk")+ }+}++func TestExpandHTMLTableRowsNonTablePassesThrough(t *testing.T) {+ item := schema.ChunkDoc{DocType: "text", Text: "just prose"}+ out := expandHTMLTableRows(item)+ if len(out) != 1 || out[0].Text != "just prose" {+ t.Fatalf("non-table not passed through: %+v", out)+ }+}diff --git a/internal/ingestion/component/chunker/table_test.go b/internal/ingestion/component/chunker/table_test.goindex 36979b2a24..596766c3f3 100644--- a/internal/ingestion/component/chunker/table_test.go+++ b/internal/ingestion/component/chunker/table_test.go@@ -17,9 +17,12 @@package chunkerimport (+ "encoding/json""fmt""strings""testing"++ "ragflow/internal/parser/tableutil")// tableChunksOf drives TableChunker.Invoke and returns the emitted maps.@@ -40,6 +43,33 @@ func tableChunksOf(t *testing.T, inputs map[string]any) []map[string]any {return chunks}+// tableItem builds a chunker JSON input item carrying the structured TableData+// contract a post-migration producer emits (derived from markup), plus any+// extra fields (positions, sheet_index, ck_type, ...). It deliberately does+// NOT set "text": the chunker derives a plain text body from TableData.+func tableItem(t *testing.T, html string, extra ...map[string]any) map[string]any {+ t.Helper()+ td, err := tableutil.ParseTableHTML(html)+ if err != nil {+ t.Fatalf("ParseTableHTML(%q): %v", html, err)+ }+ b, err := json.Marshal(td)+ if err != nil {+ t.Fatalf("marshal TableData: %v", err)+ }+ var tbl map[string]any+ if err := json.Unmarshal(b, &tbl); err != nil {+ t.Fatalf("unmarshal TableData: %v", err)+ }+ item := map[string]any{"table": tbl, "doc_type_kwd": "table"}+ if len(extra) > 0 {+ for k, v := range extra[0] {+ item[k] = v+ }+ }+ return item+}+// TestTableChunker_OneChunkPerRow ports the Python table chunker's contract:// "Every row in table will be treated as a chunk." Two upstream rows must// yield exactly two chunks, each preserving its own content + metadata.@@ -83,8 +113,8 @@ func TestTableChunker_EmptyRows(t *testing.T) {}// TestTableChunkerPreservesHeaderOnlyTable: the header row only becomes a-// chunk itself when the segment has no data rows — then the whole markup is-// the table's only searchable representation.+// chunk itself when the segment has no data rows — then the whole table's+// plain-text rendering is the table's only searchable representation.func TestTableChunkerPreservesHeaderOnlyTable(t *testing.T) {segment := spreadsheetSegmentItem("Sheet1", []string{"ID", "Name"}, nil, 1, 1)chunks := tableChunksOf(t, map[string]any{@@ -95,8 +125,14 @@ func TestTableChunkerPreservesHeaderOnlyTable(t *testing.T) {if len(chunks) != 1 {t.Fatalf("got %d chunks, want the header-only table preserved", len(chunks))}- if chunks[0]["ck_type"] != "table" || chunks[0]["text"] != segment["text"] {- t.Fatalf("chunk = %#v, want the header-only segment whole", chunks[0])+ if chunks[0]["ck_type"] != "table" {+ t.Fatalf("chunk = %#v, want a table chunk", chunks[0])+ }+ if got, ok := chunks[0]["text"].(string); !ok || got == "" {+ t.Fatalf("header-only chunk must carry a plain-text body, got %#v", chunks[0]["text"])+ }+ if chunks[0]["table"] != nil {+ t.Fatalf("structured TableData must be cleared on the emitted chunk: %#v", chunks[0]["table"])}}@@ -116,14 +152,14 @@ func TestTableChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {if chunks[0]["text"] != "- ID: A-1\n- Name: paid" {t.Fatalf("chunk0 = %#v", chunks[0])}- if chunks[1]["text"] != headerSegment["text"] {- t.Fatalf("chunk1 = %#v, want the header-only segment whole", chunks[1])+ if chunks[1]["ck_type"] != "table" || chunks[1]["table"] != nil {+ t.Fatalf("chunk1 = %#v, want the header-only segment whole with TableData cleared", chunks[1])}}// TestTableChunker_ExpandsHTMLRowsWithAlignedPositions covers the P2-// expander: one HTML <table> item with a row-aligned position matrix (one-// tuple per <tr>, header included) becomes one chunk per data row, each+// expander: one table item with a row-aligned position matrix (one+// tuple per row, header included) becomes one chunk per data row, each// carrying its own tuple and the Python "- field: value" line format.func TestTableChunker_ExpandsHTMLRowsWithAlignedPositions(t *testing.T) {table := "<table><caption>orders</caption>\n" +@@ -135,13 +171,11 @@ func TestTableChunker_ExpandsHTMLRowsWithAlignedPositions(t *testing.T) {"name": "orders.xlsx","output_format": "json","json": []map[string]any{- {- "text": table,- "doc_type_kwd": "table",- "ck_type": "table",- "sheet_index": 1,- "positions": [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}, {1, 3, 3, 1, 2}},- },+ tableItem(t, table, map[string]any{+ "ck_type": "table",+ "sheet_index": 1,+ "positions": [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}, {1, 3, 3, 1, 2}},+ }),},})if len(chunks) != 2 {@@ -172,12 +206,10 @@ func TestTableChunker_WholeTableTupleNotCopiedToRows(t *testing.T) {"name": "sheet.csv","output_format": "json","json": []map[string]any{- {- "text": table,- "doc_type_kwd": "table",- "sheet_index": 1,- "positions": [][]float64{{1, 1, 2, 1, 1}},- },+ tableItem(t, table, map[string]any{+ "sheet_index": 1,+ "positions": [][]float64{{1, 1, 2, 1, 1}},+ }),},})if len(chunks) != 2 {@@ -190,7 +222,7 @@ func TestTableChunker_WholeTableTupleNotCopiedToRows(t *testing.T) {}}-// TestTableChunker_NonSpreadsheetPositionsNotAttached: a markup item without+// TestTableChunker_NonSpreadsheetPositionsNotAttached: a table item without// spreadsheet identity must not have its positions read as per-row tuples —// PDF items write layout boxes into the same field, so a five-field matrix// that happens to align must still be ignored.@@ -200,12 +232,10 @@ func TestTableChunker_NonSpreadsheetPositionsNotAttached(t *testing.T) {"name": "document.docx","output_format": "json","json": []map[string]any{- {- "text": table,- "doc_type_kwd": "table",- "ck_type": "table",- "positions": [][]float64{{1, 1, 1, 1, 1}, {1, 2, 2, 1, 1}, {1, 3, 3, 1, 1}},- },+ tableItem(t, table, map[string]any{+ "ck_type": "table",+ "positions": [][]float64{{1, 1, 1, 1, 1}, {1, 2, 2, 1, 1}, {1, 3, 3, 1, 1}},+ }),},})if len(chunks) != 2 {@@ -219,30 +249,39 @@ func TestTableChunker_NonSpreadsheetPositionsNotAttached(t *testing.T) {}// TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload: a table whose only-// row is the header has nothing to expand; the whole markup stays one chunk.+// row is the header has nothing to expand; the whole markup stays one chunk+// carrying the plain-text rendering of the header.func TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload(t *testing.T) {table := "<table><caption>t</caption>\n<tr><th>A</th><th>B</th></tr>\n</table>\n"+ td, err := tableutil.ParseTableHTML(table)+ if err != nil {+ t.Fatalf("ParseTableHTML: %v", err)+ }+ want := tableutil.RenderTableText(td)chunks := tableChunksOf(t, map[string]any{"name": "template.xlsx","output_format": "json","json": []map[string]any{- {"text": table, "doc_type_kwd": "table", "ck_type": "table"},+ tableItem(t, table, map[string]any{"ck_type": "table"}),},})- if len(chunks) != 1 || chunks[0]["text"] != table {- t.Fatalf("header-only table must stay a single whole-payload chunk, got %#v", chunks)+ if len(chunks) != 1 {+ t.Fatalf("got %d chunks, want the single header-only chunk", len(chunks))+ }+ if chunks[0]["text"] != want {+ t.Fatalf("header-only chunk text = %q, want %q", chunks[0]["text"], want)}}// TestTableChunker_HeaderlessHTMLFirstRowIsHeader: with no <th> row the-// first row names the columns, matching SplitLargeHTMLTable's convention.+// first row names the columns, matching splitLargeTable's convention.func TestTableChunker_HeaderlessHTMLFirstRowIsHeader(t *testing.T) {table := "<table><tr><td>k</td><td>v</td></tr><tr><td>a</td><td>b</td></tr></table>"chunks := tableChunksOf(t, map[string]any{"name": "kv.csv","output_format": "json","json": []map[string]any{- {"text": table, "doc_type_kwd": "table"},+ tableItem(t, table),},})if len(chunks) != 1 {diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 279813cd9b..bee33050a1 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -84,6 +84,7 @@ import (deepdoctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/tableutil""ragflow/internal/parser/chunk")@@ -255,6 +256,20 @@ func decodeChunkerFromUpstream(inputs map[string]any) (schema.ChunkerFromUpstreaif err := out.Validate(); err != nil {return out, err}+ // A structured table carries no markup text of its own. Derive a plain+ // (tag-free) text from its cells so consumers that read Text — TokenChunker+ // and the embed path — still receive a searchable body when the producer+ // emitted only the TableData contract (decision 1-B: no HTML in Text).+ for i := range out.JSONResult {+ if out.JSONResult[i].TableData != nil && out.JSONResult[i].Text == "" {+ out.JSONResult[i].Text = tableutil.RenderTableText(out.JSONResult[i].TableData)+ }+ }+ for i := range out.Chunks {+ if out.Chunks[i].TableData != nil && out.Chunks[i].Text == "" {+ out.Chunks[i].Text = tableutil.RenderTableText(out.Chunks[i].TableData)+ }+ }return out, nil}diff --git a/internal/ingestion/component/schema/chunk_table_test.go b/internal/ingestion/component/schema/chunk_table_test.gonew file mode 100644index 0000000000..400b4c5c42--- /dev/null+++ b/internal/ingestion/component/schema/chunk_table_test.go@@ -0,0 +1,85 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package schema++import (+ "encoding/json"+ "testing"++ "ragflow/internal/entity"+)++func TestChunkDocUnmarshalTableField(t *testing.T) {+ src := `{+ "doc_type_kwd": "table",+ "table": {"rows": [["Name", "Age"], ["Alice", "30"]], "header_rows": 1, "caption": "People"},+ "text": "legacy-html-should-be-ignored"+ }`+ var cd ChunkDoc+ if err := json.Unmarshal([]byte(src), &cd); err != nil {+ t.Fatalf("Unmarshal error = %v", err)+ }+ if cd.TableData == nil {+ t.Fatalf("TableData == nil, want populated")+ }+ if cd.TableData.HeaderRows != 1 {+ t.Fatalf("HeaderRows = %d, want 1", cd.TableData.HeaderRows)+ }+ if cd.TableData.Caption != "People" {+ t.Fatalf("Caption = %q, want %q", cd.TableData.Caption, "People")+ }+ if len(cd.TableData.Rows) != 2 || cd.TableData.Rows[1][0] != "Alice" {+ t.Fatalf("Rows = %v, unexpected", cd.TableData.Rows)+ }+ // The "table" key must NOT also land in Extra (no double storage).+ if _, ok := cd.Extra["table"]; ok {+ t.Fatalf("table key leaked into Extra: %v", cd.Extra)+ }+}++func TestChunkDocMarshalRoundTripTable(t *testing.T) {+ cd := ChunkDoc{+ DocType: "table",+ TableData: &entity.TableData{+ Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}},+ HeaderRows: 1,+ },+ }+ b, err := json.Marshal(cd)+ if err != nil {+ t.Fatalf("Marshal error = %v", err)+ }+ var back ChunkDoc+ if err := json.Unmarshal(b, &back); err != nil {+ t.Fatalf("Unmarshal error = %v", err)+ }+ if back.TableData == nil || len(back.TableData.Rows) != 2 {+ t.Fatalf("round-trip lost TableData: %v", back.TableData)+ }+}++func TestChunkDocNoTableFieldUnaffected(t *testing.T) {+ src := `{"doc_type_kwd": "text", "text": "hello", "custom": 42}`+ var cd ChunkDoc+ if err := json.Unmarshal([]byte(src), &cd); err != nil {+ t.Fatalf("Unmarshal error = %v", err)+ }+ if cd.TableData != nil {+ t.Fatalf("TableData = %v, want nil", cd.TableData)+ }+ if _, ok := cd.Extra["custom"]; !ok {+ t.Fatalf("Extra lost unrelated key: %v", cd.Extra)+ }+}diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.goindex 08ec8e9821..5aa15ed2df 100644--- a/internal/ingestion/component/schema/chunk_types.go+++ b/internal/ingestion/component/schema/chunk_types.go@@ -16,7 +16,11 @@package schema-import "encoding/json"+import (+ "encoding/json"++ "ragflow/internal/entity"+)// PayloadFormat is the discriminator shared by parser/chunker/tokenizer// wire payloads.@@ -99,34 +103,39 @@ func (m ChunkerFileMeta) MarshalJSON() ([]byte, error) {// boundaries. Common fields are explicit; dynamic enrichments are// preserved in Extra for forward compatibility.type ChunkDoc struct {- Text string `json:"text,omitempty"`- DocType string `json:"doc_type_kwd,omitempty"`- CKType string `json:"ck_type,omitempty"`- TKNums *int `json:"tk_nums,omitempty"`- Mom string `json:"mom,omitempty"`- ImgID string `json:"img_id,omitempty"`- ID string `json:"id,omitempty"`- Layout string `json:"layout,omitempty"`- LayoutType string `json:"layout_type,omitempty"`- LayoutNo string `json:"layoutno,omitempty"`- Image string `json:"image,omitempty"`- ContextAbove string `json:"context_above,omitempty"`- ContextBelow string `json:"context_below,omitempty"`- Questions string `json:"questions,omitempty"`- Keywords string `json:"keywords,omitempty"`- Summary string `json:"summary,omitempty"`- ChunkOrderInt *int `json:"chunk_order_int,omitempty"`- TitleTks string `json:"title_tks,omitempty"`- TitleSmTks string `json:"title_sm_tks,omitempty"`- ContentLtks string `json:"content_ltks,omitempty"`- ContentSmLtks string `json:"content_sm_ltks,omitempty"`- PageNumber *int `json:"page_number,omitempty"`- TopInt []int `json:"top_int,omitempty"`- PDFPositions json.RawMessage `json:"_pdf_positions,omitempty"`- Positions json.RawMessage `json:"positions,omitempty"`- Sheet string `json:"sheet,omitempty"`- SheetIndex *int `json:"sheet_index,omitempty"`- Extra map[string]json.RawMessage `json:"-"`+ Text string `json:"text,omitempty"`+ DocType string `json:"doc_type_kwd,omitempty"`+ CKType string `json:"ck_type,omitempty"`+ TKNums *int `json:"tk_nums,omitempty"`+ Mom string `json:"mom,omitempty"`+ ImgID string `json:"img_id,omitempty"`+ ID string `json:"id,omitempty"`+ Layout string `json:"layout,omitempty"`+ LayoutType string `json:"layout_type,omitempty"`+ LayoutNo string `json:"layoutno,omitempty"`+ Image string `json:"image,omitempty"`+ ContextAbove string `json:"context_above,omitempty"`+ ContextBelow string `json:"context_below,omitempty"`+ Questions string `json:"questions,omitempty"`+ Keywords string `json:"keywords,omitempty"`+ Summary string `json:"summary,omitempty"`+ ChunkOrderInt *int `json:"chunk_order_int,omitempty"`+ TitleTks string `json:"title_tks,omitempty"`+ TitleSmTks string `json:"title_sm_tks,omitempty"`+ ContentLtks string `json:"content_ltks,omitempty"`+ ContentSmLtks string `json:"content_sm_ltks,omitempty"`+ PageNumber *int `json:"page_number,omitempty"`+ TopInt []int `json:"top_int,omitempty"`+ PDFPositions json.RawMessage `json:"_pdf_positions,omitempty"`+ Positions json.RawMessage `json:"positions,omitempty"`+ Sheet string `json:"sheet,omitempty"`+ SheetIndex *int `json:"sheet_index,omitempty"`+ // TableData is the structured table contract. When a chunk carries it+ // the table content is consumed directly from here instead of being+ // re-parsed from Text markup. It must stay out of Extra (see the+ // deletion list in UnmarshalJSON).+ TableData *entity.TableData `json:"table,omitempty"`+ Extra map[string]json.RawMessage `json:"-"`}func (d *ChunkDoc) UnmarshalJSON(data []byte) error {@@ -149,7 +158,7 @@ func (d *ChunkDoc) UnmarshalJSON(data []byte) error {"context_above", "context_below", "questions", "keywords", "summary","chunk_order_int", "title_tks", "title_sm_tks", "content_ltks","content_sm_ltks", "tag_kwd", "page_number", "top_int", "_pdf_positions", "positions",- "sheet", "sheet_index",+ "sheet", "sheet_index", "table",} {delete(raw, key)}diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex 5694a57e01..185f46222a 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -105,11 +105,9 @@ const ComponentNameTokenizer = "Tokenizer"// `re.sub(r"\.[a-zA-Z]+$", "", name)` in tokenizer.py:137.var titleExtRE = regexp.MustCompile(`\.[a-zA-Z]+$`)-// htmlTableRE matches HTML table-cell tags so the embedded text fed-// to the embedding model doesn't carry raw markup. Mirrors the python-// `re.sub(r"</?(table|td|caption|tr|th)( [^<>]{0,12})?>", " ", txt)` at-// tokenizer.py:79.-var htmlTableRE = regexp.MustCompile(`</?(table|td|caption|tr|th)( [^<>]{0,12})?>`)+// Table content is no longer carried as HTML markup (the parser emits the+// structured entity.TableData contract), so there is no table markup to strip+// from the embedded text. The legacy htmlTableRE was removed.// EmbeddingResult carries a vector plus the model-reported token usage// for that input batch entry.@@ -472,8 +470,7 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, nacacheHits := 0for i, ck := range chunks {raw := concatFields(ck, c.param.Fields)- txt := htmlTableRE.ReplaceAllString(raw, " ")- txt = strings.TrimSpace(txt)+ txt := strings.TrimSpace(raw)if txt == "" {continue}@@ -1077,7 +1074,7 @@ func requiresEmbeddingVector(ck schema.ChunkDoc, fields []string) bool {}func cleanEmbeddingText(text string) string {- return strings.TrimSpace(htmlTableRE.ReplaceAllString(text, " "))+ return strings.TrimSpace(text)}func hasEmbeddingVector(ck schema.ChunkDoc) bool {diff --git a/internal/parser/parser/align_test.go b/internal/parser/parser/align_test.goindex 640b14aadc..64a18407d5 100644--- a/internal/parser/parser/align_test.go+++ b/internal/parser/parser/align_test.go@@ -390,8 +390,11 @@ func tableCellSignature(text string) string {func tableSignatures(items []map[string]any) map[string]bool {sigs := map[string]bool{}for _, it := range items {+ // A structured table item carries the TableData contract (no HTML+ // text); render it back to the legacy markup so the cell signature+ // matches the Python golden, which keeps raw <table> HTML.+ text := renderedText(it)kd, _ := it["doc_type_kwd"].(string)- text, _ := it["text"].(string)if kd != "table" && !isStandaloneTable(text) {continue}diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.goindex 7eccf252f8..1445a2279f 100644--- a/internal/parser/parser/charset_test.go+++ b/internal/parser/parser/charset_test.go@@ -167,7 +167,7 @@ func TestCSVParser_GBK(t *testing.T) {}var rowText strings.Builderfor _, item := range res.JSON {- text, _ := item["text"].(string)+ text := tableItemHTML(t, item)rowText.WriteString(text)}if strings.ContainsRune(rowText.String(), '\ufffd') {diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.goindex f50c38cec9..283ac0b245 100644--- a/internal/parser/parser/csv_parser_test.go+++ b/internal/parser/parser/csv_parser_test.go@@ -35,8 +35,8 @@ func TestCSVParser_EmitsSpreadsheetRows(t *testing.T) {"<tr><td>Alice</td><td>30</td><td>New York</td></tr>\n" +"<tr><td>Bob</td><td>25</td><td>San Francisco</td></tr>\n" +"</table>\n"- if item["text"] != wantText {- t.Errorf("text = %q, want %q", item["text"], wantText)+ if rendered := tableItemHTML(t, item); rendered != wantText {+ t.Errorf("table HTML = %q, want %q", rendered, wantText)}if item["sheet"] != "Data" || item["sheet_index"] != 1 {t.Errorf("sheet metadata = %#v", item)@@ -69,7 +69,7 @@ func TestCSVParser_UTF8BOM(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("items = %#v, want one table item", res.JSON)}- text, _ := res.JSON[0]["text"].(string)+ text := tableItemHTML(t, res.JSON[0])if !strings.Contains(text, "<tr><th>Name</th><th>Age</th></tr>") {t.Errorf("header row = %q", text)}@@ -177,7 +177,7 @@ func TestCSVParserKeepsVariableRowWidths(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("items = %d, want one HTML table item", len(res.JSON))}- text, _ := res.JSON[0]["text"].(string)+ text := tableItemHTML(t, res.JSON[0])if !strings.Contains(text, "<tr><td>Q1</td><td>A1</td></tr>") ||!strings.Contains(text, "<tr><td>Q2</td><td>A2</td><td>extra</td></tr>") {t.Fatalf("row widths not preserved in markup: %q", text)@@ -201,7 +201,7 @@ func TestCSVParser_ReadsTheSeparatorTheFileWasWrittenWith(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("separator %q: items = %d, want one HTML table item", sep, len(res.JSON))}- text, _ := res.JSON[0]["text"].(string)+ text := tableItemHTML(t, res.JSON[0])if !strings.Contains(text, "<tr><th>Name</th><th>Region</th><th>Units</th></tr>") ||!strings.Contains(text, "<tr><td>Widget</td><td>EU</td><td>12</td></tr>") {t.Errorf("separator %q: row not split into three columns: %q", sep, text)@@ -272,7 +272,7 @@ func TestCSVParser_KeepsAnEmptyTabSeparatedField(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("items = %d, want one HTML table item", len(res.JSON))}- text, _ := res.JSON[0]["text"].(string)+ text := tableItemHTML(t, res.JSON[0])if !strings.Contains(text, "<tr><td>Widget</td><td></td><td>12</td></tr>") {t.Errorf("empty tab-separated field not kept: %q", text)}diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.goindex f6a2f8569c..b4188238af 100644--- a/internal/parser/parser/docx_ir.go+++ b/internal/parser/parser/docx_ir.go@@ -359,6 +359,7 @@ func buildDOCXJSONSections(irJSON string, budget *embeddedMediaBudget) []map[str"image": nil,"doc_type_kwd": "table",}+ projectPDFTable(table)tableID := ""foundImage := falsevar tableImages []docxTableImagediff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.goindex 03c98ffe2c..20a82e92c4 100644--- a/internal/parser/parser/docx_ir_test.go+++ b/internal/parser/parser/docx_ir_test.go@@ -7,6 +7,8 @@ import ("reflect""strings""testing"++ "ragflow/internal/entity")// TestBuildDOCXJSONSections_FromJSON feeds an office_oxide IR JSON@@ -30,7 +32,7 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {{"text": "Title", "image": nil, "doc_type_kwd": "text", "ck_type": "heading"},{"text": "Hello", "image": nil, "doc_type_kwd": "text"},{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},- {"text": "<table><tr><td>cell</td></tr></table>", "image": nil, "doc_type_kwd": "table"},+ {"image": nil, "doc_type_kwd": "table", "table": &entity.TableData{Rows: [][]string{{"cell"}}, HeaderRows: 1}},{"text": "item1", "image": nil, "doc_type_kwd": "text"},}if !reflect.DeepEqual(got, want) {@@ -91,9 +93,12 @@ func TestBuildDOCXJSONSections_ExtractsTableCellImages(t *testing.T) {if len(got) != 3 {t.Fatalf("sections = %+v, want table plus two cell image items", got)}- if got[0]["doc_type_kwd"] != "table" || got[0]["text"] != "<table><tr><td>beforeafter</td><td></td></tr></table>" || got[0]["source_table_id"] != "docx-table-1" {+ if got[0]["doc_type_kwd"] != "table" || got[0]["source_table_id"] != "docx-table-1" {t.Fatalf("table item = %+v", got[0])}+ if _, ok := got[0]["table"].(*entity.TableData); !ok {+ t.Fatalf("table item missing structured TableData: %+v", got[0])+ }for i, want := range []string{"aGVsbG8=", "aW1hZ2U="} {item := got[i+1]if item["doc_type_kwd"] != "image" || item["image"] != want {diff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.goindex 42e32d47d8..b222854a41 100644--- a/internal/parser/parser/docx_parser_test.go+++ b/internal/parser/parser/docx_parser_test.go@@ -23,6 +23,8 @@ import ("encoding/json""strings""testing"++ "ragflow/internal/entity")func TestBuildDOCXJSONSections_Paragraphs(t *testing.T) {@@ -95,18 +97,23 @@ func TestBuildDOCXJSONSections_Tables(t *testing.T) {t.Fatalf("got %d sections, want 1", len(sections))}item := sections[0]- html, ok := item["text"].(string)- if !ok {- t.Fatal("text field missing or not string")- }- if !strings.Contains(html, "<table>") || !strings.Contains(html, "</table>") {- t.Errorf("html = %q, missing <table> tags", html)+ td, ok := item["table"].(*entity.TableData)+ if !ok || td == nil {+ t.Fatal("table field missing or not *entity.TableData")+ }+ var all strings.Builder+ for _, row := range td.Rows {+ for _, c := range row {+ all.WriteString(c)+ }}- if !strings.Contains(html, "<tr>") || !strings.Contains(html, "</tr>") {- t.Errorf("html = %q, missing <tr> tags", html)+ s := all.String()+ if !strings.Contains(s, "A1") || !strings.Contains(s, "B1") ||+ !strings.Contains(s, "A2") || !strings.Contains(s, "B2") {+ t.Errorf("table rows = %#v, missing cell content", td.Rows)}- if !strings.Contains(html, "<td>A1</td>") || !strings.Contains(html, "<td>B2</td>") {- t.Errorf("html = %q, missing cell content", html)+ if len(td.Rows) != 2 {+ t.Errorf("table rows = %#v, want 2 rows", td.Rows)}if got, ok := item["doc_type_kwd"].(string); !ok || got != "table" {t.Errorf("doc_type_kwd = %q, want %q", got, "table")diff --git a/internal/parser/parser/html_charset_test.go b/internal/parser/parser/html_charset_test.goindex b42f6f58b6..604f4ba442 100644--- a/internal/parser/parser/html_charset_test.go+++ b/internal/parser/parser/html_charset_test.go@@ -63,10 +63,8 @@ func htmlEncodedFixture(t *testing.T, src string, enc encoding.Encoding) []bytefunc itemsText(res ParseResult) string {var sb strings.Builderfor _, it := range res.JSON {- if s, ok := it["text"].(string); ok {- sb.WriteString(s)- sb.WriteString("\n")- }+ sb.WriteString(renderedText(it))+ sb.WriteString("\n")}return sb.String()}@@ -163,7 +161,7 @@ func TestHTMLParser_DecodeTableSurvivesGBK(t *testing.T) {var tableText stringfor _, it := range res.JSON {if it["doc_type_kwd"] == "table" {- tableText, _ = it["text"].(string)+ tableText = renderedText(it)}}if tableText == "" {diff --git a/internal/parser/parser/html_media_test.go b/internal/parser/parser/html_media_test.goindex 5df4e73238..024e28a859 100644--- a/internal/parser/parser/html_media_test.go+++ b/internal/parser/parser/html_media_test.go@@ -45,12 +45,12 @@ func TestHTMLParser_EmitsTableCellImagesWithParentAndCellOrder(t *testing.T) {if result.JSON[0]["doc_type_kwd"] != "table" || result.JSON[0]["source_table_id"] != "html-table-1" {t.Errorf("table item = %+v", result.JSON[0])}- tableText, _ := result.JSON[0]["text"].(string)+ tableText := renderedText(result.JSON[0])if strings.Contains(tableText, "data:image/") {t.Error("table text duplicates the separately emitted inline image payload")}- if !strings.Contains(tableText, `alt="first"`) {- t.Errorf("table text lost the image alt text: %q", tableText)+ if !strings.Contains(tableText, "A") {+ t.Errorf("table text lost the cell text: %q", tableText)}if result.JSON[1]["image"] != htmlMediaDataURI {t.Errorf("inline image payload = %v, want data URI", result.JSON[1]["image"])diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex 1a8a94f3d1..a41404d366 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -25,6 +25,7 @@ import ("strings""golang.org/x/net/html"+ "ragflow/internal/parser/tableutil")type HTMLParser struct {@@ -405,10 +406,17 @@ func emitHTMLTable(n *html.Node, out *[]map[string]any, state *htmlWalkState) {if strings.TrimSpace(markup) == "" {return}+ // Emit the structured TableData contract so the chunker reads rows+ // directly instead of re-parsing markup. ParseTableHTML mirrors the+ // legacy chunker extraction, so the recovered rows/caption are identical+ // to what the old HTML path produced. html.Parse (used by ParseTableHTML)+ // is infallible, so the parse always yields a non-nil TableData and no+ // legacy HTML `text` is kept.+ td, _ := tableutil.ParseTableHTML(markup)table := map[string]any{- "text": markup,"doc_type_kwd": "table","ck_type": "table",+ "table": td,}tableID := ""if len(images) > 0 {diff --git a/internal/parser/parser/html_parser_align_test.go b/internal/parser/parser/html_parser_align_test.goindex 3feae909d0..daf42abfe9 100644--- a/internal/parser/parser/html_parser_align_test.go+++ b/internal/parser/parser/html_parser_align_test.go@@ -47,7 +47,7 @@ func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {trailingIdx = i}case "table":- tableText = text+ tableText = renderedText(it)tableIdx = itableCount++default:@@ -130,7 +130,7 @@ func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {trailingIdx = i}case "table":- tableText = text+ tableText = renderedText(it)tableIdx = itableCount++default:@@ -216,7 +216,7 @@ func TestHTMLParser_NestedTableWithSurroundingText(t *testing.T) {afterIdx = i}case "table":- tableText = text+ tableText = renderedText(it)tableIdx = itableCount++default:@@ -312,8 +312,8 @@ func TestHTMLParser_MultipleTablesOrdering(t *testing.T) {if !(titleIdx < tableItemIdx[0] && tableItemIdx[0] < middleIdx && middleIdx < tableItemIdx[1] && tableItemIdx[1] < endIdx) {t.Fatalf("table order wrong: tables=%v title=%d middle=%d end=%d", tableItemIdx, titleIdx, middleIdx, endIdx)}- t1, _ := res.JSON[tableItemIdx[0]]["text"].(string)- t2, _ := res.JSON[tableItemIdx[1]]["text"].(string)+ t1 := renderedText(res.JSON[tableItemIdx[0]])+ t2 := renderedText(res.JSON[tableItemIdx[1]])if !strings.Contains(t1, "x") || !strings.Contains(t1, "y") {t.Errorf("first table item missing x/y cells: %q", t1)}diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.goindex 606501a266..38843297e6 100644--- a/internal/parser/parser/markdown_parser.go+++ b/internal/parser/parser/markdown_parser.go@@ -32,6 +32,7 @@ import (markdownlib "github.com/gomarkdown/markdown""github.com/gomarkdown/markdown/ast"mdparser "github.com/gomarkdown/markdown/parser"+ "ragflow/internal/parser/tableutil")// dataURIPrefix is the MIME prefix for data URI images.@@ -438,11 +439,18 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[// embedded once and its markup does not pollute prose chunks.txt = leafText(n)if isTableHTML(txt) {- *out = append(*out, map[string]any{- "text": txt,+ // Emit the structured TableData contract so the chunker reads+ // rows directly instead of re-parsing markup. ParseTableHTML+ // mirrors the legacy chunker extraction. html.Parse is+ // infallible, so the parse always yields a non-nil TableData+ // and no legacy HTML `text` is kept.+ td, _ := tableutil.ParseTableHTML(txt)+ table := map[string]any{"doc_type_kwd": "table","ck_type": "table",- })+ "table": td,+ }+ *out = append(*out, table)continue}// Non-table HTML block: ordinary text, no ck_type.diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.goindex d91241b1e2..3365889472 100644--- a/internal/parser/parser/markdown_parser_test.go+++ b/internal/parser/parser/markdown_parser_test.go@@ -9,6 +9,8 @@ import ("strings""sync/atomic""testing"++ "ragflow/internal/entity")func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {@@ -180,8 +182,7 @@ func TestMarkdownParser_TableKeepsCellsBeyondTheHeader(t *testing.T) {}var joined strings.Builderfor _, item := range res.JSON {- text, _ := item["text"].(string)- joined.WriteString(text)+ joined.WriteString(renderedText(item))joined.WriteString("\n")}out := joined.String()@@ -275,8 +276,12 @@ func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {if ck, _ := item["ck_type"].(string); ck != "table" {t.Errorf("table item ck_type = %q, want \"table\"", ck)}- if !strings.Contains(text, "<table") {- t.Errorf("table item text is not raw <table> HTML: %q", text)+ td, ok := item["table"].(*entity.TableData)+ if !ok || td == nil {+ t.Fatalf("table item missing *entity.TableData: %#v", item)+ }+ if !tableRowsContain(td, "Check item") {+ t.Errorf("table item rows missing header label: %+v", td.Rows)}case "text":if strings.Contains(text, "<table") {@@ -634,8 +639,8 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {case "table":sawTableItem = truetableCount++- if !strings.Contains(text, "<table") {- t.Fatalf("table item text is not raw <table> HTML: %q", text)+ if _, ok := item["table"].(*entity.TableData); !ok {+ t.Fatalf("table item missing *entity.TableData: %#v", item)}if ck, _ := item["ck_type"].(string); ck != "table" {t.Fatalf("table item ck_type = %q, want \"table\"", ck)@@ -696,11 +701,11 @@ func TestMarkdownParser_TableWithSurroundingText(t *testing.T) {afterIdx = i}case "table":- tableText = text+ tableText = renderedText(item)tableIdx = itableCount++- if !strings.Contains(text, "<table") {- t.Fatalf("table item text is not raw <table> HTML: %q", text)+ if _, ok := item["table"].(*entity.TableData); !ok {+ t.Fatalf("table item missing *entity.TableData: %#v", item)}if ck, _ := item["ck_type"].(string); ck != "table" {t.Fatalf("table item ck_type = %q, want \"table\"", ck)@@ -819,8 +824,8 @@ func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {case "table":sawTableItem = truetableCount++- if !strings.Contains(text, "<table") {- t.Fatalf("raw table item text is not raw <table> HTML: %q", text)+ if _, ok := item["table"].(*entity.TableData); !ok {+ t.Fatalf("raw table item missing *entity.TableData: %#v", item)}if ck, _ := item["ck_type"].(string); ck != "table" {t.Fatalf("raw table item ck_type = %q, want \"table\"", ck)@@ -882,8 +887,8 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {if !(titleIdx < tableItemIdx[0] && tableItemIdx[0] < middleIdx && middleIdx < tableItemIdx[1] && tableItemIdx[1] < endIdx) {t.Fatalf("table order wrong: tables=%v title=%d middle=%d end=%d", tableItemIdx, titleIdx, middleIdx, endIdx)}- t1, _ := res.JSON[tableItemIdx[0]]["text"].(string)- t2, _ := res.JSON[tableItemIdx[1]]["text"].(string)+ t1 := renderedText(res.JSON[tableItemIdx[0]])+ t2 := renderedText(res.JSON[tableItemIdx[1]])if !strings.Contains(t1, "x") || !strings.Contains(t1, "y") {t.Fatalf("first table item missing x/y cells: %q", t1)}@@ -955,3 +960,17 @@ func TestMarkdownParser_AlignmentGolden(t *testing.T) {})}}++// tableRowsContain reports whether any cell of the structured table contains+// the substring want, so table-contract tests can assert cell content without+// depending on the legacy HTML markup shape.+func tableRowsContain(td *entity.TableData, want string) bool {+ for _, row := range td.Rows {+ for _, cell := range row {+ if strings.Contains(cell, want) {+ return true+ }+ }+ }+ return false+}diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex 0c22e5267b..4c074d5e97 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -19,7 +19,6 @@ package parserimport ("encoding/base64""fmt"- "html""regexp""sort""strconv"@@ -29,6 +28,7 @@ import ("go.uber.org/zap""ragflow/internal/common"+ "ragflow/internal/entity")// tableIllegalCharsRe replaces illegal control characters (everything except@@ -51,47 +51,32 @@ type spreadsheetSegmentRow struct {colStart int}-// renderTableHTML renders one sheet segment in the spreadsheet wire format: a-// captioned <table> whose header row is <th> cells and whose data rows are-// <td> cells, one row per line. It is the byte contract every consumer reads —-// the chunkers parse this shape back row by row (see-// internal/ingestion/component/chunker/html_rows.go), and the positions matrix-// emitted next to it is aligned one tuple per <tr>, header included.-func renderSpreadsheetTable(sheet string, header []string, rows [][]string) string {- var builder strings.Builder- builder.WriteString("<table><caption>")- builder.WriteString(html.EscapeString(sheet))- builder.WriteString("</caption>\n<tr>")- for _, cell := range header {- builder.WriteString("<th>")- builder.WriteString(html.EscapeString(strings.TrimSpace(cell)))- builder.WriteString("</th>")- }- builder.WriteString("</tr>\n")- for _, row := range rows {- builder.WriteString("<tr>")- for _, cell := range row {- builder.WriteString("<td>")- builder.WriteString(html.EscapeString(strings.TrimSpace(cell)))- builder.WriteString("</td>")- }- builder.WriteString("</tr>\n")- }- builder.WriteString("</table>\n")- return builder.String()+// trimCells returns a copy of row with every cell TrimSpace'd. The legacy+// spreadsheet producer HTML-escaped each TrimSpace'd cell; the structured+// TableData contract stores the already-TrimSpace'd text, which the chunker+// re-escapes when it renders chunk text (see tableutil.RenderTableHTML), so+// the rendered text matches the legacy HTML byte for byte.+func trimCells(row []string) []string {+ out := make([]string, len(row))+ for i, c := range row {+ out[i] = strings.TrimSpace(c)+ }+ return out}-// buildSheetItems renders one sheet into its wire items: HTML <table>-// segments split at image anchor boundaries, interleaved with the anchored-// image items in document order. Every segment repeats the header row (so-// segment 2+ never loses column names) and carries a row-aligned position-// matrix — one tuple per <tr> in strict markup order, header included — so-// row-level consumers index positions by <tr> number with no side channel.+// buildSheetItems renders one sheet into its wire items: the structured+// TableData contract (captioned table with a header row and data rows) split+// at image anchor boundaries, interleaved with the anchored image items in+// document order. Every segment repeats the header row (so segment 2+ never+// loses column names) and carries a row-aligned position matrix — one tuple+// per row in strict order, header included — so row-level consumers index+// positions by row number with no side channel. The chunker reads the rows+// directly; no HTML markup is emitted.//// An image lands after the last row whose (row, colStart) sorts at or before// its anchor; rows before the header's anchor keep the header in front of// the image (a header-only lead segment); fully empty rows are skipped from-// both markup and matrix.+// both rows and matrix.//// Segmentation is semantic only — sheet boundaries and image anchors, never a// size budget. The legacy html4excel path pre-cut a fixed 12 rows here, which@@ -179,14 +164,16 @@ func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow intitems := make([]map[string]any, 0, len(rows)+len(images))newSegment := func(body []spreadsheetSegmentRow) map[string]any {- cells := make([][]string, 0, len(body))matrix := make([][]float64, 0, len(body)+1)matrix = append(matrix, headerTuple)+ rowsOut := make([][]string, 0, len(body)+1)+ rowsOut = append(rowsOut, trimCells(header))for _, r := range body {- cells = append(cells, r.cells)+ rowsOut = append(rowsOut, trimCells(r.cells))matrix = append(matrix, r.tuple)}- item := NewTableJSONItem(renderSpreadsheetTable(sheet, header, cells), sheet, matrix)+ td := &entity.TableData{Rows: rowsOut, HeaderRows: 1, Caption: sheet}+ item := NewTableDataJSONItem(td, sheet, matrix)item["sheet_index"] = sheetIndexreturn item}diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.goindex 6bb517a548..a8fd74c909 100644--- a/internal/parser/parser/office_table_render_test.go+++ b/internal/parser/parser/office_table_render_test.go@@ -8,6 +8,8 @@ import ("testing""github.com/xuri/excelize/v2"+ "ragflow/internal/entity"+ "ragflow/internal/parser/tableutil")func TestXLSXParserEmitsSegmentedHTMLTable(t *testing.T) {@@ -31,14 +33,14 @@ func TestXLSXParserEmitsSegmentedHTMLTable(t *testing.T) {if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {t.Fatalf("item types = ck_type:%v doc_type:%v", item["ck_type"], item["doc_type_kwd"])}- text, _ := item["text"].(string)wantText := "<table><caption>Sheet1</caption>\n" +"<tr><th>ID</th><th>Status</th></tr>\n" +"<tr><td>A-100</td><td>paid</td></tr>\n" +"<tr><td>A-101</td><td>pending</td></tr>\n" +"</table>\n"- if text != wantText {- t.Fatalf("text = %q, want %q", text, wantText)+ rendered := tableItemHTML(t, item)+ if rendered != wantText {+ t.Fatalf("table HTML = %q, want %q", rendered, wantText)}if item["sheet"] != "Sheet1" || item["sheet_index"] != 1 {t.Fatalf("sheet metadata = %#v", item)@@ -79,7 +81,7 @@ func TestXLSXParserHTML4ExcelIsIgnored(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("items = %d, want one atomic table item", len(res.JSON))}- if res.JSON[0]["ck_type"] != "table" || !strings.Contains(res.JSON[0]["text"].(string), "<table>") {+ if res.JSON[0]["ck_type"] != "table" || res.JSON[0]["table"] == nil {t.Fatalf("html4excel item = %#v", res.JSON[0])}if !reflect.DeepEqual(res.JSON, plainRes.JSON) {@@ -111,7 +113,7 @@ func TestXLSXParserDoesNotPrechunkRows(t *testing.T) {if last[1] != float64(dataRows+1) || last[2] != float64(dataRows+1) {t.Fatalf("last tuple = %v, want source row %d", last, dataRows+1)}- text, _ := res.JSON[0]["text"].(string)+ text := tableItemHTML(t, res.JSON[0])if !strings.Contains(text, "<td>258</td>") {t.Fatalf("last data row missing from markup:\n%s", text)}@@ -170,8 +172,8 @@ func TestXLSXParserEmitsHeaderOnlySegment(t *testing.T) {wantText := "<table><caption>Sheet1</caption>\n" +"<tr><th>Name</th><th>Amount</th></tr>\n" +"</table>\n"- if item["text"] != wantText {- t.Fatalf("text = %q, want %q", item["text"], wantText)+ if rendered := tableItemHTML(t, item); rendered != wantText {+ t.Fatalf("table HTML = %q, want %q", rendered, wantText)}positions, _ := item["positions"].([][]float64)if len(positions) != 1 || !reflect.DeepEqual(positions[0], []float64{1, 1, 1, 1, 2}) {@@ -188,7 +190,7 @@ func TestBuildSheetItemsEscapesCellText(t *testing.T) {if len(items) != 1 {t.Fatalf("items = %d, want one segment", len(items))}- text, _ := items[0]["text"].(string)+ text := tableItemHTML(t, items[0])if strings.Contains(text, "<b>") || !strings.Contains(text, "<b>") {t.Fatalf("cell text was not escaped: %q", text)}@@ -244,7 +246,7 @@ func TestXLSXParserImageAnchorsSplitSegments(t *testing.T) {}for segIdx, want := range [][]string{wantFirst, wantSecond} {item := items[segIdx*2]- text, _ := item["text"].(string)+ text := tableItemHTML(t, item)rows := markupRows(t, text)if len(rows) != len(want)+1 {t.Fatalf("segment %d rows = %d, want header plus %d", segIdx, len(rows), len(want))@@ -303,11 +305,37 @@ func markupRows(t *testing.T, markup string) [][]string {return rows}+// tableItemHTML renders a parser-produced structured table item back to the+// legacy spreadsheet HTML so the byte-level assertions in these tests keep+// pinning the same wire shape. The producer emits *entity.TableData now, and+// RenderTableHTML reproduces the legacy markup exactly.+func tableItemHTML(t *testing.T, item map[string]any) string {+ t.Helper()+ td, ok := item["table"].(*entity.TableData)+ if !ok {+ t.Fatalf("item missing *entity.TableData table field: %#v", item)+ }+ return tableutil.RenderTableHTML(td)+}++// renderedText returns the legacy HTML rendering of an item's structured table+// when present, otherwise its text field. It lets table-contract tests pin+// the byte-level wire shape whether the producer emitted TableData or plain+// text (e.g. non-table items).+func renderedText(item map[string]any) string {+ if td, ok := item["table"].(*entity.TableData); ok && td != nil {+ return tableutil.RenderTableHTML(td)+ }+ text, _ := item["text"].(string)+ return text+}+func spreadsheetText(res ParseResult) string {var out strings.Builderfor _, item := range res.JSON {- text, _ := item["text"].(string)- out.WriteString(text)+ if td, ok := item["table"].(*entity.TableData); ok {+ out.WriteString(tableutil.RenderTableHTML(td))+ }}return out.String()}@@ -317,10 +345,11 @@ func spreadsheetText(res ParseResult) string {// table item exists.func spreadsheetHeaderText(res ParseResult) string {for _, item := range res.JSON {- text, _ := item["text"].(string)- if !strings.Contains(strings.ToLower(text), "<table") {+ td, ok := item["table"].(*entity.TableData)+ if !ok {continue}+ text := tableutil.RenderTableHTML(td)row := textif i := strings.Index(row, "<tr>"); i >= 0 {row = row[i+len("<tr>"):]diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.goindex 4d7752deb7..655bfd88e7 100644--- a/internal/parser/parser/parse_result.go+++ b/internal/parser/parser/parse_result.go@@ -31,7 +31,11 @@package parser-import "context"+import (+ "context"++ "ragflow/internal/entity"+)// ParseResult is the structured return value of a parse operation. JSON holds// structured backend output when available; the rendered fields carry@@ -108,10 +112,14 @@ func NewTextJSONItem(text string) map[string]any {}}-// NewTableJSONItem constructs a canonical table JSON item for parser output.-func NewTableJSONItem(html string, sheet string, positions [][]float64) map[string]any {+// NewTableDataJSONItem constructs a canonical structured table JSON item for+// parser output. It carries the parsed TableData contract (caption, header+// rows, cell text already normalized the same way the legacy chunker+// normalized HTML <td>/<th> text) instead of the legacy HTML markup string,+// so the chunker reads rows directly without re-parsing any markup.+func NewTableDataJSONItem(td *entity.TableData, sheet string, positions [][]float64) map[string]any {item := map[string]any{- "text": html,+ "table": td,DocTypeKey: DocTypeTable,"ck_type": DocTypeTable,}diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.goindex 031e378dad..380f4bf0b7 100644--- a/internal/parser/parser/parse_result_test.go+++ b/internal/parser/parser/parse_result_test.go@@ -19,10 +19,17 @@ package parserimport ("strings""testing"++ "ragflow/internal/entity")-func TestNewTableJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {- item := NewTableJSONItem("<table><tr><td>value</td></tr></table>", "Sheet1", [][]float64{{1, 2, 2, 1, 1}})+func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {+ td := &entity.TableData{+ Rows: [][]string{{"col"}, {"value"}},+ HeaderRows: 1,+ Caption: "Sheet1",+ }+ item := NewTableDataJSONItem(td, "Sheet1", [][]float64{{1, 2, 2, 1, 1}})if got, want := item[DocTypeKey], DocTypeTable; got != want {t.Fatalf("item[%q] = %v, want %v", DocTypeKey, got, want)@@ -30,6 +37,16 @@ func TestNewTableJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {if got, want := item["ck_type"], DocTypeTable; got != want {t.Fatalf("item[%q] = %v, want %v", "ck_type", got, want)}+ got, ok := item["table"].(*entity.TableData)+ if !ok {+ t.Fatalf("item[%q] = %T, want *entity.TableData", "table", item["table"])+ }+ if got.Caption != "Sheet1" || len(got.Rows) != 2 {+ t.Fatalf("item table = %+v, want caption Sheet1 and 2 rows", got)+ }+ if _, ok := item["positions"].([][]float64); !ok {+ t.Fatalf("item[%q] missing positions matrix", "positions")+ }}// TestParseResult_Contract pins the wire-shape guaranteesdiff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex bbc3d94221..dcb93f0a96 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -32,6 +32,7 @@ import (pdflayout "ragflow/internal/deepdoc/parser/pdf/layout""ragflow/internal/deepdoc/parser/pdf/util"deepdoctype "ragflow/internal/deepdoc/parser/type"+ "ragflow/internal/parser/tableutil""ragflow/internal/utility")@@ -494,6 +495,7 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR}}normalizePDFDocType(items[i])+ projectPDFTable(items[i])if img, _ := items[i]["image"].(string); img != "" {items[i]["image"] = pdflayout.InlinePNGDataURL(img)}@@ -931,6 +933,42 @@ func normalizePDFDocType(item map[string]any) {}}+// projectPDFTable upgrades a legacy pdf table item — one whose doc_type_kwd is+// "table" and whose text is <table> markup — to the structured TableData+// contract, so the downstream chunker reads Rows directly instead of+// re-parsing HTML. ParseTableHTML + RenderTableHTML round-trips the legacy+// htmlTableRE extraction exactly (entity/html unescaping is left to the+// consumer), so dropping the original HTML string loses no information: the+// markdown and search paths render it on demand from `table`. The pdf page+// number (when present) is projected onto TableData.Page.+//+// The redundant HTML `text` is deleted once `table` is projected: keeping both+// would duplicate every table's cell text in the parse result (the storage+// cost that motivated moving to the structured contract). Non-table items, and+// table items whose text is not <table> markup, are left untouched so their+// `text` is preserved as a last-resort carrier.+func projectPDFTable(item map[string]any) {+ if item == nil {+ return+ }+ if dt, _ := item["doc_type_kwd"].(string); dt != "table" {+ return+ }+ text, _ := item["text"].(string)+ if !isTableHTML(text) {+ return+ }+ td, err := tableutil.ParseTableHTML(text)+ if err != nil || td == nil {+ return+ }+ if page := numberValue(item["page_number"]); page > 0 {+ td.Page = int(page)+ }+ item["table"] = td+ delete(item, "text")+}+// ExtractPDFPositions is the single source of truth for "does this parsed item// carry a usable PDF crop region". It returns the non-empty positions matrix// (under the canonical _pdf_positions key or the legacy positions key),diff --git a/internal/parser/parser/pdf_parser_opendataloader.go b/internal/parser/parser/pdf_parser_opendataloader.goindex dc9c324466..8caa1a1d87 100644--- a/internal/parser/parser/pdf_parser_opendataloader.go+++ b/internal/parser/parser/pdf_parser_opendataloader.go@@ -149,7 +149,9 @@ func openDataLoaderNodeToItem(el map[string]any) map[string]any {if text == "" {return nil}- return map[string]any{"text": text, "doc_type_kwd": pdfTableDocType(text), "layout": "table"}+ item := map[string]any{"text": text, "doc_type_kwd": pdfTableDocType(text), "layout": "table"}+ projectPDFTable(item)+ return itemcase "image", "picture", "figure":if text == "" {text = "[Image]"diff --git a/internal/parser/parser/pdf_parser_opendataloader_test.go b/internal/parser/parser/pdf_parser_opendataloader_test.goindex ebf5f55707..b20301ca03 100644--- a/internal/parser/parser/pdf_parser_opendataloader_test.go+++ b/internal/parser/parser/pdf_parser_opendataloader_test.go@@ -7,6 +7,8 @@ import ("net/http/httptest""strings""testing"++ "ragflow/internal/entity")func TestPDFParser_ParseWithResult_OpenDataLoaderJSONIntegration(t *testing.T) {@@ -134,8 +136,18 @@ func TestOpenDataLoaderItems_TableCellsFallback(t *testing.T) {if len(items) != 1 {t.Fatalf("items len = %d, want 1", len(items))}- if got, want := items[0]["text"], "<table><tr><th >a</th><th >b</th></tr></table>"; got != want {- t.Fatalf("text = %q, want %q", got, want)+ td, ok := items[0]["table"].(*entity.TableData)+ if !ok || td == nil {+ t.Fatalf("table field missing: %#v", items[0])+ }+ if items[0]["text"] != nil {+ t.Fatalf("table item must not carry legacy HTML text: %#v", items[0])+ }+ if len(td.Rows) != 1 || len(td.Rows[0]) != 2 || td.Rows[0][0] != "a" || td.Rows[0][1] != "b" {+ t.Fatalf("table rows = %#v, want [[a b]]", td.Rows)+ }+ if td.HeaderRows != 1 {+ t.Fatalf("HeaderRows = %d, want 1", td.HeaderRows)}}diff --git a/internal/parser/parser/pdf_parser_remote_common.go b/internal/parser/parser/pdf_parser_remote_common.goindex acde5ace67..a26af71a1d 100644--- a/internal/parser/parser/pdf_parser_remote_common.go+++ b/internal/parser/parser/pdf_parser_remote_common.go@@ -3,6 +3,9 @@ package parserimport ("fmt""strings"++ "ragflow/internal/entity"+ "ragflow/internal/parser/tableutil")func pdfFileMeta(filename string, pageCount int) map[string]any {@@ -16,6 +19,23 @@ func pdfFileMeta(filename string, pageCount int) map[string]any {}}+// markdownItemText renders a single parse item to markdown. A table uses the+// structured TableData contract directly (rendered via RenderTableHTML); every+// other item uses its text. Producers emit `table` for tables and never the+// legacy HTML `text`, so a table-typed item without structured data is skipped+// rather than falling back to a raw HTML string — the contract is "tables are+// structured, not markup".+func markdownItemText(item map[string]any) string {+ if dt, _ := item["doc_type_kwd"].(string); dt == "table" {+ if td, ok := item["table"].(*entity.TableData); ok && td != nil {+ return tableutil.RenderTableHTML(td)+ }+ return ""+ }+ text, _ := item["text"].(string)+ return text+}+func pdfItemsToResult(filename string, items []map[string]any, outputFormat string, pageCount int) ParseResult {if len(items) == 0 {items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}@@ -31,7 +51,7 @@ func pdfItemsToResult(filename string, items []map[string]any, outputFormat stricase "markdown":var b strings.Builderfor _, item := range items {- text, _ := item["text"].(string)+ text := markdownItemText(item)layout, _ := item["layout"].(string)if strings.TrimSpace(text) == "" {continuediff --git a/internal/parser/parser/pdf_parser_somark.go b/internal/parser/parser/pdf_parser_somark.goindex 75dc22782b..363badbd72 100644--- a/internal/parser/parser/pdf_parser_somark.go+++ b/internal/parser/parser/pdf_parser_somark.go@@ -196,7 +196,9 @@ func soMarkBlockToItem(block map[string]any, keepHeaderFooter bool) map[string]aif content == "" {return nil}- return map[string]any{"text": content, "doc_type_kwd": pdfTableDocType(content), "layout": "table"}+ item := map[string]any{"text": content, "doc_type_kwd": pdfTableDocType(content), "layout": "table"}+ projectPDFTable(item)+ return itemcase "equation":if content == "" {return nildiff --git a/internal/parser/parser/pdf_parser_table_contract_test.go b/internal/parser/parser/pdf_parser_table_contract_test.goindex adf662418b..e6e6dcf0fe 100644--- a/internal/parser/parser/pdf_parser_table_contract_test.go+++ b/internal/parser/parser/pdf_parser_table_contract_test.go@@ -3,11 +3,15 @@ package parserimport ("strings""testing"++ "ragflow/internal/entity"+ "ragflow/internal/parser/tableutil")// Consumers of doc_type_kwd "table" parse the item text as <table> markup// (the QA chunker reads its <tr> rows), so a table item that carries other-// text loses its rows downstream.+// text loses its rows downstream. The structured TableData contract must also+// be present on every table item so the chunker no longer re-parses HTML.func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {cells := []any{map[string]any{"row": 0, "content": "What is RAGFlow?"},@@ -73,12 +77,24 @@ func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {item := items[0]docType, _ := item["doc_type_kwd"].(string)text, _ := item["text"].(string)- if docType == "table" && !strings.HasPrefix(strings.TrimSpace(text), "<table") {- t.Fatalf("doc_type_kwd table carries non-<table> text %q", text)+ if docType == "table" && text != "" {+ t.Fatalf("doc_type_kwd table must not carry legacy HTML text %q", text)}if docType != tc.wantDocType {t.Fatalf("doc_type_kwd = %q, want %q (text %q)", docType, tc.wantDocType, text)}+ if docType == "table" {+ td, ok := item["table"].(*entity.TableData)+ if !ok || td == nil {+ t.Fatalf("doc_type_kwd table item missing structured *entity.TableData: %#v", item)+ }+ if len(td.Rows) == 0 {+ t.Fatalf("structured table has no rows: %#v", td)+ }+ if got := tableutil.RenderTableText(td); !strings.Contains(got, "What is RAGFlow?") {+ t.Fatalf("structured table missing cell text %q: %#v", got, td)+ }+ }})}}diff --git a/internal/parser/parser/pdf_parser_tcadp.go b/internal/parser/parser/pdf_parser_tcadp.goindex faafe813e8..226f9bd438 100644--- a/internal/parser/parser/pdf_parser_tcadp.go+++ b/internal/parser/parser/pdf_parser_tcadp.go@@ -100,7 +100,11 @@ func tcadpAnyToItems(raw any) []map[string]any {if text == "" {return nil}- return emit(text, pdfTableDocType(text), "table")+ items := emit(text, pdfTableDocType(text), "table")+ for _, it := range items {+ projectPDFTable(it)+ }+ return itemscase "image":caption := strings.TrimSpace(stringValue(v["caption"]))if caption == "" {diff --git a/internal/parser/parser/pdf_table_no_html_text_test.go b/internal/parser/parser/pdf_table_no_html_text_test.gonew file mode 100644index 0000000000..e741e91a81--- /dev/null+++ b/internal/parser/parser/pdf_table_no_html_text_test.go@@ -0,0 +1,142 @@+package parser++import (+ "strings"+ "testing"++ "ragflow/internal/entity"+ "ragflow/internal/parser/tableutil"+)++// TestPDFTableItemsOmitHTMLText pins the structured TableData contract: a table+// item must carry `table` (parsed rows) and must NOT carry the legacy HTML+// `text`. The HTML string is a redundant, storage-costly duplicate that the+// markdown/search paths derive on demand from `table`. Reducing this catches+// any producer that regresses to emitting both.+func TestPDFTableItemsOmitHTMLText(t *testing.T) {+ const tableHTML = "<table><tr><td>What is RAGFlow?</td><td>A RAG engine.</td></tr><tr><td>Is it open source?</td><td>Yes.</td></tr></table>"+ const docxIR = `{"sections":[{"elements":[+ {"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"}]}]}]}]}+ ]}]}`++ cases := []struct {+ name string+ produce func() []map[string]any+ }{+ {"projectPDFTable (DLA)", func() []map[string]any {+ item := map[string]any{"doc_type_kwd": "table", "text": tableHTML, "page_number": 1}+ projectPDFTable(item)+ return []map[string]any{item}+ }},+ {"opendataloader html", func() []map[string]any {+ return openDataLoaderItems(map[string]any{"type": "table", "html": tableHTML})+ }},+ {"tcadp html content", func() []map[string]any {+ return tcadpAnyToItems(map[string]any{"type": "table", "content": tableHTML})+ }},+ {"somark html", func() []map[string]any {+ return []map[string]any{soMarkBlockToItem(map[string]any{"type": "table", "content": tableHTML}, false)}+ }},+ {"docx table", func() []map[string]any {+ return buildDOCXJSONSections(docxIR, newEmbeddedMediaBudget())+ }},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ items := tc.produce()+ var tableItem map[string]any+ for _, it := range items {+ if dt, _ := it["doc_type_kwd"].(string); dt == "table" {+ tableItem = it+ break+ }+ }+ if tableItem == nil {+ t.Fatalf("producer %q emitted no table item: %#v", tc.name, items)+ }+ if v, ok := tableItem["text"]; ok && v != "" {+ t.Fatalf("table item still carries legacy HTML text %q", v)+ }+ td, ok := tableItem["table"].(*entity.TableData)+ if !ok || td == nil || len(td.Rows) == 0 {+ t.Fatalf("table item missing structured TableData: %#v", tableItem)+ }+ if got := tableutil.RenderTableText(td); !strings.Contains(got, "What is RAGFlow?") && !strings.Contains(got, "cell") {+ t.Fatalf("structured table missing cell text %q: %#v", got, td)+ }+ })+ }+}++// TestPDFItemsToMarkdownRendersTableFromTableField verifies the markdown+// renderer derives table markup from `table` when the legacy HTML `text` is+// absent. This is the consumer that previously forced producers to keep `text`;+// once producers drop `text`, markdown must still embed the table. The output+// must be byte-equivalent to rendering `table` directly (stable across the+// ParseTableHTML/RenderTableHTML round-trip).+func TestPDFItemsToMarkdownRendersTableFromTableField(t *testing.T) {+ td := &entity.TableData{+ Rows: [][]string{{"What is RAGFlow?", "A RAG engine."}, {"Is it open source?", "Yes."}},+ HeaderRows: 1,+ }+ // Producer-after-cleanup shape: `table` present, `text` absent.+ item := map[string]any{"doc_type_kwd": "table", "layout": "table", "table": td}+ res := pdfItemsToResult("doc.pdf", []map[string]any{item}, "markdown", 1)+ if res.Err != nil {+ t.Fatalf("pdfItemsToResult markdown: %v", res.Err)+ }+ want := strings.TrimSpace(tableutil.RenderTableHTML(td))+ if got := strings.TrimSpace(res.Markdown); got != want {+ t.Fatalf("markdown table mismatch:\n got: %q\nwant: %q", got, want)+ }+ if !strings.Contains(res.Markdown, "What is RAGFlow?") {+ t.Fatalf("markdown lost table cell text: %q", res.Markdown)+ }+}++// TestPDFItemsToMarkdownSkipsTableWithoutStructuredData guards the+// structured-first contract: a table-typed item that carries no `table`+// (e.g. a mislabeled table region) must be skipped from markdown rather than+// falling back to a raw HTML `text`. This pins "tables are structured, not+// markup" so a future change cannot reintroduce an HTML fallback for tables.+func TestPDFItemsToMarkdownSkipsTableWithoutStructuredData(t *testing.T) {+ // doc_type_kwd "table" but no `table` field and no `text`.+ item := map[string]any{"doc_type_kwd": "table", "layout": "table"}+ res := pdfItemsToResult("doc.pdf", []map[string]any{item}, "markdown", 1)+ if res.Err != nil {+ t.Fatalf("pdfItemsToResult markdown: %v", res.Err)+ }+ if strings.TrimSpace(res.Markdown) != "" {+ t.Fatalf("markdown must skip table-typed item without structured data, got %q", res.Markdown)+ }+}++// TestProjectPDFTableKeepsTextWhenNotProjected guards the safety net: items+// that are not tables, or whose text is not <table> markup, must keep their+// `text` untouched and receive no `table` — we must never delete text for+// non-table items while stripping it from projected tables.+func TestProjectPDFTableKeepsTextWhenNotProjected(t *testing.T) {+ t.Run("non-table doc_type", func(t *testing.T) {+ item := map[string]any{"doc_type_kwd": "text", "text": "plain prose"}+ projectPDFTable(item)+ if item["text"] != "plain prose" {+ t.Fatalf("text mutated for non-table item: %#v", item)+ }+ if _, ok := item["table"]; ok {+ t.Fatalf("non-table item gained table: %#v", item)+ }+ })+ t.Run("table doc_type but non-HTML text", func(t *testing.T) {+ item := map[string]any{"doc_type_kwd": "table", "text": "not a table"}+ projectPDFTable(item)+ if item["text"] != "not a table" {+ t.Fatalf("text mutated for non-HTML table item: %#v", item)+ }+ if _, ok := item["table"]; ok {+ t.Fatalf("non-HTML table item gained table: %#v", item)+ }+ })+ t.Run("nil item", func(t *testing.T) {+ projectPDFTable(nil) // must not panic+ })+}diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.goindex b0a7ac0f4b..98b7b4ee29 100644--- a/internal/parser/parser/xls_parser_test.go+++ b/internal/parser/parser/xls_parser_test.go@@ -140,7 +140,7 @@ func TestXLSParser_PreservesFormulaErrors(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("res.JSON len = %d, want 1", len(res.JSON))}- text, _ := res.JSON[0]["text"].(string)+ text := tableItemHTML(t, res.JSON[0])if !strings.Contains(text, "<tr><td>#DIV/0!</td></tr>") {t.Fatalf("table markup = %q, want formula error row", text)}@@ -174,7 +174,7 @@ func TestXLSParser_GenuineBIFF8SpreadsheetJSONOutput(t *testing.T) {}first := res.JSON[0]- text, _ := first["text"].(string)+ text := tableItemHTML(t, first)if !strings.Contains(text, "<caption>Inventory</caption>") ||!strings.Contains(text, "<tr><th>Name</th><th>Count</th></tr>") ||!strings.Contains(text, "<tr><td>Apples</td><td>3</td></tr>") {@@ -195,7 +195,7 @@ func TestXLSParser_GenuineBIFF8SpreadsheetJSONOutput(t *testing.T) {}second := res.JSON[1]- secondText, _ := second["text"].(string)+ secondText := tableItemHTML(t, second)if !strings.Contains(secondText, "<caption>Notes</caption>") ||!strings.Contains(secondText, "<tr><td>Legacy</td><td>BIFF8</td></tr>") {t.Fatalf("second table markup = %q, want second BIFF8 sheet data", secondText)@@ -228,7 +228,7 @@ func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {if item["ck_type"] != "table" {t.Errorf("table item ck_type = %v, want table", item["ck_type"])}- text, _ := item["text"].(string)+ text := tableItemHTML(t, item)if !strings.Contains(text, "<tr><th>Header1</th><th>Header2</th></tr>") ||!strings.Contains(text, "<tr><td>Val1</td><td>Val2</td></tr>") {t.Errorf("table markup = %q, want captioned header and data row", text)diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.goindex 9374087ac1..e07b2875d7 100644--- a/internal/parser/parser/xlsx_parse_method_test.go+++ b/internal/parser/parser/xlsx_parse_method_test.go@@ -80,7 +80,7 @@ func TestXLSXParser_DeepDocParseMethod(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON item count = %d, want header-only result", len(res.JSON))}- text, _ := res.JSON[0]["text"].(string)+ text := tableItemHTML(t, res.JSON[0])if !strings.Contains(text, tc.cellValue) {t.Fatalf("JSON = %#v, want it to contain cell content %q", res.JSON, tc.cellValue)}diff --git a/internal/parser/tableutil/table_html.go b/internal/parser/tableutil/table_html.gonew file mode 100644index 0000000000..1ac54b5726--- /dev/null+++ b/internal/parser/tableutil/table_html.go@@ -0,0 +1,174 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++// Package tableutil converts between table markup and the structured+// entity.TableData contract. It intentionally mirrors the legacy chunker+// extraction in internal/ingestion/component/chunker/html_rows.go so that the+// cells it produces are byte-for-byte what consumers used to read out of the+// HTML <table> text.+package tableutil++import (+ "regexp"+ "strings"++ "golang.org/x/net/html"+ "ragflow/internal/entity"+)++// ParseTableHTML extracts a structured TableData from table markup. It mirrors+// the legacy chunker behaviour (html_rows.go tableRowsWithHeader): each <tr>+// becomes one row, each direct <td>/<th> one cell, a row carrying a <th> cell+// is a header row, <br> becomes a newline, inert elements+// (script/style/noscript/template) are skipped, and a nested table's text is+// folded into the enclosing cell (the nested <tr> does NOT become a row).+func ParseTableHTML(s string) (*entity.TableData, error) {+ lower := strings.ToLower(s)+ if strings.Contains(lower, "<tr") && !strings.Contains(lower, "<table") {+ s = "<table>" + s + "</table>"+ }+ doc, err := html.Parse(strings.NewReader(s))+ if err != nil {+ return nil, err+ }+ var rows [][]string+ var headerFlags []bool+ var caption string+ var walk func(*html.Node)+ walk = func(n *html.Node) {+ if n.Type == html.ElementNode && isInertElement(n.Data) {+ return+ }+ if isHTMLElement(n, "caption") {+ caption = cellText(n)+ return+ }+ if isHTMLElement(n, "tr") {+ var cells []string+ anyTh := false+ for c := n.FirstChild; c != nil; c = c.NextSibling {+ if isHTMLElement(c, "td") || isHTMLElement(c, "th") {+ if isHTMLElement(c, "th") {+ anyTh = true+ }+ cells = append(cells, cellText(c))+ }+ }+ rows = append(rows, cells)+ headerFlags = append(headerFlags, anyTh)+ return+ }+ for c := n.FirstChild; c != nil; c = c.NextSibling {+ walk(c)+ }+ }+ walk(doc)+ headerCount := 0+ for headerCount < len(headerFlags) && headerFlags[headerCount] {+ headerCount+++ }+ if len(rows) > 0 && headerCount == 0 {+ headerCount = 1+ }+ return &entity.TableData{Rows: rows, HeaderRows: headerCount, Caption: caption}, nil+}++// RenderTableHTML emits a deterministic <table> markup for a TableData. It is+// used only as a test oracle / round-trip helper and is NOT part of the+// persisted contract (producers emit TableData, not HTML). The emitted shape+// matches the legacy spreadsheet producer's renderSpreadsheetTable exactly+// (including the inter-row newlines) so that RenderTableText reproduces the+// legacy htmlTableRE-stripped text byte for byte.+func RenderTableHTML(td *entity.TableData) string {+ var b strings.Builder+ b.WriteString("<table>")+ if td.Caption != "" {+ b.WriteString("<caption>")+ b.WriteString(html.EscapeString(td.Caption))+ b.WriteString("</caption>")+ }+ b.WriteString("\n")+ for i, row := range td.Rows {+ b.WriteString("<tr>")+ isHeader := i < td.HeaderRows+ for _, cell := range row {+ tag := "td"+ if isHeader {+ tag = "th"+ }+ b.WriteString("<")+ b.WriteString(tag)+ b.WriteString(">")+ b.WriteString(html.EscapeString(cell))+ b.WriteString("</")+ b.WriteString(tag)+ b.WriteString(">")+ }+ b.WriteString("</tr>")+ b.WriteString("\n")+ }+ b.WriteString("</table>")+ b.WriteString("\n")+ return b.String()+}++// tableTagRE mirrors the legacy tokenizer stripping pattern+// (htmlTableRE in internal/ingestion/component/tokenizer.go) so that the+// plaintext a chunker writes for an embed-only path matches the legacy+// tag-stripped output exactly.+var tableTagRE = regexp.MustCompile(`</?(table|td|caption|tr|th)( [^<>]{0,12})?>`)++// RenderTableText renders a TableData to plain text with no markup tags, for+// the rare embed path that receives a table without a downstream TableChunker.+// It reproduces the legacy htmlTableRE stripping of RenderTableHTML(td).+func RenderTableText(td *entity.TableData) string {+ h := RenderTableHTML(td)+ h = tableTagRE.ReplaceAllString(h, " ")+ return strings.TrimSpace(h)+}++// cellText returns the visible text of a table cell: text nodes contribute+// verbatim, <br> becomes a newline, inert elements are skipped, and other+// elements are descended. The result is TrimSpace'd.+func cellText(cell *html.Node) string {+ var sb strings.Builder+ var walk func(*html.Node)+ walk = func(n *html.Node) {+ switch {+ case n.Type == html.TextNode:+ sb.WriteString(n.Data)+ case isHTMLElement(n, "br"):+ sb.WriteByte('\n')+ case n.Type == html.ElementNode && isInertElement(n.Data):+ return+ }+ for c := n.FirstChild; c != nil; c = c.NextSibling {+ walk(c)+ }+ }+ walk(cell)+ return strings.TrimSpace(sb.String())+}++func isHTMLElement(n *html.Node, tag string) bool {+ return n.Type == html.ElementNode && n.Namespace == "" && n.Data == tag+}++func isInertElement(tag string) bool {+ switch tag {+ case "script", "style", "noscript", "template":+ return true+ }+ return false+}diff --git a/internal/parser/tableutil/table_html_test.go b/internal/parser/tableutil/table_html_test.gonew file mode 100644index 0000000000..abfea73ad6--- /dev/null+++ b/internal/parser/tableutil/table_html_test.go@@ -0,0 +1,187 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package tableutil++import (+ "strings"+ "testing"++ "ragflow/internal/entity"+)++// mustParse is a test helper that fails the test on parse error.+func mustParse(t *testing.T, html string) *entity.TableData {+ t.Helper()+ td, err := ParseTableHTML(html)+ if err != nil {+ t.Fatalf("ParseTableHTML(%q) error = %v", html, err)+ }+ return td+}++func TestParseTableHTMLHappyPath(t *testing.T) {+ html := `<table><tr><th>Name</th><th>Age</th></tr>` ++ `<tr><td>Alice</td><td>30</td></tr>` ++ `<tr><td>Bob</td><td>25</td></tr></table>`+ td := mustParse(t, html)+ if td.HeaderRows != 1 {+ t.Fatalf("HeaderRows = %d, want 1", td.HeaderRows)+ }+ want := [][]string{+ {"Name", "Age"},+ {"Alice", "30"},+ {"Bob", "25"},+ }+ assertRowsEqual(t, td.Rows, want)+}++func TestParseTableHTMLMultipleHeaderRows(t *testing.T) {+ html := `<table><tr><th>H1</th></tr><tr><th>H2</th></tr>` ++ `<tr><td>a</td></tr></table>`+ td := mustParse(t, html)+ if td.HeaderRows != 2 {+ t.Fatalf("HeaderRows = %d, want 2", td.HeaderRows)+ }+}++func TestParseTableHTMLNoThTreatsFirstRowAsHeader(t *testing.T) {+ html := `<table><tr><td>Name</td><td>Age</td></tr>` ++ `<tr><td>Alice</td><td>30</td></tr></table>`+ td := mustParse(t, html)+ if td.HeaderRows != 1 {+ t.Fatalf("HeaderRows = %d, want 1 (no <th> => first row is header)", td.HeaderRows)+ }+}++func TestParseTableHTMLEmpty(t *testing.T) {+ td := mustParse(t, `<table></table>`)+ if len(td.Rows) != 0 {+ t.Fatalf("Rows = %v, want empty", td.Rows)+ }+ if td.HeaderRows != 0 {+ t.Fatalf("HeaderRows = %d, want 0", td.HeaderRows)+ }+}++func TestParseTableHTMLTrimsCellWhitespace(t *testing.T) {+ html := `<table><tr><td> spaced </td></tr></table>`+ td := mustParse(t, html)+ if got := td.Rows[0][0]; got != "spaced" {+ t.Fatalf("cell = %q, want %q", got, "spaced")+ }+}++func TestParseTableHTMLBrBecomesNewline(t *testing.T) {+ html := `<table><tr><td>line1<br>line2</td></tr></table>`+ td := mustParse(t, html)+ if got := td.Rows[0][0]; got != "line1\nline2" {+ t.Fatalf("cell = %q, want %q", got, "line1\nline2")+ }+}++func TestParseTableHTMLInertElementsIgnored(t *testing.T) {+ // script/style/template content must not appear in any cell.+ html := `<table><tr><td>keep<script>alert(1)</script>middle` ++ `<style>.x{}</style>end</td></tr></table>`+ td := mustParse(t, html)+ got := td.Rows[0][0]+ if strings.Contains(got, "alert") || strings.Contains(got, ".x{}") {+ t.Fatalf("inert content leaked into cell: %q", got)+ }+ if !strings.Contains(got, "keep") || !strings.Contains(got, "middle") || !strings.Contains(got, "end") {+ t.Fatalf("visible text lost: %q", got)+ }+}++func TestParseTableHTMLNestedTableFoldedIntoCell(t *testing.T) {+ html := `<table><tr><td>outer<table><tr><td>inner</td></tr></table></td></tr></table>`+ td := mustParse(t, html)+ // Outer <tr> yields exactly one row with one cell; the nested <tr> must+ // NOT become a separate row.+ if len(td.Rows) != 1 {+ t.Fatalf("len(Rows) = %d, want 1 (nested tr not a row)", len(td.Rows))+ }+ got := td.Rows[0][0]+ if !strings.Contains(got, "outer") || !strings.Contains(got, "inner") {+ t.Fatalf("nested table text not folded: %q", got)+ }+ if strings.Contains(got, "<table") {+ t.Fatalf("nested markup not stripped: %q", got)+ }+}++func TestParseTableHTMLBareTrWrapped(t *testing.T) {+ // Legacy behaviour: a bare <tr> fragment without <table> still parses.+ html := `<tr><td>a</td><td>b</td></tr>`+ td := mustParse(t, html)+ assertRowsEqual(t, td.Rows, [][]string{{"a", "b"}})+}++func TestParseTableHTMLColspanIgnoredLikeLegacy(t *testing.T) {+ // Legacy tableRowsWithHeader ignores colspan/rowspan and just collects+ // each <td>/<th> as one cell; a spanned cell keeps its own text.+ html := `<table><tr><td colspan="2">wide</td><td>narrow</td></tr></table>`+ td := mustParse(t, html)+ assertRowsEqual(t, td.Rows, [][]string{{"wide", "narrow"}})+}++func TestRenderTableHTMLRoundTrip(t *testing.T) {+ td := &entity.TableData{+ Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}},+ HeaderRows: 1,+ Caption: "People",+ }+ rendered := RenderTableHTML(td)+ back := mustParse(t, rendered)+ assertRowsEqual(t, back.Rows, td.Rows)+ if back.HeaderRows != td.HeaderRows {+ t.Fatalf("HeaderRows round-trip = %d, want %d", back.HeaderRows, td.HeaderRows)+ }+ if back.Caption != td.Caption {+ t.Fatalf("Caption round-trip = %q, want %q", back.Caption, td.Caption)+ }+}++func TestRenderTableTextHasNoTags(t *testing.T) {+ td := &entity.TableData{+ Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}},+ HeaderRows: 1,+ Caption: "People",+ }+ got := RenderTableText(td)+ if strings.ContainsAny(got, "<>") {+ t.Fatalf("RenderTableText contains tags: %q", got)+ }+ if !strings.Contains(got, "Name") || !strings.Contains(got, "Alice") {+ t.Fatalf("RenderTableText lost cell text: %q", got)+ }+}++func assertRowsEqual(t *testing.T, got, want [][]string) {+ t.Helper()+ if len(got) != len(want) {+ t.Fatalf("row count = %d, want %d\n got=%v\nwant=%v", len(got), len(want), got, want)+ }+ for i := range want {+ if len(got[i]) != len(want[i]) {+ t.Fatalf("row %d cell count = %d, want %d\n got=%v\nwant=%v", i, len(got[i]), len(want[i]), got[i], want[i])+ }+ for j := range want[i] {+ if got[i][j] != want[i][j] {+ t.Fatalf("cell [%d][%d] = %q, want %q", i, j, got[i][j], want[i][j])+ }+ }+ }+}diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.goindex 551c968316..faba6b6769 100644--- a/internal/service/file/file_content.go+++ b/internal/service/file/file_content.go@@ -8,6 +8,7 @@ import ("ragflow/internal/dao""ragflow/internal/entity""ragflow/internal/parser/parser"+ "ragflow/internal/parser/tableutil""ragflow/internal/storage""ragflow/internal/utility""strings"@@ -239,6 +240,10 @@ func parseResultText(res parser.ParseResult) (string, error) {parts = append(parts, text)continue}+ if td, ok := item["table"].(*entity.TableData); ok && td != nil {+ parts = append(parts, tableutil.RenderTableHTML(td))+ continue+ }raw, err := json.Marshal(item)if err != nil {return "", err