Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(parser): emit structured TableData instead of HTML for tables (#20741)

xugangqiang97b648fd32ff
diff --git a/internal/entity/table.go b/internal/entity/table.go
new file mode 100644
index 0000000000..328038480d
--- /dev/null
+++ b/internal/entity/table.go
@@ -0,0 +1,55 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package entity
+
+// TableData is the canonical structured table contract shared by every parser
+// (office / markdown / html / pdf) and consumed by the ingestion chunkers.
+// Coordinates and merge geometry are optional so coordinate-free tables
+// (spreadsheet / markdown / html) and coordinate-bearing tables (pdf) share
+// one type. The cell text in Rows is already normalized the same way the
+// legacy chunker normalized HTML <td>/<th> text (TrimSpace, <br> -> newline,
+// inert markup skipped, nested table text folded), so consumers can render
+// chunk text directly from Rows without re-parsing any markup.
+type TableData struct {
+ // Rows holds the cell text, row-major. rows[0:HeaderRows] are header rows.
+ Rows [][]string `json:"rows"`
+ // HeaderRows is the number of leading header rows. The legacy rule is:
+ // rows carrying a <th> cell count as headers; when no row uses <th> at
+ // all the first row is treated as the header (HeaderRows == 1).
+ HeaderRows int `json:"header_rows"`
+ // Caption is the table caption (pdf emits it; office uses the sheet name).
+ Caption string `json:"caption,omitempty"`
+ // Spans records merged cells. Recovered information: pdf already emits
+ // colspan/rowspan in its HTML and office loses merged-cell geometry today;
+ // consumers do not use Spans yet (preserved for a later follow-up).
+ Spans []CellSpan `json:"spans,omitempty"`
+ // Page is the pdf 0-based page number, used for layout retrieval.
+ Page int `json:"page,omitempty"`
+ // Cells carries optional per-cell coordinates (pdf only); currently lost
+ // in the legacy HTML serialization and preserved here for later use.
+ Cells []CellCoord `json:"cells,omitempty"`
+}
+
+// CellSpan identifies a merged cell: the top-left anchor at (Row, Col) spans
+// RowSpan rows and ColSpan columns.
+type CellSpan struct {
+ Row, Col, RowSpan, ColSpan int
+}
+
+// CellCoord is the bounding box of a single cell in pdf page coordinates.
+type CellCoord struct {
+ Row, Col int
+ X0, Y0, X1, Y1 float64
+}
diff --git a/internal/ingestion/component/chunker/coverage_blind_spots_test.go b/internal/ingestion/component/chunker/coverage_blind_spots_test.go
index c1c9d4023f..0841533af7 100644
--- a/internal/ingestion/component/chunker/coverage_blind_spots_test.go
+++ b/internal/ingestion/component/chunker/coverage_blind_spots_test.go
@@ -175,7 +175,12 @@ func TestGeneralChunkerKeepsHeaderOnlySheetsSeparate(t *testing.T) {
if len(chunks) != 2 {
t.Fatalf("chunks = %#v, want one header-only chunk per sheet", chunks)
}
- if chunks[0]["text"] != segOne["text"] || chunks[1]["text"] != segTwo["text"] {
- t.Fatalf("header-only chunks = %#v", chunks)
+ // Header-only segments carry no data rows; the chunk keeps a plain-text
+ // body derived from the structured TableData (caption + header cells).
+ if got, _ := chunks[0]["text"].(string); got == "" || !strings.Contains(got, "Name") {
+ t.Fatalf("header-only chunk 0 lost content: %#v", chunks[0])
+ }
+ if got, _ := chunks[1]["text"].(string); got == "" || !strings.Contains(got, "Amount") {
+ t.Fatalf("header-only chunk 1 lost content: %#v", chunks[1])
}
}
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index f533038218..b53a204cb0 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -39,6 +39,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/chunk"
+ "ragflow/internal/parser/tableutil"
)
const ComponentNameGeneralChunker = "GeneralChunker"
@@ -500,10 +501,14 @@ func generalContextSourceText(doc schema.ChunkDoc) (string, bool) {
case "text":
return doc.Text, true
case "table":
- if !hasSpreadsheetIdentity(doc) || !isTableStrictHTML(doc.Text) {
+ if !hasSpreadsheetIdentity(doc) {
return "", false
}
- rows, headerCount := tableRowsWithHeader(doc.Text)
+ td := tableDataOf(doc)
+ if td == nil {
+ return "", false
+ }
+ rows, headerCount := td.Rows, td.HeaderRows
lines := make([]string, 0, len(rows)-headerCount)
for _, row := range rows[headerCount:] {
cells := make([]string, 0, len(row))
@@ -897,7 +902,7 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream
attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
chunks := make([]schema.ChunkDoc, 0, len(units))
for _, unit := range units {
- if itemDocType(unit) == "table" && isTableStrictHTML(unit.Text) {
+ if itemDocType(unit) == "table" && tableDataOf(unit) != nil {
chunks = append(chunks, c.splitSpreadsheetTable(unit)...)
continue
}
@@ -922,7 +927,11 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream
// and deferred the header; a segment now passes through whole unless the
// budget forces a cut, and the header repeats in every part.
func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []schema.ChunkDoc {
- parts, ranges, headerRows := splitLargeHTMLTable(unit.Text, c.param.ChunkTokenSize, tokenizeStr)
+ td := tableDataOf(unit)
+ if td == nil {
+ return []schema.ChunkDoc{cloneChunkDoc(unit)}
+ }
+ parts, ranges, headerRows := splitLargeTable(td, c.param.ChunkTokenSize, tokenizeStr)
if ranges == nil {
return []schema.ChunkDoc{cloneChunkDoc(unit)}
}
@@ -934,9 +943,8 @@ func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []
matrix = nil
}
}
- // The matrix carries one five-field tuple per <tr>; the splitter cut those
- // same rows, so its last range end is the total data-row count — no second
- // full parse of the markup is needed to compare lengths.
+ // The matrix carries one five-field tuple per row; the splitter cut those
+ // same rows, so its last range end is the total data-row count.
aligned := false
if len(ranges) > 0 {
if expected := headerRows + ranges[len(ranges)-1][1]; expected > 0 && len(matrix) == expected {
@@ -952,7 +960,11 @@ func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []
out := make([]schema.ChunkDoc, 0, len(parts))
for i, part := range parts {
piece := cloneChunkDoc(unit)
- piece.Text = part
+ // Text carries a plain-text rendering (no markup) for the rare embed
+ // path without a downstream TableChunker; the structured TableData is
+ // the authoritative content for chunkers that read it.
+ piece.Text = tableutil.RenderTableText(&part)
+ piece.TableData = &part
piece.Positions = nil
lo := headerRows + ranges[i][0]
hi := headerRows + ranges[i][1]
diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.go
index c384ab8d59..daeea3f92a 100644
--- a/internal/ingestion/component/chunker/general_test.go
+++ b/internal/ingestion/component/chunker/general_test.go
@@ -34,30 +34,25 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
+ "ragflow/internal/entity"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/ingestion/task/indexdoc"
+ "ragflow/internal/parser/tableutil"
)
// spreadsheetSegmentItem builds one wire-format spreadsheet table segment:
// the markup the parser renders (captioned <table>, <th> header, <td> rows)
// plus the row-aligned positions matrix (one tuple per <tr>, header first).
// firstRow is the sheet row of the first DATA row; the header tuple covers
-// firstRow-1.
+// firstRow-1. It emits the structured TableData contract a post-migration
+// producer writes (caption = sheet name, header rows, data rows), not HTML
+// markup.
func spreadsheetSegmentItem(sheet string, header []string, rows [][]string, sheetIndex, firstRow int) map[string]any {
- var markup strings.Builder
- markup.WriteString("<table><caption>" + sheet + "</caption>\n<tr>")
- for _, cell := range header {
- markup.WriteString("<th>" + cell + "</th>")
- }
- markup.WriteString("</tr>\n")
- for _, row := range rows {
- markup.WriteString("<tr>")
- for _, cell := range row {
- markup.WriteString("<td>" + cell + "</td>")
- }
- markup.WriteString("</tr>\n")
+ allRows := make([][]string, 0, len(rows)+1)
+ if len(header) > 0 {
+ allRows = append(allRows, header)
}
- markup.WriteString("</table>\n")
+ allRows = append(allRows, rows...)
tuple := func(row int) []float64 {
return []float64{float64(sheetIndex), float64(row), float64(row), 1, float64(len(header))}
@@ -66,8 +61,16 @@ func spreadsheetSegmentItem(sheet string, header []string, rows [][]string, shee
for i := range rows {
matrix = append(matrix, tuple(firstRow+i))
}
+ td := &entity.TableData{Rows: allRows, HeaderRows: 1, Caption: sheet}
return map[string]any{
- "text": markup.String(),
+ "table": map[string]any{
+ "rows": allRows,
+ "header_rows": 1,
+ "caption": sheet,
+ },
+ // Mirror what the chunker derives from TableData so text-bound
+ // assertions (e.g. TokenChunker boundary tests) stay valid.
+ "text": tableutil.RenderTableText(td),
"doc_type_kwd": "table",
"ck_type": "table",
"sheet_index": sheetIndex,
@@ -983,7 +986,8 @@ func TestGeneralChunkerSpreadsheetHeaderOnlyPreservesHeaderChunk(t *testing.T) {
if len(chunks) != 1 {
t.Fatalf("chunks = %#v, want one header-only chunk", chunks)
}
- if chunks[0]["text"] != item["text"] || chunks[0]["ck_type"] != "table" {
+ text, _ := chunks[0]["text"].(string)
+ if chunks[0]["ck_type"] != "table" || text == "" || !strings.Contains(text, "Name") || !strings.Contains(text, "Amount") {
t.Fatalf("header-only chunk = %#v", chunks[0])
}
}
@@ -1074,9 +1078,19 @@ func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testing
}
covered := 0
for i, chunk := range chunks {
- text, _ := chunk["text"].(string)
- if !strings.Contains(text, "<th>Name</th>") {
- t.Errorf("chunk %d lost the replicated header", i)
+ // New contract: Text is the plain rendering (no markup); the
+ // replicated header lives in the structured TableData, and positions
+ // are sliced to this sub-table when the matrix is row-aligned.
+ tbl, ok := chunk["table"].(map[string]any)
+ if !ok {
+ t.Fatalf("chunk %d missing structured TableData", i)
+ }
+ tblRows, _ := tbl["rows"].([]any)
+ if len(tblRows) < 2 {
+ t.Fatalf("chunk %d has too few rows: %v", i, tblRows)
+ }
+ if !strings.Contains(chunk["text"].(string), "Name") {
+ t.Errorf("chunk %d text lost the header cell: %q", i, chunk["text"])
}
var matrix [][]float64
raw, _ := json.Marshal(chunk["positions"])
@@ -1091,11 +1105,11 @@ func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testing
if matrix[0][1] != 1 {
t.Errorf("chunk %d header tuple = %v, want sheet 1 row 1", i, matrix[0])
}
- trCount := strings.Count(text, "<tr>")
- if trCount != len(matrix) {
- t.Errorf("chunk %d has %d rows but %d position tuples", i, trCount, len(matrix))
+ dataRows := len(tblRows) - 1
+ if dataRows != len(matrix)-1 {
+ t.Errorf("chunk %d data rows %d != position tuples %d", i, dataRows, len(matrix)-1)
}
- covered += len(matrix) - 1
+ covered += dataRows
}
if covered != len(rows) {
t.Errorf("sub-tables cover %d data rows, want %d", covered, len(rows))
diff --git a/internal/ingestion/component/chunker/html_rows.go b/internal/ingestion/component/chunker/html_rows.go
deleted file mode 100644
index a1874deac0..0000000000
--- a/internal/ingestion/component/chunker/html_rows.go
+++ /dev/null
@@ -1,168 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package chunker
-
-import (
- "strings"
-
- "golang.org/x/net/html"
-)
-
-// ---------------------------------------------------------------------------
-// HTML table row extraction (shared by the Table, QA and General chunkers)
-// ---------------------------------------------------------------------------
-
-// tableRows returns every row's cell text, header rows included.
-func tableRows(htmlStr string) [][]string {
- rows, _ := tableRowsWithHeader(htmlStr)
- return rows
-}
-
-// tableRowsWithHeader returns every row's cell text in document order, plus
-// the number of leading header rows. A row counts as a header row when any of
-// its cells is a <th>. When no row uses <th> at all the first row is treated
-// as the header — the convention spreadsheet and markup tables are read with.
-//
-// The markup is parsed into a tree rather than matched with a regex because
-// this input is not guaranteed to be well formed: seven parsers render table
-// items (xlsx, csv, docx, html, pdf, …) and some of that markup originates
-// from user-supplied documents. A tree also settles the cases a tag-level
-// scan gets wrong: a nested <table> no longer terminates its enclosing row
-// early — that row keeps its own cells, with the nested table's text folded
-// into the cell holding it — and a row or cell missing its closing tag is
-// recovered rather than dropped.
-func tableRowsWithHeader(htmlStr string) (rows [][]string, headerCount int) {
- // A <tr> outside a <table> is discarded by the HTML5 "in body" insertion
- // mode, so a bare row fragment would yield nothing. Give the parser the
- // table context it needs instead of dropping the rows silently.
- lower := strings.ToLower(htmlStr)
- if strings.Contains(lower, "<tr") && !strings.Contains(lower, "<table") {
- htmlStr = "<table>" + htmlStr + "</table>"
- }
- doc, err := html.Parse(strings.NewReader(htmlStr))
- if err != nil {
- return nil, 0
- }
- var headerFlags []bool
- var walk func(*html.Node)
- walk = func(n *html.Node) {
- // An inert subtree is parsed but never rendered. <template> puts its
- // content straight into the ordinary child list (the parser has no
- // separate template-contents field), so without this its rows would
- // be read as rows of the enclosing table.
- if n.Type == html.ElementNode && isInertElement(n.Data) {
- return
- }
- if isHTMLElement(n, "tr") {
- var cells []string
- anyTh := false
- for c := n.FirstChild; c != nil; c = c.NextSibling {
- if isHTMLElement(c, "td") || isHTMLElement(c, "th") {
- if isHTMLElement(c, "th") {
- anyTh = true
- }
- cells = append(cells, cellText(c))
- }
- }
- // Return without descending: the cells above already collected
- // the nested table's text, so its rows must not be reported a
- // second time as rows of the enclosing table.
- rows = append(rows, cells)
- headerFlags = append(headerFlags, anyTh)
- return
- }
- for c := n.FirstChild; c != nil; c = c.NextSibling {
- walk(c)
- }
- }
- walk(doc)
- for len(headerFlags) > headerCount && headerFlags[headerCount] {
- headerCount++
- }
- if len(rows) > 0 && headerCount == 0 {
- headerCount = 1
- }
- return rows, headerCount
-}
-
-// isTableStrictHTML reports whether block text is an outer <table> element
-// (the inlined GFM/HTML table). Only such blocks are emitted as structured
-// table items; other raw HTML (e.g. <div>, <style>) is plain text. The
-// spreadsheet wire is the markup this reports on (see the parser's
-// renderSpreadsheetTable), so callers that slice a positions matrix use
-// this strict form rather than the candidate filter.
-func isTableStrictHTML(s string) bool {
- return strings.HasPrefix(strings.TrimSpace(strings.ToLower(s)), "<table")
-}
-
-// isTableHTML is the cheap candidate filter for table-vs-prose routing: text
-// that opens an outer <table> element, or a bare-row fragment holding <tr>.
-// It only narrows the candidates — the caller decides on tableRows' result —
-// so a block that merely opens with "<table" text and holds no row stays on
-// the prose path instead of silently pairing nothing.
-func isTableHTML(s string) bool {
- lower := strings.ToLower(s)
- return strings.HasPrefix(strings.TrimSpace(lower), "<table") || strings.Contains(lower, "<tr")
-}
-
-// cellText returns the visible text of a table cell. The parser hands text
-// nodes over already unescaped, nested markup contributes its text without
-// its tags (a nested table's cells are concatenated, not separated), and a
-// <br> becomes a newline instead of silently gluing the two halves of the
-// cell together.
-func cellText(cell *html.Node) string {
- var sb strings.Builder
- var walk func(*html.Node)
- walk = func(n *html.Node) {
- switch {
- case n.Type == html.TextNode:
- sb.WriteString(n.Data)
- case isHTMLElement(n, "br"):
- sb.WriteByte('\n')
- case n.Type == html.ElementNode && isInertElement(n.Data):
- // Stop here rather than descending: the content is parsed but
- // never rendered, so it is not text a reader of the cell sees.
- return
- }
- for c := n.FirstChild; c != nil; c = c.NextSibling {
- walk(c)
- }
- }
- walk(cell)
- return strings.TrimSpace(sb.String())
-}
-
-// isHTMLElement reports whether n is an element with the given tag name in
-// the HTML namespace. Foreign content reuses HTML tag names for unrelated
-// elements — an <svg><tr> is not a table row — so matching on the tag name
-// alone would read markup that carries no table semantics.
-func isHTMLElement(n *html.Node, tag string) bool {
- return n.Type == html.ElementNode && n.Namespace == "" && n.Data == tag
-}
-
-// isInertElement reports whether an element's content is inert — parsed, but
-// never rendered as visible text. An inline <script> or <style> inside a
-// table cell of a user-supplied document would otherwise be embedded into the
-// Q&A pair as if it were part of the sentence, and a <template>'s placeholder
-// rows would be read as real ones.
-func isInertElement(tag string) bool {
- switch tag {
- case "script", "style", "noscript", "template":
- return true
- }
- return false
-}
diff --git a/internal/ingestion/component/chunker/html_rows_test.go b/internal/ingestion/component/chunker/html_rows_test.go
deleted file mode 100644
index 76690052bb..0000000000
--- a/internal/ingestion/component/chunker/html_rows_test.go
+++ /dev/null
@@ -1,70 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package chunker
-
-import "testing"
-
-// TestIsTableStrictHTML pins the predicate's prefix semantics: it reports
-// whether the block opens with the "<table" text, exactly as the markdown
-// parser has always classified its HTML blocks. It is deliberately loose (a
-// "<tableau>" token passes), which is why routing never treats it as the final
-// word — the row walker decides.
-func TestIsTableStrictHTML(t *testing.T) {
- cases := []struct {
- text string
- want bool
- }{
- {"<table>", true},
- {" <TABLE class=\"x\">", true},
- {"<table-x>", true}, // historical prefix semantics: "<table" prefix only
- {"<tableau>", true},
- {"<div><table>", false},
- {"plain text", false},
- {"", false},
- }
- for _, tc := range cases {
- if got := isTableStrictHTML(tc.text); got != tc.want {
- t.Errorf("isTableStrictHTML(%q) = %v, want %v", tc.text, got, tc.want)
- }
- }
-}
-
-// TestIsTableHTMLCoversWhatTheWalkerReads guards the routing invariant: the
-// candidate filter must never deny a payload the row walker can read, or a
-// readable table would silently fall through to the prose extractor.
-func TestIsTableHTMLCoversWhatTheWalkerReads(t *testing.T) {
- texts := []string{
- "<table><tr><th>q</th><th>a</th></tr></table>",
- "<table><caption>c</caption>\n<tr><td>q</td><td>a</td></tr>\n</table>\n",
- "<tr><td>q</td><td>a</td></tr>", // bare fragment
- "<tr><td>q</td><td>a</td></tr><tr><td>q2</td><td>a2</td></tr>", // bare rows
- "<tbody><tr><td>q</td><td>a</td></tr></tbody>", // rows wrapped in tbody
- "<table><tr><td>q<table><tr><td>inner</td></tr></table></td></tr></table>", // nested
- "<table><template><tr><td>x</td></tr></template></table>", // template rows are inert
- "<table></table>", // nothing to read
- "plain text",
- "",
- }
- for _, text := range texts {
- if len(tableRows(text)) == 0 {
- continue
- }
- if !isTableHTML(text) {
- t.Errorf("tableRows can read %q but isTableHTML denies it", text)
- }
- }
-}
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go
index 9e9d36d785..5ef0f1a953 100644
--- a/internal/ingestion/component/chunker/qa.go
+++ b/internal/ingestion/component/chunker/qa.go
@@ -42,6 +42,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/tableutil"
"ragflow/internal/tokenizer"
)
@@ -403,8 +404,11 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {
strictCSV := strings.EqualFold(fileType, "csv")
for _, item := range items {
var tmp []qaPair
+ td := tableDataOf(item)
txt, _ := itemText(item)
- if txt == "" {
+ // A structured table with no markup text still carries rows; only a
+ // truly empty item (no text and no TableData) is skipped.
+ if txt == "" && td == nil {
continue
}
// Route on what the row walker can read, not on the type label.
@@ -423,8 +427,8 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {
// fall through to the text extractor, so documents from before this
// wire must be re-parsed rather than re-chunked.
var rows [][]string
- if isTableHTML(txt) {
- rows = tableRows(txt)
+ if td != nil {
+ rows = td.Rows
}
if len(rows) > 0 {
tmp = qaPairsFromRows(rows, strictCSV, item.Positions, item.SheetIndex != nil)
@@ -454,7 +458,11 @@ func extractQATable(htmlStr string, strictPairs bool) []qaPair {
if htmlStr == "" {
return nil
}
- return qaPairsFromRows(tableRows(htmlStr), strictPairs, nil, false)
+ td, err := tableutil.ParseTableHTML(htmlStr)
+ if err != nil || td == nil {
+ return nil
+ }
+ return qaPairsFromRows(td.Rows, strictPairs, nil, false)
}
// qaPairsFromRows builds the pairs of one table: the first two non-empty
diff --git a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
index becbdc7934..70b139eca8 100644
--- a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
+++ b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
@@ -72,11 +72,10 @@ func TestCSVJSONHTMLTableUsesStrictContinuationRules(t *testing.T) {
"name": "qa.csv",
"file_type": "csv",
"output_format": "json",
- "json": []map[string]any{{
- "doc_type_kwd": "table",
- "text": "<table><tr><td>q1</td><td>a1</td></tr>" +
- "<tr><td>q2</td><td>a2</td><td>trailing</td></tr></table>",
- }},
+ "json": []map[string]any{
+ tableItem(t, "<table><tr><td>q1</td><td>a1</td></tr>"+
+ "<tr><td>q2</td><td>a2</td><td>trailing</td></tr></table>"),
+ },
})
if err != nil {
t.Fatal(err)
diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.go
index 468edcf9c3..9a1e8785a4 100644
--- a/internal/ingestion/component/chunker/qa_test.go
+++ b/internal/ingestion/component/chunker/qa_test.go
@@ -22,9 +22,22 @@ import (
"testing"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/entity"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/tableutil"
)
+// mustTableData parses markup into the structured TableData contract a
+// post-migration producer emits, failing the test on error.
+func mustTableData(t *testing.T, html string) *entity.TableData {
+ t.Helper()
+ td, err := tableutil.ParseTableHTML(html)
+ if err != nil {
+ t.Fatalf("ParseTableHTML(%q): %v", html, err)
+ }
+ return td
+}
+
func TestQAChunker_Registered(t *testing.T) {
factory, _, _, ok := runtime.DefaultRegistry.Lookup("QAChunker")
if !ok {
@@ -394,7 +407,7 @@ func TestQAChunker_XLSXJSONRegression(t *testing.T) {
"output_format": "json",
"json": []map[string]any{
{
- "text": "<table><caption>Sheet1</caption><tr><th>question</th><th>answer</th></tr><tr><td>What is RAGFlow?</td><td>A RAG engine.</td></tr><tr><td>Where are the docs?</td><td>On the website.</td></tr></table>",
+ "table": mustTableData(t, "<table><caption>Sheet1</caption><tr><th>question</th><th>answer</th></tr><tr><td>What is RAGFlow?</td><td>A RAG engine.</td></tr><tr><td>Where are the docs?</td><td>On the website.</td></tr></table>"),
"doc_type_kwd": "table",
},
},
@@ -493,7 +506,7 @@ func TestQAChunker_PlainTablePayloadNotSwallowed(t *testing.T) {
// genuinely holds table markup is now read as a table.
func TestQAChunker_TextLabelledHTMLTableReadAsTable(t *testing.T) {
items := []schema.ChunkDoc{
- {Text: "<table><tr><td>q1</td><td>a1</td></tr><tr><td>q2</td><td>a2</td></tr></table>", DocType: "text"},
+ {TableData: mustTableData(t, "<table><tr><td>q1</td><td>a1</td></tr><tr><td>q2</td><td>a2</td></tr></table>"), DocType: "text"},
}
pairs := extractQAJSON(items, "")
if len(pairs) != 2 {
@@ -545,7 +558,7 @@ func TestQAChunkerPositionsRequireSpreadsheetIdentityAndFiveFields(t *testing.T)
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
item := schema.ChunkDoc{
- Text: table,
+ TableData: mustTableData(t, table),
DocType: "table",
Positions: json.RawMessage(tc.positions),
SheetIndex: tc.sheetIndex,
diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.go
index 986733f3c6..4947595e34 100644
--- a/internal/ingestion/component/chunker/table.go
+++ b/internal/ingestion/component/chunker/table.go
@@ -21,11 +21,11 @@
// Unlike TokenChunker (which token-shreds a row's text into multiple
// pieces) or OneChunker (which merges many rows into a single chunk),
// TableChunker keeps the row as the unit of chunking. Every table arrives as
-// rendered HTML — the spreadsheet parsers emit captioned table segments — and
-// each data row becomes one chunk whose text repeats the column names in the
-// Python "- field: value" line format, carrying its own position tuple when
-// the item is a spreadsheet item with a row-aligned matrix. A table whose only
-// row is the header stays one whole chunk.
+// structured TableData on ChunkDoc.TableData — the parsers emit captioned
+// table segments — and each data row becomes one chunk whose text repeats the
+// column names in the Python "- field: value" line format, carrying its own
+// position tuple when the item is a spreadsheet item with a row-aligned
+// matrix. A table whose only row is the header stays one whole chunk.
package chunker
import (
@@ -35,7 +35,9 @@ import (
"strings"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/entity"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/tableutil"
"gorm.io/gorm"
)
@@ -114,11 +116,11 @@ func (c *TableChunkerComponent) invoke(_ context.Context, inputs map[string]any)
}
// tableItems returns the per-row records, preferring JSONResult and
-// falling back to Chunks. Each HTML table row becomes exactly one chunk;
-// every other payload record passes through as one chunk — including
-// pre-upgrade row-IR records (ck_type: table_row/table_header with cells),
-// which hold no markup and therefore keep no per-row positions; documents
-// from before this wire must be re-parsed rather than re-chunked.
+// falling back to Chunks. Each table row becomes exactly one chunk; every
+// other payload record passes through as one chunk — including pre-upgrade
+// row-IR records (ck_type: table_row/table_header with cells), which hold no
+// markup and therefore keep no per-row positions; documents from before this
+// wire must be re-parsed rather than re-chunked.
func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {
source := items
if len(source) == 0 {
@@ -134,22 +136,30 @@ func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {
return filtered
}
-// expandHTMLTableRows turns one HTML <table> payload into one chunk per data
-// row. Non-table payloads pass through unchanged. A table whose only row is
-// the header keeps the whole markup as its single chunk: the header line is
-// then the only searchable representation.
+// expandHTMLTableRows turns one table payload into one chunk per data row.
+// Non-table payloads pass through unchanged. It reads the structured
+// TableData contract that the parser emits on ChunkDoc.TableData; items that
+// do not carry TableData (e.g. pre-upgrade HTML markup or non-table records)
+// pass through as a single chunk. A table whose only row is the header keeps
+// the whole item as its single chunk: the header line is then the only
+// searchable representation. The structured TableData is cleared on every
+// emitted chunk so the whole table is not duplicated into the index.
func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {
- if !isTableHTML(item.Text) {
+ td := item.TableData
+ if td == nil {
return []schema.ChunkDoc{item}
}
- rows, headerCount := tableRowsWithHeader(item.Text)
+ rows, headerCount := td.Rows, td.HeaderRows
if len(rows) <= headerCount {
- return []schema.ChunkDoc{item}
+ out := item
+ out.Text = tableutil.RenderTableText(td)
+ out.TableData = nil
+ return []schema.ChunkDoc{out}
}
names := rows[0]
// R1: a row chunk must carry only its own tuple. That is only sound when
// the item carries spreadsheet identity and its matrix was built
- // row-aligned (one five-field tuple per <tr>, header included); a
+ // row-aligned (one five-field tuple per row, header included); a
// whole-table tuple must not be copied onto every row, and PDF items write
// layout boxes into the same field, so misaligned or non-spreadsheet
// payloads get no positions at all.
@@ -176,6 +186,7 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {
doc.Text = text
doc.TKNums = nil
doc.Positions = nil
+ doc.TableData = nil
if aligned {
if tuple, err := json.Marshal([][]float64{matrix[headerCount+i]}); err == nil {
doc.Positions = tuple
@@ -184,11 +195,23 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {
out = append(out, doc)
}
if len(out) == 0 {
- return []schema.ChunkDoc{item}
+ single := item
+ single.Text = tableutil.RenderTableText(td)
+ single.TableData = nil
+ return []schema.ChunkDoc{single}
}
return out
}
+// tableDataOf returns the structured TableData for an item. The parser emits
+// it on ChunkDoc.TableData; items without it (pre-upgrade HTML markup or
+// non-table records) return nil and pass through unchanged. Re-chunk always
+// re-parses from the source document, so a payload never arrives with HTML
+// but no TableData, and no HTML fallback is performed (decision §9.1).
+func tableDataOf(item schema.ChunkDoc) *entity.TableData {
+ return item.TableData
+}
+
// tableRowRecordText renders one row in the Python table chunker's line format:
// "- {field}: {value}" per non-empty cell, newline joined. A cell whose
// column has no header name keeps its value alone rather than being dropped.
diff --git a/internal/ingestion/component/chunker/table_split.go b/internal/ingestion/component/chunker/table_split.go
index 0957114cef..3b086feea8 100644
--- a/internal/ingestion/component/chunker/table_split.go
+++ b/internal/ingestion/component/chunker/table_split.go
@@ -17,173 +17,74 @@
package chunker
import (
- "regexp"
"strings"
-)
-
-var trRegex = regexp.MustCompile(`(?is)<tr\b[^>]*>.*?</tr>`)
-// tableOpenTagEnd returns the index of the ">" that closes the open tag whose
-// name s begins after, skipping ">" inside quoted attribute values. It returns
-// -1 when the tag never closes before the next "<": a malformed open tag must
-// not be cut at the next ">", which may belong to a closing tag — slicing
-// "<table </table>" there would invert the slice bounds.
-func tableOpenTagEnd(s string) int {
- quote := byte(0)
- for i := 0; i < len(s); i++ {
- switch c := s[i]; {
- case quote != 0:
- if c == quote {
- quote = 0
- }
- case c == '"' || c == '\'':
- quote = c
- case c == '>':
- return i
- case c == '<':
- return -1
- }
- }
- return -1
-}
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/tableutil"
+)
-// splitLargeHTMLTable splits an HTML table into sub-tables that fit within
-// maxTokens, replicating the caption and header row(s) in each sub-table.
-// Token counting is injected (countTokens) so the caller's tokenizer decides
-// what "size" means. This is the chunker-side size budget: Python's flow path
-// never splits a spreadsheet sheet, but honouring chunk_token_size keeps a
-// wide or long table from losing its tail rows to embedding truncation.
-//
-// The text is expected to hold one outer <table> with flat rows — the shape
-// the spreadsheet parsers render. Markup this function cannot cut safely (an
-// open tag that never closes, a ">" inside an attribute value, a nested or
-// second table) is returned unchanged, so callers fall back to one whole
-// chunk instead of cutting at a guessed boundary.
+// splitLargeTable enforces the ChunkTokenSize budget on a structured table,
+// splitting it into sub-tables that fit. Splitting is row-granular: every
+// emitted sub-table is a whole number of complete rows (never a partial row),
+// so the per-row chunk produced downstream keeps full row semantics. The token
+// budget is counted on the real cell text of each row, not on any HTML markup.
//
// Alongside the parts it returns dataRanges and headerRows: for part i, the
-// half-open range [start, end) of DATA rows (rows below the header in the
-// original table) the part contains, and the number of leading header rows
-// every part replicates. A caller carrying a per-row position matrix slices
+// half-open range [start, end) of DATA rows (rows below the header) the part
+// contains, and the number of leading header rows every part replicates. A
+// caller carrying a per-row position matrix slices
// matrix[:headerRows] + matrix[headerRows+start : headerRows+end] to stay
-// aligned with the sub-table markup. headerRows and ranges are 0/nil when
-// the text is returned unchanged.
-func splitLargeHTMLTable(text string, maxTokens int, countTokens func(string) int) (parts []string, dataRanges [][2]int, headerRows int) {
- lower := strings.ToLower(text)
- startIdx := strings.Index(lower, "<table")
- endIdx := strings.LastIndex(lower, "</table>")
- if startIdx < 0 || endIdx < 0 || endIdx <= startIdx {
- return []string{text}, nil, 0
- }
-
- prefix := text[:startIdx]
- afterOpen := text[startIdx:]
- const openTag = "<table"
- // The match must be a tag name, not the prefix of a longer one such as
- // "<tableau>".
- if len(afterOpen) <= len(openTag) || !strings.ContainsRune(" \t\n\f\r/>", rune(afterOpen[len(openTag)])) {
- return []string{text}, nil, 0
- }
- tagRel := tableOpenTagEnd(afterOpen[len(openTag):])
- if tagRel < 0 {
- return []string{text}, nil, 0
- }
- tagEnd := len(openTag) + tagRel
- innerStart := startIdx + tagEnd + 1
- if innerStart > endIdx {
- return []string{text}, nil, 0
+// aligned with the sub-table. headerRows and ranges are 0/nil when the table
+// fits in a single chunk (the caller then emits the whole table unchanged).
+func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(string) int) (parts []entity.TableData, dataRanges [][2]int, headerRows int) {
+ if td == nil || len(td.Rows) == 0 {
+ return nil, nil, 0
}
- tableOpenTag := afterOpen[:tagEnd+1]
- inner := text[innerStart:endIdx]
- // A second table (or a nested one) has no modelled boundary here; leaving
- // the text alone keeps the caller on the single-chunk path.
- if innerLower := strings.ToLower(inner); strings.Contains(innerLower, "<table") || strings.Contains(innerLower, "</table>") {
- return []string{text}, nil, 0
+ headerRows = td.HeaderRows
+ if headerRows > len(td.Rows) {
+ headerRows = len(td.Rows)
}
- suffix := text[endIdx+len("</table>"):]
-
- // Extract caption if present.
- lowerInner := strings.ToLower(inner)
- capStart := strings.Index(lowerInner, "<caption")
- var captionHTML string
- if capStart >= 0 {
- capEndRel := strings.Index(lowerInner[capStart:], "</caption>")
- if capEndRel >= 0 {
- capEnd := capStart + capEndRel + len("</caption>")
- captionHTML = inner[capStart:capEnd]
- }
+ dataRows := td.Rows[headerRows:]
+ if len(dataRows) == 0 {
+ return nil, nil, 0
}
- // Extract all rows.
- rows := trRegex.FindAllString(inner, -1)
- if len(rows) <= 1 {
- return []string{text}, nil, 0
- }
-
- // Determine header rows vs data rows.
- headerEnd := 0
- for i, r := range rows {
- if strings.Contains(strings.ToLower(r), "<th") {
- headerEnd = i + 1
- } else {
- break
- }
- }
- if headerEnd == 0 || headerEnd >= len(rows) {
- headerEnd = 1
- }
- headerRowHTML := rows[:headerEnd]
- dataRows := rows[headerEnd:]
-
- baseHeader := tableOpenTag + captionHTML + strings.Join(headerRowHTML, "")
- prefixTokens := countTokens(prefix)
- baseTokens := countTokens(baseHeader + "</table>")
-
- // prefix (text before the table, e.g. a short Markdown heading) belongs to
- // the table's position in the document: repeat only the table wrapper,
- // caption and header rows in every chunk, and prepend prefix to the first.
- var chunks []string
- var ranges [][2]int
- var currentDataRows []string
- currentTokens := baseTokens + prefixTokens
- currentStart := 0
- firstChunk := true
+ // Base tokens: header rows + caption, replicated into every sub-table.
+ base := &entity.TableData{Rows: td.Rows[:headerRows], Caption: td.Caption}
+ baseTokens := countTokens(tableutil.RenderTableText(base))
+ var start int
+ var currentTokens int
flush := func(end int) {
- if len(currentDataRows) == 0 {
+ if end <= start {
return
}
- var b strings.Builder
- if firstChunk {
- b.WriteString(prefix)
- firstChunk = false
+ part := entity.TableData{
+ Caption: td.Caption,
+ HeaderRows: headerRows,
+ Rows: append(append([][]string{}, td.Rows[:headerRows]...), dataRows[start:end]...),
}
- b.WriteString(baseHeader)
- for _, dr := range currentDataRows {
- b.WriteString(dr)
- }
- b.WriteString("</table>")
- chunks = append(chunks, b.String())
- ranges = append(ranges, [2]int{currentStart, end})
- currentDataRows = nil
+ parts = append(parts, part)
+ dataRanges = append(dataRanges, [2]int{start, end})
+ start = end
currentTokens = baseTokens
- currentStart = end
}
+ currentTokens = baseTokens
for i, row := range dataRows {
- rowTokens := countTokens(row)
- if len(currentDataRows) > 0 && currentTokens+rowTokens > maxTokens {
+ rowTokens := countTokens(strings.Join(row, " "))
+ // Flush the running batch before this row if adding it would exceed the
+ // budget. A single row that alone exceeds the budget still becomes its
+ // own chunk (row integrity wins over the budget).
+ if i > start && currentTokens+rowTokens > maxTokens {
flush(i)
}
- currentDataRows = append(currentDataRows, row)
currentTokens += rowTokens
}
flush(len(dataRows))
- if len(chunks) <= 1 {
- return []string{text}, nil, 0
- }
- if suffix != "" {
- chunks[len(chunks)-1] += suffix
+ if len(parts) <= 1 {
+ return nil, nil, 0
}
- return chunks, ranges, headerEnd
+ return parts, dataRanges, headerRows
}
diff --git a/internal/ingestion/component/chunker/table_split_test.go b/internal/ingestion/component/chunker/table_split_test.go
index 652bdba8d4..f2b29f5242 100644
--- a/internal/ingestion/component/chunker/table_split_test.go
+++ b/internal/ingestion/component/chunker/table_split_test.go
@@ -1,165 +1,112 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+// http://www.apache.org/licenses/LICENSE-2.0
//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
package chunker
import (
- "strings"
"testing"
+
+ "ragflow/internal/entity"
)
-// charTokens approximates a tokenizer for tests: one token per byte.
-func charTokens(s string) int { return len(s) }
+// charTokens counts runes, a cheap deterministic stand-in for a real tokenizer.
+func charTokens(s string) int { return len([]rune(s)) }
-func buildTable(header string, data []string) string {
- var b strings.Builder
- b.WriteString("<table><caption>cap</caption>\n")
- b.WriteString("<tr>" + header + "</tr>\n")
- for _, row := range data {
- b.WriteString("<tr>" + row + "</tr>\n")
+func makeTable(dataRows int) *entity.TableData {
+ rows := [][]string{{"Name", "Age"}} // header
+ for i := 0; i < dataRows; i++ {
+ rows = append(rows, []string{"person", "30"})
}
- b.WriteString("</table>\n")
- return b.String()
+ return &entity.TableData{Rows: rows, HeaderRows: 1}
}
-func TestSplitLargeHTMLTableNoSplit(t *testing.T) {
- small := buildTable("<th>a</th><th>b</th>", []string{"<td>1</td><td>2</td>"})
- parts, ranges, headerRows := splitLargeHTMLTable(small, len(small)+100, charTokens)
- if len(parts) != 1 || parts[0] != small || ranges != nil || headerRows != 0 {
- t.Fatalf("small table must return unchanged with nil ranges, got %d parts ranges=%v headerRows=%d", len(parts), ranges, headerRows)
+func TestSplitLargeTableFitsWhole(t *testing.T) {
+ td := makeTable(3)
+ parts, ranges, headerRows := splitLargeTable(td, 1_000_000, charTokens)
+ if ranges != nil || len(parts) != 0 {
+ t.Fatalf("small table should not split: parts=%v ranges=%v", parts, ranges)
}
- parts, ranges, headerRows = splitLargeHTMLTable("no table here", 1, charTokens)
- if len(parts) != 1 || parts[0] != "no table here" || ranges != nil || headerRows != 0 {
- t.Fatalf("non-table text must pass through unchanged")
+ if headerRows != 0 {
+ t.Fatalf("headerRows = %d, want 0 when unsplit", headerRows)
}
}
-func TestSplitLargeHTMLTableReplicatesHeaderAndSlicesRanges(t *testing.T) {
- data := make([]string, 6)
- for i := range data {
- data[i] = "<td>d" + string(rune('0'+i)) + "</td><td>x</td>"
- }
- text := buildTable("<th>h1</th><th>h2</th>", data)
- // Budget that holds header + 2 data rows at most.
- rowLen := len("<td>d0</td><td>x</td>")
- budget := len("<table><caption>cap</caption><tr><th>h1</th><th>h2</th></tr></table>") + 2*rowLen
- parts, ranges, headerRows := splitLargeHTMLTable(text, budget, charTokens)
+func TestSplitLargeTableRowGranular(t *testing.T) {
+ td := makeTable(20)
+ // Budget bounded so a part holds ~3 data rows.
+ parts, ranges, headerRows := splitLargeTable(td, 22, charTokens)
if headerRows != 1 {
- t.Fatalf("headerRows = %d, want 1 (the single <th> row)", headerRows)
+ t.Fatalf("headerRows = %d, want 1", headerRows)
}
- if len(parts) < 3 {
- t.Fatalf("expected several sub-tables, got %d", len(parts))
+ if len(parts) <= 1 {
+ t.Fatalf("expected multiple parts, got %d", len(parts))
}
- if len(ranges) != len(parts) {
- t.Fatalf("ranges count %d != parts count %d", len(ranges), len(parts))
+ if len(parts) != len(ranges) {
+ t.Fatalf("parts(%d) != ranges(%d)", len(parts), len(ranges))
}
+ // Every part replicates the header and holds a whole number of rows; the
+ // data ranges are contiguous and cover all 20 data rows exactly once.
seen := 0
- for i, p := range parts {
- if !strings.HasPrefix(p, "<table><caption>cap</caption>") {
- t.Errorf("part %d lost its caption: %q", i, p)
- }
- if !strings.Contains(p, "<th>h1</th>") {
- t.Errorf("part %d lost the header row", i)
+ for i, part := range parts {
+ if part.HeaderRows != 1 {
+ t.Fatalf("part %d headerRows = %d", i, part.HeaderRows)
}
- lo, hi := ranges[i][0], ranges[i][1]
- if hi <= lo || hi > len(data) {
- t.Fatalf("part %d range [%d,%d) out of bounds", i, lo, hi)
+ if len(part.Rows) < 2 {
+ t.Fatalf("part %d has <1 data row: %v", i, part.Rows)
}
- for r := lo; r < hi; r++ {
- if !strings.Contains(p, data[r]) {
- t.Errorf("part %d missing row %d", i, r)
- }
- seen++
+ if part.Rows[0][0] != "Name" {
+ t.Fatalf("part %d missing header row", i)
}
- // Rows outside the range must not leak into this part.
- for r := 0; r < len(data); r++ {
- if (r < lo || r >= hi) && strings.Contains(p, data[r]) {
- t.Errorf("part %d leaked row %d outside [%d,%d)", i, r, lo, hi)
- }
+ r := ranges[i]
+ if r[0] != seen {
+ t.Fatalf("part %d range start %d != running %d", i, r[0], seen)
}
+ seen = r[1]
}
- if seen != len(data) {
- t.Errorf("rows covered %d, want %d", seen, len(data))
- }
-}
-
-func TestSplitLargeHTMLTableHeaderlessReplicatesFirstRow(t *testing.T) {
- // Original behaviour: with no <th> row the first row is treated as the
- // repeating header.
- var b strings.Builder
- b.WriteString("<table>")
- b.WriteString("<tr><td>r0</td></tr>")
- for i := 1; i < 5; i++ {
- b.WriteString("<tr><td>row" + string(rune('0'+i)) + "</td></tr>")
- }
- b.WriteString("</table>")
- parts, ranges, headerRows := splitLargeHTMLTable(b.String(), len("<table><tr><td>r0</td></tr><tr><td>row1</td></tr></table>")+10, charTokens)
- if headerRows != 1 {
- t.Fatalf("headerRows = %d, want 1 (first row treated as header)", headerRows)
+ if seen != 20 {
+ t.Fatalf("data rows covered = %d, want 20", seen)
}
- if len(parts) < 2 {
- t.Fatalf("expected split, got %d parts", len(parts))
+ // The concatenation of every part's data rows must equal the original rows.
+ var rebuilt [][]string
+ for _, part := range parts {
+ rebuilt = append(rebuilt, part.Rows[1:]...)
}
- for i, p := range parts {
- if !strings.Contains(p, "<td>r0</td>") {
- t.Errorf("part %d lost replicated first row", i)
+ for i, row := range td.Rows[1:] {
+ if len(rebuilt[i]) != len(row) || rebuilt[i][0] != row[0] {
+ t.Fatalf("row %d mismatch: got %v want %v", i, rebuilt[i], row)
}
}
- if ranges[0][0] != 0 {
- t.Errorf("ranges index DATA rows below the treated header, want first range to start at 0, got %v", ranges[0])
- }
}
-// TestSplitLargeHTMLTableRefusesUncuttableMarkup: an open tag that never
-// closes, a longer tag name, or a nested table is markup this splitter must
-// not cut at a guessed boundary — the text comes back unchanged. The
-// "<table </table>" case used to invert the slice bounds.
-func TestSplitLargeHTMLTableRefusesUncuttableMarkup(t *testing.T) {
- unchanged := []string{
- "<table </table>",
- "<tableau><tr><td>a</td></tr><tr><td>b</td></tr></table>",
- "<table><tr><td>a</td></tr><table><tr><td>b</td></tr></table></table>",
- }
- for _, text := range unchanged {
- parts, ranges, headerRows := splitLargeHTMLTable(text, 1, charTokens)
- if len(parts) != 1 || parts[0] != text || ranges != nil || headerRows != 0 {
- t.Errorf("text %q must pass through unchanged, got %d parts ranges=%v headerRows=%d", text, len(parts), ranges, headerRows)
- }
+func TestSplitLargeTableSingleRowOverBudgetKeepsRow(t *testing.T) {
+ // One data row alone exceeds the budget; it must still form its own chunk
+ // (row integrity wins over the budget). A single part means "not split,
+ // emit whole" — the over-budget row becomes one chunk rather than being cut.
+ td := &entity.TableData{
+ Rows: [][]string{{"H"}, {"x" + string(make([]byte, 200))}},
+ HeaderRows: 1,
+ }
+ parts, ranges, _ := splitLargeTable(td, 5, charTokens)
+ if ranges != nil || len(parts) != 0 {
+ t.Fatalf("over-budget single row must emit whole (ranges nil): parts=%v ranges=%v", parts, ranges)
}
}
-// TestSplitLargeHTMLTableQuotedAngleBracketStaysInOpenTag: a ">" inside a
-// quoted attribute value must not end the open tag — every part repeats the
-// complete tag instead of being cut at the attribute.
-func TestSplitLargeHTMLTableQuotedAngleBracketStaysInOpenTag(t *testing.T) {
- text := "<table data-meta=\"a>b\">" +
- "<tr><td>r0</td></tr><tr><td>r1</td></tr><tr><td>r2</td></tr></table>"
- parts, ranges, headerRows := splitLargeHTMLTable(text, 60, charTokens)
- if headerRows != 1 {
- t.Fatalf("headerRows = %d, want 1 (first row treated as header)", headerRows)
- }
- if len(parts) < 2 {
- t.Fatalf("expected a split, got %d parts", len(parts))
- }
- if len(ranges) != len(parts) {
- t.Fatalf("ranges count %d != parts count %d", len(ranges), len(parts))
- }
- for i, p := range parts {
- if !strings.HasPrefix(p, "<table data-meta=\"a>b\">") {
- t.Errorf("part %d lost the complete open tag: %q", i, p)
- }
+func TestSplitLargeTableHeaderOnly(t *testing.T) {
+ td := &entity.TableData{Rows: [][]string{{"Name", "Age"}}, HeaderRows: 1}
+ parts, ranges, _ := splitLargeTable(td, 10, charTokens)
+ if ranges != nil || len(parts) != 0 {
+ t.Fatalf("header-only table should not split")
}
}
diff --git a/internal/ingestion/component/chunker/table_struct_test.go b/internal/ingestion/component/chunker/table_struct_test.go
new file mode 100644
index 0000000000..a8a6cb16ed
--- /dev/null
+++ b/internal/ingestion/component/chunker/table_struct_test.go
@@ -0,0 +1,63 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunker
+
+import (
+ "testing"
+
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/component/schema"
+)
+
+func TestExpandHTMLTableRowsFromTableData(t *testing.T) {
+ td := &entity.TableData{
+ Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}, {"Bob", "25"}},
+ HeaderRows: 1,
+ }
+ out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})
+ if len(out) != 2 {
+ t.Fatalf("got %d chunks, want 2", len(out))
+ }
+ if out[0].Text != "- Name: Alice\n- Age: 30" {
+ t.Fatalf("row0 text = %q", out[0].Text)
+ }
+ if out[1].Text != "- Name: Bob\n- Age: 25" {
+ t.Fatalf("row1 text = %q", out[1].Text)
+ }
+ for _, c := range out {
+ if c.TableData != nil {
+ t.Fatalf("TableData must be cleared on emitted row chunk, got %+v", c.TableData)
+ }
+ }
+}
+
+func TestExpandHTMLTableRowsHeaderOnlySingleChunk(t *testing.T) {
+ td := &entity.TableData{Rows: [][]string{{"Name", "Age"}}, HeaderRows: 1}
+ out := expandHTMLTableRows(schema.ChunkDoc{DocType: "table", TableData: td})
+ if len(out) != 1 {
+ t.Fatalf("got %d chunks, want 1 (header-only)", len(out))
+ }
+ if out[0].TableData != nil {
+ t.Fatalf("TableData must be cleared on the single chunk")
+ }
+}
+
+func TestExpandHTMLTableRowsNonTablePassesThrough(t *testing.T) {
+ item := schema.ChunkDoc{DocType: "text", Text: "just prose"}
+ out := expandHTMLTableRows(item)
+ if len(out) != 1 || out[0].Text != "just prose" {
+ t.Fatalf("non-table not passed through: %+v", out)
+ }
+}
diff --git a/internal/ingestion/component/chunker/table_test.go b/internal/ingestion/component/chunker/table_test.go
index 36979b2a24..596766c3f3 100644
--- a/internal/ingestion/component/chunker/table_test.go
+++ b/internal/ingestion/component/chunker/table_test.go
@@ -17,9 +17,12 @@
package chunker
import (
+ "encoding/json"
"fmt"
"strings"
"testing"
+
+ "ragflow/internal/parser/tableutil"
)
// tableChunksOf drives TableChunker.Invoke and returns the emitted maps.
@@ -40,6 +43,33 @@ func tableChunksOf(t *testing.T, inputs map[string]any) []map[string]any {
return chunks
}
+// tableItem builds a chunker JSON input item carrying the structured TableData
+// contract a post-migration producer emits (derived from markup), plus any
+// extra fields (positions, sheet_index, ck_type, ...). It deliberately does
+// NOT set "text": the chunker derives a plain text body from TableData.
+func tableItem(t *testing.T, html string, extra ...map[string]any) map[string]any {
+ t.Helper()
+ td, err := tableutil.ParseTableHTML(html)
+ if err != nil {
+ t.Fatalf("ParseTableHTML(%q): %v", html, err)
+ }
+ b, err := json.Marshal(td)
+ if err != nil {
+ t.Fatalf("marshal TableData: %v", err)
+ }
+ var tbl map[string]any
+ if err := json.Unmarshal(b, &tbl); err != nil {
+ t.Fatalf("unmarshal TableData: %v", err)
+ }
+ item := map[string]any{"table": tbl, "doc_type_kwd": "table"}
+ if len(extra) > 0 {
+ for k, v := range extra[0] {
+ item[k] = v
+ }
+ }
+ return item
+}
+
// TestTableChunker_OneChunkPerRow ports the Python table chunker's contract:
// "Every row in table will be treated as a chunk." Two upstream rows must
// yield exactly two chunks, each preserving its own content + metadata.
@@ -83,8 +113,8 @@ func TestTableChunker_EmptyRows(t *testing.T) {
}
// TestTableChunkerPreservesHeaderOnlyTable: the header row only becomes a
-// chunk itself when the segment has no data rows — then the whole markup is
-// the table's only searchable representation.
+// chunk itself when the segment has no data rows — then the whole table's
+// plain-text rendering is the table's only searchable representation.
func TestTableChunkerPreservesHeaderOnlyTable(t *testing.T) {
segment := spreadsheetSegmentItem("Sheet1", []string{"ID", "Name"}, nil, 1, 1)
chunks := tableChunksOf(t, map[string]any{
@@ -95,8 +125,14 @@ func TestTableChunkerPreservesHeaderOnlyTable(t *testing.T) {
if len(chunks) != 1 {
t.Fatalf("got %d chunks, want the header-only table preserved", len(chunks))
}
- if chunks[0]["ck_type"] != "table" || chunks[0]["text"] != segment["text"] {
- t.Fatalf("chunk = %#v, want the header-only segment whole", chunks[0])
+ if chunks[0]["ck_type"] != "table" {
+ t.Fatalf("chunk = %#v, want a table chunk", chunks[0])
+ }
+ if got, ok := chunks[0]["text"].(string); !ok || got == "" {
+ t.Fatalf("header-only chunk must carry a plain-text body, got %#v", chunks[0]["text"])
+ }
+ if chunks[0]["table"] != nil {
+ t.Fatalf("structured TableData must be cleared on the emitted chunk: %#v", chunks[0]["table"])
}
}
@@ -116,14 +152,14 @@ func TestTableChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {
if chunks[0]["text"] != "- ID: A-1\n- Name: paid" {
t.Fatalf("chunk0 = %#v", chunks[0])
}
- if chunks[1]["text"] != headerSegment["text"] {
- t.Fatalf("chunk1 = %#v, want the header-only segment whole", chunks[1])
+ if chunks[1]["ck_type"] != "table" || chunks[1]["table"] != nil {
+ t.Fatalf("chunk1 = %#v, want the header-only segment whole with TableData cleared", chunks[1])
}
}
// TestTableChunker_ExpandsHTMLRowsWithAlignedPositions covers the P2
-// expander: one HTML <table> item with a row-aligned position matrix (one
-// tuple per <tr>, header included) becomes one chunk per data row, each
+// expander: one table item with a row-aligned position matrix (one
+// tuple per row, header included) becomes one chunk per data row, each
// carrying its own tuple and the Python "- field: value" line format.
func TestTableChunker_ExpandsHTMLRowsWithAlignedPositions(t *testing.T) {
table := "<table><caption>orders</caption>\n" +
@@ -135,13 +171,11 @@ func TestTableChunker_ExpandsHTMLRowsWithAlignedPositions(t *testing.T) {
"name": "orders.xlsx",
"output_format": "json",
"json": []map[string]any{
- {
- "text": table,
- "doc_type_kwd": "table",
- "ck_type": "table",
- "sheet_index": 1,
- "positions": [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}, {1, 3, 3, 1, 2}},
- },
+ tableItem(t, table, map[string]any{
+ "ck_type": "table",
+ "sheet_index": 1,
+ "positions": [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}, {1, 3, 3, 1, 2}},
+ }),
},
})
if len(chunks) != 2 {
@@ -172,12 +206,10 @@ func TestTableChunker_WholeTableTupleNotCopiedToRows(t *testing.T) {
"name": "sheet.csv",
"output_format": "json",
"json": []map[string]any{
- {
- "text": table,
- "doc_type_kwd": "table",
- "sheet_index": 1,
- "positions": [][]float64{{1, 1, 2, 1, 1}},
- },
+ tableItem(t, table, map[string]any{
+ "sheet_index": 1,
+ "positions": [][]float64{{1, 1, 2, 1, 1}},
+ }),
},
})
if len(chunks) != 2 {
@@ -190,7 +222,7 @@ func TestTableChunker_WholeTableTupleNotCopiedToRows(t *testing.T) {
}
}
-// TestTableChunker_NonSpreadsheetPositionsNotAttached: a markup item without
+// TestTableChunker_NonSpreadsheetPositionsNotAttached: a table item without
// spreadsheet identity must not have its positions read as per-row tuples —
// PDF items write layout boxes into the same field, so a five-field matrix
// that happens to align must still be ignored.
@@ -200,12 +232,10 @@ func TestTableChunker_NonSpreadsheetPositionsNotAttached(t *testing.T) {
"name": "document.docx",
"output_format": "json",
"json": []map[string]any{
- {
- "text": table,
- "doc_type_kwd": "table",
- "ck_type": "table",
- "positions": [][]float64{{1, 1, 1, 1, 1}, {1, 2, 2, 1, 1}, {1, 3, 3, 1, 1}},
- },
+ tableItem(t, table, map[string]any{
+ "ck_type": "table",
+ "positions": [][]float64{{1, 1, 1, 1, 1}, {1, 2, 2, 1, 1}, {1, 3, 3, 1, 1}},
+ }),
},
})
if len(chunks) != 2 {
@@ -219,30 +249,39 @@ func TestTableChunker_NonSpreadsheetPositionsNotAttached(t *testing.T) {
}
// TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload: a table whose only
-// row is the header has nothing to expand; the whole markup stays one chunk.
+// row is the header has nothing to expand; the whole markup stays one chunk
+// carrying the plain-text rendering of the header.
func TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload(t *testing.T) {
table := "<table><caption>t</caption>\n<tr><th>A</th><th>B</th></tr>\n</table>\n"
+ td, err := tableutil.ParseTableHTML(table)
+ if err != nil {
+ t.Fatalf("ParseTableHTML: %v", err)
+ }
+ want := tableutil.RenderTableText(td)
chunks := tableChunksOf(t, map[string]any{
"name": "template.xlsx",
"output_format": "json",
"json": []map[string]any{
- {"text": table, "doc_type_kwd": "table", "ck_type": "table"},
+ tableItem(t, table, map[string]any{"ck_type": "table"}),
},
})
- if len(chunks) != 1 || chunks[0]["text"] != table {
- t.Fatalf("header-only table must stay a single whole-payload chunk, got %#v", chunks)
+ if len(chunks) != 1 {
+ t.Fatalf("got %d chunks, want the single header-only chunk", len(chunks))
+ }
+ if chunks[0]["text"] != want {
+ t.Fatalf("header-only chunk text = %q, want %q", chunks[0]["text"], want)
}
}
// TestTableChunker_HeaderlessHTMLFirstRowIsHeader: with no <th> row the
-// first row names the columns, matching SplitLargeHTMLTable's convention.
+// first row names the columns, matching splitLargeTable's convention.
func TestTableChunker_HeaderlessHTMLFirstRowIsHeader(t *testing.T) {
table := "<table><tr><td>k</td><td>v</td></tr><tr><td>a</td><td>b</td></tr></table>"
chunks := tableChunksOf(t, map[string]any{
"name": "kv.csv",
"output_format": "json",
"json": []map[string]any{
- {"text": table, "doc_type_kwd": "table"},
+ tableItem(t, table),
},
})
if len(chunks) != 1 {
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 279813cd9b..bee33050a1 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -84,6 +84,7 @@ import (
deepdoctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/tableutil"
"ragflow/internal/parser/chunk"
)
@@ -255,6 +256,20 @@ func decodeChunkerFromUpstream(inputs map[string]any) (schema.ChunkerFromUpstrea
if err := out.Validate(); err != nil {
return out, err
}
+ // A structured table carries no markup text of its own. Derive a plain
+ // (tag-free) text from its cells so consumers that read Text — TokenChunker
+ // and the embed path — still receive a searchable body when the producer
+ // emitted only the TableData contract (decision 1-B: no HTML in Text).
+ for i := range out.JSONResult {
+ if out.JSONResult[i].TableData != nil && out.JSONResult[i].Text == "" {
+ out.JSONResult[i].Text = tableutil.RenderTableText(out.JSONResult[i].TableData)
+ }
+ }
+ for i := range out.Chunks {
+ if out.Chunks[i].TableData != nil && out.Chunks[i].Text == "" {
+ out.Chunks[i].Text = tableutil.RenderTableText(out.Chunks[i].TableData)
+ }
+ }
return out, nil
}
diff --git a/internal/ingestion/component/schema/chunk_table_test.go b/internal/ingestion/component/schema/chunk_table_test.go
new file mode 100644
index 0000000000..400b4c5c42
--- /dev/null
+++ b/internal/ingestion/component/schema/chunk_table_test.go
@@ -0,0 +1,85 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schema
+
+import (
+ "encoding/json"
+ "testing"
+
+ "ragflow/internal/entity"
+)
+
+func TestChunkDocUnmarshalTableField(t *testing.T) {
+ src := `{
+ "doc_type_kwd": "table",
+ "table": {"rows": [["Name", "Age"], ["Alice", "30"]], "header_rows": 1, "caption": "People"},
+ "text": "legacy-html-should-be-ignored"
+ }`
+ var cd ChunkDoc
+ if err := json.Unmarshal([]byte(src), &cd); err != nil {
+ t.Fatalf("Unmarshal error = %v", err)
+ }
+ if cd.TableData == nil {
+ t.Fatalf("TableData == nil, want populated")
+ }
+ if cd.TableData.HeaderRows != 1 {
+ t.Fatalf("HeaderRows = %d, want 1", cd.TableData.HeaderRows)
+ }
+ if cd.TableData.Caption != "People" {
+ t.Fatalf("Caption = %q, want %q", cd.TableData.Caption, "People")
+ }
+ if len(cd.TableData.Rows) != 2 || cd.TableData.Rows[1][0] != "Alice" {
+ t.Fatalf("Rows = %v, unexpected", cd.TableData.Rows)
+ }
+ // The "table" key must NOT also land in Extra (no double storage).
+ if _, ok := cd.Extra["table"]; ok {
+ t.Fatalf("table key leaked into Extra: %v", cd.Extra)
+ }
+}
+
+func TestChunkDocMarshalRoundTripTable(t *testing.T) {
+ cd := ChunkDoc{
+ DocType: "table",
+ TableData: &entity.TableData{
+ Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}},
+ HeaderRows: 1,
+ },
+ }
+ b, err := json.Marshal(cd)
+ if err != nil {
+ t.Fatalf("Marshal error = %v", err)
+ }
+ var back ChunkDoc
+ if err := json.Unmarshal(b, &back); err != nil {
+ t.Fatalf("Unmarshal error = %v", err)
+ }
+ if back.TableData == nil || len(back.TableData.Rows) != 2 {
+ t.Fatalf("round-trip lost TableData: %v", back.TableData)
+ }
+}
+
+func TestChunkDocNoTableFieldUnaffected(t *testing.T) {
+ src := `{"doc_type_kwd": "text", "text": "hello", "custom": 42}`
+ var cd ChunkDoc
+ if err := json.Unmarshal([]byte(src), &cd); err != nil {
+ t.Fatalf("Unmarshal error = %v", err)
+ }
+ if cd.TableData != nil {
+ t.Fatalf("TableData = %v, want nil", cd.TableData)
+ }
+ if _, ok := cd.Extra["custom"]; !ok {
+ t.Fatalf("Extra lost unrelated key: %v", cd.Extra)
+ }
+}
diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.go
index 08ec8e9821..5aa15ed2df 100644
--- a/internal/ingestion/component/schema/chunk_types.go
+++ b/internal/ingestion/component/schema/chunk_types.go
@@ -16,7 +16,11 @@
package schema
-import "encoding/json"
+import (
+ "encoding/json"
+
+ "ragflow/internal/entity"
+)
// PayloadFormat is the discriminator shared by parser/chunker/tokenizer
// wire payloads.
@@ -99,34 +103,39 @@ func (m ChunkerFileMeta) MarshalJSON() ([]byte, error) {
// boundaries. Common fields are explicit; dynamic enrichments are
// preserved in Extra for forward compatibility.
type ChunkDoc struct {
- Text string `json:"text,omitempty"`
- DocType string `json:"doc_type_kwd,omitempty"`
- CKType string `json:"ck_type,omitempty"`
- TKNums *int `json:"tk_nums,omitempty"`
- Mom string `json:"mom,omitempty"`
- ImgID string `json:"img_id,omitempty"`
- ID string `json:"id,omitempty"`
- Layout string `json:"layout,omitempty"`
- LayoutType string `json:"layout_type,omitempty"`
- LayoutNo string `json:"layoutno,omitempty"`
- Image string `json:"image,omitempty"`
- ContextAbove string `json:"context_above,omitempty"`
- ContextBelow string `json:"context_below,omitempty"`
- Questions string `json:"questions,omitempty"`
- Keywords string `json:"keywords,omitempty"`
- Summary string `json:"summary,omitempty"`
- ChunkOrderInt *int `json:"chunk_order_int,omitempty"`
- TitleTks string `json:"title_tks,omitempty"`
- TitleSmTks string `json:"title_sm_tks,omitempty"`
- ContentLtks string `json:"content_ltks,omitempty"`
- ContentSmLtks string `json:"content_sm_ltks,omitempty"`
- PageNumber *int `json:"page_number,omitempty"`
- TopInt []int `json:"top_int,omitempty"`
- PDFPositions json.RawMessage `json:"_pdf_positions,omitempty"`
- Positions json.RawMessage `json:"positions,omitempty"`
- Sheet string `json:"sheet,omitempty"`
- SheetIndex *int `json:"sheet_index,omitempty"`
- Extra map[string]json.RawMessage `json:"-"`
+ Text string `json:"text,omitempty"`
+ DocType string `json:"doc_type_kwd,omitempty"`
+ CKType string `json:"ck_type,omitempty"`
+ TKNums *int `json:"tk_nums,omitempty"`
+ Mom string `json:"mom,omitempty"`
+ ImgID string `json:"img_id,omitempty"`
+ ID string `json:"id,omitempty"`
+ Layout string `json:"layout,omitempty"`
+ LayoutType string `json:"layout_type,omitempty"`
+ LayoutNo string `json:"layoutno,omitempty"`
+ Image string `json:"image,omitempty"`
+ ContextAbove string `json:"context_above,omitempty"`
+ ContextBelow string `json:"context_below,omitempty"`
+ Questions string `json:"questions,omitempty"`
+ Keywords string `json:"keywords,omitempty"`
+ Summary string `json:"summary,omitempty"`
+ ChunkOrderInt *int `json:"chunk_order_int,omitempty"`
+ TitleTks string `json:"title_tks,omitempty"`
+ TitleSmTks string `json:"title_sm_tks,omitempty"`
+ ContentLtks string `json:"content_ltks,omitempty"`
+ ContentSmLtks string `json:"content_sm_ltks,omitempty"`
+ PageNumber *int `json:"page_number,omitempty"`
+ TopInt []int `json:"top_int,omitempty"`
+ PDFPositions json.RawMessage `json:"_pdf_positions,omitempty"`
+ Positions json.RawMessage `json:"positions,omitempty"`
+ Sheet string `json:"sheet,omitempty"`
+ SheetIndex *int `json:"sheet_index,omitempty"`
+ // TableData is the structured table contract. When a chunk carries it
+ // the table content is consumed directly from here instead of being
+ // re-parsed from Text markup. It must stay out of Extra (see the
+ // deletion list in UnmarshalJSON).
+ TableData *entity.TableData `json:"table,omitempty"`
+ Extra map[string]json.RawMessage `json:"-"`
}
func (d *ChunkDoc) UnmarshalJSON(data []byte) error {
@@ -149,7 +158,7 @@ func (d *ChunkDoc) UnmarshalJSON(data []byte) error {
"context_above", "context_below", "questions", "keywords", "summary",
"chunk_order_int", "title_tks", "title_sm_tks", "content_ltks",
"content_sm_ltks", "tag_kwd", "page_number", "top_int", "_pdf_positions", "positions",
- "sheet", "sheet_index",
+ "sheet", "sheet_index", "table",
} {
delete(raw, key)
}
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index 5694a57e01..185f46222a 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -105,11 +105,9 @@ const ComponentNameTokenizer = "Tokenizer"
// `re.sub(r"\.[a-zA-Z]+$", "", name)` in tokenizer.py:137.
var titleExtRE = regexp.MustCompile(`\.[a-zA-Z]+$`)
-// htmlTableRE matches HTML table-cell tags so the embedded text fed
-// to the embedding model doesn't carry raw markup. Mirrors the python
-// `re.sub(r"</?(table|td|caption|tr|th)( [^<>]{0,12})?>", " ", txt)` at
-// tokenizer.py:79.
-var htmlTableRE = regexp.MustCompile(`</?(table|td|caption|tr|th)( [^<>]{0,12})?>`)
+// Table content is no longer carried as HTML markup (the parser emits the
+// structured entity.TableData contract), so there is no table markup to strip
+// from the embedded text. The legacy htmlTableRE was removed.
// EmbeddingResult carries a vector plus the model-reported token usage
// for that input batch entry.
@@ -472,8 +470,7 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, na
cacheHits := 0
for i, ck := range chunks {
raw := concatFields(ck, c.param.Fields)
- txt := htmlTableRE.ReplaceAllString(raw, " ")
- txt = strings.TrimSpace(txt)
+ txt := strings.TrimSpace(raw)
if txt == "" {
continue
}
@@ -1077,7 +1074,7 @@ func requiresEmbeddingVector(ck schema.ChunkDoc, fields []string) bool {
}
func cleanEmbeddingText(text string) string {
- return strings.TrimSpace(htmlTableRE.ReplaceAllString(text, " "))
+ return strings.TrimSpace(text)
}
func hasEmbeddingVector(ck schema.ChunkDoc) bool {
diff --git a/internal/parser/parser/align_test.go b/internal/parser/parser/align_test.go
index 640b14aadc..64a18407d5 100644
--- a/internal/parser/parser/align_test.go
+++ b/internal/parser/parser/align_test.go
@@ -390,8 +390,11 @@ func tableCellSignature(text string) string {
func tableSignatures(items []map[string]any) map[string]bool {
sigs := map[string]bool{}
for _, it := range items {
+ // A structured table item carries the TableData contract (no HTML
+ // text); render it back to the legacy markup so the cell signature
+ // matches the Python golden, which keeps raw <table> HTML.
+ text := renderedText(it)
kd, _ := it["doc_type_kwd"].(string)
- text, _ := it["text"].(string)
if kd != "table" && !isStandaloneTable(text) {
continue
}
diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.go
index 7eccf252f8..1445a2279f 100644
--- a/internal/parser/parser/charset_test.go
+++ b/internal/parser/parser/charset_test.go
@@ -167,7 +167,7 @@ func TestCSVParser_GBK(t *testing.T) {
}
var rowText strings.Builder
for _, item := range res.JSON {
- text, _ := item["text"].(string)
+ text := tableItemHTML(t, item)
rowText.WriteString(text)
}
if strings.ContainsRune(rowText.String(), '\ufffd') {
diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.go
index f50c38cec9..283ac0b245 100644
--- a/internal/parser/parser/csv_parser_test.go
+++ b/internal/parser/parser/csv_parser_test.go
@@ -35,8 +35,8 @@ func TestCSVParser_EmitsSpreadsheetRows(t *testing.T) {
"<tr><td>Alice</td><td>30</td><td>New York</td></tr>\n" +
"<tr><td>Bob</td><td>25</td><td>San Francisco</td></tr>\n" +
"</table>\n"
- if item["text"] != wantText {
- t.Errorf("text = %q, want %q", item["text"], wantText)
+ if rendered := tableItemHTML(t, item); rendered != wantText {
+ t.Errorf("table HTML = %q, want %q", rendered, wantText)
}
if item["sheet"] != "Data" || item["sheet_index"] != 1 {
t.Errorf("sheet metadata = %#v", item)
@@ -69,7 +69,7 @@ func TestCSVParser_UTF8BOM(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("items = %#v, want one table item", res.JSON)
}
- text, _ := res.JSON[0]["text"].(string)
+ text := tableItemHTML(t, res.JSON[0])
if !strings.Contains(text, "<tr><th>Name</th><th>Age</th></tr>") {
t.Errorf("header row = %q", text)
}
@@ -177,7 +177,7 @@ func TestCSVParserKeepsVariableRowWidths(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("items = %d, want one HTML table item", len(res.JSON))
}
- text, _ := res.JSON[0]["text"].(string)
+ text := tableItemHTML(t, res.JSON[0])
if !strings.Contains(text, "<tr><td>Q1</td><td>A1</td></tr>") ||
!strings.Contains(text, "<tr><td>Q2</td><td>A2</td><td>extra</td></tr>") {
t.Fatalf("row widths not preserved in markup: %q", text)
@@ -201,7 +201,7 @@ func TestCSVParser_ReadsTheSeparatorTheFileWasWrittenWith(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("separator %q: items = %d, want one HTML table item", sep, len(res.JSON))
}
- text, _ := res.JSON[0]["text"].(string)
+ text := tableItemHTML(t, res.JSON[0])
if !strings.Contains(text, "<tr><th>Name</th><th>Region</th><th>Units</th></tr>") ||
!strings.Contains(text, "<tr><td>Widget</td><td>EU</td><td>12</td></tr>") {
t.Errorf("separator %q: row not split into three columns: %q", sep, text)
@@ -272,7 +272,7 @@ func TestCSVParser_KeepsAnEmptyTabSeparatedField(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("items = %d, want one HTML table item", len(res.JSON))
}
- text, _ := res.JSON[0]["text"].(string)
+ text := tableItemHTML(t, res.JSON[0])
if !strings.Contains(text, "<tr><td>Widget</td><td></td><td>12</td></tr>") {
t.Errorf("empty tab-separated field not kept: %q", text)
}
diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.go
index f6a2f8569c..b4188238af 100644
--- a/internal/parser/parser/docx_ir.go
+++ b/internal/parser/parser/docx_ir.go
@@ -359,6 +359,7 @@ func buildDOCXJSONSections(irJSON string, budget *embeddedMediaBudget) []map[str
"image": nil,
"doc_type_kwd": "table",
}
+ projectPDFTable(table)
tableID := ""
foundImage := false
var tableImages []docxTableImage
diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.go
index 03c98ffe2c..20a82e92c4 100644
--- a/internal/parser/parser/docx_ir_test.go
+++ b/internal/parser/parser/docx_ir_test.go
@@ -7,6 +7,8 @@ import (
"reflect"
"strings"
"testing"
+
+ "ragflow/internal/entity"
)
// TestBuildDOCXJSONSections_FromJSON feeds an office_oxide IR JSON
@@ -30,7 +32,7 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {
{"text": "Title", "image": nil, "doc_type_kwd": "text", "ck_type": "heading"},
{"text": "Hello", "image": nil, "doc_type_kwd": "text"},
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
- {"text": "<table><tr><td>cell</td></tr></table>", "image": nil, "doc_type_kwd": "table"},
+ {"image": nil, "doc_type_kwd": "table", "table": &entity.TableData{Rows: [][]string{{"cell"}}, HeaderRows: 1}},
{"text": "item1", "image": nil, "doc_type_kwd": "text"},
}
if !reflect.DeepEqual(got, want) {
@@ -91,9 +93,12 @@ func TestBuildDOCXJSONSections_ExtractsTableCellImages(t *testing.T) {
if len(got) != 3 {
t.Fatalf("sections = %+v, want table plus two cell image items", got)
}
- if got[0]["doc_type_kwd"] != "table" || got[0]["text"] != "<table><tr><td>beforeafter</td><td></td></tr></table>" || got[0]["source_table_id"] != "docx-table-1" {
+ if got[0]["doc_type_kwd"] != "table" || got[0]["source_table_id"] != "docx-table-1" {
t.Fatalf("table item = %+v", got[0])
}
+ if _, ok := got[0]["table"].(*entity.TableData); !ok {
+ t.Fatalf("table item missing structured TableData: %+v", got[0])
+ }
for i, want := range []string{"aGVsbG8=", "aW1hZ2U="} {
item := got[i+1]
if item["doc_type_kwd"] != "image" || item["image"] != want {
diff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.go
index 42e32d47d8..b222854a41 100644
--- a/internal/parser/parser/docx_parser_test.go
+++ b/internal/parser/parser/docx_parser_test.go
@@ -23,6 +23,8 @@ import (
"encoding/json"
"strings"
"testing"
+
+ "ragflow/internal/entity"
)
func TestBuildDOCXJSONSections_Paragraphs(t *testing.T) {
@@ -95,18 +97,23 @@ func TestBuildDOCXJSONSections_Tables(t *testing.T) {
t.Fatalf("got %d sections, want 1", len(sections))
}
item := sections[0]
- html, ok := item["text"].(string)
- if !ok {
- t.Fatal("text field missing or not string")
- }
- if !strings.Contains(html, "<table>") || !strings.Contains(html, "</table>") {
- t.Errorf("html = %q, missing <table> tags", html)
+ td, ok := item["table"].(*entity.TableData)
+ if !ok || td == nil {
+ t.Fatal("table field missing or not *entity.TableData")
+ }
+ var all strings.Builder
+ for _, row := range td.Rows {
+ for _, c := range row {
+ all.WriteString(c)
+ }
}
- if !strings.Contains(html, "<tr>") || !strings.Contains(html, "</tr>") {
- t.Errorf("html = %q, missing <tr> tags", html)
+ s := all.String()
+ if !strings.Contains(s, "A1") || !strings.Contains(s, "B1") ||
+ !strings.Contains(s, "A2") || !strings.Contains(s, "B2") {
+ t.Errorf("table rows = %#v, missing cell content", td.Rows)
}
- if !strings.Contains(html, "<td>A1</td>") || !strings.Contains(html, "<td>B2</td>") {
- t.Errorf("html = %q, missing cell content", html)
+ if len(td.Rows) != 2 {
+ t.Errorf("table rows = %#v, want 2 rows", td.Rows)
}
if got, ok := item["doc_type_kwd"].(string); !ok || got != "table" {
t.Errorf("doc_type_kwd = %q, want %q", got, "table")
diff --git a/internal/parser/parser/html_charset_test.go b/internal/parser/parser/html_charset_test.go
index b42f6f58b6..604f4ba442 100644
--- a/internal/parser/parser/html_charset_test.go
+++ b/internal/parser/parser/html_charset_test.go
@@ -63,10 +63,8 @@ func htmlEncodedFixture(t *testing.T, src string, enc encoding.Encoding) []byte
func itemsText(res ParseResult) string {
var sb strings.Builder
for _, it := range res.JSON {
- if s, ok := it["text"].(string); ok {
- sb.WriteString(s)
- sb.WriteString("\n")
- }
+ sb.WriteString(renderedText(it))
+ sb.WriteString("\n")
}
return sb.String()
}
@@ -163,7 +161,7 @@ func TestHTMLParser_DecodeTableSurvivesGBK(t *testing.T) {
var tableText string
for _, it := range res.JSON {
if it["doc_type_kwd"] == "table" {
- tableText, _ = it["text"].(string)
+ tableText = renderedText(it)
}
}
if tableText == "" {
diff --git a/internal/parser/parser/html_media_test.go b/internal/parser/parser/html_media_test.go
index 5df4e73238..024e28a859 100644
--- a/internal/parser/parser/html_media_test.go
+++ b/internal/parser/parser/html_media_test.go
@@ -45,12 +45,12 @@ func TestHTMLParser_EmitsTableCellImagesWithParentAndCellOrder(t *testing.T) {
if result.JSON[0]["doc_type_kwd"] != "table" || result.JSON[0]["source_table_id"] != "html-table-1" {
t.Errorf("table item = %+v", result.JSON[0])
}
- tableText, _ := result.JSON[0]["text"].(string)
+ tableText := renderedText(result.JSON[0])
if strings.Contains(tableText, "data:image/") {
t.Error("table text duplicates the separately emitted inline image payload")
}
- if !strings.Contains(tableText, `alt="first"`) {
- t.Errorf("table text lost the image alt text: %q", tableText)
+ if !strings.Contains(tableText, "A") {
+ t.Errorf("table text lost the cell text: %q", tableText)
}
if result.JSON[1]["image"] != htmlMediaDataURI {
t.Errorf("inline image payload = %v, want data URI", result.JSON[1]["image"])
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index 1a8a94f3d1..a41404d366 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -25,6 +25,7 @@ import (
"strings"
"golang.org/x/net/html"
+ "ragflow/internal/parser/tableutil"
)
type HTMLParser struct {
@@ -405,10 +406,17 @@ func emitHTMLTable(n *html.Node, out *[]map[string]any, state *htmlWalkState) {
if strings.TrimSpace(markup) == "" {
return
}
+ // Emit the structured TableData contract so the chunker reads rows
+ // directly instead of re-parsing markup. ParseTableHTML mirrors the
+ // legacy chunker extraction, so the recovered rows/caption are identical
+ // to what the old HTML path produced. html.Parse (used by ParseTableHTML)
+ // is infallible, so the parse always yields a non-nil TableData and no
+ // legacy HTML `text` is kept.
+ td, _ := tableutil.ParseTableHTML(markup)
table := map[string]any{
- "text": markup,
"doc_type_kwd": "table",
"ck_type": "table",
+ "table": td,
}
tableID := ""
if len(images) > 0 {
diff --git a/internal/parser/parser/html_parser_align_test.go b/internal/parser/parser/html_parser_align_test.go
index 3feae909d0..daf42abfe9 100644
--- a/internal/parser/parser/html_parser_align_test.go
+++ b/internal/parser/parser/html_parser_align_test.go
@@ -47,7 +47,7 @@ func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {
trailingIdx = i
}
case "table":
- tableText = text
+ tableText = renderedText(it)
tableIdx = i
tableCount++
default:
@@ -130,7 +130,7 @@ func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {
trailingIdx = i
}
case "table":
- tableText = text
+ tableText = renderedText(it)
tableIdx = i
tableCount++
default:
@@ -216,7 +216,7 @@ func TestHTMLParser_NestedTableWithSurroundingText(t *testing.T) {
afterIdx = i
}
case "table":
- tableText = text
+ tableText = renderedText(it)
tableIdx = i
tableCount++
default:
@@ -312,8 +312,8 @@ func TestHTMLParser_MultipleTablesOrdering(t *testing.T) {
if !(titleIdx < tableItemIdx[0] && tableItemIdx[0] < middleIdx && middleIdx < tableItemIdx[1] && tableItemIdx[1] < endIdx) {
t.Fatalf("table order wrong: tables=%v title=%d middle=%d end=%d", tableItemIdx, titleIdx, middleIdx, endIdx)
}
- t1, _ := res.JSON[tableItemIdx[0]]["text"].(string)
- t2, _ := res.JSON[tableItemIdx[1]]["text"].(string)
+ t1 := renderedText(res.JSON[tableItemIdx[0]])
+ t2 := renderedText(res.JSON[tableItemIdx[1]])
if !strings.Contains(t1, "x") || !strings.Contains(t1, "y") {
t.Errorf("first table item missing x/y cells: %q", t1)
}
diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go
index 606501a266..38843297e6 100644
--- a/internal/parser/parser/markdown_parser.go
+++ b/internal/parser/parser/markdown_parser.go
@@ -32,6 +32,7 @@ import (
markdownlib "github.com/gomarkdown/markdown"
"github.com/gomarkdown/markdown/ast"
mdparser "github.com/gomarkdown/markdown/parser"
+ "ragflow/internal/parser/tableutil"
)
// dataURIPrefix is the MIME prefix for data URI images.
@@ -438,11 +439,18 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[
// embedded once and its markup does not pollute prose chunks.
txt = leafText(n)
if isTableHTML(txt) {
- *out = append(*out, map[string]any{
- "text": txt,
+ // Emit the structured TableData contract so the chunker reads
+ // rows directly instead of re-parsing markup. ParseTableHTML
+ // mirrors the legacy chunker extraction. html.Parse is
+ // infallible, so the parse always yields a non-nil TableData
+ // and no legacy HTML `text` is kept.
+ td, _ := tableutil.ParseTableHTML(txt)
+ table := map[string]any{
"doc_type_kwd": "table",
"ck_type": "table",
- })
+ "table": td,
+ }
+ *out = append(*out, table)
continue
}
// Non-table HTML block: ordinary text, no ck_type.
diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go
index d91241b1e2..3365889472 100644
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -9,6 +9,8 @@ import (
"strings"
"sync/atomic"
"testing"
+
+ "ragflow/internal/entity"
)
func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {
@@ -180,8 +182,7 @@ func TestMarkdownParser_TableKeepsCellsBeyondTheHeader(t *testing.T) {
}
var joined strings.Builder
for _, item := range res.JSON {
- text, _ := item["text"].(string)
- joined.WriteString(text)
+ joined.WriteString(renderedText(item))
joined.WriteString("\n")
}
out := joined.String()
@@ -275,8 +276,12 @@ func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {
if ck, _ := item["ck_type"].(string); ck != "table" {
t.Errorf("table item ck_type = %q, want \"table\"", ck)
}
- if !strings.Contains(text, "<table") {
- t.Errorf("table item text is not raw <table> HTML: %q", text)
+ td, ok := item["table"].(*entity.TableData)
+ if !ok || td == nil {
+ t.Fatalf("table item missing *entity.TableData: %#v", item)
+ }
+ if !tableRowsContain(td, "Check item") {
+ t.Errorf("table item rows missing header label: %+v", td.Rows)
}
case "text":
if strings.Contains(text, "<table") {
@@ -634,8 +639,8 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {
case "table":
sawTableItem = true
tableCount++
- if !strings.Contains(text, "<table") {
- t.Fatalf("table item text is not raw <table> HTML: %q", text)
+ if _, ok := item["table"].(*entity.TableData); !ok {
+ t.Fatalf("table item missing *entity.TableData: %#v", item)
}
if ck, _ := item["ck_type"].(string); ck != "table" {
t.Fatalf("table item ck_type = %q, want \"table\"", ck)
@@ -696,11 +701,11 @@ func TestMarkdownParser_TableWithSurroundingText(t *testing.T) {
afterIdx = i
}
case "table":
- tableText = text
+ tableText = renderedText(item)
tableIdx = i
tableCount++
- if !strings.Contains(text, "<table") {
- t.Fatalf("table item text is not raw <table> HTML: %q", text)
+ if _, ok := item["table"].(*entity.TableData); !ok {
+ t.Fatalf("table item missing *entity.TableData: %#v", item)
}
if ck, _ := item["ck_type"].(string); ck != "table" {
t.Fatalf("table item ck_type = %q, want \"table\"", ck)
@@ -819,8 +824,8 @@ func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {
case "table":
sawTableItem = true
tableCount++
- if !strings.Contains(text, "<table") {
- t.Fatalf("raw table item text is not raw <table> HTML: %q", text)
+ if _, ok := item["table"].(*entity.TableData); !ok {
+ t.Fatalf("raw table item missing *entity.TableData: %#v", item)
}
if ck, _ := item["ck_type"].(string); ck != "table" {
t.Fatalf("raw table item ck_type = %q, want \"table\"", ck)
@@ -882,8 +887,8 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {
if !(titleIdx < tableItemIdx[0] && tableItemIdx[0] < middleIdx && middleIdx < tableItemIdx[1] && tableItemIdx[1] < endIdx) {
t.Fatalf("table order wrong: tables=%v title=%d middle=%d end=%d", tableItemIdx, titleIdx, middleIdx, endIdx)
}
- t1, _ := res.JSON[tableItemIdx[0]]["text"].(string)
- t2, _ := res.JSON[tableItemIdx[1]]["text"].(string)
+ t1 := renderedText(res.JSON[tableItemIdx[0]])
+ t2 := renderedText(res.JSON[tableItemIdx[1]])
if !strings.Contains(t1, "x") || !strings.Contains(t1, "y") {
t.Fatalf("first table item missing x/y cells: %q", t1)
}
@@ -955,3 +960,17 @@ func TestMarkdownParser_AlignmentGolden(t *testing.T) {
})
}
}
+
+// tableRowsContain reports whether any cell of the structured table contains
+// the substring want, so table-contract tests can assert cell content without
+// depending on the legacy HTML markup shape.
+func tableRowsContain(td *entity.TableData, want string) bool {
+ for _, row := range td.Rows {
+ for _, cell := range row {
+ if strings.Contains(cell, want) {
+ return true
+ }
+ }
+ }
+ return false
+}
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index 0c22e5267b..4c074d5e97 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -19,7 +19,6 @@ package parser
import (
"encoding/base64"
"fmt"
- "html"
"regexp"
"sort"
"strconv"
@@ -29,6 +28,7 @@ import (
"go.uber.org/zap"
"ragflow/internal/common"
+ "ragflow/internal/entity"
)
// tableIllegalCharsRe replaces illegal control characters (everything except
@@ -51,47 +51,32 @@ type spreadsheetSegmentRow struct {
colStart int
}
-// renderTableHTML renders one sheet segment in the spreadsheet wire format: a
-// captioned <table> whose header row is <th> cells and whose data rows are
-// <td> cells, one row per line. It is the byte contract every consumer reads —
-// the chunkers parse this shape back row by row (see
-// internal/ingestion/component/chunker/html_rows.go), and the positions matrix
-// emitted next to it is aligned one tuple per <tr>, header included.
-func renderSpreadsheetTable(sheet string, header []string, rows [][]string) string {
- var builder strings.Builder
- builder.WriteString("<table><caption>")
- builder.WriteString(html.EscapeString(sheet))
- builder.WriteString("</caption>\n<tr>")
- for _, cell := range header {
- builder.WriteString("<th>")
- builder.WriteString(html.EscapeString(strings.TrimSpace(cell)))
- builder.WriteString("</th>")
- }
- builder.WriteString("</tr>\n")
- for _, row := range rows {
- builder.WriteString("<tr>")
- for _, cell := range row {
- builder.WriteString("<td>")
- builder.WriteString(html.EscapeString(strings.TrimSpace(cell)))
- builder.WriteString("</td>")
- }
- builder.WriteString("</tr>\n")
- }
- builder.WriteString("</table>\n")
- return builder.String()
+// trimCells returns a copy of row with every cell TrimSpace'd. The legacy
+// spreadsheet producer HTML-escaped each TrimSpace'd cell; the structured
+// TableData contract stores the already-TrimSpace'd text, which the chunker
+// re-escapes when it renders chunk text (see tableutil.RenderTableHTML), so
+// the rendered text matches the legacy HTML byte for byte.
+func trimCells(row []string) []string {
+ out := make([]string, len(row))
+ for i, c := range row {
+ out[i] = strings.TrimSpace(c)
+ }
+ return out
}
-// buildSheetItems renders one sheet into its wire items: HTML <table>
-// segments split at image anchor boundaries, interleaved with the anchored
-// image items in document order. Every segment repeats the header row (so
-// segment 2+ never loses column names) and carries a row-aligned position
-// matrix — one tuple per <tr> in strict markup order, header included — so
-// row-level consumers index positions by <tr> number with no side channel.
+// buildSheetItems renders one sheet into its wire items: the structured
+// TableData contract (captioned table with a header row and data rows) split
+// at image anchor boundaries, interleaved with the anchored image items in
+// document order. Every segment repeats the header row (so segment 2+ never
+// loses column names) and carries a row-aligned position matrix — one tuple
+// per row in strict order, header included — so row-level consumers index
+// positions by row number with no side channel. The chunker reads the rows
+// directly; no HTML markup is emitted.
//
// An image lands after the last row whose (row, colStart) sorts at or before
// its anchor; rows before the header's anchor keep the header in front of
// the image (a header-only lead segment); fully empty rows are skipped from
-// both markup and matrix.
+// both rows and matrix.
//
// Segmentation is semantic only — sheet boundaries and image anchors, never a
// size budget. The legacy html4excel path pre-cut a fixed 12 rows here, which
@@ -179,14 +164,16 @@ func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow int
items := make([]map[string]any, 0, len(rows)+len(images))
newSegment := func(body []spreadsheetSegmentRow) map[string]any {
- cells := make([][]string, 0, len(body))
matrix := make([][]float64, 0, len(body)+1)
matrix = append(matrix, headerTuple)
+ rowsOut := make([][]string, 0, len(body)+1)
+ rowsOut = append(rowsOut, trimCells(header))
for _, r := range body {
- cells = append(cells, r.cells)
+ rowsOut = append(rowsOut, trimCells(r.cells))
matrix = append(matrix, r.tuple)
}
- item := NewTableJSONItem(renderSpreadsheetTable(sheet, header, cells), sheet, matrix)
+ td := &entity.TableData{Rows: rowsOut, HeaderRows: 1, Caption: sheet}
+ item := NewTableDataJSONItem(td, sheet, matrix)
item["sheet_index"] = sheetIndex
return item
}
diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.go
index 6bb517a548..a8fd74c909 100644
--- a/internal/parser/parser/office_table_render_test.go
+++ b/internal/parser/parser/office_table_render_test.go
@@ -8,6 +8,8 @@ import (
"testing"
"github.com/xuri/excelize/v2"
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/tableutil"
)
func TestXLSXParserEmitsSegmentedHTMLTable(t *testing.T) {
@@ -31,14 +33,14 @@ func TestXLSXParserEmitsSegmentedHTMLTable(t *testing.T) {
if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {
t.Fatalf("item types = ck_type:%v doc_type:%v", item["ck_type"], item["doc_type_kwd"])
}
- text, _ := item["text"].(string)
wantText := "<table><caption>Sheet1</caption>\n" +
"<tr><th>ID</th><th>Status</th></tr>\n" +
"<tr><td>A-100</td><td>paid</td></tr>\n" +
"<tr><td>A-101</td><td>pending</td></tr>\n" +
"</table>\n"
- if text != wantText {
- t.Fatalf("text = %q, want %q", text, wantText)
+ rendered := tableItemHTML(t, item)
+ if rendered != wantText {
+ t.Fatalf("table HTML = %q, want %q", rendered, wantText)
}
if item["sheet"] != "Sheet1" || item["sheet_index"] != 1 {
t.Fatalf("sheet metadata = %#v", item)
@@ -79,7 +81,7 @@ func TestXLSXParserHTML4ExcelIsIgnored(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("items = %d, want one atomic table item", len(res.JSON))
}
- if res.JSON[0]["ck_type"] != "table" || !strings.Contains(res.JSON[0]["text"].(string), "<table>") {
+ if res.JSON[0]["ck_type"] != "table" || res.JSON[0]["table"] == nil {
t.Fatalf("html4excel item = %#v", res.JSON[0])
}
if !reflect.DeepEqual(res.JSON, plainRes.JSON) {
@@ -111,7 +113,7 @@ func TestXLSXParserDoesNotPrechunkRows(t *testing.T) {
if last[1] != float64(dataRows+1) || last[2] != float64(dataRows+1) {
t.Fatalf("last tuple = %v, want source row %d", last, dataRows+1)
}
- text, _ := res.JSON[0]["text"].(string)
+ text := tableItemHTML(t, res.JSON[0])
if !strings.Contains(text, "<td>258</td>") {
t.Fatalf("last data row missing from markup:\n%s", text)
}
@@ -170,8 +172,8 @@ func TestXLSXParserEmitsHeaderOnlySegment(t *testing.T) {
wantText := "<table><caption>Sheet1</caption>\n" +
"<tr><th>Name</th><th>Amount</th></tr>\n" +
"</table>\n"
- if item["text"] != wantText {
- t.Fatalf("text = %q, want %q", item["text"], wantText)
+ if rendered := tableItemHTML(t, item); rendered != wantText {
+ t.Fatalf("table HTML = %q, want %q", rendered, wantText)
}
positions, _ := item["positions"].([][]float64)
if len(positions) != 1 || !reflect.DeepEqual(positions[0], []float64{1, 1, 1, 1, 2}) {
@@ -188,7 +190,7 @@ func TestBuildSheetItemsEscapesCellText(t *testing.T) {
if len(items) != 1 {
t.Fatalf("items = %d, want one segment", len(items))
}
- text, _ := items[0]["text"].(string)
+ text := tableItemHTML(t, items[0])
if strings.Contains(text, "<b>") || !strings.Contains(text, "&lt;b&gt;") {
t.Fatalf("cell text was not escaped: %q", text)
}
@@ -244,7 +246,7 @@ func TestXLSXParserImageAnchorsSplitSegments(t *testing.T) {
}
for segIdx, want := range [][]string{wantFirst, wantSecond} {
item := items[segIdx*2]
- text, _ := item["text"].(string)
+ text := tableItemHTML(t, item)
rows := markupRows(t, text)
if len(rows) != len(want)+1 {
t.Fatalf("segment %d rows = %d, want header plus %d", segIdx, len(rows), len(want))
@@ -303,11 +305,37 @@ func markupRows(t *testing.T, markup string) [][]string {
return rows
}
+// tableItemHTML renders a parser-produced structured table item back to the
+// legacy spreadsheet HTML so the byte-level assertions in these tests keep
+// pinning the same wire shape. The producer emits *entity.TableData now, and
+// RenderTableHTML reproduces the legacy markup exactly.
+func tableItemHTML(t *testing.T, item map[string]any) string {
+ t.Helper()
+ td, ok := item["table"].(*entity.TableData)
+ if !ok {
+ t.Fatalf("item missing *entity.TableData table field: %#v", item)
+ }
+ return tableutil.RenderTableHTML(td)
+}
+
+// renderedText returns the legacy HTML rendering of an item's structured table
+// when present, otherwise its text field. It lets table-contract tests pin
+// the byte-level wire shape whether the producer emitted TableData or plain
+// text (e.g. non-table items).
+func renderedText(item map[string]any) string {
+ if td, ok := item["table"].(*entity.TableData); ok && td != nil {
+ return tableutil.RenderTableHTML(td)
+ }
+ text, _ := item["text"].(string)
+ return text
+}
+
func spreadsheetText(res ParseResult) string {
var out strings.Builder
for _, item := range res.JSON {
- text, _ := item["text"].(string)
- out.WriteString(text)
+ if td, ok := item["table"].(*entity.TableData); ok {
+ out.WriteString(tableutil.RenderTableHTML(td))
+ }
}
return out.String()
}
@@ -317,10 +345,11 @@ func spreadsheetText(res ParseResult) string {
// table item exists.
func spreadsheetHeaderText(res ParseResult) string {
for _, item := range res.JSON {
- text, _ := item["text"].(string)
- if !strings.Contains(strings.ToLower(text), "<table") {
+ td, ok := item["table"].(*entity.TableData)
+ if !ok {
continue
}
+ text := tableutil.RenderTableHTML(td)
row := text
if i := strings.Index(row, "<tr>"); i >= 0 {
row = row[i+len("<tr>"):]
diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.go
index 4d7752deb7..655bfd88e7 100644
--- a/internal/parser/parser/parse_result.go
+++ b/internal/parser/parser/parse_result.go
@@ -31,7 +31,11 @@
package parser
-import "context"
+import (
+ "context"
+
+ "ragflow/internal/entity"
+)
// ParseResult is the structured return value of a parse operation. JSON holds
// structured backend output when available; the rendered fields carry
@@ -108,10 +112,14 @@ func NewTextJSONItem(text string) map[string]any {
}
}
-// NewTableJSONItem constructs a canonical table JSON item for parser output.
-func NewTableJSONItem(html string, sheet string, positions [][]float64) map[string]any {
+// NewTableDataJSONItem constructs a canonical structured table JSON item for
+// parser output. It carries the parsed TableData contract (caption, header
+// rows, cell text already normalized the same way the legacy chunker
+// normalized HTML <td>/<th> text) instead of the legacy HTML markup string,
+// so the chunker reads rows directly without re-parsing any markup.
+func NewTableDataJSONItem(td *entity.TableData, sheet string, positions [][]float64) map[string]any {
item := map[string]any{
- "text": html,
+ "table": td,
DocTypeKey: DocTypeTable,
"ck_type": DocTypeTable,
}
diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.go
index 031e378dad..380f4bf0b7 100644
--- a/internal/parser/parser/parse_result_test.go
+++ b/internal/parser/parser/parse_result_test.go
@@ -19,10 +19,17 @@ package parser
import (
"strings"
"testing"
+
+ "ragflow/internal/entity"
)
-func TestNewTableJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {
- item := NewTableJSONItem("<table><tr><td>value</td></tr></table>", "Sheet1", [][]float64{{1, 2, 2, 1, 1}})
+func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {
+ td := &entity.TableData{
+ Rows: [][]string{{"col"}, {"value"}},
+ HeaderRows: 1,
+ Caption: "Sheet1",
+ }
+ item := NewTableDataJSONItem(td, "Sheet1", [][]float64{{1, 2, 2, 1, 1}})
if got, want := item[DocTypeKey], DocTypeTable; got != want {
t.Fatalf("item[%q] = %v, want %v", DocTypeKey, got, want)
@@ -30,6 +37,16 @@ func TestNewTableJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {
if got, want := item["ck_type"], DocTypeTable; got != want {
t.Fatalf("item[%q] = %v, want %v", "ck_type", got, want)
}
+ got, ok := item["table"].(*entity.TableData)
+ if !ok {
+ t.Fatalf("item[%q] = %T, want *entity.TableData", "table", item["table"])
+ }
+ if got.Caption != "Sheet1" || len(got.Rows) != 2 {
+ t.Fatalf("item table = %+v, want caption Sheet1 and 2 rows", got)
+ }
+ if _, ok := item["positions"].([][]float64); !ok {
+ t.Fatalf("item[%q] missing positions matrix", "positions")
+ }
}
// TestParseResult_Contract pins the wire-shape guarantees
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go
index bbc3d94221..dcb93f0a96 100644
--- a/internal/parser/parser/pdf_parser_common.go
+++ b/internal/parser/parser/pdf_parser_common.go
@@ -32,6 +32,7 @@ import (
pdflayout "ragflow/internal/deepdoc/parser/pdf/layout"
"ragflow/internal/deepdoc/parser/pdf/util"
deepdoctype "ragflow/internal/deepdoc/parser/type"
+ "ragflow/internal/parser/tableutil"
"ragflow/internal/utility"
)
@@ -494,6 +495,7 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR
}
}
normalizePDFDocType(items[i])
+ projectPDFTable(items[i])
if img, _ := items[i]["image"].(string); img != "" {
items[i]["image"] = pdflayout.InlinePNGDataURL(img)
}
@@ -931,6 +933,42 @@ func normalizePDFDocType(item map[string]any) {
}
}
+// projectPDFTable upgrades a legacy pdf table item — one whose doc_type_kwd is
+// "table" and whose text is <table> markup — to the structured TableData
+// contract, so the downstream chunker reads Rows directly instead of
+// re-parsing HTML. ParseTableHTML + RenderTableHTML round-trips the legacy
+// htmlTableRE extraction exactly (entity/html unescaping is left to the
+// consumer), so dropping the original HTML string loses no information: the
+// markdown and search paths render it on demand from `table`. The pdf page
+// number (when present) is projected onto TableData.Page.
+//
+// The redundant HTML `text` is deleted once `table` is projected: keeping both
+// would duplicate every table's cell text in the parse result (the storage
+// cost that motivated moving to the structured contract). Non-table items, and
+// table items whose text is not <table> markup, are left untouched so their
+// `text` is preserved as a last-resort carrier.
+func projectPDFTable(item map[string]any) {
+ if item == nil {
+ return
+ }
+ if dt, _ := item["doc_type_kwd"].(string); dt != "table" {
+ return
+ }
+ text, _ := item["text"].(string)
+ if !isTableHTML(text) {
+ return
+ }
+ td, err := tableutil.ParseTableHTML(text)
+ if err != nil || td == nil {
+ return
+ }
+ if page := numberValue(item["page_number"]); page > 0 {
+ td.Page = int(page)
+ }
+ item["table"] = td
+ delete(item, "text")
+}
+
// ExtractPDFPositions is the single source of truth for "does this parsed item
// carry a usable PDF crop region". It returns the non-empty positions matrix
// (under the canonical _pdf_positions key or the legacy positions key),
diff --git a/internal/parser/parser/pdf_parser_opendataloader.go b/internal/parser/parser/pdf_parser_opendataloader.go
index dc9c324466..8caa1a1d87 100644
--- a/internal/parser/parser/pdf_parser_opendataloader.go
+++ b/internal/parser/parser/pdf_parser_opendataloader.go
@@ -149,7 +149,9 @@ func openDataLoaderNodeToItem(el map[string]any) map[string]any {
if text == "" {
return nil
}
- return map[string]any{"text": text, "doc_type_kwd": pdfTableDocType(text), "layout": "table"}
+ item := map[string]any{"text": text, "doc_type_kwd": pdfTableDocType(text), "layout": "table"}
+ projectPDFTable(item)
+ return item
case "image", "picture", "figure":
if text == "" {
text = "[Image]"
diff --git a/internal/parser/parser/pdf_parser_opendataloader_test.go b/internal/parser/parser/pdf_parser_opendataloader_test.go
index ebf5f55707..b20301ca03 100644
--- a/internal/parser/parser/pdf_parser_opendataloader_test.go
+++ b/internal/parser/parser/pdf_parser_opendataloader_test.go
@@ -7,6 +7,8 @@ import (
"net/http/httptest"
"strings"
"testing"
+
+ "ragflow/internal/entity"
)
func TestPDFParser_ParseWithResult_OpenDataLoaderJSONIntegration(t *testing.T) {
@@ -134,8 +136,18 @@ func TestOpenDataLoaderItems_TableCellsFallback(t *testing.T) {
if len(items) != 1 {
t.Fatalf("items len = %d, want 1", len(items))
}
- if got, want := items[0]["text"], "<table><tr><th >a</th><th >b</th></tr></table>"; got != want {
- t.Fatalf("text = %q, want %q", got, want)
+ td, ok := items[0]["table"].(*entity.TableData)
+ if !ok || td == nil {
+ t.Fatalf("table field missing: %#v", items[0])
+ }
+ if items[0]["text"] != nil {
+ t.Fatalf("table item must not carry legacy HTML text: %#v", items[0])
+ }
+ if len(td.Rows) != 1 || len(td.Rows[0]) != 2 || td.Rows[0][0] != "a" || td.Rows[0][1] != "b" {
+ t.Fatalf("table rows = %#v, want [[a b]]", td.Rows)
+ }
+ if td.HeaderRows != 1 {
+ t.Fatalf("HeaderRows = %d, want 1", td.HeaderRows)
}
}
diff --git a/internal/parser/parser/pdf_parser_remote_common.go b/internal/parser/parser/pdf_parser_remote_common.go
index acde5ace67..a26af71a1d 100644
--- a/internal/parser/parser/pdf_parser_remote_common.go
+++ b/internal/parser/parser/pdf_parser_remote_common.go
@@ -3,6 +3,9 @@ package parser
import (
"fmt"
"strings"
+
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/tableutil"
)
func pdfFileMeta(filename string, pageCount int) map[string]any {
@@ -16,6 +19,23 @@ func pdfFileMeta(filename string, pageCount int) map[string]any {
}
}
+// markdownItemText renders a single parse item to markdown. A table uses the
+// structured TableData contract directly (rendered via RenderTableHTML); every
+// other item uses its text. Producers emit `table` for tables and never the
+// legacy HTML `text`, so a table-typed item without structured data is skipped
+// rather than falling back to a raw HTML string — the contract is "tables are
+// structured, not markup".
+func markdownItemText(item map[string]any) string {
+ if dt, _ := item["doc_type_kwd"].(string); dt == "table" {
+ if td, ok := item["table"].(*entity.TableData); ok && td != nil {
+ return tableutil.RenderTableHTML(td)
+ }
+ return ""
+ }
+ text, _ := item["text"].(string)
+ return text
+}
+
func pdfItemsToResult(filename string, items []map[string]any, outputFormat string, pageCount int) ParseResult {
if len(items) == 0 {
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
@@ -31,7 +51,7 @@ func pdfItemsToResult(filename string, items []map[string]any, outputFormat stri
case "markdown":
var b strings.Builder
for _, item := range items {
- text, _ := item["text"].(string)
+ text := markdownItemText(item)
layout, _ := item["layout"].(string)
if strings.TrimSpace(text) == "" {
continue
diff --git a/internal/parser/parser/pdf_parser_somark.go b/internal/parser/parser/pdf_parser_somark.go
index 75dc22782b..363badbd72 100644
--- a/internal/parser/parser/pdf_parser_somark.go
+++ b/internal/parser/parser/pdf_parser_somark.go
@@ -196,7 +196,9 @@ func soMarkBlockToItem(block map[string]any, keepHeaderFooter bool) map[string]a
if content == "" {
return nil
}
- return map[string]any{"text": content, "doc_type_kwd": pdfTableDocType(content), "layout": "table"}
+ item := map[string]any{"text": content, "doc_type_kwd": pdfTableDocType(content), "layout": "table"}
+ projectPDFTable(item)
+ return item
case "equation":
if content == "" {
return nil
diff --git a/internal/parser/parser/pdf_parser_table_contract_test.go b/internal/parser/parser/pdf_parser_table_contract_test.go
index adf662418b..e6e6dcf0fe 100644
--- a/internal/parser/parser/pdf_parser_table_contract_test.go
+++ b/internal/parser/parser/pdf_parser_table_contract_test.go
@@ -3,11 +3,15 @@ package parser
import (
"strings"
"testing"
+
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/tableutil"
)
// Consumers of doc_type_kwd "table" parse the item text as <table> markup
// (the QA chunker reads its <tr> rows), so a table item that carries other
-// text loses its rows downstream.
+// text loses its rows downstream. The structured TableData contract must also
+// be present on every table item so the chunker no longer re-parses HTML.
func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {
cells := []any{
map[string]any{"row": 0, "content": "What is RAGFlow?"},
@@ -73,12 +77,24 @@ func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {
item := items[0]
docType, _ := item["doc_type_kwd"].(string)
text, _ := item["text"].(string)
- if docType == "table" && !strings.HasPrefix(strings.TrimSpace(text), "<table") {
- t.Fatalf("doc_type_kwd table carries non-<table> text %q", text)
+ if docType == "table" && text != "" {
+ t.Fatalf("doc_type_kwd table must not carry legacy HTML text %q", text)
}
if docType != tc.wantDocType {
t.Fatalf("doc_type_kwd = %q, want %q (text %q)", docType, tc.wantDocType, text)
}
+ if docType == "table" {
+ td, ok := item["table"].(*entity.TableData)
+ if !ok || td == nil {
+ t.Fatalf("doc_type_kwd table item missing structured *entity.TableData: %#v", item)
+ }
+ if len(td.Rows) == 0 {
+ t.Fatalf("structured table has no rows: %#v", td)
+ }
+ if got := tableutil.RenderTableText(td); !strings.Contains(got, "What is RAGFlow?") {
+ t.Fatalf("structured table missing cell text %q: %#v", got, td)
+ }
+ }
})
}
}
diff --git a/internal/parser/parser/pdf_parser_tcadp.go b/internal/parser/parser/pdf_parser_tcadp.go
index faafe813e8..226f9bd438 100644
--- a/internal/parser/parser/pdf_parser_tcadp.go
+++ b/internal/parser/parser/pdf_parser_tcadp.go
@@ -100,7 +100,11 @@ func tcadpAnyToItems(raw any) []map[string]any {
if text == "" {
return nil
}
- return emit(text, pdfTableDocType(text), "table")
+ items := emit(text, pdfTableDocType(text), "table")
+ for _, it := range items {
+ projectPDFTable(it)
+ }
+ return items
case "image":
caption := strings.TrimSpace(stringValue(v["caption"]))
if caption == "" {
diff --git a/internal/parser/parser/pdf_table_no_html_text_test.go b/internal/parser/parser/pdf_table_no_html_text_test.go
new file mode 100644
index 0000000000..e741e91a81
--- /dev/null
+++ b/internal/parser/parser/pdf_table_no_html_text_test.go
@@ -0,0 +1,142 @@
+package parser
+
+import (
+ "strings"
+ "testing"
+
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/tableutil"
+)
+
+// TestPDFTableItemsOmitHTMLText pins the structured TableData contract: a table
+// item must carry `table` (parsed rows) and must NOT carry the legacy HTML
+// `text`. The HTML string is a redundant, storage-costly duplicate that the
+// markdown/search paths derive on demand from `table`. Reducing this catches
+// any producer that regresses to emitting both.
+func TestPDFTableItemsOmitHTMLText(t *testing.T) {
+ const tableHTML = "<table><tr><td>What is RAGFlow?</td><td>A RAG engine.</td></tr><tr><td>Is it open source?</td><td>Yes.</td></tr></table>"
+ const docxIR = `{"sections":[{"elements":[
+ {"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"}]}]}]}]}
+ ]}]}`
+
+ cases := []struct {
+ name string
+ produce func() []map[string]any
+ }{
+ {"projectPDFTable (DLA)", func() []map[string]any {
+ item := map[string]any{"doc_type_kwd": "table", "text": tableHTML, "page_number": 1}
+ projectPDFTable(item)
+ return []map[string]any{item}
+ }},
+ {"opendataloader html", func() []map[string]any {
+ return openDataLoaderItems(map[string]any{"type": "table", "html": tableHTML})
+ }},
+ {"tcadp html content", func() []map[string]any {
+ return tcadpAnyToItems(map[string]any{"type": "table", "content": tableHTML})
+ }},
+ {"somark html", func() []map[string]any {
+ return []map[string]any{soMarkBlockToItem(map[string]any{"type": "table", "content": tableHTML}, false)}
+ }},
+ {"docx table", func() []map[string]any {
+ return buildDOCXJSONSections(docxIR, newEmbeddedMediaBudget())
+ }},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ items := tc.produce()
+ var tableItem map[string]any
+ for _, it := range items {
+ if dt, _ := it["doc_type_kwd"].(string); dt == "table" {
+ tableItem = it
+ break
+ }
+ }
+ if tableItem == nil {
+ t.Fatalf("producer %q emitted no table item: %#v", tc.name, items)
+ }
+ if v, ok := tableItem["text"]; ok && v != "" {
+ t.Fatalf("table item still carries legacy HTML text %q", v)
+ }
+ td, ok := tableItem["table"].(*entity.TableData)
+ if !ok || td == nil || len(td.Rows) == 0 {
+ t.Fatalf("table item missing structured TableData: %#v", tableItem)
+ }
+ if got := tableutil.RenderTableText(td); !strings.Contains(got, "What is RAGFlow?") && !strings.Contains(got, "cell") {
+ t.Fatalf("structured table missing cell text %q: %#v", got, td)
+ }
+ })
+ }
+}
+
+// TestPDFItemsToMarkdownRendersTableFromTableField verifies the markdown
+// renderer derives table markup from `table` when the legacy HTML `text` is
+// absent. This is the consumer that previously forced producers to keep `text`;
+// once producers drop `text`, markdown must still embed the table. The output
+// must be byte-equivalent to rendering `table` directly (stable across the
+// ParseTableHTML/RenderTableHTML round-trip).
+func TestPDFItemsToMarkdownRendersTableFromTableField(t *testing.T) {
+ td := &entity.TableData{
+ Rows: [][]string{{"What is RAGFlow?", "A RAG engine."}, {"Is it open source?", "Yes."}},
+ HeaderRows: 1,
+ }
+ // Producer-after-cleanup shape: `table` present, `text` absent.
+ item := map[string]any{"doc_type_kwd": "table", "layout": "table", "table": td}
+ res := pdfItemsToResult("doc.pdf", []map[string]any{item}, "markdown", 1)
+ if res.Err != nil {
+ t.Fatalf("pdfItemsToResult markdown: %v", res.Err)
+ }
+ want := strings.TrimSpace(tableutil.RenderTableHTML(td))
+ if got := strings.TrimSpace(res.Markdown); got != want {
+ t.Fatalf("markdown table mismatch:\n got: %q\nwant: %q", got, want)
+ }
+ if !strings.Contains(res.Markdown, "What is RAGFlow?") {
+ t.Fatalf("markdown lost table cell text: %q", res.Markdown)
+ }
+}
+
+// TestPDFItemsToMarkdownSkipsTableWithoutStructuredData guards the
+// structured-first contract: a table-typed item that carries no `table`
+// (e.g. a mislabeled table region) must be skipped from markdown rather than
+// falling back to a raw HTML `text`. This pins "tables are structured, not
+// markup" so a future change cannot reintroduce an HTML fallback for tables.
+func TestPDFItemsToMarkdownSkipsTableWithoutStructuredData(t *testing.T) {
+ // doc_type_kwd "table" but no `table` field and no `text`.
+ item := map[string]any{"doc_type_kwd": "table", "layout": "table"}
+ res := pdfItemsToResult("doc.pdf", []map[string]any{item}, "markdown", 1)
+ if res.Err != nil {
+ t.Fatalf("pdfItemsToResult markdown: %v", res.Err)
+ }
+ if strings.TrimSpace(res.Markdown) != "" {
+ t.Fatalf("markdown must skip table-typed item without structured data, got %q", res.Markdown)
+ }
+}
+
+// TestProjectPDFTableKeepsTextWhenNotProjected guards the safety net: items
+// that are not tables, or whose text is not <table> markup, must keep their
+// `text` untouched and receive no `table` — we must never delete text for
+// non-table items while stripping it from projected tables.
+func TestProjectPDFTableKeepsTextWhenNotProjected(t *testing.T) {
+ t.Run("non-table doc_type", func(t *testing.T) {
+ item := map[string]any{"doc_type_kwd": "text", "text": "plain prose"}
+ projectPDFTable(item)
+ if item["text"] != "plain prose" {
+ t.Fatalf("text mutated for non-table item: %#v", item)
+ }
+ if _, ok := item["table"]; ok {
+ t.Fatalf("non-table item gained table: %#v", item)
+ }
+ })
+ t.Run("table doc_type but non-HTML text", func(t *testing.T) {
+ item := map[string]any{"doc_type_kwd": "table", "text": "not a table"}
+ projectPDFTable(item)
+ if item["text"] != "not a table" {
+ t.Fatalf("text mutated for non-HTML table item: %#v", item)
+ }
+ if _, ok := item["table"]; ok {
+ t.Fatalf("non-HTML table item gained table: %#v", item)
+ }
+ })
+ t.Run("nil item", func(t *testing.T) {
+ projectPDFTable(nil) // must not panic
+ })
+}
diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.go
index b0a7ac0f4b..98b7b4ee29 100644
--- a/internal/parser/parser/xls_parser_test.go
+++ b/internal/parser/parser/xls_parser_test.go
@@ -140,7 +140,7 @@ func TestXLSParser_PreservesFormulaErrors(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("res.JSON len = %d, want 1", len(res.JSON))
}
- text, _ := res.JSON[0]["text"].(string)
+ text := tableItemHTML(t, res.JSON[0])
if !strings.Contains(text, "<tr><td>#DIV/0!</td></tr>") {
t.Fatalf("table markup = %q, want formula error row", text)
}
@@ -174,7 +174,7 @@ func TestXLSParser_GenuineBIFF8SpreadsheetJSONOutput(t *testing.T) {
}
first := res.JSON[0]
- text, _ := first["text"].(string)
+ text := tableItemHTML(t, first)
if !strings.Contains(text, "<caption>Inventory</caption>") ||
!strings.Contains(text, "<tr><th>Name</th><th>Count</th></tr>") ||
!strings.Contains(text, "<tr><td>Apples</td><td>3</td></tr>") {
@@ -195,7 +195,7 @@ func TestXLSParser_GenuineBIFF8SpreadsheetJSONOutput(t *testing.T) {
}
second := res.JSON[1]
- secondText, _ := second["text"].(string)
+ secondText := tableItemHTML(t, second)
if !strings.Contains(secondText, "<caption>Notes</caption>") ||
!strings.Contains(secondText, "<tr><td>Legacy</td><td>BIFF8</td></tr>") {
t.Fatalf("second table markup = %q, want second BIFF8 sheet data", secondText)
@@ -228,7 +228,7 @@ func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {
if item["ck_type"] != "table" {
t.Errorf("table item ck_type = %v, want table", item["ck_type"])
}
- text, _ := item["text"].(string)
+ text := tableItemHTML(t, item)
if !strings.Contains(text, "<tr><th>Header1</th><th>Header2</th></tr>") ||
!strings.Contains(text, "<tr><td>Val1</td><td>Val2</td></tr>") {
t.Errorf("table markup = %q, want captioned header and data row", text)
diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.go
index 9374087ac1..e07b2875d7 100644
--- a/internal/parser/parser/xlsx_parse_method_test.go
+++ b/internal/parser/parser/xlsx_parse_method_test.go
@@ -80,7 +80,7 @@ func TestXLSXParser_DeepDocParseMethod(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON item count = %d, want header-only result", len(res.JSON))
}
- text, _ := res.JSON[0]["text"].(string)
+ text := tableItemHTML(t, res.JSON[0])
if !strings.Contains(text, tc.cellValue) {
t.Fatalf("JSON = %#v, want it to contain cell content %q", res.JSON, tc.cellValue)
}
diff --git a/internal/parser/tableutil/table_html.go b/internal/parser/tableutil/table_html.go
new file mode 100644
index 0000000000..1ac54b5726
--- /dev/null
+++ b/internal/parser/tableutil/table_html.go
@@ -0,0 +1,174 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package tableutil converts between table markup and the structured
+// entity.TableData contract. It intentionally mirrors the legacy chunker
+// extraction in internal/ingestion/component/chunker/html_rows.go so that the
+// cells it produces are byte-for-byte what consumers used to read out of the
+// HTML <table> text.
+package tableutil
+
+import (
+ "regexp"
+ "strings"
+
+ "golang.org/x/net/html"
+ "ragflow/internal/entity"
+)
+
+// ParseTableHTML extracts a structured TableData from table markup. It mirrors
+// the legacy chunker behaviour (html_rows.go tableRowsWithHeader): each <tr>
+// becomes one row, each direct <td>/<th> one cell, a row carrying a <th> cell
+// is a header row, <br> becomes a newline, inert elements
+// (script/style/noscript/template) are skipped, and a nested table's text is
+// folded into the enclosing cell (the nested <tr> does NOT become a row).
+func ParseTableHTML(s string) (*entity.TableData, error) {
+ lower := strings.ToLower(s)
+ if strings.Contains(lower, "<tr") && !strings.Contains(lower, "<table") {
+ s = "<table>" + s + "</table>"
+ }
+ doc, err := html.Parse(strings.NewReader(s))
+ if err != nil {
+ return nil, err
+ }
+ var rows [][]string
+ var headerFlags []bool
+ var caption string
+ var walk func(*html.Node)
+ walk = func(n *html.Node) {
+ if n.Type == html.ElementNode && isInertElement(n.Data) {
+ return
+ }
+ if isHTMLElement(n, "caption") {
+ caption = cellText(n)
+ return
+ }
+ if isHTMLElement(n, "tr") {
+ var cells []string
+ anyTh := false
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ if isHTMLElement(c, "td") || isHTMLElement(c, "th") {
+ if isHTMLElement(c, "th") {
+ anyTh = true
+ }
+ cells = append(cells, cellText(c))
+ }
+ }
+ rows = append(rows, cells)
+ headerFlags = append(headerFlags, anyTh)
+ return
+ }
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ walk(c)
+ }
+ }
+ walk(doc)
+ headerCount := 0
+ for headerCount < len(headerFlags) && headerFlags[headerCount] {
+ headerCount++
+ }
+ if len(rows) > 0 && headerCount == 0 {
+ headerCount = 1
+ }
+ return &entity.TableData{Rows: rows, HeaderRows: headerCount, Caption: caption}, nil
+}
+
+// RenderTableHTML emits a deterministic <table> markup for a TableData. It is
+// used only as a test oracle / round-trip helper and is NOT part of the
+// persisted contract (producers emit TableData, not HTML). The emitted shape
+// matches the legacy spreadsheet producer's renderSpreadsheetTable exactly
+// (including the inter-row newlines) so that RenderTableText reproduces the
+// legacy htmlTableRE-stripped text byte for byte.
+func RenderTableHTML(td *entity.TableData) string {
+ var b strings.Builder
+ b.WriteString("<table>")
+ if td.Caption != "" {
+ b.WriteString("<caption>")
+ b.WriteString(html.EscapeString(td.Caption))
+ b.WriteString("</caption>")
+ }
+ b.WriteString("\n")
+ for i, row := range td.Rows {
+ b.WriteString("<tr>")
+ isHeader := i < td.HeaderRows
+ for _, cell := range row {
+ tag := "td"
+ if isHeader {
+ tag = "th"
+ }
+ b.WriteString("<")
+ b.WriteString(tag)
+ b.WriteString(">")
+ b.WriteString(html.EscapeString(cell))
+ b.WriteString("</")
+ b.WriteString(tag)
+ b.WriteString(">")
+ }
+ b.WriteString("</tr>")
+ b.WriteString("\n")
+ }
+ b.WriteString("</table>")
+ b.WriteString("\n")
+ return b.String()
+}
+
+// tableTagRE mirrors the legacy tokenizer stripping pattern
+// (htmlTableRE in internal/ingestion/component/tokenizer.go) so that the
+// plaintext a chunker writes for an embed-only path matches the legacy
+// tag-stripped output exactly.
+var tableTagRE = regexp.MustCompile(`</?(table|td|caption|tr|th)( [^<>]{0,12})?>`)
+
+// RenderTableText renders a TableData to plain text with no markup tags, for
+// the rare embed path that receives a table without a downstream TableChunker.
+// It reproduces the legacy htmlTableRE stripping of RenderTableHTML(td).
+func RenderTableText(td *entity.TableData) string {
+ h := RenderTableHTML(td)
+ h = tableTagRE.ReplaceAllString(h, " ")
+ return strings.TrimSpace(h)
+}
+
+// cellText returns the visible text of a table cell: text nodes contribute
+// verbatim, <br> becomes a newline, inert elements are skipped, and other
+// elements are descended. The result is TrimSpace'd.
+func cellText(cell *html.Node) string {
+ var sb strings.Builder
+ var walk func(*html.Node)
+ walk = func(n *html.Node) {
+ switch {
+ case n.Type == html.TextNode:
+ sb.WriteString(n.Data)
+ case isHTMLElement(n, "br"):
+ sb.WriteByte('\n')
+ case n.Type == html.ElementNode && isInertElement(n.Data):
+ return
+ }
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ walk(c)
+ }
+ }
+ walk(cell)
+ return strings.TrimSpace(sb.String())
+}
+
+func isHTMLElement(n *html.Node, tag string) bool {
+ return n.Type == html.ElementNode && n.Namespace == "" && n.Data == tag
+}
+
+func isInertElement(tag string) bool {
+ switch tag {
+ case "script", "style", "noscript", "template":
+ return true
+ }
+ return false
+}
diff --git a/internal/parser/tableutil/table_html_test.go b/internal/parser/tableutil/table_html_test.go
new file mode 100644
index 0000000000..abfea73ad6
--- /dev/null
+++ b/internal/parser/tableutil/table_html_test.go
@@ -0,0 +1,187 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tableutil
+
+import (
+ "strings"
+ "testing"
+
+ "ragflow/internal/entity"
+)
+
+// mustParse is a test helper that fails the test on parse error.
+func mustParse(t *testing.T, html string) *entity.TableData {
+ t.Helper()
+ td, err := ParseTableHTML(html)
+ if err != nil {
+ t.Fatalf("ParseTableHTML(%q) error = %v", html, err)
+ }
+ return td
+}
+
+func TestParseTableHTMLHappyPath(t *testing.T) {
+ html := `<table><tr><th>Name</th><th>Age</th></tr>` +
+ `<tr><td>Alice</td><td>30</td></tr>` +
+ `<tr><td>Bob</td><td>25</td></tr></table>`
+ td := mustParse(t, html)
+ if td.HeaderRows != 1 {
+ t.Fatalf("HeaderRows = %d, want 1", td.HeaderRows)
+ }
+ want := [][]string{
+ {"Name", "Age"},
+ {"Alice", "30"},
+ {"Bob", "25"},
+ }
+ assertRowsEqual(t, td.Rows, want)
+}
+
+func TestParseTableHTMLMultipleHeaderRows(t *testing.T) {
+ html := `<table><tr><th>H1</th></tr><tr><th>H2</th></tr>` +
+ `<tr><td>a</td></tr></table>`
+ td := mustParse(t, html)
+ if td.HeaderRows != 2 {
+ t.Fatalf("HeaderRows = %d, want 2", td.HeaderRows)
+ }
+}
+
+func TestParseTableHTMLNoThTreatsFirstRowAsHeader(t *testing.T) {
+ html := `<table><tr><td>Name</td><td>Age</td></tr>` +
+ `<tr><td>Alice</td><td>30</td></tr></table>`
+ td := mustParse(t, html)
+ if td.HeaderRows != 1 {
+ t.Fatalf("HeaderRows = %d, want 1 (no <th> => first row is header)", td.HeaderRows)
+ }
+}
+
+func TestParseTableHTMLEmpty(t *testing.T) {
+ td := mustParse(t, `<table></table>`)
+ if len(td.Rows) != 0 {
+ t.Fatalf("Rows = %v, want empty", td.Rows)
+ }
+ if td.HeaderRows != 0 {
+ t.Fatalf("HeaderRows = %d, want 0", td.HeaderRows)
+ }
+}
+
+func TestParseTableHTMLTrimsCellWhitespace(t *testing.T) {
+ html := `<table><tr><td> spaced </td></tr></table>`
+ td := mustParse(t, html)
+ if got := td.Rows[0][0]; got != "spaced" {
+ t.Fatalf("cell = %q, want %q", got, "spaced")
+ }
+}
+
+func TestParseTableHTMLBrBecomesNewline(t *testing.T) {
+ html := `<table><tr><td>line1<br>line2</td></tr></table>`
+ td := mustParse(t, html)
+ if got := td.Rows[0][0]; got != "line1\nline2" {
+ t.Fatalf("cell = %q, want %q", got, "line1\nline2")
+ }
+}
+
+func TestParseTableHTMLInertElementsIgnored(t *testing.T) {
+ // script/style/template content must not appear in any cell.
+ html := `<table><tr><td>keep<script>alert(1)</script>middle` +
+ `<style>.x{}</style>end</td></tr></table>`
+ td := mustParse(t, html)
+ got := td.Rows[0][0]
+ if strings.Contains(got, "alert") || strings.Contains(got, ".x{}") {
+ t.Fatalf("inert content leaked into cell: %q", got)
+ }
+ if !strings.Contains(got, "keep") || !strings.Contains(got, "middle") || !strings.Contains(got, "end") {
+ t.Fatalf("visible text lost: %q", got)
+ }
+}
+
+func TestParseTableHTMLNestedTableFoldedIntoCell(t *testing.T) {
+ html := `<table><tr><td>outer<table><tr><td>inner</td></tr></table></td></tr></table>`
+ td := mustParse(t, html)
+ // Outer <tr> yields exactly one row with one cell; the nested <tr> must
+ // NOT become a separate row.
+ if len(td.Rows) != 1 {
+ t.Fatalf("len(Rows) = %d, want 1 (nested tr not a row)", len(td.Rows))
+ }
+ got := td.Rows[0][0]
+ if !strings.Contains(got, "outer") || !strings.Contains(got, "inner") {
+ t.Fatalf("nested table text not folded: %q", got)
+ }
+ if strings.Contains(got, "<table") {
+ t.Fatalf("nested markup not stripped: %q", got)
+ }
+}
+
+func TestParseTableHTMLBareTrWrapped(t *testing.T) {
+ // Legacy behaviour: a bare <tr> fragment without <table> still parses.
+ html := `<tr><td>a</td><td>b</td></tr>`
+ td := mustParse(t, html)
+ assertRowsEqual(t, td.Rows, [][]string{{"a", "b"}})
+}
+
+func TestParseTableHTMLColspanIgnoredLikeLegacy(t *testing.T) {
+ // Legacy tableRowsWithHeader ignores colspan/rowspan and just collects
+ // each <td>/<th> as one cell; a spanned cell keeps its own text.
+ html := `<table><tr><td colspan="2">wide</td><td>narrow</td></tr></table>`
+ td := mustParse(t, html)
+ assertRowsEqual(t, td.Rows, [][]string{{"wide", "narrow"}})
+}
+
+func TestRenderTableHTMLRoundTrip(t *testing.T) {
+ td := &entity.TableData{
+ Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}},
+ HeaderRows: 1,
+ Caption: "People",
+ }
+ rendered := RenderTableHTML(td)
+ back := mustParse(t, rendered)
+ assertRowsEqual(t, back.Rows, td.Rows)
+ if back.HeaderRows != td.HeaderRows {
+ t.Fatalf("HeaderRows round-trip = %d, want %d", back.HeaderRows, td.HeaderRows)
+ }
+ if back.Caption != td.Caption {
+ t.Fatalf("Caption round-trip = %q, want %q", back.Caption, td.Caption)
+ }
+}
+
+func TestRenderTableTextHasNoTags(t *testing.T) {
+ td := &entity.TableData{
+ Rows: [][]string{{"Name", "Age"}, {"Alice", "30"}},
+ HeaderRows: 1,
+ Caption: "People",
+ }
+ got := RenderTableText(td)
+ if strings.ContainsAny(got, "<>") {
+ t.Fatalf("RenderTableText contains tags: %q", got)
+ }
+ if !strings.Contains(got, "Name") || !strings.Contains(got, "Alice") {
+ t.Fatalf("RenderTableText lost cell text: %q", got)
+ }
+}
+
+func assertRowsEqual(t *testing.T, got, want [][]string) {
+ t.Helper()
+ if len(got) != len(want) {
+ t.Fatalf("row count = %d, want %d\n got=%v\nwant=%v", len(got), len(want), got, want)
+ }
+ for i := range want {
+ if len(got[i]) != len(want[i]) {
+ t.Fatalf("row %d cell count = %d, want %d\n got=%v\nwant=%v", i, len(got[i]), len(want[i]), got[i], want[i])
+ }
+ for j := range want[i] {
+ if got[i][j] != want[i][j] {
+ t.Fatalf("cell [%d][%d] = %q, want %q", i, j, got[i][j], want[i][j])
+ }
+ }
+ }
+}
diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.go
index 551c968316..faba6b6769 100644
--- a/internal/service/file/file_content.go
+++ b/internal/service/file/file_content.go
@@ -8,6 +8,7 @@ import (
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/parser/parser"
+ "ragflow/internal/parser/tableutil"
"ragflow/internal/storage"
"ragflow/internal/utility"
"strings"
@@ -239,6 +240,10 @@ func parseResultText(res parser.ParseResult) (string, error) {
parts = append(parts, text)
continue
}
+ if td, ok := item["table"].(*entity.TableData); ok && td != nil {
+ parts = append(parts, tableutil.RenderTableHTML(td))
+ continue
+ }
raw, err := json.Marshal(item)
if err != nil {
return "", err