Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): emit xlsx spreadsheets as typed table chunks (#19509)

xugangqiang7c99668aee69
diff --git a/deepdoc/parser/excel_parser.py b/deepdoc/parser/excel_parser.py
index bfa688b26f..84d14e2222 100644
--- a/deepdoc/parser/excel_parser.py
+++ b/deepdoc/parser/excel_parser.py
@@ -231,6 +231,14 @@ class RAGFlowExcelParser:
# when the data-row count is an exact multiple of chunk_rows and emits
# a spurious header-only chunk.
n_data_rows = len(rows) - 1
+ if n_data_rows <= 0:
+ # A template sheet holds only its header row. Emit it as a
+ # captioned table instead of dropping the sheet, which is what
+ # Go does (recordsToHTMLTableChunkList, nData == 0). Without
+ # this the column schema of a blank template is lost.
+ tb = f"<table><caption>{sheetname}</caption>" + tb_rows_0 + "</table>"
+ tb_chunks.append((tb, (sheet_idx, 1, 1, 1, col_max)))
+ continue
for chunk_i in range((n_data_rows + chunk_rows - 1) // chunk_rows):
row_start = 2 + chunk_i * chunk_rows
row_end = min(1 + (chunk_i + 1) * chunk_rows, len(rows))
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index fd4b81f356..91cb78a25a 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -38,7 +38,13 @@ var tableIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`
// value parsing.
var numericCellRe = regexp.MustCompile(`^[\$\+\-]?[\d,]+(\.\d+)?%?$`)
-const defaultTableChunkRows = 256
+// defaultTableChunkRows is the row ceiling for one spreadsheet <table>. It is
+// deliberately far beyond any real sheet: a sheet is emitted as ONE
+// self-contained <table> and is never split by row count. Any split would cut
+// inside <td> content or, worse, drop the delimiter characters it cut on —
+// the bug the typed table item exists to prevent. Keep in sync with Python's
+// TABLE_NO_SPLIT_ROWS (rag/flow/parser/parser.py).
+const defaultTableChunkRows = 1 << 30
// extractXLSXImages returns the floating and in-cell images anchored to a
// worksheet as structured parser items. Excelize exposes both kinds through
diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.go
index 1407301720..4b6efc1e1c 100644
--- a/internal/parser/parser/office_table_render_test.go
+++ b/internal/parser/parser/office_table_render_test.go
@@ -128,6 +128,28 @@ func TestRecordsToHTMLTableChunkList_Alignment(t *testing.T) {
}
}
+// TestRecordsToHTMLTableChunkList_DefaultCeilingDoesNotSplit pins the default row
+// ceiling: a sheet is emitted as ONE self-contained <table>. Splitting it would
+// cut inside <td> content or drop the delimiter characters it cut on, and the
+// value must stay in sync with Python's TABLE_NO_SPLIT_ROWS.
+func TestRecordsToHTMLTableChunkList_DefaultCeilingDoesNotSplit(t *testing.T) {
+ const dataRows = 5000
+ records := make([][]string, 0, dataRows+1)
+ records = append(records, []string{"C1", "C2"})
+ for i := 0; i < dataRows; i++ {
+ records = append(records, []string{"x", "y"})
+ }
+
+ chunks := recordsToHTMLTableChunkList(records, defaultTableChunkRows, "S", 1)
+ if len(chunks) != 1 {
+ t.Fatalf("want 1 chunk with the default ceiling, got %d", len(chunks))
+ }
+ // The single chunk spans every data row.
+ if chunks[0].RowStart != 2 || chunks[0].RowEnd != dataRows+1 {
+ t.Fatalf("want rows 2..%d, got %d..%d", dataRows+1, chunks[0].RowStart, chunks[0].RowEnd)
+ }
+}
+
// TestRecordsToHTMLTableChunkList_Chunking asserts 256-row chunking with a repeated
// header (ceil(n_data / chunk_rows) chunks).
func TestRecordsToHTMLTableChunkList_Chunking(t *testing.T) {
diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.py
index 367f73f040..b6e18ce6d9 100644
--- a/rag/flow/parser/parser.py
+++ b/rag/flow/parser/parser.py
@@ -68,6 +68,14 @@ from rag.flow.parser.utils import (
from rag.llm.cv_model import Base as VLM
from rag.utils.base64_image import image2id
+# Row ceiling passed to ``ExcelParser.html`` for a spreadsheet sheet. It is
+# deliberately far beyond any real sheet: a sheet is emitted as ONE
+# self-contained <table> and is never split by row count or token budget.
+# Any split would cut inside ``<td>`` content or, worse, drop the delimiter
+# characters it cut on — the bug this typed parser output exists to prevent.
+# The TCADP path applies the same rule and emits one item per returned table.
+TABLE_NO_SPLIT_ROWS = 1 << 30
+
class ParserParam(ProcessParamBase):
def __init__(self):
@@ -137,7 +145,7 @@ class ParserParam(ProcessParamBase):
"spreadsheet": {
"parse_method": "deepdoc", # deepdoc/tcadp_parser
"flatten_media_to_text": False,
- "output_format": "html",
+ "output_format": "json",
"suffix": [
"xls",
"xlsx",
@@ -874,18 +882,21 @@ class Parser(ProcessBase):
htmls = spreadsheet_parser.html(blob, 1000000000)
self.set_output("html", htmls[0][0] if htmls else "")
elif conf.get("output_format") == "json":
+ # One self-contained <table> item per sheet, never split, so the
+ # downstream TokenChunker keeps every table whole instead of
+ # cutting it on a delimiter.
self.set_output(
"json",
[
{
- "text": txt,
- "doc_type_kwd": "text",
+ "text": tb,
+ "doc_type_kwd": "text" if flatten_media_to_text else "table",
# 0-based sheet. TaskExecutor and dataflow_service
# call add_positions, which stores pn+1 (1-based).
"positions": [[sheet, r1, r2, c1, c2]],
}
- for txt, (sheet, r1, r2, c1, c2) in spreadsheet_parser(blob)
- if txt
+ for tb, (sheet, r1, r2, c1, c2) in spreadsheet_parser.html(blob, TABLE_NO_SPLIT_ROWS)
+ if tb
],
)
elif conf.get("output_format") == "markdown":
diff --git a/test/unit_test/rag/flow/parser/test_spreadsheet_output.py b/test/unit_test/rag/flow/parser/test_spreadsheet_output.py
new file mode 100644
index 0000000000..c822037d26
--- /dev/null
+++ b/test/unit_test/rag/flow/parser/test_spreadsheet_output.py
@@ -0,0 +1,230 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Spreadsheet parsing must hand the chunker typed, self-contained tables.
+
+Regression guard for "TokenChunker shreds xlsx tables": the spreadsheet parser
+used to emit one *text* item per row (or a single raw HTML string under
+``output_format=html``), so the TokenChunker cut tables in the middle of
+``<td>`` text and dropped the delimiter characters it cut on.
+
+The parser now emits one ``doc_type_kwd="table"`` item per sheet chunk (Go's
+``defaultTableChunkRows`` = 256), each carrying its own caption and header row,
+and the TokenChunker keeps those items whole.
+"""
+
+import importlib
+import io
+import sys
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+from openpyxl import Workbook
+
+DEFAULT_SETUP = {
+ "parse_method": "deepdoc",
+ "flatten_media_to_text": False,
+ "output_format": "json",
+ "suffix": ["xls", "xlsx", "csv"],
+}
+
+
+def flow_modules(monkeypatch):
+ """Import the flow modules the way the sibling parser test does.
+
+ The component registry imports ``deepdoc.parser.pdf_parser`` first and
+ leaves it half-initialized, so importing ``rag.flow.parser.parser``
+ directly raises ``cannot import name 'PlainParser'``. Drop the stale
+ entries and re-import ``deepdoc.parser.pdf_parser`` first.
+ """
+ for module_name in (
+ "rag.flow.parser.parser",
+ "rag.flow.chunker.token_chunker",
+ "deepdoc.parser.pdf_parser",
+ "deepdoc.parser",
+ "deepdoc",
+ ):
+ monkeypatch.delitem(sys.modules, module_name, raising=False)
+ importlib.invalidate_caches()
+ importlib.import_module("deepdoc.parser.pdf_parser")
+ return SimpleNamespace(
+ parser=importlib.import_module("rag.flow.parser.parser"),
+ chunker=importlib.import_module("rag.flow.chunker.token_chunker"),
+ delim=importlib.import_module("rag.nlp.delim"),
+ )
+
+
+def build_xlsx(sheets: dict[str, list[list]], pad_rows: int = 0) -> bytes:
+ """Build an xlsx with one sheet per entry, padded with ``pad_rows`` data rows."""
+ wb = Workbook()
+ wb.remove(wb.active)
+ for name, rows in sheets.items():
+ ws = wb.create_sheet(name)
+ for row in rows:
+ ws.append(row)
+ for i in range(pad_rows):
+ ws.append([f"数据{i}", f"部门{i % 3}", f"备注{i};说明。"])
+ buf = io.BytesIO()
+ wb.save(buf)
+ return buf.getvalue()
+
+
+def parse(flow_modules, blob: bytes, name: str = "sheet.xlsx", **overrides) -> dict:
+ """Run ``Parser._spreadsheet`` against a minimal process double."""
+ setup = {**DEFAULT_SETUP, **overrides}
+ process = SimpleNamespace(
+ _param=SimpleNamespace(setups={"spreadsheet": setup}),
+ _canvas=SimpleNamespace(_doc_id=None, _tenant_id="tenant", _language="English"),
+ callback=Mock(),
+ outputs={},
+ set_output=lambda k, v: process.outputs.__setitem__(k, v),
+ )
+ flow_modules.parser.Parser._spreadsheet(process, name, blob)
+ return process.outputs
+
+
+def test_spreadsheet_default_output_format_is_json(flow_modules):
+ # The default must be the structured format; "html" flattens a table into a
+ # raw string the chunker can only cut on character boundaries.
+ assert flow_modules.parser.ParserParam().setups["spreadsheet"]["output_format"] == "json"
+
+
+def test_spreadsheet_emits_one_typed_table_item_per_sheet(flow_modules):
+ blob = build_xlsx({"SheetA": [["姓名", "部门"], ["张三", "研发"]], "SheetB": [["产品", "数量"], ["A", 1]]})
+ items = parse(flow_modules, blob)["json"]
+
+ assert [i["doc_type_kwd"] for i in items] == ["table", "table"]
+ # Every sheet is emitted; the old html path silently kept only the first.
+ assert [i["positions"][0][0] for i in items] == [0, 1]
+ for item in items:
+ text = item["text"].strip()
+ assert text.startswith("<table>") and text.endswith("</table>")
+ assert "<caption>" in text
+
+
+def test_spreadsheet_table_chunk_is_self_contained(flow_modules):
+ # A chunk is retrieved and sent to the LLM on its own, so it must carry the
+ # header row that gives the columns their meaning.
+ blob = build_xlsx({"Big": [["列1", "列2", "列3"]]}, pad_rows=600)
+ items = parse(flow_modules, blob)["json"]
+
+ for item in items:
+ assert item["text"].count("<th>") == 3
+ assert item["doc_type_kwd"] == "table"
+
+
+def test_large_table_is_not_split(flow_modules):
+ """A sheet is one table chunk, however many rows it has."""
+ rows = [["h"]] + [[str(i)] for i in range(2000)]
+ items = parse(flow_modules, build_xlsx({"Narrow": rows}))["json"]
+
+ assert len(items) == 1
+ # The chunk spans every data row.
+ assert items[0]["positions"] == [[0, 2, len(rows), 1, 1]]
+
+
+def test_wide_table_is_not_split(flow_modules):
+ """Even a wide table stays a single chunk.
+
+ Splitting it would cut inside ``<td>`` content or drop the delimiter
+ characters on the cut, which is the bug this parser output fixes.
+ """
+ cols = 10
+ header = [f"列{c}" for c in range(cols)]
+ rows = [[f"单元格{r}-{c}描述文本。" for c in range(cols)] for r in range(300)]
+ items = parse(flow_modules, build_xlsx({"Wide": [header] + rows}))["json"]
+
+ assert len(items) == 1
+ assert items[0]["text"].count("<th>") == cols
+ # Punctuation inside cells must survive.
+ assert "。" in items[0]["text"]
+
+
+def test_header_only_sheet_still_emits_a_table(flow_modules):
+ """A template sheet with no data rows must not vanish.
+
+ ``ExcelParser.html`` computes the chunk count from the data-row count, so a
+ sheet holding only its header row used to yield zero chunks and disappear
+ silently. Go emits a captioned header-only table in that case
+ (``office_table_render.go``, ``nData == 0``).
+ """
+ blob = build_xlsx({"Template": [["姓名", "电话", "地址"]]})
+ items = parse(flow_modules, blob)["json"]
+
+ assert len(items) == 1
+ assert items[0]["doc_type_kwd"] == "table"
+ # Row 1 is the header row; there is no data row below it.
+ assert items[0]["positions"] == [[0, 1, 1, 1, 3]]
+ text = items[0]["text"].strip()
+ assert text.startswith("<table>") and text.endswith("</table>")
+ assert text.count("<th>") == 3
+
+
+def test_header_only_sheet_is_kept_next_to_a_data_sheet(flow_modules):
+ blob = build_xlsx({"Data": [["a", "b"], ["1", "2"]], "Empty": [["x", "y", "z"]]})
+ items = parse(flow_modules, blob)["json"]
+
+ assert len(items) == 2
+ # Both sheets survive, the empty one last.
+ assert [i["positions"][0][0] for i in items] == [0, 1]
+ assert all(i["doc_type_kwd"] == "table" for i in items)
+
+
+def test_spreadsheet_flatten_media_to_text_marks_tables_as_text(flow_modules):
+ blob = build_xlsx({"S": [["a", "b"], ["1", "2"]]})
+ items = parse(flow_modules, blob, flatten_media_to_text=True)["json"]
+
+ assert [i["doc_type_kwd"] for i in items] == ["text"]
+
+
[email protected](("suffix", "blob"), [("xlsx", None), ("csv", "姓名,部门\n张三,研发\n".encode())])
+def test_spreadsheet_supported_suffixes_produce_tables(flow_modules, suffix, blob):
+ if blob is None:
+ blob = build_xlsx({"S": [["姓名", "部门"], ["张三", "研发"]]})
+ items = parse(flow_modules, blob, name=f"sheet.{suffix}")["json"]
+
+ assert items and items[0]["doc_type_kwd"] == "table"
+
+
+async def test_token_chunker_keeps_spreadsheet_table_whole(flow_modules):
+ """The reported bug: delimiter chunking must not cut inside a table."""
+ blob = build_xlsx(
+ {
+ "Sheet1": [
+ ["问题", "回答"],
+ # Cells carrying the default delimiter chars and embedded newlines.
+ ["碳排放纳入名录是否公布?", "需要公布。根据《管理办法》\n由省级主管部门确定。"],
+ ["关停企业如何报告?", "向实际所在地报告;不能独立报告的随上级法人。"],
+ ]
+ }
+ )
+ parsed = parse(flow_modules, blob)["json"]
+
+ param = flow_modules.chunker.TokenChunkerParam() # current upstream default delimiters
+ assert param.delimiters == list(flow_modules.delim.DEFAULT_DELIMITER)
+ chunker = flow_modules.chunker.TokenChunker.__new__(flow_modules.chunker.TokenChunker)
+ chunker._canvas = SimpleNamespace(_doc_id=None, _tenant_id="tenant")
+ chunker._param = param
+ chunker.callback = Mock()
+
+ await chunker._invoke(name="sheet.xlsx", output_format="json", json_result=parsed)
+ chunks = [c["text"] for c in chunker._param.outputs["chunks"]["value"]]
+
+ assert len(chunks) == 1
+ text = chunks[0].strip()
+ assert text.startswith("<table>") and text.endswith("</table>")
+ # No delimiter character may be eaten by the chunker.
+ assert "。" in text and "?" in text and ";" in text
diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx
index 9bfe909627..e0dd0dd2f6 100644
--- a/web/src/pages/agent/constant/pipeline.tsx
+++ b/web/src/pages/agent/constant/pipeline.tsx
@@ -195,7 +195,7 @@ export const initialParserValues = {
},
{
fileFormat: FileType.Spreadsheet,
- output_format: SpreadsheetOutputFormat.Html,
+ output_format: SpreadsheetOutputFormat.Json,
parse_method: ParseDocumentType.DeepDOC,
preprocess: PreprocessValue.main_content,
flatten_media_to_text: false,