Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): attach positions on TCADP spreadsheet JSON items (#19555)

li8728fad60d33403
diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.py
index b6e18ce6d9..7a8a00cff1 100644
--- a/rag/flow/parser/parser.py
+++ b/rag/flow/parser/parser.py
@@ -55,6 +55,7 @@ from rag.flow.parser.pdf_chunk_metadata import (
reorder_multi_column_bboxes,
)
from rag.flow.parser.schema import ParserFromUpstream
+from rag.flow.parser.spreadsheet_positions import TCADP_POSITION_TAG_RE, tcadp_spreadsheet_json_items
from rag.flow.parser.utils import (
enhance_media_sections_with_vision,
extract_word_outlines,
@@ -634,7 +635,7 @@ class Parser(ProcessBase):
bboxes = []
for section, position_tag in sections:
if position_tag:
- match = re.match(r"@@([0-9-]+)\t([0-9.]+)\t([0-9.]+)\t([0-9.]+)\t([0-9.]+)##", position_tag)
+ match = TCADP_POSITION_TAG_RE.match(position_tag)
if match:
pn, x0, x1, top, bott = match.groups()
bboxes.append(
@@ -846,23 +847,7 @@ class Parser(ProcessBase):
self.set_output("html", html_content)
elif output_format == "json":
- # For JSON output, create a list of text items
- result = []
- # Add sections as text
- for section, position_tag in sections:
- if section:
- result.append({"text": section, "doc_type_kwd": "text"})
- # Add tables as text
- for table in tables:
- if table:
- result.append(
- {
- "text": table,
- "doc_type_kwd": "text" if flatten_media_to_text else "table",
- }
- )
-
- self.set_output("json", result)
+ self.set_output("json", tcadp_spreadsheet_json_items(sections, tables, flatten_media_to_text))
elif output_format == "markdown":
# For markdown output, combine into markdown
diff --git a/rag/flow/parser/spreadsheet_positions.py b/rag/flow/parser/spreadsheet_positions.py
new file mode 100644
index 0000000000..c721358e5f
--- /dev/null
+++ b/rag/flow/parser/spreadsheet_positions.py
@@ -0,0 +1,97 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import re
+
+# @@page\tleft\tright\ttop\tbottom##; shared with the PDF TCADP path in parser.py.
+TCADP_POSITION_TAG_RE = re.compile(r"@@([0-9-]+)\t([0-9.]+)\t([0-9.]+)\t([0-9.]+)\t([0-9.]+)##")
+_TABLE_ROW_RE = re.compile(r"<tr\b[^>]*>(.*?)</tr>", flags=re.IGNORECASE | re.DOTALL)
+_TABLE_CELL_RE = re.compile(r"<t[dh]\b([^>]*)>", flags=re.IGNORECASE)
+_COLSPAN_RE = re.compile(r"""colspan\s*=\s*(?:["']\s*)?(\d+)""", flags=re.IGNORECASE)
+
+
+def spreadsheet_positions_from_tcadp_tag(position_tag):
+ """Map a TCADP position tag onto DeepDOC spreadsheet coordinates.
+
+ DeepDOC JSON items store a 0-based sheet index as the first component so
+ ``add_positions`` can persist a 1-based sheet. TCADP tags use a 1-based page.
+ Remaining numbers are reused as row/col when the tag carries them.
+ """
+ fallback = [[0, 1, 1, 1, 1]]
+ if not position_tag:
+ return fallback
+ match = TCADP_POSITION_TAG_RE.match(str(position_tag).strip())
+ if not match:
+ return fallback
+ pn, left, right, top, bottom = match.groups()
+ sheet = max(int(pn.split("-")[0]) - 1, 0)
+ return [[sheet, int(float(left)), int(float(right)), int(float(top)), int(float(bottom))]]
+
+
+def _cell_colspan(attrs):
+ """Visual column span for a cell; absent or invalid colspan counts as 1."""
+ match = _COLSPAN_RE.search(attrs or "")
+ if not match:
+ return 1
+ span = int(match.group(1))
+ return span if span > 0 else 1
+
+
+def _row_visual_width(row_html):
+ """Visual column count for a table row, summing each cell's colspan."""
+ return sum(_cell_colspan(attrs) for attrs in _TABLE_CELL_RE.findall(row_html))
+
+
+def html_table_row_col_span(html):
+ """Return 1-based row/col counts from a TCADP HTML table, defaulting to 1x1."""
+ if not isinstance(html, str) or not html.strip():
+ return 1, 1
+ n_rows = len(re.findall(r"<tr\b", html, flags=re.IGNORECASE))
+ n_cols = 0
+ for row_html in _TABLE_ROW_RE.findall(html):
+ n_cols = max(n_cols, _row_visual_width(row_html))
+ return max(n_rows, 1), max(n_cols, 1)
+
+
+def spreadsheet_positions_from_tcadp_table(sheet_idx, table_html):
+ """Attach at least a sheet index; use the HTML row/col span when present."""
+ n_rows, n_cols = html_table_row_col_span(table_html)
+ return [[sheet_idx, 1, n_rows, 1, n_cols]]
+
+
+def tcadp_spreadsheet_json_items(sections, tables, flatten_media_to_text=False):
+ """Turn mocked TCADP ``(sections, tables)`` into JSON items with ``positions``."""
+ result = []
+ for section, position_tag in sections:
+ if section:
+ result.append(
+ {
+ "text": section,
+ "doc_type_kwd": "text",
+ "positions": spreadsheet_positions_from_tcadp_tag(position_tag),
+ }
+ )
+ sheet_idx = 0
+ for table in tables:
+ if table:
+ result.append(
+ {
+ "text": table,
+ "doc_type_kwd": "text" if flatten_media_to_text else "table",
+ "positions": spreadsheet_positions_from_tcadp_table(sheet_idx, table),
+ }
+ )
+ sheet_idx += 1
+ return result
diff --git a/test/unit_test/rag/flow/parser/test_spreadsheet_tcadp_positions.py b/test/unit_test/rag/flow/parser/test_spreadsheet_tcadp_positions.py
new file mode 100644
index 0000000000..aa2b27d27d
--- /dev/null
+++ b/test/unit_test/rag/flow/parser/test_spreadsheet_tcadp_positions.py
@@ -0,0 +1,128 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import ast
+from pathlib import Path
+from unittest.mock import Mock
+
+import pytest
+
+from rag.flow.parser.spreadsheet_positions import (
+ spreadsheet_positions_from_tcadp_table,
+ spreadsheet_positions_from_tcadp_tag,
+ tcadp_spreadsheet_json_items,
+)
+
+REPO_ROOT = Path(__file__).resolve().parents[5]
+
+_TABLE_HTML = "<table>\n <tr>\n <th>Name</th>\n <th>Qty</th>\n </tr>\n <tr>\n <td>A</td>\n <td>1</td>\n </tr>\n <tr>\n <td>B</td>\n <td>2</td>\n </tr>\n</table>"
+
+_MOCK_SECTIONS = [
+ ("Intro text", "@@2\t3\t5\t1\t4##"),
+ ("Dummy tagged", "@@1\t0.0\t1000.0\t0.0\t100.0##"),
+ ("Untagged", ""),
+]
+_MOCK_TABLES = [_TABLE_HTML, "<table><tr><td>only</td></tr></table>"]
+
+
+def _assert_positions(item):
+ assert "positions" in item, f"missing positions on {item!r}"
+ positions = item["positions"]
+ assert isinstance(positions, list) and positions, f"empty positions on {item!r}"
+ coord = positions[0]
+ assert len(coord) == 5, f"expected 5-tuple positions, got {coord!r}"
+ assert all(isinstance(n, int) for n in coord), f"non-int positions {coord!r}"
+ assert coord[0] >= 0
+
+
+def test_spreadsheet_positions_from_tcadp_tag():
+ assert spreadsheet_positions_from_tcadp_tag("@@2\t3\t5\t1\t4##") == [[1, 3, 5, 1, 4]]
+ assert spreadsheet_positions_from_tcadp_tag("@@1\t0.0\t1000.0\t0.0\t100.0##") == [[0, 0, 1000, 0, 100]]
+ assert spreadsheet_positions_from_tcadp_tag("@@1-3\t2\t4\t1\t2##") == [[0, 2, 4, 1, 2]]
+ assert spreadsheet_positions_from_tcadp_tag("") == [[0, 1, 1, 1, 1]]
+ assert spreadsheet_positions_from_tcadp_tag("not-a-tag") == [[0, 1, 1, 1, 1]]
+
+
+def test_spreadsheet_positions_from_tcadp_table_uses_html_span():
+ assert spreadsheet_positions_from_tcadp_table(0, _TABLE_HTML) == [[0, 1, 3, 1, 2]]
+ assert spreadsheet_positions_from_tcadp_table(1, "not html") == [[1, 1, 1, 1, 1]]
+
+
+def test_spreadsheet_positions_from_tcadp_table_uses_max_colspan_width():
+ merged_then_wider = "<table><tr><td colspan='2'>merged header</td></tr><tr><td>a</td><td>b</td><td>c</td></tr></table>"
+ assert spreadsheet_positions_from_tcadp_table(0, merged_then_wider) == [[0, 1, 2, 1, 3]]
+
+ invalid_colspan = "<table><tr><td colspan='x'>a</td><td colspan='0'>b</td></tr></table>"
+ assert spreadsheet_positions_from_tcadp_table(0, invalid_colspan) == [[0, 1, 1, 1, 2]]
+
+
+def test_tcadp_spreadsheet_json_items_include_positions():
+ tcadp = Mock()
+ tcadp.parse_pdf.return_value = (_MOCK_SECTIONS, _MOCK_TABLES)
+ sections, tables = tcadp.parse_pdf(filepath="book.xlsx", binary=b"xlsx", callback=None, file_type="XLSX")
+
+ items = tcadp_spreadsheet_json_items(sections, tables)
+
+ assert len(items) == 5
+ for item in items:
+ _assert_positions(item)
+
+ assert items[0]["text"] == "Intro text"
+ assert items[0]["doc_type_kwd"] == "text"
+ assert items[0]["positions"] == [[1, 3, 5, 1, 4]]
+
+ assert items[1]["positions"] == [[0, 0, 1000, 0, 100]]
+ assert items[2]["positions"] == [[0, 1, 1, 1, 1]]
+
+ assert items[3]["doc_type_kwd"] == "table"
+ assert items[3]["positions"] == [[0, 1, 3, 1, 2]]
+ assert items[4]["positions"] == [[1, 1, 1, 1, 1]]
+
+ # add_positions (task_executor / dataflow_service) stores pn+1.
+ sheet, r1, r2, c1, c2 = items[0]["positions"][0]
+ assert (sheet + 1, r1, r2, c1, c2) == (2, 3, 5, 1, 4)
+
+
+def test_tcadp_spreadsheet_flatten_media_still_keeps_positions():
+ items = tcadp_spreadsheet_json_items(
+ [("note", "@@1\t2\t2\t1\t1##")],
+ [_TABLE_HTML],
+ flatten_media_to_text=True,
+ )
+ assert [item["doc_type_kwd"] for item in items] == ["text", "text"]
+ for item in items:
+ _assert_positions(item)
+ assert items[1]["positions"] == [[0, 1, 3, 1, 2]]
+
+
+def test_parser_spreadsheet_json_branch_calls_tcadp_spreadsheet_json_items():
+ tree = ast.parse((REPO_ROOT / "rag/flow/parser/parser.py").read_text())
+ spreadsheet = None
+ for node in tree.body:
+ if isinstance(node, ast.ClassDef) and node.name == "Parser":
+ for item in node.body:
+ if isinstance(item, ast.FunctionDef) and item.name == "_spreadsheet":
+ spreadsheet = item
+ break
+ assert spreadsheet is not None
+
+ calls = [node for node in ast.walk(spreadsheet) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "tcadp_spreadsheet_json_items"]
+ assert len(calls) == 1, "Parser._spreadsheet JSON path must attach positions via tcadp_spreadsheet_json_items"