RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): attach positions on TCADP spreadsheet JSON items (#19555)
li8728fad60d33403
diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.pyindex b6e18ce6d9..7a8a00cff1 100644--- a/rag/flow/parser/parser.py+++ b/rag/flow/parser/parser.py@@ -55,6 +55,7 @@ from rag.flow.parser.pdf_chunk_metadata import (reorder_multi_column_bboxes,)from rag.flow.parser.schema import ParserFromUpstream+from rag.flow.parser.spreadsheet_positions import TCADP_POSITION_TAG_RE, tcadp_spreadsheet_json_itemsfrom rag.flow.parser.utils import (enhance_media_sections_with_vision,extract_word_outlines,@@ -634,7 +635,7 @@ class Parser(ProcessBase):bboxes = []for section, position_tag in sections:if position_tag:- match = re.match(r"@@([0-9-]+)\t([0-9.]+)\t([0-9.]+)\t([0-9.]+)\t([0-9.]+)##", position_tag)+ match = TCADP_POSITION_TAG_RE.match(position_tag)if match:pn, x0, x1, top, bott = match.groups()bboxes.append(@@ -846,23 +847,7 @@ class Parser(ProcessBase):self.set_output("html", html_content)elif output_format == "json":- # For JSON output, create a list of text items- result = []- # Add sections as text- for section, position_tag in sections:- if section:- result.append({"text": section, "doc_type_kwd": "text"})- # Add tables as text- for table in tables:- if table:- result.append(- {- "text": table,- "doc_type_kwd": "text" if flatten_media_to_text else "table",- }- )-- self.set_output("json", result)+ self.set_output("json", tcadp_spreadsheet_json_items(sections, tables, flatten_media_to_text))elif output_format == "markdown":# For markdown output, combine into markdowndiff --git a/rag/flow/parser/spreadsheet_positions.py b/rag/flow/parser/spreadsheet_positions.pynew file mode 100644index 0000000000..c721358e5f--- /dev/null+++ b/rag/flow/parser/spreadsheet_positions.py@@ -0,0 +1,97 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import re++# @@page\tleft\tright\ttop\tbottom##; shared with the PDF TCADP path in parser.py.+TCADP_POSITION_TAG_RE = re.compile(r"@@([0-9-]+)\t([0-9.]+)\t([0-9.]+)\t([0-9.]+)\t([0-9.]+)##")+_TABLE_ROW_RE = re.compile(r"<tr\b[^>]*>(.*?)</tr>", flags=re.IGNORECASE | re.DOTALL)+_TABLE_CELL_RE = re.compile(r"<t[dh]\b([^>]*)>", flags=re.IGNORECASE)+_COLSPAN_RE = re.compile(r"""colspan\s*=\s*(?:["']\s*)?(\d+)""", flags=re.IGNORECASE)+++def spreadsheet_positions_from_tcadp_tag(position_tag):+ """Map a TCADP position tag onto DeepDOC spreadsheet coordinates.++ DeepDOC JSON items store a 0-based sheet index as the first component so+ ``add_positions`` can persist a 1-based sheet. TCADP tags use a 1-based page.+ Remaining numbers are reused as row/col when the tag carries them.+ """+ fallback = [[0, 1, 1, 1, 1]]+ if not position_tag:+ return fallback+ match = TCADP_POSITION_TAG_RE.match(str(position_tag).strip())+ if not match:+ return fallback+ pn, left, right, top, bottom = match.groups()+ sheet = max(int(pn.split("-")[0]) - 1, 0)+ return [[sheet, int(float(left)), int(float(right)), int(float(top)), int(float(bottom))]]+++def _cell_colspan(attrs):+ """Visual column span for a cell; absent or invalid colspan counts as 1."""+ match = _COLSPAN_RE.search(attrs or "")+ if not match:+ return 1+ span = int(match.group(1))+ return span if span > 0 else 1+++def _row_visual_width(row_html):+ """Visual column count for a table row, summing each cell's colspan."""+ return sum(_cell_colspan(attrs) for attrs in _TABLE_CELL_RE.findall(row_html))+++def html_table_row_col_span(html):+ """Return 1-based row/col counts from a TCADP HTML table, defaulting to 1x1."""+ if not isinstance(html, str) or not html.strip():+ return 1, 1+ n_rows = len(re.findall(r"<tr\b", html, flags=re.IGNORECASE))+ n_cols = 0+ for row_html in _TABLE_ROW_RE.findall(html):+ n_cols = max(n_cols, _row_visual_width(row_html))+ return max(n_rows, 1), max(n_cols, 1)+++def spreadsheet_positions_from_tcadp_table(sheet_idx, table_html):+ """Attach at least a sheet index; use the HTML row/col span when present."""+ n_rows, n_cols = html_table_row_col_span(table_html)+ return [[sheet_idx, 1, n_rows, 1, n_cols]]+++def tcadp_spreadsheet_json_items(sections, tables, flatten_media_to_text=False):+ """Turn mocked TCADP ``(sections, tables)`` into JSON items with ``positions``."""+ result = []+ for section, position_tag in sections:+ if section:+ result.append(+ {+ "text": section,+ "doc_type_kwd": "text",+ "positions": spreadsheet_positions_from_tcadp_tag(position_tag),+ }+ )+ sheet_idx = 0+ for table in tables:+ if table:+ result.append(+ {+ "text": table,+ "doc_type_kwd": "text" if flatten_media_to_text else "table",+ "positions": spreadsheet_positions_from_tcadp_table(sheet_idx, table),+ }+ )+ sheet_idx += 1+ return resultdiff --git a/test/unit_test/rag/flow/parser/test_spreadsheet_tcadp_positions.py b/test/unit_test/rag/flow/parser/test_spreadsheet_tcadp_positions.pynew file mode 100644index 0000000000..aa2b27d27d--- /dev/null+++ b/test/unit_test/rag/flow/parser/test_spreadsheet_tcadp_positions.py@@ -0,0 +1,128 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import ast+from pathlib import Path+from unittest.mock import Mock++import pytest++from rag.flow.parser.spreadsheet_positions import (+ spreadsheet_positions_from_tcadp_table,+ spreadsheet_positions_from_tcadp_tag,+ tcadp_spreadsheet_json_items,+)++REPO_ROOT = Path(__file__).resolve().parents[5]++_TABLE_HTML = "<table>\n <tr>\n <th>Name</th>\n <th>Qty</th>\n </tr>\n <tr>\n <td>A</td>\n <td>1</td>\n </tr>\n <tr>\n <td>B</td>\n <td>2</td>\n </tr>\n</table>"++_MOCK_SECTIONS = [+ ("Intro text", "@@2\t3\t5\t1\t4##"),+ ("Dummy tagged", "@@1\t0.0\t1000.0\t0.0\t100.0##"),+ ("Untagged", ""),+]+_MOCK_TABLES = [_TABLE_HTML, "<table><tr><td>only</td></tr></table>"]+++def _assert_positions(item):+ assert "positions" in item, f"missing positions on {item!r}"+ positions = item["positions"]+ assert isinstance(positions, list) and positions, f"empty positions on {item!r}"+ coord = positions[0]+ assert len(coord) == 5, f"expected 5-tuple positions, got {coord!r}"+ assert all(isinstance(n, int) for n in coord), f"non-int positions {coord!r}"+ assert coord[0] >= 0+++def test_spreadsheet_positions_from_tcadp_tag():+ assert spreadsheet_positions_from_tcadp_tag("@@2\t3\t5\t1\t4##") == [[1, 3, 5, 1, 4]]+ assert spreadsheet_positions_from_tcadp_tag("@@1\t0.0\t1000.0\t0.0\t100.0##") == [[0, 0, 1000, 0, 100]]+ assert spreadsheet_positions_from_tcadp_tag("@@1-3\t2\t4\t1\t2##") == [[0, 2, 4, 1, 2]]+ assert spreadsheet_positions_from_tcadp_tag("") == [[0, 1, 1, 1, 1]]+ assert spreadsheet_positions_from_tcadp_tag("not-a-tag") == [[0, 1, 1, 1, 1]]+++def test_spreadsheet_positions_from_tcadp_table_uses_html_span():+ assert spreadsheet_positions_from_tcadp_table(0, _TABLE_HTML) == [[0, 1, 3, 1, 2]]+ assert spreadsheet_positions_from_tcadp_table(1, "not html") == [[1, 1, 1, 1, 1]]+++def test_spreadsheet_positions_from_tcadp_table_uses_max_colspan_width():+ merged_then_wider = "<table><tr><td colspan='2'>merged header</td></tr><tr><td>a</td><td>b</td><td>c</td></tr></table>"+ assert spreadsheet_positions_from_tcadp_table(0, merged_then_wider) == [[0, 1, 2, 1, 3]]++ invalid_colspan = "<table><tr><td colspan='x'>a</td><td colspan='0'>b</td></tr></table>"+ assert spreadsheet_positions_from_tcadp_table(0, invalid_colspan) == [[0, 1, 1, 1, 2]]+++def test_tcadp_spreadsheet_json_items_include_positions():+ tcadp = Mock()+ tcadp.parse_pdf.return_value = (_MOCK_SECTIONS, _MOCK_TABLES)+ sections, tables = tcadp.parse_pdf(filepath="book.xlsx", binary=b"xlsx", callback=None, file_type="XLSX")++ items = tcadp_spreadsheet_json_items(sections, tables)++ assert len(items) == 5+ for item in items:+ _assert_positions(item)++ assert items[0]["text"] == "Intro text"+ assert items[0]["doc_type_kwd"] == "text"+ assert items[0]["positions"] == [[1, 3, 5, 1, 4]]++ assert items[1]["positions"] == [[0, 0, 1000, 0, 100]]+ assert items[2]["positions"] == [[0, 1, 1, 1, 1]]++ assert items[3]["doc_type_kwd"] == "table"+ assert items[3]["positions"] == [[0, 1, 3, 1, 2]]+ assert items[4]["positions"] == [[1, 1, 1, 1, 1]]++ # add_positions (task_executor / dataflow_service) stores pn+1.+ sheet, r1, r2, c1, c2 = items[0]["positions"][0]+ assert (sheet + 1, r1, r2, c1, c2) == (2, 3, 5, 1, 4)+++def test_tcadp_spreadsheet_flatten_media_still_keeps_positions():+ items = tcadp_spreadsheet_json_items(+ [("note", "@@1\t2\t2\t1\t1##")],+ [_TABLE_HTML],+ flatten_media_to_text=True,+ )+ assert [item["doc_type_kwd"] for item in items] == ["text", "text"]+ for item in items:+ _assert_positions(item)+ assert items[1]["positions"] == [[0, 1, 3, 1, 2]]+++def test_parser_spreadsheet_json_branch_calls_tcadp_spreadsheet_json_items():+ tree = ast.parse((REPO_ROOT / "rag/flow/parser/parser.py").read_text())+ spreadsheet = None+ for node in tree.body:+ if isinstance(node, ast.ClassDef) and node.name == "Parser":+ for item in node.body:+ if isinstance(item, ast.FunctionDef) and item.name == "_spreadsheet":+ spreadsheet = item+ break+ assert spreadsheet is not None++ calls = [node for node in ast.walk(spreadsheet) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "tcadp_spreadsheet_json_items"]+ assert len(calls) == 1, "Parser._spreadsheet JSON path must attach positions via tcadp_spreadsheet_json_items"