RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(deepdoc): harden opendataloader table-cell parsing against bad json (#17316)
Yurii2147622d6742ab5
diff --git a/deepdoc/parser/opendataloader_parser.py b/deepdoc/parser/opendataloader_parser.pyindex d591ad737f..21494de34f 100644--- a/deepdoc/parser/opendataloader_parser.py+++ b/deepdoc/parser/opendataloader_parser.py@@ -108,8 +108,16 @@ def _element_text(el: dict) -> str:for c in cells:if not isinstance(c, dict):continue- row = c.get("row") or c.get("row_index") or 0- rows.setdefault(int(row), []).append(str(c.get("content") or c.get("text") or ""))+ raw_row = c.get("row")+ if raw_row is None:+ raw_row = c.get("row_index")+ try:+ # a non-numeric row label must not abort the whole parse; also treat+ # row 0 as a real index rather than "missing" (the `or` chain did not).+ row_idx = int(raw_row) if raw_row is not None else 0+ except (TypeError, ValueError):+ row_idx = 0+ rows.setdefault(row_idx, []).append(str(c.get("content") or c.get("text") or ""))return "\n".join(" | ".join(v) for _, v in sorted(rows.items()))return ""@@ -122,18 +130,27 @@ def _element_html(el: dict) -> str:# Docling tables: rebuild HTML from table_cellscells = el.get("table_cells")if isinstance(cells, list) and cells:- num_rows = el.get("num_rows") or 0- num_cols = el.get("num_cols") or 0+ try:+ num_rows = int(el.get("num_rows") or 0)+ num_cols = int(el.get("num_cols") or 0)+ except (TypeError, ValueError):+ num_rows = num_cols = 0if num_rows > 0 and num_cols > 0:grid: list[list[dict]] = [[{} for _ in range(num_cols)] for _ in range(num_rows)]for c in cells:if not isinstance(c, dict):continue- r = c.get("start_row_offset_idx", 0)- col = c.get("start_col_offset_idx", 0)- row_span = max(1, c.get("row_span", 1) or 1)- col_span = max(1, c.get("col_span", 1) or 1)- text = html.escape(c.get("text", ""))+ try:+ # cells come from the external opendataloader/Docling json, so a+ # null/non-numeric offset or span must skip the cell, not crash the parse.+ r = int(c.get("start_row_offset_idx") or 0)+ col = int(c.get("start_col_offset_idx") or 0)+ row_span = max(1, int(c.get("row_span") or 1))+ col_span = max(1, int(c.get("col_span") or 1))+ except (TypeError, ValueError):+ continue+ # a cell may carry a null or non-string text; coerce before escaping.+ text = html.escape(str(c.get("text") or ""))if 0 <= r < num_rows and 0 <= col < num_cols:grid[r][col] = {"text": text, "row_span": row_span, "col_span": col_span}# Mark spanned positions as occupied so they are skippeddiff --git a/test/unit_test/deepdoc/parser/test_opendataloader_parser.py b/test/unit_test/deepdoc/parser/test_opendataloader_parser.pyindex 48628a53da..d1794b5a65 100644--- a/test/unit_test/deepdoc/parser/test_opendataloader_parser.py+++ b/test/unit_test/deepdoc/parser/test_opendataloader_parser.py@@ -460,3 +460,52 @@ class TestElementHtml:def test_returns_existing_html_content(self):el = {"html": "<table><tr><td>X</td></tr></table>"}assert _mod._element_html(el) == "<table><tr><td>X</td></tr></table>"++ def test_cell_with_null_or_nonstring_text_does_not_crash(self):+ # cells come from external json; a null or int text must not raise+ # AttributeError inside html.escape and abort the whole document parse.+ el = {+ "table_cells": [+ {"start_row_offset_idx": 0, "start_col_offset_idx": 0, "text": None},+ {"start_row_offset_idx": 0, "start_col_offset_idx": 1, "text": 42},+ ],+ "num_rows": 1,+ "num_cols": 2,+ }+ html = _mod._element_html(el)+ assert html == "<table><tr><td></td><td>42</td></tr></table>"++ def test_cell_with_null_offset_does_not_crash(self):+ # a null start_row/col_offset_idx used to raise TypeError in the+ # `0 <= r < num_rows` comparison; it must be coerced rather than aborting+ # the whole document parse.+ el = {+ "table_cells": [{"start_row_offset_idx": None, "start_col_offset_idx": 0, "text": "x"}],+ "num_rows": 1,+ "num_cols": 1,+ }+ assert _mod._element_html(el) == "<table><tr><td>x</td></tr></table>"++ def test_string_num_rows_cols_do_not_crash(self):+ # num_rows/num_cols arriving as strings must not raise on `num_rows > 0`.+ el = {+ "table_cells": [{"start_row_offset_idx": 0, "start_col_offset_idx": 0, "text": "A"}],+ "num_rows": "1",+ "num_cols": "1",+ }+ assert _mod._element_html(el) == "<table><tr><td>A</td></tr></table>"+++class TestElementText:+ """Unit tests for _element_text's cell row-joining."""++ def test_non_numeric_row_label_does_not_crash(self):+ # int(row) on a non-numeric label used to raise ValueError and abort the parse.+ out = _mod._element_text({"cells": [{"row": "header", "content": "a"}, {"row": 1, "content": "b"}]})+ assert "a" in out and "b" in out++ def test_row_zero_is_not_treated_as_missing(self):+ # the `row or row_index or 0` chain treated a real row 0 as missing and fell+ # through to row_index (here 3), mis-ordering rows; row 0 must keep its slot.+ out = _mod._element_text({"cells": [{"row": 0, "row_index": 3, "content": "first"}, {"row": 1, "content": "second"}]})+ assert out == "first\nsecond"