RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): escape DOCX table cells in the naive, manual and qa chunkers (#19746)
Lukasba693dac0346
diff --git a/rag/app/manual.py b/rag/app/manual.pyindex 80f282dedb..165e6b3263 100644--- a/rag/app/manual.py+++ b/rag/app/manual.py@@ -17,6 +17,7 @@import copyimport loggingimport re+from html import escape as html_escapefrom io import BytesIOfrom docx import Document@@ -134,7 +135,8 @@ class Docx(DocxParser):else:breaki += 1- html += f"<td>{c.text}</td>" if span == 1 else f"<td colspan='{span}'>{c.text}</td>"+ cell = html_escape(c.text)+ html += f"<td>{cell}</td>" if span == 1 else f"<td colspan='{span}'>{cell}</td>"html += "</tr>"html += "</table>"tbls.append(((None, html), ""))diff --git a/rag/app/naive.py b/rag/app/naive.pyindex 672e031ef3..02bd468dfd 100644--- a/rag/app/naive.py+++ b/rag/app/naive.py@@ -19,6 +19,7 @@ import osimport refrom collections.abc import Callablefrom functools import reduce+from html import escape as html_escapefrom io import BytesIOfrom timeit import default_timer as timerfrom typing import Any@@ -812,7 +813,7 @@ class Docx(DocxParser):title = self.__get_nearest_title(table_idx, filename)html = "<table>"if title:- html += f"<caption>Table Location: {title}</caption>"+ html += f"<caption>Table Location: {html_escape(title)}</caption>"for r in tb.rows:html += "<tr>"col_idx = 0@@ -827,7 +828,8 @@ class Docx(DocxParser):else:breakcol_idx += 1- html += f"<td>{c.text}</td>" if span == 1 else f"<td colspan='{span}'>{c.text}</td>"+ cell = html_escape(c.text)+ html += f"<td>{cell}</td>" if span == 1 else f"<td colspan='{span}'>{cell}</td>"except Exception as e:logging.warning(f"Error parsing table, ignore: {e}")html += "</tr>"diff --git a/rag/app/qa.py b/rag/app/qa.pyindex a97b890611..c02f105331 100644--- a/rag/app/qa.py+++ b/rag/app/qa.py@@ -18,6 +18,7 @@ import csvimport loggingimport refrom copy import deepcopy+from html import escape as html_escapefrom io import BytesIOfrom timeit import default_timer as timer@@ -240,7 +241,8 @@ class Docx(DocxParser):else:breaki += 1- html += f"<td>{c.text}</td>" if span == 1 else f"<td colspan='{span}'>{c.text}</td>"+ cell = html_escape(c.text)+ html += f"<td>{cell}</td>" if span == 1 else f"<td colspan='{span}'>{cell}</td>"html += "</tr>"html += "</table>"tbls.append(((None, html), ""))diff --git a/test/unit_test/rag/test_docx_table_escaping.py b/test/unit_test/rag/test_docx_table_escaping.pynew file mode 100644index 0000000000..f285243e1d--- /dev/null+++ b/test/unit_test/rag/test_docx_table_escaping.py@@ -0,0 +1,225 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""Three DOCX chunkers built their table HTML without escaping the cells.++`rag/app/laws.py` escapes them (`test_laws_docx_tables.py` pins it), and so does+`deepdoc/parser/excel_parser.py`. `naive.py`, `manual.py` and `qa.py` did not, so+a cell holding `<`, `>` or `&` -- a comparison, a formula, an XML snippet, a+company name -- was written into the table as markup.+"""++import importlib.util+import sys+import types+from io import BytesIO+from pathlib import Path+from unittest import mock++import pytest++_REPO_ROOT = Path(__file__).resolve().parents[3]++CELL_A = "a < b & c > d"+CELL_B = "<img src=x onerror=alert(1)>"+ESCAPED_A = "a < b & c > d"+ESCAPED_B = "<img src=x onerror=alert(1)>"+++class _Dummy:+ def __init__(self, *a, **k):+ pass+++def _stub(name, **attrs):+ mod = types.ModuleType(name)+ for key, value in attrs.items():+ setattr(mod, key, value)+ sys.modules[name] = mod+ return mod+++def _load(module_name, relative_path):+ spec = importlib.util.spec_from_file_location(module_name, _REPO_ROOT / relative_path)+ module = importlib.util.module_from_spec(spec)+ sys.modules[module_name] = module+ spec.loader.exec_module(module)+ return module++[email protected](scope="module")+def chunkers():+ """The DOCX chunkers, with the vision / LLM / storage siblings stubbed out."""+ saved = {name: sys.modules.get(name) for name in list(sys.modules)}+ try:+ for name in ("xgboost", "pdfplumber", "huggingface_hub", "pypdf", "sklearn", "sklearn.cluster", "sklearn.metrics", "deepdoc.vision"):+ sys.modules.setdefault(name, mock.MagicMock())++ docx_parser = _load("deepdoc.parser.docx_parser_under_test", "deepdoc/parser/docx_parser.py")++ _stub(+ "deepdoc.parser",+ DocxParser=docx_parser.RAGFlowDocxParser,+ PdfParser=_Dummy,+ HtmlParser=_Dummy,+ ExcelParser=_Dummy,+ EpubParser=_Dummy,+ JsonParser=_Dummy,+ MarkdownElementExtractor=_Dummy,+ MarkdownParser=_Dummy,+ TxtParser=_Dummy,+ )+ _stub("deepdoc.parser.utils", get_text=lambda *a, **k: "", extract_pdf_outlines=lambda *a, **k: [])+ _stub(+ "deepdoc.parser.figure_parser",+ VisionFigureParser=_Dummy,+ vision_figure_parser_pdf_wrapper=lambda *a, **k: [],+ vision_figure_parser_docx_wrapper=lambda *a, **k: [],+ vision_figure_parser_docx_wrapper_naive=lambda *a, **k: [],+ )+ _stub("deepdoc.parser.pdf_parser", PlainParser=_Dummy, VisionParser=_Dummy, RAGFlowPdfParser=_Dummy)+ _stub("deepdoc.parser.docling_parser", DoclingParser=_Dummy)+ _stub("deepdoc.parser.monkeyocrv2_parser", MonkeyOCRv2Parser=_Dummy)+ _stub("deepdoc.parser.tcadp_parser", TCADPParser=_Dummy)+ _stub(+ "api.db.joint_services.tenant_model_service",+ get_composite_model_name_by_id=lambda x: x,+ get_first_provider_model_name=lambda *a, **k: None,+ resolve_model_config=lambda *a, **k: None,+ get_tenant_default_model_by_type=lambda *a, **k: None,+ ensure_mineru_from_env=lambda *a, **k: None,+ ensure_opendataloader_from_env=lambda *a, **k: None,+ ensure_paddleocr_from_env=lambda *a, **k: None,+ ensure_somark_from_env=lambda *a, **k: None,+ )+ _stub("api.db.services.llm_service", LLMBundle=_Dummy)+ _stub(+ "rag.utils.file_utils",+ extract_embed_file=lambda *a, **k: None,+ extract_links_from_pdf=lambda *a, **k: [],+ extract_links_from_docx=lambda *a, **k: [],+ extract_html=lambda *a, **k: (None, None),+ )+ _stub("common.float_utils", get_float=lambda *a, **k: 0.0, normalize_overlapped_percent=lambda x: x)+ _stub("common.text_utils", normalize_arabic_presentation_forms=lambda x: x)+ _stub("rag.app.naive", by_plaintext=lambda *a, **k: ([], [], None), PARSERS={})+ _stub(+ "common.parser_config_utils",+ normalize_layout_recognizer=lambda x: (x, None),+ has_mineru_options=lambda *a, **k: False,+ is_tenant_model_id=lambda *a, **k: False,+ MINERU_OPTION_KEYS=(),+ )++ yield types.SimpleNamespace(+ laws=_load("rag.app.laws_under_test", "rag/app/laws.py").Docx,+ qa=_load("rag.app.qa_under_test", "rag/app/qa.py").Docx,+ manual=_load("rag.app.manual_under_test", "rag/app/manual.py").Docx,+ naive=_load("rag.app.naive_under_test", "rag/app/naive.py").Docx,+ )+ finally:+ for name in list(sys.modules):+ if name not in saved:+ del sys.modules[name]+ sys.modules.update(saved)+++def _docx_with_table(heading="Heading"):+ from docx import Document++ doc = Document()+ doc.add_heading(heading, level=1)+ table = doc.add_table(rows=1, cols=2)+ table.cell(0, 0).text = CELL_A+ table.cell(0, 1).text = CELL_B++ buf = BytesIO()+ doc.save(buf)+ return buf.getvalue()+++def _assert_escaped(html):+ assert ESCAPED_A in html+ assert ESCAPED_B in html+ # The markup a cell carried must not reach the table as markup.+ assert "<img" not in html+ assert "<td>a < b" not in html+++def test_naive_docx_escapes_cell_html(chunkers):+ lines = chunkers.naive()("table.docx", _docx_with_table())++ tables = [table for _text, _image, table in lines if table]+ assert tables+ _assert_escaped(tables[0])+++def test_naive_docx_escapes_the_table_caption(chunkers):+ """The caption carries the enclosing heading, which is document text too."""+ lines = chunkers.naive()("table.docx", _docx_with_table(heading="Q1 < Q2 & Q3"))++ tables = [table for _text, _image, table in lines if table]+ assert tables+ assert "<caption>" in tables[0]+ assert "Q1 < Q2 & Q3" in tables[0]+ assert "Q1 < Q2" not in tables[0]+++def test_manual_docx_escapes_cell_html(chunkers):+ _ti_list, tbls = chunkers.manual()("table.docx", _docx_with_table())++ assert tbls+ _assert_escaped(tbls[0][0][1])+++def test_qa_docx_escapes_cell_html(chunkers):+ _qai_list, tbls = chunkers.qa()("table.docx", _docx_with_table())++ assert tbls+ _assert_escaped(tbls[0][0][1])+++def test_laws_docx_still_escapes_cell_html(chunkers):+ """Guard: the chunker that already escaped must not change."""+ chunks = chunkers.laws()("table.docx", _docx_with_table())++ table_chunk = next(chunk for chunk in chunks if "<table>" in chunk)+ _assert_escaped(table_chunk)+++def test_a_plain_table_is_unchanged(chunkers):+ """Guard: a table with nothing to escape renders exactly as before."""+ from docx import Document++ doc = Document()+ doc.add_heading("Heading", level=1)+ table = doc.add_table(rows=1, cols=2)+ table.cell(0, 0).text = "Item"+ table.cell(0, 1).text = "Fee"+ buf = BytesIO()+ doc.save(buf)++ lines = chunkers.naive()("table.docx", buf.getvalue())+ tables = [table for _text, _image, table in lines if table]++ assert "<td>Item</td>" in tables[0]+ assert "<td>Fee</td>" in tables[0]