RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): skip table-of-contents entries when parsing docx (#19491)
wang3939b346156129c
diff --git a/rag/app/naive.py b/rag/app/naive.pyindex 690de2d81e..ce918322a3 100644--- a/rag/app/naive.py+++ b/rag/app/naive.py@@ -561,9 +561,41 @@ PARSERS = {"plaintext": by_plaintext, # default}+# ---------------------------------------------------------------------------+# Table of contents (TOC) detection+# ---------------------------------------------------------------------------+# Word-generated TOC entries are pure navigation noise for retrieval: they only+# repeat section titles and add a page number. Indexing them wastes top_k slots+# and, because an entry is a short string that literally matches the section+# title, it often outranks the real content block in vector search.+#+# Word marks TOC entries with the built-in styles ``toc 1`` / ``TOC 1`` / ...+# When a document loses its styles (e.g. converted from PDF), an entry usually+# still looks like "1.1<TAB>title<TAB>1" — section number, tab, page number.+_TOC_STYLE_RE = re.compile(r"^toc(?:[\s\d]|$)")+_TOC_LINE_RE = re.compile(r"^[\d.]+[\s\u3000]*(?:[^\t]*\t)?[^\t]*\t\s*\d+\s*$")+++def _is_toc_paragraph(text: str, style_name: str, paragraph=None) -> bool:+ """Return True when a paragraph looks like a table-of-contents entry."""+ if not text:+ return False+ if style_name and _TOC_STYLE_RE.match(style_name.strip().lower()):+ return True+ # Fallback for documents whose TOC styles were renamed or dropped. The text+ # shape alone cannot tell an entry ("1.1 Introduction<TAB>1") from a+ # tab-separated data row ("1. Price<TAB>100"), and dropping the latter loses+ # real content, so require the PAGEREF field Word writes into every TOC+ # entry. The cheap shape check runs first so body text never pays for the+ # XML scan.+ if paragraph is None or not _TOC_LINE_RE.match(text):+ return False+ return "PAGEREF" in paragraph._element.xml+class Docx(DocxParser):def __init__(self):+ """Initialize the naive DOCX parser."""passdef __clean(self, line):@@ -676,6 +708,11 @@ class Docx(DocxParser):return ""def __call__(self, filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER):+ """Parse a DOCX file into ordered (text, image, table) triples.++ Each element is a plain-text paragraph, an image, or an HTML table,+ preserving document order. Table-of-contents entries are skipped.+ """self.doc = Document(filename) if binary is None else Document(BytesIO(binary))pn = 0lines = []@@ -700,7 +737,12 @@ class Docx(DocxParser):style_name = p.style.name if p.style else ""if text:- if style_name == "Caption":+ # Skip table-of-contents entries: they are navigation+ # noise that pollutes retrieval (see _is_toc_paragraph).+ if _is_toc_paragraph(text, style_name, p):+ pass++ elif style_name == "Caption":former_image = Noneif lines and lines[-1].get("image") and lines[-1].get("style") != "Caption":diff --git a/test/unit_test/rag/app/test_docx_skip_toc.py b/test/unit_test/rag/app/test_docx_skip_toc.pynew file mode 100644index 0000000000..383ba608f2--- /dev/null+++ b/test/unit_test/rag/app/test_docx_skip_toc.py@@ -0,0 +1,133 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""Pin the DOCX table-of-contents skip added for #19491.++``Docx.__call__`` drops a paragraph that ``_is_toc_paragraph`` recognises as a+TOC entry. Indexing those entries is pure navigation noise: an entry is a short+string that literally matches its section title, so it wastes ``top_k`` slots+and can outrank the real content block in vector search.++Two signals decide it:++* Word's built-in TOC styles (``toc 1`` / ``TOC 1`` / ``TOC Heading`` / ...);+* a text-shape fallback for documents whose TOC styles were renamed or dropped+ (e.g. converted from PDF). The shape alone cannot tell an entry+ (``1.1 Introduction<TAB>1``) from a tab-separated data row+ (``1. Price<TAB>100``), so the fallback additionally requires the ``PAGEREF``+ field Word writes into every real TOC entry.++The tests below pin both signals, the false-positive guard (a data row that only+matches the shape must be kept), and the end-to-end behaviour of the parser.+"""++from __future__ import annotations++from io import BytesIO+from types import SimpleNamespace++import pytest+from docx import Document+from docx.enum.style import WD_STYLE_TYPE++from rag.app import naive+++class _Paragraph:+ """Stand-in exposing the only attribute ``_is_toc_paragraph`` reads."""++ def __init__(self, xml: str):+ self._element = SimpleNamespace(xml=xml)+++# The PAGEREF field Word writes into every TOC entry, and a plain body run.+TOC_XML = "<w:p><w:r><w:instrText> PAGEREF _Toc12345 \\h </w:instrText></w:r></w:p>"+BODY_XML = "<w:p><w:r><w:t>row</w:t></w:r></w:p>"+++def _ensure_style(document, style_name):+ """Return ``style_name``, adding it to ``document`` when absent."""+ for style in document.styles:+ if style.name == style_name:+ return style+ return document.styles.add_style(style_name, WD_STYLE_TYPE.PARAGRAPH)+++def _docx_bytes(paragraphs):+ """Build an in-memory ``.docx`` from ``(text, style_name)`` pairs."""+ document = Document()+ for text, style_name in paragraphs:+ paragraph = document.add_paragraph(text)+ paragraph.style = _ensure_style(document, style_name)+ buffer = BytesIO()+ document.save(buffer)+ return buffer.getvalue()++[email protected]("style_name", ["toc 1", "TOC 1", "Toc 2", "TOC Heading"])+def test_word_toc_style_is_skipped(style_name):+ assert naive._is_toc_paragraph("1.1 Introduction\t1", style_name) is True+++def test_style_that_only_starts_with_toc_is_kept():+ # "tocustom" is not a Word TOC style; the pattern must not swallow it.+ assert naive._is_toc_paragraph("Body text", "tocustom") is False+++def test_plain_body_paragraph_is_kept():+ assert naive._is_toc_paragraph("The quick brown fox jumps.", "Normal") is False+++def test_empty_text_is_never_a_toc_entry():+ assert naive._is_toc_paragraph("", "toc 1") is False+++def test_styleless_toc_entry_with_pageref_is_skipped():+ # The TOC style was dropped, but the PAGEREF field Word writes survives.+ assert naive._is_toc_paragraph("1.1 Introduction\t1", "Normal", _Paragraph(TOC_XML)) is True++[email protected]("row", ["1. Price\t100", "2024\t2025", "3.5 mg dose\t250"])+def test_tab_separated_data_row_without_pageref_is_kept(row):+ # Regression guard: these rows match the TOC text shape but are body+ # content. Requiring PAGEREF is what keeps them.+ assert naive._is_toc_paragraph(row, "Normal", _Paragraph(BODY_XML)) is False+++def test_toc_shaped_text_without_paragraph_object_is_kept():+ # Without a paragraph to inspect, the fallback cannot confirm PAGEREF.+ assert naive._is_toc_paragraph("1.1 Introduction\t1", "Normal", None) is False+++def test_docx_parser_skips_toc_entry_and_keeps_body():+ binary = _docx_bytes(+ [+ ("1.1 Introduction\t1", "TOC 1"),+ ("Body paragraph.", "Normal"),+ ]+ )+ lines = naive.Docx()("report.docx", binary)+ texts = [text for text, _image, _table in lines if text]+ assert "Body paragraph." in texts+ assert all("Introduction" not in text for text in texts)