Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): keep the text of DOCX content controls and tracked insertions (#4448)
Lukasc0febacb4361
diff --git a/docreader/parser/docx_parser.py b/docreader/parser/docx_parser.pyindex f9820069f..cc5bdc7e3 100644--- a/docreader/parser/docx_parser.py+++ b/docreader/parser/docx_parser.py@@ -40,6 +40,8 @@ from docx.image.exceptions import (UnrecognizedImageError,)from docx.oxml.ns import qn+from docx.table import Table+from docx.text.run import Runfrom PIL import Imagefrom docreader.config import CONFIG@@ -51,6 +53,96 @@ logger = logging.getLogger(__name__)_W_P = qn("w:p")_W_TBL = qn("w:tbl")+_W_R = qn("w:r")+_W_SDT = qn("w:sdt")+_W_SDT_PR = qn("w:sdtPr")+_W_SDT_CONTENT = qn("w:sdtContent")+_W_CUSTOM_XML = qn("w:customXml")++# Inline wrappers whose runs Word shows as part of the paragraph. python-docx's+# Paragraph.text reads only w:r and w:hyperlink children, so the text of a+# tracked insertion (w:ins) or move destination (w:moveTo), a smart tag,+# custom XML markup or a simple field was dropped, and that of an inline+# content control (w:sdt, read through _sdt_content). Deletions (w:del,+# w:moveFrom) hold their text in w:delText, not w:t, and stay out, which is+# the document as it reads with changes accepted.+_INLINE_TEXT_CONTAINERS = frozenset(+ qn(tag)+ for tag in (+ "w:hyperlink",+ "w:ins",+ "w:moveTo",+ "w:smartTag",+ "w:customXml",+ "w:fldSimple",+ )+)+++def _sdt_content(sdt: Any) -> Optional[Any]:+ """The w:sdtContent of a content control that holds document text, or None.++ A control still showing its placeholder ("Click or tap here to enter+ text.") holds none, and a table of contents repeats the headings with+ their page numbers, so both are left out as before.+ """+ props = sdt.find(_W_SDT_PR)+ if props is not None:+ if props.find(qn("w:showingPlcHdr")) is not None:+ return None+ gallery = props.find(f"{qn('w:docPartObj')}/{qn('w:docPartGallery')}")+ if gallery is not None and gallery.get(qn("w:val")) == "Table of Contents":+ return None+ return sdt.find(_W_SDT_CONTENT)+++def _paragraph_text(p_element: Any) -> str:+ """The text of a w:p as Word shows it, wrapped runs included."""+ parts: List[str] = []++ def walk(parent: Any) -> None:+ for child in parent:+ if child.tag == _W_R:+ parts.append(Run(child, None).text)+ elif child.tag == _W_SDT:+ content = _sdt_content(child)+ if content is not None:+ walk(content)+ elif child.tag in _INLINE_TEXT_CONTAINERS:+ walk(child)++ walk(p_element)+ return "".join(parts)+++def _block_children(container: Any):+ """Yield the paragraphs and tables of a body or cell in document order.++ A block content control (w:sdt) and block custom XML hold paragraphs and+ tables of their own, which Word shows in place; they are opened instead of+ skipped. Yields ``(element, direct)``, where ``direct`` is False for an+ element found inside one of them.+ """+ for child in container:+ if child.tag in (_W_P, _W_TBL):+ yield child, True+ elif child.tag == _W_SDT:+ content = _sdt_content(child)+ if content is not None:+ for element, _ in _block_children(content):+ yield element, False+ elif child.tag == _W_CUSTOM_XML:+ for element, _ in _block_children(child):+ yield element, False+++def _cell_text(cell: Any) -> str:+ """A table cell's paragraph text, content controls and insertions included."""+ return "\n".join(+ _paragraph_text(element)+ for element, _ in _block_children(cell._tc)+ if element.tag == _W_P+ )def _gfm_cell_text(text: Optional[str]) -> str:@@ -67,7 +159,7 @@ def table_to_gfm_markdown(table: Any) -> str:"""rows: List[List[str]] = []for row in table.rows:- cells = [_gfm_cell_text(cell.text) for cell in row.cells]+ cells = [_gfm_cell_text(_cell_text(cell)) for cell in row.cells]if cells:rows.append(cells)if not rows:@@ -1082,28 +1174,26 @@ class Docx:max_allowed = max(allowed) if allowed else -1parts: List[str] = []+ # Counts the body's own paragraphs, the ones the page mapping indexes.para_i = 0- tbl_i = 0- paragraphs = self.doc.paragraphs- tables = self.doc.tables- for child in self.doc.element.body:+ def in_page_range() -> bool:+ return not allowed or min_allowed <= para_i <= max_allowed + 1++ for child, direct in _block_children(self.doc.element.body):if child.tag == _W_P:- if para_i in allowed and para_i < len(paragraphs):- text = (paragraphs[para_i].text or "").strip()+ # Paragraphs inside a content control are not in the page+ # mapping; they follow the range of the paragraphs around them.+ include = para_i in allowed if direct else in_page_range()+ if direct:+ para_i += 1+ if include:+ text = _paragraph_text(child).strip()if text:parts.append(text)- para_i += 1- continue- if child.tag != _W_TBL:continue- include_table = False- if not allowed:- include_table = True- elif min_allowed <= para_i <= max_allowed + 1:- include_table = True- if include_table and tbl_i < len(tables):- table = tables[tbl_i]+ if in_page_range():+ table = Table(child, self.doc._body)if not (len(table.rows) == 0or all(len(row.cells) == 0 for row in table.rows)@@ -1111,7 +1201,6 @@ class Docx:markdown = table_to_gfm_markdown(table)if markdown:parts.append(markdown)- tbl_i += 1return "\n\n".join(parts)diff --git a/docreader/tests/test_docx_tables.py b/docreader/tests/test_docx_tables.pyindex 77f2881f1..ccfa2b039 100644--- a/docreader/tests/test_docx_tables.py+++ b/docreader/tests/test_docx_tables.py@@ -212,5 +212,112 @@ class DocxTableContentTest(unittest.TestCase):self.assertEqual(table_to_gfm_markdown(_Empty()), "")++def _xml(fragment):+ """Parse a WordprocessingML fragment, declaring the w namespace on its root."""+ from docx.oxml import parse_xml+ from docx.oxml.ns import nsdecls++ head, rest = fragment.split(">", 1)+ return parse_xml(f"{head} {nsdecls('w')}>{rest}")+++def _insert_before_sectpr(doc, element):+ body = doc.element.body+ body.insert(len(body) - 1, element)+++class DocxContentControlAndRevisionTest(unittest.TestCase):+ """Text Word shows inside content controls and tracked insertions is kept.++ python-docx's Paragraph.text reads only w:r and w:hyperlink children, and+ the body walk only handled direct w:p and w:tbl children, so the text of+ a block or inline content control (w:sdt) and of a tracked insertion+ (w:ins) was dropped from the parsed document.+ """++ def test_block_content_control_paragraphs_are_kept_in_order(self):+ def build(doc):+ doc.add_paragraph("Before the control")+ _insert_before_sectpr(+ doc,+ _xml(+ "<w:sdt><w:sdtPr><w:alias w:val='Clause'/></w:sdtPr><w:sdtContent>"+ "<w:p><w:r><w:t>Inside the control</w:t></w:r></w:p>"+ "<w:sdt><w:sdtContent><w:p><w:r><w:t>Nested control</w:t></w:r></w:p>"+ "</w:sdtContent></w:sdt></w:sdtContent></w:sdt>"+ ),+ )+ doc.add_paragraph("After the control")++ document = _parse(_docx_bytes(build))+ self.assertEqual(+ "Before the control\n\nInside the control\n\nNested control\n\nAfter the control",+ document.content,+ )++ def test_inline_controls_and_insertions_stay_in_their_paragraph(self):+ def build(doc):+ paragraph = doc.add_paragraph("Signed by ")+ paragraph._p.append(+ _xml("<w:sdt><w:sdtContent><w:r><w:t>Jane Doe</w:t></w:r></w:sdtContent></w:sdt>")+ )+ paragraph._p.append(+ _xml("<w:ins w:id='1' w:author='a'><w:r><w:t xml:space='preserve'> on Monday</w:t></w:r></w:ins>")+ )+ paragraph._p.append(+ _xml("<w:del w:id='2' w:author='a'><w:r><w:delText> on Friday</w:delText></w:r></w:del>")+ )++ document = _parse(_docx_bytes(build))+ self.assertEqual("Signed by Jane Doe on Monday", document.content)++ def test_toc_and_placeholder_controls_are_left_out(self):+ """A table of contents repeats the headings with page numbers, and a+ control still showing its placeholder holds no document text."""++ def build(doc):+ _insert_before_sectpr(+ doc,+ _xml(+ "<w:sdt><w:sdtPr><w:docPartObj>"+ "<w:docPartGallery w:val='Table of Contents'/><w:docPartUnique/>"+ "</w:docPartObj></w:sdtPr><w:sdtContent>"+ "<w:p><w:r><w:t>Contents</w:t></w:r></w:p>"+ "<w:p><w:r><w:t>Introduction 1</w:t></w:r></w:p>"+ "</w:sdtContent></w:sdt>"+ ),+ )+ paragraph = doc.add_paragraph("Name: ")+ paragraph._p.append(+ _xml(+ "<w:sdt><w:sdtPr><w:showingPlcHdr/></w:sdtPr><w:sdtContent>"+ "<w:r><w:t>Click or tap here to enter text.</w:t></w:r>"+ "</w:sdtContent></w:sdt>"+ )+ )+ doc.add_paragraph("Introduction")++ document = _parse(_docx_bytes(build))+ self.assertEqual("Name:\n\nIntroduction", document.content)++ def test_tables_in_a_control_and_cell_insertions_are_kept(self):+ def build(doc):+ doc.add_paragraph("Intro")+ _insert_before_sectpr(+ doc,+ _xml(+ "<w:sdt><w:sdtContent><w:tbl><w:tblGrid><w:gridCol/><w:gridCol/></w:tblGrid>"+ "<w:tr><w:tc><w:p><w:r><w:t>Term</w:t></w:r></w:p></w:tc>"+ "<w:tc><w:p><w:r><w:t>Value</w:t></w:r></w:p></w:tc></w:tr>"+ "<w:tr><w:tc><w:p><w:ins w:id='3' w:author='a'><w:r><w:t>Rate</w:t></w:r></w:ins></w:p></w:tc>"+ "<w:tc><w:sdt><w:sdtContent><w:p><w:r><w:t>5%</w:t></w:r></w:p></w:sdtContent></w:sdt></w:tc>"+ "</w:tr></w:tbl></w:sdtContent></w:sdt>"+ ),+ )++ document = _parse(_docx_bytes(build))+ self.assertIn("| Term | Value |\n| --- | --- |\n| Rate | 5% |", document.content)+if __name__ == "__main__":unittest.main()