Python tool for converting files and office documents to Markdown.
fix(docx): preserve namespaces when repairing stylesheets (#2467)
afourney485a7b93b6d9
diff --git a/packages/markitdown-ocr/tests/test_docx_converter.py b/packages/markitdown-ocr/tests/test_docx_converter.pyindex 2538206d..f9df439c 100644--- a/packages/markitdown-ocr/tests/test_docx_converter.py+++ b/packages/markitdown-ocr/tests/test_docx_converter.py@@ -226,6 +226,56 @@ def test_docx_no_ocr_service_no_tags() -> None:assert "[End OCR]*" not in md[email protected]("double_strike", [False, True])[email protected]("use_ocr", [False, True])+def test_docx_styles_with_redundant_default_namespace(+ svc: MockOCRService, use_ocr: bool, double_strike: bool+) -> None:+ path = TEST_DATA_DIR / "docx_image_middle.docx"+ if not path.exists():+ pytest.skip(f"Test file not found: {path}")+ original = path.read_bytes()+ fixture = io.BytesIO()+ namespace = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"+ declaration = f'xmlns:w="{namespace}"'.encode("utf-8")+ with zipfile.ZipFile(io.BytesIO(original)) as source, zipfile.ZipFile(+ fixture, "w"+ ) as target:+ for item in source.infolist():+ content = source.read(item)+ if item.filename == "word/styles.xml":+ if double_strike:+ assert content.count(b"</w:styles>") == 1+ content = content.replace(+ b"</w:styles>",+ b'<w:style w:type="character" w:styleId="DoubleStrike">'+ b'<w:name w:val="Double Strike"/>'+ b'<w:rPr><w:dstrike w:val="1"/></w:rPr>'+ b"</w:style></w:styles>",+ 1,+ )+ assert content.count(declaration) == 1+ content = content.replace(+ declaration, declaration + f' xmlns="{namespace}"'.encode(), 1+ )+ target.writestr(item, content)++ converter = DocxConverterWithOCR()+ service = svc if use_ocr else None+ expected = converter.convert(+ io.BytesIO(original), StreamInfo(extension=".docx"), ocr_service=service+ ).markdown+ fixture.seek(0)+ actual = converter.convert(+ fixture, StreamInfo(extension=".docx"), ocr_service=service+ ).markdown++ assert "# Introduction" in actual+ assert actual == expected+ if use_ocr:+ assert _MOCK_TEXT in actual++# ---------------------------------------------------------------------------# Underlined runs survive both the OCR and the non-OCR mammoth paths# ---------------------------------------------------------------------------diff --git a/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py b/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.pyindex 46847a2a..c6dc3034 100644--- a/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py+++ b/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py@@ -216,15 +216,35 @@ def _pre_process_styles(content: bytes) -> bytes:to ``paragraph``, so the attribute is filled in rather than dropping thestyle, which would discard its formatting (a heading would be emitted asplain body text). A style with no ``w:styleId`` cannot be referenced by the- document body, so it is removed.+ document body, so it is removed. Double strikethrough is normalized to+ single strikethrough in the same namespace-aware pass.++ Match elements and attributes by namespace URI, preserving their qualified+ names when repairing the XML. Return the original bytes if no repair is needed."""- soup = BeautifulSoup(content, features="xml")- for tag in soup.find_all("w:style"):- if not tag.has_attr("w:styleId"):- tag.decompose()- elif not tag.has_attr("w:type"):- tag["w:type"] = "paragraph"- return str(soup).encode()+ from lxml import etree++ namespace = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"+ parser = etree.XMLParser(resolve_entities=False, no_network=True)+ root = etree.fromstring(content, parser=parser)+ changed = False++ for style in root.findall(namespace + "style"):+ if namespace + "styleId" not in style.attrib:+ root.remove(style)+ changed = True+ elif namespace + "type" not in style.attrib:+ style.set(namespace + "type", "paragraph")+ changed = True++ for strike in root.iter(namespace + "dstrike"):+ strike.tag = namespace + "strike"+ changed = True++ if not changed:+ return content++ return etree.tostring(root.getroottree(), encoding="utf-8", xml_declaration=True)def pre_process_docx(input_docx: BinaryIO) -> BinaryIO:@@ -250,7 +270,7 @@ def pre_process_docx(input_docx: BinaryIO) -> BinaryIO:"word/document.xml": (_pre_process_strike, _pre_process_math),"word/footnotes.xml": (_pre_process_strike, _pre_process_math),"word/endnotes.xml": (_pre_process_strike, _pre_process_math),- "word/styles.xml": (_pre_process_strike, _pre_process_styles),+ "word/styles.xml": (_pre_process_styles,),}with zipfile.ZipFile(input_docx, mode="r") as zip_input:files = {name: zip_input.read(name) for name in zip_input.namelist()}diff --git a/packages/markitdown/tests/test_docx_styles.py b/packages/markitdown/tests/test_docx_styles.pynew file mode 100644index 00000000..218fe03c--- /dev/null+++ b/packages/markitdown/tests/test_docx_styles.py@@ -0,0 +1,152 @@+"""DOCX stylesheet repair must preserve namespace-qualified attributes."""++import io+from pathlib import Path+import re+import zipfile++from lxml import etree+import pytest++from markitdown import MarkItDown, StreamInfo+from markitdown.converter_utils.docx.pre_process import _pre_process_styles+++WORD_NAMESPACE = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"+W = f"{{{WORD_NAMESPACE}}}"+TEST_DOCX = Path(__file__).parent / "test_files" / "test.docx"++[email protected]("double_strike", [False, True])[email protected]("missing_type", [False, True])+def test_docx_styles_with_redundant_default_namespace(+ missing_type: bool, double_strike: bool+) -> None:+ markitdown = MarkItDown()+ expected = markitdown.convert(TEST_DOCX).markdown+ fixture = io.BytesIO()+ declaration = f'xmlns:w="{WORD_NAMESPACE}"'.encode("utf-8")++ with zipfile.ZipFile(TEST_DOCX) as source, zipfile.ZipFile(fixture, "w") as target:+ for item in source.infolist():+ content = source.read(item)+ if item.filename == "word/styles.xml":+ if double_strike:+ assert content.count(b"</w:styles>") == 1+ content = content.replace(+ b"</w:styles>",+ b'<w:style w:type="character" w:styleId="DoubleStrike">'+ b'<w:name w:val="Double Strike"/>'+ b'<w:rPr><w:dstrike w:val="1"/></w:rPr>'+ b"</w:style></w:styles>",+ 1,+ )+ assert content.count(declaration) == 1+ content = content.replace(+ declaration,+ declaration + f' xmlns="{WORD_NAMESPACE}"'.encode("utf-8"),+ 1,+ )+ if missing_type:+ content, count = re.subn(+ rb'<w:style\s+w:type="[^"]+"(\s+w:styleId="1")',+ rb"<w:style\1",+ content,+ )+ assert count == 1+ styles = etree.fromstring(content).findall(W + "style")+ assert styles+ assert all(W + "styleId" in style.attrib for style in styles)+ target.writestr(item, content)++ fixture.seek(0)+ actual = markitdown.convert_stream(+ fixture, stream_info=StreamInfo(extension=".docx")+ ).markdown++ assert "# Abstract" in actual+ assert actual == expected++[email protected](params=["w", "word"])+def styles_xml(request: pytest.FixtureRequest) -> bytes:+ prefix = request.param+ return f"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<{prefix}:styles xmlns:{prefix}="{WORD_NAMESPACE}" xmlns="{WORD_NAMESPACE}"+ xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"+ xmlns:w14="http://schemas.microsoft.com/office/word/2010/wordml"+ mc:Ignorable="w14">+ <{prefix}:style {prefix}:type="paragraph" {prefix}:styleId="Normal">+ <{prefix}:name {prefix}:val="Normal"/>+ </{prefix}:style>+</{prefix}:styles>""".encode(+ "utf-8"+ )+++def test_valid_styles_are_returned_unchanged(styles_xml: bytes) -> None:+ assert _pre_process_styles(styles_xml) == styles_xml+++def test_missing_type_repair_preserves_namespaces(styles_xml: bytes) -> None:+ malformed, count = re.subn(rb'\s+(?:w|word):type="paragraph"', b"", styles_xml)+ assert count == 1+ original = etree.fromstring(styles_xml)++ repaired = etree.fromstring(_pre_process_styles(malformed))++ assert repaired.nsmap == original.nsmap+ assert repaired.attrib == original.attrib+ style = repaired.find(W + "style")+ assert style is not None+ assert style.attrib == {W + "type": "paragraph", W + "styleId": "Normal"}+ name = style.find(W + "name")+ assert name is not None+ assert name.attrib == {W + "val": "Normal"}+++def test_styles_without_ids_are_removed(styles_xml: bytes) -> None:+ malformed, count = re.subn(rb'\s+(?:w|word):styleId="Normal"', b"", styles_xml)+ assert count == 1++ repaired = etree.fromstring(_pre_process_styles(malformed))++ assert repaired.findall(W + "style") == []++[email protected]("value", [None, "0", "1"])[email protected]("missing_type", [False, True])+def test_style_strike_repair_preserves_namespaces(+ styles_xml: bytes, value: str | None, missing_type: bool+) -> None:+ original = etree.fromstring(styles_xml)+ style = original.find(W + "style")+ assert style is not None+ if missing_type:+ del style.attrib[W + "type"]+ properties = etree.SubElement(style, W + "rPr")+ strike = etree.SubElement(properties, W + "dstrike")+ if value is not None:+ strike.set(W + "val", value)+ etree.SubElement(properties, W + "b")++ repaired = etree.fromstring(_pre_process_styles(etree.tostring(original)))++ assert repaired.nsmap == original.nsmap+ assert repaired.attrib == original.attrib+ style = repaired.find(W + "style")+ assert style is not None+ assert style.attrib == {W + "type": "paragraph", W + "styleId": "Normal"}+ assert repaired.find(".//" + W + "dstrike") is None+ strike = style.find(f"{W}rPr/{W}strike")+ assert strike is not None+ assert strike.attrib == ({} if value is None else {W + "val": value})+ assert style.find(f"{W}rPr/{W}b") is not None+++def test_other_namespace_dstrike_is_unchanged(styles_xml: bytes) -> None:+ original = etree.fromstring(styles_xml)+ etree.SubElement(original, "{urn:extension}dstrike")+ content = etree.tostring(original)++ assert _pre_process_styles(content) == content