Python tool for converting files and office documents to Markdown.
fix(docx): preserve underlined text (#2017)
Yufeng Heb752951ec7bd
diff --git a/packages/markitdown-ocr/src/markitdown_ocr/_docx_converter_with_ocr.py b/packages/markitdown-ocr/src/markitdown_ocr/_docx_converter_with_ocr.pyindex f2463de1..9fd3b07e 100644--- a/packages/markitdown-ocr/src/markitdown_ocr/_docx_converter_with_ocr.py+++ b/packages/markitdown-ocr/src/markitdown_ocr/_docx_converter_with_ocr.py@@ -29,6 +29,18 @@ except ImportError:# Must be a single token with no special markdown characters._PLACEHOLDER = "MARKITDOWNOCRBLOCK{}"+_UNDERLINE_STYLE_MAP = "u => u"+++def _read_embedded_style_map(file_stream: BinaryIO) -> Optional[str]:+ """Read the style map embedded in a .docx, if it has one."""+ position = file_stream.tell()+ file_stream.seek(0)+ try:+ return mammoth.read_embedded_style_map(file_stream)+ finally:+ file_stream.seek(position)+class DocxConverterWithOCR(HtmlConverter):"""@@ -82,6 +94,20 @@ class DocxConverterWithOCR(HtmlConverter):kwargs.get("ocr_service") or self.ocr_service)+ # Read the embedded style map and combine with any provided style map+ caller_style_map = kwargs.get("style_map")+ embedded_style_map = _read_embedded_style_map(file_stream)++ style_map = "\n".join(+ part+ for part in (+ caller_style_map,+ embedded_style_map,+ _UNDERLINE_STYLE_MAP,+ )+ if part+ )+if ocr_service:# 1. Extract and OCR images — returns raw text per imagefile_stream.seek(0)@@ -91,7 +117,9 @@ class DocxConverterWithOCR(HtmlConverter):file_stream.seek(0)pre_process_stream = pre_process_docx(file_stream)html_result = mammoth.convert_to_html(- pre_process_stream, style_map=kwargs.get("style_map")+ pre_process_stream,+ style_map=style_map,+ include_embedded_style_map=False,).value# 3. Replace <img> tags with plain placeholder tokens so that@@ -116,10 +144,13 @@ class DocxConverterWithOCR(HtmlConverter):return DocumentConverterResult(markdown=md)else:# Standard conversion without OCR- style_map = kwargs.get("style_map", None)pre_process_stream = pre_process_docx(file_stream)return self._html_converter.convert_string(- mammoth.convert_to_html(pre_process_stream, style_map=style_map).value,+ mammoth.convert_to_html(+ pre_process_stream,+ style_map=style_map,+ include_embedded_style_map=False,+ ).value,**kwargs,)diff --git a/packages/markitdown-ocr/tests/test_docx_converter.py b/packages/markitdown-ocr/tests/test_docx_converter.pyindex 0fb66650..fcd0944b 100644--- a/packages/markitdown-ocr/tests/test_docx_converter.py+++ b/packages/markitdown-ocr/tests/test_docx_converter.py@@ -11,6 +11,7 @@ OCR block format used by the converter:"""import sys+import zipfilefrom pathlib import Pathfrom typing import Any@@ -221,3 +222,60 @@ def test_docx_no_ocr_service_no_tags() -> None:md = converter.convert(f, StreamInfo(extension=".docx")).text_contentassert "*[Image OCR]" not in mdassert "[End OCR]*" not in md+++# ---------------------------------------------------------------------------+# Underlined runs survive both the OCR and the non-OCR mammoth paths+# ---------------------------------------------------------------------------+++def _underlined_docx(tmp_path: Path) -> Path:+ docx_file = tmp_path / "underlined.docx"+ document_xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">+ <w:body>+ <w:p>+ <w:r><w:t>plain </w:t></w:r>+ <w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>+ </w:p>+ </w:body>+</w:document>"""++ with zipfile.ZipFile(docx_file, "w") as archive:+ archive.writestr(+ "[Content_Types].xml",+ """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">+ <Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>+ <Default Extension="xml" ContentType="application/xml"/>+ <Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>+</Types>""",+ )+ archive.writestr(+ "_rels/.rels",+ """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">+ <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>+</Relationships>""",+ )+ archive.writestr("word/document.xml", document_xml)++ return docx_file+++def test_docx_underlined_text_is_preserved_without_ocr(tmp_path: Path) -> None:+ converter = DocxConverterWithOCR()+ with open(_underlined_docx(tmp_path), "rb") as f:+ md = converter.convert(f, StreamInfo(extension=".docx")).markdown+ assert "plain <u>underlined</u>" in md+++def test_docx_underlined_text_is_preserved_with_ocr(+ tmp_path: Path, svc: MockOCRService+) -> None:+ converter = DocxConverterWithOCR()+ with open(_underlined_docx(tmp_path), "rb") as f:+ md = converter.convert(+ f, StreamInfo(extension=".docx"), ocr_service=svc+ ).markdown+ assert "plain <u>underlined</u>" in mddiff --git a/packages/markitdown/src/markitdown/converters/_docx_converter.py b/packages/markitdown/src/markitdown/converters/_docx_converter.pyindex 0f92b9e6..8248417a 100644--- a/packages/markitdown/src/markitdown/converters/_docx_converter.py+++ b/packages/markitdown/src/markitdown/converters/_docx_converter.py@@ -2,7 +2,7 @@ import sysimport iofrom warnings import warn-from typing import BinaryIO, Any+from typing import BinaryIO, Any, Optionalfrom ._html_converter import HtmlConverterfrom ..converter_utils.docx.pre_process import pre_process_docx@@ -27,6 +27,18 @@ ACCEPTED_MIME_TYPE_PREFIXES = [ACCEPTED_FILE_EXTENSIONS = [".docx"]+_UNDERLINE_STYLE_MAP = "u => u"+++def _read_embedded_style_map(file_stream: BinaryIO) -> Optional[str]:+ """Read the style map embedded in a .docx, if it has one."""+ position = file_stream.tell()+ file_stream.seek(0)+ try:+ return mammoth.read_embedded_style_map(file_stream)+ finally:+ file_stream.seek(position)+class DocxConverter(HtmlConverter):"""@@ -75,9 +87,25 @@ class DocxConverter(HtmlConverter):_dependency_exc_info[2])- style_map = kwargs.get("style_map", None)pre_process_stream = pre_process_docx(file_stream)- return self._html_converter.convert_string(- mammoth.convert_to_html(pre_process_stream, style_map=style_map).value,- **kwargs,++ caller_style_map = kwargs.get("style_map")+ embedded_style_map = _read_embedded_style_map(pre_process_stream)++ style_map = "\n".join(+ part+ for part in (+ caller_style_map,+ embedded_style_map,+ _UNDERLINE_STYLE_MAP,+ )+ if part)++ html_result = mammoth.convert_to_html(+ pre_process_stream,+ style_map=style_map,+ include_embedded_style_map=False,+ ).value++ return self._html_converter.convert_string(html_result, **kwargs)diff --git a/packages/markitdown/src/markitdown/converters/_markdownify.py b/packages/markitdown/src/markitdown/converters/_markdownify.pyindex 0e9fd3bf..4fff2e2c 100644--- a/packages/markitdown/src/markitdown/converters/_markdownify.py+++ b/packages/markitdown/src/markitdown/converters/_markdownify.py@@ -145,6 +145,18 @@ class _CustomMarkdownify(markdownify.MarkdownConverter):return "[x] " if el.has_attr("checked") else "[ ] "return ""+ def convert_u(+ self,+ el: Any,+ text: str,+ convert_as_inline: Optional[bool] = False,+ **kwargs,+ ) -> str:+ prefix, suffix, text = markdownify.chomp(text) # type: ignore+ if not text:+ return ""+ return f"{prefix}<u>{text}</u>{suffix}"+def convert_strike(self, el: Any, text: str, *args, **kwargs) -> str:"""Obsolete <strike> is still in the wild; treat it like <s>/<del>."""return self.convert_s(el, text, *args, **kwargs) # type: ignorediff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.pyindex 57fd6523..83fc05e9 100644--- a/packages/markitdown/tests/test_module_misc.py+++ b/packages/markitdown/tests/test_module_misc.py@@ -8,6 +8,7 @@ import shutilimport zipfileimport pytestfrom types import SimpleNamespace+from typing import Optionalfrom unittest.mock import MagicMockimport markitdown._uri_utils as uri_utils@@ -323,6 +324,83 @@ def test_docx_comments() -> None:validate_strings(result, DOCX_COMMENT_TEST_STRINGS)+def _write_underlined_docx(path, embedded_style_map: Optional[str] = None) -> str:+ """Write a minimal .docx holding one underlined run, and return its path."""+ document_xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">+ <w:body>+ <w:p>+ <w:r><w:t>plain </w:t></w:r>+ <w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>+ </w:p>+ </w:body>+</w:document>"""++ with zipfile.ZipFile(path, "w") as archive:+ archive.writestr(+ "[Content_Types].xml",+ """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">+ <Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>+ <Default Extension="xml" ContentType="application/xml"/>+ <Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>+</Types>""",+ )+ archive.writestr(+ "_rels/.rels",+ """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">+ <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>+</Relationships>""",+ )+ archive.writestr(+ "word/_rels/document.xml.rels",+ """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"></Relationships>""",+ )+ archive.writestr("word/document.xml", document_xml)++ if embedded_style_map is not None:+ import mammoth++ with open(path, "r+b") as f:+ mammoth.embed_style_map(f, embedded_style_map)++ return str(path)+++def test_docx_underlined_text_is_preserved(tmp_path) -> None:+ docx_file = _write_underlined_docx(tmp_path / "underlined.docx")++ result = MarkItDown().convert(docx_file)++ assert "plain <u>underlined</u>" in result.markdown+++def test_docx_embedded_style_map_overrides_underline_default(tmp_path) -> None:+ # A style map embedded in the document takes precedence over the default+ # "u => u" mapping that preserves underlines.+ docx_file = _write_underlined_docx(+ tmp_path / "embedded.docx", embedded_style_map="u => em"+ )++ result = MarkItDown().convert(docx_file)++ assert "plain *underlined*" in result.markdown+ assert "<u>" not in result.markdown+++def test_docx_caller_style_map_overrides_embedded_style_map(tmp_path) -> None:+ # ... and a caller-supplied style map still outranks the embedded one.+ docx_file = _write_underlined_docx(+ tmp_path / "embedded.docx", embedded_style_map="u => em"+ )++ result = MarkItDown(style_map="u => strong").convert(docx_file)++ assert "plain **underlined**" in result.markdown++def test_html_strikethrough_variants(tmp_path) -> None:html = """<!doctype html><html><body>