Python tool for converting files and office documents to Markdown.
Preserve whitepace underlines (#2477)
afourneycc0ca9edd8e2
diff --git a/packages/markitdown-ocr/tests/test_docx_converter.py b/packages/markitdown-ocr/tests/test_docx_converter.pyindex f9df439c..8988c033 100644--- a/packages/markitdown-ocr/tests/test_docx_converter.py+++ b/packages/markitdown-ocr/tests/test_docx_converter.py@@ -281,15 +281,19 @@ def test_docx_styles_with_redundant_default_namespace(# ----------------------------------------------------------------------------def _underlined_docx(tmp_path: Path) -> Path:+def _underlined_docx(+ tmp_path: Path,+ *,+ paragraph_xml: str = (+ "<w:r><w:t>plain </w:t></w:r>"+ '<w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>'+ ),+) -> Path:docx_file = tmp_path / "underlined.docx"- document_xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+ document_xml = f"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>- <w:p>- <w:r><w:t>plain </w:t></w:r>- <w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>- </w:p>+ <w:p>{paragraph_xml}</w:p></w:body></w:document>"""@@ -333,6 +337,41 @@ def test_docx_underlined_text_is_preserved_with_ocr(assert "plain <u>underlined</u>" in md[email protected]("use_ocr", [False, True])+ ("run_xml", "expected"),+ [+ ('<w:t xml:space="preserve"> </w:t>', "First Last"),+ ("<w:tab/>", "First Last"),+ ("<w:t> </w:t>", "First\u00a0Last"),+ # Direct conversion keeps the two-space hard break; the dispatcher strips it.+ ("<w:br/>", "First \nLast"),+ ],+)+def test_docx_underlined_whitespace_is_preserved(+ tmp_path: Path,+ svc: MockOCRService,+ use_ocr: bool,+ run_xml: str,+ expected: str,+) -> None:+ path = _underlined_docx(+ tmp_path,+ paragraph_xml=(+ "<w:r><w:t>First</w:t></w:r>"+ f'<w:r><w:rPr><w:u w:val="single"/></w:rPr>{run_xml}</w:r>'+ "<w:r><w:t>Last</w:t></w:r>"+ ),+ )+ converter = DocxConverterWithOCR()+ with path.open("rb") as stream:+ result = converter.convert(+ stream, StreamInfo(extension=".docx"), ocr_service=svc if use_ocr else None+ )++ assert result.markdown == expected++# ---------------------------------------------------------------------------# ZIP local file header casing mismatch# ---------------------------------------------------------------------------diff --git a/packages/markitdown/src/markitdown/converters/_markdownify.py b/packages/markitdown/src/markitdown/converters/_markdownify.pyindex e0bbcaf2..ed341448 100644--- a/packages/markitdown/src/markitdown/converters/_markdownify.py+++ b/packages/markitdown/src/markitdown/converters/_markdownify.py@@ -163,6 +163,9 @@ class _CustomMarkdownify(markdownify.MarkdownConverter):convert_as_inline: Optional[bool] = False,**kwargs,) -> str:+ if not text.strip():+ return text+prefix, suffix, text = markdownify.chomp(text) # type: ignoreif not text:return ""diff --git a/packages/markitdown/tests/test_html_converter.py b/packages/markitdown/tests/test_html_converter.pyindex e5643519..3c5d8976 100644--- a/packages/markitdown/tests/test_html_converter.py+++ b/packages/markitdown/tests/test_html_converter.py@@ -1,6 +1,10 @@import io+import pytest+from bs4 import BeautifulSoup+from markitdown import MarkItDown+from markitdown.converters._markdownify import _CustomMarkdownifydef _convert_html(html: str, **kwargs) -> str:@@ -12,6 +16,31 @@ def _convert_html(html: str, **kwargs) -> str:return result.markdown+ "whitespace", ["", " ", " ", "\t", "\n", "\r\n", "\u00a0", " \t\n\u00a0 "]+)+def test_underline_preserves_whitespace_verbatim(whitespace: str) -> None:+ element = BeautifulSoup("<u></u>", "html.parser").u++ assert _CustomMarkdownify().convert_u(element, whitespace) == whitespace+++ ("content", "expected"),+ [+ ("", "FirstLast"),+ (" ", "First Last"),+ ("\t", "First Last"),+ (" ", "First\u00a0Last"),+ ("<br>", "First\nLast"),+ ("word", "First<u>word</u>Last"),+ (" word ", "First <u>word</u> Last"),+ ],+)+def test_html_underlined_content_is_preserved(content: str, expected: str) -> None:+ assert _convert_html(f"<p>First<u>{content}</u>Last</p>") == expected++def test_preserves_non_utf8_percent_encoded_href_path() -> None:href = "https://abc.com/hist/" "%a5%c8%a5%c3%a5%d7%a5%da%a1%bc%a5%b8"html = f'<a href="{href}">example</a>'diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.pyindex 18690d18..434cd3c5 100644--- a/packages/markitdown/tests/test_module_misc.py+++ b/packages/markitdown/tests/test_module_misc.py@@ -387,15 +387,20 @@ def test_docx_comments() -> None:validate_strings(result, DOCX_COMMENT_TEST_STRINGS)-def _write_underlined_docx(path, embedded_style_map: Optional[str] = None) -> str:+def _write_underlined_docx(+ path,+ embedded_style_map: Optional[str] = None,+ *,+ paragraph_xml: str = (+ "<w:r><w:t>plain </w:t></w:r>"+ '<w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>'+ ),+) -> str:"""Write a minimal .docx holding one underlined run, and return its path."""- document_xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+ document_xml = f"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>- <w:p>- <w:r><w:t>plain </w:t></w:r>- <w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>- </w:p>+ <w:p>{paragraph_xml}</w:p></w:body></w:document>"""@@ -440,6 +445,30 @@ def test_docx_underlined_text_is_preserved(tmp_path) -> None:assert "plain <u>underlined</u>" in result.markdown+ ("run_xml", "expected"),+ [+ ('<w:t xml:space="preserve"> </w:t>', "First Last"),+ ("<w:tab/>", "First Last"),+ ("<w:t> </w:t>", "First\u00a0Last"),+ ("<w:br/>", "First\nLast"),+ ],+)+def test_docx_underlined_whitespace_is_preserved(+ tmp_path, run_xml: str, expected: str+) -> None:+ docx_file = _write_underlined_docx(+ tmp_path / "underlined_whitespace.docx",+ paragraph_xml=(+ "<w:r><w:t>First</w:t></w:r>"+ f'<w:r><w:rPr><w:u w:val="single"/></w:rPr>{run_xml}</w:r>'+ "<w:r><w:t>Last</w:t></w:r>"+ ),+ )++ assert MarkItDown().convert(docx_file).markdown == expected++def test_docx_embedded_style_map_overrides_underline_default(tmp_path) -> None:# A style map embedded in the document takes precedence over the default# "u => u" mapping that preserves underlines.