Skip to content

Python tool for converting files and office documents to Markdown.

Preserve whitepace underlines (#2477)

afourneycc0ca9edd8e2
diff --git a/packages/markitdown-ocr/tests/test_docx_converter.py b/packages/markitdown-ocr/tests/test_docx_converter.py
index f9df439c..8988c033 100644
--- a/packages/markitdown-ocr/tests/test_docx_converter.py
+++ b/packages/markitdown-ocr/tests/test_docx_converter.py
@@ -281,15 +281,19 @@ def test_docx_styles_with_redundant_default_namespace(
# ---------------------------------------------------------------------------
-def _underlined_docx(tmp_path: Path) -> Path:
+def _underlined_docx(
+ tmp_path: Path,
+ *,
+ paragraph_xml: str = (
+ "<w:r><w:t>plain </w:t></w:r>"
+ '<w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>'
+ ),
+) -> Path:
docx_file = tmp_path / "underlined.docx"
- document_xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ document_xml = f"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
<w:body>
- <w:p>
- <w:r><w:t>plain </w:t></w:r>
- <w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>
- </w:p>
+ <w:p>{paragraph_xml}</w:p>
</w:body>
</w:document>"""
@@ -333,6 +337,41 @@ def test_docx_underlined_text_is_preserved_with_ocr(
assert "plain <u>underlined</u>" in md
[email protected]("use_ocr", [False, True])
+ ("run_xml", "expected"),
+ [
+ ('<w:t xml:space="preserve"> </w:t>', "First Last"),
+ ("<w:tab/>", "First Last"),
+ ("<w:t>&#160;</w:t>", "First\u00a0Last"),
+ # Direct conversion keeps the two-space hard break; the dispatcher strips it.
+ ("<w:br/>", "First \nLast"),
+ ],
+)
+def test_docx_underlined_whitespace_is_preserved(
+ tmp_path: Path,
+ svc: MockOCRService,
+ use_ocr: bool,
+ run_xml: str,
+ expected: str,
+) -> None:
+ path = _underlined_docx(
+ tmp_path,
+ paragraph_xml=(
+ "<w:r><w:t>First</w:t></w:r>"
+ f'<w:r><w:rPr><w:u w:val="single"/></w:rPr>{run_xml}</w:r>'
+ "<w:r><w:t>Last</w:t></w:r>"
+ ),
+ )
+ converter = DocxConverterWithOCR()
+ with path.open("rb") as stream:
+ result = converter.convert(
+ stream, StreamInfo(extension=".docx"), ocr_service=svc if use_ocr else None
+ )
+
+ assert result.markdown == expected
+
+
# ---------------------------------------------------------------------------
# ZIP local file header casing mismatch
# ---------------------------------------------------------------------------
diff --git a/packages/markitdown/src/markitdown/converters/_markdownify.py b/packages/markitdown/src/markitdown/converters/_markdownify.py
index e0bbcaf2..ed341448 100644
--- a/packages/markitdown/src/markitdown/converters/_markdownify.py
+++ b/packages/markitdown/src/markitdown/converters/_markdownify.py
@@ -163,6 +163,9 @@ class _CustomMarkdownify(markdownify.MarkdownConverter):
convert_as_inline: Optional[bool] = False,
**kwargs,
) -> str:
+ if not text.strip():
+ return text
+
prefix, suffix, text = markdownify.chomp(text) # type: ignore
if not text:
return ""
diff --git a/packages/markitdown/tests/test_html_converter.py b/packages/markitdown/tests/test_html_converter.py
index e5643519..3c5d8976 100644
--- a/packages/markitdown/tests/test_html_converter.py
+++ b/packages/markitdown/tests/test_html_converter.py
@@ -1,6 +1,10 @@
import io
+import pytest
+from bs4 import BeautifulSoup
+
from markitdown import MarkItDown
+from markitdown.converters._markdownify import _CustomMarkdownify
def _convert_html(html: str, **kwargs) -> str:
@@ -12,6 +16,31 @@ def _convert_html(html: str, **kwargs) -> str:
return result.markdown
+ "whitespace", ["", " ", " ", "\t", "\n", "\r\n", "\u00a0", " \t\n\u00a0 "]
+)
+def test_underline_preserves_whitespace_verbatim(whitespace: str) -> None:
+ element = BeautifulSoup("<u></u>", "html.parser").u
+
+ assert _CustomMarkdownify().convert_u(element, whitespace) == whitespace
+
+
+ ("content", "expected"),
+ [
+ ("", "FirstLast"),
+ (" ", "First Last"),
+ ("\t", "First Last"),
+ ("&#160;", "First\u00a0Last"),
+ ("<br>", "First\nLast"),
+ ("word", "First<u>word</u>Last"),
+ (" word ", "First <u>word</u> Last"),
+ ],
+)
+def test_html_underlined_content_is_preserved(content: str, expected: str) -> None:
+ assert _convert_html(f"<p>First<u>{content}</u>Last</p>") == expected
+
+
def test_preserves_non_utf8_percent_encoded_href_path() -> None:
href = "https://abc.com/hist/" "%a5%c8%a5%c3%a5%d7%a5%da%a1%bc%a5%b8"
html = f'<a href="{href}">example</a>'
diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py
index 18690d18..434cd3c5 100644
--- a/packages/markitdown/tests/test_module_misc.py
+++ b/packages/markitdown/tests/test_module_misc.py
@@ -387,15 +387,20 @@ def test_docx_comments() -> None:
validate_strings(result, DOCX_COMMENT_TEST_STRINGS)
-def _write_underlined_docx(path, embedded_style_map: Optional[str] = None) -> str:
+def _write_underlined_docx(
+ path,
+ embedded_style_map: Optional[str] = None,
+ *,
+ paragraph_xml: str = (
+ "<w:r><w:t>plain </w:t></w:r>"
+ '<w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>'
+ ),
+) -> str:
"""Write a minimal .docx holding one underlined run, and return its path."""
- document_xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ document_xml = f"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
<w:body>
- <w:p>
- <w:r><w:t>plain </w:t></w:r>
- <w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>
- </w:p>
+ <w:p>{paragraph_xml}</w:p>
</w:body>
</w:document>"""
@@ -440,6 +445,30 @@ def test_docx_underlined_text_is_preserved(tmp_path) -> None:
assert "plain <u>underlined</u>" in result.markdown
+ ("run_xml", "expected"),
+ [
+ ('<w:t xml:space="preserve"> </w:t>', "First Last"),
+ ("<w:tab/>", "First Last"),
+ ("<w:t>&#160;</w:t>", "First\u00a0Last"),
+ ("<w:br/>", "First\nLast"),
+ ],
+)
+def test_docx_underlined_whitespace_is_preserved(
+ tmp_path, run_xml: str, expected: str
+) -> None:
+ docx_file = _write_underlined_docx(
+ tmp_path / "underlined_whitespace.docx",
+ paragraph_xml=(
+ "<w:r><w:t>First</w:t></w:r>"
+ f'<w:r><w:rPr><w:u w:val="single"/></w:rPr>{run_xml}</w:r>'
+ "<w:r><w:t>Last</w:t></w:r>"
+ ),
+ )
+
+ assert MarkItDown().convert(docx_file).markdown == expected
+
+
def test_docx_embedded_style_map_overrides_underline_default(tmp_path) -> None:
# A style map embedded in the document takes precedence over the default
# "u => u" mapping that preserves underlines.