Skip to content

Python tool for converting files and office documents to Markdown.

fix(docx): preserve underlined text (#2017)

Yufeng Heb752951ec7bd
diff --git a/packages/markitdown-ocr/src/markitdown_ocr/_docx_converter_with_ocr.py b/packages/markitdown-ocr/src/markitdown_ocr/_docx_converter_with_ocr.py
index f2463de1..9fd3b07e 100644
--- a/packages/markitdown-ocr/src/markitdown_ocr/_docx_converter_with_ocr.py
+++ b/packages/markitdown-ocr/src/markitdown_ocr/_docx_converter_with_ocr.py
@@ -29,6 +29,18 @@ except ImportError:
# Must be a single token with no special markdown characters.
_PLACEHOLDER = "MARKITDOWNOCRBLOCK{}"
+_UNDERLINE_STYLE_MAP = "u => u"
+
+
+def _read_embedded_style_map(file_stream: BinaryIO) -> Optional[str]:
+ """Read the style map embedded in a .docx, if it has one."""
+ position = file_stream.tell()
+ file_stream.seek(0)
+ try:
+ return mammoth.read_embedded_style_map(file_stream)
+ finally:
+ file_stream.seek(position)
+
class DocxConverterWithOCR(HtmlConverter):
"""
@@ -82,6 +94,20 @@ class DocxConverterWithOCR(HtmlConverter):
kwargs.get("ocr_service") or self.ocr_service
)
+ # Read the embedded style map and combine with any provided style map
+ caller_style_map = kwargs.get("style_map")
+ embedded_style_map = _read_embedded_style_map(file_stream)
+
+ style_map = "\n".join(
+ part
+ for part in (
+ caller_style_map,
+ embedded_style_map,
+ _UNDERLINE_STYLE_MAP,
+ )
+ if part
+ )
+
if ocr_service:
# 1. Extract and OCR images — returns raw text per image
file_stream.seek(0)
@@ -91,7 +117,9 @@ class DocxConverterWithOCR(HtmlConverter):
file_stream.seek(0)
pre_process_stream = pre_process_docx(file_stream)
html_result = mammoth.convert_to_html(
- pre_process_stream, style_map=kwargs.get("style_map")
+ pre_process_stream,
+ style_map=style_map,
+ include_embedded_style_map=False,
).value
# 3. Replace <img> tags with plain placeholder tokens so that
@@ -116,10 +144,13 @@ class DocxConverterWithOCR(HtmlConverter):
return DocumentConverterResult(markdown=md)
else:
# Standard conversion without OCR
- style_map = kwargs.get("style_map", None)
pre_process_stream = pre_process_docx(file_stream)
return self._html_converter.convert_string(
- mammoth.convert_to_html(pre_process_stream, style_map=style_map).value,
+ mammoth.convert_to_html(
+ pre_process_stream,
+ style_map=style_map,
+ include_embedded_style_map=False,
+ ).value,
**kwargs,
)
diff --git a/packages/markitdown-ocr/tests/test_docx_converter.py b/packages/markitdown-ocr/tests/test_docx_converter.py
index 0fb66650..fcd0944b 100644
--- a/packages/markitdown-ocr/tests/test_docx_converter.py
+++ b/packages/markitdown-ocr/tests/test_docx_converter.py
@@ -11,6 +11,7 @@ OCR block format used by the converter:
"""
import sys
+import zipfile
from pathlib import Path
from typing import Any
@@ -221,3 +222,60 @@ def test_docx_no_ocr_service_no_tags() -> None:
md = converter.convert(f, StreamInfo(extension=".docx")).text_content
assert "*[Image OCR]" not in md
assert "[End OCR]*" not in md
+
+
+# ---------------------------------------------------------------------------
+# Underlined runs survive both the OCR and the non-OCR mammoth paths
+# ---------------------------------------------------------------------------
+
+
+def _underlined_docx(tmp_path: Path) -> Path:
+ docx_file = tmp_path / "underlined.docx"
+ document_xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
+ <w:body>
+ <w:p>
+ <w:r><w:t>plain </w:t></w:r>
+ <w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>
+ </w:p>
+ </w:body>
+</w:document>"""
+
+ with zipfile.ZipFile(docx_file, "w") as archive:
+ archive.writestr(
+ "[Content_Types].xml",
+ """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
+ <Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
+ <Default Extension="xml" ContentType="application/xml"/>
+ <Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>
+</Types>""",
+ )
+ archive.writestr(
+ "_rels/.rels",
+ """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
+ <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>
+</Relationships>""",
+ )
+ archive.writestr("word/document.xml", document_xml)
+
+ return docx_file
+
+
+def test_docx_underlined_text_is_preserved_without_ocr(tmp_path: Path) -> None:
+ converter = DocxConverterWithOCR()
+ with open(_underlined_docx(tmp_path), "rb") as f:
+ md = converter.convert(f, StreamInfo(extension=".docx")).markdown
+ assert "plain <u>underlined</u>" in md
+
+
+def test_docx_underlined_text_is_preserved_with_ocr(
+ tmp_path: Path, svc: MockOCRService
+) -> None:
+ converter = DocxConverterWithOCR()
+ with open(_underlined_docx(tmp_path), "rb") as f:
+ md = converter.convert(
+ f, StreamInfo(extension=".docx"), ocr_service=svc
+ ).markdown
+ assert "plain <u>underlined</u>" in md
diff --git a/packages/markitdown/src/markitdown/converters/_docx_converter.py b/packages/markitdown/src/markitdown/converters/_docx_converter.py
index 0f92b9e6..8248417a 100644
--- a/packages/markitdown/src/markitdown/converters/_docx_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_docx_converter.py
@@ -2,7 +2,7 @@ import sys
import io
from warnings import warn
-from typing import BinaryIO, Any
+from typing import BinaryIO, Any, Optional
from ._html_converter import HtmlConverter
from ..converter_utils.docx.pre_process import pre_process_docx
@@ -27,6 +27,18 @@ ACCEPTED_MIME_TYPE_PREFIXES = [
ACCEPTED_FILE_EXTENSIONS = [".docx"]
+_UNDERLINE_STYLE_MAP = "u => u"
+
+
+def _read_embedded_style_map(file_stream: BinaryIO) -> Optional[str]:
+ """Read the style map embedded in a .docx, if it has one."""
+ position = file_stream.tell()
+ file_stream.seek(0)
+ try:
+ return mammoth.read_embedded_style_map(file_stream)
+ finally:
+ file_stream.seek(position)
+
class DocxConverter(HtmlConverter):
"""
@@ -75,9 +87,25 @@ class DocxConverter(HtmlConverter):
_dependency_exc_info[2]
)
- style_map = kwargs.get("style_map", None)
pre_process_stream = pre_process_docx(file_stream)
- return self._html_converter.convert_string(
- mammoth.convert_to_html(pre_process_stream, style_map=style_map).value,
- **kwargs,
+
+ caller_style_map = kwargs.get("style_map")
+ embedded_style_map = _read_embedded_style_map(pre_process_stream)
+
+ style_map = "\n".join(
+ part
+ for part in (
+ caller_style_map,
+ embedded_style_map,
+ _UNDERLINE_STYLE_MAP,
+ )
+ if part
)
+
+ html_result = mammoth.convert_to_html(
+ pre_process_stream,
+ style_map=style_map,
+ include_embedded_style_map=False,
+ ).value
+
+ return self._html_converter.convert_string(html_result, **kwargs)
diff --git a/packages/markitdown/src/markitdown/converters/_markdownify.py b/packages/markitdown/src/markitdown/converters/_markdownify.py
index 0e9fd3bf..4fff2e2c 100644
--- a/packages/markitdown/src/markitdown/converters/_markdownify.py
+++ b/packages/markitdown/src/markitdown/converters/_markdownify.py
@@ -145,6 +145,18 @@ class _CustomMarkdownify(markdownify.MarkdownConverter):
return "[x] " if el.has_attr("checked") else "[ ] "
return ""
+ def convert_u(
+ self,
+ el: Any,
+ text: str,
+ convert_as_inline: Optional[bool] = False,
+ **kwargs,
+ ) -> str:
+ prefix, suffix, text = markdownify.chomp(text) # type: ignore
+ if not text:
+ return ""
+ return f"{prefix}<u>{text}</u>{suffix}"
+
def convert_strike(self, el: Any, text: str, *args, **kwargs) -> str:
"""Obsolete <strike> is still in the wild; treat it like <s>/<del>."""
return self.convert_s(el, text, *args, **kwargs) # type: ignore
diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py
index 57fd6523..83fc05e9 100644
--- a/packages/markitdown/tests/test_module_misc.py
+++ b/packages/markitdown/tests/test_module_misc.py
@@ -8,6 +8,7 @@ import shutil
import zipfile
import pytest
from types import SimpleNamespace
+from typing import Optional
from unittest.mock import MagicMock
import markitdown._uri_utils as uri_utils
@@ -323,6 +324,83 @@ def test_docx_comments() -> None:
validate_strings(result, DOCX_COMMENT_TEST_STRINGS)
+def _write_underlined_docx(path, embedded_style_map: Optional[str] = None) -> str:
+ """Write a minimal .docx holding one underlined run, and return its path."""
+ document_xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
+ <w:body>
+ <w:p>
+ <w:r><w:t>plain </w:t></w:r>
+ <w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>
+ </w:p>
+ </w:body>
+</w:document>"""
+
+ with zipfile.ZipFile(path, "w") as archive:
+ archive.writestr(
+ "[Content_Types].xml",
+ """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
+ <Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
+ <Default Extension="xml" ContentType="application/xml"/>
+ <Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>
+</Types>""",
+ )
+ archive.writestr(
+ "_rels/.rels",
+ """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
+ <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>
+</Relationships>""",
+ )
+ archive.writestr(
+ "word/_rels/document.xml.rels",
+ """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"></Relationships>""",
+ )
+ archive.writestr("word/document.xml", document_xml)
+
+ if embedded_style_map is not None:
+ import mammoth
+
+ with open(path, "r+b") as f:
+ mammoth.embed_style_map(f, embedded_style_map)
+
+ return str(path)
+
+
+def test_docx_underlined_text_is_preserved(tmp_path) -> None:
+ docx_file = _write_underlined_docx(tmp_path / "underlined.docx")
+
+ result = MarkItDown().convert(docx_file)
+
+ assert "plain <u>underlined</u>" in result.markdown
+
+
+def test_docx_embedded_style_map_overrides_underline_default(tmp_path) -> None:
+ # A style map embedded in the document takes precedence over the default
+ # "u => u" mapping that preserves underlines.
+ docx_file = _write_underlined_docx(
+ tmp_path / "embedded.docx", embedded_style_map="u => em"
+ )
+
+ result = MarkItDown().convert(docx_file)
+
+ assert "plain *underlined*" in result.markdown
+ assert "<u>" not in result.markdown
+
+
+def test_docx_caller_style_map_overrides_embedded_style_map(tmp_path) -> None:
+ # ... and a caller-supplied style map still outranks the embedded one.
+ docx_file = _write_underlined_docx(
+ tmp_path / "embedded.docx", embedded_style_map="u => em"
+ )
+
+ result = MarkItDown(style_map="u => strong").convert(docx_file)
+
+ assert "plain **underlined**" in result.markdown
+
+
def test_html_strikethrough_variants(tmp_path) -> None:
html = """<!doctype html>
<html><body>