Skip to content

Python tool for converting files and office documents to Markdown.

Fix ANSI Outlook MSG decoding for Japanese code pages and padded strings (#2462)

afourney40658cdcc50f
diff --git a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py
index 79d7656e..e3668651 100644
--- a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py
@@ -1,4 +1,5 @@
import codecs
+import re
import struct
import sys
from typing import Any, Dict, Union, BinaryIO
@@ -62,8 +63,8 @@ CODEPAGE_CODECS = {
10079: "mac_iceland",
10081: "mac_turkish",
50220: "iso2022_jp",
- 50221: "iso2022_jp",
- 50222: "iso2022_jp",
+ 50221: "iso2022_jp_ext", # Supports ESC ( I for halfwidth Katakana.
+ 50222: "iso2022_jp_ext", # SO/SI also need normalization before decoding.
50225: "iso2022_kr",
51932: "euc_jp",
51936: "gb2312",
@@ -277,11 +278,16 @@ class OutlookMsgConverter(DocumentConverter):
except Exception:
return None
+ # Some writers include trailing NUL terminators or padding. Remove them
+ # before charset detection as well as decoding; str.strip() keeps NULs.
+ data = data.rstrip(b"\x00")
if not data:
return None
if encoding is not None:
try:
+ if encoding == "iso2022_jp_ext":
+ return self._decode_iso2022_jp(data).strip()
return data.decode(encoding).strip()
except (UnicodeDecodeError, LookupError):
pass # The declared code page does not fit; fall back to detection
@@ -291,6 +297,33 @@ class OutlookMsgConverter(DocumentConverter):
return str(detected).strip()
return data.decode("utf-8", errors="ignore").strip()
+ def _decode_iso2022_jp(self, data: bytes) -> str:
+ """Decode Katakana in Windows code pages 50221 and 50222.
+
+ Python's extended codec supports 50221's ESC ( I designation, but leaves
+ 50222's SO/SI controls untouched. Translate those controls to equivalent
+ designations, retaining the preceding mode so SI can restore it even
+ when it is JIS Roman or double-byte Japanese rather than ASCII.
+ """
+ mode = b"\x1b(B"
+
+ def replace_control(match: re.Match[bytes]) -> bytes:
+ nonlocal mode
+ control = match.group()
+ if control == b"\x0e": # SO: temporarily select halfwidth Katakana.
+ return b"\x1b(I"
+ if control == b"\x0f": # SI: restore the preceding designation.
+ return mode
+ mode = control
+ return control
+
+ normalized = re.sub(
+ rb"\x1b(?:\([BJI]|\$[@B]|\$\(D)|[\x0e\x0f]",
+ replace_control,
+ data,
+ )
+ return normalized.decode("iso2022_jp_ext")
+
def _get_stream_data(self, msg: Any, stream_path: str) -> Union[str, None]:
"""Helper to safely extract and decode stream data from the MSG file."""
assert olefile is not None
diff --git a/packages/markitdown/tests/test_outlook_msg_ansi.py b/packages/markitdown/tests/test_outlook_msg_ansi.py
index 57b4db15..0c397233 100644
--- a/packages/markitdown/tests/test_outlook_msg_ansi.py
+++ b/packages/markitdown/tests/test_outlook_msg_ansi.py
@@ -7,8 +7,9 @@ import struct
from unittest.mock import patch
import olefile
+import pytest
-from markitdown import MarkItDown
+from markitdown import DocumentConverterResult, MarkItDown
from markitdown._stream_info import StreamInfo
from markitdown.converters._outlook_msg_converter import OutlookMsgConverter
@@ -104,15 +105,146 @@ def _fake_olefile(streams: dict):
return _FakeOleFileIO
-def _convert(streams: dict) -> str:
+def _convert_result(streams: dict) -> DocumentConverterResult:
with patch.object(olefile, "OleFileIO", _fake_olefile(streams)):
- return (
- OutlookMsgConverter()
- .convert(io.BytesIO(b""), StreamInfo(extension=".msg"))
- .markdown
+ return OutlookMsgConverter().convert(
+ io.BytesIO(b""), StreamInfo(extension=".msg")
)
+def _convert(streams: dict) -> str:
+ return _convert_result(streams).markdown
+
+
[email protected]("codepage", [1252, 0, 99999, 20127])
[email protected]("terminator", [b"\x00", b"\x00\x00"])
+def test_ansi_terminators_are_removed(codepage: int, terminator: bytes) -> None:
+ """Strip terminators with declared, detected, and contradicted code pages."""
+ streams = _ansi_streams(codepage=codepage)
+ for path in streams:
+ if path.endswith("001E"):
+ streams[path] = b" \t" + streams[path] + b"\r\n " + terminator
+
+ result = _convert_result(streams)
+
+ assert result.title == SUBJECT
+ assert result.markdown == (
+ f"# Email Message\n\n**From:** {SENDER}\n**To:** {RECIPIENT}\n"
+ f"**Subject:** {SUBJECT}\n\n## Content\n\n{BODY}"
+ )
+
+
[email protected]("codepage", [1252, 0])
[email protected]("value", [b"", b"\x00", b"\x00\x00"])
+def test_empty_ansi_properties_are_omitted(codepage: int, value: bytes) -> None:
+ streams = _ansi_streams(codepage=codepage)
+ for path in streams:
+ if path.endswith("001E"):
+ streams[path] = value
+
+ result = _convert_result(streams)
+
+ assert not result.title
+ assert result.markdown == "# Email Message\n\n\n## Content"
+
+
+def test_ansi_terminators_are_removed_without_a_detected_charset() -> None:
+ streams = _ansi_streams(encoding="utf-8", codepage=0)
+ for path in streams:
+ streams[path] += b"\x00"
+ with patch("markitdown.converters._outlook_msg_converter.from_bytes") as detect:
+ detect.return_value.best.return_value = None
+ result = _convert_result(streams)
+
+ assert result.title == SUBJECT
+ assert result.markdown.endswith(BODY)
+ assert "\x00" not in result.markdown
+
+
+ "codepage, value, expected",
+ [
+ (50220, b"\x1b$BF|K\\8l\x1b(B", "日本語"),
+ (50221, b"\x1b(I6@6E\x1b(B", "カタカナ"),
+ (50221, b"\x1b$BF|K\\\x1b(I6@6E\x1b(B ABC", "日本カタカナ ABC"),
+ (50222, b"Plain ASCII", "Plain ASCII"),
+ (50222, b"\x0e6@6E\x0f", "カタカナ"),
+ (50222, b"ABC \x0e6@6E\x0f XYZ", "ABC カタカナ XYZ"),
+ # SI must restore the preceding designation, which may not be ASCII.
+ (50222, b"\x1b$BF|\x0e6@6E\x0fK\\\x1b(B", "日カタカナ本"),
+ (50222, b"\x1b(J\\\x0e6@6E\x0f~\x1b(B", "¥カタカナ‾"),
+ (50222, b"\x0e6@\x0f/\x0e6E\x0f", "カタ/カナ"),
+ (50222, b"\x0e6@6E", "カタカナ"),
+ ],
+)
+def test_japanese_codepages_decode_without_detection(
+ codepage: int, value: bytes, expected: str
+) -> None:
+ """Use literal wire bytes so an incorrect encoder cannot mask a decoder bug."""
+ streams = {
+ f"__substg1.0_{tag}001E": value + b"\x00"
+ for tag in (SENDER_TAG, RECIPIENT_TAG, SUBJECT_TAG, BODY_TAG)
+ }
+ streams["__properties_version1.0"] = _properties_stream(
+ {PR_MESSAGE_CODEPAGE: codepage}
+ )
+ with patch(
+ "markitdown.converters._outlook_msg_converter.from_bytes",
+ side_effect=AssertionError("The declared Japanese code page must be honored"),
+ ):
+ result = _convert_result(streams)
+
+ assert result.title == expected
+ assert result.markdown == (
+ f"# Email Message\n\n**From:** {expected}\n**To:** {expected}\n"
+ f"**Subject:** {expected}\n\n## Content\n\n{expected}"
+ )
+
+
+ "codepage, body",
+ [(50221, b"\x1b(I6@6E\x1b(B"), (50222, b"\x0e6@6E\x0f")],
+)
+def test_japanese_internet_codepage_decodes_body(codepage: int, body: bytes) -> None:
+ streams = {
+ "__substg1.0_0037001E": AMBIGUOUS_LATIN.encode("cp1252") + b"\x00",
+ "__substg1.0_1000001E": body + b"\x00",
+ "__properties_version1.0": _properties_stream(
+ {PR_MESSAGE_CODEPAGE: 1252, PR_INTERNET_CPID: codepage}
+ ),
+ }
+ with patch(
+ "markitdown.converters._outlook_msg_converter.from_bytes",
+ side_effect=AssertionError("The declared code pages must be honored"),
+ ):
+ result = _convert_result(streams)
+
+ assert result.title == AMBIGUOUS_LATIN
+ assert result.markdown == (
+ f"# Email Message\n\n**Subject:** {AMBIGUOUS_LATIN}\n\n## Content\n\nカタカナ"
+ )
+
+
+ "codepage, value",
+ [(50221, b"\x1b(I~\x1b(B"), (50222, b"\x0e~\x0f")],
+)
+def test_invalid_japanese_bytes_fall_back_to_detection(
+ codepage: int, value: bytes
+) -> None:
+ """Invalid Katakana must still reach detection with the original bytes."""
+ streams = {
+ "__substg1.0_0037001E": value + b"\x00",
+ "__properties_version1.0": _properties_stream({PR_MESSAGE_CODEPAGE: codepage}),
+ }
+ with patch("markitdown.converters._outlook_msg_converter.from_bytes") as detect:
+ detect.return_value.best.return_value = "Recovered subject"
+ result = _convert_result(streams)
+
+ detect.assert_called_once_with(value)
+ assert result.title == "Recovered subject"
+
+
def test_ansi_message_keeps_headers_and_body() -> None:
"""A non-Unicode .msg must convert like its Unicode counterpart."""
markdown = _convert(_ansi_streams())
@@ -259,6 +391,4 @@ def test_real_unicode_fixture_still_converts() -> None:
if __name__ == "__main__":
- import pytest
-
pytest.main([__file__, "-v"])