Skip to content

Python tool for converting files and office documents to Markdown.

Mitigate UnicodeDecodeError due to wrong ASCII charset guess for long files (#2360)

Fabio Catalano4243c26b3abb
diff --git a/packages/markitdown/src/markitdown/_exceptions.py b/packages/markitdown/src/markitdown/_exceptions.py
index cd5c5ec0..baa2a08b 100644
--- a/packages/markitdown/src/markitdown/_exceptions.py
+++ b/packages/markitdown/src/markitdown/_exceptions.py
@@ -73,4 +73,10 @@ class FileConversionException(MarkItDownException):
else:
message += f" - {type(attempt.converter).__name__} threw {attempt.exc_info[0].__name__} with message: {attempt.exc_info[1]}\n"
+ if isinstance(attempt.exc_info[1], UnicodeDecodeError):
+ message += (
+ "The charset may have been automatically guessed incorrectly. "
+ "If you didn't specify the charset, please retry with an explicitly defined one.\n"
+ )
+
super().__init__(message)
diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.py
index 2b4f2a69..ca435d5f 100644
--- a/packages/markitdown/src/markitdown/_markitdown.py
+++ b/packages/markitdown/src/markitdown/_markitdown.py
@@ -726,9 +726,9 @@ class MarkItDown:
# If it's text, also guess the charset
charset = None
if result.prediction.output.is_text:
- # Read the first 4k to guess the charset
+ # Read the first 64k to guess the charset
file_stream.seek(cur_pos)
- stream_page = file_stream.read(4096)
+ stream_page = file_stream.read(65536)
charset_result = charset_normalizer.from_bytes(stream_page).best()
if charset_result is not None:
diff --git a/packages/markitdown/src/markitdown/converters/_ipynb_converter.py b/packages/markitdown/src/markitdown/converters/_ipynb_converter.py
index b15e77aa..dae0643d 100644
--- a/packages/markitdown/src/markitdown/converters/_ipynb_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_ipynb_converter.py
@@ -38,6 +38,8 @@ class IpynbConverter(DocumentConverter):
"nbformat" in notebook_content
and "nbformat_minor" in notebook_content
)
+ except Exception as e:
+ pass
finally:
file_stream.seek(cur_pos)
diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py
index f966eee1..6b46f612 100644
--- a/packages/markitdown/tests/test_module_misc.py
+++ b/packages/markitdown/tests/test_module_misc.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python3 -m pytest
import io
+import json
import os
import re
import shutil
@@ -587,6 +588,24 @@ def test_markitdown_llm() -> None:
validate_strings(result, PPTX_TEST_STRINGS)
+def test_json_with_late_non_ascii_character(tmp_path) -> None:
+ payload = {
+ "record": {
+ "title": "Example record",
+ "abstract": "This is sample test. " * 500,
+ },
+ "notes": "non-ASCII character: è",
+ }
+ json_path = tmp_path / "input.json"
+ json_path.write_text(
+ json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
+ )
+
+ result = MarkItDown().convert(str(json_path))
+
+ assert "non-ASCII character: è" in result.text_content
+
+
if __name__ == "__main__":
"""Runs this file's tests from the command line."""
for test in [