Python tool for converting files and office documents to Markdown.
Mitigate UnicodeDecodeError due to wrong ASCII charset guess for long files (#2360)
Fabio Catalano4243c26b3abb
diff --git a/packages/markitdown/src/markitdown/_exceptions.py b/packages/markitdown/src/markitdown/_exceptions.pyindex cd5c5ec0..baa2a08b 100644--- a/packages/markitdown/src/markitdown/_exceptions.py+++ b/packages/markitdown/src/markitdown/_exceptions.py@@ -73,4 +73,10 @@ class FileConversionException(MarkItDownException):else:message += f" - {type(attempt.converter).__name__} threw {attempt.exc_info[0].__name__} with message: {attempt.exc_info[1]}\n"+ if isinstance(attempt.exc_info[1], UnicodeDecodeError):+ message += (+ "The charset may have been automatically guessed incorrectly. "+ "If you didn't specify the charset, please retry with an explicitly defined one.\n"+ )+super().__init__(message)diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.pyindex 2b4f2a69..ca435d5f 100644--- a/packages/markitdown/src/markitdown/_markitdown.py+++ b/packages/markitdown/src/markitdown/_markitdown.py@@ -726,9 +726,9 @@ class MarkItDown:# If it's text, also guess the charsetcharset = Noneif result.prediction.output.is_text:- # Read the first 4k to guess the charset+ # Read the first 64k to guess the charsetfile_stream.seek(cur_pos)- stream_page = file_stream.read(4096)+ stream_page = file_stream.read(65536)charset_result = charset_normalizer.from_bytes(stream_page).best()if charset_result is not None:diff --git a/packages/markitdown/src/markitdown/converters/_ipynb_converter.py b/packages/markitdown/src/markitdown/converters/_ipynb_converter.pyindex b15e77aa..dae0643d 100644--- a/packages/markitdown/src/markitdown/converters/_ipynb_converter.py+++ b/packages/markitdown/src/markitdown/converters/_ipynb_converter.py@@ -38,6 +38,8 @@ class IpynbConverter(DocumentConverter):"nbformat" in notebook_contentand "nbformat_minor" in notebook_content)+ except Exception as e:+ passfinally:file_stream.seek(cur_pos)diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.pyindex f966eee1..6b46f612 100644--- a/packages/markitdown/tests/test_module_misc.py+++ b/packages/markitdown/tests/test_module_misc.py@@ -1,5 +1,6 @@#!/usr/bin/env python3 -m pytestimport io+import jsonimport osimport reimport shutil@@ -587,6 +588,24 @@ def test_markitdown_llm() -> None:validate_strings(result, PPTX_TEST_STRINGS)+def test_json_with_late_non_ascii_character(tmp_path) -> None:+ payload = {+ "record": {+ "title": "Example record",+ "abstract": "This is sample test. " * 500,+ },+ "notes": "non-ASCII character: è",+ }+ json_path = tmp_path / "input.json"+ json_path.write_text(+ json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"+ )++ result = MarkItDown().convert(str(json_path))++ assert "non-ASCII character: è" in result.text_content++if __name__ == "__main__":"""Runs this file's tests from the command line."""for test in [