Python tool for converting files and office documents to Markdown.
Do not convert an undecodable file to the word "None" (#2418)
kevin24b9e122ea7d
diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.pyindex f028cfb2..82fd2974 100644--- a/packages/markitdown/src/markitdown/converters/_csv_converter.py+++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py@@ -76,7 +76,13 @@ class CsvConverter(DocumentConverter):if stream_info.charset:content = file_stream.read().decode(stream_info.charset)else:- content = str(from_bytes(file_stream.read()).best())+ data = file_stream.read()+ detected = from_bytes(data).best()+ content = (+ str(detected)+ if detected is not None+ else data.decode("utf-8", errors="ignore")+ )# Excel and other tools prepend a UTF-8 BOM to CSV exports; strip it so# it does not end up inside the first header cell.diff --git a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py b/packages/markitdown/src/markitdown/converters/_plain_text_converter.pyindex 520a0cea..6a26560a 100644--- a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py+++ b/packages/markitdown/src/markitdown/converters/_plain_text_converter.py@@ -55,6 +55,12 @@ class PlainTextConverter(DocumentConverter):if stream_info.charset:text_content = file_stream.read().decode(stream_info.charset)else:- text_content = str(from_bytes(file_stream.read()).best())+ data = file_stream.read()+ detected = from_bytes(data).best()+ text_content = (+ str(detected)+ if detected is not None+ else data.decode("utf-8", errors="ignore")+ )return DocumentConverterResult(markdown=text_content)diff --git a/packages/markitdown/tests/test_undetectable_charset.py b/packages/markitdown/tests/test_undetectable_charset.pynew file mode 100644index 00000000..eb87a3cf--- /dev/null+++ b/packages/markitdown/tests/test_undetectable_charset.py@@ -0,0 +1,74 @@+#!/usr/bin/env python3 -m pytest+"""Tests for bytes no charset decodes.++``charset_normalizer.from_bytes(data).best()`` returns ``None`` when nothing+decodes the bytes, and ``str(None)`` is the four-character string ``"None"``,+so a binary file handed over with a text extension became a document whose+entire content was the word "None" -- four characters that were never in the+file.+"""++import io++from charset_normalizer import from_bytes++from markitdown import MarkItDown, StreamInfo++# Random bytes: no charset claims them, which is what makes `best()` answer None.+# Fixed rather than generated, so the test does not depend on chance.+UNDECODABLE = bytes((7 * i * i + 251 * i + 193) % 256 for i in range(4096))+++def test_the_fixture_really_is_undecodable() -> None:+ """Guards the guard: if some charset claimed these bytes the rest would pass+ for the wrong reason."""+ assert from_bytes(UNDECODABLE).best() is None+++def test_binary_with_a_text_extension_is_not_the_word_none() -> None:+ markitdown = MarkItDown()+ for extension in (".txt", ".md"):+ result = markitdown.convert_stream(+ io.BytesIO(UNDECODABLE), file_extension=extension+ )+ assert result.markdown != "None"+++def test_binary_with_a_csv_extension_is_not_a_table_of_none() -> None:+ markitdown = MarkItDown()+ result = markitdown.convert_stream(io.BytesIO(UNDECODABLE), file_extension=".csv")+ assert result.markdown != "| None |\n| --- |"+++def test_ordinary_text_is_unchanged() -> None:+ """The fallback only runs when detection fails; everything else is as it was."""+ markitdown = MarkItDown()++ text = markitdown.convert_stream(+ io.BytesIO(b"hello, world\n"), file_extension=".txt"+ )+ assert text.markdown == "hello, world\n"++ table = markitdown.convert_stream(io.BytesIO(b"a,b\n1,2\n"), file_extension=".csv")+ assert table.markdown == "| a | b |\n| --- | --- |\n| 1 | 2 |"+++def test_text_containing_the_word_none_survives() -> None:+ """The bug was a whole document reading "None", not the word appearing in+ one."""+ markitdown = MarkItDown()+ result = markitdown.convert_stream(+ io.BytesIO(b"value,note\nNone,missing\n"), file_extension=".csv"+ )+ assert result.markdown == "| value | note |\n| --- | --- |\n| None | missing |"+++def test_a_declared_charset_still_wins() -> None:+ """`stream_info.charset` short-circuits detection and is untouched."""+ markitdown = MarkItDown()+ body = "名称,代码\n浦发银行,600000\n".encode("gb18030")+ result = markitdown.convert_stream(+ io.BytesIO(body),+ stream_info=StreamInfo(extension=".csv", charset="gb18030"),+ )+ assert "浦发银行" in result.markdown