Skip to content

Python tool for converting files and office documents to Markdown.

Do not convert an undecodable file to the word "None" (#2418)

kevin24b9e122ea7d
diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py
index f028cfb2..82fd2974 100644
--- a/packages/markitdown/src/markitdown/converters/_csv_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py
@@ -76,7 +76,13 @@ class CsvConverter(DocumentConverter):
if stream_info.charset:
content = file_stream.read().decode(stream_info.charset)
else:
- content = str(from_bytes(file_stream.read()).best())
+ data = file_stream.read()
+ detected = from_bytes(data).best()
+ content = (
+ str(detected)
+ if detected is not None
+ else data.decode("utf-8", errors="ignore")
+ )
# Excel and other tools prepend a UTF-8 BOM to CSV exports; strip it so
# it does not end up inside the first header cell.
diff --git a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py b/packages/markitdown/src/markitdown/converters/_plain_text_converter.py
index 520a0cea..6a26560a 100644
--- a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_plain_text_converter.py
@@ -55,6 +55,12 @@ class PlainTextConverter(DocumentConverter):
if stream_info.charset:
text_content = file_stream.read().decode(stream_info.charset)
else:
- text_content = str(from_bytes(file_stream.read()).best())
+ data = file_stream.read()
+ detected = from_bytes(data).best()
+ text_content = (
+ str(detected)
+ if detected is not None
+ else data.decode("utf-8", errors="ignore")
+ )
return DocumentConverterResult(markdown=text_content)
diff --git a/packages/markitdown/tests/test_undetectable_charset.py b/packages/markitdown/tests/test_undetectable_charset.py
new file mode 100644
index 00000000..eb87a3cf
--- /dev/null
+++ b/packages/markitdown/tests/test_undetectable_charset.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3 -m pytest
+"""Tests for bytes no charset decodes.
+
+``charset_normalizer.from_bytes(data).best()`` returns ``None`` when nothing
+decodes the bytes, and ``str(None)`` is the four-character string ``"None"``,
+so a binary file handed over with a text extension became a document whose
+entire content was the word "None" -- four characters that were never in the
+file.
+"""
+
+import io
+
+from charset_normalizer import from_bytes
+
+from markitdown import MarkItDown, StreamInfo
+
+# Random bytes: no charset claims them, which is what makes `best()` answer None.
+# Fixed rather than generated, so the test does not depend on chance.
+UNDECODABLE = bytes((7 * i * i + 251 * i + 193) % 256 for i in range(4096))
+
+
+def test_the_fixture_really_is_undecodable() -> None:
+ """Guards the guard: if some charset claimed these bytes the rest would pass
+ for the wrong reason."""
+ assert from_bytes(UNDECODABLE).best() is None
+
+
+def test_binary_with_a_text_extension_is_not_the_word_none() -> None:
+ markitdown = MarkItDown()
+ for extension in (".txt", ".md"):
+ result = markitdown.convert_stream(
+ io.BytesIO(UNDECODABLE), file_extension=extension
+ )
+ assert result.markdown != "None"
+
+
+def test_binary_with_a_csv_extension_is_not_a_table_of_none() -> None:
+ markitdown = MarkItDown()
+ result = markitdown.convert_stream(io.BytesIO(UNDECODABLE), file_extension=".csv")
+ assert result.markdown != "| None |\n| --- |"
+
+
+def test_ordinary_text_is_unchanged() -> None:
+ """The fallback only runs when detection fails; everything else is as it was."""
+ markitdown = MarkItDown()
+
+ text = markitdown.convert_stream(
+ io.BytesIO(b"hello, world\n"), file_extension=".txt"
+ )
+ assert text.markdown == "hello, world\n"
+
+ table = markitdown.convert_stream(io.BytesIO(b"a,b\n1,2\n"), file_extension=".csv")
+ assert table.markdown == "| a | b |\n| --- | --- |\n| 1 | 2 |"
+
+
+def test_text_containing_the_word_none_survives() -> None:
+ """The bug was a whole document reading "None", not the word appearing in
+ one."""
+ markitdown = MarkItDown()
+ result = markitdown.convert_stream(
+ io.BytesIO(b"value,note\nNone,missing\n"), file_extension=".csv"
+ )
+ assert result.markdown == "| value | note |\n| --- | --- |\n| None | missing |"
+
+
+def test_a_declared_charset_still_wins() -> None:
+ """`stream_info.charset` short-circuits detection and is untouched."""
+ markitdown = MarkItDown()
+ body = "名称,代码\n浦发银行,600000\n".encode("gb18030")
+ result = markitdown.convert_stream(
+ io.BytesIO(body),
+ stream_info=StreamInfo(extension=".csv", charset="gb18030"),
+ )
+ assert "浦发银行" in result.markdown