Skip to content

Python tool for converting files and office documents to Markdown.

Fix character decoding issues with text-like files

Divyansh Singh52b723724c33
diff --git a/pyproject.toml b/pyproject.toml
index 74df032b..756380a5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -38,6 +38,7 @@ dependencies = [
"youtube-transcript-api",
"SpeechRecognition",
"pathvalidate",
+ "charset-normalizer",
]
[project.urls]
diff --git a/src/markitdown/_markitdown.py b/src/markitdown/_markitdown.py
index 96997cf3..25786f68 100644
--- a/src/markitdown/_markitdown.py
+++ b/src/markitdown/_markitdown.py
@@ -26,6 +26,7 @@ import pptx
import puremagic
import requests
from bs4 import BeautifulSoup
+from charset_normalizer import from_path
# Optional Transcription support
try:
@@ -161,9 +162,7 @@ class PlainTextConverter(DocumentConverter):
elif "text/" not in content_type.lower():
return None
- text_content = ""
- with open(local_path, "rt", encoding="utf-8") as fh:
- text_content = fh.read()
+ text_content = str(from_path(local_path).best())
return DocumentConverterResult(
title=None,
text_content=text_content,
diff --git a/tests/test_files/test_mskanji.csv b/tests/test_files/test_mskanji.csv
new file mode 100644
index 00000000..d67f5a31
--- /dev/null
+++ b/tests/test_files/test_mskanji.csv
@@ -0,0 +1,4 @@
+���O,�N��,�Z��
+�������Y,30,����
+�O�؉p�q,25,���
+�����~,35,����
diff --git a/tests/test_markitdown.py b/tests/test_markitdown.py
index 94fd8862..ac088204 100644
--- a/tests/test_markitdown.py
+++ b/tests/test_markitdown.py
@@ -87,6 +87,13 @@ SERP_TEST_EXCLUDES = [
"data:image/svg+xml,%3Csvg%20width%3D",
]
+CSV_CP932_TEST_STRINGS = [
+ "名前,年齢,住所",
+ "佐藤太郎,30,東京",
+ "三木英子,25,大阪",
+ "髙橋淳,35,名古屋",
+]
+
@pytest.mark.skipif(
skip_remote,
@@ -164,6 +171,12 @@ def test_markitdown_local() -> None:
for test_string in SERP_TEST_STRINGS:
assert test_string in text_content
+ ## Test non-UTF-8 encoding
+ result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_mskanji.csv"))
+ text_content = result.text_content.replace("\\", "")
+ for test_string in CSV_CP932_TEST_STRINGS:
+ assert test_string in text_content
+
@pytest.mark.skipif(
skip_exiftool,