Python tool for converting files and office documents to Markdown.
Fix character decoding issues with text-like files
Divyansh Singh52b723724c33
diff --git a/pyproject.toml b/pyproject.tomlindex 74df032b..756380a5 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -38,6 +38,7 @@ dependencies = ["youtube-transcript-api","SpeechRecognition","pathvalidate",+ "charset-normalizer",][project.urls]diff --git a/src/markitdown/_markitdown.py b/src/markitdown/_markitdown.pyindex 96997cf3..25786f68 100644--- a/src/markitdown/_markitdown.py+++ b/src/markitdown/_markitdown.py@@ -26,6 +26,7 @@ import pptximport puremagicimport requestsfrom bs4 import BeautifulSoup+from charset_normalizer import from_path# Optional Transcription supporttry:@@ -161,9 +162,7 @@ class PlainTextConverter(DocumentConverter):elif "text/" not in content_type.lower():return None- text_content = ""- with open(local_path, "rt", encoding="utf-8") as fh:- text_content = fh.read()+ text_content = str(from_path(local_path).best())return DocumentConverterResult(title=None,text_content=text_content,diff --git a/tests/test_files/test_mskanji.csv b/tests/test_files/test_mskanji.csvnew file mode 100644index 00000000..d67f5a31--- /dev/null+++ b/tests/test_files/test_mskanji.csv@@ -0,0 +1,4 @@+���O,�N��,�Z��+�������Y,30,����+�O�؉p�q,25,���+�����~,35,���É�diff --git a/tests/test_markitdown.py b/tests/test_markitdown.pyindex 94fd8862..ac088204 100644--- a/tests/test_markitdown.py+++ b/tests/test_markitdown.py@@ -87,6 +87,13 @@ SERP_TEST_EXCLUDES = ["data:image/svg+xml,%3Csvg%20width%3D",]+CSV_CP932_TEST_STRINGS = [+ "名前,年齢,住所",+ "佐藤太郎,30,東京",+ "三木英子,25,大阪",+ "髙橋淳,35,名古屋",+]+@pytest.mark.skipif(skip_remote,@@ -164,6 +171,12 @@ def test_markitdown_local() -> None:for test_string in SERP_TEST_STRINGS:assert test_string in text_content+ ## Test non-UTF-8 encoding+ result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_mskanji.csv"))+ text_content = result.text_content.replace("\\", "")+ for test_string in CSV_CP932_TEST_STRINGS:+ assert test_string in text_content+@pytest.mark.skipif(skip_exiftool,