Python tool for converting files and office documents to Markdown.
Have magika read from the stream. (#1136)
afourneyc5f70b904ffc
diff --git a/packages/markitdown/pyproject.toml b/packages/markitdown/pyproject.tomlindex 0324ed46..bd381939 100644--- a/packages/markitdown/pyproject.toml+++ b/packages/markitdown/pyproject.toml@@ -27,7 +27,7 @@ dependencies = ["beautifulsoup4","requests","markdownify",- "magika>=0.6.1rc2",+ "magika>=0.6.1rc3","charset-normalizer",]diff --git a/packages/markitdown/src/markitdown/__about__.py b/packages/markitdown/src/markitdown/__about__.pyindex 463e0c70..e54f3bc2 100644--- a/packages/markitdown/src/markitdown/__about__.py+++ b/packages/markitdown/src/markitdown/__about__.py@@ -1,4 +1,4 @@# SPDX-FileCopyrightText: 2024-present Adam Fourney <[email protected]>## SPDX-License-Identifier: MIT-__version__ = "0.1.0a3"+__version__ = "0.1.0a4"diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.pyindex 2e9965a6..78319eb5 100644--- a/packages/markitdown/src/markitdown/_markitdown.py+++ b/packages/markitdown/src/markitdown/_markitdown.py@@ -610,14 +610,16 @@ class MarkItDown:# Call magika to guess from the streamcur_pos = file_stream.tell()try:- stream_bytes = file_stream.read()-- result = self._magika.identify_bytes(stream_bytes)+ result = self._magika.identify_stream(file_stream)if result.status == "ok" and result.prediction.output.label != "unknown":# If it's text, also guess the charsetcharset = Noneif result.prediction.output.is_text:- charset_result = charset_normalizer.from_bytes(stream_bytes).best()+ # Read the first 4k to guess the charset+ file_stream.seek(cur_pos)+ stream_page = file_stream.read(4096)+ charset_result = charset_normalizer.from_bytes(stream_page).best()+if charset_result is not None:charset = self._normalize_charset(charset_result.encoding)diff --git a/packages/markitdown/tests/test_module_vectors.py b/packages/markitdown/tests/test_module_vectors.pyindex 873be753..9afffa54 100644--- a/packages/markitdown/tests/test_module_vectors.py+++ b/packages/markitdown/tests/test_module_vectors.py@@ -47,7 +47,6 @@ def test_guess_stream_info(test_vector):# mimetype or extension, so we'll special-case them here.if test_vector.filename in ["test_outlook_msg.msg",- "test_mskanji.csv", # See: https://github.com/google/magika/issues/983]:return@@ -96,15 +95,6 @@ def test_convert_stream_without_hints(test_vector):"""Test the conversion of a stream with no stream info."""markitdown = MarkItDown()- # For some limited exceptions, we can't guarantee the exact- # mimetype or extension, so we'll special-case them here.- if test_vector.filename in [- # This appears to be a subtle bug in magika.- # See: https://github.com/google/magika/issues/983- "test_mskanji.csv",- ]:- return-with open(os.path.join(TEST_FILES_DIR, test_vector.filename), "rb") as stream:result = markitdown.convert(stream, url=test_vector.url)for string in test_vector.must_include: