Python tool for converting files and office documents to Markdown.
fix: buffer CLI stdin before format detection on Windows (#2351)
Mo Huibebea8bddd09
diff --git a/packages/markitdown/src/markitdown/__main__.py b/packages/markitdown/src/markitdown/__main__.pyindex b2575540..31c0aea8 100644--- a/packages/markitdown/src/markitdown/__main__.py+++ b/packages/markitdown/src/markitdown/__main__.py@@ -5,6 +5,7 @@ import argparseimport osimport sysimport codecs+import iofrom typing import Any, Dictfrom textwrap import dedentfrom importlib.metadata import entry_points@@ -248,8 +249,9 @@ def main():markitdown = MarkItDown(enable_plugins=args.use_plugins)if args.filename is None:+ # Windows pipe-backed stdin can report seekable() even though it cannot rewind.result = markitdown.convert_stream(- sys.stdin.buffer,+ io.BytesIO(sys.stdin.buffer.read()),stream_info=stream_info,keep_data_uris=args.keep_data_uris,)diff --git a/packages/markitdown/tests/test_cli_misc.py b/packages/markitdown/tests/test_cli_misc.pyindex cf6c9ccc..c059d744 100644--- a/packages/markitdown/tests/test_cli_misc.py+++ b/packages/markitdown/tests/test_cli_misc.py@@ -1,6 +1,11 @@#!/usr/bin/env python3 -m pytest+import ioimport subprocess+import sys+from types import SimpleNamespace+from markitdown import __version__+from markitdown.__main__ import main# This file contains CLI tests that are not directly tested by the FileTestVectors.# This includes things like help messages, version numbers, and invalid flags.@@ -8,7 +13,9 @@ from markitdown import __version__def test_version() -> None:result = subprocess.run(- ["python", "-m", "markitdown", "--version"], capture_output=True, text=True+ [sys.executable, "-m", "markitdown", "--version"],+ capture_output=True,+ text=True,)assert result.returncode == 0, f"CLI exited with error: {result.stderr}"@@ -17,7 +24,9 @@ def test_version() -> None:def test_invalid_flag() -> None:result = subprocess.run(- ["python", "-m", "markitdown", "--foobar"], capture_output=True, text=True+ [sys.executable, "-m", "markitdown", "--foobar"],+ capture_output=True,+ text=True,)assert result.returncode != 0, f"CLI exited with error: {result.stderr}"@@ -27,6 +36,25 @@ def test_invalid_flag() -> None:assert "SYNTAX" in result.stderr, "Expected 'SYNTAX' to appear in STDERR"+def test_windows_pipe_input_is_buffered_before_conversion(monkeypatch, capsys) -> None:+ class WindowsPipe(io.BytesIO):+ def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:+ if whence == io.SEEK_END:+ return super().seek(offset, whence)+ return 0++ stdin = SimpleNamespace(+ buffer=WindowsPipe(b"<html><body><h1>Test HTML</h1></body></html>")+ )+ monkeypatch.setattr(sys, "stdin", stdin)+ monkeypatch.setattr(sys, "argv", ["markitdown", "-x", "html"])++ main()++ captured = capsys.readouterr()+ assert captured.out.strip() == "# Test HTML"++if __name__ == "__main__":"""Runs this file's tests from the command line."""test_version()diff --git a/packages/markitdown/tests/test_cli_vectors.py b/packages/markitdown/tests/test_cli_vectors.pyindex 547c7c00..04e0754b 100644--- a/packages/markitdown/tests/test_cli_vectors.py+++ b/packages/markitdown/tests/test_cli_vectors.py@@ -4,6 +4,7 @@ import timeimport pytestimport subprocessimport locale+import sysfrom typing import Listif __name__ == "__main__":@@ -46,7 +47,7 @@ def test_output_to_stdout(shared_tmp_dir, test_vector) -> None:result = subprocess.run([- "python",+ sys.executable,"-m","markitdown",os.path.join(TEST_FILES_DIR, test_vector.filename),@@ -69,7 +70,7 @@ def test_output_to_file(shared_tmp_dir, test_vector) -> None:output_file = os.path.join(shared_tmp_dir, test_vector.filename + ".output")result = subprocess.run([- "python",+ sys.executable,"-m","markitdown","-o",@@ -83,7 +84,7 @@ def test_output_to_file(shared_tmp_dir, test_vector) -> None:assert result.returncode == 0, f"CLI exited with error: {result.stderr}"assert os.path.exists(output_file), f"Output file not created: {output_file}"- with open(output_file, "r") as f:+ with open(output_file, "r", encoding="utf-8") as f:output_data = f.read()for test_string in test_vector.must_include:assert test_string in output_data@@ -104,10 +105,9 @@ def test_input_from_stdin_without_hints(shared_tmp_dir, test_vector) -> None:result = subprocess.run([- "python",+ sys.executable,"-m","markitdown",- os.path.join(TEST_FILES_DIR, test_vector.filename),],input=test_input,capture_output=True,@@ -135,7 +135,12 @@ def test_convert_url(shared_tmp_dir, test_vector):time.sleep(1) # Ensure we don't hit rate limitsresult = subprocess.run(- ["python", "-m", "markitdown", TEST_FILES_URL + "/" + test_vector.filename],+ [+ sys.executable,+ "-m",+ "markitdown",+ TEST_FILES_URL + "/" + test_vector.filename,+ ],capture_output=True,text=False,)@@ -155,7 +160,7 @@ def test_output_to_file_with_data_uris(shared_tmp_dir, test_vector) -> None:output_file = os.path.join(shared_tmp_dir, test_vector.filename + ".output")result = subprocess.run([- "python",+ sys.executable,"-m","markitdown","--keep-data-uris",@@ -170,7 +175,7 @@ def test_output_to_file_with_data_uris(shared_tmp_dir, test_vector) -> None:assert result.returncode == 0, f"CLI exited with error: {result.stderr}"assert os.path.exists(output_file), f"Output file not created: {output_file}"- with open(output_file, "r") as f:+ with open(output_file, "r", encoding="utf-8") as f:output_data = f.read()for test_string in test_vector.must_include:assert test_string in output_datadiff --git a/packages/markitdown/tests/test_cu_converter.py b/packages/markitdown/tests/test_cu_converter.pyindex 20b517f7..487407bb 100644--- a/packages/markitdown/tests/test_cu_converter.py+++ b/packages/markitdown/tests/test_cu_converter.py@@ -932,11 +932,11 @@ class TestCLIArgs:enable_plugins=False,cu_endpoint="https://fake-cu",)- markitdown_instance.convert_stream.assert_called_once_with(- input_buffer,- stream_info=None,- keep_data_uris=False,- )++ assert markitdown_instance.convert_stream.call_count == 1+ call_args, call_kwargs = markitdown_instance.convert_stream.call_args+ assert call_args[0].read() == b"fake pdf"+ assert call_kwargs == {"stream_info": None, "keep_data_uris": False}assert capsys.readouterr().out == "converted\n"