Python tool for converting files and office documents to Markdown.
Throw an error when llm client fails with image converter. (#2476)
afourney3a6ce44358e6
diff --git a/README.md b/README.mdindex 62d526f6..5c5a0cca 100644--- a/README.md+++ b/README.md@@ -287,12 +287,16 @@ To use Large Language Models for image descriptions (currently only for pptx andfrom markitdown import MarkItDownfrom openai import OpenAI-client = OpenAI()+client = OpenAI(max_retries=5)md = MarkItDown(llm_client=client, llm_model="gpt-4o", llm_prompt="optional custom prompt")result = md.convert("example.jpg")print(result.markdown)```+`max_retries` controls the OpenAI client's automatic retries for retryable errors (the default is 2); `5` allows up to six attempts with backoff. See the [OpenAI SDK retry documentation](https://github.com/openai/openai-python#retries).++If any attempt succeeds, image conversion continues normally. If the client raises an error after exhausting its retries, or encounters a non-retryable error, MarkItDown tries other applicable converters and raises `FileConversionException` only if none succeeds.+### Docker```shdiff --git a/packages/markitdown/src/markitdown/converters/_image_converter.py b/packages/markitdown/src/markitdown/converters/_image_converter.pyindex cd49b96d..dd8fbac6 100644--- a/packages/markitdown/src/markitdown/converters/_image_converter.py+++ b/packages/markitdown/src/markitdown/converters/_image_converter.py@@ -134,8 +134,5 @@ class ImageConverter(DocumentConverter):]# Call the OpenAI API- try:- response = client.chat.completions.create(model=model, messages=messages)- return response.choices[0].message.content- except Exception:- return None+ response = client.chat.completions.create(model=model, messages=messages)+ return response.choices[0].message.contentdiff --git a/packages/markitdown/tests/test_image_converter.py b/packages/markitdown/tests/test_image_converter.pynew file mode 100644index 00000000..ffb10102--- /dev/null+++ b/packages/markitdown/tests/test_image_converter.py@@ -0,0 +1,137 @@+import io+from pathlib import Path+from typing import Any, BinaryIO, Optional+from unittest.mock import MagicMock++import pytest++from markitdown import (+ DocumentConverterResult,+ FileConversionException,+ MarkItDown,+ StreamInfo,+)+from markitdown.converters import ImageConverter+import markitdown.converters._image_converter as image_module++IMAGE_FILE = Path(__file__).parent / "test_files" / "test.jpg"++[email protected]("metadata", [{}, {"Title": "Image title"}])[email protected]("use_dispatcher", [False, True])+def test_image_caption_errors_propagate(+ metadata: dict[str, str],+ use_dispatcher: bool,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(+ image_module, "exiftool_metadata", MagicMock(return_value=metadata)+ )+ error = RuntimeError("Caption request failed after client retries")+ client = MagicMock()+ client.chat.completions.create.side_effect = error++ if use_dispatcher:+ md = MarkItDown(llm_client=client, llm_model="test-model")+ with pytest.raises(FileConversionException, match=str(error)) as conversion_exc:+ md.convert(IMAGE_FILE)++ attempts = conversion_exc.value.attempts+ assert attempts is not None+ assert any(+ isinstance(attempt.converter, ImageConverter)+ and attempt.exc_info is not None+ and attempt.exc_info[1] is error+ for attempt in attempts+ )+ else:+ prefix = b"ignored prefix"+ stream = io.BytesIO(prefix + IMAGE_FILE.read_bytes())+ stream.seek(len(prefix))+ with pytest.raises(RuntimeError) as caption_exc:+ ImageConverter().convert(+ stream,+ StreamInfo(extension=".jpg", mimetype="image/jpeg"),+ llm_client=client,+ llm_model="test-model",+ )++ assert caption_exc.value is error+ assert stream.tell() == len(prefix)+ client.chat.completions.create.assert_called_once()+++def test_image_caption_error_allows_fallback() -> None:+ client = MagicMock()+ client.chat.completions.create.side_effect = RuntimeError("Caption request failed")+ image = IMAGE_FILE.read_bytes()++ class FallbackImageConverter(ImageConverter):+ def convert(+ self,+ file_stream: BinaryIO,+ stream_info: StreamInfo,+ **kwargs: Any,+ ) -> DocumentConverterResult:+ assert client.chat.completions.create.called+ assert file_stream.read() == image+ return DocumentConverterResult(markdown="Fallback image description")++ md = MarkItDown(llm_client=client, llm_model="test-model", exiftool_path="")+ md.register_converter(FallbackImageConverter(), priority=10)+ prefix = b"ignored prefix"+ stream = io.BytesIO(prefix + image)+ stream.seek(len(prefix))++ result = md.convert_stream(+ stream, stream_info=StreamInfo(extension=".jpg", mimetype="image/jpeg")+ )++ assert result.markdown == "Fallback image description"+ assert stream.tell() == len(prefix)++[email protected]("metadata", [{}, {"Title": "Image title"}])[email protected]("caption", [None, " Image caption. "])+def test_image_successful_caption_response_is_unchanged(+ metadata: dict[str, str],+ caption: Optional[str],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(+ image_module, "exiftool_metadata", MagicMock(return_value=metadata)+ )+ client = MagicMock()+ client.chat.completions.create.return_value.choices[0].message.content = caption++ result = ImageConverter().convert(+ io.BytesIO(IMAGE_FILE.read_bytes()),+ StreamInfo(extension=".jpg", mimetype="image/jpeg"),+ llm_client=client,+ llm_model="test-model",+ )++ expected = "Title: Image title\n" if metadata else ""+ if caption is not None:+ expected += "\n# Description:\nImage caption.\n"+ assert result.markdown == expected+ client.chat.completions.create.assert_called_once()++[email protected]("metadata", [{}, {"Title": "Image title"}])+def test_image_without_llm_is_unchanged(+ metadata: dict[str, str], monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setattr(+ image_module, "exiftool_metadata", MagicMock(return_value=metadata)+ )+ description = MagicMock()+ monkeypatch.setattr(ImageConverter, "_get_llm_description", description)++ result = ImageConverter().convert(+ io.BytesIO(IMAGE_FILE.read_bytes()),+ StreamInfo(extension=".jpg", mimetype="image/jpeg"),+ )++ assert result.markdown == ("Title: Image title\n" if metadata else "")+ description.assert_not_called()