Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Fix/mistral ocr image file type handling (#84)

Louis Choquelb9254c591cde
diff --git a/pipelex/plugins/mistral/mistral_ocr_worker.py b/pipelex/plugins/mistral/mistral_ocr_worker.py
index 9c0b74192..59611dcca 100644
--- a/pipelex/plugins/mistral/mistral_ocr_worker.py
+++ b/pipelex/plugins/mistral/mistral_ocr_worker.py
@@ -14,6 +14,7 @@ from pipelex.plugins.mistral.mistral_factory import MistralFactory
from pipelex.plugins.mistral.mistral_utils import upload_file_for_ocr
from pipelex.reporting.reporting_protocol import ReportingProtocol
from pipelex.tools.misc.base_64_utils import load_binary_as_base64_async
+from pipelex.tools.misc.filetype_utils import detect_file_type_from_base64
from pipelex.tools.misc.path_utils import clarify_path_or_url
@@ -124,10 +125,12 @@ class MistralOcrWorker(OcrWorkerAbstract):
) -> OcrOutput:
b64 = await load_binary_as_base64_async(path=image_path)
- # TODO: set proper image format
+ file_type = detect_file_type_from_base64(b64=b64)
+ mime_type = file_type.mime
+
ocr_response = await self.mistral_client.ocr.process_async(
model=self.ocr_engine.ocr_model_name,
- document={"type": "image_url", "image_url": f"data:image/jpeg;base64,{b64.decode('utf-8')}"},
+ document={"type": "image_url", "image_url": f"data:{mime_type};base64,{b64.decode('utf-8')}"},
)
ocr_output = await MistralFactory.make_ocr_output_from_mistral_response(
mistral_ocr_response=ocr_response,
diff --git a/tests/data/documents/solar_system.jpg b/tests/data/documents/solar_system.jpg
new file mode 100644
index 000000000..950d73e65
Binary files /dev/null and b/tests/data/documents/solar_system.jpg differ
diff --git a/tests/pipelex/cogt_asynch/test_content_generator.py b/tests/pipelex/cogt_asynch/test_content_generator.py
index 37503d0a5..735895f4f 100644
--- a/tests/pipelex/cogt_asynch/test_content_generator.py
+++ b/tests/pipelex/cogt_asynch/test_content_generator.py
@@ -145,7 +145,7 @@ class TestContentGenerator:
top_job_id=cast(str, request.node.originalname), # pyright: ignore[reportUnknownMemberType]
),
ocr_handle=OcrHandle.MISTRAL_OCR,
- ocr_input=OcrInput(image_uri=ImageTestCases.IMAGE_FILE_PATH),
+ ocr_input=OcrInput(image_uri=ImageTestCases.IMAGE_FILE_PATH_PNG),
ocr_job_params=OcrJobParams.make_default_ocr_job_params(),
ocr_job_config=OcrJobConfig(),
)
diff --git a/tests/pipelex/cogt_asynch/test_ocr.py b/tests/pipelex/cogt_asynch/test_ocr.py
index ca3c56dbc..4e42f0f7b 100644
--- a/tests/pipelex/cogt_asynch/test_ocr.py
+++ b/tests/pipelex/cogt_asynch/test_ocr.py
@@ -36,7 +36,7 @@ class TestOcr:
pretty_print(ocr_output, title="OCR Output")
assert ocr_output.pages
- @pytest.mark.parametrize("file_path", [ImageTestCases.IMAGE_FILE_PATH])
+ @pytest.mark.parametrize("file_path", ImageTestCases.IMAGE_FILE_PATHS)
async def test_ocr_image_file(self, file_path: str):
ocr_worker = get_ocr_worker(ocr_handle=OcrHandle.MISTRAL_OCR)
ocr_job = OcrJobFactory.make_ocr_job(
@@ -46,7 +46,7 @@ class TestOcr:
pretty_print(ocr_output, title="OCR Output")
assert ocr_output.pages
- @pytest.mark.parametrize("url", [ImageTestCases.IMAGE_URL])
+ @pytest.mark.parametrize("url", ImageTestCases.IMAGE_URLS)
async def test_ocr_image_url(self, url: str):
ocr_worker = get_ocr_worker(ocr_handle=OcrHandle.MISTRAL_OCR)
ocr_job = OcrJobFactory.make_ocr_job(
diff --git a/tests/pipelex/test_data.py b/tests/pipelex/test_data.py
index 0aca7eff2..32f04e5ba 100644
--- a/tests/pipelex/test_data.py
+++ b/tests/pipelex/test_data.py
@@ -304,8 +304,8 @@ class LibraryTestCases:
class PipeOcrTestCases:
PIPE_OCR_IMAGE_TEST_CASES: ClassVar[List[str]] = [
- ImageTestCases.IMAGE_FILE_PATH,
- ImageTestCases.IMAGE_URL,
+ ImageTestCases.IMAGE_FILE_PATH_PNG,
+ ImageTestCases.IMAGE_URL_PNG,
]
PIPE_OCR_PDF_TEST_CASES: ClassVar[List[str]] = PDFTestCases.DOCUMENT_FILE_PATHS + PDFTestCases.DOCUMENT_URLS
diff --git a/tests/test_data.py b/tests/test_data.py
index 28ea3cbe6..babdbbe0c 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -13,5 +13,15 @@ class PDFTestCases:
# TODO: move the other shared test images here from other test_data.py files
class ImageTestCases:
- IMAGE_FILE_PATH = "tests/data/documents/solar_system.png"
- IMAGE_URL = "https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/documents/solar_system.png"
+ IMAGE_FILE_PATH_PNG = "tests/data/documents/solar_system.png"
+ IMAGE_FILE_PATH_JPG = "tests/data/documents/solar_system.jpg"
+ IMAGE_FILE_PATHS: ClassVar[List[str]] = [
+ IMAGE_FILE_PATH_PNG,
+ IMAGE_FILE_PATH_JPG,
+ ]
+ IMAGE_URL_PNG = "https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/documents/solar_system.png"
+ IMAGE_URL_JPG = "https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/documents/solar_system.jpg"
+ IMAGE_URLS: ClassVar[List[str]] = [
+ IMAGE_URL_PNG,
+ IMAGE_URL_JPG,
+ ]