🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Add new Mistral Document AI for advanced OCR processing (#2479)
JINO ROHITb834d4016401
diff --git a/camel/loaders/__init__.py b/camel/loaders/__init__.pyindex fce954487..0b47a9cfb 100644--- a/camel/loaders/__init__.py+++ b/camel/loaders/__init__.py@@ -20,6 +20,7 @@ from .firecrawl_reader import Firecrawlfrom .jina_url_reader import JinaURLReaderfrom .markitdown import MarkItDownLoaderfrom .mineru_extractor import MinerU+from .mistral_reader import MistralReaderfrom .pandas_reader import PandasReaderfrom .scrapegraph_reader import ScrapeGraphAIfrom .unstructured_io import UnstructuredIO@@ -39,4 +40,5 @@ __all__ = ['Crawl4AI','MarkItDownLoader','ScrapeGraphAI',+ 'MistralReader',]diff --git a/camel/loaders/mistral_reader.py b/camel/loaders/mistral_reader.pynew file mode 100644index 000000000..ec3264c96--- /dev/null+++ b/camel/loaders/mistral_reader.py@@ -0,0 +1,148 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import os+from typing import TYPE_CHECKING, List, Optional++if TYPE_CHECKING:+ from mistralai.models import OCRResponse++from camel.logger import get_logger+from camel.utils import api_keys_required++logger = get_logger(__name__)+++class MistralReader:+ r"""Mistral Document Loader."""++ @api_keys_required(+ [+ ("api_key", "MISTRAL_API_KEY"),+ ]+ )+ def __init__(+ self,+ api_key: Optional[str] = None,+ model: Optional[str] = "mistral-ocr-latest",+ ) -> None:+ r"""Initialize the MistralReader.++ Args:+ api_key (Optional[str]): The API key for the Mistral API.+ (default: :obj:`None`)+ model (Optional[str]): The model to use for OCR.+ (default: :obj:`"mistral-ocr-latest"`)+ """+ from mistralai import Mistral++ self._api_key = api_key or os.environ.get("MISTRAL_API_KEY")+ self.client = Mistral(api_key=self._api_key)+ self.model = model++ def _encode_file(self, file_path: str) -> str:+ r"""Encode the pdf to base64.++ Args:+ file_path (str): Path to the input file.++ Returns:+ str: base64 version of the file.+ """++ import base64++ try:+ with open(file_path, "rb") as pdf_file:+ return base64.b64encode(pdf_file.read()).decode('utf-8')+ except FileNotFoundError:+ logger.error(f"Error: The file {file_path} was not found.")+ return ""+ except Exception as e:+ logger.error(f"Error: {e}")+ return ""++ def extract_text(+ self,+ file_path: str,+ is_image: bool = False,+ pages: Optional[List[int]] = None,+ include_image_base64: Optional[bool] = None,+ ) -> "OCRResponse":+ r"""Converts the given file to Markdown format.++ Args:+ file_path (str): Path to the input file or a remote URL.+ is_image (bool): Whether the file or URL is an image. If True,+ uses image_url type instead of document_url.+ (default: :obj:`False`)+ pages (Optional[List[int]]): Specific pages user wants to process+ in various formats: single number, range, or list of both.+ Starts from 0. (default: :obj:`None`)+ include_image_base64 (Optional[bool]): Whether to include image+ URLs in response. (default: :obj:`None`)++ Returns:+ OCRResponse: page wise extractions.++ Raises:+ FileNotFoundError: If the specified local file does not exist.+ ValueError: If the file format is not supported.+ Exception: For other errors during conversion.+ """+ # Check if the input is a URL (starts with http:// or https://)+ is_url = file_path.startswith(('http://', 'https://'))++ if not is_url and not os.path.isfile(file_path):+ logger.error(f"File not found: {file_path}")+ raise FileNotFoundError(f"File not found: {file_path}")+ try:+ if is_url:+ logger.info(f"Processing URL: {file_path}")+ if is_image:+ document_config = {+ "type": "image_url",+ "image_url": file_path,+ }+ else:+ document_config = {+ "type": "document_url",+ "document_url": file_path,+ }+ else:+ logger.info(f"Converting local file: {file_path}")+ base64_file = self._encode_file(file_path)+ if is_image:+ document_config = {+ "type": "image_url",+ "image_url": f"data:image/jpeg;base64,{base64_file}",+ }+ else:+ document_config = {+ "type": "document_url",+ "document_url": f"data:application/"+ f"pdf;base64,{base64_file}",+ }++ ocr_response = self.client.ocr.process(+ model=self.model,+ document=document_config, # type: ignore[arg-type]+ pages=None if is_image else pages,+ include_image_base64=include_image_base64,+ )++ logger.info(f"Processing completed successfully for: {file_path}")+ return ocr_response+ except Exception as e:+ logger.error(f"Error processing '{file_path}': {e}")+ raise ValueError(f"Error processing '{file_path}': {e}")diff --git a/docs/key_modules/loaders.md b/docs/key_modules/loaders.mdindex eb7630cf0..e42234753 100644--- a/docs/key_modules/loaders.md+++ b/docs/key_modules/loaders.md@@ -1,6 +1,6 @@## 1. ConceptCAMEL introduced two IO modules, `Base IO` and `Unstructured IO` which are designed for handling various file types and unstructured data processing.-Additionally, four new data readers were added, `Apify Reader`,`Chunkr Reader`, `Firecrawl Reader`, and `Jina_url Reader`, which enable retrieval of external data for improved data integration and analysis.+Additionally, several data readers were added, including `Apify Reader`, `Chunkr Reader`, `Firecrawl Reader`, `Jina_url Reader`, and `Mistral Reader`, which enable retrieval of external data for improved data integration and analysis.## 2. Types@@ -26,6 +26,9 @@ JinaURL Reader is a Python client for Jina AI's URL reading service, optimized t### 2.7 MarkitDown ReaderMarkItDown is a lightweight Python utility for converting various files to Markdown for use with LLMs and related text analysis pipelines.+### 2.8 Mistral Reader+Mistral Reader is a Python client for interacting with Mistral AI's OCR service, which can extract text from PDFs and images. It provides capabilities to process both local files and remote URLs, supporting various file formats.+## 3. Get Started### 3.1. Using `Base IO`@@ -319,7 +322,6 @@ print(response)```markdown>>>Welcome to CAMEL’s documentation! — CAMEL 0.2.61 documentation-===============[Skip to main content](https://docs.camel-ai.org/#main-content)@@ -345,3 +347,46 @@ Key Modules* [Models](https://docs.camel-ai...)```++### 3.7. Using `Mistral Reader`++Initialize the Mistral Reader client with your API key (or it will use the MISTRAL_API_KEY environment variable) and extract text from a PDF URL:++```python+from camel.loaders import MistralReader++mistral_reader = MistralReader()++# Extract text from a PDF URL+url_ocr_response = mistral_reader.extract_text(+ file_path="https://arxiv.org/pdf/2201.04234", pages=[5]+)+print(url_ocr_response)+```++Extract text from an image URL by setting `is_image=True`:++```python+# Extract text from an image URL+image_ocr_response = mistral_reader.extract_text(+ file_path="https://raw.githubusercontent.com/mistralai/cookbook/refs/heads/main/mistral/ocr/receipt.png",+ is_image=True,+)+print(image_ocr_response)+```++Process a local PDF file:++```python+# Extract text from a local PDF file+local_ocr_response = mistral_reader.extract_text("path/to/your/document.pdf")+print(local_ocr_response)+```++The response contains structured information about the extracted text, including the markdown-formatted content, page dimensions, and usage information:++```markdown+>>>pages=[OCRPageObject(index=5, markdown='\n\nFigure 2: Scatter plot of predicted accuracy versus (true) OOD accuracy. Each point denotes a dif...',++images=[OCRImageObject(id='img-0.jpeg', top_left_x=294, top_left_y=180, bottom_right_x=1387, bottom_right_y=558, image_base64=None, image_annotation=None)], dimensions=OCRPageDimensions(dpi=200, height=2200, width=1700))] model='mistral-ocr-2505-completion' usage_info=OCRUsageInfo(pages_processed=1, doc_size_bytes=3002783) document_annotation=None+```diff --git a/examples/loaders/mistral_example.py b/examples/loaders/mistral_example.pynew file mode 100644index 000000000..25ec022c7--- /dev/null+++ b/examples/loaders/mistral_example.py@@ -0,0 +1,154 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import os+import tempfile++from camel.loaders import MistralReader++mistral_reader = MistralReader()+url_ocr_response = mistral_reader.extract_text(+ file_path="https://arxiv.org/pdf/2201.04234", pages=[5]+)+print(url_ocr_response)+"""+============================================================================+pages=[OCRPageObject(index=5, markdown='\n\nFigure 2:+Scatter plot of predicted accuracy versus (true) OOD accuracy. Each point+denotes a different OOD dataset, all evaluated with the same DenseNet121+model. We only plot the best three methods. With ATC (ours), we refer to+ATC-NE. We observe that ATC significantly outperforms other methods and with+ATC, we recover the desired line $y=x$ with a robust linear fit. Aggregated+estimation error in Table 1 and plots for other datasets and architectures in+App. H.\nof the target accuracy with various methods given access to only+unlabeled data from the target. Unless noted otherwise, all models are trained+only on samples from the source distribution with the main exception of+pre-training on a different distribution. We use labeled examples from the+target distribution to only obtain true error estimates.\n\nDatasets. First,+we consider synthetic shifts induced due to different visual corruptions (e.+g., shot noise, motion blur etc.) under ImageNet-C (Hendrycks \\& Dietterich,+2019). Next, we consider natural shifts due to differences in the data+collection process of ImageNet (Russakovsky et al., 2015), e.g, ImageNetv2+(Recht et al., 2019). We also consider images with artistic renditions of+object classes, i.e., ImageNet-R (Hendrycks et al., 2021) and ImageNet-Sketch+(Wang et al., 2019). Note that renditions dataset only contains a subset 200+classes from ImageNet. To include renditions dataset in our testbed, we+include results on ImageNet restricted to these 200 classes (which we call+ImageNet-200) along with full ImageNet.\n\nSecond, we consider BREEDS+(Santurkar et al., 2020) to assess robustness to subpopulation shifts, in+particular, to understand how accuracy estimation methods behave when novel+subpopulations not observed during training are introduced. BREEDS leverages+class hierarchy in ImageNet to create 4 datasets ENTITY-13, ENTITY-30,+LIVING-17, NON-LIVING-26. We focus on natural and synthetic shifts as in+ImageNet on same and different subpopulations in BREEDs. Third, from WILDS+(Koh et al., 2021) benchmark, we consider FMoW-WILDS (Christie et al., 2018),+RxRx1-WILDS (Taylor et al., 2019), Amazon-WILDS (Ni et al., 2019),+CivilComments-WILDS (Borkan et al., 2019) to consider distribution shifts+faced in the wild.\n\nFinally, similar to ImageNet, we consider (i) synthetic+shifts (CIFAR-10-C) due to common corruptions; and (ii) natural shift (i.e.,+CIFARv2 (Recht et al., 2018)) on CIFAR-10 (Krizhevsky \\& Hinton, 2009). On+CIFAR-100, we just have synthetic shifts due to common corruptions. For+completeness, we also consider natural shifts on MNIST (LeCun et al., 1998) as+in the prior work (Deng \\& Zheng, 2021). We use three real shifted datasets,+i.e., USPS (Hull, 1994), SVHN (Netzer et al., 2011) and QMNIST (Yadav \\&+Bottou, 2019). We give a detailed overview of our setup in App. F.+\n\nArchitectures and Evaluation. For ImageNet, BREEDS, CIFAR, FMoW-WILDS,+RxRx1-WILDS datasets, we use DenseNet121 (Huang et al., 2017) and ResNet50 (He+et al., 2016) architectures. For Amazon-WILDS and CivilComments-WILDS, we+fine-tune a DistilBERT-base-uncased (Sanh et al., 2019) model. For MNIST, we+train a fully connected multilayer perceptron. We use standard training with+benchmarked hyperparameters. To compare methods, we report average absolute+difference between the true accuracy on the target data and the estimated+accuracy on the same unlabeled examples. We refer to this metric as Mean+Absolute estimation Error (MAE). Along with MAE, we also show scatter plots to+visualize performance at individual target sets. Refer to App. G for+additional details on the setup.\n\nMethods With ATC-NE, we denote ATC with+negative entropy score function and with ATC-MC, we denote ATC with maximum+confidence score function. For all methods, we implement post-hoc calibration+on validation source data with Temperature Scaling (TS; Guo et al. (2017)).+Below we briefly discuss baselines methods compared in our work and relegate+details to App. E.', images=[OCRImageObject(id='img-0.jpeg', top_left_x=294,+top_left_y=180, bottom_right_x=1387, bottom_right_y=558, image_base64=None,+image_annotation=None)], dimensions=OCRPageDimensions(dpi=200, height=2200,+width=1700))] model='mistral-ocr-2505-completion' usage_info=OCRUsageInfo+(pages_processed=1, doc_size_bytes=3002783) document_annotation=None+============================================================================+"""+++image_ocr_response = mistral_reader.extract_text(+ file_path="https://raw.githubusercontent.com/mistralai/cookbook/refs/heads/main/mistral/ocr/receipt.png",+ is_image=True,+)+print(image_ocr_response)+"""+============================================================================+pages=[OCRPageObject(index=0, markdown='PLACE FACE UP ON DASH\nCITY OF PALO+ALTO\nNOT VALID FOR\nONSTREET PARKING\n\nExpiration Date/Time\n11:59 PM\nAUG+19, 2024\n\nPurchase Date/Time: 01:34pm Aug 19, 2024\nTotal Due: $15.00\nTotal+Paid: $15.00\nTicket #: 00005883\nS/N #: 520117260957\nSetting: Permit+Machines\nMach Name: Civic Center\n\n#****-1224, Visa\nDISPLAY FACE UP ON+DASH\n\nPERMIT EXPIRES\nAT MIDNIGHT', images=[], dimensions=OCRPageDimensions+(dpi=200, height=3210, width=1806))] model='mistral-ocr-2505-completion'+usage_info=OCRUsageInfo(pages_processed=1, doc_size_bytes=3110191)+document_annotation=None+============================================================================+"""+++mock_files = {+ "report.pdf": "%PDF-1.4\n%âãÏÓ\n1 0 obj\n<</Type/Catalog/Pages 2 0"+ " R>>\nendobj\n2 0 obj\n<</Type/Pages/Kids[3 0 R]/Count 1>>\nendobj\n3 "+ "0 obj\n<</Type/Page/MediaBox[0 0 595 842]/Resources<<>>/Contents 4 0 "+ "R/Parent 2 0 R>>\nendobj\n4 0 obj\n<</Length 150>>\nstream\nBT\n/F1 "+ "12 Tf\n50 750 Td\n(CAMEL AI Research Report) Tj\n0 -20 Td\n(This is "+ "a complex mock PDF document for testing OCR capabilities.) Tj\n0 -20 "+ "Td\n(It contains multiple lines of text and "+ "formatting.) Tj\nET\nendstream\nendobj\nxref\n0 5\n0000000000 "+ "65535 f\n0000000010 00000 n\n0000000056 00000 n\n0000000111 "+ "00000 n\n0000000212 00000 n\ntrailer\n<</Size 5/Root 1 "+ "0 R>>\nstartxref\n362\n%%EOF",+}++with tempfile.TemporaryDirectory() as temp_dir:+ file_paths = {}++ for filename, content in mock_files.items():+ file_path = os.path.join(temp_dir, filename)+ with open(file_path, "w") as f:+ f.write(content)+ file_paths[filename] = file_path++ local_ocr_response = mistral_reader.extract_text(file_paths["report.pdf"])+ print(local_ocr_response)++"""+============================================================================+pages=[OCRPageObject(index=0, markdown='PLACE FACE UP ON DASH\nCITY OF PALO+ALTO\nNOT VALID FOR\nONSTREET PARKING\n\nExpiration Date/Time\n11:59 PM\nAUG+19, 2024\n\nPurchase Date/Time: 01:34pm Aug 19, 2024\nTotal Due: $15.00\nTotal+Paid: $15.00\nTicket #: 00005883\nS/N #: 520117260957\nSetting: Permit+Machines\nMach Name: Civic Center\n\n#****-1224, Visa\nDISPLAY FACE UP ON+DASH\n\nPERMIT EXPIRES\nAT MIDNIGHT', images=[], dimensions=OCRPageDimensions+(dpi=200, height=3210, width=1806))] model='mistral-ocr-2505-completion'+usage_info=OCRUsageInfo(pages_processed=1, doc_size_bytes=3110191)+document_annotation=None+pages=[OCRPageObject(index=0, markdown='CAMEL AI Research Report\nThis is a+complex mock PDF document for testing OCR capabilities.\nIt contains multiple+lines of text and formatting.', images=[], dimensions=OCRPageDimensions+(dpi=200, height=2339, width=1653))] model='mistral-ocr-2505-completion'+usage_info=OCRUsageInfo(pages_processed=1, doc_size_bytes=612)+document_annotation=None+============================================================================+"""diff --git a/test/loaders/test_mistral_reader.py b/test/loaders/test_mistral_reader.pynew file mode 100644index 000000000..158176b2b--- /dev/null+++ b/test/loaders/test_mistral_reader.py@@ -0,0 +1,173 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import base64+import os+import tempfile+from unittest.mock import MagicMock, patch++import pytest++from camel.loaders.mistral_reader import MistralReader+++ os.environ.get("MISTRAL_API_KEY") is None,+ reason="MISTRAL_API_KEY not available",+)+def test_init_with_env_variable():+ r"""Test MistralReader initialization with environment variable."""+ with patch("mistralai.Mistral") as mock_mistral:+ reader = MistralReader()+ mock_mistral.assert_called_once_with(+ api_key=os.environ.get("MISTRAL_API_KEY")+ )+ assert reader.model == "mistral-ocr-latest"+++def test_init_with_api_key():+ r"""Test MistralReader initialization with provided API key."""+ test_api_key = "test_api_key"+ test_model = "test-model"++ with patch("mistralai.Mistral") as mock_mistral:+ reader = MistralReader(api_key=test_api_key, model=test_model)+ mock_mistral.assert_called_once_with(api_key=test_api_key)+ assert reader.model == test_model+++def test_encode_file_success():+ r"""Test file encoding with a valid file."""+ reader = MistralReader(api_key="dummy_key")++ # Create a temporary file for testing+ with tempfile.NamedTemporaryFile(delete=False) as temp_file:+ temp_file.write(b"test content")+ temp_file_path = temp_file.name++ try:+ # Test encoding+ encoded = reader._encode_file(temp_file_path)+ assert encoded == base64.b64encode(b"test content").decode('utf-8')+ finally:+ # Clean up+ if os.path.exists(temp_file_path):+ os.remove(temp_file_path)+++def test_encode_file_not_found():+ r"""Test file encoding with a non-existent file."""+ reader = MistralReader(api_key="dummy_key")++ # Test with a non-existent file+ with patch("camel.loaders.mistral_reader.logger") as mock_logger:+ encoded = reader._encode_file("/non/existent/file.pdf")+ assert encoded == ""+ mock_logger.error.assert_called_once()+++def test_extract_text_local_file():+ r"""Test extract_text with a local file."""+ # Mock OCR response+ mock_ocr_response = MagicMock()++ # Create a temporary file+ with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as temp_file:+ temp_file.write(b"test pdf content")+ temp_file_path = temp_file.name++ try:+ with patch("mistralai.Mistral") as mock_mistral_class:+ # Set up the mock chain+ mock_mistral = MagicMock()+ mock_mistral_class.return_value = mock_mistral+ mock_mistral.ocr = MagicMock()+ mock_mistral.ocr.process.return_value = mock_ocr_response++ # Create reader and test extract_text+ reader = MistralReader(api_key="dummy_key")+ result = reader.extract_text(temp_file_path)++ # Verify the result and that the correct methods were called+ assert result == mock_ocr_response+ mock_mistral.ocr.process.assert_called_once()++ # Verify document_config was correctly constructed+ call_args = mock_mistral.ocr.process.call_args[1]+ assert call_args["model"] == "mistral-ocr-latest"+ assert call_args["document"]["type"] == "document_url"+ assert (+ "data:application/pdf;base64,"+ in call_args["document"]["document_url"]+ )+ finally:+ # Clean up+ if os.path.exists(temp_file_path):+ os.remove(temp_file_path)+++def test_extract_text_url():+ r"""Test extract_text with a URL."""+ # Mock OCR response+ mock_ocr_response = MagicMock()+ test_url = "https://example.com/document.pdf"++ with patch("mistralai.Mistral") as mock_mistral_class:+ # Set up the mock chain+ mock_mistral = MagicMock()+ mock_mistral_class.return_value = mock_mistral+ mock_mistral.ocr = MagicMock()+ mock_mistral.ocr.process.return_value = mock_ocr_response++ # Create reader and test extract_text with URL+ reader = MistralReader(api_key="dummy_key")+ result = reader.extract_text(test_url)++ # Verify the result and that the correct methods were called+ assert result == mock_ocr_response+ mock_mistral.ocr.process.assert_called_once()++ # Verify document_config was correctly constructed+ call_args = mock_mistral.ocr.process.call_args[1]+ assert call_args["model"] == "mistral-ocr-latest"+ assert call_args["document"]["type"] == "document_url"+ assert call_args["document"]["document_url"] == test_url+++def test_extract_text_image():+ r"""Test extract_text with an image file."""+ # Mock OCR response+ mock_ocr_response = MagicMock()+ test_url = "https://example.com/image.jpg"++ with patch("mistralai.Mistral") as mock_mistral_class:+ # Set up the mock chain+ mock_mistral = MagicMock()+ mock_mistral_class.return_value = mock_mistral+ mock_mistral.ocr = MagicMock()+ mock_mistral.ocr.process.return_value = mock_ocr_response++ # Create reader and test extract_text with image URL+ reader = MistralReader(api_key="dummy_key")+ result = reader.extract_text(test_url, is_image=True)++ # Verify the result and that the correct methods were called+ assert result == mock_ocr_response+ mock_mistral.ocr.process.assert_called_once()++ # Verify document_config was correctly constructed+ call_args = mock_mistral.ocr.process.call_args[1]+ assert call_args["model"] == "mistral-ocr-latest"+ assert call_args["document"]["type"] == "image_url"+ assert call_args["document"]["image_url"] == test_url