š« CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
chunkr integrated (#1103)
yaoxie22057daccf4deb1
diff --git a/camel/loaders/__init__.py b/camel/loaders/__init__.pyindex 57b432ea1..6e66dff25 100644--- a/camel/loaders/__init__.py+++ b/camel/loaders/__init__.py@@ -13,6 +13,7 @@# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========from .base_io import File+from .chunkr_reader import ChunkrReaderfrom .firecrawl_reader import Firecrawlfrom .jina_url_reader import JinaURLReaderfrom .unstructured_io import UnstructuredIO@@ -22,4 +23,5 @@ __all__ = ['UnstructuredIO','JinaURLReader','Firecrawl',+ 'ChunkrReader',]diff --git a/camel/loaders/chunkr_reader.py b/camel/loaders/chunkr_reader.pynew file mode 100644index 000000000..4ceec4b24--- /dev/null+++ b/camel/loaders/chunkr_reader.py@@ -0,0 +1,165 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========++import json+import logging+import os+import time+from typing import IO, Any, Optional, Union++import requests++from camel.utils import api_keys_required++logging.basicConfig(level=logging.INFO)+++class ChunkrReader:+ r"""Chunkr Reader for processing documents and returning content+ in various formats.++ Args:+ api_key (Optional[str], optional): The API key for Chunkr API. If not+ provided, it will be retrieved from the environment variable+ `CHUNKR_API_KEY`. (default: :obj:`None`)+ url (Optional[str], optional): The url to the Chunkr service.+ (default: :obj:`https://api.chunkr.ai/api/v1/task`)+ timeout (int, optional): The maximum time in seconds to wait for the+ API responses. (default: :obj:`30`)+ **kwargs (Any): Additional keyword arguments for request headers.+ """++ @api_keys_required("CHUNKR_API_KEY")+ def __init__(+ self,+ api_key: Optional[str] = None,+ url: Optional[str] = "https://api.chunkr.ai/api/v1/task",+ timeout: int = 30,+ **kwargs: Any,+ ) -> None:+ self._api_key = api_key or os.getenv('CHUNKR_API_KEY')+ self._url = os.getenv('CHUNKR_API_URL') or url+ self._headers = {+ "Authorization": f"{self._api_key}",+ **kwargs,+ }+ self.timeout = timeout++ def submit_task(+ self,+ file_path: str,+ model: str = "Fast",+ ocr_strategy: str = "Auto",+ target_chunk_length: str = "512",+ ) -> str:+ r"""Submits a file to the Chunkr API and returns the task ID.++ Args:+ file_path (str): The path to the file to be uploaded.+ model (str, optional): The model to be used for the task.+ (default: :obj:`Fast`)+ ocr_strategy (str, optional): The OCR strategy. Defaults to 'Auto'.+ target_chunk_length (str, optional): The target chunk length.+ (default: :obj:`512`)++ Returns:+ str: The task ID.+ """+ with open(file_path, 'rb') as file:+ files: dict[+ str, Union[tuple[None, IO[bytes]], tuple[None, str]]+ ] = {+ 'file': (+ None,+ file,+ ), # Properly pass the file as a binary stream+ 'model': (None, model),+ 'ocr_strategy': (None, ocr_strategy),+ 'target_chunk_length': (None, target_chunk_length),+ }+ try:+ response = requests.post(+ self._url, # type: ignore[arg-type]+ headers=self._headers,+ files=files,+ timeout=self.timeout,+ )+ response.raise_for_status()+ task_id = response.json().get('task_id')+ if not task_id:+ raise ValueError("Task ID not returned in the response.")+ logging.info(+ f"Task submitted successfully. Task ID: {task_id}"+ )+ return task_id+ except Exception as e:+ logging.error(f"Failed to submit task: {e}")+ raise ValueError(f"Failed to submit task: {e}") from e++ def get_task_output(self, task_id: str, max_retries: int = 5) -> str:+ r"""Polls the Chunkr API to check the task status and returns the task+ result.++ Args:+ task_id (str): The task ID to check the status for.+ max_retries (int, optional): Maximum number of retry attempts.+ (default: :obj:`5`)++ Returns:+ str: The formatted task result in JSON format.++ Raises:+ ValueError: If the task status cannot be retrieved.+ RuntimeError: If the maximum number of retries is reached without+ a successful task completion.+ """+ url_get = f"{self._url}/{task_id}"+ attempts = 0++ while attempts < max_retries:+ try:+ response = requests.get(+ url_get, headers=self._headers, timeout=self.timeout+ )+ response.raise_for_status()+ task_status = response.json().get('status')++ if task_status == "Succeeded":+ logging.info(f"Task {task_id} completed successfully.")+ return self._pretty_print_response(response.json())+ else:+ logging.info(+ f"Task {task_id} is still {task_status}. Retrying "+ "in 5 seconds..."+ )+ except Exception as e:+ logging.error(f"Failed to retrieve task status: {e}")+ raise ValueError(f"Failed to retrieve task status: {e}") from e++ attempts += 1+ time.sleep(5)++ logging.error(f"Max retries reached for task {task_id}.")+ raise RuntimeError(f"Max retries reached for task {task_id}.")++ def _pretty_print_response(self, response_json: dict) -> str:+ r"""Pretty prints the JSON response.++ Args:+ response_json (dict): The response JSON to pretty print.++ Returns:+ str: Formatted JSON as a string.+ """+ return json.dumps(response_json, indent=4)diff --git a/examples/loaders/chunkr_example.py b/examples/loaders/chunkr_example.pynew file mode 100644index 000000000..e28a9e2d4--- /dev/null+++ b/examples/loaders/chunkr_example.py@@ -0,0 +1,73 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========++from camel.loaders import ChunkrReader+++def read_with_different_model():+ r"""Reads a document using the Chunkr API."""+ print("Choose a model for processing the file:")+ print("1. Fast")+ print("2. HighQuality")+ choice = input("Enter your choice (1-2): ")+ models = {"1": "Fast", "2": "HighQuality"}++ if choice not in models:+ print("Invalid choice. Exiting.")+ return++ model = models[choice]++ print("Choose an OCR strategy:")+ print("1. Auto")+ print("2. All")+ print("3. Off")+ ocr_choice = input("Enter your choice (1-3) [Default: Auto]: ")+ ocr_strategies = {"1": "Auto", "2": "All", "3": "Off"}++ if ocr_choice not in ocr_strategies:+ ocr_strategy = "Auto"+ else:+ ocr_strategy = ocr_strategies[ocr_choice]++ while True:+ target_chunk_length = (+ input("Enter the target chunk length [Default: 512]: ") or "512"+ )+ if target_chunk_length.isdigit() and int(target_chunk_length) > 0:+ break+ else:+ print("Invalid input. Please enter a valid positive integer.")++ file_path = input("Please provide the file path: ")++ chunkr_reader = ChunkrReader()++ try:+ task_id = chunkr_reader.submit_task(+ file_path,+ model=model,+ ocr_strategy=ocr_strategy,+ target_chunk_length=target_chunk_length,+ )++ result = chunkr_reader.get_task_output(task_id)+ print(result)++ except Exception as e:+ print(f"An error occurred: {e}")+++if __name__ == "__main__":+ read_with_different_model()diff --git a/test/loaders/test_chunkr_reader.py b/test/loaders/test_chunkr_reader.pynew file mode 100644index 000000000..28ff767e9--- /dev/null+++ b/test/loaders/test_chunkr_reader.py@@ -0,0 +1,86 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========++import unittest+from unittest.mock import mock_open, patch++from camel.loaders import ChunkrReader+++class TestChunkrReader(unittest.TestCase):+ @patch('requests.post')+ @patch(+ 'builtins.open', new_callable=mock_open, read_data='fake file content'+ )+ def test_submit_task_success(self, mock_file, mock_post):+ mock_post.return_value.ok = True+ mock_post.return_value.json.return_value = {'task_id': '12345'}++ reader = ChunkrReader(api_key='test_api_key')+ task_id = reader.submit_task('fake_path.txt')++ self.assertEqual(task_id, '12345')+ mock_post.assert_called_once()+ mock_file.assert_called_once_with('fake_path.txt', 'rb')++ @patch('requests.post')+ @patch(+ 'builtins.open', new_callable=mock_open, read_data='fake file content'+ )+ def test_submit_task_no_task_id(self, mock_file, mock_post):+ mock_post.return_value.ok = True+ mock_post.return_value.json.return_value = {}++ reader = ChunkrReader(api_key='test_api_key')++ with self.assertRaises(ValueError) as context:+ reader.submit_task('fake_path.txt')++ self.assertEqual(+ str(context.exception),+ "Failed to submit task: Task ID not returned in the response.",+ )++ @patch('requests.get')+ def test_get_task_output_success(self, mock_get):+ mock_get.return_value.ok = True+ mock_get.return_value.json.return_value = {+ 'status': 'Succeeded',+ 'result': 'Some result',+ }++ reader = ChunkrReader(api_key='test_api_key')+ result = reader.get_task_output('12345')++ self.assertIn('result', result)+ mock_get.assert_called_once_with(+ 'https://api.chunkr.ai/api/v1/task/12345',+ headers=reader._headers,+ timeout=reader.timeout,+ )++ @patch('requests.get')+ def test_get_task_output_max_retries(self, mock_get):+ mock_get.return_value.ok = True+ mock_get.return_value.json.return_value = {'status': 'Pending'}++ reader = ChunkrReader(api_key='test_api_key')++ with self.assertRaises(RuntimeError) as context:+ reader.get_task_output('12345', max_retries=2)++ self.assertEqual(+ str(context.exception), "Max retries reached for task 12345."+ )+ self.assertEqual(mock_get.call_count, 2)