Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

chunkr integrated (#1103)

yaoxie22057daccf4deb1
diff --git a/camel/loaders/__init__.py b/camel/loaders/__init__.py
index 57b432ea1..6e66dff25 100644
--- a/camel/loaders/__init__.py
+++ b/camel/loaders/__init__.py
@@ -13,6 +13,7 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from .base_io import File
+from .chunkr_reader import ChunkrReader
from .firecrawl_reader import Firecrawl
from .jina_url_reader import JinaURLReader
from .unstructured_io import UnstructuredIO
@@ -22,4 +23,5 @@ __all__ = [
'UnstructuredIO',
'JinaURLReader',
'Firecrawl',
+ 'ChunkrReader',
]
diff --git a/camel/loaders/chunkr_reader.py b/camel/loaders/chunkr_reader.py
new file mode 100644
index 000000000..4ceec4b24
--- /dev/null
+++ b/camel/loaders/chunkr_reader.py
@@ -0,0 +1,165 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+import json
+import logging
+import os
+import time
+from typing import IO, Any, Optional, Union
+
+import requests
+
+from camel.utils import api_keys_required
+
+logging.basicConfig(level=logging.INFO)
+
+
+class ChunkrReader:
+ r"""Chunkr Reader for processing documents and returning content
+ in various formats.
+
+ Args:
+ api_key (Optional[str], optional): The API key for Chunkr API. If not
+ provided, it will be retrieved from the environment variable
+ `CHUNKR_API_KEY`. (default: :obj:`None`)
+ url (Optional[str], optional): The url to the Chunkr service.
+ (default: :obj:`https://api.chunkr.ai/api/v1/task`)
+ timeout (int, optional): The maximum time in seconds to wait for the
+ API responses. (default: :obj:`30`)
+ **kwargs (Any): Additional keyword arguments for request headers.
+ """
+
+ @api_keys_required("CHUNKR_API_KEY")
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ url: Optional[str] = "https://api.chunkr.ai/api/v1/task",
+ timeout: int = 30,
+ **kwargs: Any,
+ ) -> None:
+ self._api_key = api_key or os.getenv('CHUNKR_API_KEY')
+ self._url = os.getenv('CHUNKR_API_URL') or url
+ self._headers = {
+ "Authorization": f"{self._api_key}",
+ **kwargs,
+ }
+ self.timeout = timeout
+
+ def submit_task(
+ self,
+ file_path: str,
+ model: str = "Fast",
+ ocr_strategy: str = "Auto",
+ target_chunk_length: str = "512",
+ ) -> str:
+ r"""Submits a file to the Chunkr API and returns the task ID.
+
+ Args:
+ file_path (str): The path to the file to be uploaded.
+ model (str, optional): The model to be used for the task.
+ (default: :obj:`Fast`)
+ ocr_strategy (str, optional): The OCR strategy. Defaults to 'Auto'.
+ target_chunk_length (str, optional): The target chunk length.
+ (default: :obj:`512`)
+
+ Returns:
+ str: The task ID.
+ """
+ with open(file_path, 'rb') as file:
+ files: dict[
+ str, Union[tuple[None, IO[bytes]], tuple[None, str]]
+ ] = {
+ 'file': (
+ None,
+ file,
+ ), # Properly pass the file as a binary stream
+ 'model': (None, model),
+ 'ocr_strategy': (None, ocr_strategy),
+ 'target_chunk_length': (None, target_chunk_length),
+ }
+ try:
+ response = requests.post(
+ self._url, # type: ignore[arg-type]
+ headers=self._headers,
+ files=files,
+ timeout=self.timeout,
+ )
+ response.raise_for_status()
+ task_id = response.json().get('task_id')
+ if not task_id:
+ raise ValueError("Task ID not returned in the response.")
+ logging.info(
+ f"Task submitted successfully. Task ID: {task_id}"
+ )
+ return task_id
+ except Exception as e:
+ logging.error(f"Failed to submit task: {e}")
+ raise ValueError(f"Failed to submit task: {e}") from e
+
+ def get_task_output(self, task_id: str, max_retries: int = 5) -> str:
+ r"""Polls the Chunkr API to check the task status and returns the task
+ result.
+
+ Args:
+ task_id (str): The task ID to check the status for.
+ max_retries (int, optional): Maximum number of retry attempts.
+ (default: :obj:`5`)
+
+ Returns:
+ str: The formatted task result in JSON format.
+
+ Raises:
+ ValueError: If the task status cannot be retrieved.
+ RuntimeError: If the maximum number of retries is reached without
+ a successful task completion.
+ """
+ url_get = f"{self._url}/{task_id}"
+ attempts = 0
+
+ while attempts < max_retries:
+ try:
+ response = requests.get(
+ url_get, headers=self._headers, timeout=self.timeout
+ )
+ response.raise_for_status()
+ task_status = response.json().get('status')
+
+ if task_status == "Succeeded":
+ logging.info(f"Task {task_id} completed successfully.")
+ return self._pretty_print_response(response.json())
+ else:
+ logging.info(
+ f"Task {task_id} is still {task_status}. Retrying "
+ "in 5 seconds..."
+ )
+ except Exception as e:
+ logging.error(f"Failed to retrieve task status: {e}")
+ raise ValueError(f"Failed to retrieve task status: {e}") from e
+
+ attempts += 1
+ time.sleep(5)
+
+ logging.error(f"Max retries reached for task {task_id}.")
+ raise RuntimeError(f"Max retries reached for task {task_id}.")
+
+ def _pretty_print_response(self, response_json: dict) -> str:
+ r"""Pretty prints the JSON response.
+
+ Args:
+ response_json (dict): The response JSON to pretty print.
+
+ Returns:
+ str: Formatted JSON as a string.
+ """
+ return json.dumps(response_json, indent=4)
diff --git a/examples/loaders/chunkr_example.py b/examples/loaders/chunkr_example.py
new file mode 100644
index 000000000..e28a9e2d4
--- /dev/null
+++ b/examples/loaders/chunkr_example.py
@@ -0,0 +1,73 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+from camel.loaders import ChunkrReader
+
+
+def read_with_different_model():
+ r"""Reads a document using the Chunkr API."""
+ print("Choose a model for processing the file:")
+ print("1. Fast")
+ print("2. HighQuality")
+ choice = input("Enter your choice (1-2): ")
+ models = {"1": "Fast", "2": "HighQuality"}
+
+ if choice not in models:
+ print("Invalid choice. Exiting.")
+ return
+
+ model = models[choice]
+
+ print("Choose an OCR strategy:")
+ print("1. Auto")
+ print("2. All")
+ print("3. Off")
+ ocr_choice = input("Enter your choice (1-3) [Default: Auto]: ")
+ ocr_strategies = {"1": "Auto", "2": "All", "3": "Off"}
+
+ if ocr_choice not in ocr_strategies:
+ ocr_strategy = "Auto"
+ else:
+ ocr_strategy = ocr_strategies[ocr_choice]
+
+ while True:
+ target_chunk_length = (
+ input("Enter the target chunk length [Default: 512]: ") or "512"
+ )
+ if target_chunk_length.isdigit() and int(target_chunk_length) > 0:
+ break
+ else:
+ print("Invalid input. Please enter a valid positive integer.")
+
+ file_path = input("Please provide the file path: ")
+
+ chunkr_reader = ChunkrReader()
+
+ try:
+ task_id = chunkr_reader.submit_task(
+ file_path,
+ model=model,
+ ocr_strategy=ocr_strategy,
+ target_chunk_length=target_chunk_length,
+ )
+
+ result = chunkr_reader.get_task_output(task_id)
+ print(result)
+
+ except Exception as e:
+ print(f"An error occurred: {e}")
+
+
+if __name__ == "__main__":
+ read_with_different_model()
diff --git a/test/loaders/test_chunkr_reader.py b/test/loaders/test_chunkr_reader.py
new file mode 100644
index 000000000..28ff767e9
--- /dev/null
+++ b/test/loaders/test_chunkr_reader.py
@@ -0,0 +1,86 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+import unittest
+from unittest.mock import mock_open, patch
+
+from camel.loaders import ChunkrReader
+
+
+class TestChunkrReader(unittest.TestCase):
+ @patch('requests.post')
+ @patch(
+ 'builtins.open', new_callable=mock_open, read_data='fake file content'
+ )
+ def test_submit_task_success(self, mock_file, mock_post):
+ mock_post.return_value.ok = True
+ mock_post.return_value.json.return_value = {'task_id': '12345'}
+
+ reader = ChunkrReader(api_key='test_api_key')
+ task_id = reader.submit_task('fake_path.txt')
+
+ self.assertEqual(task_id, '12345')
+ mock_post.assert_called_once()
+ mock_file.assert_called_once_with('fake_path.txt', 'rb')
+
+ @patch('requests.post')
+ @patch(
+ 'builtins.open', new_callable=mock_open, read_data='fake file content'
+ )
+ def test_submit_task_no_task_id(self, mock_file, mock_post):
+ mock_post.return_value.ok = True
+ mock_post.return_value.json.return_value = {}
+
+ reader = ChunkrReader(api_key='test_api_key')
+
+ with self.assertRaises(ValueError) as context:
+ reader.submit_task('fake_path.txt')
+
+ self.assertEqual(
+ str(context.exception),
+ "Failed to submit task: Task ID not returned in the response.",
+ )
+
+ @patch('requests.get')
+ def test_get_task_output_success(self, mock_get):
+ mock_get.return_value.ok = True
+ mock_get.return_value.json.return_value = {
+ 'status': 'Succeeded',
+ 'result': 'Some result',
+ }
+
+ reader = ChunkrReader(api_key='test_api_key')
+ result = reader.get_task_output('12345')
+
+ self.assertIn('result', result)
+ mock_get.assert_called_once_with(
+ 'https://api.chunkr.ai/api/v1/task/12345',
+ headers=reader._headers,
+ timeout=reader.timeout,
+ )
+
+ @patch('requests.get')
+ def test_get_task_output_max_retries(self, mock_get):
+ mock_get.return_value.ok = True
+ mock_get.return_value.json.return_value = {'status': 'Pending'}
+
+ reader = ChunkrReader(api_key='test_api_key')
+
+ with self.assertRaises(RuntimeError) as context:
+ reader.get_task_output('12345', max_retries=2)
+
+ self.assertEqual(
+ str(context.exception), "Max retries reached for task 12345."
+ )
+ self.assertEqual(mock_get.call_count, 2)