Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

Fixes #2998 add kwargs to ChunkrReaderConfig (#3228)

Feng Xue80e8b83189bc
diff --git a/camel/loaders/chunkr_reader.py b/camel/loaders/chunkr_reader.py
index 2be01bd1f..279c97b4e 100644
--- a/camel/loaders/chunkr_reader.py
+++ b/camel/loaders/chunkr_reader.py
@@ -34,6 +34,10 @@ class ChunkrReaderConfig:
high_resolution (bool, optional): Whether to use high resolution OCR.
(default: :obj:`True`)
ocr_strategy (str, optional): The OCR strategy. Defaults to 'Auto'.
+ **kwargs: Additional keyword arguments to pass to the Chunkr Configuration.
+ This accepts all other Configuration parameters such as expires_in,
+ pipeline, segment_processing, segmentation_strategy, etc.
+ See: https://github.com/lumina-ai-inc/chunkr/blob/main/core/src/models/task.rs#L749
"""
def __init__(
@@ -41,10 +45,12 @@ class ChunkrReaderConfig:
chunk_processing: int = 512,
high_resolution: bool = True,
ocr_strategy: str = "Auto",
+ **kwargs,
):
self.chunk_processing = chunk_processing
self.high_resolution = high_resolution
self.ocr_strategy = ocr_strategy
+ self.kwargs = kwargs
class ChunkrReader:
@@ -175,11 +181,7 @@ class ChunkrReader:
Returns:
Configuration: Chunkr SDK configuration.
"""
- from chunkr_ai.models import (
- ChunkProcessing,
- Configuration,
- OcrStrategy,
- )
+ from chunkr_ai.models import ChunkProcessing, Configuration, OcrStrategy
return Configuration(
chunk_processing=ChunkProcessing(
@@ -190,4 +192,5 @@ class ChunkrReader:
"Auto": OcrStrategy.AUTO,
"All": OcrStrategy.ALL,
}.get(chunkr_config.ocr_strategy, OcrStrategy.ALL),
+ **chunkr_config.kwargs,
)
diff --git a/test/loaders/test_chunkr_reader.py b/test/loaders/test_chunkr_reader.py
index 357cf35e2..fc5a646e5 100644
--- a/test/loaders/test_chunkr_reader.py
+++ b/test/loaders/test_chunkr_reader.py
@@ -14,7 +14,7 @@
import json
import unittest
-from unittest.mock import AsyncMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
from chunkr_ai.models import Status
@@ -119,6 +119,92 @@ class TestChunkrReader(unittest.TestCase):
mock_chunkr_instance.get_task.assert_called_once_with("test_task_id")
mock_task.poll.assert_called_once()
+ def test_chunkr_reader_config_defaults(self):
+ """Test ChunkrReaderConfig with default values."""
+ config = ChunkrReaderConfig()
+
+ self.assertEqual(config.chunk_processing, 512)
+ self.assertEqual(config.high_resolution, True)
+ self.assertEqual(config.ocr_strategy, "Auto")
+ self.assertEqual(config.kwargs, {})
+
+ def test_chunkr_reader_config_custom_values(self):
+ """Test ChunkrReaderConfig with custom values."""
+ config = ChunkrReaderConfig(
+ chunk_processing=1024,
+ high_resolution=False,
+ ocr_strategy="All",
+ )
+
+ self.assertEqual(config.chunk_processing, 1024)
+ self.assertEqual(config.high_resolution, False)
+ self.assertEqual(config.ocr_strategy, "All")
+
+ def test_chunkr_reader_config_with_kwargs(self):
+ """Test ChunkrReaderConfig with additional kwargs."""
+ config = ChunkrReaderConfig(
+ chunk_processing=2048,
+ expires_in=3600,
+ pipeline="custom_pipeline",
+ )
+
+ self.assertEqual(config.chunk_processing, 2048)
+ self.assertEqual(config.kwargs["expires_in"], 3600)
+ self.assertEqual(config.kwargs["pipeline"], "custom_pipeline")
+
+ @patch('chunkr_ai.models.Configuration')
+ @patch('chunkr_ai.models.ChunkProcessing')
+ @patch('chunkr_ai.models.OcrStrategy')
+ def test_to_chunkr_configuration_auto_strategy(
+ self, mock_ocr_strategy, mock_chunk_processing, mock_configuration
+ ):
+ """Test _to_chunkr_configuration with Auto OCR strategy."""
+ config = ChunkrReaderConfig(
+ chunk_processing=512, high_resolution=True, ocr_strategy="Auto"
+ )
+
+ mock_chunk_processing_instance = MagicMock()
+ mock_chunk_processing.return_value = mock_chunk_processing_instance
+ mock_ocr_strategy.AUTO = "AUTO"
+
+ self.reader._to_chunkr_configuration(config)
+
+ mock_chunk_processing.assert_called_once_with(target_length=512)
+ mock_configuration.assert_called_once_with(
+ chunk_processing=mock_chunk_processing_instance,
+ high_resolution=True,
+ ocr_strategy="AUTO",
+ )
+
+ @patch('chunkr_ai.models.Configuration')
+ @patch('chunkr_ai.models.ChunkProcessing')
+ @patch('chunkr_ai.models.OcrStrategy')
+ def test_to_chunkr_configuration_with_kwargs(
+ self, mock_ocr_strategy, mock_chunk_processing, mock_configuration
+ ):
+ """Test _to_chunkr_configuration with additional kwargs."""
+ config = ChunkrReaderConfig(
+ chunk_processing=512,
+ high_resolution=True,
+ ocr_strategy="Auto",
+ expires_in=7200,
+ pipeline="test_pipeline",
+ )
+
+ mock_chunk_processing_instance = MagicMock()
+ mock_chunk_processing.return_value = mock_chunk_processing_instance
+ mock_ocr_strategy.AUTO = "AUTO"
+
+ result = self.reader._to_chunkr_configuration(config)
+
+ mock_chunk_processing.assert_called_once_with(target_length=512)
+ mock_configuration.assert_called_once_with(
+ chunk_processing=mock_chunk_processing_instance,
+ high_resolution=True,
+ ocr_strategy="AUTO",
+ expires_in=7200,
+ pipeline="test_pipeline",
+ )
if __name__ == "__main__":
unittest.main()