🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
refactor: Iterable dataset (#1900)
Apokryphosxbfd79bb6c86c
diff --git a/camel/datasets/base_generator.py b/camel/datasets/base_generator.pyindex de5aa75fb..e9ca6411f 100644--- a/camel/datasets/base_generator.py+++ b/camel/datasets/base_generator.py@@ -13,10 +13,14 @@# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import abc+import asyncioimport jsonimport randomfrom pathlib import Path-from typing import List, Union+from typing import Any, Dict, List, Union++from pydantic import ValidationError+from torch.utils.data import IterableDatasetfrom camel.logger import get_logger@@ -25,23 +29,47 @@ from .models import DataPointlogger = get_logger(__name__)-class BaseGenerator(abc.ABC):+class BaseGenerator(abc.ABC, IterableDataset):r"""Abstract base class for data generators.This class defines the interface for generating synthetic datapoints.Concrete implementations should provide specific generation strategies."""- def __init__(self, seed: int = 42, **kwargs):+ def __init__(+ self,+ seed: int = 42,+ cache: Union[str, Path, None] = None,+ data_path: Union[str, Path, None] = None,+ **kwargs,+ ):r"""Initialize the base generator.Args:seed (int): Random seed for reproducibility. (default: :obj:`42`)+ cache (Union[str, Path, None]): Optional path to save generated+ datapoints during iteration. If None is provided, datapoints+ will be discarded every 100 generations.+ data_path (Union[str, Path, None]): Optional path to a JSONL file+ to initialize the dataset from.**kwargs: Additional generator parameters."""self._rng = random.Random(seed)+ self.cache = Path(cache) if cache else Noneself._data: List[DataPoint] = []+ self._batch_to_save: List[DataPoint] = []++ if data_path:+ file_path = Path(data_path)+ raw_data = self._init_from_jsonl(file_path)+ try:+ data_points = [DataPoint(**item) for item in raw_data]+ self._data.extend(data_points)+ except ValidationError as e:+ raise ValueError(+ f"Failed to create DataPoint from JSONL data: {e}"+ )@abc.abstractmethodasync def generate_new(self, n: int, **kwargs) -> List[DataPoint]:@@ -56,34 +84,112 @@ class BaseGenerator(abc.ABC):"""pass- def __len__(self) -> int:- r"""Return the size of the generated dataset."""- return len(self._data)+ def __aiter__(self):+ r"""Async iterator that yields datapoints dynamically.- def __getitem__(self, idx: int) -> DataPoint:- r"""Retrieve a datapoint by index.+ If a `data_path` was provided during initialization, those datapoints+ are yielded first. When self._data is empty, 20 new datapoints+ are generated. Every 100 yields, the batch is appended to the+ JSONL file or discarded if `cache` is None.- Args:- idx (int): Index of the datapoint.+ Yields:+ DataPoint: A single datapoint.+ """- Returns:- DataPoint: The datapoint corresponding to the given index.+ async def generator():+ while True:+ if not self._data:+ new_datapoints = await self.generate_new(20)+ self._data.extend(new_datapoints)+ datapoint = self._data.pop(0)+ yield datapoint+ self._batch_to_save.append(datapoint)+ if len(self._batch_to_save) == 100:+ if self.cache:+ with self.cache.open("a", encoding="utf-8") as f:+ for dp in self._batch_to_save:+ json.dump(dp.to_dict(), f, ensure_ascii=False)+ f.write("\n")+ self._batch_to_save = []- Raises:- IndexError: If idx is out of bounds.+ return generator()++ def __iter__(self):+ r"""Synchronous iterator for PyTorch IterableDataset compatibility.++ If a `data_path` was provided during initialization, those datapoints+ are yielded first. When self._data is empty, 20 new datapoints+ are generated. Every 100 yields, the batch is appended to the+ JSONL file or discarded if `cache` is None.++ Yields:+ DataPoint: A single datapoint."""- if idx < 0 or idx >= len(self._data):- raise IndexError(- f"Index {idx} out of bounds for dataset of "- f"size {len(self._data)}"- )- return self._data[idx]+ try:+ if asyncio.get_event_loop().is_running():+ raise RuntimeError(+ "Cannot use synchronous iteration (__iter__) in an async "+ "context; use 'async for' with __aiter__ instead"+ )+ except RuntimeError as e:+ if "no running event loop" not in str(e):+ raise++ while True:+ if not self._data:+ new_datapoints = asyncio.run(self.generate_new(20))+ self._data.extend(new_datapoints)+ datapoint = self._data.pop(0)+ yield datapoint+ self._batch_to_save.append(datapoint)+ if len(self._batch_to_save) == 100:+ if self.cache:+ with self.cache.open("a", encoding="utf-8") as f:+ for dp in self._batch_to_save:+ json.dump(dp.to_dict(), f, ensure_ascii=False)+ f.write("\n")+ self._batch_to_save = []def sample(self) -> DataPoint:- if len(self._data) == 0:- raise RuntimeError("Dataset is empty, cannot sample.")- idx = self._rng.randint(0, len(self._data) - 1)- return self[idx]+ r"""Returns the next datapoint from the current dataset+ synchronously.++ Raises:+ RuntimeError: If called in an async context.++ Returns:+ DataPoint: The next DataPoint.++ Note:+ This method is intended for synchronous contexts.+ Use 'async_sample' in asynchronous contexts to+ avoid blocking or runtime errors.+ """+ try:+ if asyncio.get_event_loop().is_running():+ raise RuntimeError(+ "Cannot use synchronous sampling (sample) "+ "in an async context; use async_sample instead"+ )+ except RuntimeError as e:+ if "no running event loop" not in str(e):+ raise++ return next(iter(self))++ async def async_sample(self) -> DataPoint:+ r"""Returns the next datapoint from the current dataset asynchronously.++ Returns:+ DataPoint: The next datapoint.++ Note:+ This method is intended for asynchronous contexts. Use 'sample'+ in synchronous contexts.+ """++ async_iter = self.__aiter__()+ return await async_iter.__anext__()def save_to_jsonl(self, file_path: Union[str, Path]) -> None:r"""Saves the generated datapoints to a JSONL (JSON Lines) file.@@ -99,7 +205,7 @@ class BaseGenerator(abc.ABC):Notes:- Uses `self._data`, which contains the generated datapoints.- - Overwrites the file if it already exists.+ - Appends to the file if it already exists.- Ensures compatibility with large datasets by using JSONL format."""if not self._data:@@ -108,11 +214,66 @@ class BaseGenerator(abc.ABC):file_path = Path(file_path)try:- with file_path.open("w", encoding="utf-8") as f:+ with file_path.open("a", encoding="utf-8") as f:for datapoint in self._data:json.dump(datapoint.to_dict(), f, ensure_ascii=False)- f.write("\n") # Ensure each entry is on a new line+ f.write("\n")logger.info(f"Dataset saved successfully to {file_path}")except IOError as e:logger.error(f"Error writing to file {file_path}: {e}")raise++ def flush(self, file_path: Union[str, Path]) -> None:+ r"""Flush the current data to a JSONL file and clear the data.++ Args:+ file_path (Union[str, Path]): Path to save the JSONL file.++ Notes:+ - Uses `save_to_jsonl` to save `self._data`.+ """++ self.save_to_jsonl(file_path)+ self._data = []+ logger.info(f"Data flushed to {file_path} and cleared from the memory")++ def _init_from_jsonl(self, file_path: Path) -> List[Dict[str, Any]]:+ r"""Load and parse a dataset from a JSONL file.++ Args:+ file_path (Path): Path to the JSONL file.++ Returns:+ List[Dict[str, Any]]: A list of datapoint dictionaries.++ Raises:+ FileNotFoundError: If the specified JSONL file does not exist.+ ValueError: If a line contains invalid JSON or is not a dictionary.+ """+ if not file_path.exists():+ raise FileNotFoundError(f"JSONL file not found: {file_path}")++ raw_data = []+ logger.debug(f"Loading JSONL from {file_path}")+ with file_path.open('r', encoding='utf-8') as f:+ for line_number, line in enumerate(f, start=1):+ line = line.strip()+ if not line:+ continue # Skip blank lines+ try:+ record = json.loads(line)+ except json.JSONDecodeError as e:+ raise ValueError(+ f"Invalid JSON on line {line_number} "+ f"in file {file_path}: {e}"+ )+ if not isinstance(record, dict):+ raise ValueError(+ f"Expected a dictionary at line {line_number}, "+ f"got {type(record).__name__}"+ )+ raw_data.append(record)+ logger.info(+ f"Successfully loaded {len(raw_data)} items from {file_path}"+ )+ return raw_datadiff --git a/test/datasets/test_base_dataset.py b/test/datasets/test_base_dataset.pyindex dcce0669b..9c298b451 100644--- a/test/datasets/test_base_dataset.py+++ b/test/datasets/test_base_dataset.py@@ -27,7 +27,9 @@ from camel.datasets import (FewShotGenerator,StaticDataset,)+from camel.models.base_model import BaseModelBackendfrom camel.models.stub_model import StubModel+from camel.verifiers import BaseVerifier# ruff: noqa: RUF001@@ -1427,3 +1429,356 @@ async def test_few_shot_generator_save_to_jsonl(tmp_path):]with pytest.raises(IOError, match="No such file or directory"):dataset.save_to_jsonl(invalid_path)+++async def test_few_shot_generator_flush(tmp_path):+ r"""Test FewShotGenerator's flush method with mocked data.++ This test verifies that:+ 1. Data can be successfully flushed to a JSONL file and+ the internal data is cleared.+ 2. The method handles empty datasets appropriately.+ 3. IO errors are caught and raised as expected.++ Args:+ tmp_path: Pytest fixture providing a temporary directory.+ """+ # Mock the seed dataset+ mock_static_dataset = MagicMock(spec=StaticDataset)+ mock_static_dataset.__len__.return_value = 5+ mock_static_dataset.__getitem__.side_effect = lambda i: DataPoint(+ question=f"Question {i}",+ rationale=f"Rationale {i}",+ final_answer=f"Answer {i}",+ )++ # Mock the verifier+ mock_verifier = MagicMock()+ mock_verifier.verify = AsyncMock()+ mock_verifier.verify.return_value = MagicMock(result="Verified Answer")++ # Mock the agent+ mock_agent = MagicMock()+ mock_agent.step.return_value = MagicMock(+ msgs=[+ MagicMock(+ parsed={+ 'question': 'Generated Question',+ 'rationale': 'Generated Rationale',+ }+ )+ ]+ )+ mock_agent.reset = MagicMock()++ model = StubModel("OpenAI")++ # Create FewShotGenerator with mocks+ dataset = FewShotGenerator(+ seed_dataset=mock_static_dataset,+ verifier=mock_verifier,+ model=model,+ )++ dataset.agent = mock_agent++ # Mock data+ dataset._data = [+ DataPoint(+ question="What is 2 + 2?",+ rationale="Adding 2 and 2 gives 4.",+ final_answer="4",+ metadata={"created": "2025-03-12T10:00:00"},+ ),+ DataPoint(+ question="What color is the sky?",+ rationale="The sky appears blue due to Rayleigh scattering.",+ final_answer="Blue",+ metadata={"created": "2025-03-12T10:01:00"},+ ),+ DataPoint(+ question="How many sides does a triangle have?",+ rationale="A triangle is defined as a shape with three sides.",+ final_answer="3",+ metadata={"created": "2025-03-12T10:02:00"},+ ),+ DataPoint(+ question="What is the capital of France?",+ rationale="France is a country in Europe, "+ "and its capital is well-known.",+ final_answer="Paris",+ metadata={"created": "2025-03-12T10:03:00"},+ ),+ ]++ # **Test 1: Successful flush**+ file_path = tmp_path / "test_dataset.jsonl"+ dataset.flush(file_path)++ # Verify file exists and content is correct+ assert file_path.exists(), "JSONL file was not created"+ with file_path.open("r", encoding="utf-8") as f:+ lines = f.readlines()+ assert len(lines) == 4, "Incorrect number of lines in JSONL file"++ # Parse and verify all lines+ for i, line in enumerate(lines):+ parsed_line = json.loads(line)+ expected_dp = dataset._data[i] if i < len(dataset._data) else None+ # Since flush clears the data, we can't+ # directly compare with dataset._data+ # Instead, we check against the original data structure we set up+ original_data = [+ {+ "question": "What is 2 + 2?",+ "rationale": "Adding 2 and 2 gives 4.",+ "final_answer": "4",+ "metadata": {"created": "2025-03-12T10:00:00"},+ },+ {+ "question": "What color is the sky?",+ "rationale": "The sky appears blue due to "+ "Rayleigh scattering.",+ "final_answer": "Blue",+ "metadata": {"created": "2025-03-12T10:01:00"},+ },+ {+ "question": "How many sides does a triangle have?",+ "rationale": "A triangle is defined as a "+ "shape with three sides.",+ "final_answer": "3",+ "metadata": {"created": "2025-03-12T10:02:00"},+ },+ {+ "question": "What is the capital of France?",+ "rationale": "France is a country in Europe, "+ "and its capital is well-known.",+ "final_answer": "Paris",+ "metadata": {"created": "2025-03-12T10:03:00"},+ },+ ]+ expected_dp = original_data[i]+ assert (+ parsed_line["question"] == expected_dp["question"]+ ), f"Question mismatch at line {i+1}"+ assert (+ parsed_line["rationale"] == expected_dp["rationale"]+ ), f"Rationale mismatch at line {i+1}"+ assert (+ parsed_line["final_answer"] == expected_dp["final_answer"]+ ), f"Final answer mismatch at line {i+1}"+ assert "metadata" in parsed_line, f"Metadata missing at line {i+1}"+ assert (+ parsed_line["metadata"]["created"]+ == expected_dp["metadata"]["created"]+ ), f"Metadata mismatch at line {i+1}"++ # Verify that internal data is cleared+ assert len(dataset._data) == 0, "Internal data was not cleared after flush"++ # **Test 2: Flush with empty dataset**+ dataset._data = []+ with pytest.raises(ValueError, match="Dataset is empty. No data to save."):+ dataset.flush(file_path)++ # **Test 3: IO Error handling with flush**+ invalid_path = tmp_path / "nonexistent" / "test.jsonl"+ dataset._data = [+ DataPoint(+ question="Test",+ rationale="Test rationale",+ final_answer="Test answer",+ )+ ]+ with pytest.raises(IOError, match="No such file or directory"):+ dataset.flush(invalid_path)+++# Fixture for a valid JSONL file with 4 datapoints+# Needed for other tests+def mock_jsonl_file(tmp_path: Path) -> Path:+ r"""Creates a temporary JSONL file with four valid datapoints."""+ file_path = tmp_path / "mock_data.jsonl"+ datapoints = [+ {+ "question": "What is 2 + 2?",+ "rationale": "Adding 2 and 2 gives 4.",+ "final_answer": "4",+ },+ {+ "question": "What color is the sky?",+ "rationale": "The sky appears blue due to scattering.",+ "final_answer": "Blue",+ },+ {+ "question": "How many sides does a triangle have?",+ "rationale": "A triangle has three sides.",+ "final_answer": "3",+ },+ {+ "question": "What is the capital of France?",+ "rationale": "France’s capital is well-known.",+ "final_answer": "Paris",+ },+ ]+ with file_path.open("w", encoding="utf-8") as f:+ for dp in datapoints:+ json.dump(dp, f)+ f.write("\n")+ return file_path+++# Fixture for an invalid JSONL file+# Needed for other tests+def mock_wrong_jsonl_file(tmp_path: Path) -> Path:+ r"""Creates a temporary JSONL file with+ invalid data for error handling tests."""++ file_path = tmp_path / "mock_wrong_data.jsonl"+ invalid_data = [+ '{"question": "Invalid", "rationale": "Missing final_answer"}',+ '{"question": "Invalid", "final_answer": "Answer"}',+ "Not a JSON object",+ ]+ with file_path.open("w", encoding="utf-8") as f:+ for line in invalid_data:+ f.write(line + "\n")+ return file_path+++async def test_few_shot_generator_async(+ mock_jsonl_file: Path, mock_wrong_jsonl_file: Path+):+ r"""Tests FewShotGenerator in an asynchronous context: initialization,+ async sampling, async iteration, and error handling."""++ mock_static_dataset = MagicMock(spec=StaticDataset)+ mock_verifier = MagicMock(spec=BaseVerifier)+ mock_model = MagicMock(spec=BaseModelBackend)+ mock_model.model_type = "mock_model_type"++ # --- Test 1: Initialization from a valid JSONL file ---+ generator = FewShotGenerator(+ seed_dataset=mock_static_dataset,+ verifier=mock_verifier,+ model=mock_model,+ data_path=mock_jsonl_file,+ )+ assert (+ len(generator._data) == 4+ ), "Expected 4 datapoints to be loaded from JSONL"+ assert all(+ isinstance(dp, DataPoint) for dp in generator._data+ ), "All loaded items should be DataPoint instances"+ original_data = generator._data.copy()++ # --- Test 2: Async sampling one datapoint ---+ sampled_dp = await generator.async_sample()+ assert isinstance(+ sampled_dp, DataPoint+ ), "Sampled item should be a DataPoint"+ assert (+ sampled_dp in original_data+ ), "Sampled item should be one of the loaded datapoints"+ assert (+ len(generator._data) == 3+ ), "Async sampling should reduce the number of datapoints by 1"++ # --- Test 3: Asynchronous iteration (__aiter__) ---+ collected_data = []+ async for dp in generator:+ collected_data.append(dp)+ if len(collected_data) == 3:+ break+ assert (+ len(collected_data) == 3+ ), "Asynchronous iteration should yield the remaining 3 datapoints"+ assert all(+ isinstance(dp, DataPoint) for dp in collected_data+ ), "All iterated items should be DataPoint instances"+ assert sorted([dp.question for dp in collected_data]) == sorted(+ [dp.question for dp in original_data[1:]]+ ), (+ "Remaining datapoints should match the "+ "original data (minus the sampled one)"+ )++ # --- Test 4: Error handling with an invalid JSONL file ---+ with pytest.raises(ValueError):+ FewShotGenerator(+ seed_dataset=mock_static_dataset,+ verifier=mock_verifier,+ model=mock_model,+ data_path=mock_wrong_jsonl_file,+ )+++def test_few_shot_generator_sync(+ mock_jsonl_file: Path, mock_wrong_jsonl_file: Path+):+ r"""Tests FewShotGenerator in a synchronous context: initialization,+ sampling, iteration, and error handling."""++ mock_static_dataset = MagicMock(spec=StaticDataset)+ mock_verifier = MagicMock(spec=BaseVerifier)+ mock_model = MagicMock(spec=BaseModelBackend)+ mock_model.model_type = "mock_model_type"++ # --- Test 1: Initialization from a valid JSONL file ---+ generator = FewShotGenerator(+ seed_dataset=mock_static_dataset,+ verifier=mock_verifier,+ model=mock_model,+ data_path=mock_jsonl_file,+ )+ assert (+ len(generator._data) == 4+ ), "Expected 4 datapoints to be loaded from JSONL"+ assert all(+ isinstance(dp, DataPoint) for dp in generator._data+ ), "All loaded items should be DataPoint instances"+ original_data = generator._data.copy()++ # --- Test 2: Sampling one datapoint ---+ sampled_dp = generator.sample()+ assert isinstance(+ sampled_dp, DataPoint+ ), "Sampled item should be a DataPoint"+ assert (+ sampled_dp in original_data+ ), "Sampled item should be one of the loaded datapoints"+ assert (+ len(generator._data) == 3+ ), "Sampling should reduce the number of datapoints by 1"++ # --- Test 3: Synchronous iteration (__iter__) ---+ collected_data = []+ for dp in generator:+ collected_data.append(dp)+ if len(collected_data) == 3:+ break+ assert (+ len(collected_data) == 3+ ), "Synchronous iteration should yield the remaining 3 datapoints"+ assert all(+ isinstance(dp, DataPoint) for dp in collected_data+ ), "All iterated items should be DataPoint instances"+ assert sorted([dp.question for dp in collected_data]) == sorted(+ [dp.question for dp in original_data[1:]]+ ), (+ "Remaining datapoints should match the "+ "original data (minus the sampled one)"+ )++ # --- Test 4: Error handling with an invalid JSONL file ---+ with pytest.raises(ValueError):+ FewShotGenerator(+ seed_dataset=mock_static_dataset,+ verifier=mock_verifier,+ model=mock_model,+ data_path=mock_wrong_jsonl_file,+ )