Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

refactor: Iterable dataset (#1900)

Apokryphosxbfd79bb6c86c
diff --git a/camel/datasets/base_generator.py b/camel/datasets/base_generator.py
index de5aa75fb..e9ca6411f 100644
--- a/camel/datasets/base_generator.py
+++ b/camel/datasets/base_generator.py
@@ -13,10 +13,14 @@
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import abc
+import asyncio
import json
import random
from pathlib import Path
-from typing import List, Union
+from typing import Any, Dict, List, Union
+
+from pydantic import ValidationError
+from torch.utils.data import IterableDataset
from camel.logger import get_logger
@@ -25,23 +29,47 @@ from .models import DataPoint
logger = get_logger(__name__)
-class BaseGenerator(abc.ABC):
+class BaseGenerator(abc.ABC, IterableDataset):
r"""Abstract base class for data generators.
This class defines the interface for generating synthetic datapoints.
Concrete implementations should provide specific generation strategies.
"""
- def __init__(self, seed: int = 42, **kwargs):
+ def __init__(
+ self,
+ seed: int = 42,
+ cache: Union[str, Path, None] = None,
+ data_path: Union[str, Path, None] = None,
+ **kwargs,
+ ):
r"""Initialize the base generator.
Args:
seed (int): Random seed for reproducibility. (default: :obj:`42`)
+ cache (Union[str, Path, None]): Optional path to save generated
+ datapoints during iteration. If None is provided, datapoints
+ will be discarded every 100 generations.
+ data_path (Union[str, Path, None]): Optional path to a JSONL file
+ to initialize the dataset from.
**kwargs: Additional generator parameters.
"""
self._rng = random.Random(seed)
+ self.cache = Path(cache) if cache else None
self._data: List[DataPoint] = []
+ self._batch_to_save: List[DataPoint] = []
+
+ if data_path:
+ file_path = Path(data_path)
+ raw_data = self._init_from_jsonl(file_path)
+ try:
+ data_points = [DataPoint(**item) for item in raw_data]
+ self._data.extend(data_points)
+ except ValidationError as e:
+ raise ValueError(
+ f"Failed to create DataPoint from JSONL data: {e}"
+ )
@abc.abstractmethod
async def generate_new(self, n: int, **kwargs) -> List[DataPoint]:
@@ -56,34 +84,112 @@ class BaseGenerator(abc.ABC):
"""
pass
- def __len__(self) -> int:
- r"""Return the size of the generated dataset."""
- return len(self._data)
+ def __aiter__(self):
+ r"""Async iterator that yields datapoints dynamically.
- def __getitem__(self, idx: int) -> DataPoint:
- r"""Retrieve a datapoint by index.
+ If a `data_path` was provided during initialization, those datapoints
+ are yielded first. When self._data is empty, 20 new datapoints
+ are generated. Every 100 yields, the batch is appended to the
+ JSONL file or discarded if `cache` is None.
- Args:
- idx (int): Index of the datapoint.
+ Yields:
+ DataPoint: A single datapoint.
+ """
- Returns:
- DataPoint: The datapoint corresponding to the given index.
+ async def generator():
+ while True:
+ if not self._data:
+ new_datapoints = await self.generate_new(20)
+ self._data.extend(new_datapoints)
+ datapoint = self._data.pop(0)
+ yield datapoint
+ self._batch_to_save.append(datapoint)
+ if len(self._batch_to_save) == 100:
+ if self.cache:
+ with self.cache.open("a", encoding="utf-8") as f:
+ for dp in self._batch_to_save:
+ json.dump(dp.to_dict(), f, ensure_ascii=False)
+ f.write("\n")
+ self._batch_to_save = []
- Raises:
- IndexError: If idx is out of bounds.
+ return generator()
+
+ def __iter__(self):
+ r"""Synchronous iterator for PyTorch IterableDataset compatibility.
+
+ If a `data_path` was provided during initialization, those datapoints
+ are yielded first. When self._data is empty, 20 new datapoints
+ are generated. Every 100 yields, the batch is appended to the
+ JSONL file or discarded if `cache` is None.
+
+ Yields:
+ DataPoint: A single datapoint.
"""
- if idx < 0 or idx >= len(self._data):
- raise IndexError(
- f"Index {idx} out of bounds for dataset of "
- f"size {len(self._data)}"
- )
- return self._data[idx]
+ try:
+ if asyncio.get_event_loop().is_running():
+ raise RuntimeError(
+ "Cannot use synchronous iteration (__iter__) in an async "
+ "context; use 'async for' with __aiter__ instead"
+ )
+ except RuntimeError as e:
+ if "no running event loop" not in str(e):
+ raise
+
+ while True:
+ if not self._data:
+ new_datapoints = asyncio.run(self.generate_new(20))
+ self._data.extend(new_datapoints)
+ datapoint = self._data.pop(0)
+ yield datapoint
+ self._batch_to_save.append(datapoint)
+ if len(self._batch_to_save) == 100:
+ if self.cache:
+ with self.cache.open("a", encoding="utf-8") as f:
+ for dp in self._batch_to_save:
+ json.dump(dp.to_dict(), f, ensure_ascii=False)
+ f.write("\n")
+ self._batch_to_save = []
def sample(self) -> DataPoint:
- if len(self._data) == 0:
- raise RuntimeError("Dataset is empty, cannot sample.")
- idx = self._rng.randint(0, len(self._data) - 1)
- return self[idx]
+ r"""Returns the next datapoint from the current dataset
+ synchronously.
+
+ Raises:
+ RuntimeError: If called in an async context.
+
+ Returns:
+ DataPoint: The next DataPoint.
+
+ Note:
+ This method is intended for synchronous contexts.
+ Use 'async_sample' in asynchronous contexts to
+ avoid blocking or runtime errors.
+ """
+ try:
+ if asyncio.get_event_loop().is_running():
+ raise RuntimeError(
+ "Cannot use synchronous sampling (sample) "
+ "in an async context; use async_sample instead"
+ )
+ except RuntimeError as e:
+ if "no running event loop" not in str(e):
+ raise
+
+ return next(iter(self))
+
+ async def async_sample(self) -> DataPoint:
+ r"""Returns the next datapoint from the current dataset asynchronously.
+
+ Returns:
+ DataPoint: The next datapoint.
+
+ Note:
+ This method is intended for asynchronous contexts. Use 'sample'
+ in synchronous contexts.
+ """
+
+ async_iter = self.__aiter__()
+ return await async_iter.__anext__()
def save_to_jsonl(self, file_path: Union[str, Path]) -> None:
r"""Saves the generated datapoints to a JSONL (JSON Lines) file.
@@ -99,7 +205,7 @@ class BaseGenerator(abc.ABC):
Notes:
- Uses `self._data`, which contains the generated datapoints.
- - Overwrites the file if it already exists.
+ - Appends to the file if it already exists.
- Ensures compatibility with large datasets by using JSONL format.
"""
if not self._data:
@@ -108,11 +214,66 @@ class BaseGenerator(abc.ABC):
file_path = Path(file_path)
try:
- with file_path.open("w", encoding="utf-8") as f:
+ with file_path.open("a", encoding="utf-8") as f:
for datapoint in self._data:
json.dump(datapoint.to_dict(), f, ensure_ascii=False)
- f.write("\n") # Ensure each entry is on a new line
+ f.write("\n")
logger.info(f"Dataset saved successfully to {file_path}")
except IOError as e:
logger.error(f"Error writing to file {file_path}: {e}")
raise
+
+ def flush(self, file_path: Union[str, Path]) -> None:
+ r"""Flush the current data to a JSONL file and clear the data.
+
+ Args:
+ file_path (Union[str, Path]): Path to save the JSONL file.
+
+ Notes:
+ - Uses `save_to_jsonl` to save `self._data`.
+ """
+
+ self.save_to_jsonl(file_path)
+ self._data = []
+ logger.info(f"Data flushed to {file_path} and cleared from the memory")
+
+ def _init_from_jsonl(self, file_path: Path) -> List[Dict[str, Any]]:
+ r"""Load and parse a dataset from a JSONL file.
+
+ Args:
+ file_path (Path): Path to the JSONL file.
+
+ Returns:
+ List[Dict[str, Any]]: A list of datapoint dictionaries.
+
+ Raises:
+ FileNotFoundError: If the specified JSONL file does not exist.
+ ValueError: If a line contains invalid JSON or is not a dictionary.
+ """
+ if not file_path.exists():
+ raise FileNotFoundError(f"JSONL file not found: {file_path}")
+
+ raw_data = []
+ logger.debug(f"Loading JSONL from {file_path}")
+ with file_path.open('r', encoding='utf-8') as f:
+ for line_number, line in enumerate(f, start=1):
+ line = line.strip()
+ if not line:
+ continue # Skip blank lines
+ try:
+ record = json.loads(line)
+ except json.JSONDecodeError as e:
+ raise ValueError(
+ f"Invalid JSON on line {line_number} "
+ f"in file {file_path}: {e}"
+ )
+ if not isinstance(record, dict):
+ raise ValueError(
+ f"Expected a dictionary at line {line_number}, "
+ f"got {type(record).__name__}"
+ )
+ raw_data.append(record)
+ logger.info(
+ f"Successfully loaded {len(raw_data)} items from {file_path}"
+ )
+ return raw_data
diff --git a/test/datasets/test_base_dataset.py b/test/datasets/test_base_dataset.py
index dcce0669b..9c298b451 100644
--- a/test/datasets/test_base_dataset.py
+++ b/test/datasets/test_base_dataset.py
@@ -27,7 +27,9 @@ from camel.datasets import (
FewShotGenerator,
StaticDataset,
)
+from camel.models.base_model import BaseModelBackend
from camel.models.stub_model import StubModel
+from camel.verifiers import BaseVerifier
# ruff: noqa: RUF001
@@ -1427,3 +1429,356 @@ async def test_few_shot_generator_save_to_jsonl(tmp_path):
]
with pytest.raises(IOError, match="No such file or directory"):
dataset.save_to_jsonl(invalid_path)
+
+
+async def test_few_shot_generator_flush(tmp_path):
+ r"""Test FewShotGenerator's flush method with mocked data.
+
+ This test verifies that:
+ 1. Data can be successfully flushed to a JSONL file and
+ the internal data is cleared.
+ 2. The method handles empty datasets appropriately.
+ 3. IO errors are caught and raised as expected.
+
+ Args:
+ tmp_path: Pytest fixture providing a temporary directory.
+ """
+ # Mock the seed dataset
+ mock_static_dataset = MagicMock(spec=StaticDataset)
+ mock_static_dataset.__len__.return_value = 5
+ mock_static_dataset.__getitem__.side_effect = lambda i: DataPoint(
+ question=f"Question {i}",
+ rationale=f"Rationale {i}",
+ final_answer=f"Answer {i}",
+ )
+
+ # Mock the verifier
+ mock_verifier = MagicMock()
+ mock_verifier.verify = AsyncMock()
+ mock_verifier.verify.return_value = MagicMock(result="Verified Answer")
+
+ # Mock the agent
+ mock_agent = MagicMock()
+ mock_agent.step.return_value = MagicMock(
+ msgs=[
+ MagicMock(
+ parsed={
+ 'question': 'Generated Question',
+ 'rationale': 'Generated Rationale',
+ }
+ )
+ ]
+ )
+ mock_agent.reset = MagicMock()
+
+ model = StubModel("OpenAI")
+
+ # Create FewShotGenerator with mocks
+ dataset = FewShotGenerator(
+ seed_dataset=mock_static_dataset,
+ verifier=mock_verifier,
+ model=model,
+ )
+
+ dataset.agent = mock_agent
+
+ # Mock data
+ dataset._data = [
+ DataPoint(
+ question="What is 2 + 2?",
+ rationale="Adding 2 and 2 gives 4.",
+ final_answer="4",
+ metadata={"created": "2025-03-12T10:00:00"},
+ ),
+ DataPoint(
+ question="What color is the sky?",
+ rationale="The sky appears blue due to Rayleigh scattering.",
+ final_answer="Blue",
+ metadata={"created": "2025-03-12T10:01:00"},
+ ),
+ DataPoint(
+ question="How many sides does a triangle have?",
+ rationale="A triangle is defined as a shape with three sides.",
+ final_answer="3",
+ metadata={"created": "2025-03-12T10:02:00"},
+ ),
+ DataPoint(
+ question="What is the capital of France?",
+ rationale="France is a country in Europe, "
+ "and its capital is well-known.",
+ final_answer="Paris",
+ metadata={"created": "2025-03-12T10:03:00"},
+ ),
+ ]
+
+ # **Test 1: Successful flush**
+ file_path = tmp_path / "test_dataset.jsonl"
+ dataset.flush(file_path)
+
+ # Verify file exists and content is correct
+ assert file_path.exists(), "JSONL file was not created"
+ with file_path.open("r", encoding="utf-8") as f:
+ lines = f.readlines()
+ assert len(lines) == 4, "Incorrect number of lines in JSONL file"
+
+ # Parse and verify all lines
+ for i, line in enumerate(lines):
+ parsed_line = json.loads(line)
+ expected_dp = dataset._data[i] if i < len(dataset._data) else None
+ # Since flush clears the data, we can't
+ # directly compare with dataset._data
+ # Instead, we check against the original data structure we set up
+ original_data = [
+ {
+ "question": "What is 2 + 2?",
+ "rationale": "Adding 2 and 2 gives 4.",
+ "final_answer": "4",
+ "metadata": {"created": "2025-03-12T10:00:00"},
+ },
+ {
+ "question": "What color is the sky?",
+ "rationale": "The sky appears blue due to "
+ "Rayleigh scattering.",
+ "final_answer": "Blue",
+ "metadata": {"created": "2025-03-12T10:01:00"},
+ },
+ {
+ "question": "How many sides does a triangle have?",
+ "rationale": "A triangle is defined as a "
+ "shape with three sides.",
+ "final_answer": "3",
+ "metadata": {"created": "2025-03-12T10:02:00"},
+ },
+ {
+ "question": "What is the capital of France?",
+ "rationale": "France is a country in Europe, "
+ "and its capital is well-known.",
+ "final_answer": "Paris",
+ "metadata": {"created": "2025-03-12T10:03:00"},
+ },
+ ]
+ expected_dp = original_data[i]
+ assert (
+ parsed_line["question"] == expected_dp["question"]
+ ), f"Question mismatch at line {i+1}"
+ assert (
+ parsed_line["rationale"] == expected_dp["rationale"]
+ ), f"Rationale mismatch at line {i+1}"
+ assert (
+ parsed_line["final_answer"] == expected_dp["final_answer"]
+ ), f"Final answer mismatch at line {i+1}"
+ assert "metadata" in parsed_line, f"Metadata missing at line {i+1}"
+ assert (
+ parsed_line["metadata"]["created"]
+ == expected_dp["metadata"]["created"]
+ ), f"Metadata mismatch at line {i+1}"
+
+ # Verify that internal data is cleared
+ assert len(dataset._data) == 0, "Internal data was not cleared after flush"
+
+ # **Test 2: Flush with empty dataset**
+ dataset._data = []
+ with pytest.raises(ValueError, match="Dataset is empty. No data to save."):
+ dataset.flush(file_path)
+
+ # **Test 3: IO Error handling with flush**
+ invalid_path = tmp_path / "nonexistent" / "test.jsonl"
+ dataset._data = [
+ DataPoint(
+ question="Test",
+ rationale="Test rationale",
+ final_answer="Test answer",
+ )
+ ]
+ with pytest.raises(IOError, match="No such file or directory"):
+ dataset.flush(invalid_path)
+
+
+# Fixture for a valid JSONL file with 4 datapoints
+# Needed for other tests
+def mock_jsonl_file(tmp_path: Path) -> Path:
+ r"""Creates a temporary JSONL file with four valid datapoints."""
+ file_path = tmp_path / "mock_data.jsonl"
+ datapoints = [
+ {
+ "question": "What is 2 + 2?",
+ "rationale": "Adding 2 and 2 gives 4.",
+ "final_answer": "4",
+ },
+ {
+ "question": "What color is the sky?",
+ "rationale": "The sky appears blue due to scattering.",
+ "final_answer": "Blue",
+ },
+ {
+ "question": "How many sides does a triangle have?",
+ "rationale": "A triangle has three sides.",
+ "final_answer": "3",
+ },
+ {
+ "question": "What is the capital of France?",
+ "rationale": "France’s capital is well-known.",
+ "final_answer": "Paris",
+ },
+ ]
+ with file_path.open("w", encoding="utf-8") as f:
+ for dp in datapoints:
+ json.dump(dp, f)
+ f.write("\n")
+ return file_path
+
+
+# Fixture for an invalid JSONL file
+# Needed for other tests
+def mock_wrong_jsonl_file(tmp_path: Path) -> Path:
+ r"""Creates a temporary JSONL file with
+ invalid data for error handling tests."""
+
+ file_path = tmp_path / "mock_wrong_data.jsonl"
+ invalid_data = [
+ '{"question": "Invalid", "rationale": "Missing final_answer"}',
+ '{"question": "Invalid", "final_answer": "Answer"}',
+ "Not a JSON object",
+ ]
+ with file_path.open("w", encoding="utf-8") as f:
+ for line in invalid_data:
+ f.write(line + "\n")
+ return file_path
+
+
+async def test_few_shot_generator_async(
+ mock_jsonl_file: Path, mock_wrong_jsonl_file: Path
+):
+ r"""Tests FewShotGenerator in an asynchronous context: initialization,
+ async sampling, async iteration, and error handling."""
+
+ mock_static_dataset = MagicMock(spec=StaticDataset)
+ mock_verifier = MagicMock(spec=BaseVerifier)
+ mock_model = MagicMock(spec=BaseModelBackend)
+ mock_model.model_type = "mock_model_type"
+
+ # --- Test 1: Initialization from a valid JSONL file ---
+ generator = FewShotGenerator(
+ seed_dataset=mock_static_dataset,
+ verifier=mock_verifier,
+ model=mock_model,
+ data_path=mock_jsonl_file,
+ )
+ assert (
+ len(generator._data) == 4
+ ), "Expected 4 datapoints to be loaded from JSONL"
+ assert all(
+ isinstance(dp, DataPoint) for dp in generator._data
+ ), "All loaded items should be DataPoint instances"
+ original_data = generator._data.copy()
+
+ # --- Test 2: Async sampling one datapoint ---
+ sampled_dp = await generator.async_sample()
+ assert isinstance(
+ sampled_dp, DataPoint
+ ), "Sampled item should be a DataPoint"
+ assert (
+ sampled_dp in original_data
+ ), "Sampled item should be one of the loaded datapoints"
+ assert (
+ len(generator._data) == 3
+ ), "Async sampling should reduce the number of datapoints by 1"
+
+ # --- Test 3: Asynchronous iteration (__aiter__) ---
+ collected_data = []
+ async for dp in generator:
+ collected_data.append(dp)
+ if len(collected_data) == 3:
+ break
+ assert (
+ len(collected_data) == 3
+ ), "Asynchronous iteration should yield the remaining 3 datapoints"
+ assert all(
+ isinstance(dp, DataPoint) for dp in collected_data
+ ), "All iterated items should be DataPoint instances"
+ assert sorted([dp.question for dp in collected_data]) == sorted(
+ [dp.question for dp in original_data[1:]]
+ ), (
+ "Remaining datapoints should match the "
+ "original data (minus the sampled one)"
+ )
+
+ # --- Test 4: Error handling with an invalid JSONL file ---
+ with pytest.raises(ValueError):
+ FewShotGenerator(
+ seed_dataset=mock_static_dataset,
+ verifier=mock_verifier,
+ model=mock_model,
+ data_path=mock_wrong_jsonl_file,
+ )
+
+
+def test_few_shot_generator_sync(
+ mock_jsonl_file: Path, mock_wrong_jsonl_file: Path
+):
+ r"""Tests FewShotGenerator in a synchronous context: initialization,
+ sampling, iteration, and error handling."""
+
+ mock_static_dataset = MagicMock(spec=StaticDataset)
+ mock_verifier = MagicMock(spec=BaseVerifier)
+ mock_model = MagicMock(spec=BaseModelBackend)
+ mock_model.model_type = "mock_model_type"
+
+ # --- Test 1: Initialization from a valid JSONL file ---
+ generator = FewShotGenerator(
+ seed_dataset=mock_static_dataset,
+ verifier=mock_verifier,
+ model=mock_model,
+ data_path=mock_jsonl_file,
+ )
+ assert (
+ len(generator._data) == 4
+ ), "Expected 4 datapoints to be loaded from JSONL"
+ assert all(
+ isinstance(dp, DataPoint) for dp in generator._data
+ ), "All loaded items should be DataPoint instances"
+ original_data = generator._data.copy()
+
+ # --- Test 2: Sampling one datapoint ---
+ sampled_dp = generator.sample()
+ assert isinstance(
+ sampled_dp, DataPoint
+ ), "Sampled item should be a DataPoint"
+ assert (
+ sampled_dp in original_data
+ ), "Sampled item should be one of the loaded datapoints"
+ assert (
+ len(generator._data) == 3
+ ), "Sampling should reduce the number of datapoints by 1"
+
+ # --- Test 3: Synchronous iteration (__iter__) ---
+ collected_data = []
+ for dp in generator:
+ collected_data.append(dp)
+ if len(collected_data) == 3:
+ break
+ assert (
+ len(collected_data) == 3
+ ), "Synchronous iteration should yield the remaining 3 datapoints"
+ assert all(
+ isinstance(dp, DataPoint) for dp in collected_data
+ ), "All iterated items should be DataPoint instances"
+ assert sorted([dp.question for dp in collected_data]) == sorted(
+ [dp.question for dp in original_data[1:]]
+ ), (
+ "Remaining datapoints should match the "
+ "original data (minus the sampled one)"
+ )
+
+ # --- Test 4: Error handling with an invalid JSONL file ---
+ with pytest.raises(ValueError):
+ FewShotGenerator(
+ seed_dataset=mock_static_dataset,
+ verifier=mock_verifier,
+ model=mock_model,
+ data_path=mock_wrong_jsonl_file,
+ )