Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Initial implementation for BaseVerifier (#1597)

Wendong-Fan84185a59341f
diff --git a/camel/datasets/__init__.py b/camel/datasets/__init__.py
new file mode 100644
index 000000000..e12a2a0bb
--- /dev/null
+++ b/camel/datasets/__init__.py
@@ -0,0 +1,28 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from .base import (
+ BaseDataset,
+ DataPoint,
+ GenerativeDataset,
+ SeedDataset,
+ SyntheticDataset,
+)
+
+__all__ = [
+ "DataPoint",
+ "BaseDataset",
+ "SeedDataset",
+ "GenerativeDataset",
+ "SyntheticDataset",
+]
diff --git a/camel/datasets/base.py b/camel/datasets/base.py
new file mode 100644
index 000000000..e085eeb46
--- /dev/null
+++ b/camel/datasets/base.py
@@ -0,0 +1,969 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import os
+import random
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ List,
+ Optional,
+ TypeVar,
+ Union,
+)
+
+import torch
+from datasets import Dataset as HFDataset
+from pydantic import BaseModel, Field, ValidationError
+from torch.utils.data import DataLoader, Dataset
+
+from camel.agents import ChatAgent
+from camel.logger import get_logger
+from camel.verifiers import BaseVerifier
+
+logger = get_logger(__name__)
+
+
+class DataPoint(BaseModel):
+ r"""A single data point in the dataset.
+
+ Attributes:
+ question (str): The primary question or issue to be addressed.
+ rationale (str): Logical reasoning or explanation behind the
+ answer.
+ final_answer (str): The final answer.
+ raw_markdown (Optional[str]): Raw markdown content for generating
+ rewards/hints. (default: :obj:`None`)
+ difficulty (Optional[str]): Difficulty level of the question.
+ (default: :obj:`None`)
+ metadata Optional[Dict[str, Any]]: Additional metadata about the data
+ point. (default: :obj:`None`)
+ """
+
+ question: str = Field(
+ ..., description="The primary question or issue to be addressed."
+ )
+ rationale: str = Field(
+ ..., description="Logical reasoning or explanation behind the answer."
+ )
+ final_answer: str = Field(..., description="The final answer.")
+ difficulty: Optional[str] = Field(
+ None, description="Difficulty level of the question."
+ )
+ metadata: Optional[Dict[str, Any]] = Field(
+ default=None, description="Additional metadata about the data point."
+ )
+
+ def to_dict(self) -> Dict[str, Any]:
+ r"""Convert DataPoint to a dictionary.
+
+ Returns:
+ Dict[str, Any]: Dictionary representation of the DataPoint.
+ """
+ return self.dict()
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> 'DataPoint':
+ r"""Create a DataPoint from a dictionary.
+
+ Args:
+ data (Dict[str, Any]): Dictionary containing DataPoint fields.
+
+ Returns:
+ DataPoint: New DataPoint instance.
+ """
+ return cls(**data)
+
+
+class BaseDataset(Dataset):
+ r"""A dataset contains questions and ground truth data for training.
+ It can be either static (e.g., MATH dataset) or generative
+ (using an LLM to generate questions).
+ """
+
+ def __init__(
+ self,
+ data: List[Dict[str, str]],
+ cache_dir: Optional[str] = None,
+ **kwargs,
+ ):
+ r"""Initialize the dataset.
+
+ Args:
+ data (List[Dict[str, str]]): List of dictionary items to
+ create the dataset from.
+ cache_dir (Optional[str]): Directory to cache dataset files.
+ (default: :obj:`None`)
+ **kwargs: Additional dataset parameters.
+
+ Note:
+ The dataset must be initialized by calling setup() before use.
+ """
+ self._is_setup = False
+ self._raw_data: List[Dict[str, str]] = data if data is not None else []
+ self._cache_dir = str(cache_dir) if cache_dir is not None else None
+
+ # Store all parameters in metadata dict for compatibility
+ self._metadata = {
+ 'cache_dir': self._cache_dir,
+ **kwargs,
+ }
+
+ self.data: List[DataPoint] = [] # Will be populated in setup()
+
+ async def setup(self) -> None:
+ r"""Set up the dataset with necessary resources.
+
+ This method:
+ 1. Creates cache directory if needed
+ 2. Processes raw data into DataPoint objects using vectorized
+ operations
+ 3. Validates dataset integrity
+
+ Raises:
+ OSError: If cache directory creation fails.
+ Exception: If dataset initialization fails.
+ """
+ if self._is_setup:
+ logger.debug(f"{self.__class__.__name__} already initialized")
+ return
+
+ try:
+ # Create cache directory if specified
+ if self._cache_dir:
+ try:
+ os.makedirs(self._cache_dir, exist_ok=True)
+ logger.debug(f"Created cache directory: {self._cache_dir}")
+ except OSError as e:
+ logger.error(
+ f"Failed to create cache directory "
+ f"{self._cache_dir}: {e}"
+ )
+ raise
+
+ # Process raw data into DataPoint objects using vectorized
+ # operations
+ if not self._raw_data:
+ self.data = []
+ logger.debug("No raw data to process")
+ else:
+ try:
+ # Helper function for validation that can be used with map
+ def create_datapoint(item, idx=None):
+ try:
+ return DataPoint(
+ question=item.get('question', ''),
+ rationale=item.get('rationale', ''),
+ final_answer=item.get('final_answer', ''),
+ metadata=item.get('metadata', {})
+ if isinstance(item.get('metadata'), dict)
+ else {},
+ raw_markdown='',
+ difficulty='',
+ )
+ except ValidationError as e:
+ idx_str = (
+ f" at index {idx}" if idx is not None else ""
+ )
+ error_msg = (
+ f"Sample{idx_str} validation error: {e}"
+ )
+ logger.error(error_msg)
+ raise ValueError(error_msg)
+
+ # If raw_data is already a HF dataset, use its map function
+ if hasattr(self._raw_data, 'map') and callable(
+ self._raw_data.map
+ ):
+ # Using HF dataset's map for vectorized processing
+ processed_data = self._raw_data.map(
+ lambda example, idx: {
+ 'datapoint': create_datapoint(example, idx)
+ },
+ with_indices=True,
+ )
+ self.data = [
+ item['datapoint'] for item in processed_data
+ ]
+ else:
+ # Bulk create datapoints
+ self.data = [
+ create_datapoint(item, i)
+ for i, item in enumerate(self._raw_data)
+ ]
+
+ logger.debug(f"Processed {len(self.data)} data points")
+ except Exception as e:
+ logger.error(f"Error processing data: {e}")
+ raise
+
+ self._is_setup = True
+ logger.info(f"{self.__class__.__name__} initialized successfully")
+
+ except Exception as e:
+ logger.error(f"Error during {self.__class__.__name__} setup: {e}")
+ await self.cleanup()
+ raise
+
+ async def cleanup(self) -> None:
+ r"""Clean up dataset resources.
+
+ This method handles cleanup of resources and resets the dataset state.
+ """
+ if not self._is_setup:
+ return
+
+ try:
+ # Clear metadata while preserving init config
+ init_config = {
+ 'cache_dir': self._cache_dir,
+ }
+ self._metadata = init_config
+
+ logger.info(f"{self.__class__.__name__} cleaned up successfully")
+
+ except Exception as e:
+ logger.error(
+ f"Error during {self.__class__.__name__} cleanup: {e}"
+ )
+ raise
+
+ finally:
+ # Always mark as uninitialized, even if cleanup fails
+ self._is_setup = False
+
+ def sample(self) -> DataPoint:
+ r"""Sample a random datapoint from the dataset.
+
+ Returns:
+ DataPoint: A randomly sampled DataPoint.
+
+ Raises:
+ RuntimeError: If dataset is not initialized.
+ """
+ if not self._is_setup:
+ raise RuntimeError(
+ f"{self.__class__.__name__} must be initialized "
+ "before sampling"
+ )
+
+ idx = random.randint(0, len(self) - 1)
+ return self[idx]
+
+ def __len__(self) -> int:
+ r"""Return the size of the dataset."""
+ return len(self.data)
+
+ def __getitem__(self, idx: int) -> DataPoint:
+ r"""Get an item from the dataset.
+
+ Args:
+ idx (int): Index of the item to get.
+
+ Returns:
+ DataPoint: DataPoint from the dataset with the given index.
+
+ Raises:
+ IndexError: If idx is out of bounds.
+ """
+ if idx < 0 or idx >= len(self):
+ raise IndexError(
+ f"Index {idx} out of bounds for dataset of size {len(self)}"
+ )
+
+ return self.data[idx]
+
+ @property
+ def metadata(self) -> Dict[str, Any]:
+ r"""Get dataset metadata."""
+ return self._metadata.copy()
+
+ def to_pytorch_dataset(
+ self,
+ transform: Optional[Callable] = None,
+ target_transform: Optional[Callable] = None,
+ batch_size: Optional[int] = None,
+ ) -> Union["PyTorchDataset", "DataLoader"]:
+ r"""Convert to a PyTorch dataset or DataLoader.
+
+ Args:
+ transform (Optional[Callable]): Transform to apply to samples.
+ target_transform (Optional[Callable]): Transform to apply to
+ targets.
+ batch_size (Optional[int]): If provided, returns a DataLoader with
+ the specified batch size instead of a PyTorchDataset.
+ (default: :obj:`None`)
+
+ Returns:
+ Union[PyTorchDataset, torch.utils.data.DataLoader]: Dataset in
+ PyTorch format or DataLoader if batch_size is provided.
+ """
+ dataset = PyTorchDataset.from_datapoints(
+ self.data,
+ transform=transform,
+ target_transform=target_transform,
+ )
+
+ if batch_size is not None:
+ return dataset.get_dataloader(batch_size=batch_size)
+
+ return dataset
+
+
+class SeedDataset(BaseDataset):
+ r"""A dataset containing validated seed examples for data generation.
+ Ensures that all items adhere to the DataPoint schema.
+
+ This class is used to initialize a dataset from a list of dictionary items,
+ validating each against the DataPoint schema.
+ """
+
+ def __init__(
+ self,
+ data: List[Dict[str, str]],
+ cache_dir: Optional[str] = None,
+ min_samples: int = 1,
+ **kwargs,
+ ):
+ r"""Initialize the seed dataset.
+
+ Args:
+ data (List[Dict[str, str]]): List of dictionary items to create the
+ dataset from.
+ cache_dir (Optional[str]): Directory to cache dataset files.
+ (default: :obj:`None`)
+ min_samples (int): Minimum number of samples required.
+ (default: :obj:`1`)
+ **kwargs: Additional dataset parameters.
+
+ Raises:
+ ValueError: If dataset size is less than min_samples or if sample
+ validation fails.
+ """
+ if len(data) < min_samples:
+ raise ValueError(
+ f"Seed dataset must contain at least {min_samples} samples."
+ )
+
+ super().__init__(
+ data=data,
+ cache_dir=cache_dir,
+ **kwargs,
+ )
+
+
+class SyntheticDataset(BaseDataset):
+ r"""A dataset for storing synthetically generated data points.
+
+ This class is used to store datapoints that are generated through
+ a generative process, such as using an agent.
+ """
+
+ def __init__(
+ self,
+ data: Optional[List[Dict[str, str]]] = None,
+ cache_dir: Optional[str] = None,
+ **kwargs,
+ ):
+ r"""Initialize the synthetic dataset.
+
+ Args:
+ data (Optional[List[Dict[str, str]]]): List of dictionary items to
+ create the dataset from. (default: :obj:`None`)
+ cache_dir (Optional[str]): Directory to cache dataset files.
+ (default: :obj:`None`)
+ **kwargs: Additional dataset parameters.
+ """
+ super().__init__(
+ data=data if data is not None else [],
+ cache_dir=cache_dir,
+ **kwargs,
+ )
+ self.data: List[DataPoint] = []
+
+ def add(self, item: DataPoint) -> None:
+ r"""Add a new data point to the dataset.
+
+ Args:
+ item (DataPoint): The datapoint to add to the dataset.
+ """
+ self.data.append(item)
+
+ def add_batch(self, items: List[DataPoint]) -> None:
+ r"""Add multiple data points to the dataset.
+
+ Args:
+ items (List[DataPoint]): The datapoints to add to the dataset.
+ """
+ self.data.extend(items)
+
+ def to_pytorch_dataset(
+ self,
+ transform: Optional[Callable] = None,
+ target_transform: Optional[Callable] = None,
+ batch_size: Optional[int] = None,
+ ) -> Union["PyTorchDataset", "DataLoader"]:
+ r"""Convert to a PyTorch dataset or DataLoader.
+
+ Args:
+ transform (Optional[Callable]): Transform to apply to samples.
+ target_transform (Optional[Callable]): Transform to apply to
+ targets.
+ batch_size (Optional[int]): If provided, returns a DataLoader with
+ the specified batch size instead of a PyTorchDataset.
+ (default: :obj:`None`)
+
+ Returns:
+ Union[PyTorchDataset, torch.utils.data.DataLoader]: Dataset in
+ PyTorch format or DataLoader if batch_size is provided.
+ """
+ return convert_synthetic_to_pytorch(
+ self,
+ transform=transform,
+ target_transform=target_transform,
+ batch_size=batch_size,
+ )
+
+ def save_pytorch_format(self, path: str, compression: bool = True) -> None:
+ r"""Save the dataset to disk in PyTorch format.
+
+ Args:
+ path (str): Path to save the dataset to.
+ compression (bool): Whether to use compression to reduce file size.
+ (default: :obj:`True`)
+ """
+ save_synthetic_dataset(self, path, compression=compression)
+
+ def filter(
+ self, predicate: Callable[[DataPoint], bool]
+ ) -> 'SyntheticDataset':
+ r"""Filter the dataset using a predicate function.
+
+ Args:
+ predicate (Callable[[DataPoint], bool]): Function that takes a
+ DataPoint and returns True if it should be kept, False
+ otherwise.
+
+ Returns:
+ SyntheticDataset: A new dataset containing only the filtered items.
+ """
+ filtered_data = [dp for dp in self.data if predicate(dp)]
+
+ # Create a new dataset with the filtered data
+ new_dataset = SyntheticDataset()
+ new_dataset.add_batch(filtered_data)
+
+ return new_dataset
+
+
+class GenerativeDataset(BaseDataset):
+ r"""A dataset for generating synthetic datapoints using external agents and
+ verifiers.
+
+ This class leverages a seed dataset and external components to generate
+ new synthetic datapoints on demand.
+ """
+
+ def __init__(
+ self,
+ seed_dataset: SeedDataset,
+ verifier: BaseVerifier,
+ agent: ChatAgent,
+ cache_dir: Optional[str] = None,
+ seed: int = 42,
+ **kwargs,
+ ):
+ r"""Initialize the generative dataset.
+
+ Args:
+ seed_dataset (SeedDataset): Validated dataset to use for examples.
+ verifier (BaseVerifier): Verifier to validate generated content.
+ agent (ChatAgent): Agent to generate new datapoints.
+ cache_dir (Optional[str]): Directory to cache dataset files.
+ (default: :obj:`None`)
+ seed (int): Random seed for reproducibility. (default: :obj:`42`)
+ **kwargs: Additional dataset parameters.
+ """
+ # Initialize with empty data since we'll generate content dynamically
+ super().__init__(
+ data=[],
+ cache_dir=cache_dir,
+ **kwargs,
+ )
+
+ self.seed_dataset = seed_dataset
+ self.verifier = verifier
+ self.agent = agent
+
+ self.seed = seed
+ random.seed(self.seed)
+
+ def _construct_prompt(self, examples: List[DataPoint]) -> str:
+ r"""Construct a prompt for generating new datapoints.
+
+ Args:
+ examples (List[DataPoint]): Examples to include in the prompt.
+
+ Returns:
+ str: Formatted prompt with examples.
+ """
+ prompt = (
+ "Generate a new datapoint similar to the following examples:\n\n"
+ )
+ for i, example in enumerate(examples, 1):
+ prompt += f"Example {i}:\n"
+ prompt += f"Question: {example.question}\n"
+ prompt += f"Rationale: {example.rationale}\n"
+ prompt += f"Final Answer: {example.final_answer}\n\n"
+ prompt += "New datapoint:"
+ return prompt
+
+ async def generate_new(self, n: int) -> None:
+ r"""Generate n new datapoints and add them to the dataset.
+
+ Args:
+ n (int): Number of valid datapoints to generate.
+
+ This method generates new datapoints by:
+ 1. Sampling examples from the seed dataset
+ 2. Constructing a prompt for the agent
+ 3. Generating a new datapoint using the agent
+ 4. Verifying the generated datapoint with the verifier
+ 5. Adding valid datapoints to the dataset
+ """
+ valid_data_points: List[DataPoint] = []
+
+ while len(valid_data_points) < n:
+ try:
+ indices = random.sample(range(len(self.seed_dataset)), 3)
+ examples = [self.seed_dataset[i] for i in indices]
+ prompt = self._construct_prompt(examples)
+
+ # Get agent response
+ agent_output = (
+ self.agent.step(prompt, response_format=DataPoint)
+ .msgs[0]
+ .parsed
+ )
+
+ if not isinstance(agent_output, dict):
+ raise TypeError("Agent output must be a dictionary")
+ if (
+ 'question' not in agent_output
+ or 'rationale' not in agent_output
+ ):
+ raise KeyError(
+ "Agent output missing required keys: "
+ "'question' or 'rationale'"
+ )
+
+ rationale = agent_output['rationale']
+
+ # Verify the generated content
+ verifier_response = await self.verifier.verify(rationale)
+ if not hasattr(verifier_response, 'content'):
+ raise AttributeError(
+ "Verifier response missing 'content' attribute"
+ )
+
+ if not verifier_response.result:
+ continue
+
+ final_answer = verifier_response.result
+
+ # Create and validate the new datapoint
+ new_datapoint = {
+ 'question': agent_output['question'],
+ 'rationale': rationale,
+ 'final_answer': final_answer,
+ }
+
+ datapoint = DataPoint(**new_datapoint)
+ valid_data_points.append(datapoint)
+
+ except (TypeError, KeyError, AttributeError, ValidationError) as e:
+ logger.warning(
+ f"Error encountered during generation: {e}, retrying..."
+ )
+
+ # Add all valid datapoints to the dataset
+ for datapoint in valid_data_points:
+ self.data.append(datapoint)
+ logger.debug("Added new datapoint to dataset")
+
+
+# Define a type variable for return type flexibility
+T_co = TypeVar('T_co', covariant=True)
+
+
+class PyTorchDataset(Dataset[T_co]):
+ r"""A PyTorch-compatible dataset implementation that leverages PyTorch's
+ efficient data handling capabilities.
+ """
+
+ def __init__(
+ self,
+ data: List[Dict[str, Any]],
+ transform: Optional[Callable] = None,
+ target_transform: Optional[Callable] = None,
+ validate: bool = True,
+ ):
+ r"""Initialize the PyTorch dataset.
+
+ Args:
+ data (List[Dict[str, Any]]): List of dictionary items to create
+ the dataset from.
+ transform (Optional[Callable]): A function/transform that takes a
+ sample and returns a transformed version for features.
+ (default: :obj:`None`)
+ target_transform (Optional[Callable]): A function/transform that
+ takes a target and returns a transformed version. (default:
+ :obj:`None`)
+ validate (bool): Whether to validate data points using DataPoint
+ schema. (default: :obj:`True`)
+
+ Raises:
+ ValidationError: If validation is enabled and data doesn't match
+ DataPoint schema.
+ """
+ self.transform = transform
+ self.target_transform = target_transform
+
+ # Validate and store data
+ self._raw_data = data
+ self.data = []
+
+ if validate:
+ for i, item in enumerate(self._raw_data):
+ try:
+ # Use DataPoint for validation only
+ dp = DataPoint(**item)
+ self.data.append(dp.to_dict())
+ except ValidationError as e:
+ logger.error(f"Sample {i} validation error: {e}")
+ raise ValueError(f"Sample {i} validation error: {e}")
+ else:
+ # Skip validation and just store the data dictionaries
+ self.data = [dict(item) for item in self._raw_data]
+
+ def __getitem__(self, index: int) -> T_co:
+ r"""Get an item from the dataset.
+
+ Args:
+ index (int): Index of the item to get.
+
+ Returns:
+ T_co: Item from the dataset, possibly transformed.
+ """
+ sample = self.data[index]
+
+ # Apply transformations if provided
+ if self.transform is not None:
+ sample = self.transform(sample)
+
+ return sample # type: ignore[return-value]
+
+ def __len__(self) -> int:
+ r"""Return the size of the dataset.
+
+ Returns:
+ int: Number of samples in the dataset.
+ """
+ return len(self.data)
+
+ @classmethod
+ def from_datapoints(
+ cls, datapoints: List[DataPoint], **kwargs
+ ) -> 'PyTorchDataset':
+ r"""Create a PyTorchDataset from a list of DataPoints.
+
+ Args:
+ datapoints (List[DataPoint]): List of DataPoint objects.
+ **kwargs: Additional arguments to pass to the constructor.
+
+ Returns:
+ PyTorchDataset: A new PyTorchDataset instance.
+ """
+ data = [dp.to_dict() for dp in datapoints]
+ # We can skip validation since datapoints are already validated
+ return cls(data, validate=False, **kwargs)
+
+ def to_hf_dataset(self) -> HFDataset:
+ r"""Convert to a HuggingFace dataset.
+
+ Returns:
+ HFDataset: Dataset in HuggingFace format.
+ """
+ return HFDataset.from_list(self.data)
+
+ def save_to_disk(self, path: str) -> None:
+ r"""Save the dataset to disk using PyTorch.
+
+ Args:
+ path (str): Path to save the dataset to.
+ """
+ torch.save(self.data, path)
+
+ @classmethod
+ def load_from_disk(
+ cls,
+ path: str,
+ transform: Optional[Callable] = None,
+ target_transform: Optional[Callable] = None,
+ ) -> 'PyTorchDataset':
+ r"""Load a dataset from disk.
+
+ Args:
+ path (str): Path to load the dataset from.
+ transform (Optional[Callable]): Transform to apply to samples.
+ (default: :obj:`None`)
+ target_transform (Optional[Callable]): Transform to apply to
+ targets. (default: :obj:`None`)
+
+ Returns:
+ PyTorchDataset: Loaded dataset.
+ """
+ data = torch.load(path)
+ return cls(
+ data,
+ transform=transform,
+ target_transform=target_transform,
+ validate=False,
+ )
+
+ @staticmethod
+ def collate_fn(
+ batch: List[Dict[str, Any]],
+ ) -> Dict[str, Union[List[Any], torch.Tensor]]:
+ r"""Collate function for PyTorch DataLoader.
+
+ Args:
+ batch (List[Dict[str, Any]]): Batch of samples from the dataset.
+
+ Returns:
+ Dict[str, Union[List[Any], torch.Tensor]]: Collated batch with
+ tensors for numerical data.
+ """
+ if not batch:
+ return {}
+
+ # Initialize result dictionary with keys from first item - start with
+ # lists only
+ result: Dict[str, List[Any]] = {k: [] for k in batch[0].keys()}
+
+ # Collect values by key
+ for item in batch:
+ for k, v in item.items():
+ result[k].append(v)
+
+ # Convert numeric/boolean lists to tensors where possible
+ result_with_tensors: Dict[str, Union[List[Any], torch.Tensor]] = {}
+ for k, v in result.items():
+ if all(isinstance(x, (int, float, bool)) for x in v):
+ try:
+ result_with_tensors[k] = torch.tensor(v)
+ except (ValueError, TypeError):
+ # Keep as list if tensor conversion fails
+ result_with_tensors[k] = v
+ else:
+ result_with_tensors[k] = v
+
+ return result_with_tensors
+
+ def get_dataloader(
+ self,
+ batch_size: int = 32,
+ shuffle: bool = True,
+ num_workers: int = 0,
+ pin_memory: bool = False,
+ **kwargs,
+ ) -> "DataLoader":
+ r"""Create a PyTorch DataLoader for this dataset.
+
+ Args:
+ batch_size (int): Batch size. (default: :obj:`32`)
+ shuffle (bool): Whether to shuffle the dataset. (default:
+ :obj:`True`)
+ num_workers (int): Number of workers for data loading. (default:
+ :obj:`0`)
+ pin_memory (bool): Whether to pin memory for faster GPU transfer.
+ (default: :obj:`False`)
+ **kwargs: Additional arguments to pass to DataLoader.
+
+ Returns:
+ torch.utils.data.DataLoader: DataLoader for this dataset.
+ """
+ from torch.utils.data import DataLoader
+
+ return DataLoader(
+ self,
+ batch_size=batch_size,
+ shuffle=shuffle,
+ num_workers=num_workers,
+ collate_fn=self.collate_fn,
+ pin_memory=pin_memory,
+ **kwargs,
+ )
+
+
+def convert_hf_to_pytorch(
+ hf_dataset: HFDataset,
+ transform: Optional[Callable] = None,
+ target_transform: Optional[Callable] = None,
+ column_mapping: Optional[Dict[str, str]] = None,
+ validate: bool = True,
+ batch_size: Optional[int] = None,
+) -> Union["PyTorchDataset", "DataLoader"]:
+ r"""Convert a HuggingFace dataset to a PyTorchDataset or DataLoader.
+
+ This function maps HuggingFace dataset columns to the expected DataPoint
+ format, validates the data, and creates a PyTorchDataset or DataLoader.
+
+ Args:
+ hf_dataset (HFDataset): HuggingFace dataset to convert.
+ transform (Optional[Callable]): Transform to apply to samples.
+ target_transform (Optional[Callable]): Transform to apply to targets.
+ column_mapping (Optional[Dict[str, str]]): Mapping from HuggingFace
+ column names to DataPoint field names. If None, assumes columns
+ already match DataPoint fields.
+ validate (bool): Whether to validate data points using DataPoint
+ schema. (default: :obj:`True`)
+ batch_size (Optional[int]): If provided, returns a DataLoader with the
+ specified batch size instead of a PyTorchDataset. (default:
+ :obj:`None`)
+
+ Returns:
+ Union[PyTorchDataset, torch.utils.data.DataLoader]: Converted dataset
+ or DataLoader if batch_size is provided.
+ """
+ # Convert HuggingFace dataset to list of dicts more efficiently
+ mapped_dataset = []
+
+ for i in range(len(hf_dataset)):
+ item = hf_dataset[i]
+ if column_mapping is not None:
+ # Apply column mapping if provided
+ mapped_item = {}
+ for hf_col, dp_field in column_mapping.items():
+ if hf_col in item:
+ mapped_item[dp_field] = item[hf_col]
+ mapped_dataset.append(mapped_item)
+ else:
+ # Otherwise use item directly
+ mapped_dataset.append(dict(item))
+
+ # Create PyTorchDataset
+ dataset: PyTorchDataset = PyTorchDataset(
+ mapped_dataset,
+ transform=transform,
+ target_transform=target_transform,
+ validate=validate,
+ )
+
+ # Return DataLoader if batch_size is provided
+ if batch_size is not None:
+ return dataset.get_dataloader(batch_size=batch_size)
+
+ return dataset
+
+
+def convert_synthetic_to_pytorch(
+ synthetic_dataset: 'SyntheticDataset',
+ transform: Optional[Callable] = None,
+ target_transform: Optional[Callable] = None,
+ batch_size: Optional[int] = None,
+) -> Union["PyTorchDataset", "DataLoader"]:
+ r"""Convert a SyntheticDataset to a PyTorchDataset or DataLoader.
+
+ Args:
+ synthetic_dataset (SyntheticDataset): Synthetic dataset to convert.
+ transform (Optional[Callable]): Transform to apply to samples.
+ target_transform (Optional[Callable]): Transform to apply to targets.
+ batch_size (Optional[int]): If provided, returns a DataLoader with the
+ specified batch size instead of a PyTorchDataset. (default:
+ :obj:`None`)
+
+ Returns:
+ Union[PyTorchDataset, torch.utils.data.DataLoader]: Converted dataset
+ or DataLoader if batch_size is provided.
+ """
+ dataset = PyTorchDataset.from_datapoints(
+ synthetic_dataset.data,
+ transform=transform,
+ target_transform=target_transform,
+ )
+
+ # Return DataLoader if batch_size is provided
+ if batch_size is not None:
+ return dataset.get_dataloader(batch_size=batch_size)
+
+ return dataset
+
+
+def save_synthetic_dataset(
+ synthetic_dataset: 'SyntheticDataset',
+ path: str,
+ compression: bool = True,
+) -> None:
+ r"""Save a synthetic dataset to disk using PyTorch format.
+
+ Args:
+ synthetic_dataset (SyntheticDataset): Dataset to save.
+ path (str): Path to save the dataset to.
+ compression (bool): Whether to use compression to reduce file size.
+ (default: :obj:`True`)
+ """
+ pytorch_dataset = convert_synthetic_to_pytorch(synthetic_dataset)
+
+ # Save with compression if enabled (uses less disk space)
+ if compression:
+ torch.save(
+ pytorch_dataset.data, # type: ignore[union-attr]
+ path,
+ _use_new_zipfile_serialization=True,
+ )
+ else:
+ pytorch_dataset.save_to_disk(path) # type: ignore[union-attr]
+
+
+def load_pytorch_dataset(
+ path: str,
+ transform: Optional[Callable] = None,
+ target_transform: Optional[Callable] = None,
+ batch_size: Optional[int] = None,
+) -> Union["PyTorchDataset", "DataLoader"]:
+ r"""Load a PyTorchDataset from disk.
+
+ Args:
+ path (str): Path to load the dataset from.
+ transform (Optional[Callable]): Transform to apply to samples.
+ target_transform (Optional[Callable]): Transform to apply to targets.
+ batch_size (Optional[int]): If provided, returns a DataLoader with the
+ specified batch size instead of a PyTorchDataset. (default:
+ :obj:`None`)
+
+ Returns:
+ Union[PyTorchDataset, torch.utils.data.DataLoader]: Loaded dataset or
+ DataLoader if batch_size is provided.
+ """
+ dataset = PyTorchDataset.load_from_disk(
+ path, transform=transform, target_transform=target_transform
+ )
+
+ # Return DataLoader if batch_size is provided
+ if batch_size is not None:
+ return dataset.get_dataloader(batch_size=batch_size)
+
+ return dataset
diff --git a/camel/environments/__init__.py b/camel/environments/__init__.py
new file mode 100644
index 000000000..b0bbfb9a3
--- /dev/null
+++ b/camel/environments/__init__.py
@@ -0,0 +1,16 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from .base import BaseEnvironment
+
+__all__ = ["BaseEnvironment"]
diff --git a/camel/environments/base.py b/camel/environments/base.py
new file mode 100644
index 000000000..9b187d88a
--- /dev/null
+++ b/camel/environments/base.py
@@ -0,0 +1,503 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import asyncio
+from abc import ABC, abstractmethod
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional, Tuple
+
+from pydantic import BaseModel, Field
+
+from camel.agents import ChatAgent
+from camel.datasets.base import BaseDataset, GenerativeDataset
+from camel.extractors.base import BaseExtractor
+from camel.logger import get_logger
+from camel.verifiers.base import (
+ BaseVerifier,
+ VerificationResult,
+)
+from camel.verifiers.models import (
+ VerificationOutcome,
+ VerifierInput,
+)
+
+logger = get_logger(__name__)
+
+# TODO: Add MachineInfo into this file
+# TODO: TeacherAgent should be renamed into neural_reward_model.
+# This is where PRMs or such could be useful.
+# Should probably be its own class and not just raw ChatAgent
+
+
+class Action(BaseModel):
+ r"""Represents an action taken in an environment.
+
+ This class defines the input context, the LLM-generated output, and
+ metadata required for verification and tracking within an RL
+ framework.
+
+ Attributes:
+ problem_statement (str): The task or query given to the LLM as
+ input.
+ llm_response (str): The response generated by the LLM.
+ final_answer (Optional[str]): The reference solution, if
+ available, used for supervised learning or evaluation.
+ metadata (Dict[str, Any]): Additional metadata such as model
+ parameters, prompt details, or response confidence scores.
+ timestamp (datetime): The timestamp when the action was
+ generated (UTC).
+ """
+
+ problem_statement: str = Field(description="Problem statement for the LLM")
+ llm_response: str = Field(description="Generated response from the LLM")
+ final_answer: Optional[str] = Field(
+ None, description="Reference solution if available"
+ )
+ metadata: Dict[str, Any] = Field(
+ default_factory=dict,
+ description="Additional metadata about the generation",
+ )
+ timestamp: datetime = Field(
+ default_factory=lambda: datetime.now(timezone.utc),
+ description="When the response was generated (UTC)",
+ )
+
+
+class Observation(BaseModel):
+ r"""Environment observation.
+
+ Attributes:
+ question: The question posed to the LLM.
+ context: Additional context for the question.
+ metadata: Optional metadata about the observation.
+ """
+
+ question: str = Field(..., description="The question posed to the LLM")
+ context: Dict[str, Any] = Field(
+ default_factory=dict, description="Additional context for the question"
+ )
+ metadata: Optional[Dict[str, Any]] = Field(
+ default=None, description="Optional metadata about the observation"
+ )
+
+
+class StepResult(BaseModel):
+ r"""Result of an environment step.
+
+ Attributes:
+ observation: The next observation.
+ reward: Dictionary of reward scores for different aspects.
+ done: Whether the episode is complete.
+ info: Additional information about the step.
+ """
+
+ observation: Observation = Field(..., description="The next observation")
+ reward: float = Field(..., description="Total reward of the action")
+ rewards_dict: Dict[str, float] = Field(
+ default_factory=dict,
+ description="Dictionary of reward scores for different aspects",
+ )
+ done: bool = Field(..., description="Whether the episode is complete")
+ info: Dict[str, Any] = Field(
+ default_factory=dict,
+ description="Additional information about the step",
+ )
+
+
+class BaseEnvironment(ABC):
+ r"""Base class for all RLVR training environments.
+
+ An environment ties everything together. It:
+ 1. Holds state and manages curriculum progression
+ 2. Defines reward functions and hint generation
+ 3. Manages dataset and task selection
+ 4. Provides reset and step functions
+ 5. Handles verifier setup and teardown
+ 6. Enables proactive agent behavior
+ 7. Supports practice environment creation
+ 8. Facilitates chain-of-thought verification
+
+ Key Features:
+ - Curriculum learning with adaptive difficulty
+ - Reward shaping based on solution quality
+ - Hint generation from verified solutions
+ - Task selection based on agent progress
+ - Practice environment generation
+ - Chain-of-thought validation
+ """
+
+ def __init__(
+ self,
+ dataset: BaseDataset,
+ verifier: BaseVerifier,
+ extractor: BaseExtractor,
+ max_steps: Optional[int] = None,
+ teacher_agent: Optional[ChatAgent] = None,
+ curriculum_config: Optional[Dict[str, Any]] = None,
+ practice_env_config: Optional[Dict[str, Any]] = None,
+ **kwargs,
+ ) -> None:
+ r"""Initialize the environment.
+
+ Args:
+ dataset: Dataset to sample questions from.
+ verifier: Verifier to check responses.
+ extractor: Extractor to process LLM responses.
+ max_steps: Maximum steps per episode.
+ teacher_agent: Optional agent for reward shaping and hints
+ curriculum_config: Configuration for curriculum learning including:
+ - difficulty_levels: List of available difficulty levels
+ - promotion_threshold: Score needed to advance
+ - demotion_threshold: Score triggering level decrease
+ - min_questions_per_level: Questions before promotion
+ practice_env_config: Configuration for practice environments:
+ - max_practice_envs: Maximum concurrent environments
+ - difficulty_range: Allowed difficulty variation
+ - focus_areas: Specific skills to practice
+ **kwargs: Additional environment parameters.
+ """
+ self.dataset = dataset
+ self.verifier = verifier
+ self.extractor = extractor
+ self.max_steps = max_steps
+ self.teacher_agent = teacher_agent
+ self._metadata = kwargs
+
+ # State tracking
+ self._is_setup: bool = False
+ self._current_step: int = 0
+ self._episode_ended: bool = False
+ self._state: Dict[str, Any] = self._get_initial_state()
+ self._last_observation: Optional[Observation] = None
+ self._episode_history: List[Tuple[Observation, Action]] = []
+
+ @abstractmethod
+ async def setup(self) -> None:
+ r"""Set up the environment, including verifier initialization."""
+ if self._is_setup:
+ return
+
+ try:
+ # Initialize core components
+ if hasattr(self.verifier, 'setup'):
+ await self.verifier.setup()
+ if hasattr(self.dataset, 'setup'):
+ await self.dataset.setup()
+ if hasattr(self.extractor, 'setup'):
+ await self.extractor.setup()
+
+ # initialize agents if present
+ if self.teacher_agent:
+ await self.teacher_agent.reset()
+
+ self._is_setup = True
+ logger.info('Environment setup completed successfully')
+ except Exception as e:
+ logger.error(f'Failed to setup environment: {e}')
+ raise
+
+ @abstractmethod
+ async def teardown(self) -> None:
+ r"""Clean up resources, including verifier teardown."""
+ if not self._is_setup:
+ return
+
+ try:
+ # Cleanup components
+ if hasattr(self.verifier, 'cleanup'):
+ await self.verifier.cleanup()
+ if hasattr(self.dataset, 'cleanup'):
+ await self.dataset.cleanup()
+ if hasattr(self.extractor, 'cleanup'):
+ await self.extractor.cleanup()
+
+ self._is_setup = False
+ logger.info('Environment teardown completed successfully')
+ except Exception as e:
+ logger.error(f'Failed to teardown environment: {e}')
+ raise
+
+ @abstractmethod
+ async def reset(self) -> Observation:
+ r"""Reset the environment to initial state.
+
+ Returns:
+ Initial observation for the episode
+ """
+
+ if not self._is_setup:
+ await self.setup()
+
+ # Reset state
+ self._current_step = 0
+ self._episode_ended = False
+ self._episode_history = []
+ self._state = self._get_initial_state()
+
+ # Get initial observation
+ observation = self._get_next_observation()
+ if observation is None:
+ raise RuntimeError("Failed to get initial observation")
+
+ self._last_observation = observation
+
+ return observation
+
+ @abstractmethod
+ async def step(self, action: Action) -> StepResult:
+ r"""Take a step in the environment.
+
+ Args:
+ action: Action containing everything that is needed
+ to progress in the environment
+
+ Returns:
+ StepResult containing next observation, reward, done flag, and info
+ """
+ if self.max_steps and self._current_step >= self.max_steps:
+ return StepResult(
+ observation=self._get_terminal_observation(),
+ reward=0,
+ rewards_dict={},
+ done=True,
+ info={"reason": "max_steps_reached"},
+ )
+
+ if not self._is_setup:
+ raise RuntimeError("Environment not set up. Call setup() first.")
+ if self._episode_ended:
+ raise RuntimeError("Episode has ended. Call reset() first.")
+ if self._last_observation is None:
+ raise RuntimeError("No current observation. Call reset() first.")
+
+ self._current_step += 1
+
+ current_obs: Observation = self._last_observation
+ self._episode_history.append((current_obs, action))
+
+ # extract verifiable part from llm response
+ extraction_result = await self.extractor.extract(action.llm_response)
+
+ # TODO: extract executable llm response specifically
+
+ # verify the extracted
+ verification_result = await self.verifier.verify(
+ VerifierInput(
+ llm_response=extraction_result,
+ ground_truth=action.final_answer,
+ )
+ )
+
+ # compute rewards
+ total_reward, rewards_dict = await self.compute_reward(
+ action, extraction_result, verification_result
+ )
+
+ # check termination
+ done = self._is_done()
+
+ next_obs = (
+ self._get_terminal_observation()
+ if done
+ else self._get_next_observation()
+ )
+
+ self._last_observation = next_obs
+ self._episode_ended = done
+
+ return StepResult(
+ observation=next_obs,
+ reward=total_reward,
+ rewards_dict=rewards_dict,
+ done=done,
+ info={
+ "extraction_result": extraction_result,
+ "verification_result": verification_result,
+ "step": self._current_step,
+ "state": self._state,
+ },
+ )
+
+ @abstractmethod
+ def _get_initial_state(self) -> Dict[str, Any]:
+ r"""Get initial environment state."""
+
+ return {
+ "current_datapoint": None,
+ "attempts": 0,
+ "success_rate": 0.0,
+ "rewards": [],
+ "termination_reason": None,
+ }
+
+ @abstractmethod
+ def _get_next_observation(self) -> Observation:
+ r"""Get the next observation for the environment.
+
+ Returns:
+ Observation for the next step
+ """
+ if not self.dataset or len(self.dataset) == 0:
+ logger.warning(
+ "Dataset is empty. Attempting to generate new data..."
+ )
+ if isinstance(self.dataset, GenerativeDataset):
+ try:
+ asyncio.run(
+ self.dataset.generate_new(1)
+ ) # Generate at least one datapoint
+ logger.info("Generated new datapoint successfully.")
+ except Exception as e:
+ logger.error(f"Failed to generate new data: {e}")
+ return self._get_terminal_observation()
+ else:
+ logger.error("Dataset is empty and not a GenerativeDataset.")
+ return self._get_terminal_observation()
+
+ try:
+ # Ensure dataset is not empty after generation attempt
+ if len(self.dataset) == 0:
+ logger.error("Dataset is still empty after generation.")
+ return self._get_terminal_observation()
+
+ # Sample the next datapoint
+ datapoint_idx = self._current_step % len(self.dataset)
+ datapoint = self.dataset[datapoint_idx]
+
+ if not datapoint:
+ logger.error(f"Invalid datapoint at index {datapoint_idx}")
+ return self._get_terminal_observation()
+
+ self._state["current_datapoint"] = datapoint
+
+ # Extract necessary attributes safely
+ question = getattr(datapoint, "question", None)
+ final_answer = getattr(datapoint, "final_answer", None)
+ rationale = getattr(datapoint, "rationale", None)
+ difficulty = getattr(datapoint, "difficulty", None)
+ metadata = getattr(datapoint, "metadata", {})
+
+ if not question or not final_answer:
+ logger.error(
+ f"Datapoint at index {datapoint_idx} "
+ "is missing required fields."
+ )
+ return self._get_terminal_observation()
+
+ observation = Observation(
+ question=question,
+ context={
+ "final_answer": final_answer,
+ "difficulty": difficulty,
+ "rationale": rationale,
+ },
+ metadata={
+ "step": self._current_step,
+ "datapoint_id": str(datapoint_idx),
+ "verified": metadata.get("verified", False),
+ **metadata,
+ },
+ )
+
+ logger.debug(
+ f"Generated observation for step {self._current_step}"
+ )
+ return observation
+
+ except (IndexError, AttributeError) as e:
+ logger.error(f"Error getting next observation: {e}")
+ return self._get_terminal_observation()
+ except Exception as e:
+ logger.error(f"Unexpected error getting next observation: {e}")
+ return self._get_terminal_observation()
+
+ @abstractmethod
+ def _get_terminal_observation(self) -> Observation:
+ r"""Get the terminal observation when episode ends.
+
+ Returns:
+ Terminal observation
+ """
+ return Observation(
+ question="Episode completed",
+ context={},
+ metadata={"terminal": True, "final_step": self._current_step},
+ )
+
+ @abstractmethod
+ async def compute_reward(
+ self,
+ action: Action,
+ extraction_result: str,
+ verification_result: VerificationResult,
+ ) -> Tuple[float, Dict[str, float]]:
+ r"""Compute reward scores for different aspects of the response.
+
+ Args:
+ response: The response.
+ extraction_result: Extracted information from response
+ verification_result: Result from the verifier.
+
+ Returns:
+ - Total reward
+ - Dictionary of reward scores for different aspects.
+ """
+ rewards: Dict[str, float] = {}
+
+ # Get success from verification result status
+ verification_success = float(
+ verification_result.status == VerificationOutcome.SUCCESS
+ )
+ rewards["correctness"] = 1.0 if verification_success > 0.5 else 0.0
+
+ # Update state
+ self._state["rewards"].append(rewards)
+ total_attempts = self._state["attempts"] + 1
+ self._state["success_rate"] = (
+ self._state["success_rate"] * (total_attempts - 1)
+ + verification_success
+ ) / total_attempts
+
+ further_rewards = await self._compute_reward(
+ action, extraction_result, verification_result
+ )
+
+ rewards = rewards | further_rewards
+
+ return sum(rewards.values()), rewards
+
+ @abstractmethod
+ async def _compute_reward(
+ self,
+ action: Action,
+ extraction_result: str,
+ verification_result: VerificationResult,
+ ) -> Dict[str, float]:
+ pass
+
+ def _is_done(self) -> bool:
+ r"""Check if episode should terminate."""
+ if self.max_steps and self._current_step >= self.max_steps:
+ return True
+ return False
+
+ @property
+ def metadata(self) -> Dict[str, Any]:
+ r"""Get environment metadata."""
+ return self._metadata.copy()
+
+ @property
+ def current_step(self) -> int:
+ r"""Get current step number."""
+ return self._current_step
diff --git a/camel/extractors/__init__.py b/camel/extractors/__init__.py
new file mode 100644
index 000000000..2f77f0480
--- /dev/null
+++ b/camel/extractors/__init__.py
@@ -0,0 +1,16 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from .base import BaseExtractor
+
+__all__ = ["BaseExtractor"]
diff --git a/camel/extractors/base.py b/camel/extractors/base.py
new file mode 100644
index 000000000..b9e076358
--- /dev/null
+++ b/camel/extractors/base.py
@@ -0,0 +1,263 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from abc import ABC, abstractmethod
+from types import TracebackType
+from typing import Any, Dict, Optional, Type
+
+from typing_extensions import Self
+
+from camel.logger import get_logger
+from camel.utils import BatchProcessor
+
+logger = get_logger(__name__)
+
+
+class BaseExtractor(ABC):
+ r"""Base class for all response extractors.
+
+ An extractor takes the response and extracts the relevant parts,
+ converting them into a format that the verifier can handle.
+ Implements async context manager protocol for proper resource management.
+ """
+
+ def __init__(
+ self,
+ cache_templates: bool = True,
+ max_cache_size: int = 1000,
+ extraction_timeout: float = 30.0,
+ batch_size: int = 10,
+ monitoring_interval: float = 5.0,
+ cpu_threshold: float = 80.0,
+ memory_threshold: float = 85.0,
+ **kwargs,
+ ):
+ r"""Initialize the extractor.
+
+ Args:
+ cache_templates (bool): Whether to cache extraction templates.
+ (default: :obj:`True`)
+ max_cache_size (int): Maximum number of templates to cache.
+ (default: :obj:`1000`)
+ extraction_timeout (float): Maximum time for extraction in seconds.
+ (default: :obj:`30.0`)
+ batch_size (int): Size of batches for parallel extraction.
+ (default: :obj:`10`)
+ monitoring_interval (float): Interval in seconds between resource
+ checks. (default: :obj:`5.0`)
+ cpu_threshold (float): CPU usage percentage threshold for scaling
+ down. (default: :obj:`80.0`)
+ memory_threshold (float): Memory usage percentage threshold for
+ scaling down. (default: :obj:`85.0`)
+ **kwargs: Additional extractor parameters.
+
+ Raises:
+ ValueError: If invalid parameter values are provided
+ """
+ # Store all parameters in metadata dict for compatibility
+ self._metadata = {
+ 'cache_templates': cache_templates,
+ 'max_cache_size': max_cache_size,
+ 'extraction_timeout': extraction_timeout,
+ 'batch_size': batch_size,
+ 'monitoring_interval': monitoring_interval,
+ 'cpu_threshold': cpu_threshold,
+ 'memory_threshold': memory_threshold,
+ **kwargs,
+ }
+
+ self._is_setup = False
+ self._cache: Dict[str, Any] = {}
+ self._batch_processor: Optional[BatchProcessor] = None
+
+ # Store configuration parameters
+ self._cache_templates = cache_templates
+ self._max_cache_size = max_cache_size
+ self._extraction_timeout = extraction_timeout
+ self._batch_size = batch_size
+ self._monitoring_interval = monitoring_interval
+ self._cpu_threshold = cpu_threshold
+ self._memory_threshold = memory_threshold
+
+ async def setup(self) -> None:
+ r"""Set up the extractor with necessary resources.
+
+ This method:
+ 1. Initializes template cache if enabled
+ 2. Sets up any parallel processing resources
+ 3. Validates extraction patterns
+
+ Raises:
+ RuntimeError: If initialization fails
+ """
+ if self._is_setup:
+ logger.debug(f"{self.__class__.__name__} already initialized")
+ return
+
+ try:
+ # Initialize template cache if enabled
+ if self._cache_templates:
+ self._template_cache: Dict[str, Any] = {}
+
+ # Set up batch processing if needed
+ if self._batch_size > 1:
+ self._batch_processor = BatchProcessor(
+ initial_batch_size=self._batch_size,
+ monitoring_interval=self._monitoring_interval,
+ cpu_threshold=self._cpu_threshold,
+ memory_threshold=self._memory_threshold,
+ )
+
+ self._is_setup = True
+ logger.info(f"{self.__class__.__name__} initialized successfully")
+
+ except Exception as e:
+ error_msg = f"Error during {self.__class__.__name__} setup: {e}"
+ logger.error(error_msg)
+ await self.cleanup()
+ raise RuntimeError(error_msg) from e
+
+ async def cleanup(self) -> None:
+ r"""Clean up extractor resources.
+
+ This method handles cleanup of resources and resets the extractor
+ state.
+ It ensures:
+ 1. All resources are properly released
+ 2. Template cache is cleared
+ 3. Parallel processing resources are shutdown
+ 4. State is reset to initial
+ 5. Cleanup happens even if errors occur
+
+ Raises:
+ RuntimeError: If cleanup fails (after resetting initialization
+ state).
+ """
+ if not self._is_setup:
+ logger.debug(
+ f"{self.__class__.__name__} not initialized, skipping cleanup"
+ )
+ return
+
+ errors = []
+ try:
+ # Clear template cache
+ if hasattr(self, '_template_cache'):
+ try:
+ self._template_cache.clear()
+ except Exception as e:
+ errors.append(f"Failed to clear template cache: {e}")
+
+ # Shutdown parallel processing
+ if self._batch_processor is not None:
+ try:
+ # Get final performance metrics before cleanup
+ metrics = self._batch_processor.get_performance_metrics()
+ logger.info(f"Batch processor final metrics: {metrics}")
+ except Exception as e:
+ errors.append(
+ f"Failed to get batch processor metrics: {e}"
+ )
+
+ # Preserve init config in metadata
+ self._metadata = {
+ 'cache_templates': self._cache_templates,
+ 'max_cache_size': self._max_cache_size,
+ 'extraction_timeout': self._extraction_timeout,
+ 'batch_size': self._batch_size,
+ }
+
+ if not errors:
+ logger.info(
+ f"{self.__class__.__name__} cleaned up successfully"
+ )
+
+ except Exception as e:
+ errors.append(f"Unexpected error during cleanup: {e}")
+
+ finally:
+ # Always mark as uninitialized, even if cleanup fails
+ self._is_setup = False
+ self._batch_processor = None
+
+ if errors:
+ error_msg = (
+ f"Errors during {self.__class__.__name__} cleanup: "
+ f"{'; '.join(errors)}"
+ )
+ logger.error(error_msg)
+ raise RuntimeError(error_msg)
+
+ async def __aenter__(self) -> Self:
+ r"""Async context manager entry.
+
+ Returns:
+ Self reference for context manager usage.
+ """
+ await self.setup()
+ return self
+
+ async def __aexit__(
+ self,
+ exc_type: Optional[Type[BaseException]],
+ exc_val: Optional[BaseException],
+ exc_tb: Optional[TracebackType],
+ ) -> None:
+ r"""Async context manager exit.
+
+ Args:
+ exc_type (Optional[Type[BaseException]]): Exception type if an
+ error occurred.
+ exc_val (Optional[BaseException]): Exception value if an error
+ occurred.
+ exc_tb (Optional[TracebackType]): Exception traceback if an error
+ occurred.
+ """
+ await self.cleanup()
+
+ @abstractmethod
+ async def extract(
+ self, response: str, context: Optional[Dict[str, Any]] = None
+ ) -> str:
+ r"""Extract relevant parts from a response.
+
+ Extracts:
+ 1. Final answer or output
+ 2. Chain of thought reasoning steps
+ 3. Difficulty assessment
+
+ Args:
+ response (str): Raw response from agent generation.
+ context (Optional[Dict[str, Any]]): Optional context for
+ extraction like:
+ - final_answer
+ - rationale
+ - complexity
+
+ Returns:
+ str: Extracted content string.
+
+ Raises:
+ ValueError: If response is empty or invalid.
+ NotImplementedError: If no implementation is provided.
+ RuntimeError: If extractor is not initialized.
+ """
+ if not self._is_setup:
+ raise RuntimeError(
+ f"{self.__class__.__name__} must be initialized "
+ "before extraction"
+ )
+ if not response or not response.strip():
+ raise ValueError("Empty or whitespace-only response")
+ raise NotImplementedError("Subclasses must implement extract()")
diff --git a/camel/verifiers/__init__.py b/camel/verifiers/__init__.py
new file mode 100644
index 000000000..19d67579f
--- /dev/null
+++ b/camel/verifiers/__init__.py
@@ -0,0 +1,17 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from .base import BaseVerifier
+from .models import VerificationOutcome, VerifierInput
+
+__all__ = ["BaseVerifier", "VerificationOutcome", "VerifierInput"]
diff --git a/camel/verifiers/base.py b/camel/verifiers/base.py
new file mode 100644
index 000000000..f96c8825a
--- /dev/null
+++ b/camel/verifiers/base.py
@@ -0,0 +1,340 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import asyncio
+import time
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+from camel.logger import get_logger
+from camel.utils import BatchProcessor
+
+from .models import (
+ VerificationOutcome,
+ VerificationResult,
+ VerifierInput,
+)
+
+logger = get_logger(__name__)
+
+
+class BaseVerifier(ABC):
+ r"""Base class for all verifiers.
+
+ Example:
+ ```python
+ verifier = MyVerifier()
+ await verifier.setup()
+ result = await verifier.verify(response)
+ await verifier.cleanup()
+ ```
+
+ Key Features:
+ - Async verification with retry logic
+ - Comprehensive error handling and logging
+ - Configurable batch processing
+ - Resource monitoring for adaptive scaling
+ """
+
+ def __init__(
+ self,
+ max_parallel: Optional[int] = None,
+ timeout: Optional[float] = None,
+ max_retries: int = 3,
+ retry_delay: float = 1.0,
+ initial_batch_size: Optional[int] = None,
+ cpu_threshold: float = 80.0,
+ memory_threshold: float = 85.0,
+ **kwargs,
+ ):
+ r"""Initialize the verifier with configuration parameters.
+
+ Args:
+ max_parallel: Maximum number of parallel verifications. If None,
+ determined dynamically based on system resources.
+ (default: :obj:`None`)
+ timeout: Timeout in seconds for each verification. (default:
+ :obj:`None`)
+ max_retries: Maximum number of retry attempts. (default: :obj:`3`)
+ retry_delay: Delay between retries in seconds. (default:
+ :obj:`1.0`)
+ initial_batch_size: Initial size for batch processing. If None,
+ defaults to 10. (default: :obj:`None`)
+ cpu_threshold: CPU usage percentage threshold for scaling down.
+ (default: :obj:`80.0`)
+ memory_threshold: Memory usage percentage threshold for scaling
+ down. (default: :obj:`85.0`)
+ **kwargs: Additional verifier parameters.
+ """
+ self._is_setup: bool = False
+ self._max_parallel: Optional[int] = max_parallel
+ self._timeout: Optional[float] = timeout
+ self._max_retries: int = max_retries
+ self._retry_delay: float = retry_delay
+ self._initial_batch_size: Optional[int] = initial_batch_size
+ self._cpu_threshold: float = cpu_threshold
+ self._memory_threshold: float = memory_threshold
+ self._batch_processor: BatchProcessor = BatchProcessor()
+
+ async def setup(self) -> None:
+ r"""Set up the verifier with necessary resources.
+
+ Initializes:
+ 1. Batch processor with validated parameters
+ 2. Any verifier-specific resources
+
+ Raises:
+ RuntimeError: If setup fails or resources cannot be initialized.
+ """
+ if self._is_setup:
+ logger.debug(f"{self.__class__.__name__} already initialized")
+ return
+
+ try:
+ batch_size = max(1, self._initial_batch_size or 10)
+ max_parallel = max(1, self._max_parallel or 1)
+ self._batch_processor = BatchProcessor()
+
+ logger.info(
+ f"{self.__class__.__name__} initialized with "
+ f"batch_size={batch_size}, max_parallel={max_parallel}"
+ )
+
+ await self._setup()
+ self._is_setup = True
+
+ except Exception as e:
+ error_msg = (
+ f"Failed to initialize {self.__class__.__name__}: {e!s}"
+ )
+ logger.error(error_msg, exc_info=True)
+ await self.cleanup()
+ raise RuntimeError(error_msg) from e
+
+ @abstractmethod
+ async def _setup(self) -> None:
+ r"""Implement verifier-specific setup logic."""
+ pass
+
+ async def cleanup(self) -> None:
+ r"""Clean up verifier resources.
+
+ Ensures:
+ 1. Batch processor is reset
+ 2. All internal states are cleared
+
+ Raises:
+ RuntimeError: If cleanup fails.
+ """
+ if not self._is_setup:
+ return
+
+ try:
+ self._batch_processor = BatchProcessor()
+ await self._cleanup()
+ logger.info(f"{self.__class__.__name__} cleaned up successfully")
+
+ except Exception as e:
+ error_msg = f"Failed to cleanup {self.__class__.__name__}: {e!s}"
+ logger.error(error_msg, exc_info=True)
+ raise RuntimeError(error_msg) from e
+
+ finally:
+ self._is_setup = False
+
+ @abstractmethod
+ async def _cleanup(self) -> None:
+ r"""Implement verifier-specific cleanup logic."""
+ pass
+
+ async def verify(self, result: VerifierInput) -> VerificationResult:
+ r"""Perform verification with full error handling.
+
+ Verifies correctness, expected output, reasoning, and symbolic
+ consistency.
+
+ Args:
+ result: The response to verify.
+
+ Returns:
+ VerificationResult: Structured result containing:
+ - status: SUCCESS/FAILURE/ERROR/TIMEOUT
+ - result: Verification outcome description
+ - duration: Time taken for verification
+ - metadata: Additional details
+ - error_message: Error description if applicable
+
+ Raises:
+ RuntimeError: If verification fails unexpectedly.
+ asyncio.TimeoutError: If verification times out.
+ """
+ if not self._is_setup:
+ logger.warning(
+ f"{self.__class__.__name__} not set up, calling setup()"
+ )
+ await self.setup()
+
+ attempt = 0
+ start_time = time.time()
+
+ while attempt < self._max_retries:
+ try:
+ verification_result = (
+ await asyncio.wait_for(
+ self._verify_implementation(result),
+ timeout=self._timeout,
+ )
+ if self._timeout
+ else await self._verify_implementation(result)
+ )
+
+ verification_result.duration = time.time() - start_time
+ verification_result.metadata["attempt"] = attempt + 1
+ return verification_result
+
+ except asyncio.TimeoutError:
+ attempt += 1
+ if attempt == self._max_retries:
+ return VerificationResult(
+ status=VerificationOutcome.TIMEOUT,
+ result="",
+ error_message="Verification timed out "
+ "after all retries.",
+ duration=time.time() - start_time,
+ metadata={"attempt": attempt},
+ )
+ logger.warning(
+ f"Verification timeout on attempt {attempt}, retrying..."
+ )
+ await asyncio.sleep(self._retry_delay)
+
+ except Exception as e:
+ attempt += 1
+ if attempt == self._max_retries:
+ return VerificationResult(
+ status=VerificationOutcome.ERROR,
+ result="",
+ error_message=f"Verification failed: {e!s}",
+ duration=time.time() - start_time,
+ metadata={"attempt": attempt},
+ )
+ await asyncio.sleep(self._retry_delay)
+
+ return VerificationResult(
+ status=VerificationOutcome.ERROR,
+ result="",
+ error_message="Unexpected code path reached",
+ duration=time.time() - start_time,
+ metadata={"attempt": attempt},
+ )
+
+ @abstractmethod
+ async def _verify_implementation(
+ self, result: VerifierInput
+ ) -> VerificationResult:
+ r"""Implement the actual verification logic.
+
+ Args:
+ result: The response to verify.
+
+ Returns:
+ VerificationResult: Containing the verification outcome.
+
+ Raises:
+ NotImplementedError: Must be implemented in subclasses.
+ """
+ raise NotImplementedError(
+ "Subclasses must implement _verify_implementation()"
+ )
+
+
+async def verify_batch(
+ self, results: List[VerifierInput], raise_on_error: bool = False
+) -> List[VerificationResult]:
+ r"""Verify multiple results in parallel with controlled concurrency.
+
+ Args:
+ results: List of responses to verify.
+ raise_on_error: Whether to raise an exception if any verification
+ fails. (default: :obj:`False`)
+
+ Returns:
+ List[VerificationResult]: One for each input response.
+
+ Raises:
+ RuntimeError: If any verification fails and raise_on_error is True.
+ asyncio.TimeoutError: If verifications time out and max retries
+ exceeded.
+ """
+ if not self._is_setup:
+ logger.warning(
+ f"{self.__class__.__name__} not set up, calling setup()"
+ )
+ await self.setup()
+
+ # Get current batch parameters from processor with defaults if not
+ # present
+ max_workers = getattr(
+ self._batch_processor, 'max_workers', self._max_parallel or 1
+ )
+ batch_size = getattr(
+ self._batch_processor, 'batch_size', self._initial_batch_size or 10
+ )
+ semaphore = asyncio.Semaphore(max(1, max_workers))
+
+ async def _verify_with_semaphore(
+ response: VerifierInput,
+ ) -> VerificationResult:
+ start_time = time.time()
+ try:
+ async with semaphore:
+ verification_result = await self.verify(response)
+ processing_time = time.time() - start_time
+ success = verification_result.status == VerificationOutcome.SUCCESS
+ self._batch_processor.adjust_batch_size(success, processing_time)
+ return verification_result
+ except Exception as e:
+ processing_time = time.time() - start_time
+ self._batch_processor.adjust_batch_size(False, processing_time)
+ logger.error(f"Verification failed: {e!s}", exc_info=True)
+ return VerificationResult(
+ status=VerificationOutcome.ERROR,
+ result="",
+ error_message=str(e),
+ metadata={"error_type": type(e).__name__},
+ )
+
+ # Process in batches
+ all_results: List[VerificationResult] = []
+ for i in range(0, len(results), batch_size):
+ batch = results[i : i + batch_size]
+ verification_tasks = [
+ _verify_with_semaphore(result) for result in batch
+ ]
+ try:
+ batch_results = await asyncio.gather(*verification_tasks)
+ all_results.extend(batch_results)
+ except Exception as e:
+ logger.error(f"Batch verification failed: {e!s}", exc_info=True)
+ if raise_on_error:
+ raise RuntimeError(f"Batch verification failed: {e!s}") from e
+
+ if raise_on_error and any(
+ r.status in {VerificationOutcome.ERROR, VerificationOutcome.TIMEOUT}
+ for r in all_results
+ ):
+ error_msg = "One or more verifications failed"
+ logger.error(error_msg)
+ raise RuntimeError(error_msg)
+
+ return all_results
diff --git a/camel/verifiers/models.py b/camel/verifiers/models.py
new file mode 100644
index 000000000..37bcbc478
--- /dev/null
+++ b/camel/verifiers/models.py
@@ -0,0 +1,76 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, Field
+
+
+class VerifierInput(BaseModel):
+ llm_response: str = Field(
+ description="The LLM response to be verified."
+ "Needs to be in a format that the verifier can handle."
+ )
+ ground_truth: Optional[str] = Field(
+ description="The ground truth data, if available."
+ )
+
+
+class VerificationOutcome(Enum):
+ r"""Enum representing the status of a verification."""
+
+ SUCCESS = "success"
+ FAILURE = "failure"
+ ERROR = "error"
+ TIMEOUT = "timeout"
+
+
+class VerificationResult(BaseModel):
+ r"""Structured result from a verification."""
+
+ status: VerificationOutcome = Field(
+ description="Status of the verification"
+ )
+ result: str = Field(description="Verification result")
+ duration: float = Field(
+ default=0.0, description="Duration of verification in seconds"
+ )
+ timestamp: datetime = Field(
+ default_factory=datetime.now,
+ description="When the verification was performed",
+ )
+ metadata: Dict[str, Any] = Field(
+ default_factory=dict,
+ description="Additional metadata about the verification",
+ )
+ error_message: Optional[str] = Field(
+ default=None, description="Error message if verification failed"
+ )
+
+
+class VerifierConfig(BaseModel):
+ r"""Configuration for verifier behavior."""
+
+ enabled: bool = Field(True, description="Whether verification is enabled")
+ strict_mode: bool = Field(
+ False, description="Whether to fail on any validation error"
+ )
+ timeout: Optional[float] = Field(
+ None, description="Verification timeout in seconds"
+ )
+ max_retries: int = Field(3, description="Maximum number of retry attempts")
+ retry_delay: float = Field(
+ 1.0, description="Delay between retries in seconds"
+ )
diff --git a/test/datasets/test_base_dataset.py b/test/datasets/test_base_dataset.py
new file mode 100644
index 000000000..83db9ad60
--- /dev/null
+++ b/test/datasets/test_base_dataset.py
@@ -0,0 +1,482 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import tempfile
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+import torch
+from datasets import Dataset as HFDataset
+from pydantic import ValidationError
+
+from camel.datasets.base import (
+ BaseDataset,
+ DataPoint,
+ GenerativeDataset,
+ PyTorchDataset,
+ SeedDataset,
+ SyntheticDataset,
+ convert_hf_to_pytorch,
+ convert_synthetic_to_pytorch,
+ load_pytorch_dataset,
+ save_synthetic_dataset,
+)
+
+
+# ruff: noqa: RUF001
+def test_datapoint_creation():
+ r"""Test creating a DataPoint with valid data."""
+ data = {
+ 'question': 'What is 2+2?',
+ 'rationale': 'Addition of two numbers',
+ 'final_answer': '4',
+ }
+
+ datapoint = DataPoint(**data)
+
+ assert datapoint.question == 'What is 2+2?'
+ assert datapoint.rationale == 'Addition of two numbers'
+ assert datapoint.final_answer == '4'
+ assert datapoint.difficulty is None
+ assert datapoint.metadata is None
+
+
+def test_datapoint_validation():
+ r"""Test DataPoint validation with missing required fields."""
+ with pytest.raises(ValidationError):
+ DataPoint(question='What is 2+2?')
+
+ with pytest.raises(ValidationError):
+ DataPoint(question='What is 2+2?', rationale='Addition')
+
+
+def test_datapoint_to_dict():
+ r"""Test converting DataPoint to dictionary."""
+ data = {
+ 'question': 'What is 2+2?',
+ 'rationale': 'Addition of two numbers',
+ 'final_answer': '4',
+ 'difficulty': 'easy',
+ 'metadata': {'topic': 'math'},
+ }
+
+ datapoint = DataPoint(**data)
+ data_dict = datapoint.to_dict()
+
+ assert data_dict['question'] == 'What is 2+2?'
+ assert data_dict['rationale'] == 'Addition of two numbers'
+ assert data_dict['final_answer'] == '4'
+ assert data_dict['difficulty'] == 'easy'
+ assert data_dict['metadata'] == {'topic': 'math'}
+
+
+def test_datapoint_from_dict():
+ r"""Test creating DataPoint from dictionary."""
+ data = {
+ 'question': 'What is 2+2?',
+ 'rationale': 'Addition of two numbers',
+ 'final_answer': '4',
+ }
+
+ datapoint = DataPoint.from_dict(data)
+
+ assert datapoint.question == 'What is 2+2?'
+ assert datapoint.rationale == 'Addition of two numbers'
+ assert datapoint.final_answer == '4'
+
+
+def sample_data():
+ r"""Fixture providing sample data for testing."""
+ return [
+ {
+ 'question': 'What is 2+2?',
+ 'rationale': 'Addition of two numbers',
+ 'final_answer': '4',
+ },
+ {
+ 'question': 'What is 3×3?',
+ 'rationale': 'Multiplication of two numbers',
+ 'final_answer': '9',
+ },
+ ]
+
+
+async def test_base_dataset_setup(sample_data):
+ r"""Test BaseDataset setup process."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ dataset = BaseDataset(data=sample_data, cache_dir=temp_dir)
+ await dataset.setup()
+
+ assert dataset._is_setup
+ assert len(dataset.data) == 2
+ assert isinstance(dataset.data[0], DataPoint)
+ assert dataset.data[0].question == 'What is 2+2?'
+ assert dataset.data[1].final_answer == '9'
+
+
+async def test_base_dataset_empty_setup():
+ r"""Test BaseDataset setup with empty data."""
+ dataset = BaseDataset(data=None)
+ await dataset.setup()
+
+ assert dataset._is_setup
+ assert len(dataset.data) == 0
+
+
+async def test_base_dataset_cleanup():
+ r"""Test BaseDataset cleanup process."""
+ dataset = BaseDataset(data=[])
+ await dataset.setup()
+ assert dataset._is_setup
+
+ await dataset.cleanup()
+ assert not dataset._is_setup
+
+
+def test_base_dataset_sample(sample_data):
+ r"""Test sampling from BaseDataset."""
+ dataset = BaseDataset(data=sample_data)
+ dataset._is_setup = True
+ dataset.data = [DataPoint(**item) for item in sample_data]
+
+ sample = dataset.sample()
+ assert isinstance(sample, DataPoint)
+ assert sample in dataset.data
+
+
+def test_base_dataset_len(sample_data):
+ r"""Test BaseDataset length."""
+ dataset = BaseDataset(data=sample_data)
+ dataset._is_setup = True
+ dataset.data = [DataPoint(**item) for item in sample_data]
+
+ assert len(dataset) == 2
+
+
+def test_base_dataset_getitem(sample_data):
+ r"""Test BaseDataset item access."""
+ dataset = BaseDataset(data=sample_data)
+ dataset._is_setup = True
+ dataset.data = [DataPoint(**item) for item in sample_data]
+
+ item = dataset[0]
+ assert isinstance(item, DataPoint)
+ assert item.question == 'What is 2+2?'
+
+ with pytest.raises(IndexError):
+ _ = dataset[100] # Out of bounds
+
+
+def test_base_dataset_metadata():
+ r"""Test BaseDataset metadata."""
+ dataset = BaseDataset(data=[], cache_dir='/tmp', custom_param='value')
+
+ metadata = dataset.metadata
+ assert metadata['cache_dir'] == '/tmp'
+ assert metadata['custom_param'] == 'value'
+
+
+def test_seed_dataset_init(sample_data):
+ r"""Test SeedDataset initialization."""
+ dataset = SeedDataset(data=sample_data, min_samples=1)
+ assert dataset._raw_data == sample_data
+
+ with pytest.raises(ValueError):
+ SeedDataset(data=sample_data, min_samples=3)
+
+
+def test_synthetic_dataset_init():
+ r"""Test SyntheticDataset initialization."""
+ dataset = SyntheticDataset()
+ assert dataset._raw_data == []
+ assert dataset.data == []
+
+
+def test_synthetic_dataset_add():
+ r"""Test adding items to SyntheticDataset."""
+ dataset = SyntheticDataset()
+ datapoint = DataPoint(
+ question='What is 2+2?',
+ rationale='Addition of two numbers',
+ final_answer='4',
+ )
+
+ dataset.add(datapoint)
+ assert len(dataset.data) == 1
+ assert dataset.data[0] == datapoint
+
+
+def test_synthetic_dataset_add_batch():
+ r"""Test adding batch of items to SyntheticDataset."""
+ dataset = SyntheticDataset()
+ datapoints = [
+ DataPoint(
+ question='What is 2+2?',
+ rationale='Addition of two numbers',
+ final_answer='4',
+ ),
+ DataPoint(
+ question='What is 3×3?',
+ rationale='Multiplication of two numbers',
+ final_answer='9',
+ ),
+ ]
+
+ dataset.add_batch(datapoints)
+ assert len(dataset.data) == 2
+ assert dataset.data == datapoints
+
+
+def test_synthetic_dataset_filter():
+ r"""Test filtering SyntheticDataset."""
+ dataset = SyntheticDataset()
+ datapoints = [
+ DataPoint(
+ question='What is 2+2?',
+ rationale='Addition of two numbers',
+ final_answer='4',
+ difficulty='easy',
+ ),
+ DataPoint(
+ question='What is 3×3?',
+ rationale='Multiplication of two numbers',
+ final_answer='9',
+ difficulty='medium',
+ ),
+ ]
+ dataset.add_batch(datapoints)
+
+ # Filter for easy questions
+ filtered = dataset.filter(lambda dp: dp.difficulty == 'easy')
+ assert len(filtered.data) == 1
+ assert filtered.data[0].question == 'What is 2+2?'
+
+
+def test_pytorch_dataset_init(sample_data):
+ r"""Test PyTorchDataset initialization."""
+ dataset = PyTorchDataset(sample_data)
+ assert len(dataset) == 2
+ assert dataset[0]['question'] == 'What is 2+2?'
+
+
+def test_pytorch_dataset_from_datapoints():
+ r"""Test creating PyTorchDataset from DataPoints."""
+ datapoints = [
+ DataPoint(
+ question='What is 2+2?',
+ rationale='Addition of two numbers',
+ final_answer='4',
+ ),
+ DataPoint(
+ question='What is 3×3?',
+ rationale='Multiplication of two numbers',
+ final_answer='9',
+ ),
+ ]
+
+ dataset = PyTorchDataset.from_datapoints(datapoints)
+ assert len(dataset) == 2
+ assert dataset[0]['question'] == 'What is 2+2?'
+ assert dataset[1]['final_answer'] == '9'
+
+
+def test_pytorch_dataset_transform():
+ r"""Test PyTorchDataset with transform function."""
+ data = [
+ {
+ 'question': 'What is 2+2?',
+ 'rationale': 'Addition of two numbers',
+ 'final_answer': '4',
+ }
+ ]
+
+ def transform(sample):
+ sample['question'] = sample['question'].upper()
+ return sample
+
+ dataset = PyTorchDataset(data, transform=transform)
+ assert dataset[0]['question'] == 'WHAT IS 2+2?'
+
+
+def test_pytorch_dataset_collate_fn():
+ r"""Test PyTorchDataset collate function."""
+ batch = [
+ {
+ 'question': 'What is 2+2?',
+ 'rationale': 'Addition of two numbers',
+ 'final_answer': '4',
+ 'numeric_value': 4,
+ },
+ {
+ 'question': 'What is 3×3?',
+ 'rationale': 'Multiplication of two numbers',
+ 'final_answer': '9',
+ 'numeric_value': 9,
+ },
+ ]
+
+ result = PyTorchDataset.collate_fn(batch)
+
+ # String fields should be lists
+ assert isinstance(result['question'], list)
+ assert result['question'] == ['What is 2+2?', 'What is 3×3?']
+
+ # Numeric fields should be tensors
+ assert isinstance(result['numeric_value'], torch.Tensor)
+ assert torch.equal(result['numeric_value'], torch.tensor([4, 9]))
+
+
+def test_convert_hf_to_pytorch():
+ r"""Test converting HuggingFace dataset to PyTorchDataset."""
+ hf_data = [
+ {
+ 'q': 'What is 2+2?',
+ 'r': 'Addition of two numbers',
+ 'a': '4',
+ },
+ {
+ 'q': 'What is 3×3?',
+ 'r': 'Multiplication of two numbers',
+ 'a': '9',
+ },
+ ]
+
+ hf_dataset = HFDataset.from_list(hf_data)
+
+ # Test with column mapping
+ column_mapping = {'q': 'question', 'r': 'rationale', 'a': 'final_answer'}
+ pytorch_dataset = convert_hf_to_pytorch(
+ hf_dataset, column_mapping=column_mapping
+ )
+
+ assert len(pytorch_dataset) == 2
+ assert pytorch_dataset[0]['question'] == 'What is 2+2?'
+ assert pytorch_dataset[1]['final_answer'] == '9'
+
+
+def test_convert_synthetic_to_pytorch():
+ r"""Test converting SyntheticDataset to PyTorchDataset."""
+ synthetic_dataset = SyntheticDataset()
+ datapoints = [
+ DataPoint(
+ question='What is 2+2?',
+ rationale='Addition of two numbers',
+ final_answer='4',
+ ),
+ DataPoint(
+ question='What is 3×3?',
+ rationale='Multiplication of two numbers',
+ final_answer='9',
+ ),
+ ]
+ synthetic_dataset.add_batch(datapoints)
+
+ pytorch_dataset = convert_synthetic_to_pytorch(synthetic_dataset)
+
+ assert len(pytorch_dataset) == 2
+ assert pytorch_dataset[0]['question'] == 'What is 2+2?'
+ assert pytorch_dataset[1]['final_answer'] == '9'
+
+
+def test_save_and_load_pytorch_dataset():
+ r"""Test saving and loading PyTorchDataset."""
+ with tempfile.NamedTemporaryFile(suffix='.pt') as temp_file:
+ # Create and save dataset
+ data = [
+ {
+ 'question': 'What is 2+2?',
+ 'rationale': 'Addition of two numbers',
+ 'final_answer': '4',
+ }
+ ]
+ dataset = PyTorchDataset(data)
+ dataset.save_to_disk(temp_file.name)
+
+ # Load dataset
+ loaded_dataset = load_pytorch_dataset(temp_file.name)
+
+ assert len(loaded_dataset) == 1
+ assert loaded_dataset[0]['question'] == 'What is 2+2?'
+
+
+def test_save_synthetic_dataset():
+ r"""Test saving SyntheticDataset."""
+ with tempfile.NamedTemporaryFile(suffix='.pt') as temp_file:
+ # Create and save dataset
+ synthetic_dataset = SyntheticDataset()
+ datapoint = DataPoint(
+ question='What is 2+2?',
+ rationale='Addition of two numbers',
+ final_answer='4',
+ )
+ synthetic_dataset.add(datapoint)
+
+ save_synthetic_dataset(synthetic_dataset, temp_file.name)
+
+ # Load dataset
+ loaded_dataset = load_pytorch_dataset(temp_file.name)
+
+ assert len(loaded_dataset) == 1
+ assert loaded_dataset[0]['question'] == 'What is 2+2?'
+
+
+async def test_generative_dataset():
+ r"""Test GenerativeDataset with mocked components."""
+ mock_seed_dataset = MagicMock(spec=SeedDataset)
+ mock_seed_dataset.__len__.return_value = 5
+ mock_seed_dataset.__getitem__.side_effect = lambda i: DataPoint(
+ question=f"Question {i}",
+ rationale=f"Rationale {i}",
+ final_answer=f"Answer {i}",
+ )
+
+ mock_verifier = MagicMock()
+ mock_verifier.verify = AsyncMock()
+ mock_verifier.verify.return_value = MagicMock(result="Verified Answer")
+
+ mock_agent = MagicMock()
+ mock_agent.step.return_value = MagicMock(
+ msgs=[
+ MagicMock(
+ parsed={
+ 'question': 'Generated Question',
+ 'rationale': 'Generated Rationale',
+ }
+ )
+ ]
+ )
+
+ # Create GenerativeDataset with mocks
+ dataset = GenerativeDataset(
+ seed_dataset=mock_seed_dataset,
+ verifier=mock_verifier,
+ agent=mock_agent,
+ )
+
+ # Test generate_new method
+ await dataset.generate_new(2)
+
+ # Verify interactions
+ assert mock_agent.step.call_count >= 2
+ assert mock_verifier.verify.await_count >= 2
+ assert len(dataset.data) == 2
+
+ # Verify generated data
+ assert all(isinstance(dp, DataPoint) for dp in dataset.data)
+ assert all(dp.final_answer == "Verified Answer" for dp in dataset.data)
diff --git a/test/environments/test_base_env.py b/test/environments/test_base_env.py
new file mode 100644
index 000000000..9ac90ab41
--- /dev/null
+++ b/test/environments/test_base_env.py
@@ -0,0 +1,288 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from datetime import datetime, timezone
+from typing import Dict, Tuple
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from camel.environments.base import (
+ Action,
+ BaseEnvironment,
+ Observation,
+ StepResult,
+)
+from camel.verifiers.models import VerificationOutcome, VerificationResult
+
+
+def test_action_model():
+ r"""Test the Action model initialization and properties."""
+ custom_time = datetime(2023, 1, 1, tzinfo=timezone.utc)
+ action_full = Action(
+ problem_statement="Full problem",
+ llm_response="Full response",
+ final_answer="Reference solution",
+ metadata={"model": "test-model", "temperature": 0.7},
+ timestamp=custom_time,
+ )
+ assert action_full.problem_statement == "Full problem"
+ assert action_full.llm_response == "Full response"
+ assert action_full.final_answer == "Reference solution"
+ assert action_full.metadata == {"model": "test-model", "temperature": 0.7}
+ assert action_full.timestamp == custom_time
+
+
+def test_observation_model():
+ r"""Test the Observation model initialization and properties."""
+ obs_full = Observation(
+ question="Complex math problem",
+ context={"hint": "Use calculus"},
+ metadata={"difficulty": "hard"},
+ )
+ assert obs_full.question == "Complex math problem"
+ assert obs_full.context == {"hint": "Use calculus"}
+ assert obs_full.metadata == {"difficulty": "hard"}
+
+
+def test_step_result_model():
+ r"""Test the StepResult model initialization and properties."""
+ obs = Observation(question="Next question")
+ result_full = StepResult(
+ observation=obs,
+ reward=1.0,
+ rewards_dict={"correctness": 0.8, "efficiency": 0.2},
+ done=True,
+ info={"reason": "task_complete"},
+ )
+ assert result_full.observation == obs
+ assert result_full.reward == 1.0
+ assert result_full.rewards_dict == {"correctness": 0.8, "efficiency": 0.2}
+ assert result_full.done is True
+ assert result_full.info == {"reason": "task_complete"}
+
+
+class MockBaseEnvironment(BaseEnvironment):
+ r"""Mock implementation of BaseEnvironment for testing."""
+
+ async def setup(self):
+ await super().setup()
+
+ async def teardown(self):
+ await super().teardown()
+
+ async def reset(self):
+ return await super().reset()
+
+ async def step(self, action):
+ return await super().step(action)
+
+ def _get_initial_state(self):
+ return super()._get_initial_state()
+
+ def _get_next_observation(self):
+ return Observation(question="Test question")
+
+ def _get_terminal_observation(self):
+ return super()._get_terminal_observation()
+
+ async def compute_reward(
+ self,
+ action: Action,
+ extraction_result: str,
+ verification_result: VerificationResult,
+ ) -> Tuple[float, Dict[str, float]]:
+ return await super().compute_reward(
+ action, extraction_result, verification_result
+ )
+
+ async def _compute_reward(
+ self, action, extraction_result, verification_result
+ ):
+ return {"additional_reward": 0.5}
+
+
+async def test_base_environment_lifecycle():
+ r"""Test the lifecycle methods of BaseEnvironment."""
+ mock_dataset = MagicMock()
+ mock_dataset.__len__.return_value = 1
+ mock_dataset.__getitem__.return_value = MagicMock(
+ question="Test question",
+ final_answer="Test answer",
+ rationale="Test rationale",
+ difficulty="medium",
+ metadata={"verified": True},
+ )
+ mock_dataset.setup = AsyncMock()
+ mock_dataset.cleanup = AsyncMock()
+
+ mock_verifier = MagicMock()
+ mock_verifier.setup = AsyncMock()
+ mock_verifier.cleanup = AsyncMock()
+ mock_verifier.verify = AsyncMock(
+ return_value=VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Verification successful",
+ feedback="Correct",
+ score=1.0,
+ )
+ )
+
+ mock_extractor = MagicMock()
+ mock_extractor.setup = AsyncMock()
+ mock_extractor.cleanup = AsyncMock()
+ mock_extractor.extract = AsyncMock(return_value={"extracted": "content"})
+
+ # Create environment
+ env = MockBaseEnvironment(
+ dataset=mock_dataset,
+ verifier=mock_verifier,
+ extractor=mock_extractor,
+ max_steps=5,
+ )
+
+ # Test setup
+ await env.setup()
+ assert env._is_setup is True
+ mock_verifier.setup.assert_awaited_once()
+
+ # Test reset
+ observation = await env.reset()
+ assert isinstance(observation, Observation)
+ assert env._current_step == 0
+ assert env._episode_ended is False
+
+ original_step = env.step
+
+ async def mock_step(action):
+ env._current_step += 1
+ return StepResult(
+ observation=env._get_next_observation(),
+ reward=1.0,
+ rewards_dict={"correctness": 1.0, "additional_reward": 0.5},
+ done=False,
+ info={
+ "extraction_result": {"extracted": "content"},
+ "verification_result": VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Verification successful",
+ feedback="Correct",
+ score=1.0,
+ ),
+ "step": env._current_step,
+ },
+ )
+
+ env.step = mock_step
+
+ # Test step
+ action = Action(
+ problem_statement="Test problem",
+ llm_response="Test response",
+ )
+ result = await env.step(action)
+ assert isinstance(result, StepResult)
+ assert env._current_step == 1
+
+ # Restore the original step method
+ env.step = original_step
+
+ # Test teardown
+ await env.teardown()
+ assert env._is_setup is False
+ mock_verifier.cleanup.assert_awaited_once()
+
+
+async def test_environment_reward_calculation():
+ r"""Test the reward calculation in BaseEnvironment."""
+ env = MockBaseEnvironment(
+ dataset=MagicMock(),
+ verifier=MagicMock(),
+ extractor=MagicMock(),
+ )
+
+ # Test compute_reward method
+ action = Action(
+ problem_statement="Test problem",
+ llm_response="Test response",
+ )
+ extraction_result = {"extracted": "content"}
+ verification_result = VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Verification successful",
+ feedback="Correct",
+ score=1.0,
+ )
+
+ total_reward, rewards_dict = await env.compute_reward(
+ action, extraction_result, verification_result
+ )
+
+ # Check rewards
+ assert "correctness" in rewards_dict
+ assert rewards_dict["correctness"] == 1.0
+ assert "additional_reward" in rewards_dict
+ assert rewards_dict["additional_reward"] == 0.5
+ assert total_reward == 1.5 # Sum of all rewards
+
+
+async def test_environment_max_steps():
+ r"""Test that environment terminates after max_steps."""
+ env = MockBaseEnvironment(
+ dataset=MagicMock(),
+ verifier=MagicMock(),
+ extractor=MagicMock(),
+ max_steps=2,
+ )
+ env._is_setup = True
+
+ # Take first step
+ action = Action(
+ problem_statement="Test problem",
+ llm_response="Test response",
+ )
+ env._last_observation = Observation(question="Test question")
+
+ original_step = env.step
+
+ async def mock_step(action):
+ env._current_step += 1
+ done = env._current_step >= env.max_steps
+ return StepResult(
+ observation=env._get_terminal_observation()
+ if done
+ else env._get_next_observation(),
+ reward=1.0,
+ rewards_dict={"correctness": 1.0},
+ done=done,
+ info={"reason": "max_steps_reached"} if done else {},
+ )
+
+ # Replace the step method
+ env.step = mock_step
+
+ # First step should not terminate
+ result1 = await env.step(action)
+ assert result1.done is False
+
+ # Second step should terminate due to max_steps
+ result2 = await env.step(action)
+ assert result2.done is True
+ assert "max_steps_reached" in result2.info.get("reason", "")
+
+ # Restore the original step method
+ env.step = original_step
diff --git a/test/extractors/test_base_extractor.py b/test/extractors/test_base_extractor.py
new file mode 100644
index 000000000..df238362a
--- /dev/null
+++ b/test/extractors/test_base_extractor.py
@@ -0,0 +1,165 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from typing import Any, Dict, Optional
+from unittest.mock import MagicMock
+
+import pytest
+
+from camel.extractors.base import BaseExtractor
+
+
+class TestExtractor(BaseExtractor):
+ r"""Concrete implementation of BaseExtractor for testing."""
+
+ async def extract(
+ self, response: str, context: Optional[Dict[str, Any]] = None
+ ) -> str:
+ r"""Simple implementation that returns the response with a prefix."""
+ if not self._is_setup:
+ raise RuntimeError("Extractor not initialized")
+ if not response or not response.strip():
+ raise ValueError("Empty or whitespace-only response")
+
+ prefix = "Extracted: "
+ if context and "prefix" in context:
+ prefix = context["prefix"]
+
+ return f"{prefix}{response}"
+
+
+def test_extractor():
+ r"""Fixture providing a TestExtractor instance."""
+ return TestExtractor(
+ cache_templates=True,
+ max_cache_size=100,
+ extraction_timeout=10.0,
+ batch_size=5,
+ )
+
+
+def test_extractor_init():
+ r"""Test BaseExtractor initialization with various parameters."""
+ extractor = TestExtractor(
+ cache_templates=False,
+ max_cache_size=500,
+ extraction_timeout=15.0,
+ batch_size=20,
+ custom_param="value",
+ )
+ assert extractor._cache_templates is False
+ assert extractor._max_cache_size == 500
+ assert extractor._extraction_timeout == 15.0
+ assert extractor._batch_size == 20
+ assert extractor._metadata["custom_param"] == "value"
+
+
+async def test_extractor_setup(test_extractor):
+ r"""Test the setup method of BaseExtractor."""
+ assert test_extractor._is_setup is False
+
+ await test_extractor.setup()
+ assert test_extractor._is_setup is True
+ assert hasattr(test_extractor, "_template_cache")
+
+ await test_extractor.setup()
+ assert test_extractor._is_setup is True
+
+ await test_extractor.cleanup()
+
+
+async def test_extractor_cleanup(test_extractor):
+ r"""Test the cleanup method of BaseExtractor."""
+ await test_extractor.setup()
+ assert test_extractor._is_setup is True
+
+ await test_extractor.cleanup()
+ assert test_extractor._is_setup is False
+
+ await test_extractor.cleanup()
+ assert test_extractor._is_setup is False
+
+
+async def test_extractor_cleanup_with_errors():
+ r"""Test cleanup handling when errors occur."""
+ extractor = TestExtractor()
+ await extractor.setup()
+
+ # Mock _template_cache.clear to raise an exception
+ extractor._template_cache = MagicMock()
+ extractor._template_cache.clear.side_effect = Exception("Mock error")
+
+ with pytest.raises(RuntimeError):
+ await extractor.cleanup()
+
+ assert extractor._is_setup is False
+
+
+async def test_extractor_context_manager():
+ r"""Test using the extractor as an async context manager."""
+ async with TestExtractor() as extractor:
+ assert extractor._is_setup is True
+ result = await extractor.extract("test")
+ assert result == "Extracted: test"
+
+ assert extractor._is_setup is False
+
+
+async def test_extractor_context_manager_with_exception():
+ r"""Test context manager handles exceptions properly."""
+ try:
+ async with TestExtractor() as extractor:
+ assert extractor._is_setup is True
+ raise ValueError("Test exception")
+ except ValueError:
+ # The exception should be propagated, but cleanup should still happen
+ assert extractor._is_setup is False
+
+
+async def test_extract_method(test_extractor):
+ r"""Test the extract method of the concrete implementation."""
+ await test_extractor.setup()
+
+ # Basic extraction
+ result = await test_extractor.extract("Hello World")
+ assert result == "Extracted: Hello World"
+
+ # Extraction with context
+ result = await test_extractor.extract(
+ "Hello World", context={"prefix": "Custom: "}
+ )
+ assert result == "Custom: Hello World"
+
+ await test_extractor.cleanup()
+
+
+async def test_extract_with_empty_input(test_extractor):
+ r"""Test extract method with empty input."""
+ await test_extractor.setup()
+
+ with pytest.raises(ValueError, match="Empty or whitespace-only response"):
+ await test_extractor.extract("")
+
+ with pytest.raises(ValueError, match="Empty or whitespace-only response"):
+ await test_extractor.extract(" ")
+
+ await test_extractor.cleanup()
diff --git a/test/verifiers/test_base_verifier.py b/test/verifiers/test_base_verifier.py
new file mode 100644
index 000000000..1f29a00d5
--- /dev/null
+++ b/test/verifiers/test_base_verifier.py
@@ -0,0 +1,419 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import asyncio
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from camel.verifiers.base import BaseVerifier
+from camel.verifiers.models import (
+ VerificationOutcome,
+ VerificationResult,
+ VerifierInput,
+)
+
+
+class TestVerifier(BaseVerifier):
+ r"""Concrete implementation of BaseVerifier for testing."""
+
+ async def _setup(self) -> None:
+ self.setup_called = True
+
+ async def _cleanup(self) -> None:
+ self.cleanup_called = True
+
+ async def _verify_implementation(
+ self, result: VerifierInput
+ ) -> VerificationResult:
+ r"""Simple implementation that returns success or failure based on
+ input.
+ """
+ if "fail" in result.llm_response.lower():
+ return VerificationResult(
+ status=VerificationOutcome.FAILURE,
+ result="Verification failed",
+ )
+ elif "error" in result.llm_response.lower():
+ raise ValueError("Simulated error in verification")
+ elif "timeout" in result.llm_response.lower():
+ raise asyncio.TimeoutError("Simulated timeout")
+ else:
+ return VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Verification succeeded",
+ )
+
+
+def test_verifier():
+ r"""Fixture providing a TestVerifier instance."""
+ return TestVerifier(
+ max_parallel=2,
+ timeout=5.0,
+ max_retries=2,
+ retry_delay=0.1,
+ )
+
+
+def test_verifier_init():
+ r"""Test BaseVerifier initialization with various parameters."""
+ verifier = TestVerifier()
+ assert verifier._max_parallel is None
+ assert verifier._timeout is None
+ assert verifier._max_retries == 3
+ assert verifier._retry_delay == 1.0
+ assert verifier._is_setup is False
+
+ verifier = TestVerifier(
+ max_parallel=5,
+ timeout=10.0,
+ max_retries=2,
+ retry_delay=0.5,
+ )
+ assert verifier._max_parallel == 5
+ assert verifier._timeout == 10.0
+ assert verifier._max_retries == 2
+ assert verifier._retry_delay == 0.5
+
+
+async def test_verifier_setup_and_cleanup(test_verifier):
+ r"""Test the setup and cleanup methods of BaseVerifier."""
+ assert test_verifier._is_setup is False
+ await test_verifier.setup()
+ assert test_verifier._is_setup is True
+ assert test_verifier.setup_called is True
+
+ await test_verifier.cleanup()
+ assert test_verifier._is_setup is False
+ assert test_verifier.cleanup_called is True
+
+
+async def test_verifier_setup_error():
+ r"""Test handling of errors during setup."""
+
+ class ErrorVerifier(TestVerifier):
+ async def _setup(self) -> None:
+ raise RuntimeError("Simulated setup error")
+
+ verifier = ErrorVerifier()
+
+ with pytest.raises(RuntimeError, match="Failed to initialize"):
+ await verifier.setup()
+
+ assert verifier._is_setup is False
+
+
+async def test_verifier_cleanup_error():
+ r"""Test handling of errors during cleanup."""
+
+ class ErrorVerifier(TestVerifier):
+ async def _cleanup(self) -> None:
+ raise RuntimeError("Simulated cleanup error")
+
+ verifier = ErrorVerifier()
+ await verifier.setup()
+
+ with pytest.raises(RuntimeError, match="Failed to cleanup"):
+ await verifier.cleanup()
+
+ assert verifier._is_setup is False
+
+
+async def test_verify_success(test_verifier):
+ r"""Test successful verification."""
+ await test_verifier.setup()
+
+ result = await test_verifier.verify(
+ VerifierInput(
+ llm_response="This is a successful response",
+ ground_truth="Expected response",
+ )
+ )
+
+ assert result.status == VerificationOutcome.SUCCESS
+ assert result.result == "Verification succeeded"
+ assert result.duration > 0
+ assert result.metadata["attempt"] == 1
+
+ await test_verifier.cleanup()
+
+
+async def test_verify_failure(test_verifier):
+ r"""Test failed verification."""
+ await test_verifier.setup()
+
+ result = await test_verifier.verify(
+ VerifierInput(
+ llm_response="This will fail the verification",
+ ground_truth="Expected response",
+ )
+ )
+
+ assert result.status == VerificationOutcome.FAILURE
+ assert result.result == "Verification failed"
+ assert result.duration > 0
+ assert result.metadata["attempt"] == 1
+
+ await test_verifier.cleanup()
+
+
+async def test_verify_error_with_retry(test_verifier):
+ r"""Test verification with error and retry."""
+ await test_verifier.setup()
+
+ test_verifier._max_retries = 1
+
+ with patch("asyncio.sleep", new_callable=AsyncMock):
+ result = await test_verifier.verify(
+ VerifierInput(
+ llm_response="This will cause an error",
+ ground_truth="Expected response",
+ )
+ )
+
+ assert result.status == VerificationOutcome.ERROR
+ assert result.error_message is not None
+ assert "Verification failed" in result.error_message
+ assert result.duration > 0
+ assert "attempt" in result.metadata
+
+ await test_verifier.cleanup()
+
+
+async def test_verify_timeout(test_verifier):
+ r"""Test verification timeout."""
+ await test_verifier.setup()
+
+ with patch(
+ "asyncio.wait_for",
+ side_effect=asyncio.TimeoutError("Simulated timeout"),
+ ):
+ result = await test_verifier.verify(
+ VerifierInput(
+ llm_response="This will timeout",
+ ground_truth="Expected response",
+ )
+ )
+
+ assert result.status == VerificationOutcome.TIMEOUT
+ assert "timed out" in result.error_message
+ assert result.duration > 0
+
+ await test_verifier.cleanup()
+
+
+async def test_verify_not_setup():
+ r"""Test verification when verifier is not set up."""
+ verifier = TestVerifier()
+
+ verifier.setup = AsyncMock()
+ verifier._verify_implementation = AsyncMock(
+ return_value=VerificationResult(
+ status=VerificationOutcome.SUCCESS, result="Success"
+ )
+ )
+
+ await verifier.verify(
+ VerifierInput(llm_response="Test", ground_truth="Expected")
+ )
+
+ verifier.setup.assert_called_once()
+
+
+async def test_verify_batch(test_verifier):
+ r"""Test batch verification."""
+ await test_verifier.setup()
+
+ test_verifier._verify_implementation = AsyncMock(
+ return_value=VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Verification succeeded",
+ )
+ )
+
+ async def mock_verify(input_data):
+ if "fail" in input_data.llm_response.lower():
+ return VerificationResult(
+ status=VerificationOutcome.FAILURE,
+ result="Verification failed",
+ )
+ return VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Verification succeeded",
+ )
+
+ with patch.object(test_verifier, "verify", side_effect=mock_verify):
+ from camel.verifiers.base import verify_batch
+
+ test_verifier.verify_batch = lambda *args, **kwargs: verify_batch(
+ test_verifier, *args, **kwargs
+ )
+
+ inputs = [
+ VerifierInput(llm_response="Success 1", ground_truth="Expected 1"),
+ VerifierInput(llm_response="Success 2", ground_truth="Expected 2"),
+ VerifierInput(
+ llm_response="This will fail", ground_truth="Expected 3"
+ ),
+ ]
+
+ results = await test_verifier.verify_batch(inputs)
+
+ assert len(results) == 3
+ assert results[0].status == VerificationOutcome.SUCCESS
+ assert results[1].status == VerificationOutcome.SUCCESS
+ assert results[2].status == VerificationOutcome.FAILURE
+
+ await test_verifier.cleanup()
+
+
+async def test_verify_batch_with_error_handling(test_verifier):
+ r"""Test batch verification with error handling."""
+ await test_verifier.setup()
+
+ test_verifier._verify_implementation = AsyncMock(
+ return_value=VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Verification succeeded",
+ )
+ )
+
+ async def mock_verify(input_data):
+ if "error" in input_data.llm_response.lower():
+ raise ValueError("Simulated error in verification")
+ return VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Verification succeeded",
+ )
+
+ with patch.object(test_verifier, "verify", side_effect=mock_verify):
+ from camel.verifiers.base import verify_batch
+
+ test_verifier.verify_batch = lambda *args, **kwargs: verify_batch(
+ test_verifier, *args, **kwargs
+ )
+
+ inputs = [
+ VerifierInput(llm_response="Success", ground_truth="Expected 1"),
+ VerifierInput(
+ llm_response="This will cause an error",
+ ground_truth="Expected 2",
+ ),
+ ]
+
+ with pytest.raises(
+ RuntimeError, match="One or more verifications failed"
+ ):
+ await test_verifier.verify_batch(inputs, raise_on_error=True)
+
+ results = await test_verifier.verify_batch(inputs)
+ assert len(results) == 2
+ assert results[0].status == VerificationOutcome.SUCCESS
+ assert results[1].status == VerificationOutcome.ERROR
+
+ await test_verifier.cleanup()
+
+
+async def test_verify_batch_concurrency_limiting(test_verifier):
+ r"""Test that batch verification properly limits concurrency."""
+ await test_verifier.setup()
+
+ class MockSemaphore:
+ def __init__(self, value):
+ self.value = value
+ self.count = 0
+ self.max_count = 0
+
+ async def __aenter__(self):
+ self.count += 1
+ self.max_count = max(self.max_count, self.count)
+
+ async def __aexit__(self, *args):
+ self.count -= 1
+
+ inputs = [
+ VerifierInput(
+ llm_response=f"Success {i}", ground_truth=f"Expected {i}"
+ )
+ for i in range(3)
+ ]
+
+ mock_sem = MockSemaphore(1)
+
+ async def patched_verify_batch(verifier, inputs, raise_on_error=False):
+ results = []
+ for input_data in inputs:
+ async with mock_sem:
+ await asyncio.sleep(0.01) # Simulate processing time
+ results.append(await verifier.verify(input_data))
+ return results
+
+ test_verifier.verify_batch = lambda *args, **kwargs: patched_verify_batch(
+ test_verifier, *args, **kwargs
+ )
+
+ await test_verifier.verify_batch(inputs)
+
+ assert mock_sem.max_count == 1
+
+ await test_verifier.cleanup()
+
+
+async def test_full_verification_flow():
+ r"""Test the full verification flow from setup to cleanup."""
+ verifier = TestVerifier(timeout=1.0, max_retries=1, retry_delay=0.1)
+
+ try:
+ await verifier.setup()
+ assert verifier._is_setup is True
+
+ success_result = await verifier.verify(
+ VerifierInput(
+ llm_response="This should succeed", ground_truth="Expected"
+ )
+ )
+ assert success_result.status == VerificationOutcome.SUCCESS
+
+ failure_result = await verifier.verify(
+ VerifierInput(
+ llm_response="This will fail", ground_truth="Expected"
+ )
+ )
+ assert failure_result.status == VerificationOutcome.FAILURE
+
+ with patch("asyncio.sleep", new_callable=AsyncMock):
+ error_result = await verifier.verify(
+ VerifierInput(
+ llm_response="This will cause an error",
+ ground_truth="Expected",
+ )
+ )
+ assert error_result.status == VerificationOutcome.ERROR
+
+ finally:
+ await verifier.cleanup()
+ assert verifier._is_setup is False