🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Initial implementation for BaseVerifier (#1597)
Wendong-Fan84185a59341f
diff --git a/camel/datasets/__init__.py b/camel/datasets/__init__.pynew file mode 100644index 000000000..e12a2a0bb--- /dev/null+++ b/camel/datasets/__init__.py@@ -0,0 +1,28 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from .base import (+ BaseDataset,+ DataPoint,+ GenerativeDataset,+ SeedDataset,+ SyntheticDataset,+)++__all__ = [+ "DataPoint",+ "BaseDataset",+ "SeedDataset",+ "GenerativeDataset",+ "SyntheticDataset",+]diff --git a/camel/datasets/base.py b/camel/datasets/base.pynew file mode 100644index 000000000..e085eeb46--- /dev/null+++ b/camel/datasets/base.py@@ -0,0 +1,969 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import os+import random+from typing import (+ Any,+ Callable,+ Dict,+ List,+ Optional,+ TypeVar,+ Union,+)++import torch+from datasets import Dataset as HFDataset+from pydantic import BaseModel, Field, ValidationError+from torch.utils.data import DataLoader, Dataset++from camel.agents import ChatAgent+from camel.logger import get_logger+from camel.verifiers import BaseVerifier++logger = get_logger(__name__)+++class DataPoint(BaseModel):+ r"""A single data point in the dataset.++ Attributes:+ question (str): The primary question or issue to be addressed.+ rationale (str): Logical reasoning or explanation behind the+ answer.+ final_answer (str): The final answer.+ raw_markdown (Optional[str]): Raw markdown content for generating+ rewards/hints. (default: :obj:`None`)+ difficulty (Optional[str]): Difficulty level of the question.+ (default: :obj:`None`)+ metadata Optional[Dict[str, Any]]: Additional metadata about the data+ point. (default: :obj:`None`)+ """++ question: str = Field(+ ..., description="The primary question or issue to be addressed."+ )+ rationale: str = Field(+ ..., description="Logical reasoning or explanation behind the answer."+ )+ final_answer: str = Field(..., description="The final answer.")+ difficulty: Optional[str] = Field(+ None, description="Difficulty level of the question."+ )+ metadata: Optional[Dict[str, Any]] = Field(+ default=None, description="Additional metadata about the data point."+ )++ def to_dict(self) -> Dict[str, Any]:+ r"""Convert DataPoint to a dictionary.++ Returns:+ Dict[str, Any]: Dictionary representation of the DataPoint.+ """+ return self.dict()++ @classmethod+ def from_dict(cls, data: Dict[str, Any]) -> 'DataPoint':+ r"""Create a DataPoint from a dictionary.++ Args:+ data (Dict[str, Any]): Dictionary containing DataPoint fields.++ Returns:+ DataPoint: New DataPoint instance.+ """+ return cls(**data)+++class BaseDataset(Dataset):+ r"""A dataset contains questions and ground truth data for training.+ It can be either static (e.g., MATH dataset) or generative+ (using an LLM to generate questions).+ """++ def __init__(+ self,+ data: List[Dict[str, str]],+ cache_dir: Optional[str] = None,+ **kwargs,+ ):+ r"""Initialize the dataset.++ Args:+ data (List[Dict[str, str]]): List of dictionary items to+ create the dataset from.+ cache_dir (Optional[str]): Directory to cache dataset files.+ (default: :obj:`None`)+ **kwargs: Additional dataset parameters.++ Note:+ The dataset must be initialized by calling setup() before use.+ """+ self._is_setup = False+ self._raw_data: List[Dict[str, str]] = data if data is not None else []+ self._cache_dir = str(cache_dir) if cache_dir is not None else None++ # Store all parameters in metadata dict for compatibility+ self._metadata = {+ 'cache_dir': self._cache_dir,+ **kwargs,+ }++ self.data: List[DataPoint] = [] # Will be populated in setup()++ async def setup(self) -> None:+ r"""Set up the dataset with necessary resources.++ This method:+ 1. Creates cache directory if needed+ 2. Processes raw data into DataPoint objects using vectorized+ operations+ 3. Validates dataset integrity++ Raises:+ OSError: If cache directory creation fails.+ Exception: If dataset initialization fails.+ """+ if self._is_setup:+ logger.debug(f"{self.__class__.__name__} already initialized")+ return++ try:+ # Create cache directory if specified+ if self._cache_dir:+ try:+ os.makedirs(self._cache_dir, exist_ok=True)+ logger.debug(f"Created cache directory: {self._cache_dir}")+ except OSError as e:+ logger.error(+ f"Failed to create cache directory "+ f"{self._cache_dir}: {e}"+ )+ raise++ # Process raw data into DataPoint objects using vectorized+ # operations+ if not self._raw_data:+ self.data = []+ logger.debug("No raw data to process")+ else:+ try:+ # Helper function for validation that can be used with map+ def create_datapoint(item, idx=None):+ try:+ return DataPoint(+ question=item.get('question', ''),+ rationale=item.get('rationale', ''),+ final_answer=item.get('final_answer', ''),+ metadata=item.get('metadata', {})+ if isinstance(item.get('metadata'), dict)+ else {},+ raw_markdown='',+ difficulty='',+ )+ except ValidationError as e:+ idx_str = (+ f" at index {idx}" if idx is not None else ""+ )+ error_msg = (+ f"Sample{idx_str} validation error: {e}"+ )+ logger.error(error_msg)+ raise ValueError(error_msg)++ # If raw_data is already a HF dataset, use its map function+ if hasattr(self._raw_data, 'map') and callable(+ self._raw_data.map+ ):+ # Using HF dataset's map for vectorized processing+ processed_data = self._raw_data.map(+ lambda example, idx: {+ 'datapoint': create_datapoint(example, idx)+ },+ with_indices=True,+ )+ self.data = [+ item['datapoint'] for item in processed_data+ ]+ else:+ # Bulk create datapoints+ self.data = [+ create_datapoint(item, i)+ for i, item in enumerate(self._raw_data)+ ]++ logger.debug(f"Processed {len(self.data)} data points")+ except Exception as e:+ logger.error(f"Error processing data: {e}")+ raise++ self._is_setup = True+ logger.info(f"{self.__class__.__name__} initialized successfully")++ except Exception as e:+ logger.error(f"Error during {self.__class__.__name__} setup: {e}")+ await self.cleanup()+ raise++ async def cleanup(self) -> None:+ r"""Clean up dataset resources.++ This method handles cleanup of resources and resets the dataset state.+ """+ if not self._is_setup:+ return++ try:+ # Clear metadata while preserving init config+ init_config = {+ 'cache_dir': self._cache_dir,+ }+ self._metadata = init_config++ logger.info(f"{self.__class__.__name__} cleaned up successfully")++ except Exception as e:+ logger.error(+ f"Error during {self.__class__.__name__} cleanup: {e}"+ )+ raise++ finally:+ # Always mark as uninitialized, even if cleanup fails+ self._is_setup = False++ def sample(self) -> DataPoint:+ r"""Sample a random datapoint from the dataset.++ Returns:+ DataPoint: A randomly sampled DataPoint.++ Raises:+ RuntimeError: If dataset is not initialized.+ """+ if not self._is_setup:+ raise RuntimeError(+ f"{self.__class__.__name__} must be initialized "+ "before sampling"+ )++ idx = random.randint(0, len(self) - 1)+ return self[idx]++ def __len__(self) -> int:+ r"""Return the size of the dataset."""+ return len(self.data)++ def __getitem__(self, idx: int) -> DataPoint:+ r"""Get an item from the dataset.++ Args:+ idx (int): Index of the item to get.++ Returns:+ DataPoint: DataPoint from the dataset with the given index.++ Raises:+ IndexError: If idx is out of bounds.+ """+ if idx < 0 or idx >= len(self):+ raise IndexError(+ f"Index {idx} out of bounds for dataset of size {len(self)}"+ )++ return self.data[idx]++ @property+ def metadata(self) -> Dict[str, Any]:+ r"""Get dataset metadata."""+ return self._metadata.copy()++ def to_pytorch_dataset(+ self,+ transform: Optional[Callable] = None,+ target_transform: Optional[Callable] = None,+ batch_size: Optional[int] = None,+ ) -> Union["PyTorchDataset", "DataLoader"]:+ r"""Convert to a PyTorch dataset or DataLoader.++ Args:+ transform (Optional[Callable]): Transform to apply to samples.+ target_transform (Optional[Callable]): Transform to apply to+ targets.+ batch_size (Optional[int]): If provided, returns a DataLoader with+ the specified batch size instead of a PyTorchDataset.+ (default: :obj:`None`)++ Returns:+ Union[PyTorchDataset, torch.utils.data.DataLoader]: Dataset in+ PyTorch format or DataLoader if batch_size is provided.+ """+ dataset = PyTorchDataset.from_datapoints(+ self.data,+ transform=transform,+ target_transform=target_transform,+ )++ if batch_size is not None:+ return dataset.get_dataloader(batch_size=batch_size)++ return dataset+++class SeedDataset(BaseDataset):+ r"""A dataset containing validated seed examples for data generation.+ Ensures that all items adhere to the DataPoint schema.++ This class is used to initialize a dataset from a list of dictionary items,+ validating each against the DataPoint schema.+ """++ def __init__(+ self,+ data: List[Dict[str, str]],+ cache_dir: Optional[str] = None,+ min_samples: int = 1,+ **kwargs,+ ):+ r"""Initialize the seed dataset.++ Args:+ data (List[Dict[str, str]]): List of dictionary items to create the+ dataset from.+ cache_dir (Optional[str]): Directory to cache dataset files.+ (default: :obj:`None`)+ min_samples (int): Minimum number of samples required.+ (default: :obj:`1`)+ **kwargs: Additional dataset parameters.++ Raises:+ ValueError: If dataset size is less than min_samples or if sample+ validation fails.+ """+ if len(data) < min_samples:+ raise ValueError(+ f"Seed dataset must contain at least {min_samples} samples."+ )++ super().__init__(+ data=data,+ cache_dir=cache_dir,+ **kwargs,+ )+++class SyntheticDataset(BaseDataset):+ r"""A dataset for storing synthetically generated data points.++ This class is used to store datapoints that are generated through+ a generative process, such as using an agent.+ """++ def __init__(+ self,+ data: Optional[List[Dict[str, str]]] = None,+ cache_dir: Optional[str] = None,+ **kwargs,+ ):+ r"""Initialize the synthetic dataset.++ Args:+ data (Optional[List[Dict[str, str]]]): List of dictionary items to+ create the dataset from. (default: :obj:`None`)+ cache_dir (Optional[str]): Directory to cache dataset files.+ (default: :obj:`None`)+ **kwargs: Additional dataset parameters.+ """+ super().__init__(+ data=data if data is not None else [],+ cache_dir=cache_dir,+ **kwargs,+ )+ self.data: List[DataPoint] = []++ def add(self, item: DataPoint) -> None:+ r"""Add a new data point to the dataset.++ Args:+ item (DataPoint): The datapoint to add to the dataset.+ """+ self.data.append(item)++ def add_batch(self, items: List[DataPoint]) -> None:+ r"""Add multiple data points to the dataset.++ Args:+ items (List[DataPoint]): The datapoints to add to the dataset.+ """+ self.data.extend(items)++ def to_pytorch_dataset(+ self,+ transform: Optional[Callable] = None,+ target_transform: Optional[Callable] = None,+ batch_size: Optional[int] = None,+ ) -> Union["PyTorchDataset", "DataLoader"]:+ r"""Convert to a PyTorch dataset or DataLoader.++ Args:+ transform (Optional[Callable]): Transform to apply to samples.+ target_transform (Optional[Callable]): Transform to apply to+ targets.+ batch_size (Optional[int]): If provided, returns a DataLoader with+ the specified batch size instead of a PyTorchDataset.+ (default: :obj:`None`)++ Returns:+ Union[PyTorchDataset, torch.utils.data.DataLoader]: Dataset in+ PyTorch format or DataLoader if batch_size is provided.+ """+ return convert_synthetic_to_pytorch(+ self,+ transform=transform,+ target_transform=target_transform,+ batch_size=batch_size,+ )++ def save_pytorch_format(self, path: str, compression: bool = True) -> None:+ r"""Save the dataset to disk in PyTorch format.++ Args:+ path (str): Path to save the dataset to.+ compression (bool): Whether to use compression to reduce file size.+ (default: :obj:`True`)+ """+ save_synthetic_dataset(self, path, compression=compression)++ def filter(+ self, predicate: Callable[[DataPoint], bool]+ ) -> 'SyntheticDataset':+ r"""Filter the dataset using a predicate function.++ Args:+ predicate (Callable[[DataPoint], bool]): Function that takes a+ DataPoint and returns True if it should be kept, False+ otherwise.++ Returns:+ SyntheticDataset: A new dataset containing only the filtered items.+ """+ filtered_data = [dp for dp in self.data if predicate(dp)]++ # Create a new dataset with the filtered data+ new_dataset = SyntheticDataset()+ new_dataset.add_batch(filtered_data)++ return new_dataset+++class GenerativeDataset(BaseDataset):+ r"""A dataset for generating synthetic datapoints using external agents and+ verifiers.++ This class leverages a seed dataset and external components to generate+ new synthetic datapoints on demand.+ """++ def __init__(+ self,+ seed_dataset: SeedDataset,+ verifier: BaseVerifier,+ agent: ChatAgent,+ cache_dir: Optional[str] = None,+ seed: int = 42,+ **kwargs,+ ):+ r"""Initialize the generative dataset.++ Args:+ seed_dataset (SeedDataset): Validated dataset to use for examples.+ verifier (BaseVerifier): Verifier to validate generated content.+ agent (ChatAgent): Agent to generate new datapoints.+ cache_dir (Optional[str]): Directory to cache dataset files.+ (default: :obj:`None`)+ seed (int): Random seed for reproducibility. (default: :obj:`42`)+ **kwargs: Additional dataset parameters.+ """+ # Initialize with empty data since we'll generate content dynamically+ super().__init__(+ data=[],+ cache_dir=cache_dir,+ **kwargs,+ )++ self.seed_dataset = seed_dataset+ self.verifier = verifier+ self.agent = agent++ self.seed = seed+ random.seed(self.seed)++ def _construct_prompt(self, examples: List[DataPoint]) -> str:+ r"""Construct a prompt for generating new datapoints.++ Args:+ examples (List[DataPoint]): Examples to include in the prompt.++ Returns:+ str: Formatted prompt with examples.+ """+ prompt = (+ "Generate a new datapoint similar to the following examples:\n\n"+ )+ for i, example in enumerate(examples, 1):+ prompt += f"Example {i}:\n"+ prompt += f"Question: {example.question}\n"+ prompt += f"Rationale: {example.rationale}\n"+ prompt += f"Final Answer: {example.final_answer}\n\n"+ prompt += "New datapoint:"+ return prompt++ async def generate_new(self, n: int) -> None:+ r"""Generate n new datapoints and add them to the dataset.++ Args:+ n (int): Number of valid datapoints to generate.++ This method generates new datapoints by:+ 1. Sampling examples from the seed dataset+ 2. Constructing a prompt for the agent+ 3. Generating a new datapoint using the agent+ 4. Verifying the generated datapoint with the verifier+ 5. Adding valid datapoints to the dataset+ """+ valid_data_points: List[DataPoint] = []++ while len(valid_data_points) < n:+ try:+ indices = random.sample(range(len(self.seed_dataset)), 3)+ examples = [self.seed_dataset[i] for i in indices]+ prompt = self._construct_prompt(examples)++ # Get agent response+ agent_output = (+ self.agent.step(prompt, response_format=DataPoint)+ .msgs[0]+ .parsed+ )++ if not isinstance(agent_output, dict):+ raise TypeError("Agent output must be a dictionary")+ if (+ 'question' not in agent_output+ or 'rationale' not in agent_output+ ):+ raise KeyError(+ "Agent output missing required keys: "+ "'question' or 'rationale'"+ )++ rationale = agent_output['rationale']++ # Verify the generated content+ verifier_response = await self.verifier.verify(rationale)+ if not hasattr(verifier_response, 'content'):+ raise AttributeError(+ "Verifier response missing 'content' attribute"+ )++ if not verifier_response.result:+ continue++ final_answer = verifier_response.result++ # Create and validate the new datapoint+ new_datapoint = {+ 'question': agent_output['question'],+ 'rationale': rationale,+ 'final_answer': final_answer,+ }++ datapoint = DataPoint(**new_datapoint)+ valid_data_points.append(datapoint)++ except (TypeError, KeyError, AttributeError, ValidationError) as e:+ logger.warning(+ f"Error encountered during generation: {e}, retrying..."+ )++ # Add all valid datapoints to the dataset+ for datapoint in valid_data_points:+ self.data.append(datapoint)+ logger.debug("Added new datapoint to dataset")+++# Define a type variable for return type flexibility+T_co = TypeVar('T_co', covariant=True)+++class PyTorchDataset(Dataset[T_co]):+ r"""A PyTorch-compatible dataset implementation that leverages PyTorch's+ efficient data handling capabilities.+ """++ def __init__(+ self,+ data: List[Dict[str, Any]],+ transform: Optional[Callable] = None,+ target_transform: Optional[Callable] = None,+ validate: bool = True,+ ):+ r"""Initialize the PyTorch dataset.++ Args:+ data (List[Dict[str, Any]]): List of dictionary items to create+ the dataset from.+ transform (Optional[Callable]): A function/transform that takes a+ sample and returns a transformed version for features.+ (default: :obj:`None`)+ target_transform (Optional[Callable]): A function/transform that+ takes a target and returns a transformed version. (default:+ :obj:`None`)+ validate (bool): Whether to validate data points using DataPoint+ schema. (default: :obj:`True`)++ Raises:+ ValidationError: If validation is enabled and data doesn't match+ DataPoint schema.+ """+ self.transform = transform+ self.target_transform = target_transform++ # Validate and store data+ self._raw_data = data+ self.data = []++ if validate:+ for i, item in enumerate(self._raw_data):+ try:+ # Use DataPoint for validation only+ dp = DataPoint(**item)+ self.data.append(dp.to_dict())+ except ValidationError as e:+ logger.error(f"Sample {i} validation error: {e}")+ raise ValueError(f"Sample {i} validation error: {e}")+ else:+ # Skip validation and just store the data dictionaries+ self.data = [dict(item) for item in self._raw_data]++ def __getitem__(self, index: int) -> T_co:+ r"""Get an item from the dataset.++ Args:+ index (int): Index of the item to get.++ Returns:+ T_co: Item from the dataset, possibly transformed.+ """+ sample = self.data[index]++ # Apply transformations if provided+ if self.transform is not None:+ sample = self.transform(sample)++ return sample # type: ignore[return-value]++ def __len__(self) -> int:+ r"""Return the size of the dataset.++ Returns:+ int: Number of samples in the dataset.+ """+ return len(self.data)++ @classmethod+ def from_datapoints(+ cls, datapoints: List[DataPoint], **kwargs+ ) -> 'PyTorchDataset':+ r"""Create a PyTorchDataset from a list of DataPoints.++ Args:+ datapoints (List[DataPoint]): List of DataPoint objects.+ **kwargs: Additional arguments to pass to the constructor.++ Returns:+ PyTorchDataset: A new PyTorchDataset instance.+ """+ data = [dp.to_dict() for dp in datapoints]+ # We can skip validation since datapoints are already validated+ return cls(data, validate=False, **kwargs)++ def to_hf_dataset(self) -> HFDataset:+ r"""Convert to a HuggingFace dataset.++ Returns:+ HFDataset: Dataset in HuggingFace format.+ """+ return HFDataset.from_list(self.data)++ def save_to_disk(self, path: str) -> None:+ r"""Save the dataset to disk using PyTorch.++ Args:+ path (str): Path to save the dataset to.+ """+ torch.save(self.data, path)++ @classmethod+ def load_from_disk(+ cls,+ path: str,+ transform: Optional[Callable] = None,+ target_transform: Optional[Callable] = None,+ ) -> 'PyTorchDataset':+ r"""Load a dataset from disk.++ Args:+ path (str): Path to load the dataset from.+ transform (Optional[Callable]): Transform to apply to samples.+ (default: :obj:`None`)+ target_transform (Optional[Callable]): Transform to apply to+ targets. (default: :obj:`None`)++ Returns:+ PyTorchDataset: Loaded dataset.+ """+ data = torch.load(path)+ return cls(+ data,+ transform=transform,+ target_transform=target_transform,+ validate=False,+ )++ @staticmethod+ def collate_fn(+ batch: List[Dict[str, Any]],+ ) -> Dict[str, Union[List[Any], torch.Tensor]]:+ r"""Collate function for PyTorch DataLoader.++ Args:+ batch (List[Dict[str, Any]]): Batch of samples from the dataset.++ Returns:+ Dict[str, Union[List[Any], torch.Tensor]]: Collated batch with+ tensors for numerical data.+ """+ if not batch:+ return {}++ # Initialize result dictionary with keys from first item - start with+ # lists only+ result: Dict[str, List[Any]] = {k: [] for k in batch[0].keys()}++ # Collect values by key+ for item in batch:+ for k, v in item.items():+ result[k].append(v)++ # Convert numeric/boolean lists to tensors where possible+ result_with_tensors: Dict[str, Union[List[Any], torch.Tensor]] = {}+ for k, v in result.items():+ if all(isinstance(x, (int, float, bool)) for x in v):+ try:+ result_with_tensors[k] = torch.tensor(v)+ except (ValueError, TypeError):+ # Keep as list if tensor conversion fails+ result_with_tensors[k] = v+ else:+ result_with_tensors[k] = v++ return result_with_tensors++ def get_dataloader(+ self,+ batch_size: int = 32,+ shuffle: bool = True,+ num_workers: int = 0,+ pin_memory: bool = False,+ **kwargs,+ ) -> "DataLoader":+ r"""Create a PyTorch DataLoader for this dataset.++ Args:+ batch_size (int): Batch size. (default: :obj:`32`)+ shuffle (bool): Whether to shuffle the dataset. (default:+ :obj:`True`)+ num_workers (int): Number of workers for data loading. (default:+ :obj:`0`)+ pin_memory (bool): Whether to pin memory for faster GPU transfer.+ (default: :obj:`False`)+ **kwargs: Additional arguments to pass to DataLoader.++ Returns:+ torch.utils.data.DataLoader: DataLoader for this dataset.+ """+ from torch.utils.data import DataLoader++ return DataLoader(+ self,+ batch_size=batch_size,+ shuffle=shuffle,+ num_workers=num_workers,+ collate_fn=self.collate_fn,+ pin_memory=pin_memory,+ **kwargs,+ )+++def convert_hf_to_pytorch(+ hf_dataset: HFDataset,+ transform: Optional[Callable] = None,+ target_transform: Optional[Callable] = None,+ column_mapping: Optional[Dict[str, str]] = None,+ validate: bool = True,+ batch_size: Optional[int] = None,+) -> Union["PyTorchDataset", "DataLoader"]:+ r"""Convert a HuggingFace dataset to a PyTorchDataset or DataLoader.++ This function maps HuggingFace dataset columns to the expected DataPoint+ format, validates the data, and creates a PyTorchDataset or DataLoader.++ Args:+ hf_dataset (HFDataset): HuggingFace dataset to convert.+ transform (Optional[Callable]): Transform to apply to samples.+ target_transform (Optional[Callable]): Transform to apply to targets.+ column_mapping (Optional[Dict[str, str]]): Mapping from HuggingFace+ column names to DataPoint field names. If None, assumes columns+ already match DataPoint fields.+ validate (bool): Whether to validate data points using DataPoint+ schema. (default: :obj:`True`)+ batch_size (Optional[int]): If provided, returns a DataLoader with the+ specified batch size instead of a PyTorchDataset. (default:+ :obj:`None`)++ Returns:+ Union[PyTorchDataset, torch.utils.data.DataLoader]: Converted dataset+ or DataLoader if batch_size is provided.+ """+ # Convert HuggingFace dataset to list of dicts more efficiently+ mapped_dataset = []++ for i in range(len(hf_dataset)):+ item = hf_dataset[i]+ if column_mapping is not None:+ # Apply column mapping if provided+ mapped_item = {}+ for hf_col, dp_field in column_mapping.items():+ if hf_col in item:+ mapped_item[dp_field] = item[hf_col]+ mapped_dataset.append(mapped_item)+ else:+ # Otherwise use item directly+ mapped_dataset.append(dict(item))++ # Create PyTorchDataset+ dataset: PyTorchDataset = PyTorchDataset(+ mapped_dataset,+ transform=transform,+ target_transform=target_transform,+ validate=validate,+ )++ # Return DataLoader if batch_size is provided+ if batch_size is not None:+ return dataset.get_dataloader(batch_size=batch_size)++ return dataset+++def convert_synthetic_to_pytorch(+ synthetic_dataset: 'SyntheticDataset',+ transform: Optional[Callable] = None,+ target_transform: Optional[Callable] = None,+ batch_size: Optional[int] = None,+) -> Union["PyTorchDataset", "DataLoader"]:+ r"""Convert a SyntheticDataset to a PyTorchDataset or DataLoader.++ Args:+ synthetic_dataset (SyntheticDataset): Synthetic dataset to convert.+ transform (Optional[Callable]): Transform to apply to samples.+ target_transform (Optional[Callable]): Transform to apply to targets.+ batch_size (Optional[int]): If provided, returns a DataLoader with the+ specified batch size instead of a PyTorchDataset. (default:+ :obj:`None`)++ Returns:+ Union[PyTorchDataset, torch.utils.data.DataLoader]: Converted dataset+ or DataLoader if batch_size is provided.+ """+ dataset = PyTorchDataset.from_datapoints(+ synthetic_dataset.data,+ transform=transform,+ target_transform=target_transform,+ )++ # Return DataLoader if batch_size is provided+ if batch_size is not None:+ return dataset.get_dataloader(batch_size=batch_size)++ return dataset+++def save_synthetic_dataset(+ synthetic_dataset: 'SyntheticDataset',+ path: str,+ compression: bool = True,+) -> None:+ r"""Save a synthetic dataset to disk using PyTorch format.++ Args:+ synthetic_dataset (SyntheticDataset): Dataset to save.+ path (str): Path to save the dataset to.+ compression (bool): Whether to use compression to reduce file size.+ (default: :obj:`True`)+ """+ pytorch_dataset = convert_synthetic_to_pytorch(synthetic_dataset)++ # Save with compression if enabled (uses less disk space)+ if compression:+ torch.save(+ pytorch_dataset.data, # type: ignore[union-attr]+ path,+ _use_new_zipfile_serialization=True,+ )+ else:+ pytorch_dataset.save_to_disk(path) # type: ignore[union-attr]+++def load_pytorch_dataset(+ path: str,+ transform: Optional[Callable] = None,+ target_transform: Optional[Callable] = None,+ batch_size: Optional[int] = None,+) -> Union["PyTorchDataset", "DataLoader"]:+ r"""Load a PyTorchDataset from disk.++ Args:+ path (str): Path to load the dataset from.+ transform (Optional[Callable]): Transform to apply to samples.+ target_transform (Optional[Callable]): Transform to apply to targets.+ batch_size (Optional[int]): If provided, returns a DataLoader with the+ specified batch size instead of a PyTorchDataset. (default:+ :obj:`None`)++ Returns:+ Union[PyTorchDataset, torch.utils.data.DataLoader]: Loaded dataset or+ DataLoader if batch_size is provided.+ """+ dataset = PyTorchDataset.load_from_disk(+ path, transform=transform, target_transform=target_transform+ )++ # Return DataLoader if batch_size is provided+ if batch_size is not None:+ return dataset.get_dataloader(batch_size=batch_size)++ return datasetdiff --git a/camel/environments/__init__.py b/camel/environments/__init__.pynew file mode 100644index 000000000..b0bbfb9a3--- /dev/null+++ b/camel/environments/__init__.py@@ -0,0 +1,16 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from .base import BaseEnvironment++__all__ = ["BaseEnvironment"]diff --git a/camel/environments/base.py b/camel/environments/base.pynew file mode 100644index 000000000..9b187d88a--- /dev/null+++ b/camel/environments/base.py@@ -0,0 +1,503 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import asyncio+from abc import ABC, abstractmethod+from datetime import datetime, timezone+from typing import Any, Dict, List, Optional, Tuple++from pydantic import BaseModel, Field++from camel.agents import ChatAgent+from camel.datasets.base import BaseDataset, GenerativeDataset+from camel.extractors.base import BaseExtractor+from camel.logger import get_logger+from camel.verifiers.base import (+ BaseVerifier,+ VerificationResult,+)+from camel.verifiers.models import (+ VerificationOutcome,+ VerifierInput,+)++logger = get_logger(__name__)++# TODO: Add MachineInfo into this file+# TODO: TeacherAgent should be renamed into neural_reward_model.+# This is where PRMs or such could be useful.+# Should probably be its own class and not just raw ChatAgent+++class Action(BaseModel):+ r"""Represents an action taken in an environment.++ This class defines the input context, the LLM-generated output, and+ metadata required for verification and tracking within an RL+ framework.++ Attributes:+ problem_statement (str): The task or query given to the LLM as+ input.+ llm_response (str): The response generated by the LLM.+ final_answer (Optional[str]): The reference solution, if+ available, used for supervised learning or evaluation.+ metadata (Dict[str, Any]): Additional metadata such as model+ parameters, prompt details, or response confidence scores.+ timestamp (datetime): The timestamp when the action was+ generated (UTC).+ """++ problem_statement: str = Field(description="Problem statement for the LLM")+ llm_response: str = Field(description="Generated response from the LLM")+ final_answer: Optional[str] = Field(+ None, description="Reference solution if available"+ )+ metadata: Dict[str, Any] = Field(+ default_factory=dict,+ description="Additional metadata about the generation",+ )+ timestamp: datetime = Field(+ default_factory=lambda: datetime.now(timezone.utc),+ description="When the response was generated (UTC)",+ )+++class Observation(BaseModel):+ r"""Environment observation.++ Attributes:+ question: The question posed to the LLM.+ context: Additional context for the question.+ metadata: Optional metadata about the observation.+ """++ question: str = Field(..., description="The question posed to the LLM")+ context: Dict[str, Any] = Field(+ default_factory=dict, description="Additional context for the question"+ )+ metadata: Optional[Dict[str, Any]] = Field(+ default=None, description="Optional metadata about the observation"+ )+++class StepResult(BaseModel):+ r"""Result of an environment step.++ Attributes:+ observation: The next observation.+ reward: Dictionary of reward scores for different aspects.+ done: Whether the episode is complete.+ info: Additional information about the step.+ """++ observation: Observation = Field(..., description="The next observation")+ reward: float = Field(..., description="Total reward of the action")+ rewards_dict: Dict[str, float] = Field(+ default_factory=dict,+ description="Dictionary of reward scores for different aspects",+ )+ done: bool = Field(..., description="Whether the episode is complete")+ info: Dict[str, Any] = Field(+ default_factory=dict,+ description="Additional information about the step",+ )+++class BaseEnvironment(ABC):+ r"""Base class for all RLVR training environments.++ An environment ties everything together. It:+ 1. Holds state and manages curriculum progression+ 2. Defines reward functions and hint generation+ 3. Manages dataset and task selection+ 4. Provides reset and step functions+ 5. Handles verifier setup and teardown+ 6. Enables proactive agent behavior+ 7. Supports practice environment creation+ 8. Facilitates chain-of-thought verification++ Key Features:+ - Curriculum learning with adaptive difficulty+ - Reward shaping based on solution quality+ - Hint generation from verified solutions+ - Task selection based on agent progress+ - Practice environment generation+ - Chain-of-thought validation+ """++ def __init__(+ self,+ dataset: BaseDataset,+ verifier: BaseVerifier,+ extractor: BaseExtractor,+ max_steps: Optional[int] = None,+ teacher_agent: Optional[ChatAgent] = None,+ curriculum_config: Optional[Dict[str, Any]] = None,+ practice_env_config: Optional[Dict[str, Any]] = None,+ **kwargs,+ ) -> None:+ r"""Initialize the environment.++ Args:+ dataset: Dataset to sample questions from.+ verifier: Verifier to check responses.+ extractor: Extractor to process LLM responses.+ max_steps: Maximum steps per episode.+ teacher_agent: Optional agent for reward shaping and hints+ curriculum_config: Configuration for curriculum learning including:+ - difficulty_levels: List of available difficulty levels+ - promotion_threshold: Score needed to advance+ - demotion_threshold: Score triggering level decrease+ - min_questions_per_level: Questions before promotion+ practice_env_config: Configuration for practice environments:+ - max_practice_envs: Maximum concurrent environments+ - difficulty_range: Allowed difficulty variation+ - focus_areas: Specific skills to practice+ **kwargs: Additional environment parameters.+ """+ self.dataset = dataset+ self.verifier = verifier+ self.extractor = extractor+ self.max_steps = max_steps+ self.teacher_agent = teacher_agent+ self._metadata = kwargs++ # State tracking+ self._is_setup: bool = False+ self._current_step: int = 0+ self._episode_ended: bool = False+ self._state: Dict[str, Any] = self._get_initial_state()+ self._last_observation: Optional[Observation] = None+ self._episode_history: List[Tuple[Observation, Action]] = []++ @abstractmethod+ async def setup(self) -> None:+ r"""Set up the environment, including verifier initialization."""+ if self._is_setup:+ return++ try:+ # Initialize core components+ if hasattr(self.verifier, 'setup'):+ await self.verifier.setup()+ if hasattr(self.dataset, 'setup'):+ await self.dataset.setup()+ if hasattr(self.extractor, 'setup'):+ await self.extractor.setup()++ # initialize agents if present+ if self.teacher_agent:+ await self.teacher_agent.reset()++ self._is_setup = True+ logger.info('Environment setup completed successfully')+ except Exception as e:+ logger.error(f'Failed to setup environment: {e}')+ raise++ @abstractmethod+ async def teardown(self) -> None:+ r"""Clean up resources, including verifier teardown."""+ if not self._is_setup:+ return++ try:+ # Cleanup components+ if hasattr(self.verifier, 'cleanup'):+ await self.verifier.cleanup()+ if hasattr(self.dataset, 'cleanup'):+ await self.dataset.cleanup()+ if hasattr(self.extractor, 'cleanup'):+ await self.extractor.cleanup()++ self._is_setup = False+ logger.info('Environment teardown completed successfully')+ except Exception as e:+ logger.error(f'Failed to teardown environment: {e}')+ raise++ @abstractmethod+ async def reset(self) -> Observation:+ r"""Reset the environment to initial state.++ Returns:+ Initial observation for the episode+ """++ if not self._is_setup:+ await self.setup()++ # Reset state+ self._current_step = 0+ self._episode_ended = False+ self._episode_history = []+ self._state = self._get_initial_state()++ # Get initial observation+ observation = self._get_next_observation()+ if observation is None:+ raise RuntimeError("Failed to get initial observation")++ self._last_observation = observation++ return observation++ @abstractmethod+ async def step(self, action: Action) -> StepResult:+ r"""Take a step in the environment.++ Args:+ action: Action containing everything that is needed+ to progress in the environment++ Returns:+ StepResult containing next observation, reward, done flag, and info+ """+ if self.max_steps and self._current_step >= self.max_steps:+ return StepResult(+ observation=self._get_terminal_observation(),+ reward=0,+ rewards_dict={},+ done=True,+ info={"reason": "max_steps_reached"},+ )++ if not self._is_setup:+ raise RuntimeError("Environment not set up. Call setup() first.")+ if self._episode_ended:+ raise RuntimeError("Episode has ended. Call reset() first.")+ if self._last_observation is None:+ raise RuntimeError("No current observation. Call reset() first.")++ self._current_step += 1++ current_obs: Observation = self._last_observation+ self._episode_history.append((current_obs, action))++ # extract verifiable part from llm response+ extraction_result = await self.extractor.extract(action.llm_response)++ # TODO: extract executable llm response specifically++ # verify the extracted+ verification_result = await self.verifier.verify(+ VerifierInput(+ llm_response=extraction_result,+ ground_truth=action.final_answer,+ )+ )++ # compute rewards+ total_reward, rewards_dict = await self.compute_reward(+ action, extraction_result, verification_result+ )++ # check termination+ done = self._is_done()++ next_obs = (+ self._get_terminal_observation()+ if done+ else self._get_next_observation()+ )++ self._last_observation = next_obs+ self._episode_ended = done++ return StepResult(+ observation=next_obs,+ reward=total_reward,+ rewards_dict=rewards_dict,+ done=done,+ info={+ "extraction_result": extraction_result,+ "verification_result": verification_result,+ "step": self._current_step,+ "state": self._state,+ },+ )++ @abstractmethod+ def _get_initial_state(self) -> Dict[str, Any]:+ r"""Get initial environment state."""++ return {+ "current_datapoint": None,+ "attempts": 0,+ "success_rate": 0.0,+ "rewards": [],+ "termination_reason": None,+ }++ @abstractmethod+ def _get_next_observation(self) -> Observation:+ r"""Get the next observation for the environment.++ Returns:+ Observation for the next step+ """+ if not self.dataset or len(self.dataset) == 0:+ logger.warning(+ "Dataset is empty. Attempting to generate new data..."+ )+ if isinstance(self.dataset, GenerativeDataset):+ try:+ asyncio.run(+ self.dataset.generate_new(1)+ ) # Generate at least one datapoint+ logger.info("Generated new datapoint successfully.")+ except Exception as e:+ logger.error(f"Failed to generate new data: {e}")+ return self._get_terminal_observation()+ else:+ logger.error("Dataset is empty and not a GenerativeDataset.")+ return self._get_terminal_observation()++ try:+ # Ensure dataset is not empty after generation attempt+ if len(self.dataset) == 0:+ logger.error("Dataset is still empty after generation.")+ return self._get_terminal_observation()++ # Sample the next datapoint+ datapoint_idx = self._current_step % len(self.dataset)+ datapoint = self.dataset[datapoint_idx]++ if not datapoint:+ logger.error(f"Invalid datapoint at index {datapoint_idx}")+ return self._get_terminal_observation()++ self._state["current_datapoint"] = datapoint++ # Extract necessary attributes safely+ question = getattr(datapoint, "question", None)+ final_answer = getattr(datapoint, "final_answer", None)+ rationale = getattr(datapoint, "rationale", None)+ difficulty = getattr(datapoint, "difficulty", None)+ metadata = getattr(datapoint, "metadata", {})++ if not question or not final_answer:+ logger.error(+ f"Datapoint at index {datapoint_idx} "+ "is missing required fields."+ )+ return self._get_terminal_observation()++ observation = Observation(+ question=question,+ context={+ "final_answer": final_answer,+ "difficulty": difficulty,+ "rationale": rationale,+ },+ metadata={+ "step": self._current_step,+ "datapoint_id": str(datapoint_idx),+ "verified": metadata.get("verified", False),+ **metadata,+ },+ )++ logger.debug(+ f"Generated observation for step {self._current_step}"+ )+ return observation++ except (IndexError, AttributeError) as e:+ logger.error(f"Error getting next observation: {e}")+ return self._get_terminal_observation()+ except Exception as e:+ logger.error(f"Unexpected error getting next observation: {e}")+ return self._get_terminal_observation()++ @abstractmethod+ def _get_terminal_observation(self) -> Observation:+ r"""Get the terminal observation when episode ends.++ Returns:+ Terminal observation+ """+ return Observation(+ question="Episode completed",+ context={},+ metadata={"terminal": True, "final_step": self._current_step},+ )++ @abstractmethod+ async def compute_reward(+ self,+ action: Action,+ extraction_result: str,+ verification_result: VerificationResult,+ ) -> Tuple[float, Dict[str, float]]:+ r"""Compute reward scores for different aspects of the response.++ Args:+ response: The response.+ extraction_result: Extracted information from response+ verification_result: Result from the verifier.++ Returns:+ - Total reward+ - Dictionary of reward scores for different aspects.+ """+ rewards: Dict[str, float] = {}++ # Get success from verification result status+ verification_success = float(+ verification_result.status == VerificationOutcome.SUCCESS+ )+ rewards["correctness"] = 1.0 if verification_success > 0.5 else 0.0++ # Update state+ self._state["rewards"].append(rewards)+ total_attempts = self._state["attempts"] + 1+ self._state["success_rate"] = (+ self._state["success_rate"] * (total_attempts - 1)+ + verification_success+ ) / total_attempts++ further_rewards = await self._compute_reward(+ action, extraction_result, verification_result+ )++ rewards = rewards | further_rewards++ return sum(rewards.values()), rewards++ @abstractmethod+ async def _compute_reward(+ self,+ action: Action,+ extraction_result: str,+ verification_result: VerificationResult,+ ) -> Dict[str, float]:+ pass++ def _is_done(self) -> bool:+ r"""Check if episode should terminate."""+ if self.max_steps and self._current_step >= self.max_steps:+ return True+ return False++ @property+ def metadata(self) -> Dict[str, Any]:+ r"""Get environment metadata."""+ return self._metadata.copy()++ @property+ def current_step(self) -> int:+ r"""Get current step number."""+ return self._current_stepdiff --git a/camel/extractors/__init__.py b/camel/extractors/__init__.pynew file mode 100644index 000000000..2f77f0480--- /dev/null+++ b/camel/extractors/__init__.py@@ -0,0 +1,16 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from .base import BaseExtractor++__all__ = ["BaseExtractor"]diff --git a/camel/extractors/base.py b/camel/extractors/base.pynew file mode 100644index 000000000..b9e076358--- /dev/null+++ b/camel/extractors/base.py@@ -0,0 +1,263 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from abc import ABC, abstractmethod+from types import TracebackType+from typing import Any, Dict, Optional, Type++from typing_extensions import Self++from camel.logger import get_logger+from camel.utils import BatchProcessor++logger = get_logger(__name__)+++class BaseExtractor(ABC):+ r"""Base class for all response extractors.++ An extractor takes the response and extracts the relevant parts,+ converting them into a format that the verifier can handle.+ Implements async context manager protocol for proper resource management.+ """++ def __init__(+ self,+ cache_templates: bool = True,+ max_cache_size: int = 1000,+ extraction_timeout: float = 30.0,+ batch_size: int = 10,+ monitoring_interval: float = 5.0,+ cpu_threshold: float = 80.0,+ memory_threshold: float = 85.0,+ **kwargs,+ ):+ r"""Initialize the extractor.++ Args:+ cache_templates (bool): Whether to cache extraction templates.+ (default: :obj:`True`)+ max_cache_size (int): Maximum number of templates to cache.+ (default: :obj:`1000`)+ extraction_timeout (float): Maximum time for extraction in seconds.+ (default: :obj:`30.0`)+ batch_size (int): Size of batches for parallel extraction.+ (default: :obj:`10`)+ monitoring_interval (float): Interval in seconds between resource+ checks. (default: :obj:`5.0`)+ cpu_threshold (float): CPU usage percentage threshold for scaling+ down. (default: :obj:`80.0`)+ memory_threshold (float): Memory usage percentage threshold for+ scaling down. (default: :obj:`85.0`)+ **kwargs: Additional extractor parameters.++ Raises:+ ValueError: If invalid parameter values are provided+ """+ # Store all parameters in metadata dict for compatibility+ self._metadata = {+ 'cache_templates': cache_templates,+ 'max_cache_size': max_cache_size,+ 'extraction_timeout': extraction_timeout,+ 'batch_size': batch_size,+ 'monitoring_interval': monitoring_interval,+ 'cpu_threshold': cpu_threshold,+ 'memory_threshold': memory_threshold,+ **kwargs,+ }++ self._is_setup = False+ self._cache: Dict[str, Any] = {}+ self._batch_processor: Optional[BatchProcessor] = None++ # Store configuration parameters+ self._cache_templates = cache_templates+ self._max_cache_size = max_cache_size+ self._extraction_timeout = extraction_timeout+ self._batch_size = batch_size+ self._monitoring_interval = monitoring_interval+ self._cpu_threshold = cpu_threshold+ self._memory_threshold = memory_threshold++ async def setup(self) -> None:+ r"""Set up the extractor with necessary resources.++ This method:+ 1. Initializes template cache if enabled+ 2. Sets up any parallel processing resources+ 3. Validates extraction patterns++ Raises:+ RuntimeError: If initialization fails+ """+ if self._is_setup:+ logger.debug(f"{self.__class__.__name__} already initialized")+ return++ try:+ # Initialize template cache if enabled+ if self._cache_templates:+ self._template_cache: Dict[str, Any] = {}++ # Set up batch processing if needed+ if self._batch_size > 1:+ self._batch_processor = BatchProcessor(+ initial_batch_size=self._batch_size,+ monitoring_interval=self._monitoring_interval,+ cpu_threshold=self._cpu_threshold,+ memory_threshold=self._memory_threshold,+ )++ self._is_setup = True+ logger.info(f"{self.__class__.__name__} initialized successfully")++ except Exception as e:+ error_msg = f"Error during {self.__class__.__name__} setup: {e}"+ logger.error(error_msg)+ await self.cleanup()+ raise RuntimeError(error_msg) from e++ async def cleanup(self) -> None:+ r"""Clean up extractor resources.++ This method handles cleanup of resources and resets the extractor+ state.+ It ensures:+ 1. All resources are properly released+ 2. Template cache is cleared+ 3. Parallel processing resources are shutdown+ 4. State is reset to initial+ 5. Cleanup happens even if errors occur++ Raises:+ RuntimeError: If cleanup fails (after resetting initialization+ state).+ """+ if not self._is_setup:+ logger.debug(+ f"{self.__class__.__name__} not initialized, skipping cleanup"+ )+ return++ errors = []+ try:+ # Clear template cache+ if hasattr(self, '_template_cache'):+ try:+ self._template_cache.clear()+ except Exception as e:+ errors.append(f"Failed to clear template cache: {e}")++ # Shutdown parallel processing+ if self._batch_processor is not None:+ try:+ # Get final performance metrics before cleanup+ metrics = self._batch_processor.get_performance_metrics()+ logger.info(f"Batch processor final metrics: {metrics}")+ except Exception as e:+ errors.append(+ f"Failed to get batch processor metrics: {e}"+ )++ # Preserve init config in metadata+ self._metadata = {+ 'cache_templates': self._cache_templates,+ 'max_cache_size': self._max_cache_size,+ 'extraction_timeout': self._extraction_timeout,+ 'batch_size': self._batch_size,+ }++ if not errors:+ logger.info(+ f"{self.__class__.__name__} cleaned up successfully"+ )++ except Exception as e:+ errors.append(f"Unexpected error during cleanup: {e}")++ finally:+ # Always mark as uninitialized, even if cleanup fails+ self._is_setup = False+ self._batch_processor = None++ if errors:+ error_msg = (+ f"Errors during {self.__class__.__name__} cleanup: "+ f"{'; '.join(errors)}"+ )+ logger.error(error_msg)+ raise RuntimeError(error_msg)++ async def __aenter__(self) -> Self:+ r"""Async context manager entry.++ Returns:+ Self reference for context manager usage.+ """+ await self.setup()+ return self++ async def __aexit__(+ self,+ exc_type: Optional[Type[BaseException]],+ exc_val: Optional[BaseException],+ exc_tb: Optional[TracebackType],+ ) -> None:+ r"""Async context manager exit.++ Args:+ exc_type (Optional[Type[BaseException]]): Exception type if an+ error occurred.+ exc_val (Optional[BaseException]): Exception value if an error+ occurred.+ exc_tb (Optional[TracebackType]): Exception traceback if an error+ occurred.+ """+ await self.cleanup()++ @abstractmethod+ async def extract(+ self, response: str, context: Optional[Dict[str, Any]] = None+ ) -> str:+ r"""Extract relevant parts from a response.++ Extracts:+ 1. Final answer or output+ 2. Chain of thought reasoning steps+ 3. Difficulty assessment++ Args:+ response (str): Raw response from agent generation.+ context (Optional[Dict[str, Any]]): Optional context for+ extraction like:+ - final_answer+ - rationale+ - complexity++ Returns:+ str: Extracted content string.++ Raises:+ ValueError: If response is empty or invalid.+ NotImplementedError: If no implementation is provided.+ RuntimeError: If extractor is not initialized.+ """+ if not self._is_setup:+ raise RuntimeError(+ f"{self.__class__.__name__} must be initialized "+ "before extraction"+ )+ if not response or not response.strip():+ raise ValueError("Empty or whitespace-only response")+ raise NotImplementedError("Subclasses must implement extract()")diff --git a/camel/verifiers/__init__.py b/camel/verifiers/__init__.pynew file mode 100644index 000000000..19d67579f--- /dev/null+++ b/camel/verifiers/__init__.py@@ -0,0 +1,17 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from .base import BaseVerifier+from .models import VerificationOutcome, VerifierInput++__all__ = ["BaseVerifier", "VerificationOutcome", "VerifierInput"]diff --git a/camel/verifiers/base.py b/camel/verifiers/base.pynew file mode 100644index 000000000..f96c8825a--- /dev/null+++ b/camel/verifiers/base.py@@ -0,0 +1,340 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import asyncio+import time+from abc import ABC, abstractmethod+from typing import List, Optional++from camel.logger import get_logger+from camel.utils import BatchProcessor++from .models import (+ VerificationOutcome,+ VerificationResult,+ VerifierInput,+)++logger = get_logger(__name__)+++class BaseVerifier(ABC):+ r"""Base class for all verifiers.++ Example:+ ```python+ verifier = MyVerifier()+ await verifier.setup()+ result = await verifier.verify(response)+ await verifier.cleanup()+ ```++ Key Features:+ - Async verification with retry logic+ - Comprehensive error handling and logging+ - Configurable batch processing+ - Resource monitoring for adaptive scaling+ """++ def __init__(+ self,+ max_parallel: Optional[int] = None,+ timeout: Optional[float] = None,+ max_retries: int = 3,+ retry_delay: float = 1.0,+ initial_batch_size: Optional[int] = None,+ cpu_threshold: float = 80.0,+ memory_threshold: float = 85.0,+ **kwargs,+ ):+ r"""Initialize the verifier with configuration parameters.++ Args:+ max_parallel: Maximum number of parallel verifications. If None,+ determined dynamically based on system resources.+ (default: :obj:`None`)+ timeout: Timeout in seconds for each verification. (default:+ :obj:`None`)+ max_retries: Maximum number of retry attempts. (default: :obj:`3`)+ retry_delay: Delay between retries in seconds. (default:+ :obj:`1.0`)+ initial_batch_size: Initial size for batch processing. If None,+ defaults to 10. (default: :obj:`None`)+ cpu_threshold: CPU usage percentage threshold for scaling down.+ (default: :obj:`80.0`)+ memory_threshold: Memory usage percentage threshold for scaling+ down. (default: :obj:`85.0`)+ **kwargs: Additional verifier parameters.+ """+ self._is_setup: bool = False+ self._max_parallel: Optional[int] = max_parallel+ self._timeout: Optional[float] = timeout+ self._max_retries: int = max_retries+ self._retry_delay: float = retry_delay+ self._initial_batch_size: Optional[int] = initial_batch_size+ self._cpu_threshold: float = cpu_threshold+ self._memory_threshold: float = memory_threshold+ self._batch_processor: BatchProcessor = BatchProcessor()++ async def setup(self) -> None:+ r"""Set up the verifier with necessary resources.++ Initializes:+ 1. Batch processor with validated parameters+ 2. Any verifier-specific resources++ Raises:+ RuntimeError: If setup fails or resources cannot be initialized.+ """+ if self._is_setup:+ logger.debug(f"{self.__class__.__name__} already initialized")+ return++ try:+ batch_size = max(1, self._initial_batch_size or 10)+ max_parallel = max(1, self._max_parallel or 1)+ self._batch_processor = BatchProcessor()++ logger.info(+ f"{self.__class__.__name__} initialized with "+ f"batch_size={batch_size}, max_parallel={max_parallel}"+ )++ await self._setup()+ self._is_setup = True++ except Exception as e:+ error_msg = (+ f"Failed to initialize {self.__class__.__name__}: {e!s}"+ )+ logger.error(error_msg, exc_info=True)+ await self.cleanup()+ raise RuntimeError(error_msg) from e++ @abstractmethod+ async def _setup(self) -> None:+ r"""Implement verifier-specific setup logic."""+ pass++ async def cleanup(self) -> None:+ r"""Clean up verifier resources.++ Ensures:+ 1. Batch processor is reset+ 2. All internal states are cleared++ Raises:+ RuntimeError: If cleanup fails.+ """+ if not self._is_setup:+ return++ try:+ self._batch_processor = BatchProcessor()+ await self._cleanup()+ logger.info(f"{self.__class__.__name__} cleaned up successfully")++ except Exception as e:+ error_msg = f"Failed to cleanup {self.__class__.__name__}: {e!s}"+ logger.error(error_msg, exc_info=True)+ raise RuntimeError(error_msg) from e++ finally:+ self._is_setup = False++ @abstractmethod+ async def _cleanup(self) -> None:+ r"""Implement verifier-specific cleanup logic."""+ pass++ async def verify(self, result: VerifierInput) -> VerificationResult:+ r"""Perform verification with full error handling.++ Verifies correctness, expected output, reasoning, and symbolic+ consistency.++ Args:+ result: The response to verify.++ Returns:+ VerificationResult: Structured result containing:+ - status: SUCCESS/FAILURE/ERROR/TIMEOUT+ - result: Verification outcome description+ - duration: Time taken for verification+ - metadata: Additional details+ - error_message: Error description if applicable++ Raises:+ RuntimeError: If verification fails unexpectedly.+ asyncio.TimeoutError: If verification times out.+ """+ if not self._is_setup:+ logger.warning(+ f"{self.__class__.__name__} not set up, calling setup()"+ )+ await self.setup()++ attempt = 0+ start_time = time.time()++ while attempt < self._max_retries:+ try:+ verification_result = (+ await asyncio.wait_for(+ self._verify_implementation(result),+ timeout=self._timeout,+ )+ if self._timeout+ else await self._verify_implementation(result)+ )++ verification_result.duration = time.time() - start_time+ verification_result.metadata["attempt"] = attempt + 1+ return verification_result++ except asyncio.TimeoutError:+ attempt += 1+ if attempt == self._max_retries:+ return VerificationResult(+ status=VerificationOutcome.TIMEOUT,+ result="",+ error_message="Verification timed out "+ "after all retries.",+ duration=time.time() - start_time,+ metadata={"attempt": attempt},+ )+ logger.warning(+ f"Verification timeout on attempt {attempt}, retrying..."+ )+ await asyncio.sleep(self._retry_delay)++ except Exception as e:+ attempt += 1+ if attempt == self._max_retries:+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result="",+ error_message=f"Verification failed: {e!s}",+ duration=time.time() - start_time,+ metadata={"attempt": attempt},+ )+ await asyncio.sleep(self._retry_delay)++ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result="",+ error_message="Unexpected code path reached",+ duration=time.time() - start_time,+ metadata={"attempt": attempt},+ )++ @abstractmethod+ async def _verify_implementation(+ self, result: VerifierInput+ ) -> VerificationResult:+ r"""Implement the actual verification logic.++ Args:+ result: The response to verify.++ Returns:+ VerificationResult: Containing the verification outcome.++ Raises:+ NotImplementedError: Must be implemented in subclasses.+ """+ raise NotImplementedError(+ "Subclasses must implement _verify_implementation()"+ )+++async def verify_batch(+ self, results: List[VerifierInput], raise_on_error: bool = False+) -> List[VerificationResult]:+ r"""Verify multiple results in parallel with controlled concurrency.++ Args:+ results: List of responses to verify.+ raise_on_error: Whether to raise an exception if any verification+ fails. (default: :obj:`False`)++ Returns:+ List[VerificationResult]: One for each input response.++ Raises:+ RuntimeError: If any verification fails and raise_on_error is True.+ asyncio.TimeoutError: If verifications time out and max retries+ exceeded.+ """+ if not self._is_setup:+ logger.warning(+ f"{self.__class__.__name__} not set up, calling setup()"+ )+ await self.setup()++ # Get current batch parameters from processor with defaults if not+ # present+ max_workers = getattr(+ self._batch_processor, 'max_workers', self._max_parallel or 1+ )+ batch_size = getattr(+ self._batch_processor, 'batch_size', self._initial_batch_size or 10+ )+ semaphore = asyncio.Semaphore(max(1, max_workers))++ async def _verify_with_semaphore(+ response: VerifierInput,+ ) -> VerificationResult:+ start_time = time.time()+ try:+ async with semaphore:+ verification_result = await self.verify(response)+ processing_time = time.time() - start_time+ success = verification_result.status == VerificationOutcome.SUCCESS+ self._batch_processor.adjust_batch_size(success, processing_time)+ return verification_result+ except Exception as e:+ processing_time = time.time() - start_time+ self._batch_processor.adjust_batch_size(False, processing_time)+ logger.error(f"Verification failed: {e!s}", exc_info=True)+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result="",+ error_message=str(e),+ metadata={"error_type": type(e).__name__},+ )++ # Process in batches+ all_results: List[VerificationResult] = []+ for i in range(0, len(results), batch_size):+ batch = results[i : i + batch_size]+ verification_tasks = [+ _verify_with_semaphore(result) for result in batch+ ]+ try:+ batch_results = await asyncio.gather(*verification_tasks)+ all_results.extend(batch_results)+ except Exception as e:+ logger.error(f"Batch verification failed: {e!s}", exc_info=True)+ if raise_on_error:+ raise RuntimeError(f"Batch verification failed: {e!s}") from e++ if raise_on_error and any(+ r.status in {VerificationOutcome.ERROR, VerificationOutcome.TIMEOUT}+ for r in all_results+ ):+ error_msg = "One or more verifications failed"+ logger.error(error_msg)+ raise RuntimeError(error_msg)++ return all_resultsdiff --git a/camel/verifiers/models.py b/camel/verifiers/models.pynew file mode 100644index 000000000..37bcbc478--- /dev/null+++ b/camel/verifiers/models.py@@ -0,0 +1,76 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from datetime import datetime+from enum import Enum+from typing import Any, Dict, Optional++from pydantic import BaseModel, Field+++class VerifierInput(BaseModel):+ llm_response: str = Field(+ description="The LLM response to be verified."+ "Needs to be in a format that the verifier can handle."+ )+ ground_truth: Optional[str] = Field(+ description="The ground truth data, if available."+ )+++class VerificationOutcome(Enum):+ r"""Enum representing the status of a verification."""++ SUCCESS = "success"+ FAILURE = "failure"+ ERROR = "error"+ TIMEOUT = "timeout"+++class VerificationResult(BaseModel):+ r"""Structured result from a verification."""++ status: VerificationOutcome = Field(+ description="Status of the verification"+ )+ result: str = Field(description="Verification result")+ duration: float = Field(+ default=0.0, description="Duration of verification in seconds"+ )+ timestamp: datetime = Field(+ default_factory=datetime.now,+ description="When the verification was performed",+ )+ metadata: Dict[str, Any] = Field(+ default_factory=dict,+ description="Additional metadata about the verification",+ )+ error_message: Optional[str] = Field(+ default=None, description="Error message if verification failed"+ )+++class VerifierConfig(BaseModel):+ r"""Configuration for verifier behavior."""++ enabled: bool = Field(True, description="Whether verification is enabled")+ strict_mode: bool = Field(+ False, description="Whether to fail on any validation error"+ )+ timeout: Optional[float] = Field(+ None, description="Verification timeout in seconds"+ )+ max_retries: int = Field(3, description="Maximum number of retry attempts")+ retry_delay: float = Field(+ 1.0, description="Delay between retries in seconds"+ )diff --git a/test/datasets/test_base_dataset.py b/test/datasets/test_base_dataset.pynew file mode 100644index 000000000..83db9ad60--- /dev/null+++ b/test/datasets/test_base_dataset.py@@ -0,0 +1,482 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import tempfile+from unittest.mock import AsyncMock, MagicMock++import pytest+import torch+from datasets import Dataset as HFDataset+from pydantic import ValidationError++from camel.datasets.base import (+ BaseDataset,+ DataPoint,+ GenerativeDataset,+ PyTorchDataset,+ SeedDataset,+ SyntheticDataset,+ convert_hf_to_pytorch,+ convert_synthetic_to_pytorch,+ load_pytorch_dataset,+ save_synthetic_dataset,+)+++# ruff: noqa: RUF001+def test_datapoint_creation():+ r"""Test creating a DataPoint with valid data."""+ data = {+ 'question': 'What is 2+2?',+ 'rationale': 'Addition of two numbers',+ 'final_answer': '4',+ }++ datapoint = DataPoint(**data)++ assert datapoint.question == 'What is 2+2?'+ assert datapoint.rationale == 'Addition of two numbers'+ assert datapoint.final_answer == '4'+ assert datapoint.difficulty is None+ assert datapoint.metadata is None+++def test_datapoint_validation():+ r"""Test DataPoint validation with missing required fields."""+ with pytest.raises(ValidationError):+ DataPoint(question='What is 2+2?')++ with pytest.raises(ValidationError):+ DataPoint(question='What is 2+2?', rationale='Addition')+++def test_datapoint_to_dict():+ r"""Test converting DataPoint to dictionary."""+ data = {+ 'question': 'What is 2+2?',+ 'rationale': 'Addition of two numbers',+ 'final_answer': '4',+ 'difficulty': 'easy',+ 'metadata': {'topic': 'math'},+ }++ datapoint = DataPoint(**data)+ data_dict = datapoint.to_dict()++ assert data_dict['question'] == 'What is 2+2?'+ assert data_dict['rationale'] == 'Addition of two numbers'+ assert data_dict['final_answer'] == '4'+ assert data_dict['difficulty'] == 'easy'+ assert data_dict['metadata'] == {'topic': 'math'}+++def test_datapoint_from_dict():+ r"""Test creating DataPoint from dictionary."""+ data = {+ 'question': 'What is 2+2?',+ 'rationale': 'Addition of two numbers',+ 'final_answer': '4',+ }++ datapoint = DataPoint.from_dict(data)++ assert datapoint.question == 'What is 2+2?'+ assert datapoint.rationale == 'Addition of two numbers'+ assert datapoint.final_answer == '4'+++def sample_data():+ r"""Fixture providing sample data for testing."""+ return [+ {+ 'question': 'What is 2+2?',+ 'rationale': 'Addition of two numbers',+ 'final_answer': '4',+ },+ {+ 'question': 'What is 3×3?',+ 'rationale': 'Multiplication of two numbers',+ 'final_answer': '9',+ },+ ]+++async def test_base_dataset_setup(sample_data):+ r"""Test BaseDataset setup process."""+ with tempfile.TemporaryDirectory() as temp_dir:+ dataset = BaseDataset(data=sample_data, cache_dir=temp_dir)+ await dataset.setup()++ assert dataset._is_setup+ assert len(dataset.data) == 2+ assert isinstance(dataset.data[0], DataPoint)+ assert dataset.data[0].question == 'What is 2+2?'+ assert dataset.data[1].final_answer == '9'+++async def test_base_dataset_empty_setup():+ r"""Test BaseDataset setup with empty data."""+ dataset = BaseDataset(data=None)+ await dataset.setup()++ assert dataset._is_setup+ assert len(dataset.data) == 0+++async def test_base_dataset_cleanup():+ r"""Test BaseDataset cleanup process."""+ dataset = BaseDataset(data=[])+ await dataset.setup()+ assert dataset._is_setup++ await dataset.cleanup()+ assert not dataset._is_setup+++def test_base_dataset_sample(sample_data):+ r"""Test sampling from BaseDataset."""+ dataset = BaseDataset(data=sample_data)+ dataset._is_setup = True+ dataset.data = [DataPoint(**item) for item in sample_data]++ sample = dataset.sample()+ assert isinstance(sample, DataPoint)+ assert sample in dataset.data+++def test_base_dataset_len(sample_data):+ r"""Test BaseDataset length."""+ dataset = BaseDataset(data=sample_data)+ dataset._is_setup = True+ dataset.data = [DataPoint(**item) for item in sample_data]++ assert len(dataset) == 2+++def test_base_dataset_getitem(sample_data):+ r"""Test BaseDataset item access."""+ dataset = BaseDataset(data=sample_data)+ dataset._is_setup = True+ dataset.data = [DataPoint(**item) for item in sample_data]++ item = dataset[0]+ assert isinstance(item, DataPoint)+ assert item.question == 'What is 2+2?'++ with pytest.raises(IndexError):+ _ = dataset[100] # Out of bounds+++def test_base_dataset_metadata():+ r"""Test BaseDataset metadata."""+ dataset = BaseDataset(data=[], cache_dir='/tmp', custom_param='value')++ metadata = dataset.metadata+ assert metadata['cache_dir'] == '/tmp'+ assert metadata['custom_param'] == 'value'+++def test_seed_dataset_init(sample_data):+ r"""Test SeedDataset initialization."""+ dataset = SeedDataset(data=sample_data, min_samples=1)+ assert dataset._raw_data == sample_data++ with pytest.raises(ValueError):+ SeedDataset(data=sample_data, min_samples=3)+++def test_synthetic_dataset_init():+ r"""Test SyntheticDataset initialization."""+ dataset = SyntheticDataset()+ assert dataset._raw_data == []+ assert dataset.data == []+++def test_synthetic_dataset_add():+ r"""Test adding items to SyntheticDataset."""+ dataset = SyntheticDataset()+ datapoint = DataPoint(+ question='What is 2+2?',+ rationale='Addition of two numbers',+ final_answer='4',+ )++ dataset.add(datapoint)+ assert len(dataset.data) == 1+ assert dataset.data[0] == datapoint+++def test_synthetic_dataset_add_batch():+ r"""Test adding batch of items to SyntheticDataset."""+ dataset = SyntheticDataset()+ datapoints = [+ DataPoint(+ question='What is 2+2?',+ rationale='Addition of two numbers',+ final_answer='4',+ ),+ DataPoint(+ question='What is 3×3?',+ rationale='Multiplication of two numbers',+ final_answer='9',+ ),+ ]++ dataset.add_batch(datapoints)+ assert len(dataset.data) == 2+ assert dataset.data == datapoints+++def test_synthetic_dataset_filter():+ r"""Test filtering SyntheticDataset."""+ dataset = SyntheticDataset()+ datapoints = [+ DataPoint(+ question='What is 2+2?',+ rationale='Addition of two numbers',+ final_answer='4',+ difficulty='easy',+ ),+ DataPoint(+ question='What is 3×3?',+ rationale='Multiplication of two numbers',+ final_answer='9',+ difficulty='medium',+ ),+ ]+ dataset.add_batch(datapoints)++ # Filter for easy questions+ filtered = dataset.filter(lambda dp: dp.difficulty == 'easy')+ assert len(filtered.data) == 1+ assert filtered.data[0].question == 'What is 2+2?'+++def test_pytorch_dataset_init(sample_data):+ r"""Test PyTorchDataset initialization."""+ dataset = PyTorchDataset(sample_data)+ assert len(dataset) == 2+ assert dataset[0]['question'] == 'What is 2+2?'+++def test_pytorch_dataset_from_datapoints():+ r"""Test creating PyTorchDataset from DataPoints."""+ datapoints = [+ DataPoint(+ question='What is 2+2?',+ rationale='Addition of two numbers',+ final_answer='4',+ ),+ DataPoint(+ question='What is 3×3?',+ rationale='Multiplication of two numbers',+ final_answer='9',+ ),+ ]++ dataset = PyTorchDataset.from_datapoints(datapoints)+ assert len(dataset) == 2+ assert dataset[0]['question'] == 'What is 2+2?'+ assert dataset[1]['final_answer'] == '9'+++def test_pytorch_dataset_transform():+ r"""Test PyTorchDataset with transform function."""+ data = [+ {+ 'question': 'What is 2+2?',+ 'rationale': 'Addition of two numbers',+ 'final_answer': '4',+ }+ ]++ def transform(sample):+ sample['question'] = sample['question'].upper()+ return sample++ dataset = PyTorchDataset(data, transform=transform)+ assert dataset[0]['question'] == 'WHAT IS 2+2?'+++def test_pytorch_dataset_collate_fn():+ r"""Test PyTorchDataset collate function."""+ batch = [+ {+ 'question': 'What is 2+2?',+ 'rationale': 'Addition of two numbers',+ 'final_answer': '4',+ 'numeric_value': 4,+ },+ {+ 'question': 'What is 3×3?',+ 'rationale': 'Multiplication of two numbers',+ 'final_answer': '9',+ 'numeric_value': 9,+ },+ ]++ result = PyTorchDataset.collate_fn(batch)++ # String fields should be lists+ assert isinstance(result['question'], list)+ assert result['question'] == ['What is 2+2?', 'What is 3×3?']++ # Numeric fields should be tensors+ assert isinstance(result['numeric_value'], torch.Tensor)+ assert torch.equal(result['numeric_value'], torch.tensor([4, 9]))+++def test_convert_hf_to_pytorch():+ r"""Test converting HuggingFace dataset to PyTorchDataset."""+ hf_data = [+ {+ 'q': 'What is 2+2?',+ 'r': 'Addition of two numbers',+ 'a': '4',+ },+ {+ 'q': 'What is 3×3?',+ 'r': 'Multiplication of two numbers',+ 'a': '9',+ },+ ]++ hf_dataset = HFDataset.from_list(hf_data)++ # Test with column mapping+ column_mapping = {'q': 'question', 'r': 'rationale', 'a': 'final_answer'}+ pytorch_dataset = convert_hf_to_pytorch(+ hf_dataset, column_mapping=column_mapping+ )++ assert len(pytorch_dataset) == 2+ assert pytorch_dataset[0]['question'] == 'What is 2+2?'+ assert pytorch_dataset[1]['final_answer'] == '9'+++def test_convert_synthetic_to_pytorch():+ r"""Test converting SyntheticDataset to PyTorchDataset."""+ synthetic_dataset = SyntheticDataset()+ datapoints = [+ DataPoint(+ question='What is 2+2?',+ rationale='Addition of two numbers',+ final_answer='4',+ ),+ DataPoint(+ question='What is 3×3?',+ rationale='Multiplication of two numbers',+ final_answer='9',+ ),+ ]+ synthetic_dataset.add_batch(datapoints)++ pytorch_dataset = convert_synthetic_to_pytorch(synthetic_dataset)++ assert len(pytorch_dataset) == 2+ assert pytorch_dataset[0]['question'] == 'What is 2+2?'+ assert pytorch_dataset[1]['final_answer'] == '9'+++def test_save_and_load_pytorch_dataset():+ r"""Test saving and loading PyTorchDataset."""+ with tempfile.NamedTemporaryFile(suffix='.pt') as temp_file:+ # Create and save dataset+ data = [+ {+ 'question': 'What is 2+2?',+ 'rationale': 'Addition of two numbers',+ 'final_answer': '4',+ }+ ]+ dataset = PyTorchDataset(data)+ dataset.save_to_disk(temp_file.name)++ # Load dataset+ loaded_dataset = load_pytorch_dataset(temp_file.name)++ assert len(loaded_dataset) == 1+ assert loaded_dataset[0]['question'] == 'What is 2+2?'+++def test_save_synthetic_dataset():+ r"""Test saving SyntheticDataset."""+ with tempfile.NamedTemporaryFile(suffix='.pt') as temp_file:+ # Create and save dataset+ synthetic_dataset = SyntheticDataset()+ datapoint = DataPoint(+ question='What is 2+2?',+ rationale='Addition of two numbers',+ final_answer='4',+ )+ synthetic_dataset.add(datapoint)++ save_synthetic_dataset(synthetic_dataset, temp_file.name)++ # Load dataset+ loaded_dataset = load_pytorch_dataset(temp_file.name)++ assert len(loaded_dataset) == 1+ assert loaded_dataset[0]['question'] == 'What is 2+2?'+++async def test_generative_dataset():+ r"""Test GenerativeDataset with mocked components."""+ mock_seed_dataset = MagicMock(spec=SeedDataset)+ mock_seed_dataset.__len__.return_value = 5+ mock_seed_dataset.__getitem__.side_effect = lambda i: DataPoint(+ question=f"Question {i}",+ rationale=f"Rationale {i}",+ final_answer=f"Answer {i}",+ )++ mock_verifier = MagicMock()+ mock_verifier.verify = AsyncMock()+ mock_verifier.verify.return_value = MagicMock(result="Verified Answer")++ mock_agent = MagicMock()+ mock_agent.step.return_value = MagicMock(+ msgs=[+ MagicMock(+ parsed={+ 'question': 'Generated Question',+ 'rationale': 'Generated Rationale',+ }+ )+ ]+ )++ # Create GenerativeDataset with mocks+ dataset = GenerativeDataset(+ seed_dataset=mock_seed_dataset,+ verifier=mock_verifier,+ agent=mock_agent,+ )++ # Test generate_new method+ await dataset.generate_new(2)++ # Verify interactions+ assert mock_agent.step.call_count >= 2+ assert mock_verifier.verify.await_count >= 2+ assert len(dataset.data) == 2++ # Verify generated data+ assert all(isinstance(dp, DataPoint) for dp in dataset.data)+ assert all(dp.final_answer == "Verified Answer" for dp in dataset.data)diff --git a/test/environments/test_base_env.py b/test/environments/test_base_env.pynew file mode 100644index 000000000..9ac90ab41--- /dev/null+++ b/test/environments/test_base_env.py@@ -0,0 +1,288 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from datetime import datetime, timezone+from typing import Dict, Tuple+from unittest.mock import AsyncMock, MagicMock++import pytest++from camel.environments.base import (+ Action,+ BaseEnvironment,+ Observation,+ StepResult,+)+from camel.verifiers.models import VerificationOutcome, VerificationResult+++def test_action_model():+ r"""Test the Action model initialization and properties."""+ custom_time = datetime(2023, 1, 1, tzinfo=timezone.utc)+ action_full = Action(+ problem_statement="Full problem",+ llm_response="Full response",+ final_answer="Reference solution",+ metadata={"model": "test-model", "temperature": 0.7},+ timestamp=custom_time,+ )+ assert action_full.problem_statement == "Full problem"+ assert action_full.llm_response == "Full response"+ assert action_full.final_answer == "Reference solution"+ assert action_full.metadata == {"model": "test-model", "temperature": 0.7}+ assert action_full.timestamp == custom_time+++def test_observation_model():+ r"""Test the Observation model initialization and properties."""+ obs_full = Observation(+ question="Complex math problem",+ context={"hint": "Use calculus"},+ metadata={"difficulty": "hard"},+ )+ assert obs_full.question == "Complex math problem"+ assert obs_full.context == {"hint": "Use calculus"}+ assert obs_full.metadata == {"difficulty": "hard"}+++def test_step_result_model():+ r"""Test the StepResult model initialization and properties."""+ obs = Observation(question="Next question")+ result_full = StepResult(+ observation=obs,+ reward=1.0,+ rewards_dict={"correctness": 0.8, "efficiency": 0.2},+ done=True,+ info={"reason": "task_complete"},+ )+ assert result_full.observation == obs+ assert result_full.reward == 1.0+ assert result_full.rewards_dict == {"correctness": 0.8, "efficiency": 0.2}+ assert result_full.done is True+ assert result_full.info == {"reason": "task_complete"}+++class MockBaseEnvironment(BaseEnvironment):+ r"""Mock implementation of BaseEnvironment for testing."""++ async def setup(self):+ await super().setup()++ async def teardown(self):+ await super().teardown()++ async def reset(self):+ return await super().reset()++ async def step(self, action):+ return await super().step(action)++ def _get_initial_state(self):+ return super()._get_initial_state()++ def _get_next_observation(self):+ return Observation(question="Test question")++ def _get_terminal_observation(self):+ return super()._get_terminal_observation()++ async def compute_reward(+ self,+ action: Action,+ extraction_result: str,+ verification_result: VerificationResult,+ ) -> Tuple[float, Dict[str, float]]:+ return await super().compute_reward(+ action, extraction_result, verification_result+ )++ async def _compute_reward(+ self, action, extraction_result, verification_result+ ):+ return {"additional_reward": 0.5}+++async def test_base_environment_lifecycle():+ r"""Test the lifecycle methods of BaseEnvironment."""+ mock_dataset = MagicMock()+ mock_dataset.__len__.return_value = 1+ mock_dataset.__getitem__.return_value = MagicMock(+ question="Test question",+ final_answer="Test answer",+ rationale="Test rationale",+ difficulty="medium",+ metadata={"verified": True},+ )+ mock_dataset.setup = AsyncMock()+ mock_dataset.cleanup = AsyncMock()++ mock_verifier = MagicMock()+ mock_verifier.setup = AsyncMock()+ mock_verifier.cleanup = AsyncMock()+ mock_verifier.verify = AsyncMock(+ return_value=VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Verification successful",+ feedback="Correct",+ score=1.0,+ )+ )++ mock_extractor = MagicMock()+ mock_extractor.setup = AsyncMock()+ mock_extractor.cleanup = AsyncMock()+ mock_extractor.extract = AsyncMock(return_value={"extracted": "content"})++ # Create environment+ env = MockBaseEnvironment(+ dataset=mock_dataset,+ verifier=mock_verifier,+ extractor=mock_extractor,+ max_steps=5,+ )++ # Test setup+ await env.setup()+ assert env._is_setup is True+ mock_verifier.setup.assert_awaited_once()++ # Test reset+ observation = await env.reset()+ assert isinstance(observation, Observation)+ assert env._current_step == 0+ assert env._episode_ended is False++ original_step = env.step++ async def mock_step(action):+ env._current_step += 1+ return StepResult(+ observation=env._get_next_observation(),+ reward=1.0,+ rewards_dict={"correctness": 1.0, "additional_reward": 0.5},+ done=False,+ info={+ "extraction_result": {"extracted": "content"},+ "verification_result": VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Verification successful",+ feedback="Correct",+ score=1.0,+ ),+ "step": env._current_step,+ },+ )++ env.step = mock_step++ # Test step+ action = Action(+ problem_statement="Test problem",+ llm_response="Test response",+ )+ result = await env.step(action)+ assert isinstance(result, StepResult)+ assert env._current_step == 1++ # Restore the original step method+ env.step = original_step++ # Test teardown+ await env.teardown()+ assert env._is_setup is False+ mock_verifier.cleanup.assert_awaited_once()+++async def test_environment_reward_calculation():+ r"""Test the reward calculation in BaseEnvironment."""+ env = MockBaseEnvironment(+ dataset=MagicMock(),+ verifier=MagicMock(),+ extractor=MagicMock(),+ )++ # Test compute_reward method+ action = Action(+ problem_statement="Test problem",+ llm_response="Test response",+ )+ extraction_result = {"extracted": "content"}+ verification_result = VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Verification successful",+ feedback="Correct",+ score=1.0,+ )++ total_reward, rewards_dict = await env.compute_reward(+ action, extraction_result, verification_result+ )++ # Check rewards+ assert "correctness" in rewards_dict+ assert rewards_dict["correctness"] == 1.0+ assert "additional_reward" in rewards_dict+ assert rewards_dict["additional_reward"] == 0.5+ assert total_reward == 1.5 # Sum of all rewards+++async def test_environment_max_steps():+ r"""Test that environment terminates after max_steps."""+ env = MockBaseEnvironment(+ dataset=MagicMock(),+ verifier=MagicMock(),+ extractor=MagicMock(),+ max_steps=2,+ )+ env._is_setup = True++ # Take first step+ action = Action(+ problem_statement="Test problem",+ llm_response="Test response",+ )+ env._last_observation = Observation(question="Test question")++ original_step = env.step++ async def mock_step(action):+ env._current_step += 1+ done = env._current_step >= env.max_steps+ return StepResult(+ observation=env._get_terminal_observation()+ if done+ else env._get_next_observation(),+ reward=1.0,+ rewards_dict={"correctness": 1.0},+ done=done,+ info={"reason": "max_steps_reached"} if done else {},+ )++ # Replace the step method+ env.step = mock_step++ # First step should not terminate+ result1 = await env.step(action)+ assert result1.done is False++ # Second step should terminate due to max_steps+ result2 = await env.step(action)+ assert result2.done is True+ assert "max_steps_reached" in result2.info.get("reason", "")++ # Restore the original step method+ env.step = original_stepdiff --git a/test/extractors/test_base_extractor.py b/test/extractors/test_base_extractor.pynew file mode 100644index 000000000..df238362a--- /dev/null+++ b/test/extractors/test_base_extractor.py@@ -0,0 +1,165 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from typing import Any, Dict, Optional+from unittest.mock import MagicMock++import pytest++from camel.extractors.base import BaseExtractor+++class TestExtractor(BaseExtractor):+ r"""Concrete implementation of BaseExtractor for testing."""++ async def extract(+ self, response: str, context: Optional[Dict[str, Any]] = None+ ) -> str:+ r"""Simple implementation that returns the response with a prefix."""+ if not self._is_setup:+ raise RuntimeError("Extractor not initialized")+ if not response or not response.strip():+ raise ValueError("Empty or whitespace-only response")++ prefix = "Extracted: "+ if context and "prefix" in context:+ prefix = context["prefix"]++ return f"{prefix}{response}"+++def test_extractor():+ r"""Fixture providing a TestExtractor instance."""+ return TestExtractor(+ cache_templates=True,+ max_cache_size=100,+ extraction_timeout=10.0,+ batch_size=5,+ )+++def test_extractor_init():+ r"""Test BaseExtractor initialization with various parameters."""+ extractor = TestExtractor(+ cache_templates=False,+ max_cache_size=500,+ extraction_timeout=15.0,+ batch_size=20,+ custom_param="value",+ )+ assert extractor._cache_templates is False+ assert extractor._max_cache_size == 500+ assert extractor._extraction_timeout == 15.0+ assert extractor._batch_size == 20+ assert extractor._metadata["custom_param"] == "value"+++async def test_extractor_setup(test_extractor):+ r"""Test the setup method of BaseExtractor."""+ assert test_extractor._is_setup is False++ await test_extractor.setup()+ assert test_extractor._is_setup is True+ assert hasattr(test_extractor, "_template_cache")++ await test_extractor.setup()+ assert test_extractor._is_setup is True++ await test_extractor.cleanup()+++async def test_extractor_cleanup(test_extractor):+ r"""Test the cleanup method of BaseExtractor."""+ await test_extractor.setup()+ assert test_extractor._is_setup is True++ await test_extractor.cleanup()+ assert test_extractor._is_setup is False++ await test_extractor.cleanup()+ assert test_extractor._is_setup is False+++async def test_extractor_cleanup_with_errors():+ r"""Test cleanup handling when errors occur."""+ extractor = TestExtractor()+ await extractor.setup()++ # Mock _template_cache.clear to raise an exception+ extractor._template_cache = MagicMock()+ extractor._template_cache.clear.side_effect = Exception("Mock error")++ with pytest.raises(RuntimeError):+ await extractor.cleanup()++ assert extractor._is_setup is False+++async def test_extractor_context_manager():+ r"""Test using the extractor as an async context manager."""+ async with TestExtractor() as extractor:+ assert extractor._is_setup is True+ result = await extractor.extract("test")+ assert result == "Extracted: test"++ assert extractor._is_setup is False+++async def test_extractor_context_manager_with_exception():+ r"""Test context manager handles exceptions properly."""+ try:+ async with TestExtractor() as extractor:+ assert extractor._is_setup is True+ raise ValueError("Test exception")+ except ValueError:+ # The exception should be propagated, but cleanup should still happen+ assert extractor._is_setup is False+++async def test_extract_method(test_extractor):+ r"""Test the extract method of the concrete implementation."""+ await test_extractor.setup()++ # Basic extraction+ result = await test_extractor.extract("Hello World")+ assert result == "Extracted: Hello World"++ # Extraction with context+ result = await test_extractor.extract(+ "Hello World", context={"prefix": "Custom: "}+ )+ assert result == "Custom: Hello World"++ await test_extractor.cleanup()+++async def test_extract_with_empty_input(test_extractor):+ r"""Test extract method with empty input."""+ await test_extractor.setup()++ with pytest.raises(ValueError, match="Empty or whitespace-only response"):+ await test_extractor.extract("")++ with pytest.raises(ValueError, match="Empty or whitespace-only response"):+ await test_extractor.extract(" ")++ await test_extractor.cleanup()diff --git a/test/verifiers/test_base_verifier.py b/test/verifiers/test_base_verifier.pynew file mode 100644index 000000000..1f29a00d5--- /dev/null+++ b/test/verifiers/test_base_verifier.py@@ -0,0 +1,419 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import asyncio+from unittest.mock import AsyncMock, patch++import pytest++from camel.verifiers.base import BaseVerifier+from camel.verifiers.models import (+ VerificationOutcome,+ VerificationResult,+ VerifierInput,+)+++class TestVerifier(BaseVerifier):+ r"""Concrete implementation of BaseVerifier for testing."""++ async def _setup(self) -> None:+ self.setup_called = True++ async def _cleanup(self) -> None:+ self.cleanup_called = True++ async def _verify_implementation(+ self, result: VerifierInput+ ) -> VerificationResult:+ r"""Simple implementation that returns success or failure based on+ input.+ """+ if "fail" in result.llm_response.lower():+ return VerificationResult(+ status=VerificationOutcome.FAILURE,+ result="Verification failed",+ )+ elif "error" in result.llm_response.lower():+ raise ValueError("Simulated error in verification")+ elif "timeout" in result.llm_response.lower():+ raise asyncio.TimeoutError("Simulated timeout")+ else:+ return VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Verification succeeded",+ )+++def test_verifier():+ r"""Fixture providing a TestVerifier instance."""+ return TestVerifier(+ max_parallel=2,+ timeout=5.0,+ max_retries=2,+ retry_delay=0.1,+ )+++def test_verifier_init():+ r"""Test BaseVerifier initialization with various parameters."""+ verifier = TestVerifier()+ assert verifier._max_parallel is None+ assert verifier._timeout is None+ assert verifier._max_retries == 3+ assert verifier._retry_delay == 1.0+ assert verifier._is_setup is False++ verifier = TestVerifier(+ max_parallel=5,+ timeout=10.0,+ max_retries=2,+ retry_delay=0.5,+ )+ assert verifier._max_parallel == 5+ assert verifier._timeout == 10.0+ assert verifier._max_retries == 2+ assert verifier._retry_delay == 0.5+++async def test_verifier_setup_and_cleanup(test_verifier):+ r"""Test the setup and cleanup methods of BaseVerifier."""+ assert test_verifier._is_setup is False+ await test_verifier.setup()+ assert test_verifier._is_setup is True+ assert test_verifier.setup_called is True++ await test_verifier.cleanup()+ assert test_verifier._is_setup is False+ assert test_verifier.cleanup_called is True+++async def test_verifier_setup_error():+ r"""Test handling of errors during setup."""++ class ErrorVerifier(TestVerifier):+ async def _setup(self) -> None:+ raise RuntimeError("Simulated setup error")++ verifier = ErrorVerifier()++ with pytest.raises(RuntimeError, match="Failed to initialize"):+ await verifier.setup()++ assert verifier._is_setup is False+++async def test_verifier_cleanup_error():+ r"""Test handling of errors during cleanup."""++ class ErrorVerifier(TestVerifier):+ async def _cleanup(self) -> None:+ raise RuntimeError("Simulated cleanup error")++ verifier = ErrorVerifier()+ await verifier.setup()++ with pytest.raises(RuntimeError, match="Failed to cleanup"):+ await verifier.cleanup()++ assert verifier._is_setup is False+++async def test_verify_success(test_verifier):+ r"""Test successful verification."""+ await test_verifier.setup()++ result = await test_verifier.verify(+ VerifierInput(+ llm_response="This is a successful response",+ ground_truth="Expected response",+ )+ )++ assert result.status == VerificationOutcome.SUCCESS+ assert result.result == "Verification succeeded"+ assert result.duration > 0+ assert result.metadata["attempt"] == 1++ await test_verifier.cleanup()+++async def test_verify_failure(test_verifier):+ r"""Test failed verification."""+ await test_verifier.setup()++ result = await test_verifier.verify(+ VerifierInput(+ llm_response="This will fail the verification",+ ground_truth="Expected response",+ )+ )++ assert result.status == VerificationOutcome.FAILURE+ assert result.result == "Verification failed"+ assert result.duration > 0+ assert result.metadata["attempt"] == 1++ await test_verifier.cleanup()+++async def test_verify_error_with_retry(test_verifier):+ r"""Test verification with error and retry."""+ await test_verifier.setup()++ test_verifier._max_retries = 1++ with patch("asyncio.sleep", new_callable=AsyncMock):+ result = await test_verifier.verify(+ VerifierInput(+ llm_response="This will cause an error",+ ground_truth="Expected response",+ )+ )++ assert result.status == VerificationOutcome.ERROR+ assert result.error_message is not None+ assert "Verification failed" in result.error_message+ assert result.duration > 0+ assert "attempt" in result.metadata++ await test_verifier.cleanup()+++async def test_verify_timeout(test_verifier):+ r"""Test verification timeout."""+ await test_verifier.setup()++ with patch(+ "asyncio.wait_for",+ side_effect=asyncio.TimeoutError("Simulated timeout"),+ ):+ result = await test_verifier.verify(+ VerifierInput(+ llm_response="This will timeout",+ ground_truth="Expected response",+ )+ )++ assert result.status == VerificationOutcome.TIMEOUT+ assert "timed out" in result.error_message+ assert result.duration > 0++ await test_verifier.cleanup()+++async def test_verify_not_setup():+ r"""Test verification when verifier is not set up."""+ verifier = TestVerifier()++ verifier.setup = AsyncMock()+ verifier._verify_implementation = AsyncMock(+ return_value=VerificationResult(+ status=VerificationOutcome.SUCCESS, result="Success"+ )+ )++ await verifier.verify(+ VerifierInput(llm_response="Test", ground_truth="Expected")+ )++ verifier.setup.assert_called_once()+++async def test_verify_batch(test_verifier):+ r"""Test batch verification."""+ await test_verifier.setup()++ test_verifier._verify_implementation = AsyncMock(+ return_value=VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Verification succeeded",+ )+ )++ async def mock_verify(input_data):+ if "fail" in input_data.llm_response.lower():+ return VerificationResult(+ status=VerificationOutcome.FAILURE,+ result="Verification failed",+ )+ return VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Verification succeeded",+ )++ with patch.object(test_verifier, "verify", side_effect=mock_verify):+ from camel.verifiers.base import verify_batch++ test_verifier.verify_batch = lambda *args, **kwargs: verify_batch(+ test_verifier, *args, **kwargs+ )++ inputs = [+ VerifierInput(llm_response="Success 1", ground_truth="Expected 1"),+ VerifierInput(llm_response="Success 2", ground_truth="Expected 2"),+ VerifierInput(+ llm_response="This will fail", ground_truth="Expected 3"+ ),+ ]++ results = await test_verifier.verify_batch(inputs)++ assert len(results) == 3+ assert results[0].status == VerificationOutcome.SUCCESS+ assert results[1].status == VerificationOutcome.SUCCESS+ assert results[2].status == VerificationOutcome.FAILURE++ await test_verifier.cleanup()+++async def test_verify_batch_with_error_handling(test_verifier):+ r"""Test batch verification with error handling."""+ await test_verifier.setup()++ test_verifier._verify_implementation = AsyncMock(+ return_value=VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Verification succeeded",+ )+ )++ async def mock_verify(input_data):+ if "error" in input_data.llm_response.lower():+ raise ValueError("Simulated error in verification")+ return VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Verification succeeded",+ )++ with patch.object(test_verifier, "verify", side_effect=mock_verify):+ from camel.verifiers.base import verify_batch++ test_verifier.verify_batch = lambda *args, **kwargs: verify_batch(+ test_verifier, *args, **kwargs+ )++ inputs = [+ VerifierInput(llm_response="Success", ground_truth="Expected 1"),+ VerifierInput(+ llm_response="This will cause an error",+ ground_truth="Expected 2",+ ),+ ]++ with pytest.raises(+ RuntimeError, match="One or more verifications failed"+ ):+ await test_verifier.verify_batch(inputs, raise_on_error=True)++ results = await test_verifier.verify_batch(inputs)+ assert len(results) == 2+ assert results[0].status == VerificationOutcome.SUCCESS+ assert results[1].status == VerificationOutcome.ERROR++ await test_verifier.cleanup()+++async def test_verify_batch_concurrency_limiting(test_verifier):+ r"""Test that batch verification properly limits concurrency."""+ await test_verifier.setup()++ class MockSemaphore:+ def __init__(self, value):+ self.value = value+ self.count = 0+ self.max_count = 0++ async def __aenter__(self):+ self.count += 1+ self.max_count = max(self.max_count, self.count)++ async def __aexit__(self, *args):+ self.count -= 1++ inputs = [+ VerifierInput(+ llm_response=f"Success {i}", ground_truth=f"Expected {i}"+ )+ for i in range(3)+ ]++ mock_sem = MockSemaphore(1)++ async def patched_verify_batch(verifier, inputs, raise_on_error=False):+ results = []+ for input_data in inputs:+ async with mock_sem:+ await asyncio.sleep(0.01) # Simulate processing time+ results.append(await verifier.verify(input_data))+ return results++ test_verifier.verify_batch = lambda *args, **kwargs: patched_verify_batch(+ test_verifier, *args, **kwargs+ )++ await test_verifier.verify_batch(inputs)++ assert mock_sem.max_count == 1++ await test_verifier.cleanup()+++async def test_full_verification_flow():+ r"""Test the full verification flow from setup to cleanup."""+ verifier = TestVerifier(timeout=1.0, max_retries=1, retry_delay=0.1)++ try:+ await verifier.setup()+ assert verifier._is_setup is True++ success_result = await verifier.verify(+ VerifierInput(+ llm_response="This should succeed", ground_truth="Expected"+ )+ )+ assert success_result.status == VerificationOutcome.SUCCESS++ failure_result = await verifier.verify(+ VerifierInput(+ llm_response="This will fail", ground_truth="Expected"+ )+ )+ assert failure_result.status == VerificationOutcome.FAILURE++ with patch("asyncio.sleep", new_callable=AsyncMock):+ error_result = await verifier.verify(+ VerifierInput(+ llm_response="This will cause an error",+ ground_truth="Expected",+ )+ )+ assert error_result.status == VerificationOutcome.ERROR++ finally:+ await verifier.cleanup()+ assert verifier._is_setup is False