Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

docs: Add cookbook for how to use `SingleStepEnv` (#1922)

hallerite59dff6aa6d83
diff --git a/camel/datasets/static_dataset.py b/camel/datasets/static_dataset.py
index 8949f0fbd..9ff726746 100644
--- a/camel/datasets/static_dataset.py
+++ b/camel/datasets/static_dataset.py
@@ -153,17 +153,6 @@ class StaticDataset(Dataset):
return None
rationale = item.get('rationale')
- if not isinstance(rationale, str):
- if self._strict:
- raise ValueError(
- f"Sample at index {idx} has invalid 'rationale': "
- f"expected str, got {type(rationale)}"
- )
- else:
- logger.warning(
- f"Skipping sample at index {idx}: invalid 'rationale'"
- )
- return None
final_answer = item.get('final_answer')
if not isinstance(final_answer, str):
@@ -207,25 +196,33 @@ class StaticDataset(Dataset):
r"""Return the size of the dataset."""
return self._length
- def __getitem__(self, idx: int) -> DataPoint:
- r"""Retrieve a datapoint by index.
+ def __getitem__(
+ self, idx: Union[int, slice]
+ ) -> Union[DataPoint, List[DataPoint]]:
+ r"""Retrieve a datapoint or a batch of datapoints by index or slice.
Args:
- idx (int): Index of the datapoint.
+ idx (Union[int, slice]): Index or slice of the datapoint(s).
Returns:
- DataPoint: The datapoint corresponding to the given index.
+ List[DataPoint]: A list of `DataPoint` objects.
Raises:
- IndexError: If :obj:`idx` is out of bounds (negative or greater
- than dataset length - 1).
+ IndexError: If an integer `idx` is out of bounds.
"""
+ if isinstance(idx, int):
+ if idx < 0 or idx >= self._length:
+ raise IndexError(
+ f"Index {idx} out of bounds for dataset "
+ f"of size {self._length}"
+ )
+ return self.data[idx]
- if idx < 0 or idx >= self._length:
- raise IndexError(
- f"Index {idx} out of bounds for dataset of size {self._length}"
- )
- return self.data[idx]
+ elif isinstance(idx, slice):
+ return self.data[idx.start : idx.stop : idx.step]
+
+ else:
+ raise TypeError(f"Indexing type {type(idx)} not supported.")
def sample(self) -> DataPoint:
r"""Sample a random datapoint from the dataset.
@@ -240,7 +237,12 @@ class StaticDataset(Dataset):
if self._length == 0:
raise RuntimeError("Dataset is empty, cannot sample.")
idx = self._rng.randint(0, self._length - 1)
- return self[idx]
+ sample = self[idx]
+ if not isinstance(sample, DataPoint):
+ raise TypeError(
+ f"Expected DataPoint instance, got {type(sample).__name__}"
+ )
+ return sample
@property
def metadata(self) -> Dict[str, Any]:
diff --git a/camel/environments/models.py b/camel/environments/models.py
index 3de9bc9b1..fe99904a2 100644
--- a/camel/environments/models.py
+++ b/camel/environments/models.py
@@ -33,6 +33,9 @@ class Action(BaseModel):
generated (UTC).
"""
+ index: int = Field(
+ ..., description="Index of the state this action is performed upon"
+ )
llm_response: str = Field(description="Generated response from the LLM")
metadata: Dict[str, Any] = Field(
default_factory=dict,
diff --git a/camel/environments/single_step.py b/camel/environments/single_step.py
index 814bd28db..f6a4445b8 100644
--- a/camel/environments/single_step.py
+++ b/camel/environments/single_step.py
@@ -12,12 +12,10 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-
-from abc import abstractmethod
-from typing import Any, Dict, Optional, Tuple, Union
+import random
+from typing import Any, Dict, List, Optional, Tuple, Union, cast
from camel.datasets import BaseGenerator, DataPoint, StaticDataset
-from camel.extractors.base import BaseExtractor
from camel.logger import get_logger
from camel.verifiers.base import (
BaseVerifier,
@@ -30,18 +28,23 @@ logger = get_logger(__name__)
class SingleStepEnv:
- r"""A single-step environment for reinforcement learning with LLMs.
+ r"""A lightweight environment for single-step RL with LLMs as policy.
+
+ This environment models a single interaction between an LLM-based agent
+ and a problem drawn from a dataset—such as a question-answering or
+ math problem—where the agent produces one response and receives feedback.
+
+ Core Flow:
+ - A question is sampled from a (possibly infinitely long) dataset.
+ - The LLM generates a single-step response (the action).
+ - The response is verified against the ground truth.
+ - A reward is computed based on correctness and optional custom logic.
Key Features:
- - Samples questions from a dataset and asks the LLM
- - Extracts verifiable information from model responses.
- - Verifies extracted responses against ground truth.
- - Computes and assigns rewards based on correctness.
- - Supports async setup, teardown, and cleanup of resources.
-
- This class is intended as a foundation for RL experiments involving
- LLM-based policies, ensuring structured interactions between model
- actions and verification mechanisms.
+ - Batched evaluation with per-sample state tracking.
+ - Async setup and teardown for verifiers and related resources.
+ - Supports deterministic sampling via local RNG (optional seed).
+ - Extensible reward computation via subclassing.
"""
PLACEHOLDER_OBS = Observation(
@@ -54,43 +57,47 @@ class SingleStepEnv:
self,
dataset: Union[StaticDataset, BaseGenerator],
verifier: BaseVerifier,
- extractor: BaseExtractor,
**kwargs,
) -> None:
- r"""Initialize the environment.
+ r"""Initialize the SingleStepEnv.
Args:
- dataset: Dataset to sample questions from.
- verifier: Verifier to check responses.
- extractor: Extractor to process LLM responses.
- **kwargs: Additional environment parameters.
+ dataset (Union[StaticDataset, BaseGenerator]): Dataset to sample
+ problems from.
+ verifier (BaseVerifier): Verifier used to evaluate LLM responses
+ against ground-truth answers.
+ **kwargs: Optional metadata or configuration values.
+
+ Notes:
+ This class assumes all interactions are single-step: one question,
+ one LLM response, one reward.
"""
self.dataset = dataset
self.verifier = verifier
- self.extractor = extractor
self._metadata = kwargs
# State tracking
self._is_setup: bool = False
- self._state: Optional[DataPoint] = None
- self._episode_ended: bool = False
+ self._states: List[DataPoint] = []
+ self._states_done: List[bool] = []
+ self.current_batch_size: int = 0
async def setup(self) -> None:
- r"""Set up the environment by initializing the verifier and extractor.
+ r"""Set up the environment by initializing the verifier.
This method ensures that the environment is ready for interaction.
- It sets up necessary components, including the verifier and extractor.
+ It sets up necessary components, including the verifier.
Raises:
Exception: If setup fails due to an internal error.
"""
if self._is_setup:
+ logger.warning("Environment has already been set up")
return
try:
await self.verifier.setup()
- await self.extractor.setup()
self._is_setup = True
logger.info('Environment setup completed successfully')
@@ -101,7 +108,7 @@ class SingleStepEnv:
async def close(self) -> None:
r"""Clean up and close all resources used by the environment.
- This method shuts down the verifier and extractor, resets the internal
+ This method shuts down the verifier, resets the internal
state, and ensures that the environment is properly closed.
Raises:
@@ -109,170 +116,250 @@ class SingleStepEnv:
"""
if not self._is_setup:
+ logger.warning(
+ "Not closing environment - has not been set up yet."
+ )
return
try:
self._is_setup = False
await self.verifier.cleanup()
- await self.extractor.cleanup()
- self._state = None
- self._episode_ended = False
+ self._states = []
+ self._states_done = []
logger.info('Environment closed successfully')
except Exception as e:
logger.error(f'Failed to close environment: {e}')
raise
- async def reset(self) -> Observation:
- r"""Reset the environment and start a new episode.
+ async def reset(
+ self, batch_size: int = 1, seed: Optional[int] = None
+ ) -> Union[Observation, List[Observation]]:
+ r"""Resets the environment and starts a new episode.
+
+ This method samples a new batch of data points from the dataset and
+ returns the corresponding initial observations.
- This method samples a new data point from the dataset and returns the
- initial observation.
+ If a seed is provided, a local random number generator is initialized
+ for deterministic sampling. The global random state is not affected.
+
+ Args:
+ batch_size (int): Number of data points to sample.
+ (default: :obj:`1`)
+ seed (Optional[int]): Seed for deterministic sampling. If None,
+ sampling is non-deterministic. (default: :obj:`None`)
Returns:
- Observation: The first observation of the new episode, including
- the question.
+ Observation or List[Observation]: Initial observation(s) for the
+ episode.
Raises:
- Exception: If the environment is not set up properly.
+ RuntimeError: If called before all previous states are processed.
+ ValueError: If batch size exceeds dataset size.
+ TypeError: If the dataset is of an unsupported type.
"""
if not self._is_setup:
+ logger.warning(
+ "reset() called on un-setup environment. Setting up..."
+ )
await self.setup()
- self._episode_ended = False
-
- # Sample a datapoint
-
- self._state = self.dataset.sample()
-
- observation = Observation(
- question=self._state.question, context={}, metadata={}
- )
-
- return observation
-
- async def step(self, action: Action) -> StepResult:
- r"""Take a step in the environment using the given action.
-
- This method processes the LLM response, extracts verifiable content,
- verifies correctness, computes rewards, and ends the episode.
+ if self._batch_started() and not self._batch_done():
+ logger.error(
+ "Reset called before all states were processed. "
+ "Call step on remaining states first."
+ )
+ raise RuntimeError(
+ "reset() called before all states in batch were processed."
+ )
+
+ if seed is not None:
+ rng = random.Random(seed)
+ else:
+ rng = random.Random()
+
+ if isinstance(self.dataset, StaticDataset):
+ dataset_len = len(self.dataset)
+
+ if batch_size > dataset_len:
+ raise ValueError(
+ f"Batch size {batch_size} is too large for dataset "
+ f"of size {dataset_len}"
+ )
+
+ start_idx = rng.randint(0, dataset_len - batch_size)
+ idx_slice = slice(start_idx, start_idx + batch_size)
+ val = self.dataset[idx_slice]
+ self._states = [val] if isinstance(val, DataPoint) else val
+
+ self.current_batch_size = len(self._states)
+ self._states_done = [False] * self.current_batch_size
+
+ observations = [
+ Observation(question=sample.question, context={}, metadata={})
+ for sample in self._states
+ ]
+
+ return observations[0] if batch_size == 1 else observations
+
+ elif isinstance(self.dataset, BaseGenerator):
+ raise NotImplementedError(
+ "Reset not yet implemented for BaseGenerator datasets."
+ )
+
+ else:
+ raise TypeError(f"Unsupported dataset type: {type(self.dataset)}")
+
+ async def step(
+ self, action: Union[Action, List[Action]]
+ ) -> Union[StepResult, List[StepResult]]:
+ r"""Process actions for a subset of states and update their
+ finished status.
Args:
- action (Action): The action containing the LLM response to
- evaluate.
+ action: Single action or list of actions, where each action
+ contains an index indicating which state it corresponds to.
+ The index must be a valid position in the internal _states list
+ that was populated during the reset() call.
+
Returns:
- StepResult: Contains the next observation (placeholder), total
- reward, reward breakdown, completion flag, and additional
- information.
+ Union[StepResult, List[StepResult]]: StepResult or list of
+ StepResults for the processed states.
Raises:
- RuntimeError: If the environment is not set up, the episode has
- ended, or there is no valid current observation.
+ RuntimeError: If environment isn't set up or episode has ended.
+ ValueError: If indices are invalid, duplicate, or correspond to
+ finished states.
"""
-
if not self._is_setup:
raise RuntimeError("Environment not set up. Call setup() first.")
- if self._episode_ended:
- raise RuntimeError("Episode has ended. Call reset() first.")
- if self._state is None:
+ if self._batch_done():
+ raise RuntimeError(
+ "Episodes have ended for batch. Call reset() first."
+ )
+ if not self._states:
raise RuntimeError("No current observation. Call reset() first.")
- # extract verifiable part from llm response
- extraction_result = await self.extractor.extract(action.llm_response)
-
- if not extraction_result:
- raise RuntimeError(f"Couldn't extract from {action.llm_response}")
-
- # verify the extracted
- verification_result = await self.verifier.verify(
- solution=extraction_result, ground_truth=self._state.final_answer
+ # Normalize everything to list
+ actions = [action] if isinstance(action, Action) else action
+ indices = [act.index for act in actions]
+
+ if len(set(indices)) != len(indices):
+ raise ValueError("Duplicate state indices in actions.")
+ for idx in indices:
+ if idx < 0 or idx >= len(self._states):
+ raise ValueError(f"Invalid state index {idx}.")
+ if self._states_done[idx]:
+ raise ValueError(f"State at index {idx} is already finished.")
+
+ num_actions = len(actions)
+
+ if self.current_batch_size % num_actions != 0:
+ logger.warning(
+ f"Number of actions ({num_actions}) is not a divisor of "
+ f"total batch size ({self.current_batch_size})"
+ )
+
+ proposed_solutions = [act.llm_response for act in actions]
+ ground_truths: List[str] = [
+ self._states[idx].final_answer for idx in indices
+ ]
+ verification_results = await self.verifier.verify_batch(
+ solutions=proposed_solutions,
+ ground_truths=cast(
+ list[str | None], ground_truths
+ ), # to satisfy mypy
+ raise_on_error=True,
)
- # compute rewards
- total_reward, rewards_dict = await self._compute_reward(
- action, extraction_result, verification_result
+ total_rewards, rewards_dicts = await self._compute_reward_batch(
+ proposed_solutions, verification_results
)
- self._episode_ended = True
-
- return StepResult(
- observation=self.PLACEHOLDER_OBS,
- reward=total_reward,
- rewards_dict=rewards_dict,
- done=True,
- info={
- "extraction_result": extraction_result,
- "verification_result": verification_result,
- "state": self._state,
- },
- )
-
- async def _compute_reward(
+ step_results = []
+ # TODO: batch this
+ for i, action in enumerate(actions):
+ idx = action.index
+ step_result = StepResult(
+ observation=self.PLACEHOLDER_OBS,
+ reward=total_rewards[i],
+ rewards_dict=rewards_dicts[i],
+ done=True,
+ info={
+ "proposed_solution": proposed_solutions[i],
+ "verification_result": verification_results[i],
+ "state": self._states[idx],
+ },
+ )
+ step_results.append(step_result)
+ self._states_done[idx] = True
+
+ return step_results[0] if len(step_results) == 1 else step_results
+
+ async def _compute_reward_batch(
self,
- action: Action,
- extraction_result: str,
- verification_result: VerificationResult,
- ) -> Tuple[float, Dict[str, float]]:
- r"""Compute reward scores based on verification results.
-
- This method calculates the reward based on correctness and any
- additional custom reward components.
+ proposed_solutions: List[str],
+ verification_results: List[VerificationResult],
+ ) -> Tuple[List[float], List[Dict[str, float]]]:
+ r"""Compute rewards for a batch of proposed solutions based on
+ verification results.
Args:
- action (Action): The action taken in the environment.
- extraction_result (str): The extracted verifiable content from the
- LLM response.
- verification_result (VerificationResult): The result of verifying
- the extracted response.
+ proposed_solutions (List[str]): List of LLM-generated responses to
+ evaluate.
+ verification_results (List[VerificationResult]): List of
+ verification outcomes for each solution.
Returns:
- Tuple[float, Dict[str, float]]: A tuple containing:
- - Total reward (float)
- - Dictionary of individual reward components.
-
- Raises:
- Exception: If an error occurs while computing rewards.
+ Tuple containing:
+ - List of total rewards for each solution.
+ - List of reward component dictionaries for each solution.
"""
+ total_rewards = []
+ rewards_dicts = []
- rewards: Dict[str, float] = {}
+ for solution, verification_result in zip(
+ proposed_solutions, verification_results
+ ):
+ rewards: Dict[str, float] = {}
- rewards["correctness"] = (
- self.ACCURACY_REWARD if verification_result.status else 0.0
- )
+ rewards["correctness"] = (
+ self.ACCURACY_REWARD if verification_result.status else 0.0
+ )
- further_rewards = await self._compute_custom_reward(
- action, extraction_result, verification_result
- )
+ further_rewards = await self._compute_custom_reward(
+ solution, verification_result
+ )
+ rewards = {**rewards, **further_rewards}
- rewards = rewards | further_rewards
+ total_reward = sum(rewards.values())
+ total_rewards.append(total_reward)
+ rewards_dicts.append(rewards)
- return sum(rewards.values()), rewards
+ return total_rewards, rewards_dicts
- @abstractmethod
async def _compute_custom_reward(
- self,
- action: Action,
- extraction_result: str,
- verification_result: VerificationResult,
+ self, proposed_solution: str, verification_result: VerificationResult
) -> Dict[str, float]:
- r"""Compute additional custom reward components.
+ r"""Compute additional custom reward components for a single solution.
- This method should be implemented by subclasses to define
- domain-specific reward calculations.
+ To be overridden by subclasses for domain-specific rewards.
Args:
- action (Action): The action taken in the environment.
- extraction_result (str): The extracted verifiable content from the
- LLM response.
- verification_result (VerificationResult): The result of verifying
- the extracted response.
+ proposed_solution (str): The LLM-generated response.
+ verification_result (VerificationResult): The verification outcome.
Returns:
- Dict[str, float]: A dictionary mapping custom reward categories
- to their values.
+ Dict[str, float]: Dictionary of custom reward components.
"""
- pass
+ return {}
+
+ def _batch_done(self) -> bool:
+ return all(self._states_done)
+
+ def _batch_started(self) -> bool:
+ return any(self._states_done)
@property
def metadata(self) -> Dict[str, Any]:
diff --git a/camel/extractors/__init__.py b/camel/extractors/__init__.py
index 51ab870ac..6a6a1bfe4 100644
--- a/camel/extractors/__init__.py
+++ b/camel/extractors/__init__.py
@@ -12,5 +12,20 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
from .base import BaseExtractor, BaseExtractorStrategy
+from .python_strategies import (
+ BoxedStrategy,
+ PythonDictStrategy,
+ PythonListStrategy,
+ PythonSetStrategy,
+ PythonTupleStrategy,
+)
-__all__ = ["BaseExtractor", "BaseExtractorStrategy"]
+__all__ = [
+ "BaseExtractor",
+ "BaseExtractorStrategy",
+ "BoxedStrategy",
+ "PythonListStrategy",
+ "PythonDictStrategy",
+ "PythonSetStrategy",
+ "PythonTupleStrategy",
+]
diff --git a/camel/verifiers/base.py b/camel/verifiers/base.py
index 687ca6756..16b80a772 100644
--- a/camel/verifiers/base.py
+++ b/camel/verifiers/base.py
@@ -16,6 +16,7 @@ import time
from abc import ABC, abstractmethod
from typing import List, Optional
+from camel.extractors.base import BaseExtractor
from camel.logger import get_logger
from camel.utils import BatchProcessor
@@ -44,6 +45,7 @@ class BaseVerifier(ABC):
def __init__(
self,
+ extractor: Optional[BaseExtractor] = None,
max_parallel: Optional[int] = None,
timeout: Optional[float] = None,
max_retries: int = 3,
@@ -72,6 +74,9 @@ class BaseVerifier(ABC):
down. (default: :obj:`85.0`)
**kwargs: Additional verifier parameters.
"""
+
+ self.extractor = extractor
+
self._is_setup: bool = False
self._max_parallel: Optional[int] = max_parallel
self._timeout: Optional[float] = timeout
@@ -82,7 +87,7 @@ class BaseVerifier(ABC):
self._memory_threshold: float = memory_threshold
self._batch_processor: BatchProcessor = BatchProcessor()
- async def setup(self) -> None:
+ async def setup(self, **kwargs) -> None:
r"""Set up the verifier with necessary resources.
Initializes:
@@ -97,6 +102,8 @@ class BaseVerifier(ABC):
return
try:
+ if self.extractor:
+ await self.extractor.setup()
batch_size = max(1, self._initial_batch_size or 10)
max_parallel = max(1, self._max_parallel or 1)
self._batch_processor = BatchProcessor()
@@ -106,7 +113,7 @@ class BaseVerifier(ABC):
f"batch_size={batch_size}, max_parallel={max_parallel}"
)
- await self._setup()
+ await self._setup(**kwargs)
self._is_setup = True
except Exception as e:
@@ -118,7 +125,7 @@ class BaseVerifier(ABC):
raise RuntimeError(error_msg) from e
@abstractmethod
- async def _setup(self) -> None:
+ async def _setup(self, **kwargs) -> None:
r"""Implement verifier-specific setup logic."""
pass
@@ -136,6 +143,8 @@ class BaseVerifier(ABC):
return
try:
+ if self.extractor:
+ await self.extractor.cleanup()
self._batch_processor = BatchProcessor()
await self._cleanup()
logger.info(f"{self.__class__.__name__} cleaned up successfully")
@@ -191,15 +200,28 @@ class BaseVerifier(ABC):
start_time = time.time()
while attempt < self._max_retries:
+ # Extract verifiable part of the proposed solution,
+ # if verifier has been initialized with extractor.
+ verifiable_solution = (
+ await self.extractor.extract(solution)
+ if self.extractor
+ else solution
+ )
+
+ if not verifiable_solution:
+ continue
+
try:
verification_result = (
await asyncio.wait_for(
- self._verify_implementation(solution, ground_truth),
+ self._verify_implementation(
+ verifiable_solution, ground_truth
+ ),
timeout=self._timeout,
)
if self._timeout
else await self._verify_implementation(
- solution, ground_truth
+ verifiable_solution, ground_truth
)
)
@@ -267,6 +289,7 @@ class BaseVerifier(ABC):
"Subclasses must implement _verify_implementation()"
)
+ # TODO: check again
async def verify_batch(
self,
solutions: List[str],
diff --git a/camel/verifiers/python_verifier.py b/camel/verifiers/python_verifier.py
index 627c5b51f..d653ef322 100644
--- a/camel/verifiers/python_verifier.py
+++ b/camel/verifiers/python_verifier.py
@@ -12,15 +12,16 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import ast
import asyncio
import os
import shutil
import subprocess
+import sys
import tempfile
import venv
-from typing import List, Optional
+from typing import List, Optional, Tuple
-from camel.extractors import BaseExtractor
from camel.logger import get_logger
from camel.verifiers import BaseVerifier
@@ -48,7 +49,6 @@ class PythonVerifier(BaseVerifier):
self,
timeout: Optional[float] = 30.0,
required_packages: Optional[List[str]] = None,
- extractor: Optional[BaseExtractor] = None,
):
r"""Initializes the PythonVerifier.
@@ -69,25 +69,39 @@ class PythonVerifier(BaseVerifier):
else: # Unix-like systems
self.bin_dir = 'bin'
- async def _setup(self) -> None:
- r"""Set up a virtual environment for execution
- and install required packages.
- """
+ async def _setup(self, **kwargs) -> None:
+ r"""Set up a virtual environment and install required packages."""
+ uv = kwargs.get('uv', False)
+ if uv or self._is_uv_environment():
+ logger.info("[UV] Detected uv environment. Using uv for setup.")
+ self._setup_with_uv()
+ return
+
self.venv_path = tempfile.mkdtemp()
- venv.create(self.venv_path, with_pip=True)
- logger.info(f"Virtual environment created at {self.venv_path}")
+ try:
+ venv.create(self.venv_path, with_pip=True)
+ logger.info(f"Virtual environment created at {self.venv_path}")
+ except Exception as e:
+ logger.error(f"Failed to create virtual environment: {e}")
+ # Clean up resources before re-raising
+ if self.venv_path and os.path.exists(self.venv_path):
+ shutil.rmtree(self.venv_path)
+ self.venv_path = None
+ raise
venv_pip = os.path.join(self.venv_path, self.bin_dir, "pip")
if self.required_packages:
try:
+ # Add timeout to subprocess call
subprocess.run(
[venv_pip, "install", *self.required_packages],
check=True,
capture_output=True,
+ timeout=self._timeout,
)
logger.info(
- "Installed required packages:"
+ "Installed required packages: "
f"{', '.join(self.required_packages)}"
)
except subprocess.CalledProcessError as e:
@@ -95,6 +109,97 @@ class PythonVerifier(BaseVerifier):
"Failed to install required packages: "
f"{e.stderr.decode().strip()}"
)
+ # Clean up resources before re-raising
+ if self.venv_path and os.path.exists(self.venv_path):
+ shutil.rmtree(self.venv_path)
+ self.venv_path = None
+ raise
+ except subprocess.TimeoutExpired:
+ logger.error(
+ f"Package installation timed out "
+ f"after {self._timeout} seconds"
+ )
+ if self.venv_path and os.path.exists(self.venv_path):
+ shutil.rmtree(self.venv_path)
+ self.venv_path = None
+ raise
+
+ def _is_uv_environment(self) -> bool:
+ r"""Detect whether the current Python runtime is managed by uv."""
+ return "UV_CACHE_DIR" in os.environ or "uv" in sys.executable
+
+ def _setup_with_uv(self) -> None:
+ r"""Create virtual environment and install packages using uv."""
+ self.venv_path = tempfile.mkdtemp()
+ try:
+ subprocess.run(
+ ["uv", "venv", self.venv_path],
+ check=True,
+ capture_output=True,
+ timeout=self._timeout,
+ )
+ logger.info(
+ f"[UV] Virtual environment created at {self.venv_path}"
+ )
+ except subprocess.CalledProcessError as e:
+ logger.error(
+ "[UV] Failed to create virtual environment:\n"
+ f"{e.stderr.decode().strip()}"
+ )
+ # Clean up resources before re-raising
+ if self.venv_path and os.path.exists(self.venv_path):
+ shutil.rmtree(self.venv_path)
+ self.venv_path = None
+ raise
+ except subprocess.TimeoutExpired:
+ logger.error(
+ f"[UV] Virtual environment creation timed "
+ f"out after {self._timeout} seconds"
+ )
+ if self.venv_path and os.path.exists(self.venv_path):
+ shutil.rmtree(self.venv_path)
+ self.venv_path = None
+ raise
+
+ if self.required_packages:
+ venv_python = os.path.join(self.venv_path, self.bin_dir, "python")
+ try:
+ subprocess.run(
+ [
+ "uv",
+ "pip",
+ "install",
+ "--python",
+ venv_python,
+ *self.required_packages,
+ ],
+ check=True,
+ capture_output=True,
+ timeout=self._timeout,
+ )
+ logger.info(
+ "[UV] Installed required packages via uv: "
+ f"{', '.join(self.required_packages)}"
+ )
+ except subprocess.CalledProcessError as e:
+ logger.error(
+ "[UV] Failed to install required packages via uv:\n"
+ f"{e.stderr.decode().strip()}"
+ )
+ # Clean up resources before re-raising
+ if self.venv_path and os.path.exists(self.venv_path):
+ shutil.rmtree(self.venv_path)
+ self.venv_path = None
+ raise
+ except subprocess.TimeoutExpired:
+ logger.error(
+ f"[UV] Package installation timed "
+ f"out after {self._timeout} seconds"
+ )
+ if self.venv_path and os.path.exists(self.venv_path):
+ shutil.rmtree(self.venv_path)
+ self.venv_path = None
+ raise
async def _cleanup(self) -> None:
r"""Clean up the virtual environment."""
@@ -106,32 +211,27 @@ class PythonVerifier(BaseVerifier):
async def _verify_implementation(
self, solution: str, ground_truth: Optional[str]
) -> VerificationResult:
- r"""Executes and verifies the LLM-generated Python solution in an
- isolated virtual environment.
+ r"""Executes the provided Python solution in an isolated environment
+ and verifies its output against an expected ground truth expression.
+
+ This method runs the solution in a subprocess inside a virtual
+ environment. The ground truth is assumed to be a pure Python
+ expression and is evaluated directly in the verifier process.
- This method runs the given Python solution inside a controlled virtual
- environment, captures its execution output, and optionally compares it
- against a provided ground truth. Handles timeouts and execution errors.
+ If both executions are successful, the actual output is compared
+ against the evaluated ground truth using semantic equality. If
+ evaluation fails, string comparison is used as a fallback.
Args:
- solution (str): The Python code to execute and verify.
- ground_truth (Optional[str]): The expected output for comparison.
- If None, verification is based only on execution success.
+ solution (str): The Python code or expression to execute and
+ verify.
+ ground_truth (Optional[str]): The expected value as a Python
+ expression. If None, only execution success is verified.
Returns:
- VerificationResult: A structured object containing:
- - status (VerificationOutcome): SUCCESS, FAILURE, ERROR,
- or TIMEOUT.
- - result (str): The execution output of the solution.
- - error_message (Optional[str]): Captured error message,
- if any.
- - duration (float, optional): Execution time (set externally).
-
- Raises:
- asyncio.TimeoutError: If execution exceeds the configured timeout.
- Exception: Any unexpected errors are caught and converted to an
- ERROR verification result.
+ VerificationResult: Result of the verification process.
"""
+ # Check for virtual environment setup
if not self.venv_path:
return VerificationResult(
status=VerificationOutcome.ERROR,
@@ -139,9 +239,47 @@ class PythonVerifier(BaseVerifier):
error_message="Virtual environment is not set up.",
)
- script = solution.strip()
- venv_python = os.path.join(self.venv_path, self.bin_dir, "python")
+ # If the solution is an expression, evaluate it directly
+ if self._is_expression(solution):
+ try:
+ sol_val = ast.literal_eval(solution)
+ except Exception as e:
+ return VerificationResult(
+ status=VerificationOutcome.ERROR,
+ result="",
+ error_message=f"Expression evaluation error: {e}",
+ )
+
+ if ground_truth is not None:
+ try:
+ gt_val = ast.literal_eval(ground_truth)
+ except Exception as e:
+ return VerificationResult(
+ status=VerificationOutcome.ERROR,
+ result="",
+ error_message=f"Ground truth evaluation error: {e}",
+ )
+ if sol_val == gt_val:
+ return VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result=str(sol_val),
+ )
+ else:
+ return VerificationResult(
+ status=VerificationOutcome.FAILURE,
+ result=str(sol_val),
+ error_message="Output mismatch: "
+ f"{sol_val} != {gt_val}",
+ )
+ else:
+ return VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result=str(sol_val),
+ )
+ # Otherwise, run the code block,
+ # which should already include a print(...) in the end
+ venv_python = os.path.join(self.venv_path, self.bin_dir, "python")
if not os.path.exists(venv_python):
return VerificationResult(
status=VerificationOutcome.ERROR,
@@ -150,64 +288,166 @@ class PythonVerifier(BaseVerifier):
)
try:
- process = await asyncio.create_subprocess_exec(
- venv_python,
- "-c",
- script,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- )
-
- stdout, stderr = await asyncio.wait_for(
- process.communicate(), timeout=self._timeout
+ sol_out, sol_err, sol_code = await self._run_code_block(
+ solution, venv_python
)
+ if sol_code != 0:
+ return VerificationResult(
+ status=VerificationOutcome.ERROR,
+ result=sol_out,
+ error_message=f"Solution code error:\n{sol_err}",
+ )
- output_result = stdout.decode().strip()
- error_output = stderr.decode().strip()
-
- if process.returncode == 0:
- # If ground truth is provided, compare it with the result
- if ground_truth is not None:
- # Normalize both strings by removing extra whitespace
- normalized_output = ' '.join(output_result.strip().split())
- normalized_truth = ' '.join(
- str(ground_truth).strip().split()
+ if ground_truth is not None:
+ try:
+ # First, try to evaluate the output as-is.
+ sol_val = ast.literal_eval(sol_out)
+ except Exception as e:
+ logger.warning(
+ f"Direct eval failed: {e}. Trying repr on output."
)
+ try:
+ # Try to convert sol_out to a literal
+ # by wrapping it with repr.
+ # FIXME: may be unnecessary
+ sol_val = ast.literal_eval(repr(sol_out))
+ except Exception as e2:
+ logger.warning(
+ f"repr eval also failed: {e2}."
+ "Falling back to string comparison."
+ )
+ sol_val = None
- if normalized_output == normalized_truth:
+ if sol_val is not None:
+ try:
+ gt_val = ast.literal_eval(ground_truth)
+ except Exception as e:
+ return VerificationResult(
+ status=VerificationOutcome.ERROR,
+ result="",
+ error_message="Ground truth evaluation error:"
+ f"{e}",
+ )
+ if sol_val == gt_val:
return VerificationResult(
status=VerificationOutcome.SUCCESS,
- result=output_result,
+ result=sol_out,
)
else:
return VerificationResult(
status=VerificationOutcome.FAILURE,
- error_message="Output doesn't match ground truth",
- result=output_result,
+ result=sol_out,
+ error_message="Output mismatch: "
+ f"{sol_val} != {gt_val}",
)
else:
- return VerificationResult(
- status=VerificationOutcome.SUCCESS,
- result=output_result,
- )
-
+ # Fallback: string comparison
+ if sol_out.strip() == ground_truth.strip():
+ return VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result=sol_out,
+ )
+ else:
+ return VerificationResult(
+ status=VerificationOutcome.FAILURE,
+ result=sol_out,
+ error_message="Fallback string mismatch: "
+ f"'{sol_out}' != '{ground_truth}'",
+ )
else:
return VerificationResult(
- status=VerificationOutcome.ERROR,
- error_message=error_output,
- result=output_result,
+ status=VerificationOutcome.SUCCESS,
+ result=sol_out,
)
-
except asyncio.TimeoutError:
return VerificationResult(
status=VerificationOutcome.TIMEOUT,
result="",
error_message="Execution timed out.",
)
-
except Exception as e:
return VerificationResult(
status=VerificationOutcome.ERROR,
result="",
- error_message=f"Execution error: {e}",
+ error_message=f"Unexpected error: {e}",
)
+
+ async def _run_code_block(
+ self, code: str, venv_path: str
+ ) -> Tuple[str, str, int]:
+ r"""Executes a block of Python code in the virtual environment.
+
+ The code is written to a temporary file, executed using the Python
+ interpreter from the specified virtual environment, and
+ its output and error streams are captured.
+
+ Args:
+ code (str): The Python code to execute.
+ venv_path (str): The path to the virtual environment's Python
+ binary.
+
+ Returns:
+ Tuple[str, str, int]: A tuple containing the stdout output,
+ stderr output, and return code from the executed script.
+ """
+ # No longer checking for expressions since they're handled separately
+ with tempfile.NamedTemporaryFile(
+ "w+", suffix=".py", delete=False
+ ) as tmp:
+ tmp.write(code)
+ tmp_path = tmp.name
+
+ proc = await asyncio.create_subprocess_exec(
+ venv_path,
+ tmp_path,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ stdout, stderr = await asyncio.wait_for(
+ proc.communicate(), timeout=self._timeout
+ )
+ os.remove(tmp_path)
+ return (
+ stdout.decode().strip(),
+ stderr.decode().strip(),
+ proc.returncode if proc.returncode is not None else -1,
+ )
+
+ def _is_expression(self, code: str) -> bool:
+ r"""Determines whether a given string of code is a single expression.
+
+ This utility uses Python's AST module to parse the code and checks if
+ it consists of a single expression node.
+
+ Args:
+ code (str): The Python code to analyze.
+
+ Returns:
+ bool: True if the code is a single expression, False otherwise.
+ """
+ # Skip empty or whitespace-only strings
+ if not code or code.isspace():
+ return False
+
+ try:
+ # First try parsing as an expression - this is more reliable than
+ # starting with literal_eval
+ tree = ast.parse(code.strip(), mode='eval')
+ # Check if it's a function call (like print()) - these should not
+ # be treated as expressions
+ if isinstance(tree.body, ast.Call):
+ return False
+ # If parsing succeeds in 'eval' mode and it's not a function call,
+ # it's a valid expression
+ return True
+ except SyntaxError:
+ # If parsing as expression fails, it's not a valid expression
+ return False
+ except Exception:
+ # For any other parsing errors, try literal_eval as fallback for
+ # simple literals
+ try:
+ ast.literal_eval(code)
+ return True
+ except Exception:
+ return False
diff --git a/examples/environments/demo.ipynb b/examples/environments/demo.ipynb
new file mode 100644
index 000000000..416885c85
--- /dev/null
+++ b/examples/environments/demo.ipynb
@@ -0,0 +1,211 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Single Step Environment in Camel"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Single Step environments are the most widespread type of environment when doing RL with an LLM as policy.\n",
+ "\n",
+ "It's called *single step* environment, because the agent only does one step. It gets a question sampled from the dataset (the initial state / observation) and then answers. The answer is then scored according to the reward function. Recently, rules-based reward functions, i.e. functions without any learnable parameters, have been successfully used to do RL with LLMs as as policy.\n",
+ "\n",
+ "First, we have to load a dataset from which we will sample questions. The dataset can be either a `StaticDataset`, which is finite and the length is known at runtime, or it can be a `BaseGenerator`, which is an infinite supply of question - answer pairs, synthetically generated in some way (depending on the implementation).\n",
+ "\n",
+ "For the sake of simplicity, we will start by loading the MATH dataset, remove unnecessary columns and rename the remaining ones, such that we can easily turn it into a `StaticDataset`, which `SingleStepEnv` can deal with. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/student/.cache/pypoetry/virtualenvs/camel-ai-80B0TVyJ-py3.11/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+ " from .autonotebook import tqdm as notebook_tqdm\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "['question', 'final_answer']\n",
+ "Example datapoint: question='Let \\\\[f(x) = \\\\left\\\\{\\n\\\\begin{array}{cl} ax+3, &\\\\text{ if }x>2, \\\\\\\\\\nx-5 &\\\\text{ if } -2 \\\\le x \\\\le 2, \\\\\\\\\\n2x-b &\\\\text{ if } x <-2.\\n\\\\end{array}\\n\\\\right.\\\\]Find $a+b$ if the piecewise function is continuous (which means that its graph can be drawn without lifting your pencil from the paper).' final_answer='For the piecewise function to be continuous, the cases must \"meet\" at $2$ and $-2$. For example, $ax+3$ and $x-5$ must be equal when $x=2$. This implies $a(2)+3=2-5$, which we solve to get $2a=-6 \\\\Rightarrow a=-3$. Similarly, $x-5$ and $2x-b$ must be equal when $x=-2$. Substituting, we get $-2-5=2(-2)-b$, which implies $b=3$. So $a+b=-3+3=\\\\boxed{0}$.' rationale=None metadata=None\n"
+ ]
+ }
+ ],
+ "source": [
+ "from datasets import load_dataset\n",
+ "\n",
+ "from camel.datasets import StaticDataset\n",
+ "from camel.logger import get_logger\n",
+ "\n",
+ "logger = get_logger(__name__)\n",
+ "\n",
+ "\n",
+ "dataset = load_dataset(\"EleutherAI/hendrycks_math\", \"algebra\")\n",
+ "\n",
+ "# Preprocess\n",
+ "dataset[\"train\"] = dataset[\"train\"].rename_column(\"problem\", \"question\")\n",
+ "dataset[\"train\"] = dataset[\"train\"].rename_column(\"solution\", \"final_answer\")\n",
+ "dataset[\"train\"] = dataset[\"train\"].remove_columns([\"type\", \"level\"])\n",
+ "\n",
+ "# This should now print \"['question', 'final_answer']\"\n",
+ "print(dataset[\"train\"].column_names)\n",
+ "seed_dataset = StaticDataset(dataset['train'])\n",
+ "\n",
+ "print(\"Example datapoint:\", seed_dataset[0])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Next, we will have to define an *extractor*. An extractor takes the LLM response and extracts the verifiable part out of it. Extractors can be initialized with different strategies which modifies the extraction behavior.\n",
+ "\n",
+ "In the case of the MATH dataset, the final answer is wrapped inside a `\\boxed{...}`, hence we should use the pre-built `BoxedStrategy`.\n",
+ "\n",
+ "Sadly, MATH answers are rather complicated and a more general Math verifier to compare, for example, equations has not yet been implemented. Hence, we shall prune the dataset to only contain those rows where the content of `\\boxed{...}` is an int. That way, we can do simple verification using the vanilla PythonVerifier!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Number of datapoints with integer answers: 1228\n"
+ ]
+ }
+ ],
+ "source": [
+ "from camel.extractors import BaseExtractor, BoxedStrategy\n",
+ "\n",
+ "# Initialize extractor\n",
+ "extractor = BaseExtractor([[BoxedStrategy()]])\n",
+ "await extractor.setup()\n",
+ "\n",
+ "valid_datapoints = []\n",
+ "\n",
+ "# Iterate through dataset, checking for datapoints with integer answers\n",
+ "for datapoint in seed_dataset:\n",
+ " extracted_value = await extractor.extract(response=datapoint.final_answer)\n",
+ "\n",
+ " if not extracted_value:\n",
+ " continue\n",
+ " if extracted_value.isdigit() or (\n",
+ " extracted_value.startswith('-') and extracted_value[1:].isdigit()\n",
+ " ):\n",
+ " valid_datapoints.append(\n",
+ " {\n",
+ " \"question\": datapoint.question,\n",
+ " \"final_answer\": datapoint.final_answer,\n",
+ " }\n",
+ " )\n",
+ "\n",
+ "# We should now have `1228` valid data points.\n",
+ "print(f\"Number of datapoints with integer answers: {len(valid_datapoints)}\")\n",
+ "\n",
+ "filtered_dataset = StaticDataset(valid_datapoints, seed=42)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let's create a Python verifier to later compare answers."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from camel.verifiers import PythonVerifier\n",
+ "\n",
+ "verifier = PythonVerifier()\n",
+ "await verifier.setup()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let's now initalize the single step environment with our filtered dataset, our verifier and our extractor.\n",
+ "\n",
+ "We can then call `env.reset()` to draw from the initial state distribution and return an observation, which can then be fed into the agent."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from camel.environments import Action, SingleStepEnv\n",
+ "\n",
+ "env = SingleStepEnv(filtered_dataset, verifier, extractor)\n",
+ "\n",
+ "obs = await env.reset()\n",
+ "\n",
+ "print(obs)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The agent would then process this observation and select an action, which it would feed into the `step` function. An action in this case would simply be the answer to the question, wrapped in `\\boxed{}`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "await env.step(Action(llm_response=\"\\\\boxed{5}\"))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "As you can see, the output of the `step` function contains the next observation (which in this case is just a placeholder, since the episode is over), a reward, as well as a reward dict, showing exactly which rubric brought which reward, a `done` flag, indicating that the episode is over and some additional info."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "camel-ai-80B0TVyJ-py3.11",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.6"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/examples/verifier/python_verifier_example.py b/examples/verifier/python_verifier_example.py
index d104085e9..7c6f4979c 100644
--- a/examples/verifier/python_verifier_example.py
+++ b/examples/verifier/python_verifier_example.py
@@ -16,7 +16,7 @@ import asyncio
from camel.verifiers import PythonVerifier
verifier = PythonVerifier(required_packages=["numpy"])
-asyncio.run(verifier.setup())
+asyncio.run(verifier.setup(uv=True))
numpy_test_code = """
import numpy as np
@@ -32,7 +32,7 @@ print(result)
result = asyncio.run(
verifier.verify(solution=numpy_test_code, ground_truth="32")
)
-print(f"Result: {result.status}")
+print(f"Result: {result}")
result = asyncio.run(
verifier.verify(solution=numpy_test_code, ground_truth="40")
@@ -40,6 +40,6 @@ result = asyncio.run(
# Now we expect the VerificationOutcome to be a failure,
# because the answer is wrong.
-print(f"Result: {result.status}")
+print(f"Result: {result}")
asyncio.run(verifier.cleanup())
diff --git a/test/datasets/test_base_dataset.py b/test/datasets/test_base_dataset.py
index 9c298b451..75c381246 100644
--- a/test/datasets/test_base_dataset.py
+++ b/test/datasets/test_base_dataset.py
@@ -168,13 +168,11 @@ def test_static_dataset_init_from_hf_dataset():
# **Test 2: Initialization with invalid data**
# Sub-test 2a: Missing required field with strict=True
- invalid_data_missing = [
- {"question": "What is 3 + 3?", "final_answer": "6"}
- ]
+ invalid_data_missing = [{"question": "What is 3 + 3?"}]
hf_invalid_missing = HFDataset.from_list(invalid_data_missing)
with pytest.raises(
ValueError,
- match="Sample at index 0 has invalid 'rationale': "
+ match="Sample at index 0 has invalid 'final_answer': "
"expected str, got <class 'NoneType'>",
):
StaticDataset(data=hf_invalid_missing, min_samples=1, strict=True)
@@ -300,13 +298,12 @@ def test_static_dataset_init_from_pytorch_dataset():
invalid_data_missing = [
{
"question": "What is 3 + 3?",
- "final_answer": "6",
- } # Missing rationale
+ } # Missing final_answer
]
pytorch_invalid_missing = MockPyTorchDataset(invalid_data_missing)
with pytest.raises(
ValueError,
- match="Sample at index 0 has invalid 'rationale': "
+ match="Sample at index 0 has invalid 'final_answer': "
"expected str, got <class 'NoneType'>",
):
StaticDataset(data=pytorch_invalid_missing, min_samples=1, strict=True)
@@ -567,15 +564,14 @@ def test_static_dataset_init_from_json_file():
invalid_data = [
{
"question": "What is 3 + 3?",
- "final_answer": "6",
- } # Missing rationale
+ } # Missing final answer
]
json.dump(invalid_data, tmp_file)
tmp_file_path = Path(tmp_file.name)
with pytest.raises(
ValueError,
- match="Sample at index 0 has invalid 'rationale': "
+ match="Sample at index 0 has invalid 'final_answer': "
"expected str, got <class 'NoneType'>",
):
StaticDataset(data=tmp_file_path, min_samples=1, strict=True)
@@ -587,8 +583,7 @@ def test_static_dataset_init_from_json_file():
invalid_data = [
{
"question": "What is 3 + 3?",
- "final_answer": "6",
- } # Missing rationale
+ } # Missing final_answer
]
json.dump(invalid_data, tmp_file)
tmp_file_path = Path(tmp_file.name)
@@ -763,12 +758,11 @@ def test_static_dataset_init_from_jsonl_file():
with tempfile.NamedTemporaryFile(
mode='w', suffix='.jsonl', delete=False
) as tmp_file:
- # Missing 'rationale' field.
+ # Missing 'final_answer' field.
tmp_file.write(
json.dumps(
{
"question": "What is 3 + 3?",
- "final_answer": "6",
}
)
+ "\n"
@@ -777,7 +771,7 @@ def test_static_dataset_init_from_jsonl_file():
with pytest.raises(
ValueError,
- match="Sample at index 0 has invalid 'rationale'",
+ match="Sample at index 0 has invalid 'final_answer'",
):
StaticDataset(data=tmp_file_path, min_samples=1, strict=True)
@@ -790,7 +784,6 @@ def test_static_dataset_init_from_jsonl_file():
json.dumps(
{
"question": "What is 3 + 3?",
- "final_answer": "6",
}
)
+ "\n"
@@ -878,12 +871,11 @@ def test_static_dataset_init_from_list():
invalid_data_missing = [
{
"question": "What is 3 + 3?",
- "final_answer": "6",
- } # Missing rationale
+ } # Missing final_answer
]
with pytest.raises(
ValueError,
- match="Sample at index 0 has invalid 'rationale': "
+ match="Sample at index 0 has invalid 'final_answer': "
"expected str, got <class 'NoneType'>",
):
StaticDataset(data=invalid_data_missing, min_samples=1, strict=True)
@@ -937,9 +929,8 @@ def test_static_dataset_init_from_list():
"final_answer": "Valid",
},
{
- "question": "Invalid question",
- "final_answer": "Invalid",
- }, # Missing rationale
+ "question": "Question",
+ }, # Missing final_answer
]
dataset_mixed = StaticDataset(data=mixed_data, min_samples=1, strict=False)
assert (
@@ -1025,6 +1016,197 @@ def test_static_dataset_methods():
empty_dataset.sample()
+def test_static_dataset_getitem_slice():
+ mock_data = [
+ {
+ "question": "What is 1+1?",
+ "rationale": "Basic addition.",
+ "final_answer": "2",
+ },
+ {
+ "question": "Is the Earth round?",
+ "rationale": "Scientific consensus.",
+ "final_answer": "yes",
+ },
+ {
+ "question": "What is the capital of Japan?",
+ "rationale": "Geography knowledge.",
+ "final_answer": "Tokyo",
+ },
+ {
+ "question": "How many sides does a triangle have?",
+ "rationale": "Definition of a triangle.",
+ "final_answer": "3",
+ },
+ ]
+ dataset = StaticDataset(data=mock_data, min_samples=1, strict=True)
+
+ sliced = dataset[1:4:2]
+ assert isinstance(sliced, list), "Slicing should return a list."
+ assert len(sliced) == 2, "Slice [1:4:2] should return 2 items."
+ assert all(
+ isinstance(item, DataPoint) for item in sliced
+ ), "All items should be DataPoint instances."
+ assert (
+ sliced[0].question == "Is the Earth round?"
+ ), "Index 1 question should match."
+ assert (
+ sliced[1].question == "How many sides does a triangle have?"
+ ), "Index 3 question should match."
+ assert (
+ sliced[0] is dataset[1]
+ ), "Slicing should return references to the same objects."
+
+ # Test slice with only start and stop
+ sliced = dataset[1:3]
+ assert len(sliced) == 2, "Slice [1:3] should return 2 items."
+ assert (
+ sliced[0].question == "Is the Earth round?"
+ ), "Index 1 question should match."
+ assert (
+ sliced[1].question == "What is the capital of Japan?"
+ ), "Index 2 question should match."
+
+ # Test slice with negative indices
+ sliced = dataset[-2:]
+ assert len(sliced) == 2, "Slice [-2:] should return 2 items."
+ assert (
+ sliced[0].question == "What is the capital of Japan?"
+ ), "Index -2 question should match."
+ assert (
+ sliced[1].question == "How many sides does a triangle have?"
+ ), "Index -1 question should match."
+
+ sliced = dataset[:-1]
+ assert len(sliced) == 3, "Slice [:-1] should return 3 items."
+ assert (
+ sliced[0].question == "What is 1+1?"
+ ), "Index 0 question should match."
+ assert (
+ sliced[2].question == "What is the capital of Japan?"
+ ), "Index 2 question should match."
+
+ # Test slice with step
+ sliced = dataset[::2]
+ assert len(sliced) == 2, "Slice [::2] should return 2 items."
+ assert (
+ sliced[0].question == "What is 1+1?"
+ ), "Index 0 question should match."
+ assert (
+ sliced[1].question == "What is the capital of Japan?"
+ ), "Index 2 question should match."
+
+ # Test slice with negative step
+ sliced = dataset[3:0:-1]
+ assert len(sliced) == 3, "Slice [3:0:-1] should return 3 items."
+ assert (
+ sliced[0].question == "How many sides does a triangle have?"
+ ), "Index 3 question should match."
+ assert (
+ sliced[2].question == "Is the Earth round?"
+ ), "Index 1 question should match."
+
+ sliced = dataset[::-1]
+ assert (
+ len(sliced) == 4
+ ), "Slice [::-1] should return all 4 items in reverse."
+ assert (
+ sliced[0].question == "How many sides does a triangle have?"
+ ), "Index 3 question should match."
+ assert (
+ sliced[3].question == "What is 1+1?"
+ ), "Index 0 question should match."
+
+ # Test slicing with out-of-bound indices
+ sliced = dataset[10:20]
+ assert (
+ sliced == []
+ ), "Slice [10:20] should return an empty list when out of bounds."
+
+ sliced = dataset[-10:10]
+ assert len(sliced) == 4, "Slice [-10:10] should return all items."
+ assert (
+ sliced[0].question == "What is 1+1?"
+ ), "Index 0 question should match."
+ assert (
+ sliced[3].question == "How many sides does a triangle have?"
+ ), "Index 3 question should match."
+
+ sliced = dataset[2:100]
+ assert (
+ len(sliced) == 2
+ ), "Slice [2:100] should return items from index 2 to end."
+ assert (
+ sliced[0].question == "What is the capital of Japan?"
+ ), "Index 2 question should match."
+ assert (
+ sliced[1].question == "How many sides does a triangle have?"
+ ), "Index 3 question should match."
+
+ # Test non-integer, non-slice inputs
+ with pytest.raises(
+ TypeError, match="Indexing type <class 'str'> not supported."
+ ):
+ dataset["invalid"]
+
+ with pytest.raises(
+ TypeError, match="Indexing type <class 'float'> not supported."
+ ):
+ dataset[1.5]
+
+ with pytest.raises(
+ TypeError, match="Indexing type <class 'NoneType'> not supported."
+ ):
+ dataset[None]
+
+ with pytest.raises(
+ TypeError, match="Indexing type <class 'list'> not supported."
+ ):
+ dataset[[1, 2]]
+
+ # Test slice with step=0
+ with pytest.raises(ValueError, match="slice step cannot be zero"):
+ dataset[::0]
+
+ # Test slice with non-integer start, stop, or step
+ invalid_slice = slice("a", 3, 1)
+ with pytest.raises(
+ TypeError,
+ match="slice indices must be integers or None or have "
+ "an __index__ method",
+ ):
+ dataset[invalid_slice]
+
+ invalid_slice = slice(0, "b", 1)
+ with pytest.raises(
+ TypeError,
+ match="slice indices must be integers or None "
+ "or have an __index__ method",
+ ):
+ dataset[invalid_slice]
+
+ invalid_slice = slice(0, 3, "a")
+ with pytest.raises(
+ TypeError,
+ match="slice indices must be integers or None "
+ "or have an __index__ method",
+ ):
+ dataset[invalid_slice]
+
+ # Test slicing an empty dataset
+ empty_dataset = StaticDataset(data=[], min_samples=0, strict=True)
+ assert len(empty_dataset) == 0, "Empty dataset should have length 0."
+ assert (
+ empty_dataset[:] == []
+ ), "Full slice of empty dataset should return empty list."
+ assert (
+ empty_dataset[0:0] == []
+ ), "Zero-length slice should return empty list."
+ assert (
+ empty_dataset[1:2] == []
+ ), "Out-of-bound slice should return empty list."
+
+
@pytest.mark.asyncio
async def test_few_shot_generator():
r"""Test FewShotGenerator with mocked components."""
diff --git a/test/environments/test_multi_step_env.py b/test/environments/test_multi_step_env.py
index 81911b922..06a33b687 100644
--- a/test/environments/test_multi_step_env.py
+++ b/test/environments/test_multi_step_env.py
@@ -77,7 +77,8 @@ async def test_multi_step_env_lifecycle():
assert observation.question == "Step 0"
# Test first step
- action1 = Action(llm_response="Response 1")
+ action1 = Action(index=0, llm_response="Response 1")
+ # FIXME: index set to 0 temporarily until we have batched MultiStep
result1 = await env.step(action1)
assert isinstance(result1, StepResult)
assert result1.reward == 1.0
@@ -88,7 +89,8 @@ async def test_multi_step_env_lifecycle():
mock_extractor.extract.assert_called_once_with("Response 1")
# Test second step
- action2 = Action(llm_response="Response 2")
+ action2 = Action(index=0, llm_response="Response 2")
+ # FIXME: index set to 0 temporarily until we have batched MultiStep
result2 = await env.step(action2)
assert isinstance(result2, StepResult)
assert result2.reward == 1.0
@@ -112,7 +114,8 @@ async def test_multi_step_env_error_handling():
mock_extractor.extract = MagicMock(return_value="extracted_answer")
env = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)
- action = Action(llm_response="Test response")
+ action = Action(index=0, llm_response="Test response")
+ # FIXME: index set to 0 temporarily until we have batched MultiStep
# **Test Case 1: Environment not set up**
# Attempt to step without setup, expect RuntimeError
diff --git a/test/environments/test_single_stage_env.py b/test/environments/test_single_stage_env.py
index 047e9af27..d67831284 100644
--- a/test/environments/test_single_stage_env.py
+++ b/test/environments/test_single_stage_env.py
@@ -30,16 +30,14 @@ from camel.verifiers.models import VerificationOutcome, VerificationResult
class MockSingleStepEnv(SingleStepEnv):
async def _compute_custom_reward(
self,
- action: Action,
- extraction_result: str,
+ proposed_solution: str,
verification_result: VerificationResult,
) -> Dict[str, float]:
- return {"custom_reward": 0.5}
+ return {"custom_reward": 5}
@pytest.mark.asyncio
async def test_single_step_env_lifecycle():
- # Define dataset
data = [
{
"question": "What is 2 + 2?",
@@ -53,69 +51,141 @@ async def test_single_step_env_lifecycle():
"rationale": "Paris is known as the capital city of France.",
"metadata": {"difficulty": "easy"},
},
+ {
+ "question": "Who wrote 'Romeo and Juliet'?",
+ "final_answer": "Shakespeare",
+ "rationale": "William Shakespeare is the author of 'Romeo and "
+ "Juliet'.",
+ "metadata": {"difficulty": "medium"},
+ },
+ {
+ "question": "What is the boiling point of water in Celsius?",
+ "final_answer": "100",
+ "rationale": "Water boils at 100 degrees Celsius at standard "
+ "pressure.",
+ "metadata": {"difficulty": "easy"},
+ },
+ {
+ "question": "What is the chemical symbol for gold?",
+ "final_answer": "Au",
+ "rationale": "The chemical symbol for gold is Au, from the Latin "
+ "'aurum'.",
+ "metadata": {"difficulty": "medium"},
+ },
+ {
+ "question": "How many continents are there?",
+ "final_answer": "7",
+ "rationale": "There are seven continents: Asia, Africa, North "
+ "America, South America, Antarctica, Europe, and Australia.",
+ "metadata": {"difficulty": "easy"},
+ },
+ {
+ "question": "What is the largest planet in our solar system?",
+ "final_answer": "Jupiter",
+ "rationale": "Jupiter is the largest planet, with a diameter of "
+ "about 142,984 km.",
+ "metadata": {"difficulty": "medium"},
+ },
+ {
+ "question": "Who painted the Mona Lisa?",
+ "final_answer": "Leonardo da Vinci",
+ "rationale": "Leonardo da Vinci painted the Mona Lisa "
+ "between 1503 and 1506.",
+ "metadata": {"difficulty": "medium"},
+ },
]
dataset = StaticDataset(data)
-
- # Define mock verifier
mock_verifier = MagicMock()
mock_verifier.setup = AsyncMock()
mock_verifier.cleanup = AsyncMock()
- mock_verifier.verify = AsyncMock(
- return_value=VerificationResult(
- status=VerificationOutcome.SUCCESS,
- result="Verification successful",
- feedback="Correct",
- score=1.0,
- )
+ mock_verifier.verify_batch = AsyncMock(
+ side_effect=lambda solutions, ground_truths, **kwargs: [
+ VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Verification successful",
+ feedback="Correct",
+ score=1.0,
+ )
+ for _ in solutions
+ ]
)
- # Define mock extractor
- mock_extractor = MagicMock()
- mock_extractor.setup = AsyncMock()
- mock_extractor.cleanup = AsyncMock()
- mock_extractor.extract = AsyncMock(return_value="extracted_answer")
+ env = MockSingleStepEnv(dataset=dataset, verifier=mock_verifier)
- # Initialize the environment with mocked dependencies
- env = MockSingleStepEnv(
- dataset=dataset,
- verifier=mock_verifier,
- extractor=mock_extractor,
- )
-
- # Test setup
await env.setup()
assert env._is_setup is True
mock_verifier.setup.assert_awaited_once()
- mock_extractor.setup.assert_awaited_once()
# Test reset
- observation = await env.reset()
- assert isinstance(observation, Observation)
- assert observation.question in [
- "What is 2 + 2?",
- "What is the capital of France?",
+ observations = await env.reset(batch_size=4)
+ assert isinstance(observations, list)
+ assert len(observations) == 4
+ for obs in observations:
+ assert isinstance(obs, Observation)
+ assert obs.question in [dp["question"] for dp in data]
+
+ # Test step with 2 actions
+ actions = [
+ Action(index=0, llm_response="4"), # "What is 2 + 2?"
+ Action(
+ index=2, llm_response="Shakespeare"
+ ), # "Who wrote 'Romeo and Juliet'?"
]
+ results = await env.step(actions)
+ assert isinstance(results, list)
+ assert len(results) == 2
+ for result in results:
+ assert isinstance(result, StepResult)
+ assert result.reward == 15
+ assert result.done is True
+ assert result.observation == MockSingleStepEnv.PLACEHOLDER_OBS
- # Test step
- action = Action(llm_response="Test response")
- result = await env.step(action)
- assert isinstance(result, StepResult)
- assert result.reward == 10.5 # correctness: 10.0 + custom_reward: 0.5
- assert result.done is True
- assert result.observation == SingleStepEnv.PLACEHOLDER_OBS
- mock_extractor.extract.assert_awaited_once_with("Test response")
- mock_verifier.verify.assert_awaited_once()
+ # Test step with remaining two actions
+ actions = [
+ Action(
+ index=1, llm_response="Paris"
+ ), # "What is the capital of France?"
+ Action(
+ index=3, llm_response="100"
+ ), # "What is the boiling point of water?"
+ ]
+ results = await env.step(actions)
+ assert isinstance(results, list)
+ assert len(results) == 2
+ for result in results:
+ assert isinstance(result, StepResult)
+ assert result.reward == 15
+ assert result.done is True
+ assert result.observation == MockSingleStepEnv.PLACEHOLDER_OBS
- # Test close
+ assert env._batch_done()
await env.close()
assert env._is_setup is False
mock_verifier.cleanup.assert_awaited_once()
- mock_extractor.cleanup.assert_awaited_once()
+
+
+def create_mock_verifier():
+ verifier = MagicMock()
+ verifier.setup = AsyncMock()
+ verifier.cleanup = AsyncMock()
+ verifier.verify_batch = AsyncMock(
+ side_effect=lambda solutions, ground_truths, **kwargs: [
+ VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Mock verification",
+ feedback="Mock feedback",
+ score=1.0,
+ )
+ for _ in solutions
+ ]
+ )
+ return verifier
@pytest.mark.asyncio
async def test_single_step_env_error_handling():
# **1. Test Faulty Dataset**
+ # Ensure we don't silently handle the error
faulty_data = [
{
"question": "What is 2 + 2?",
@@ -130,15 +200,11 @@ async def test_single_step_env_error_handling():
"metadata": {"difficulty": "easy"},
},
]
- try:
- StaticDataset(faulty_data)
- except Exception as e:
- assert isinstance(
- e, ValueError
- ), "Expected ValueError for faulty dataset"
+ with pytest.raises(ValueError):
+ StaticDataset(faulty_data, strict=True)
- # **Valid Dataset for Subsequent Tests**
- valid_data = [
+ # Define valid dataset for subsequent tests
+ data = [
{
"question": "What is 2 + 2?",
"final_answer": "4",
@@ -151,109 +217,189 @@ async def test_single_step_env_error_handling():
"rationale": "Paris is known as the capital city of France.",
"metadata": {"difficulty": "easy"},
},
+ {
+ "question": "Who wrote 'Romeo and Juliet'?",
+ "final_answer": "Shakespeare",
+ "rationale": "William Shakespeare is the author of 'Romeo and "
+ "Juliet'.",
+ "metadata": {"difficulty": "medium"},
+ },
+ {
+ "question": "What is the boiling point of water in Celsius?",
+ "final_answer": "100",
+ "rationale": "Water boils at 100 degrees Celsius at standard "
+ "pressure.",
+ "metadata": {"difficulty": "easy"},
+ },
+ {
+ "question": "What is the chemical symbol for gold?",
+ "final_answer": "Au",
+ "rationale": "The chemical symbol for gold is Au, from the Latin "
+ "'aurum'.",
+ "metadata": {"difficulty": "medium"},
+ },
+ {
+ "question": "How many continents are there?",
+ "final_answer": "7",
+ "rationale": "There are seven continents: Asia, Africa, North "
+ "America, South America, Antarctica, Europe, and Australia.",
+ "metadata": {"difficulty": "easy"},
+ },
+ {
+ "question": "What is the largest planet in our solar system?",
+ "final_answer": "Jupiter",
+ "rationale": "Jupiter is the largest planet, with a diameter of "
+ "about 142,984 km.",
+ "metadata": {"difficulty": "medium"},
+ },
+ {
+ "question": "Who painted the Mona Lisa?",
+ "final_answer": "Leonardo da Vinci",
+ "rationale": "Leonardo da Vinci painted the Mona Lisa "
+ "between 1503 and 1506.",
+ "metadata": {"difficulty": "medium"},
+ },
]
- dataset = StaticDataset(valid_data)
-
- # **2. Test Faulty Extractor**
- # Mock verifier that fails verification
- mock_verifier_fail = MagicMock()
- mock_verifier_fail.setup = AsyncMock()
- mock_verifier_fail.cleanup = AsyncMock()
- mock_verifier_fail.verify = AsyncMock(
- return_value=VerificationResult(
- status=VerificationOutcome.FAILURE,
- result="Verification failed",
- feedback="Incorrect",
- score=0.0,
- )
- )
- # Mock extractor that returns None (extraction failure)
- mock_extractor_fail = MagicMock()
- mock_extractor_fail.setup = AsyncMock()
- mock_extractor_fail.cleanup = AsyncMock()
- mock_extractor_fail.extract = AsyncMock(return_value=None)
-
- env_fail_extractor = MockSingleStepEnv(
- dataset=dataset,
- verifier=mock_verifier_fail,
- extractor=mock_extractor_fail,
- )
- await env_fail_extractor.setup()
- observation = await env_fail_extractor.reset()
- assert isinstance(
- observation, Observation
- ), "Reset should still return an Observation"
-
- action = Action(llm_response="Test response")
- with pytest.raises(RuntimeError, match="Couldn't extract from"):
- await env_fail_extractor.step(action)
+ dataset = StaticDataset(data)
- # **3. Test Faulty Verifier**
- # Mock verifier that raises an exception
+ # **2. Test Faulty Verifier**
+ # Ensure errors in verifier are propagated
mock_verifier_exception = MagicMock()
mock_verifier_exception.setup = AsyncMock()
mock_verifier_exception.cleanup = AsyncMock()
- mock_verifier_exception.verify = AsyncMock(
+ mock_verifier_exception.verify_batch = AsyncMock(
side_effect=Exception("Verifier error")
)
- # Valid extractor
- mock_extractor = MagicMock()
- mock_extractor.setup = AsyncMock()
- mock_extractor.cleanup = AsyncMock()
- mock_extractor.extract = AsyncMock(return_value="extracted_answer")
-
- env_fail_verifier = MockSingleStepEnv(
+ env_fail_verifier = SingleStepEnv(
dataset=dataset,
verifier=mock_verifier_exception,
- extractor=mock_extractor,
)
await env_fail_verifier.setup()
- observation = await env_fail_verifier.reset()
- assert isinstance(
- observation, Observation
- ), "Reset should still return an Observation"
-
+ await env_fail_verifier.reset(batch_size=1)
+ action = Action(index=0, llm_response="4")
with pytest.raises(Exception, match="Verifier error"):
await env_fail_verifier.step(action)
- # **4. Test State Mismanagement Scenarios**
- # a) Environment not set up
- env_not_setup = MockSingleStepEnv(
+ # **3. Test State Mismanagement Scenarios**
+ # a) Step without setup
+ env_not_setup = SingleStepEnv(
dataset=dataset,
- verifier=mock_verifier_fail,
- extractor=mock_extractor,
+ verifier=create_mock_verifier(),
)
with pytest.raises(
RuntimeError,
match=re.escape("Environment not set up. Call setup() first."),
):
- await env_not_setup.step(action)
+ await env_not_setup.step(Action(index=0, llm_response="4"))
- # b) Episode ended
- env_episode_ended = MockSingleStepEnv(
+ # b) Step after batch is done
+ env_episode_ended = SingleStepEnv(
dataset=dataset,
- verifier=mock_verifier_fail,
- extractor=mock_extractor,
+ verifier=create_mock_verifier(),
)
await env_episode_ended.setup()
- await env_episode_ended.reset()
- env_episode_ended._episode_ended = True
+ await env_episode_ended.reset(batch_size=1)
+ env_episode_ended._states_done = [True] # Simulate batch completion
+ with pytest.raises(
+ RuntimeError,
+ match=re.escape("Episodes have ended for batch. Call reset() first."),
+ ):
+ await env_episode_ended.step(Action(index=0, llm_response="4"))
+
+ # c) Step without reset
+ env_no_reset = SingleStepEnv(
+ dataset=dataset,
+ verifier=create_mock_verifier(),
+ )
+ await env_no_reset.setup()
with pytest.raises(
RuntimeError,
- match=re.escape("Episode has ended. " "Call reset() first."),
+ match=re.escape("Episodes have ended for batch. Call reset() first."),
):
- await env_episode_ended.step(action)
+ await env_no_reset.step(Action(index=0, llm_response="4"))
- # c) No current observation
- env_no_state = MockSingleStepEnv(
+ # d) Reset before all states processed
+ env_partial_batch = SingleStepEnv(
dataset=dataset,
- verifier=mock_verifier_fail,
- extractor=mock_extractor,
+ verifier=create_mock_verifier(),
)
- await env_no_state.setup()
- env_no_state._state = None
+ await env_partial_batch.setup()
+ await env_partial_batch.reset(batch_size=2)
+ await env_partial_batch.step(
+ Action(index=0, llm_response="4")
+ ) # Process only one state
with pytest.raises(
RuntimeError,
- match=re.escape("No current observation. Call reset() first."),
+ match=re.escape(
+ "reset() called before all states in batch were processed."
+ ),
):
- await env_no_state.step(action)
+ await env_partial_batch.reset(batch_size=1)
+
+ # **4. Test Invalid Actions**
+ env_invalid_actions = SingleStepEnv(
+ dataset=dataset,
+ verifier=create_mock_verifier(),
+ )
+ await env_invalid_actions.setup()
+ await env_invalid_actions.reset(batch_size=2)
+
+ # a) Action with invalid index (out of range)
+ with pytest.raises(ValueError, match="Invalid state index 2."):
+ await env_invalid_actions.step(Action(index=2, llm_response="4"))
+
+ # b) Actions with duplicate indices
+ with pytest.raises(
+ ValueError, match="Duplicate state indices in actions."
+ ):
+ await env_invalid_actions.step(
+ [
+ Action(index=0, llm_response="4"),
+ Action(index=0, llm_response="Paris"),
+ ]
+ )
+
+ # c) Action on already finished state
+ await env_invalid_actions.step(Action(index=0, llm_response="4"))
+ with pytest.raises(
+ ValueError, match="State at index 0 is already finished."
+ ):
+ await env_invalid_actions.step(Action(index=0, llm_response="Paris"))
+
+ # **5. Test Batch Size Issues**
+ env_large_batch = SingleStepEnv(
+ dataset=dataset,
+ verifier=create_mock_verifier(),
+ )
+ await env_large_batch.setup()
+ with pytest.raises(
+ ValueError, match="Batch size 9 is too large for dataset of size 8"
+ ):
+ await env_large_batch.reset(batch_size=9)
+
+ # **6. Test Setup and Close Failures**
+ # a) Setup failure
+ mock_verifier_setup_fail = MagicMock()
+ mock_verifier_setup_fail.setup = AsyncMock(
+ side_effect=Exception("Setup failed")
+ )
+ env_setup_fail = SingleStepEnv(
+ dataset=dataset,
+ verifier=mock_verifier_setup_fail,
+ )
+ with pytest.raises(Exception, match="Setup failed"):
+ await env_setup_fail.setup()
+
+ # b) Close failure
+ mock_verifier_close_fail = MagicMock()
+ mock_verifier_close_fail.setup = AsyncMock()
+ mock_verifier_close_fail.cleanup = AsyncMock(
+ side_effect=Exception("Cleanup failed")
+ )
+ env_close_fail = SingleStepEnv(
+ dataset=dataset,
+ verifier=mock_verifier_close_fail,
+ )
+ await env_close_fail.setup()
+ with pytest.raises(Exception, match="Cleanup failed"):
+ await env_close_fail.close()
diff --git a/test/verifiers/test_base_verifier.py b/test/verifiers/test_base_verifier.py
index f009c2a90..715fff7bb 100644
--- a/test/verifiers/test_base_verifier.py
+++ b/test/verifiers/test_base_verifier.py
@@ -25,7 +25,7 @@ from camel.verifiers.models import VerificationOutcome, VerificationResult
class TestVerifier(BaseVerifier):
r"""Concrete implementation of BaseVerifier for testing."""
- async def _setup(self) -> None:
+ async def _setup(self, **kwargs) -> None:
self.setup_called = True
async def _cleanup(self) -> None:
@@ -103,7 +103,7 @@ async def test_verifier_setup_error():
r"""Test handling of errors during setup."""
class ErrorVerifier(TestVerifier):
- async def _setup(self) -> None:
+ async def _setup(self, **kwargs) -> None:
raise RuntimeError("Simulated setup error")
verifier = ErrorVerifier()
@@ -251,8 +251,6 @@ async def test_verify_batch(test_verifier):
)
with patch.object(test_verifier, "verify", side_effect=mock_verify):
- from camel.verifiers.base import BaseVerifier
-
test_verifier.verify_batch = (
lambda *args, **kwargs: BaseVerifier.verify_batch(
test_verifier, *args, **kwargs
@@ -293,8 +291,6 @@ async def test_verify_batch_with_error_handling(test_verifier):
)
with patch.object(test_verifier, "verify", side_effect=mock_verify):
- from camel.verifiers.base import BaseVerifier
-
test_verifier.verify_batch = (
lambda *args, **kwargs: BaseVerifier.verify_batch(
test_verifier, *args, **kwargs
diff --git a/test/verifiers/test_python_verifier.py b/test/verifiers/test_python_verifier.py
index 525d10fb9..4bed297e6 100644
--- a/test/verifiers/test_python_verifier.py
+++ b/test/verifiers/test_python_verifier.py
@@ -28,13 +28,15 @@ def python_verifier():
@pytest.mark.asyncio
-async def test_python_verifier_setup_and_cleanup(python_verifier):
+async def test_python_verifiersetup_andcleanup(
+ python_verifier: PythonVerifier,
+):
r"""Test setup and cleanup of PythonVerifier."""
- await python_verifier._setup()
+ await python_verifier.setup(uv=True)
assert python_verifier.venv_path is not None
assert os.path.exists(python_verifier.venv_path)
- await python_verifier._cleanup()
+ await python_verifier.cleanup()
assert python_verifier.venv_path is None or not os.path.exists(
python_verifier.venv_path
)
@@ -43,23 +45,20 @@ async def test_python_verifier_setup_and_cleanup(python_verifier):
@pytest.mark.asyncio
async def test_python_verifier_execution_success(python_verifier):
r"""Test successful execution of a Python script."""
- await python_verifier._setup()
+ await python_verifier.setup(uv=True)
script = 'print("Hello, World!")'
- result = await python_verifier._verify_implementation(
- solution=script, ground_truth=None
- )
-
+ result = await python_verifier.verify(solution=script, ground_truth=None)
assert result.status == VerificationOutcome.SUCCESS
assert result.result == "Hello, World!"
- await python_verifier._cleanup()
+ await python_verifier.cleanup()
@pytest.mark.asyncio
async def test_python_verifier_execution_failure(python_verifier):
r"""Test execution failure due to syntax error."""
- await python_verifier._setup()
+ await python_verifier.setup(uv=True)
script = 'print("Hello"' # Syntax error
result = await python_verifier._verify_implementation(
@@ -69,14 +68,14 @@ async def test_python_verifier_execution_failure(python_verifier):
assert result.status == VerificationOutcome.ERROR
assert "SyntaxError" in result.error_message
- await python_verifier._cleanup()
+ await python_verifier.cleanup()
@pytest.mark.asyncio
async def test_python_verifier_execution_timeout(python_verifier):
r"""Test execution timeout handling."""
python_verifier.timeout = 1 # Short timeout for testing
- await python_verifier._setup()
+ await python_verifier.setup(uv=True)
script = 'import time; time.sleep(5)'
result = await python_verifier._verify_implementation(
@@ -86,39 +85,42 @@ async def test_python_verifier_execution_timeout(python_verifier):
assert result.status == VerificationOutcome.TIMEOUT
assert "Execution timed out" in result.error_message
- await python_verifier._cleanup()
+ await python_verifier.cleanup()
@pytest.mark.asyncio
async def test_python_verifier_output_mismatch(python_verifier):
r"""Test verification when output does not match ground truth."""
- await python_verifier._setup()
+ await python_verifier.setup(uv=True)
script = 'print("Wrong output")'
result = await python_verifier._verify_implementation(
- solution=script, ground_truth="Expected output"
+ solution=script, ground_truth="'Expected output'"
)
assert result.status == VerificationOutcome.FAILURE
- assert result.error_message == "Output doesn't match ground truth"
+ assert (
+ result.error_message
+ == "Output mismatch: Wrong output != Expected output"
+ )
- await python_verifier._cleanup()
+ await python_verifier.cleanup()
@pytest.mark.asyncio
async def test_python_verifier_correct_output_matching(python_verifier):
r"""Test verification when output matches ground truth."""
- await python_verifier._setup()
+ await python_verifier.setup(uv=True)
script = 'print("Expected output")'
result = await python_verifier._verify_implementation(
- solution=script, ground_truth="Expected output"
+ solution=script, ground_truth="'Expected output'"
)
assert result.status == VerificationOutcome.SUCCESS
assert result.result == "Expected output"
- await python_verifier._cleanup()
+ await python_verifier.cleanup()
@pytest.mark.asyncio
@@ -137,7 +139,7 @@ async def test_python_verifier_no_venv_error():
@pytest.mark.asyncio
async def test_python_verifier_with_numpy(python_verifier):
r"""Test execution with numpy installed in the virtual environment."""
- await python_verifier._setup()
+ await python_verifier.setup(uv=True)
script = 'import numpy as np; print(np.array([1, 2, 3]))'
result = await python_verifier._verify_implementation(
@@ -146,4 +148,4 @@ async def test_python_verifier_with_numpy(python_verifier):
assert result.status == VerificationOutcome.SUCCESS
assert "[1 2 3]" in result.result
- await python_verifier._cleanup()
+ await python_verifier.cleanup()