🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
docs: Add cookbook for how to use `SingleStepEnv` (#1922)
hallerite59dff6aa6d83
diff --git a/camel/datasets/static_dataset.py b/camel/datasets/static_dataset.pyindex 8949f0fbd..9ff726746 100644--- a/camel/datasets/static_dataset.py+++ b/camel/datasets/static_dataset.py@@ -153,17 +153,6 @@ class StaticDataset(Dataset):return Nonerationale = item.get('rationale')- if not isinstance(rationale, str):- if self._strict:- raise ValueError(- f"Sample at index {idx} has invalid 'rationale': "- f"expected str, got {type(rationale)}"- )- else:- logger.warning(- f"Skipping sample at index {idx}: invalid 'rationale'"- )- return Nonefinal_answer = item.get('final_answer')if not isinstance(final_answer, str):@@ -207,25 +196,33 @@ class StaticDataset(Dataset):r"""Return the size of the dataset."""return self._length- def __getitem__(self, idx: int) -> DataPoint:- r"""Retrieve a datapoint by index.+ def __getitem__(+ self, idx: Union[int, slice]+ ) -> Union[DataPoint, List[DataPoint]]:+ r"""Retrieve a datapoint or a batch of datapoints by index or slice.Args:- idx (int): Index of the datapoint.+ idx (Union[int, slice]): Index or slice of the datapoint(s).Returns:- DataPoint: The datapoint corresponding to the given index.+ List[DataPoint]: A list of `DataPoint` objects.Raises:- IndexError: If :obj:`idx` is out of bounds (negative or greater- than dataset length - 1).+ IndexError: If an integer `idx` is out of bounds."""+ if isinstance(idx, int):+ if idx < 0 or idx >= self._length:+ raise IndexError(+ f"Index {idx} out of bounds for dataset "+ f"of size {self._length}"+ )+ return self.data[idx]- if idx < 0 or idx >= self._length:- raise IndexError(- f"Index {idx} out of bounds for dataset of size {self._length}"- )- return self.data[idx]+ elif isinstance(idx, slice):+ return self.data[idx.start : idx.stop : idx.step]++ else:+ raise TypeError(f"Indexing type {type(idx)} not supported.")def sample(self) -> DataPoint:r"""Sample a random datapoint from the dataset.@@ -240,7 +237,12 @@ class StaticDataset(Dataset):if self._length == 0:raise RuntimeError("Dataset is empty, cannot sample.")idx = self._rng.randint(0, self._length - 1)- return self[idx]+ sample = self[idx]+ if not isinstance(sample, DataPoint):+ raise TypeError(+ f"Expected DataPoint instance, got {type(sample).__name__}"+ )+ return sample@propertydef metadata(self) -> Dict[str, Any]:diff --git a/camel/environments/models.py b/camel/environments/models.pyindex 3de9bc9b1..fe99904a2 100644--- a/camel/environments/models.py+++ b/camel/environments/models.py@@ -33,6 +33,9 @@ class Action(BaseModel):generated (UTC)."""+ index: int = Field(+ ..., description="Index of the state this action is performed upon"+ )llm_response: str = Field(description="Generated response from the LLM")metadata: Dict[str, Any] = Field(default_factory=dict,diff --git a/camel/environments/single_step.py b/camel/environments/single_step.pyindex 814bd28db..f6a4445b8 100644--- a/camel/environments/single_step.py+++ b/camel/environments/single_step.py@@ -12,12 +12,10 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========--from abc import abstractmethod-from typing import Any, Dict, Optional, Tuple, Union+import random+from typing import Any, Dict, List, Optional, Tuple, Union, castfrom camel.datasets import BaseGenerator, DataPoint, StaticDataset-from camel.extractors.base import BaseExtractorfrom camel.logger import get_loggerfrom camel.verifiers.base import (BaseVerifier,@@ -30,18 +28,23 @@ logger = get_logger(__name__)class SingleStepEnv:- r"""A single-step environment for reinforcement learning with LLMs.+ r"""A lightweight environment for single-step RL with LLMs as policy.++ This environment models a single interaction between an LLM-based agent+ and a problem drawn from a dataset—such as a question-answering or+ math problem—where the agent produces one response and receives feedback.++ Core Flow:+ - A question is sampled from a (possibly infinitely long) dataset.+ - The LLM generates a single-step response (the action).+ - The response is verified against the ground truth.+ - A reward is computed based on correctness and optional custom logic.Key Features:- - Samples questions from a dataset and asks the LLM- - Extracts verifiable information from model responses.- - Verifies extracted responses against ground truth.- - Computes and assigns rewards based on correctness.- - Supports async setup, teardown, and cleanup of resources.-- This class is intended as a foundation for RL experiments involving- LLM-based policies, ensuring structured interactions between model- actions and verification mechanisms.+ - Batched evaluation with per-sample state tracking.+ - Async setup and teardown for verifiers and related resources.+ - Supports deterministic sampling via local RNG (optional seed).+ - Extensible reward computation via subclassing."""PLACEHOLDER_OBS = Observation(@@ -54,43 +57,47 @@ class SingleStepEnv:self,dataset: Union[StaticDataset, BaseGenerator],verifier: BaseVerifier,- extractor: BaseExtractor,**kwargs,) -> None:- r"""Initialize the environment.+ r"""Initialize the SingleStepEnv.Args:- dataset: Dataset to sample questions from.- verifier: Verifier to check responses.- extractor: Extractor to process LLM responses.- **kwargs: Additional environment parameters.+ dataset (Union[StaticDataset, BaseGenerator]): Dataset to sample+ problems from.+ verifier (BaseVerifier): Verifier used to evaluate LLM responses+ against ground-truth answers.+ **kwargs: Optional metadata or configuration values.++ Notes:+ This class assumes all interactions are single-step: one question,+ one LLM response, one reward."""self.dataset = datasetself.verifier = verifier- self.extractor = extractorself._metadata = kwargs# State trackingself._is_setup: bool = False- self._state: Optional[DataPoint] = None- self._episode_ended: bool = False+ self._states: List[DataPoint] = []+ self._states_done: List[bool] = []+ self.current_batch_size: int = 0async def setup(self) -> None:- r"""Set up the environment by initializing the verifier and extractor.+ r"""Set up the environment by initializing the verifier.This method ensures that the environment is ready for interaction.- It sets up necessary components, including the verifier and extractor.+ It sets up necessary components, including the verifier.Raises:Exception: If setup fails due to an internal error."""if self._is_setup:+ logger.warning("Environment has already been set up")returntry:await self.verifier.setup()- await self.extractor.setup()self._is_setup = Truelogger.info('Environment setup completed successfully')@@ -101,7 +108,7 @@ class SingleStepEnv:async def close(self) -> None:r"""Clean up and close all resources used by the environment.- This method shuts down the verifier and extractor, resets the internal+ This method shuts down the verifier, resets the internalstate, and ensures that the environment is properly closed.Raises:@@ -109,170 +116,250 @@ class SingleStepEnv:"""if not self._is_setup:+ logger.warning(+ "Not closing environment - has not been set up yet."+ )returntry:self._is_setup = Falseawait self.verifier.cleanup()- await self.extractor.cleanup()- self._state = None- self._episode_ended = False+ self._states = []+ self._states_done = []logger.info('Environment closed successfully')except Exception as e:logger.error(f'Failed to close environment: {e}')raise- async def reset(self) -> Observation:- r"""Reset the environment and start a new episode.+ async def reset(+ self, batch_size: int = 1, seed: Optional[int] = None+ ) -> Union[Observation, List[Observation]]:+ r"""Resets the environment and starts a new episode.++ This method samples a new batch of data points from the dataset and+ returns the corresponding initial observations.- This method samples a new data point from the dataset and returns the- initial observation.+ If a seed is provided, a local random number generator is initialized+ for deterministic sampling. The global random state is not affected.++ Args:+ batch_size (int): Number of data points to sample.+ (default: :obj:`1`)+ seed (Optional[int]): Seed for deterministic sampling. If None,+ sampling is non-deterministic. (default: :obj:`None`)Returns:- Observation: The first observation of the new episode, including- the question.+ Observation or List[Observation]: Initial observation(s) for the+ episode.Raises:- Exception: If the environment is not set up properly.+ RuntimeError: If called before all previous states are processed.+ ValueError: If batch size exceeds dataset size.+ TypeError: If the dataset is of an unsupported type."""if not self._is_setup:+ logger.warning(+ "reset() called on un-setup environment. Setting up..."+ )await self.setup()- self._episode_ended = False-- # Sample a datapoint-- self._state = self.dataset.sample()-- observation = Observation(- question=self._state.question, context={}, metadata={}- )-- return observation-- async def step(self, action: Action) -> StepResult:- r"""Take a step in the environment using the given action.-- This method processes the LLM response, extracts verifiable content,- verifies correctness, computes rewards, and ends the episode.+ if self._batch_started() and not self._batch_done():+ logger.error(+ "Reset called before all states were processed. "+ "Call step on remaining states first."+ )+ raise RuntimeError(+ "reset() called before all states in batch were processed."+ )++ if seed is not None:+ rng = random.Random(seed)+ else:+ rng = random.Random()++ if isinstance(self.dataset, StaticDataset):+ dataset_len = len(self.dataset)++ if batch_size > dataset_len:+ raise ValueError(+ f"Batch size {batch_size} is too large for dataset "+ f"of size {dataset_len}"+ )++ start_idx = rng.randint(0, dataset_len - batch_size)+ idx_slice = slice(start_idx, start_idx + batch_size)+ val = self.dataset[idx_slice]+ self._states = [val] if isinstance(val, DataPoint) else val++ self.current_batch_size = len(self._states)+ self._states_done = [False] * self.current_batch_size++ observations = [+ Observation(question=sample.question, context={}, metadata={})+ for sample in self._states+ ]++ return observations[0] if batch_size == 1 else observations++ elif isinstance(self.dataset, BaseGenerator):+ raise NotImplementedError(+ "Reset not yet implemented for BaseGenerator datasets."+ )++ else:+ raise TypeError(f"Unsupported dataset type: {type(self.dataset)}")++ async def step(+ self, action: Union[Action, List[Action]]+ ) -> Union[StepResult, List[StepResult]]:+ r"""Process actions for a subset of states and update their+ finished status.Args:- action (Action): The action containing the LLM response to- evaluate.+ action: Single action or list of actions, where each action+ contains an index indicating which state it corresponds to.+ The index must be a valid position in the internal _states list+ that was populated during the reset() call.+Returns:- StepResult: Contains the next observation (placeholder), total- reward, reward breakdown, completion flag, and additional- information.+ Union[StepResult, List[StepResult]]: StepResult or list of+ StepResults for the processed states.Raises:- RuntimeError: If the environment is not set up, the episode has- ended, or there is no valid current observation.+ RuntimeError: If environment isn't set up or episode has ended.+ ValueError: If indices are invalid, duplicate, or correspond to+ finished states."""-if not self._is_setup:raise RuntimeError("Environment not set up. Call setup() first.")- if self._episode_ended:- raise RuntimeError("Episode has ended. Call reset() first.")- if self._state is None:+ if self._batch_done():+ raise RuntimeError(+ "Episodes have ended for batch. Call reset() first."+ )+ if not self._states:raise RuntimeError("No current observation. Call reset() first.")- # extract verifiable part from llm response- extraction_result = await self.extractor.extract(action.llm_response)-- if not extraction_result:- raise RuntimeError(f"Couldn't extract from {action.llm_response}")-- # verify the extracted- verification_result = await self.verifier.verify(- solution=extraction_result, ground_truth=self._state.final_answer+ # Normalize everything to list+ actions = [action] if isinstance(action, Action) else action+ indices = [act.index for act in actions]++ if len(set(indices)) != len(indices):+ raise ValueError("Duplicate state indices in actions.")+ for idx in indices:+ if idx < 0 or idx >= len(self._states):+ raise ValueError(f"Invalid state index {idx}.")+ if self._states_done[idx]:+ raise ValueError(f"State at index {idx} is already finished.")++ num_actions = len(actions)++ if self.current_batch_size % num_actions != 0:+ logger.warning(+ f"Number of actions ({num_actions}) is not a divisor of "+ f"total batch size ({self.current_batch_size})"+ )++ proposed_solutions = [act.llm_response for act in actions]+ ground_truths: List[str] = [+ self._states[idx].final_answer for idx in indices+ ]+ verification_results = await self.verifier.verify_batch(+ solutions=proposed_solutions,+ ground_truths=cast(+ list[str | None], ground_truths+ ), # to satisfy mypy+ raise_on_error=True,)- # compute rewards- total_reward, rewards_dict = await self._compute_reward(- action, extraction_result, verification_result+ total_rewards, rewards_dicts = await self._compute_reward_batch(+ proposed_solutions, verification_results)- self._episode_ended = True-- return StepResult(- observation=self.PLACEHOLDER_OBS,- reward=total_reward,- rewards_dict=rewards_dict,- done=True,- info={- "extraction_result": extraction_result,- "verification_result": verification_result,- "state": self._state,- },- )-- async def _compute_reward(+ step_results = []+ # TODO: batch this+ for i, action in enumerate(actions):+ idx = action.index+ step_result = StepResult(+ observation=self.PLACEHOLDER_OBS,+ reward=total_rewards[i],+ rewards_dict=rewards_dicts[i],+ done=True,+ info={+ "proposed_solution": proposed_solutions[i],+ "verification_result": verification_results[i],+ "state": self._states[idx],+ },+ )+ step_results.append(step_result)+ self._states_done[idx] = True++ return step_results[0] if len(step_results) == 1 else step_results++ async def _compute_reward_batch(self,- action: Action,- extraction_result: str,- verification_result: VerificationResult,- ) -> Tuple[float, Dict[str, float]]:- r"""Compute reward scores based on verification results.-- This method calculates the reward based on correctness and any- additional custom reward components.+ proposed_solutions: List[str],+ verification_results: List[VerificationResult],+ ) -> Tuple[List[float], List[Dict[str, float]]]:+ r"""Compute rewards for a batch of proposed solutions based on+ verification results.Args:- action (Action): The action taken in the environment.- extraction_result (str): The extracted verifiable content from the- LLM response.- verification_result (VerificationResult): The result of verifying- the extracted response.+ proposed_solutions (List[str]): List of LLM-generated responses to+ evaluate.+ verification_results (List[VerificationResult]): List of+ verification outcomes for each solution.Returns:- Tuple[float, Dict[str, float]]: A tuple containing:- - Total reward (float)- - Dictionary of individual reward components.-- Raises:- Exception: If an error occurs while computing rewards.+ Tuple containing:+ - List of total rewards for each solution.+ - List of reward component dictionaries for each solution."""+ total_rewards = []+ rewards_dicts = []- rewards: Dict[str, float] = {}+ for solution, verification_result in zip(+ proposed_solutions, verification_results+ ):+ rewards: Dict[str, float] = {}- rewards["correctness"] = (- self.ACCURACY_REWARD if verification_result.status else 0.0- )+ rewards["correctness"] = (+ self.ACCURACY_REWARD if verification_result.status else 0.0+ )- further_rewards = await self._compute_custom_reward(- action, extraction_result, verification_result- )+ further_rewards = await self._compute_custom_reward(+ solution, verification_result+ )+ rewards = {**rewards, **further_rewards}- rewards = rewards | further_rewards+ total_reward = sum(rewards.values())+ total_rewards.append(total_reward)+ rewards_dicts.append(rewards)- return sum(rewards.values()), rewards+ return total_rewards, rewards_dicts- @abstractmethodasync def _compute_custom_reward(- self,- action: Action,- extraction_result: str,- verification_result: VerificationResult,+ self, proposed_solution: str, verification_result: VerificationResult) -> Dict[str, float]:- r"""Compute additional custom reward components.+ r"""Compute additional custom reward components for a single solution.- This method should be implemented by subclasses to define- domain-specific reward calculations.+ To be overridden by subclasses for domain-specific rewards.Args:- action (Action): The action taken in the environment.- extraction_result (str): The extracted verifiable content from the- LLM response.- verification_result (VerificationResult): The result of verifying- the extracted response.+ proposed_solution (str): The LLM-generated response.+ verification_result (VerificationResult): The verification outcome.Returns:- Dict[str, float]: A dictionary mapping custom reward categories- to their values.+ Dict[str, float]: Dictionary of custom reward components."""- pass+ return {}++ def _batch_done(self) -> bool:+ return all(self._states_done)++ def _batch_started(self) -> bool:+ return any(self._states_done)@propertydef metadata(self) -> Dict[str, Any]:diff --git a/camel/extractors/__init__.py b/camel/extractors/__init__.pyindex 51ab870ac..6a6a1bfe4 100644--- a/camel/extractors/__init__.py+++ b/camel/extractors/__init__.py@@ -12,5 +12,20 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========from .base import BaseExtractor, BaseExtractorStrategy+from .python_strategies import (+ BoxedStrategy,+ PythonDictStrategy,+ PythonListStrategy,+ PythonSetStrategy,+ PythonTupleStrategy,+)-__all__ = ["BaseExtractor", "BaseExtractorStrategy"]+__all__ = [+ "BaseExtractor",+ "BaseExtractorStrategy",+ "BoxedStrategy",+ "PythonListStrategy",+ "PythonDictStrategy",+ "PythonSetStrategy",+ "PythonTupleStrategy",+]diff --git a/camel/verifiers/base.py b/camel/verifiers/base.pyindex 687ca6756..16b80a772 100644--- a/camel/verifiers/base.py+++ b/camel/verifiers/base.py@@ -16,6 +16,7 @@ import timefrom abc import ABC, abstractmethodfrom typing import List, Optional+from camel.extractors.base import BaseExtractorfrom camel.logger import get_loggerfrom camel.utils import BatchProcessor@@ -44,6 +45,7 @@ class BaseVerifier(ABC):def __init__(self,+ extractor: Optional[BaseExtractor] = None,max_parallel: Optional[int] = None,timeout: Optional[float] = None,max_retries: int = 3,@@ -72,6 +74,9 @@ class BaseVerifier(ABC):down. (default: :obj:`85.0`)**kwargs: Additional verifier parameters."""++ self.extractor = extractor+self._is_setup: bool = Falseself._max_parallel: Optional[int] = max_parallelself._timeout: Optional[float] = timeout@@ -82,7 +87,7 @@ class BaseVerifier(ABC):self._memory_threshold: float = memory_thresholdself._batch_processor: BatchProcessor = BatchProcessor()- async def setup(self) -> None:+ async def setup(self, **kwargs) -> None:r"""Set up the verifier with necessary resources.Initializes:@@ -97,6 +102,8 @@ class BaseVerifier(ABC):returntry:+ if self.extractor:+ await self.extractor.setup()batch_size = max(1, self._initial_batch_size or 10)max_parallel = max(1, self._max_parallel or 1)self._batch_processor = BatchProcessor()@@ -106,7 +113,7 @@ class BaseVerifier(ABC):f"batch_size={batch_size}, max_parallel={max_parallel}")- await self._setup()+ await self._setup(**kwargs)self._is_setup = Trueexcept Exception as e:@@ -118,7 +125,7 @@ class BaseVerifier(ABC):raise RuntimeError(error_msg) from e@abstractmethod- async def _setup(self) -> None:+ async def _setup(self, **kwargs) -> None:r"""Implement verifier-specific setup logic."""pass@@ -136,6 +143,8 @@ class BaseVerifier(ABC):returntry:+ if self.extractor:+ await self.extractor.cleanup()self._batch_processor = BatchProcessor()await self._cleanup()logger.info(f"{self.__class__.__name__} cleaned up successfully")@@ -191,15 +200,28 @@ class BaseVerifier(ABC):start_time = time.time()while attempt < self._max_retries:+ # Extract verifiable part of the proposed solution,+ # if verifier has been initialized with extractor.+ verifiable_solution = (+ await self.extractor.extract(solution)+ if self.extractor+ else solution+ )++ if not verifiable_solution:+ continue+try:verification_result = (await asyncio.wait_for(- self._verify_implementation(solution, ground_truth),+ self._verify_implementation(+ verifiable_solution, ground_truth+ ),timeout=self._timeout,)if self._timeoutelse await self._verify_implementation(- solution, ground_truth+ verifiable_solution, ground_truth))@@ -267,6 +289,7 @@ class BaseVerifier(ABC):"Subclasses must implement _verify_implementation()")+ # TODO: check againasync def verify_batch(self,solutions: List[str],diff --git a/camel/verifiers/python_verifier.py b/camel/verifiers/python_verifier.pyindex 627c5b51f..d653ef322 100644--- a/camel/verifiers/python_verifier.py+++ b/camel/verifiers/python_verifier.py@@ -12,15 +12,16 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import astimport asyncioimport osimport shutilimport subprocess+import sysimport tempfileimport venv-from typing import List, Optional+from typing import List, Optional, Tuple-from camel.extractors import BaseExtractorfrom camel.logger import get_loggerfrom camel.verifiers import BaseVerifier@@ -48,7 +49,6 @@ class PythonVerifier(BaseVerifier):self,timeout: Optional[float] = 30.0,required_packages: Optional[List[str]] = None,- extractor: Optional[BaseExtractor] = None,):r"""Initializes the PythonVerifier.@@ -69,25 +69,39 @@ class PythonVerifier(BaseVerifier):else: # Unix-like systemsself.bin_dir = 'bin'- async def _setup(self) -> None:- r"""Set up a virtual environment for execution- and install required packages.- """+ async def _setup(self, **kwargs) -> None:+ r"""Set up a virtual environment and install required packages."""+ uv = kwargs.get('uv', False)+ if uv or self._is_uv_environment():+ logger.info("[UV] Detected uv environment. Using uv for setup.")+ self._setup_with_uv()+ return+self.venv_path = tempfile.mkdtemp()- venv.create(self.venv_path, with_pip=True)- logger.info(f"Virtual environment created at {self.venv_path}")+ try:+ venv.create(self.venv_path, with_pip=True)+ logger.info(f"Virtual environment created at {self.venv_path}")+ except Exception as e:+ logger.error(f"Failed to create virtual environment: {e}")+ # Clean up resources before re-raising+ if self.venv_path and os.path.exists(self.venv_path):+ shutil.rmtree(self.venv_path)+ self.venv_path = None+ raisevenv_pip = os.path.join(self.venv_path, self.bin_dir, "pip")if self.required_packages:try:+ # Add timeout to subprocess callsubprocess.run([venv_pip, "install", *self.required_packages],check=True,capture_output=True,+ timeout=self._timeout,)logger.info(- "Installed required packages:"+ "Installed required packages: "f"{', '.join(self.required_packages)}")except subprocess.CalledProcessError as e:@@ -95,6 +109,97 @@ class PythonVerifier(BaseVerifier):"Failed to install required packages: "f"{e.stderr.decode().strip()}")+ # Clean up resources before re-raising+ if self.venv_path and os.path.exists(self.venv_path):+ shutil.rmtree(self.venv_path)+ self.venv_path = None+ raise+ except subprocess.TimeoutExpired:+ logger.error(+ f"Package installation timed out "+ f"after {self._timeout} seconds"+ )+ if self.venv_path and os.path.exists(self.venv_path):+ shutil.rmtree(self.venv_path)+ self.venv_path = None+ raise++ def _is_uv_environment(self) -> bool:+ r"""Detect whether the current Python runtime is managed by uv."""+ return "UV_CACHE_DIR" in os.environ or "uv" in sys.executable++ def _setup_with_uv(self) -> None:+ r"""Create virtual environment and install packages using uv."""+ self.venv_path = tempfile.mkdtemp()+ try:+ subprocess.run(+ ["uv", "venv", self.venv_path],+ check=True,+ capture_output=True,+ timeout=self._timeout,+ )+ logger.info(+ f"[UV] Virtual environment created at {self.venv_path}"+ )+ except subprocess.CalledProcessError as e:+ logger.error(+ "[UV] Failed to create virtual environment:\n"+ f"{e.stderr.decode().strip()}"+ )+ # Clean up resources before re-raising+ if self.venv_path and os.path.exists(self.venv_path):+ shutil.rmtree(self.venv_path)+ self.venv_path = None+ raise+ except subprocess.TimeoutExpired:+ logger.error(+ f"[UV] Virtual environment creation timed "+ f"out after {self._timeout} seconds"+ )+ if self.venv_path and os.path.exists(self.venv_path):+ shutil.rmtree(self.venv_path)+ self.venv_path = None+ raise++ if self.required_packages:+ venv_python = os.path.join(self.venv_path, self.bin_dir, "python")+ try:+ subprocess.run(+ [+ "uv",+ "pip",+ "install",+ "--python",+ venv_python,+ *self.required_packages,+ ],+ check=True,+ capture_output=True,+ timeout=self._timeout,+ )+ logger.info(+ "[UV] Installed required packages via uv: "+ f"{', '.join(self.required_packages)}"+ )+ except subprocess.CalledProcessError as e:+ logger.error(+ "[UV] Failed to install required packages via uv:\n"+ f"{e.stderr.decode().strip()}"+ )+ # Clean up resources before re-raising+ if self.venv_path and os.path.exists(self.venv_path):+ shutil.rmtree(self.venv_path)+ self.venv_path = None+ raise+ except subprocess.TimeoutExpired:+ logger.error(+ f"[UV] Package installation timed "+ f"out after {self._timeout} seconds"+ )+ if self.venv_path and os.path.exists(self.venv_path):+ shutil.rmtree(self.venv_path)+ self.venv_path = None+ raiseasync def _cleanup(self) -> None:r"""Clean up the virtual environment."""@@ -106,32 +211,27 @@ class PythonVerifier(BaseVerifier):async def _verify_implementation(self, solution: str, ground_truth: Optional[str]) -> VerificationResult:- r"""Executes and verifies the LLM-generated Python solution in an- isolated virtual environment.+ r"""Executes the provided Python solution in an isolated environment+ and verifies its output against an expected ground truth expression.++ This method runs the solution in a subprocess inside a virtual+ environment. The ground truth is assumed to be a pure Python+ expression and is evaluated directly in the verifier process.- This method runs the given Python solution inside a controlled virtual- environment, captures its execution output, and optionally compares it- against a provided ground truth. Handles timeouts and execution errors.+ If both executions are successful, the actual output is compared+ against the evaluated ground truth using semantic equality. If+ evaluation fails, string comparison is used as a fallback.Args:- solution (str): The Python code to execute and verify.- ground_truth (Optional[str]): The expected output for comparison.- If None, verification is based only on execution success.+ solution (str): The Python code or expression to execute and+ verify.+ ground_truth (Optional[str]): The expected value as a Python+ expression. If None, only execution success is verified.Returns:- VerificationResult: A structured object containing:- - status (VerificationOutcome): SUCCESS, FAILURE, ERROR,- or TIMEOUT.- - result (str): The execution output of the solution.- - error_message (Optional[str]): Captured error message,- if any.- - duration (float, optional): Execution time (set externally).-- Raises:- asyncio.TimeoutError: If execution exceeds the configured timeout.- Exception: Any unexpected errors are caught and converted to an- ERROR verification result.+ VerificationResult: Result of the verification process."""+ # Check for virtual environment setupif not self.venv_path:return VerificationResult(status=VerificationOutcome.ERROR,@@ -139,9 +239,47 @@ class PythonVerifier(BaseVerifier):error_message="Virtual environment is not set up.",)- script = solution.strip()- venv_python = os.path.join(self.venv_path, self.bin_dir, "python")+ # If the solution is an expression, evaluate it directly+ if self._is_expression(solution):+ try:+ sol_val = ast.literal_eval(solution)+ except Exception as e:+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result="",+ error_message=f"Expression evaluation error: {e}",+ )++ if ground_truth is not None:+ try:+ gt_val = ast.literal_eval(ground_truth)+ except Exception as e:+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result="",+ error_message=f"Ground truth evaluation error: {e}",+ )+ if sol_val == gt_val:+ return VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result=str(sol_val),+ )+ else:+ return VerificationResult(+ status=VerificationOutcome.FAILURE,+ result=str(sol_val),+ error_message="Output mismatch: "+ f"{sol_val} != {gt_val}",+ )+ else:+ return VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result=str(sol_val),+ )+ # Otherwise, run the code block,+ # which should already include a print(...) in the end+ venv_python = os.path.join(self.venv_path, self.bin_dir, "python")if not os.path.exists(venv_python):return VerificationResult(status=VerificationOutcome.ERROR,@@ -150,64 +288,166 @@ class PythonVerifier(BaseVerifier):)try:- process = await asyncio.create_subprocess_exec(- venv_python,- "-c",- script,- stdout=asyncio.subprocess.PIPE,- stderr=asyncio.subprocess.PIPE,- )-- stdout, stderr = await asyncio.wait_for(- process.communicate(), timeout=self._timeout+ sol_out, sol_err, sol_code = await self._run_code_block(+ solution, venv_python)+ if sol_code != 0:+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result=sol_out,+ error_message=f"Solution code error:\n{sol_err}",+ )- output_result = stdout.decode().strip()- error_output = stderr.decode().strip()-- if process.returncode == 0:- # If ground truth is provided, compare it with the result- if ground_truth is not None:- # Normalize both strings by removing extra whitespace- normalized_output = ' '.join(output_result.strip().split())- normalized_truth = ' '.join(- str(ground_truth).strip().split()+ if ground_truth is not None:+ try:+ # First, try to evaluate the output as-is.+ sol_val = ast.literal_eval(sol_out)+ except Exception as e:+ logger.warning(+ f"Direct eval failed: {e}. Trying repr on output.")+ try:+ # Try to convert sol_out to a literal+ # by wrapping it with repr.+ # FIXME: may be unnecessary+ sol_val = ast.literal_eval(repr(sol_out))+ except Exception as e2:+ logger.warning(+ f"repr eval also failed: {e2}."+ "Falling back to string comparison."+ )+ sol_val = None- if normalized_output == normalized_truth:+ if sol_val is not None:+ try:+ gt_val = ast.literal_eval(ground_truth)+ except Exception as e:+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result="",+ error_message="Ground truth evaluation error:"+ f"{e}",+ )+ if sol_val == gt_val:return VerificationResult(status=VerificationOutcome.SUCCESS,- result=output_result,+ result=sol_out,)else:return VerificationResult(status=VerificationOutcome.FAILURE,- error_message="Output doesn't match ground truth",- result=output_result,+ result=sol_out,+ error_message="Output mismatch: "+ f"{sol_val} != {gt_val}",)else:- return VerificationResult(- status=VerificationOutcome.SUCCESS,- result=output_result,- )-+ # Fallback: string comparison+ if sol_out.strip() == ground_truth.strip():+ return VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result=sol_out,+ )+ else:+ return VerificationResult(+ status=VerificationOutcome.FAILURE,+ result=sol_out,+ error_message="Fallback string mismatch: "+ f"'{sol_out}' != '{ground_truth}'",+ )else:return VerificationResult(- status=VerificationOutcome.ERROR,- error_message=error_output,- result=output_result,+ status=VerificationOutcome.SUCCESS,+ result=sol_out,)-except asyncio.TimeoutError:return VerificationResult(status=VerificationOutcome.TIMEOUT,result="",error_message="Execution timed out.",)-except Exception as e:return VerificationResult(status=VerificationOutcome.ERROR,result="",- error_message=f"Execution error: {e}",+ error_message=f"Unexpected error: {e}",)++ async def _run_code_block(+ self, code: str, venv_path: str+ ) -> Tuple[str, str, int]:+ r"""Executes a block of Python code in the virtual environment.++ The code is written to a temporary file, executed using the Python+ interpreter from the specified virtual environment, and+ its output and error streams are captured.++ Args:+ code (str): The Python code to execute.+ venv_path (str): The path to the virtual environment's Python+ binary.++ Returns:+ Tuple[str, str, int]: A tuple containing the stdout output,+ stderr output, and return code from the executed script.+ """+ # No longer checking for expressions since they're handled separately+ with tempfile.NamedTemporaryFile(+ "w+", suffix=".py", delete=False+ ) as tmp:+ tmp.write(code)+ tmp_path = tmp.name++ proc = await asyncio.create_subprocess_exec(+ venv_path,+ tmp_path,+ stdout=asyncio.subprocess.PIPE,+ stderr=asyncio.subprocess.PIPE,+ )+ stdout, stderr = await asyncio.wait_for(+ proc.communicate(), timeout=self._timeout+ )+ os.remove(tmp_path)+ return (+ stdout.decode().strip(),+ stderr.decode().strip(),+ proc.returncode if proc.returncode is not None else -1,+ )++ def _is_expression(self, code: str) -> bool:+ r"""Determines whether a given string of code is a single expression.++ This utility uses Python's AST module to parse the code and checks if+ it consists of a single expression node.++ Args:+ code (str): The Python code to analyze.++ Returns:+ bool: True if the code is a single expression, False otherwise.+ """+ # Skip empty or whitespace-only strings+ if not code or code.isspace():+ return False++ try:+ # First try parsing as an expression - this is more reliable than+ # starting with literal_eval+ tree = ast.parse(code.strip(), mode='eval')+ # Check if it's a function call (like print()) - these should not+ # be treated as expressions+ if isinstance(tree.body, ast.Call):+ return False+ # If parsing succeeds in 'eval' mode and it's not a function call,+ # it's a valid expression+ return True+ except SyntaxError:+ # If parsing as expression fails, it's not a valid expression+ return False+ except Exception:+ # For any other parsing errors, try literal_eval as fallback for+ # simple literals+ try:+ ast.literal_eval(code)+ return True+ except Exception:+ return Falsediff --git a/examples/environments/demo.ipynb b/examples/environments/demo.ipynbnew file mode 100644index 000000000..416885c85--- /dev/null+++ b/examples/environments/demo.ipynb@@ -0,0 +1,211 @@+{+ "cells": [+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "# Single Step Environment in Camel"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Single Step environments are the most widespread type of environment when doing RL with an LLM as policy.\n",+ "\n",+ "It's called *single step* environment, because the agent only does one step. It gets a question sampled from the dataset (the initial state / observation) and then answers. The answer is then scored according to the reward function. Recently, rules-based reward functions, i.e. functions without any learnable parameters, have been successfully used to do RL with LLMs as as policy.\n",+ "\n",+ "First, we have to load a dataset from which we will sample questions. The dataset can be either a `StaticDataset`, which is finite and the length is known at runtime, or it can be a `BaseGenerator`, which is an infinite supply of question - answer pairs, synthetically generated in some way (depending on the implementation).\n",+ "\n",+ "For the sake of simplicity, we will start by loading the MATH dataset, remove unnecessary columns and rename the remaining ones, such that we can easily turn it into a `StaticDataset`, which `SingleStepEnv` can deal with. "+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "/home/student/.cache/pypoetry/virtualenvs/camel-ai-80B0TVyJ-py3.11/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",+ " from .autonotebook import tqdm as notebook_tqdm\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "['question', 'final_answer']\n",+ "Example datapoint: question='Let \\\\[f(x) = \\\\left\\\\{\\n\\\\begin{array}{cl} ax+3, &\\\\text{ if }x>2, \\\\\\\\\\nx-5 &\\\\text{ if } -2 \\\\le x \\\\le 2, \\\\\\\\\\n2x-b &\\\\text{ if } x <-2.\\n\\\\end{array}\\n\\\\right.\\\\]Find $a+b$ if the piecewise function is continuous (which means that its graph can be drawn without lifting your pencil from the paper).' final_answer='For the piecewise function to be continuous, the cases must \"meet\" at $2$ and $-2$. For example, $ax+3$ and $x-5$ must be equal when $x=2$. This implies $a(2)+3=2-5$, which we solve to get $2a=-6 \\\\Rightarrow a=-3$. Similarly, $x-5$ and $2x-b$ must be equal when $x=-2$. Substituting, we get $-2-5=2(-2)-b$, which implies $b=3$. So $a+b=-3+3=\\\\boxed{0}$.' rationale=None metadata=None\n"+ ]+ }+ ],+ "source": [+ "from datasets import load_dataset\n",+ "\n",+ "from camel.datasets import StaticDataset\n",+ "from camel.logger import get_logger\n",+ "\n",+ "logger = get_logger(__name__)\n",+ "\n",+ "\n",+ "dataset = load_dataset(\"EleutherAI/hendrycks_math\", \"algebra\")\n",+ "\n",+ "# Preprocess\n",+ "dataset[\"train\"] = dataset[\"train\"].rename_column(\"problem\", \"question\")\n",+ "dataset[\"train\"] = dataset[\"train\"].rename_column(\"solution\", \"final_answer\")\n",+ "dataset[\"train\"] = dataset[\"train\"].remove_columns([\"type\", \"level\"])\n",+ "\n",+ "# This should now print \"['question', 'final_answer']\"\n",+ "print(dataset[\"train\"].column_names)\n",+ "seed_dataset = StaticDataset(dataset['train'])\n",+ "\n",+ "print(\"Example datapoint:\", seed_dataset[0])"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Next, we will have to define an *extractor*. An extractor takes the LLM response and extracts the verifiable part out of it. Extractors can be initialized with different strategies which modifies the extraction behavior.\n",+ "\n",+ "In the case of the MATH dataset, the final answer is wrapped inside a `\\boxed{...}`, hence we should use the pre-built `BoxedStrategy`.\n",+ "\n",+ "Sadly, MATH answers are rather complicated and a more general Math verifier to compare, for example, equations has not yet been implemented. Hence, we shall prune the dataset to only contain those rows where the content of `\\boxed{...}` is an int. That way, we can do simple verification using the vanilla PythonVerifier!"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Number of datapoints with integer answers: 1228\n"+ ]+ }+ ],+ "source": [+ "from camel.extractors import BaseExtractor, BoxedStrategy\n",+ "\n",+ "# Initialize extractor\n",+ "extractor = BaseExtractor([[BoxedStrategy()]])\n",+ "await extractor.setup()\n",+ "\n",+ "valid_datapoints = []\n",+ "\n",+ "# Iterate through dataset, checking for datapoints with integer answers\n",+ "for datapoint in seed_dataset:\n",+ " extracted_value = await extractor.extract(response=datapoint.final_answer)\n",+ "\n",+ " if not extracted_value:\n",+ " continue\n",+ " if extracted_value.isdigit() or (\n",+ " extracted_value.startswith('-') and extracted_value[1:].isdigit()\n",+ " ):\n",+ " valid_datapoints.append(\n",+ " {\n",+ " \"question\": datapoint.question,\n",+ " \"final_answer\": datapoint.final_answer,\n",+ " }\n",+ " )\n",+ "\n",+ "# We should now have `1228` valid data points.\n",+ "print(f\"Number of datapoints with integer answers: {len(valid_datapoints)}\")\n",+ "\n",+ "filtered_dataset = StaticDataset(valid_datapoints, seed=42)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Let's create a Python verifier to later compare answers."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [],+ "source": [+ "from camel.verifiers import PythonVerifier\n",+ "\n",+ "verifier = PythonVerifier()\n",+ "await verifier.setup()"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Let's now initalize the single step environment with our filtered dataset, our verifier and our extractor.\n",+ "\n",+ "We can then call `env.reset()` to draw from the initial state distribution and return an observation, which can then be fed into the agent."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "from camel.environments import Action, SingleStepEnv\n",+ "\n",+ "env = SingleStepEnv(filtered_dataset, verifier, extractor)\n",+ "\n",+ "obs = await env.reset()\n",+ "\n",+ "print(obs)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "The agent would then process this observation and select an action, which it would feed into the `step` function. An action in this case would simply be the answer to the question, wrapped in `\\boxed{}`"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "await env.step(Action(llm_response=\"\\\\boxed{5}\"))"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "As you can see, the output of the `step` function contains the next observation (which in this case is just a placeholder, since the episode is over), a reward, as well as a reward dict, showing exactly which rubric brought which reward, a `done` flag, indicating that the episode is over and some additional info."+ ]+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": "camel-ai-80B0TVyJ-py3.11",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.11.6"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 2+}diff --git a/examples/verifier/python_verifier_example.py b/examples/verifier/python_verifier_example.pyindex d104085e9..7c6f4979c 100644--- a/examples/verifier/python_verifier_example.py+++ b/examples/verifier/python_verifier_example.py@@ -16,7 +16,7 @@ import asynciofrom camel.verifiers import PythonVerifierverifier = PythonVerifier(required_packages=["numpy"])-asyncio.run(verifier.setup())+asyncio.run(verifier.setup(uv=True))numpy_test_code = """import numpy as np@@ -32,7 +32,7 @@ print(result)result = asyncio.run(verifier.verify(solution=numpy_test_code, ground_truth="32"))-print(f"Result: {result.status}")+print(f"Result: {result}")result = asyncio.run(verifier.verify(solution=numpy_test_code, ground_truth="40")@@ -40,6 +40,6 @@ result = asyncio.run(# Now we expect the VerificationOutcome to be a failure,# because the answer is wrong.-print(f"Result: {result.status}")+print(f"Result: {result}")asyncio.run(verifier.cleanup())diff --git a/test/datasets/test_base_dataset.py b/test/datasets/test_base_dataset.pyindex 9c298b451..75c381246 100644--- a/test/datasets/test_base_dataset.py+++ b/test/datasets/test_base_dataset.py@@ -168,13 +168,11 @@ def test_static_dataset_init_from_hf_dataset():# **Test 2: Initialization with invalid data**# Sub-test 2a: Missing required field with strict=True- invalid_data_missing = [- {"question": "What is 3 + 3?", "final_answer": "6"}- ]+ invalid_data_missing = [{"question": "What is 3 + 3?"}]hf_invalid_missing = HFDataset.from_list(invalid_data_missing)with pytest.raises(ValueError,- match="Sample at index 0 has invalid 'rationale': "+ match="Sample at index 0 has invalid 'final_answer': ""expected str, got <class 'NoneType'>",):StaticDataset(data=hf_invalid_missing, min_samples=1, strict=True)@@ -300,13 +298,12 @@ def test_static_dataset_init_from_pytorch_dataset():invalid_data_missing = [{"question": "What is 3 + 3?",- "final_answer": "6",- } # Missing rationale+ } # Missing final_answer]pytorch_invalid_missing = MockPyTorchDataset(invalid_data_missing)with pytest.raises(ValueError,- match="Sample at index 0 has invalid 'rationale': "+ match="Sample at index 0 has invalid 'final_answer': ""expected str, got <class 'NoneType'>",):StaticDataset(data=pytorch_invalid_missing, min_samples=1, strict=True)@@ -567,15 +564,14 @@ def test_static_dataset_init_from_json_file():invalid_data = [{"question": "What is 3 + 3?",- "final_answer": "6",- } # Missing rationale+ } # Missing final answer]json.dump(invalid_data, tmp_file)tmp_file_path = Path(tmp_file.name)with pytest.raises(ValueError,- match="Sample at index 0 has invalid 'rationale': "+ match="Sample at index 0 has invalid 'final_answer': ""expected str, got <class 'NoneType'>",):StaticDataset(data=tmp_file_path, min_samples=1, strict=True)@@ -587,8 +583,7 @@ def test_static_dataset_init_from_json_file():invalid_data = [{"question": "What is 3 + 3?",- "final_answer": "6",- } # Missing rationale+ } # Missing final_answer]json.dump(invalid_data, tmp_file)tmp_file_path = Path(tmp_file.name)@@ -763,12 +758,11 @@ def test_static_dataset_init_from_jsonl_file():with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as tmp_file:- # Missing 'rationale' field.+ # Missing 'final_answer' field.tmp_file.write(json.dumps({"question": "What is 3 + 3?",- "final_answer": "6",})+ "\n"@@ -777,7 +771,7 @@ def test_static_dataset_init_from_jsonl_file():with pytest.raises(ValueError,- match="Sample at index 0 has invalid 'rationale'",+ match="Sample at index 0 has invalid 'final_answer'",):StaticDataset(data=tmp_file_path, min_samples=1, strict=True)@@ -790,7 +784,6 @@ def test_static_dataset_init_from_jsonl_file():json.dumps({"question": "What is 3 + 3?",- "final_answer": "6",})+ "\n"@@ -878,12 +871,11 @@ def test_static_dataset_init_from_list():invalid_data_missing = [{"question": "What is 3 + 3?",- "final_answer": "6",- } # Missing rationale+ } # Missing final_answer]with pytest.raises(ValueError,- match="Sample at index 0 has invalid 'rationale': "+ match="Sample at index 0 has invalid 'final_answer': ""expected str, got <class 'NoneType'>",):StaticDataset(data=invalid_data_missing, min_samples=1, strict=True)@@ -937,9 +929,8 @@ def test_static_dataset_init_from_list():"final_answer": "Valid",},{- "question": "Invalid question",- "final_answer": "Invalid",- }, # Missing rationale+ "question": "Question",+ }, # Missing final_answer]dataset_mixed = StaticDataset(data=mixed_data, min_samples=1, strict=False)assert (@@ -1025,6 +1016,197 @@ def test_static_dataset_methods():empty_dataset.sample()+def test_static_dataset_getitem_slice():+ mock_data = [+ {+ "question": "What is 1+1?",+ "rationale": "Basic addition.",+ "final_answer": "2",+ },+ {+ "question": "Is the Earth round?",+ "rationale": "Scientific consensus.",+ "final_answer": "yes",+ },+ {+ "question": "What is the capital of Japan?",+ "rationale": "Geography knowledge.",+ "final_answer": "Tokyo",+ },+ {+ "question": "How many sides does a triangle have?",+ "rationale": "Definition of a triangle.",+ "final_answer": "3",+ },+ ]+ dataset = StaticDataset(data=mock_data, min_samples=1, strict=True)++ sliced = dataset[1:4:2]+ assert isinstance(sliced, list), "Slicing should return a list."+ assert len(sliced) == 2, "Slice [1:4:2] should return 2 items."+ assert all(+ isinstance(item, DataPoint) for item in sliced+ ), "All items should be DataPoint instances."+ assert (+ sliced[0].question == "Is the Earth round?"+ ), "Index 1 question should match."+ assert (+ sliced[1].question == "How many sides does a triangle have?"+ ), "Index 3 question should match."+ assert (+ sliced[0] is dataset[1]+ ), "Slicing should return references to the same objects."++ # Test slice with only start and stop+ sliced = dataset[1:3]+ assert len(sliced) == 2, "Slice [1:3] should return 2 items."+ assert (+ sliced[0].question == "Is the Earth round?"+ ), "Index 1 question should match."+ assert (+ sliced[1].question == "What is the capital of Japan?"+ ), "Index 2 question should match."++ # Test slice with negative indices+ sliced = dataset[-2:]+ assert len(sliced) == 2, "Slice [-2:] should return 2 items."+ assert (+ sliced[0].question == "What is the capital of Japan?"+ ), "Index -2 question should match."+ assert (+ sliced[1].question == "How many sides does a triangle have?"+ ), "Index -1 question should match."++ sliced = dataset[:-1]+ assert len(sliced) == 3, "Slice [:-1] should return 3 items."+ assert (+ sliced[0].question == "What is 1+1?"+ ), "Index 0 question should match."+ assert (+ sliced[2].question == "What is the capital of Japan?"+ ), "Index 2 question should match."++ # Test slice with step+ sliced = dataset[::2]+ assert len(sliced) == 2, "Slice [::2] should return 2 items."+ assert (+ sliced[0].question == "What is 1+1?"+ ), "Index 0 question should match."+ assert (+ sliced[1].question == "What is the capital of Japan?"+ ), "Index 2 question should match."++ # Test slice with negative step+ sliced = dataset[3:0:-1]+ assert len(sliced) == 3, "Slice [3:0:-1] should return 3 items."+ assert (+ sliced[0].question == "How many sides does a triangle have?"+ ), "Index 3 question should match."+ assert (+ sliced[2].question == "Is the Earth round?"+ ), "Index 1 question should match."++ sliced = dataset[::-1]+ assert (+ len(sliced) == 4+ ), "Slice [::-1] should return all 4 items in reverse."+ assert (+ sliced[0].question == "How many sides does a triangle have?"+ ), "Index 3 question should match."+ assert (+ sliced[3].question == "What is 1+1?"+ ), "Index 0 question should match."++ # Test slicing with out-of-bound indices+ sliced = dataset[10:20]+ assert (+ sliced == []+ ), "Slice [10:20] should return an empty list when out of bounds."++ sliced = dataset[-10:10]+ assert len(sliced) == 4, "Slice [-10:10] should return all items."+ assert (+ sliced[0].question == "What is 1+1?"+ ), "Index 0 question should match."+ assert (+ sliced[3].question == "How many sides does a triangle have?"+ ), "Index 3 question should match."++ sliced = dataset[2:100]+ assert (+ len(sliced) == 2+ ), "Slice [2:100] should return items from index 2 to end."+ assert (+ sliced[0].question == "What is the capital of Japan?"+ ), "Index 2 question should match."+ assert (+ sliced[1].question == "How many sides does a triangle have?"+ ), "Index 3 question should match."++ # Test non-integer, non-slice inputs+ with pytest.raises(+ TypeError, match="Indexing type <class 'str'> not supported."+ ):+ dataset["invalid"]++ with pytest.raises(+ TypeError, match="Indexing type <class 'float'> not supported."+ ):+ dataset[1.5]++ with pytest.raises(+ TypeError, match="Indexing type <class 'NoneType'> not supported."+ ):+ dataset[None]++ with pytest.raises(+ TypeError, match="Indexing type <class 'list'> not supported."+ ):+ dataset[[1, 2]]++ # Test slice with step=0+ with pytest.raises(ValueError, match="slice step cannot be zero"):+ dataset[::0]++ # Test slice with non-integer start, stop, or step+ invalid_slice = slice("a", 3, 1)+ with pytest.raises(+ TypeError,+ match="slice indices must be integers or None or have "+ "an __index__ method",+ ):+ dataset[invalid_slice]++ invalid_slice = slice(0, "b", 1)+ with pytest.raises(+ TypeError,+ match="slice indices must be integers or None "+ "or have an __index__ method",+ ):+ dataset[invalid_slice]++ invalid_slice = slice(0, 3, "a")+ with pytest.raises(+ TypeError,+ match="slice indices must be integers or None "+ "or have an __index__ method",+ ):+ dataset[invalid_slice]++ # Test slicing an empty dataset+ empty_dataset = StaticDataset(data=[], min_samples=0, strict=True)+ assert len(empty_dataset) == 0, "Empty dataset should have length 0."+ assert (+ empty_dataset[:] == []+ ), "Full slice of empty dataset should return empty list."+ assert (+ empty_dataset[0:0] == []+ ), "Zero-length slice should return empty list."+ assert (+ empty_dataset[1:2] == []+ ), "Out-of-bound slice should return empty list."++@pytest.mark.asyncioasync def test_few_shot_generator():r"""Test FewShotGenerator with mocked components."""diff --git a/test/environments/test_multi_step_env.py b/test/environments/test_multi_step_env.pyindex 81911b922..06a33b687 100644--- a/test/environments/test_multi_step_env.py+++ b/test/environments/test_multi_step_env.py@@ -77,7 +77,8 @@ async def test_multi_step_env_lifecycle():assert observation.question == "Step 0"# Test first step- action1 = Action(llm_response="Response 1")+ action1 = Action(index=0, llm_response="Response 1")+ # FIXME: index set to 0 temporarily until we have batched MultiStepresult1 = await env.step(action1)assert isinstance(result1, StepResult)assert result1.reward == 1.0@@ -88,7 +89,8 @@ async def test_multi_step_env_lifecycle():mock_extractor.extract.assert_called_once_with("Response 1")# Test second step- action2 = Action(llm_response="Response 2")+ action2 = Action(index=0, llm_response="Response 2")+ # FIXME: index set to 0 temporarily until we have batched MultiStepresult2 = await env.step(action2)assert isinstance(result2, StepResult)assert result2.reward == 1.0@@ -112,7 +114,8 @@ async def test_multi_step_env_error_handling():mock_extractor.extract = MagicMock(return_value="extracted_answer")env = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)- action = Action(llm_response="Test response")+ action = Action(index=0, llm_response="Test response")+ # FIXME: index set to 0 temporarily until we have batched MultiStep# **Test Case 1: Environment not set up**# Attempt to step without setup, expect RuntimeErrordiff --git a/test/environments/test_single_stage_env.py b/test/environments/test_single_stage_env.pyindex 047e9af27..d67831284 100644--- a/test/environments/test_single_stage_env.py+++ b/test/environments/test_single_stage_env.py@@ -30,16 +30,14 @@ from camel.verifiers.models import VerificationOutcome, VerificationResultclass MockSingleStepEnv(SingleStepEnv):async def _compute_custom_reward(self,- action: Action,- extraction_result: str,+ proposed_solution: str,verification_result: VerificationResult,) -> Dict[str, float]:- return {"custom_reward": 0.5}+ return {"custom_reward": 5}@pytest.mark.asyncioasync def test_single_step_env_lifecycle():- # Define datasetdata = [{"question": "What is 2 + 2?",@@ -53,69 +51,141 @@ async def test_single_step_env_lifecycle():"rationale": "Paris is known as the capital city of France.","metadata": {"difficulty": "easy"},},+ {+ "question": "Who wrote 'Romeo and Juliet'?",+ "final_answer": "Shakespeare",+ "rationale": "William Shakespeare is the author of 'Romeo and "+ "Juliet'.",+ "metadata": {"difficulty": "medium"},+ },+ {+ "question": "What is the boiling point of water in Celsius?",+ "final_answer": "100",+ "rationale": "Water boils at 100 degrees Celsius at standard "+ "pressure.",+ "metadata": {"difficulty": "easy"},+ },+ {+ "question": "What is the chemical symbol for gold?",+ "final_answer": "Au",+ "rationale": "The chemical symbol for gold is Au, from the Latin "+ "'aurum'.",+ "metadata": {"difficulty": "medium"},+ },+ {+ "question": "How many continents are there?",+ "final_answer": "7",+ "rationale": "There are seven continents: Asia, Africa, North "+ "America, South America, Antarctica, Europe, and Australia.",+ "metadata": {"difficulty": "easy"},+ },+ {+ "question": "What is the largest planet in our solar system?",+ "final_answer": "Jupiter",+ "rationale": "Jupiter is the largest planet, with a diameter of "+ "about 142,984 km.",+ "metadata": {"difficulty": "medium"},+ },+ {+ "question": "Who painted the Mona Lisa?",+ "final_answer": "Leonardo da Vinci",+ "rationale": "Leonardo da Vinci painted the Mona Lisa "+ "between 1503 and 1506.",+ "metadata": {"difficulty": "medium"},+ },]dataset = StaticDataset(data)-- # Define mock verifiermock_verifier = MagicMock()mock_verifier.setup = AsyncMock()mock_verifier.cleanup = AsyncMock()- mock_verifier.verify = AsyncMock(- return_value=VerificationResult(- status=VerificationOutcome.SUCCESS,- result="Verification successful",- feedback="Correct",- score=1.0,- )+ mock_verifier.verify_batch = AsyncMock(+ side_effect=lambda solutions, ground_truths, **kwargs: [+ VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Verification successful",+ feedback="Correct",+ score=1.0,+ )+ for _ in solutions+ ])- # Define mock extractor- mock_extractor = MagicMock()- mock_extractor.setup = AsyncMock()- mock_extractor.cleanup = AsyncMock()- mock_extractor.extract = AsyncMock(return_value="extracted_answer")+ env = MockSingleStepEnv(dataset=dataset, verifier=mock_verifier)- # Initialize the environment with mocked dependencies- env = MockSingleStepEnv(- dataset=dataset,- verifier=mock_verifier,- extractor=mock_extractor,- )-- # Test setupawait env.setup()assert env._is_setup is Truemock_verifier.setup.assert_awaited_once()- mock_extractor.setup.assert_awaited_once()# Test reset- observation = await env.reset()- assert isinstance(observation, Observation)- assert observation.question in [- "What is 2 + 2?",- "What is the capital of France?",+ observations = await env.reset(batch_size=4)+ assert isinstance(observations, list)+ assert len(observations) == 4+ for obs in observations:+ assert isinstance(obs, Observation)+ assert obs.question in [dp["question"] for dp in data]++ # Test step with 2 actions+ actions = [+ Action(index=0, llm_response="4"), # "What is 2 + 2?"+ Action(+ index=2, llm_response="Shakespeare"+ ), # "Who wrote 'Romeo and Juliet'?"]+ results = await env.step(actions)+ assert isinstance(results, list)+ assert len(results) == 2+ for result in results:+ assert isinstance(result, StepResult)+ assert result.reward == 15+ assert result.done is True+ assert result.observation == MockSingleStepEnv.PLACEHOLDER_OBS- # Test step- action = Action(llm_response="Test response")- result = await env.step(action)- assert isinstance(result, StepResult)- assert result.reward == 10.5 # correctness: 10.0 + custom_reward: 0.5- assert result.done is True- assert result.observation == SingleStepEnv.PLACEHOLDER_OBS- mock_extractor.extract.assert_awaited_once_with("Test response")- mock_verifier.verify.assert_awaited_once()+ # Test step with remaining two actions+ actions = [+ Action(+ index=1, llm_response="Paris"+ ), # "What is the capital of France?"+ Action(+ index=3, llm_response="100"+ ), # "What is the boiling point of water?"+ ]+ results = await env.step(actions)+ assert isinstance(results, list)+ assert len(results) == 2+ for result in results:+ assert isinstance(result, StepResult)+ assert result.reward == 15+ assert result.done is True+ assert result.observation == MockSingleStepEnv.PLACEHOLDER_OBS- # Test close+ assert env._batch_done()await env.close()assert env._is_setup is Falsemock_verifier.cleanup.assert_awaited_once()- mock_extractor.cleanup.assert_awaited_once()+++def create_mock_verifier():+ verifier = MagicMock()+ verifier.setup = AsyncMock()+ verifier.cleanup = AsyncMock()+ verifier.verify_batch = AsyncMock(+ side_effect=lambda solutions, ground_truths, **kwargs: [+ VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Mock verification",+ feedback="Mock feedback",+ score=1.0,+ )+ for _ in solutions+ ]+ )+ return verifier@pytest.mark.asyncioasync def test_single_step_env_error_handling():# **1. Test Faulty Dataset**+ # Ensure we don't silently handle the errorfaulty_data = [{"question": "What is 2 + 2?",@@ -130,15 +200,11 @@ async def test_single_step_env_error_handling():"metadata": {"difficulty": "easy"},},]- try:- StaticDataset(faulty_data)- except Exception as e:- assert isinstance(- e, ValueError- ), "Expected ValueError for faulty dataset"+ with pytest.raises(ValueError):+ StaticDataset(faulty_data, strict=True)- # **Valid Dataset for Subsequent Tests**- valid_data = [+ # Define valid dataset for subsequent tests+ data = [{"question": "What is 2 + 2?","final_answer": "4",@@ -151,109 +217,189 @@ async def test_single_step_env_error_handling():"rationale": "Paris is known as the capital city of France.","metadata": {"difficulty": "easy"},},+ {+ "question": "Who wrote 'Romeo and Juliet'?",+ "final_answer": "Shakespeare",+ "rationale": "William Shakespeare is the author of 'Romeo and "+ "Juliet'.",+ "metadata": {"difficulty": "medium"},+ },+ {+ "question": "What is the boiling point of water in Celsius?",+ "final_answer": "100",+ "rationale": "Water boils at 100 degrees Celsius at standard "+ "pressure.",+ "metadata": {"difficulty": "easy"},+ },+ {+ "question": "What is the chemical symbol for gold?",+ "final_answer": "Au",+ "rationale": "The chemical symbol for gold is Au, from the Latin "+ "'aurum'.",+ "metadata": {"difficulty": "medium"},+ },+ {+ "question": "How many continents are there?",+ "final_answer": "7",+ "rationale": "There are seven continents: Asia, Africa, North "+ "America, South America, Antarctica, Europe, and Australia.",+ "metadata": {"difficulty": "easy"},+ },+ {+ "question": "What is the largest planet in our solar system?",+ "final_answer": "Jupiter",+ "rationale": "Jupiter is the largest planet, with a diameter of "+ "about 142,984 km.",+ "metadata": {"difficulty": "medium"},+ },+ {+ "question": "Who painted the Mona Lisa?",+ "final_answer": "Leonardo da Vinci",+ "rationale": "Leonardo da Vinci painted the Mona Lisa "+ "between 1503 and 1506.",+ "metadata": {"difficulty": "medium"},+ },]- dataset = StaticDataset(valid_data)-- # **2. Test Faulty Extractor**- # Mock verifier that fails verification- mock_verifier_fail = MagicMock()- mock_verifier_fail.setup = AsyncMock()- mock_verifier_fail.cleanup = AsyncMock()- mock_verifier_fail.verify = AsyncMock(- return_value=VerificationResult(- status=VerificationOutcome.FAILURE,- result="Verification failed",- feedback="Incorrect",- score=0.0,- )- )- # Mock extractor that returns None (extraction failure)- mock_extractor_fail = MagicMock()- mock_extractor_fail.setup = AsyncMock()- mock_extractor_fail.cleanup = AsyncMock()- mock_extractor_fail.extract = AsyncMock(return_value=None)-- env_fail_extractor = MockSingleStepEnv(- dataset=dataset,- verifier=mock_verifier_fail,- extractor=mock_extractor_fail,- )- await env_fail_extractor.setup()- observation = await env_fail_extractor.reset()- assert isinstance(- observation, Observation- ), "Reset should still return an Observation"-- action = Action(llm_response="Test response")- with pytest.raises(RuntimeError, match="Couldn't extract from"):- await env_fail_extractor.step(action)+ dataset = StaticDataset(data)- # **3. Test Faulty Verifier**- # Mock verifier that raises an exception+ # **2. Test Faulty Verifier**+ # Ensure errors in verifier are propagatedmock_verifier_exception = MagicMock()mock_verifier_exception.setup = AsyncMock()mock_verifier_exception.cleanup = AsyncMock()- mock_verifier_exception.verify = AsyncMock(+ mock_verifier_exception.verify_batch = AsyncMock(side_effect=Exception("Verifier error"))- # Valid extractor- mock_extractor = MagicMock()- mock_extractor.setup = AsyncMock()- mock_extractor.cleanup = AsyncMock()- mock_extractor.extract = AsyncMock(return_value="extracted_answer")-- env_fail_verifier = MockSingleStepEnv(+ env_fail_verifier = SingleStepEnv(dataset=dataset,verifier=mock_verifier_exception,- extractor=mock_extractor,)await env_fail_verifier.setup()- observation = await env_fail_verifier.reset()- assert isinstance(- observation, Observation- ), "Reset should still return an Observation"-+ await env_fail_verifier.reset(batch_size=1)+ action = Action(index=0, llm_response="4")with pytest.raises(Exception, match="Verifier error"):await env_fail_verifier.step(action)- # **4. Test State Mismanagement Scenarios**- # a) Environment not set up- env_not_setup = MockSingleStepEnv(+ # **3. Test State Mismanagement Scenarios**+ # a) Step without setup+ env_not_setup = SingleStepEnv(dataset=dataset,- verifier=mock_verifier_fail,- extractor=mock_extractor,+ verifier=create_mock_verifier(),)with pytest.raises(RuntimeError,match=re.escape("Environment not set up. Call setup() first."),):- await env_not_setup.step(action)+ await env_not_setup.step(Action(index=0, llm_response="4"))- # b) Episode ended- env_episode_ended = MockSingleStepEnv(+ # b) Step after batch is done+ env_episode_ended = SingleStepEnv(dataset=dataset,- verifier=mock_verifier_fail,- extractor=mock_extractor,+ verifier=create_mock_verifier(),)await env_episode_ended.setup()- await env_episode_ended.reset()- env_episode_ended._episode_ended = True+ await env_episode_ended.reset(batch_size=1)+ env_episode_ended._states_done = [True] # Simulate batch completion+ with pytest.raises(+ RuntimeError,+ match=re.escape("Episodes have ended for batch. Call reset() first."),+ ):+ await env_episode_ended.step(Action(index=0, llm_response="4"))++ # c) Step without reset+ env_no_reset = SingleStepEnv(+ dataset=dataset,+ verifier=create_mock_verifier(),+ )+ await env_no_reset.setup()with pytest.raises(RuntimeError,- match=re.escape("Episode has ended. " "Call reset() first."),+ match=re.escape("Episodes have ended for batch. Call reset() first."),):- await env_episode_ended.step(action)+ await env_no_reset.step(Action(index=0, llm_response="4"))- # c) No current observation- env_no_state = MockSingleStepEnv(+ # d) Reset before all states processed+ env_partial_batch = SingleStepEnv(dataset=dataset,- verifier=mock_verifier_fail,- extractor=mock_extractor,+ verifier=create_mock_verifier(),)- await env_no_state.setup()- env_no_state._state = None+ await env_partial_batch.setup()+ await env_partial_batch.reset(batch_size=2)+ await env_partial_batch.step(+ Action(index=0, llm_response="4")+ ) # Process only one statewith pytest.raises(RuntimeError,- match=re.escape("No current observation. Call reset() first."),+ match=re.escape(+ "reset() called before all states in batch were processed."+ ),):- await env_no_state.step(action)+ await env_partial_batch.reset(batch_size=1)++ # **4. Test Invalid Actions**+ env_invalid_actions = SingleStepEnv(+ dataset=dataset,+ verifier=create_mock_verifier(),+ )+ await env_invalid_actions.setup()+ await env_invalid_actions.reset(batch_size=2)++ # a) Action with invalid index (out of range)+ with pytest.raises(ValueError, match="Invalid state index 2."):+ await env_invalid_actions.step(Action(index=2, llm_response="4"))++ # b) Actions with duplicate indices+ with pytest.raises(+ ValueError, match="Duplicate state indices in actions."+ ):+ await env_invalid_actions.step(+ [+ Action(index=0, llm_response="4"),+ Action(index=0, llm_response="Paris"),+ ]+ )++ # c) Action on already finished state+ await env_invalid_actions.step(Action(index=0, llm_response="4"))+ with pytest.raises(+ ValueError, match="State at index 0 is already finished."+ ):+ await env_invalid_actions.step(Action(index=0, llm_response="Paris"))++ # **5. Test Batch Size Issues**+ env_large_batch = SingleStepEnv(+ dataset=dataset,+ verifier=create_mock_verifier(),+ )+ await env_large_batch.setup()+ with pytest.raises(+ ValueError, match="Batch size 9 is too large for dataset of size 8"+ ):+ await env_large_batch.reset(batch_size=9)++ # **6. Test Setup and Close Failures**+ # a) Setup failure+ mock_verifier_setup_fail = MagicMock()+ mock_verifier_setup_fail.setup = AsyncMock(+ side_effect=Exception("Setup failed")+ )+ env_setup_fail = SingleStepEnv(+ dataset=dataset,+ verifier=mock_verifier_setup_fail,+ )+ with pytest.raises(Exception, match="Setup failed"):+ await env_setup_fail.setup()++ # b) Close failure+ mock_verifier_close_fail = MagicMock()+ mock_verifier_close_fail.setup = AsyncMock()+ mock_verifier_close_fail.cleanup = AsyncMock(+ side_effect=Exception("Cleanup failed")+ )+ env_close_fail = SingleStepEnv(+ dataset=dataset,+ verifier=mock_verifier_close_fail,+ )+ await env_close_fail.setup()+ with pytest.raises(Exception, match="Cleanup failed"):+ await env_close_fail.close()diff --git a/test/verifiers/test_base_verifier.py b/test/verifiers/test_base_verifier.pyindex f009c2a90..715fff7bb 100644--- a/test/verifiers/test_base_verifier.py+++ b/test/verifiers/test_base_verifier.py@@ -25,7 +25,7 @@ from camel.verifiers.models import VerificationOutcome, VerificationResultclass TestVerifier(BaseVerifier):r"""Concrete implementation of BaseVerifier for testing."""- async def _setup(self) -> None:+ async def _setup(self, **kwargs) -> None:self.setup_called = Trueasync def _cleanup(self) -> None:@@ -103,7 +103,7 @@ async def test_verifier_setup_error():r"""Test handling of errors during setup."""class ErrorVerifier(TestVerifier):- async def _setup(self) -> None:+ async def _setup(self, **kwargs) -> None:raise RuntimeError("Simulated setup error")verifier = ErrorVerifier()@@ -251,8 +251,6 @@ async def test_verify_batch(test_verifier):)with patch.object(test_verifier, "verify", side_effect=mock_verify):- from camel.verifiers.base import BaseVerifier-test_verifier.verify_batch = (lambda *args, **kwargs: BaseVerifier.verify_batch(test_verifier, *args, **kwargs@@ -293,8 +291,6 @@ async def test_verify_batch_with_error_handling(test_verifier):)with patch.object(test_verifier, "verify", side_effect=mock_verify):- from camel.verifiers.base import BaseVerifier-test_verifier.verify_batch = (lambda *args, **kwargs: BaseVerifier.verify_batch(test_verifier, *args, **kwargsdiff --git a/test/verifiers/test_python_verifier.py b/test/verifiers/test_python_verifier.pyindex 525d10fb9..4bed297e6 100644--- a/test/verifiers/test_python_verifier.py+++ b/test/verifiers/test_python_verifier.py@@ -28,13 +28,15 @@ def python_verifier():@pytest.mark.asyncio-async def test_python_verifier_setup_and_cleanup(python_verifier):+async def test_python_verifiersetup_andcleanup(+ python_verifier: PythonVerifier,+):r"""Test setup and cleanup of PythonVerifier."""- await python_verifier._setup()+ await python_verifier.setup(uv=True)assert python_verifier.venv_path is not Noneassert os.path.exists(python_verifier.venv_path)- await python_verifier._cleanup()+ await python_verifier.cleanup()assert python_verifier.venv_path is None or not os.path.exists(python_verifier.venv_path)@@ -43,23 +45,20 @@ async def test_python_verifier_setup_and_cleanup(python_verifier):@pytest.mark.asyncioasync def test_python_verifier_execution_success(python_verifier):r"""Test successful execution of a Python script."""- await python_verifier._setup()+ await python_verifier.setup(uv=True)script = 'print("Hello, World!")'- result = await python_verifier._verify_implementation(- solution=script, ground_truth=None- )-+ result = await python_verifier.verify(solution=script, ground_truth=None)assert result.status == VerificationOutcome.SUCCESSassert result.result == "Hello, World!"- await python_verifier._cleanup()+ await python_verifier.cleanup()@pytest.mark.asyncioasync def test_python_verifier_execution_failure(python_verifier):r"""Test execution failure due to syntax error."""- await python_verifier._setup()+ await python_verifier.setup(uv=True)script = 'print("Hello"' # Syntax errorresult = await python_verifier._verify_implementation(@@ -69,14 +68,14 @@ async def test_python_verifier_execution_failure(python_verifier):assert result.status == VerificationOutcome.ERRORassert "SyntaxError" in result.error_message- await python_verifier._cleanup()+ await python_verifier.cleanup()@pytest.mark.asyncioasync def test_python_verifier_execution_timeout(python_verifier):r"""Test execution timeout handling."""python_verifier.timeout = 1 # Short timeout for testing- await python_verifier._setup()+ await python_verifier.setup(uv=True)script = 'import time; time.sleep(5)'result = await python_verifier._verify_implementation(@@ -86,39 +85,42 @@ async def test_python_verifier_execution_timeout(python_verifier):assert result.status == VerificationOutcome.TIMEOUTassert "Execution timed out" in result.error_message- await python_verifier._cleanup()+ await python_verifier.cleanup()@pytest.mark.asyncioasync def test_python_verifier_output_mismatch(python_verifier):r"""Test verification when output does not match ground truth."""- await python_verifier._setup()+ await python_verifier.setup(uv=True)script = 'print("Wrong output")'result = await python_verifier._verify_implementation(- solution=script, ground_truth="Expected output"+ solution=script, ground_truth="'Expected output'")assert result.status == VerificationOutcome.FAILURE- assert result.error_message == "Output doesn't match ground truth"+ assert (+ result.error_message+ == "Output mismatch: Wrong output != Expected output"+ )- await python_verifier._cleanup()+ await python_verifier.cleanup()@pytest.mark.asyncioasync def test_python_verifier_correct_output_matching(python_verifier):r"""Test verification when output matches ground truth."""- await python_verifier._setup()+ await python_verifier.setup(uv=True)script = 'print("Expected output")'result = await python_verifier._verify_implementation(- solution=script, ground_truth="Expected output"+ solution=script, ground_truth="'Expected output'")assert result.status == VerificationOutcome.SUCCESSassert result.result == "Expected output"- await python_verifier._cleanup()+ await python_verifier.cleanup()@pytest.mark.asyncio@@ -137,7 +139,7 @@ async def test_python_verifier_no_venv_error():@pytest.mark.asyncioasync def test_python_verifier_with_numpy(python_verifier):r"""Test execution with numpy installed in the virtual environment."""- await python_verifier._setup()+ await python_verifier.setup(uv=True)script = 'import numpy as np; print(np.array([1, 2, 3]))'result = await python_verifier._verify_implementation(@@ -146,4 +148,4 @@ async def test_python_verifier_with_numpy(python_verifier):assert result.status == VerificationOutcome.SUCCESSassert "[1 2 3]" in result.result- await python_verifier._cleanup()+ await python_verifier.cleanup()