🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
refactor BaseEnvironment into SingleStep and MultiStep environments (#1810)
halleritedac4d318bb75
diff --git a/camel/datasets/base.py b/camel/datasets/base.pyindex ec96cad4c..f65fb2582 100644--- a/camel/datasets/base.py+++ b/camel/datasets/base.py@@ -432,8 +432,7 @@ class GenerativeDataset(Dataset):self.verifier = verifierself.agent = agent- self.seed = seed- random.seed(self.seed)+ self._rng = random.Random(seed)self._data: List[DataPoint] = []@@ -606,6 +605,12 @@ class GenerativeDataset(Dataset):)return self._data[idx]+ def sample(self) -> DataPoint:+ if len(self._data) == 0:+ raise RuntimeError("Dataset is empty, cannot sample.")+ idx = self._rng.randint(0, len(self._data) - 1)+ return self[idx]+def save_to_jsonl(self, file_path: Union[str, Path]) -> None:r"""Saves the dataset to a JSONL (JSON Lines) file.diff --git a/camel/environments/__init__.py b/camel/environments/__init__.pyindex b0bbfb9a3..c580dd577 100644--- a/camel/environments/__init__.py+++ b/camel/environments/__init__.py@@ -11,6 +11,15 @@# See the License for the specific language governing permissions and# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-from .base import BaseEnvironment+from .models import Action, Environment, Observation, StepResult+from .multi_step import MultiStepEnv+from .single_step import SingleStepEnv-__all__ = ["BaseEnvironment"]+__all__ = [+ "Environment",+ "SingleStepEnv",+ "MultiStepEnv",+ "Action",+ "Observation",+ "StepResult",+]diff --git a/camel/environments/base.py b/camel/environments/base.pydeleted file mode 100644index edd0ce021..000000000--- a/camel/environments/base.py+++ /dev/null@@ -1,509 +0,0 @@-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========--import asyncio-from abc import ABC, abstractmethod-from datetime import datetime, timezone-from typing import Any, Dict, List, Optional, Tuple--from pydantic import BaseModel, Field--from camel.agents import ChatAgent-from camel.datasets.base import GenerativeDataset, StaticDataset-from camel.extractors.base import BaseExtractor-from camel.logger import get_logger-from camel.verifiers.base import (- BaseVerifier,- VerificationResult,-)-from camel.verifiers.models import (- VerificationOutcome,- VerifierInput,-)--logger = get_logger(__name__)--# TODO: Add MachineInfo into this file-# TODO: TeacherAgent should be renamed into neural_reward_model.-# This is where PRMs or such could be useful.-# Should probably be its own class and not just raw ChatAgent---class Action(BaseModel):- r"""Represents an action taken in an environment.-- This class defines the input context, the LLM-generated output, and- metadata required for verification and tracking within an RL- framework.-- Attributes:- problem_statement (str): The task or query given to the LLM as- input.- llm_response (str): The response generated by the LLM.- final_answer (Optional[str]): The reference solution, if- available, used for supervised learning or evaluation.- metadata (Dict[str, Any]): Additional metadata such as model- parameters, prompt details, or response confidence scores.- timestamp (datetime): The timestamp when the action was- generated (UTC).- """-- problem_statement: str = Field(description="Problem statement for the LLM")- llm_response: str = Field(description="Generated response from the LLM")- final_answer: Optional[str] = Field(- None, description="Reference solution if available"- )- metadata: Dict[str, Any] = Field(- default_factory=dict,- description="Additional metadata about the generation",- )- timestamp: datetime = Field(- default_factory=lambda: datetime.now(timezone.utc),- description="When the response was generated (UTC)",- )---class Observation(BaseModel):- r"""Environment observation.-- Attributes:- question: The question posed to the LLM.- context: Additional context for the question.- metadata: Optional metadata about the observation.- """-- question: str = Field(..., description="The question posed to the LLM")- context: Dict[str, Any] = Field(- default_factory=dict, description="Additional context for the question"- )- metadata: Optional[Dict[str, Any]] = Field(- default=None, description="Optional metadata about the observation"- )---class StepResult(BaseModel):- r"""Result of an environment step.-- Attributes:- observation: The next observation.- reward: Dictionary of reward scores for different aspects.- done: Whether the episode is complete.- info: Additional information about the step.- """-- observation: Observation = Field(..., description="The next observation")- reward: float = Field(..., description="Total reward of the action")- rewards_dict: Dict[str, float] = Field(- default_factory=dict,- description="Dictionary of reward scores for different aspects",- )- done: bool = Field(..., description="Whether the episode is complete")- info: Dict[str, Any] = Field(- default_factory=dict,- description="Additional information about the step",- )---class BaseEnvironment(ABC):- r"""Base class for all RLVR training environments.-- An environment ties everything together. It:- 1. Holds state and manages curriculum progression- 2. Defines reward functions and hint generation- 3. Manages dataset and task selection- 4. Provides reset and step functions- 5. Handles verifier setup and teardown- 6. Enables proactive agent behavior- 7. Supports practice environment creation- 8. Facilitates chain-of-thought verification-- Key Features:- - Curriculum learning with adaptive difficulty- - Reward shaping based on solution quality- - Hint generation from verified solutions- - Task selection based on agent progress- - Practice environment generation- - Chain-of-thought validation- """-- def __init__(- self,- dataset: StaticDataset,- verifier: BaseVerifier,- extractor: BaseExtractor,- max_steps: Optional[int] = None,- teacher_agent: Optional[ChatAgent] = None,- curriculum_config: Optional[Dict[str, Any]] = None,- practice_env_config: Optional[Dict[str, Any]] = None,- **kwargs,- ) -> None:- r"""Initialize the environment.-- Args:- dataset (StaticDataset): Dataset to sample questions from.- verifier (BaseVerifier): Verifier to check responses.- extractor (BaseExtractor): Extractor to process LLM responses.- max_steps (Optional[int]): Maximum steps per episode. (default:- :obj:`None`)- teacher_agent (Optional[ChatAgent]): Optional agent for reward- shaping and hints. (default: :obj:`None`)- curriculum_config (Optional[Dict[str, Any]]): Configuration for- curriculum learning including:- - difficulty_levels: List of available difficulty levels- - promotion_threshold: Score needed to advance- - demotion_threshold: Score triggering level decrease- - min_questions_per_level: Questions before promotion- (default: :obj:`None`)- practice_env_config (Optional[Dict[str, Any]]): Configuration for- practice environments:- - max_practice_envs: Maximum concurrent environments- - difficulty_range: Allowed difficulty variation- - focus_areas: Specific skills to practice- (default: :obj:`None`)- **kwargs: Additional environment parameters.- """- self.dataset = dataset- self.verifier = verifier- self.extractor = extractor- self.max_steps = max_steps- self.teacher_agent = teacher_agent- self._metadata = kwargs-- # State tracking- self._is_setup: bool = False- self._current_step: int = 0- self._episode_ended: bool = False- self._state: Dict[str, Any] = self._get_initial_state()- self._last_observation: Optional[Observation] = None- self._episode_history: List[Tuple[Observation, Action]] = []-- @abstractmethod- async def setup(self) -> None:- r"""Set up the environment, including verifier initialization."""- if self._is_setup:- return-- try:- # Initialize core components- if hasattr(self.verifier, 'setup'):- await self.verifier.setup()- if hasattr(self.dataset, 'setup'):- await self.dataset.setup()- if hasattr(self.extractor, 'setup'):- await self.extractor.setup()-- # initialize agents if present- if self.teacher_agent:- await self.teacher_agent.reset()-- self._is_setup = True- logger.info('Environment setup completed successfully')- except Exception as e:- logger.error(f'Failed to setup environment: {e}')- raise-- @abstractmethod- async def teardown(self) -> None:- r"""Clean up resources, including verifier teardown."""- if not self._is_setup:- return-- try:- # Cleanup components- if hasattr(self.verifier, 'cleanup'):- await self.verifier.cleanup()- if hasattr(self.dataset, 'cleanup'):- await self.dataset.cleanup()- if hasattr(self.extractor, 'cleanup'):- await self.extractor.cleanup()-- self._is_setup = False- logger.info('Environment teardown completed successfully')- except Exception as e:- logger.error(f'Failed to teardown environment: {e}')- raise-- @abstractmethod- async def reset(self) -> Observation:- r"""Reset the environment to initial state.-- Returns:- Initial observation for the episode- """-- if not self._is_setup:- await self.setup()-- # Reset state- self._current_step = 0- self._episode_ended = False- self._episode_history = []- self._state = self._get_initial_state()-- # Get initial observation- observation = self._get_next_observation()- if observation is None:- raise RuntimeError("Failed to get initial observation")-- self._last_observation = observation-- return observation-- @abstractmethod- async def step(self, action: Action) -> StepResult:- r"""Take a step in the environment.-- Args:- action: Action containing everything that is needed- to progress in the environment-- Returns:- StepResult containing next observation, reward, done flag, and info- """- if self.max_steps and self._current_step >= self.max_steps:- return StepResult(- observation=self._get_terminal_observation(),- reward=0,- rewards_dict={},- done=True,- info={"reason": "max_steps_reached"},- )-- if not self._is_setup:- raise RuntimeError("Environment not set up. Call setup() first.")- if self._episode_ended:- raise RuntimeError("Episode has ended. Call reset() first.")- if self._last_observation is None:- raise RuntimeError("No current observation. Call reset() first.")-- self._current_step += 1-- current_obs: Observation = self._last_observation- self._episode_history.append((current_obs, action))-- # extract verifiable part from llm response- extraction_result = await self.extractor.extract(action.llm_response)-- # Ensure extraction_result is a string- if extraction_result is None:- extraction_result = ""-- # verify the extracted- verification_result = await self.verifier.verify(- VerifierInput(- llm_response=extraction_result,- ground_truth=action.final_answer,- )- )-- # compute rewards- total_reward, rewards_dict = await self.compute_reward(- action, extraction_result, verification_result- )-- # check termination- done = self._is_done()-- next_obs = (- self._get_terminal_observation()- if done- else self._get_next_observation()- )-- self._last_observation = next_obs- self._episode_ended = done-- return StepResult(- observation=next_obs,- reward=total_reward,- rewards_dict=rewards_dict,- done=done,- info={- "extraction_result": extraction_result,- "verification_result": verification_result,- "step": self._current_step,- "state": self._state,- },- )-- @abstractmethod- def _get_initial_state(self) -> Dict[str, Any]:- r"""Get initial environment state."""-- return {- "current_datapoint": None,- "attempts": 0,- "success_rate": 0.0,- "rewards": [],- "termination_reason": None,- }-- @abstractmethod- def _get_next_observation(self) -> Observation:- r"""Get the next observation for the environment.-- Returns:- Observation for the next step- """- if not self.dataset or len(self.dataset) == 0:- logger.warning(- "Dataset is empty. Attempting to generate new data..."- )- if isinstance(self.dataset, GenerativeDataset):- try:- asyncio.run(- self.dataset.generate_new(1)- ) # Generate at least one datapoint- logger.info("Generated new datapoint successfully.")- except Exception as e:- logger.error(f"Failed to generate new data: {e}")- return self._get_terminal_observation()- else:- logger.error("Dataset is empty and not a GenerativeDataset.")- return self._get_terminal_observation()-- try:- # Ensure dataset is not empty after generation attempt- if len(self.dataset) == 0:- logger.error("Dataset is still empty after generation.")- return self._get_terminal_observation()-- # Sample the next datapoint- datapoint_idx = self._current_step % len(self.dataset)- datapoint = self.dataset[datapoint_idx]-- if not datapoint:- logger.error(f"Invalid datapoint at index {datapoint_idx}")- return self._get_terminal_observation()-- self._state["current_datapoint"] = datapoint-- # Extract necessary attributes safely- question = getattr(datapoint, "question", None)- final_answer = getattr(datapoint, "final_answer", None)- rationale = getattr(datapoint, "rationale", None)- metadata = getattr(datapoint, "metadata", {})-- if not question or not final_answer:- logger.error(- f"Datapoint at index {datapoint_idx} "- "is missing required fields."- )- return self._get_terminal_observation()-- observation = Observation(- question=question,- context={- "final_answer": final_answer,- "rationale": rationale,- },- metadata={- "step": self._current_step,- "datapoint_id": str(datapoint_idx),- "verified": metadata.get("verified", False),- **metadata,- },- )-- logger.debug(- f"Generated observation for step {self._current_step}"- )- return observation-- except (IndexError, AttributeError) as e:- logger.error(f"Error getting next observation: {e}")- return self._get_terminal_observation()- except Exception as e:- logger.error(f"Unexpected error getting next observation: {e}")- return self._get_terminal_observation()-- @abstractmethod- def _get_terminal_observation(self) -> Observation:- r"""Get the terminal observation when episode ends.-- Returns:- Terminal observation- """- return Observation(- question="Episode completed",- context={},- metadata={"terminal": True, "final_step": self._current_step},- )-- @abstractmethod- async def compute_reward(- self,- action: Action,- extraction_result: str,- verification_result: VerificationResult,- ) -> Tuple[float, Dict[str, float]]:- r"""Compute reward scores for different aspects of the response.-- Args:- response: The response.- extraction_result: Extracted information from response- verification_result: Result from the verifier.-- Returns:- - Total reward- - Dictionary of reward scores for different aspects.- """- rewards: Dict[str, float] = {}-- # Get success from verification result status- verification_success = float(- verification_result.status == VerificationOutcome.SUCCESS- )- rewards["correctness"] = 1.0 if verification_success > 0.5 else 0.0-- # Update state- self._state["rewards"].append(rewards)- total_attempts = self._state["attempts"] + 1- self._state["success_rate"] = (- self._state["success_rate"] * (total_attempts - 1)- + verification_success- ) / total_attempts-- further_rewards = await self._compute_reward(- action, extraction_result, verification_result- )-- rewards = rewards | further_rewards-- return sum(rewards.values()), rewards-- @abstractmethod- async def _compute_reward(- self,- action: Action,- extraction_result: str,- verification_result: VerificationResult,- ) -> Dict[str, float]:- pass-- def _is_done(self) -> bool:- r"""Check if episode should terminate."""- if self.max_steps and self._current_step >= self.max_steps:- return True- return False-- @property- def metadata(self) -> Dict[str, Any]:- r"""Get environment metadata."""- return self._metadata.copy()-- @property- def current_step(self) -> int:- r"""Get current step number."""- return self._current_stepdiff --git a/camel/environments/models.py b/camel/environments/models.pynew file mode 100644index 000000000..3de9bc9b1--- /dev/null+++ b/camel/environments/models.py@@ -0,0 +1,111 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from datetime import datetime, timezone+from typing import Any, Dict, Optional, Protocol++from pydantic import BaseModel, Field+++class Action(BaseModel):+ r"""Represents an action taken in an environment.++ This class defines the input context, the LLM-generated output, and+ metadata required for verification and tracking within an RL+ framework.++ Attributes:+ llm_response (str): The response generated by the LLM.+ metadata (Dict[str, Any]): Additional metadata such as model+ parameters, prompt details, or response confidence scores.+ timestamp (datetime): The timestamp when the action was+ generated (UTC).+ """++ llm_response: str = Field(description="Generated response from the LLM")+ metadata: Dict[str, Any] = Field(+ default_factory=dict,+ description="Additional metadata about the generation",+ )+ timestamp: datetime = Field(+ default_factory=lambda: datetime.now(timezone.utc),+ description="When the response was generated (UTC)",+ )+++class Observation(BaseModel):+ r"""Environment observation.++ Attributes:+ question: The question posed to the LLM.+ context: Additional context for the question.+ metadata: Optional metadata about the observation.+ """++ question: str = Field(..., description="The question posed to the LLM")+ context: Dict[str, Any] = Field(+ default_factory=dict, description="Additional context for the question"+ )+ metadata: Optional[Dict[str, Any]] = Field(+ default=None, description="Optional metadata about the observation"+ )+++class StepResult(BaseModel):+ r"""Result of an environment step.++ Attributes:+ observation: The next observation.+ reward: Dictionary of reward scores for different aspects.+ done: Whether the episode is complete.+ info: Additional information about the step.+ """++ observation: Observation = Field(..., description="The next observation")+ reward: float = Field(..., description="Total reward of the action")+ rewards_dict: Dict[str, float] = Field(+ default_factory=dict,+ description="Dictionary of reward scores for different aspects",+ )+ done: bool = Field(..., description="Whether the episode is complete")+ info: Dict[str, Any] = Field(+ default_factory=dict,+ description="Additional information about the step",+ )+++class Environment(Protocol):+ async def reset(self) -> Observation:+ r"""Reset the environment to an initial state.++ Returns:+ Initial observation for the episode+ """+ ...++ async def step(self, action: Action) -> StepResult:+ r"""Take a step in the environment.++ Args:+ action: Action containing everything that is needed+ to progress in the environment++ Returns:+ StepResult containing next observation, reward, done flag, and info+ """+ ...++ async def close(self) -> None:+ r"""Perform a full cleanup of all environment resources."""+ ...diff --git a/camel/environments/multi_step.py b/camel/environments/multi_step.pynew file mode 100644index 000000000..df0013954--- /dev/null+++ b/camel/environments/multi_step.py@@ -0,0 +1,271 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from abc import ABC, abstractmethod+from typing import Any, Dict, List, Optional, Tuple++from camel.extractors.base import BaseExtractor+from camel.logger import get_logger++from .models import Action, Observation, StepResult++logger = get_logger(__name__)+++class MultiStepEnv(ABC):+ r"""A multi-step environment for reinforcement learning with LLMs."""++ def __init__(+ self,+ extractor: BaseExtractor,+ max_steps: Optional[int] = None,+ **kwargs,+ ) -> None:+ r"""Initialize the environment.++ Args:+ extractor: Extractor to process LLM responses.+ max_steps: Maximum steps per episode.+ **kwargs: Additional environment parameters.+ """+ self.extractor = extractor+ self.max_steps = max_steps+ self._metadata = kwargs++ # State tracking+ self._is_setup: bool = False+ self._current_step: int = 0+ self._episode_ended: bool = False+ self._state: Dict[str, Any] = self._get_initial_state()+ self._last_observation: Optional[Observation] = None+ self._episode_history: List[Tuple[Observation, Action]] = []++ async def setup(self) -> None:+ r"""Set up the environment by initializing the verifier and extractor.++ This method ensures that the environment is ready for interaction.+ It sets up necessary components, including the verifier and extractor.++ Raises:+ Exception: If setup fails due to an internal error.+ """++ if self._is_setup:+ return++ try:+ await self.extractor.setup()+ await self._setup()+ self._is_setup = True+ logger.info('Environment setup completed successfully')+ except Exception as e:+ logger.error(f'Failed to setup environment: {e}')+ raise++ @abstractmethod+ async def _setup(self) -> None:+ pass++ async def close(self) -> None:+ r"""Clean up and close all resources used by the environment.+ This method shuts down the verifier, calls the internal+ close function that is implemented in any MultiStepEnv,+ and ensures that the environment is properly closed.++ Raises:+ Exception: If an error occurs while closing the environment.+ """+ if not self._is_setup:+ return++ try:+ await self.extractor.cleanup()++ await self._close()++ self._is_setup = False+ logger.info('Environment teardown completed successfully')+ except Exception as e:+ logger.error(f'Failed to teardown environment: {e}')+ raise++ @abstractmethod+ async def _close(self) -> None:+ pass++ async def reset(self) -> Observation:+ r"""Reset the environment to an initial state.++ Returns:+ Observation: The initial observation for the episode.++ Raises:+ RuntimeError: If we fail to get the initial observation.+ """++ if not self._is_setup:+ await self.setup()++ # Reset state+ self._current_step = 0+ self._episode_ended = False+ self._episode_history = []+ self._state = self._get_initial_state()++ # Get initial observation+ observation = self._get_next_observation()+ if observation is None:+ raise RuntimeError("Failed to get initial observation")++ self._last_observation = observation++ return observation++ async def step(self, action: Action) -> StepResult:+ r"""Take a step in the environment using the given action.++ This method updates the environment state based on the LLM's response,+ computes rewards, checks if the episode is done, and based on that+ gets the next or final observation.++ Args:+ action (Action): The action containing the LLM response.++ Returns:+ StepResult containing next observation, total reward, a dictionary+ of rewards, done flag, and info.++ Raises:+ RuntimeError: If the environment is not set up, the episode has+ ended, or there is no valid current observation.+ """+ if self.max_steps and self._current_step >= self.max_steps:+ return StepResult(+ observation=self._get_terminal_observation(),+ reward=0,+ rewards_dict={},+ done=True,+ info={"reason": "max_steps_reached"},+ )++ if not self._is_setup:+ raise RuntimeError("Environment not set up. Call setup() first.")+ if self._episode_ended:+ raise RuntimeError("Episode has ended. Call reset() first.")+ if self._last_observation is None:+ raise RuntimeError("No current observation. Call reset() first.")++ self._current_step += 1++ current_obs: Observation = self._last_observation+ self._episode_history.append((current_obs, action))++ # Update the environment state based on the action+ await self._update_state(action)++ # Compute rewards+ total_reward, rewards_dict = await self.compute_reward()++ # Check termination+ done = self.is_done()++ # Get next observation based on the updated state+ next_obs = (+ self._get_terminal_observation()+ if done+ else self._get_next_observation()+ )++ self._last_observation = next_obs+ self._episode_ended = done++ return StepResult(+ observation=next_obs,+ reward=total_reward,+ rewards_dict=rewards_dict,+ done=done,+ info={+ "extraction_result": self.extractor.extract(+ action.llm_response+ ),+ "step": self._current_step,+ "state": self._state, # Updated state+ },+ )++ @abstractmethod+ def _get_initial_state(self) -> Dict[str, Any]:+ pass++ @abstractmethod+ async def _update_state(self, action: Action) -> None:+ pass++ @abstractmethod+ def _get_next_observation(self) -> Observation:+ pass++ @abstractmethod+ def _get_terminal_observation(self) -> Observation:+ pass++ @abstractmethod+ async def compute_reward(+ self,+ ) -> Tuple[float, Dict[str, float]]:+ pass++ def is_done(self) -> bool:+ r"""Check if the episode should terminate.++ This function terminates the episode if the maximum number of+ steps is reached or if any other terminating criterion is met.++ Returns:+ bool: A boolean flag.+ """++ # After too many steps+ if self.max_steps and self._current_step >= self.max_steps:+ return True++ # Further termination logic can be implemented in subclass+ if self._is_done():+ return True++ return False++ @abstractmethod+ def _is_done(self) -> bool:+ pass++ @property+ def metadata(self) -> Dict[str, Any]:+ r"""Retrieve the metadata of the environment.++ This provides additional parameters and configuration details.++ Returns:+ Dict[str, Any]: A copy of the environment's metadata.+ """+ return self._metadata.copy()++ @property+ def current_step(self) -> int:+ r"""Get the current step number.++ Returns:+ int: The number of the step we are currently in.+ """+ return self._current_stepdiff --git a/camel/environments/single_step.py b/camel/environments/single_step.pynew file mode 100644index 000000000..6eadf5630--- /dev/null+++ b/camel/environments/single_step.py@@ -0,0 +1,293 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+++from abc import abstractmethod+from typing import Any, Dict, Optional, Tuple++from camel.datasets.base import DataPoint, GenerativeDataset, StaticDataset+from camel.extractors.base import BaseExtractor+from camel.logger import get_logger+from camel.verifiers.base import (+ BaseVerifier,+ VerificationResult,+)+from camel.verifiers.models import (+ VerifierInput,+)++from .models import Action, Observation, StepResult++logger = get_logger(__name__)+++class SingleStepEnv:+ r"""A single-step environment for reinforcement learning with LLMs.++ Key Features:+ - Samples questions from a dataset and asks the LLM+ - Extracts verifiable information from model responses.+ - Verifies extracted responses against ground truth.+ - Computes and assigns rewards based on correctness.+ - Supports async setup, teardown, and cleanup of resources.++ This class is intended as a foundation for RL experiments involving+ LLM-based policies, ensuring structured interactions between model+ actions and verification mechanisms.+ """++ PLACEHOLDER_OBS = Observation(+ question="Episode ended. This is just a placeholder."+ )++ ACCURACY_REWARD = 10++ def __init__(+ self,+ dataset: StaticDataset | GenerativeDataset,+ verifier: BaseVerifier,+ extractor: BaseExtractor,+ **kwargs,+ ) -> None:+ r"""Initialize the environment.++ Args:+ dataset: Dataset to sample questions from.+ verifier: Verifier to check responses.+ extractor: Extractor to process LLM responses.+ **kwargs: Additional environment parameters.+ """+ self.dataset = dataset+ self.verifier = verifier+ self.extractor = extractor+ self._metadata = kwargs++ # State tracking+ self._is_setup: bool = False+ self._state: Optional[DataPoint] = None+ self._episode_ended: bool = False++ async def setup(self) -> None:+ r"""Set up the environment by initializing the verifier and extractor.++ This method ensures that the environment is ready for interaction.+ It sets up necessary components, including the verifier and extractor.++ Raises:+ Exception: If setup fails due to an internal error.+ """++ if self._is_setup:+ return++ try:+ await self.verifier.setup()+ await self.extractor.setup()++ self._is_setup = True+ logger.info('Environment setup completed successfully')+ except Exception as e:+ logger.error(f'Failed to setup environment: {e}')+ raise++ async def close(self) -> None:+ r"""Clean up and close all resources used by the environment.++ This method shuts down the verifier and extractor, resets the internal+ state, and ensures that the environment is properly closed.++ Raises:+ Exception: If an error occurs while closing the environment.+ """++ if not self._is_setup:+ return++ try:+ self._is_setup = False+ await self.verifier.cleanup()+ await self.extractor.cleanup()+ self._state = None+ self._episode_ended = False+ logger.info('Environment closed successfully')+ except Exception as e:+ logger.error(f'Failed to close environment: {e}')+ raise++ async def reset(self) -> Observation:+ r"""Reset the environment and start a new episode.++ This method samples a new data point from the dataset and returns the+ initial observation.++ Returns:+ Observation: The first observation of the new episode, including+ the question.++ Raises:+ Exception: If the environment is not set up properly.+ """++ if not self._is_setup:+ await self.setup()++ self._episode_ended = False++ # Sample a datapoint++ self._state = self.dataset.sample()++ observation = Observation(+ question=self._state.question, context={}, metadata={}+ )++ return observation++ async def step(self, action: Action) -> StepResult:+ r"""Take a step in the environment using the given action.++ This method processes the LLM response, extracts verifiable content,+ verifies correctness, computes rewards, and ends the episode.++ Args:+ action (Action): The action containing the LLM response to+ evaluate.++ Returns:+ StepResult: Contains the next observation (placeholder), total+ reward, reward breakdown, completion flag, and additional+ information.++ Raises:+ RuntimeError: If the environment is not set up, the episode has+ ended, or there is no valid current observation.+ """++ if not self._is_setup:+ raise RuntimeError("Environment not set up. Call setup() first.")+ if self._episode_ended:+ raise RuntimeError("Episode has ended. Call reset() first.")+ if self._state is None:+ raise RuntimeError("No current observation. Call reset() first.")++ # extract verifiable part from llm response+ extraction_result = await self.extractor.extract(action.llm_response)++ if not extraction_result:+ raise RuntimeError(f"Couldn't extract from {action.llm_response}")++ # verify the extracted+ verification_result = await self.verifier.verify(+ VerifierInput(+ llm_response=extraction_result,+ ground_truth=self._state.final_answer,+ )+ )++ # compute rewards+ total_reward, rewards_dict = await self._compute_reward(+ action, extraction_result, verification_result+ )++ self._episode_ended = True++ return StepResult(+ observation=self.PLACEHOLDER_OBS,+ reward=total_reward,+ rewards_dict=rewards_dict,+ done=True,+ info={+ "extraction_result": extraction_result,+ "verification_result": verification_result,+ "state": self._state,+ },+ )++ async def _compute_reward(+ self,+ action: Action,+ extraction_result: str,+ verification_result: VerificationResult,+ ) -> Tuple[float, Dict[str, float]]:+ r"""Compute reward scores based on verification results.++ This method calculates the reward based on correctness and any+ additional custom reward components.++ Args:+ action (Action): The action taken in the environment.+ extraction_result (str): The extracted verifiable content from the+ LLM response.+ verification_result (VerificationResult): The result of verifying+ the extracted response.++ Returns:+ Tuple[float, Dict[str, float]]: A tuple containing:+ - Total reward (float)+ - Dictionary of individual reward components.++ Raises:+ Exception: If an error occurs while computing rewards.+ """++ rewards: Dict[str, float] = {}++ rewards["correctness"] = (+ self.ACCURACY_REWARD if verification_result.status else 0.0+ )++ further_rewards = await self._compute_custom_reward(+ action, extraction_result, verification_result+ )++ rewards = rewards | further_rewards++ return sum(rewards.values()), rewards++ @abstractmethod+ async def _compute_custom_reward(+ self,+ action: Action,+ extraction_result: str,+ verification_result: VerificationResult,+ ) -> Dict[str, float]:+ r"""Compute additional custom reward components.++ This method should be implemented by subclasses to define+ domain-specific reward calculations.++ Args:+ action (Action): The action taken in the environment.+ extraction_result (str): The extracted verifiable content from the+ LLM response.+ verification_result (VerificationResult): The result of verifying+ the extracted response.++ Returns:+ Dict[str, float]: A dictionary mapping custom reward categories+ to their values.+ """+ pass++ @property+ def metadata(self) -> Dict[str, Any]:+ r"""Retrieve the metadata of the environment.++ This provides additional parameters and configuration details.++ Returns:+ Dict[str, Any]: A copy of the environment's metadata.+ """++ return self._metadata.copy()diff --git a/test/environments/test_base_env.py b/test/environments/test_base_env.pydeleted file mode 100644index 9ac90ab41..000000000--- a/test/environments/test_base_env.py+++ /dev/null@@ -1,288 +0,0 @@-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========--from datetime import datetime, timezone-from typing import Dict, Tuple-from unittest.mock import AsyncMock, MagicMock--import pytest--from camel.environments.base import (- Action,- BaseEnvironment,- Observation,- StepResult,-)-from camel.verifiers.models import VerificationOutcome, VerificationResult---def test_action_model():- r"""Test the Action model initialization and properties."""- custom_time = datetime(2023, 1, 1, tzinfo=timezone.utc)- action_full = Action(- problem_statement="Full problem",- llm_response="Full response",- final_answer="Reference solution",- metadata={"model": "test-model", "temperature": 0.7},- timestamp=custom_time,- )- assert action_full.problem_statement == "Full problem"- assert action_full.llm_response == "Full response"- assert action_full.final_answer == "Reference solution"- assert action_full.metadata == {"model": "test-model", "temperature": 0.7}- assert action_full.timestamp == custom_time---def test_observation_model():- r"""Test the Observation model initialization and properties."""- obs_full = Observation(- question="Complex math problem",- context={"hint": "Use calculus"},- metadata={"difficulty": "hard"},- )- assert obs_full.question == "Complex math problem"- assert obs_full.context == {"hint": "Use calculus"}- assert obs_full.metadata == {"difficulty": "hard"}---def test_step_result_model():- r"""Test the StepResult model initialization and properties."""- obs = Observation(question="Next question")- result_full = StepResult(- observation=obs,- reward=1.0,- rewards_dict={"correctness": 0.8, "efficiency": 0.2},- done=True,- info={"reason": "task_complete"},- )- assert result_full.observation == obs- assert result_full.reward == 1.0- assert result_full.rewards_dict == {"correctness": 0.8, "efficiency": 0.2}- assert result_full.done is True- assert result_full.info == {"reason": "task_complete"}---class MockBaseEnvironment(BaseEnvironment):- r"""Mock implementation of BaseEnvironment for testing."""-- async def setup(self):- await super().setup()-- async def teardown(self):- await super().teardown()-- async def reset(self):- return await super().reset()-- async def step(self, action):- return await super().step(action)-- def _get_initial_state(self):- return super()._get_initial_state()-- def _get_next_observation(self):- return Observation(question="Test question")-- def _get_terminal_observation(self):- return super()._get_terminal_observation()-- async def compute_reward(- self,- action: Action,- extraction_result: str,- verification_result: VerificationResult,- ) -> Tuple[float, Dict[str, float]]:- return await super().compute_reward(- action, extraction_result, verification_result- )-- async def _compute_reward(- self, action, extraction_result, verification_result- ):- return {"additional_reward": 0.5}---async def test_base_environment_lifecycle():- r"""Test the lifecycle methods of BaseEnvironment."""- mock_dataset = MagicMock()- mock_dataset.__len__.return_value = 1- mock_dataset.__getitem__.return_value = MagicMock(- question="Test question",- final_answer="Test answer",- rationale="Test rationale",- difficulty="medium",- metadata={"verified": True},- )- mock_dataset.setup = AsyncMock()- mock_dataset.cleanup = AsyncMock()-- mock_verifier = MagicMock()- mock_verifier.setup = AsyncMock()- mock_verifier.cleanup = AsyncMock()- mock_verifier.verify = AsyncMock(- return_value=VerificationResult(- status=VerificationOutcome.SUCCESS,- result="Verification successful",- feedback="Correct",- score=1.0,- )- )-- mock_extractor = MagicMock()- mock_extractor.setup = AsyncMock()- mock_extractor.cleanup = AsyncMock()- mock_extractor.extract = AsyncMock(return_value={"extracted": "content"})-- # Create environment- env = MockBaseEnvironment(- dataset=mock_dataset,- verifier=mock_verifier,- extractor=mock_extractor,- max_steps=5,- )-- # Test setup- await env.setup()- assert env._is_setup is True- mock_verifier.setup.assert_awaited_once()-- # Test reset- observation = await env.reset()- assert isinstance(observation, Observation)- assert env._current_step == 0- assert env._episode_ended is False-- original_step = env.step-- async def mock_step(action):- env._current_step += 1- return StepResult(- observation=env._get_next_observation(),- reward=1.0,- rewards_dict={"correctness": 1.0, "additional_reward": 0.5},- done=False,- info={- "extraction_result": {"extracted": "content"},- "verification_result": VerificationResult(- status=VerificationOutcome.SUCCESS,- result="Verification successful",- feedback="Correct",- score=1.0,- ),- "step": env._current_step,- },- )-- env.step = mock_step-- # Test step- action = Action(- problem_statement="Test problem",- llm_response="Test response",- )- result = await env.step(action)- assert isinstance(result, StepResult)- assert env._current_step == 1-- # Restore the original step method- env.step = original_step-- # Test teardown- await env.teardown()- assert env._is_setup is False- mock_verifier.cleanup.assert_awaited_once()---async def test_environment_reward_calculation():- r"""Test the reward calculation in BaseEnvironment."""- env = MockBaseEnvironment(- dataset=MagicMock(),- verifier=MagicMock(),- extractor=MagicMock(),- )-- # Test compute_reward method- action = Action(- problem_statement="Test problem",- llm_response="Test response",- )- extraction_result = {"extracted": "content"}- verification_result = VerificationResult(- status=VerificationOutcome.SUCCESS,- result="Verification successful",- feedback="Correct",- score=1.0,- )-- total_reward, rewards_dict = await env.compute_reward(- action, extraction_result, verification_result- )-- # Check rewards- assert "correctness" in rewards_dict- assert rewards_dict["correctness"] == 1.0- assert "additional_reward" in rewards_dict- assert rewards_dict["additional_reward"] == 0.5- assert total_reward == 1.5 # Sum of all rewards---async def test_environment_max_steps():- r"""Test that environment terminates after max_steps."""- env = MockBaseEnvironment(- dataset=MagicMock(),- verifier=MagicMock(),- extractor=MagicMock(),- max_steps=2,- )- env._is_setup = True-- # Take first step- action = Action(- problem_statement="Test problem",- llm_response="Test response",- )- env._last_observation = Observation(question="Test question")-- original_step = env.step-- async def mock_step(action):- env._current_step += 1- done = env._current_step >= env.max_steps- return StepResult(- observation=env._get_terminal_observation()- if done- else env._get_next_observation(),- reward=1.0,- rewards_dict={"correctness": 1.0},- done=done,- info={"reason": "max_steps_reached"} if done else {},- )-- # Replace the step method- env.step = mock_step-- # First step should not terminate- result1 = await env.step(action)- assert result1.done is False-- # Second step should terminate due to max_steps- result2 = await env.step(action)- assert result2.done is True- assert "max_steps_reached" in result2.info.get("reason", "")-- # Restore the original step method- env.step = original_stepdiff --git a/test/environments/test_multi_step_env.py b/test/environments/test_multi_step_env.pynew file mode 100644index 000000000..81911b922--- /dev/null+++ b/test/environments/test_multi_step_env.py@@ -0,0 +1,170 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import re+from typing import Any, Dict, Tuple+from unittest.mock import AsyncMock, MagicMock++import pytest++from camel.environments import (+ Action,+ Observation,+ StepResult,+)+from camel.environments.multi_step import MultiStepEnv+++class MockMultiStepEnv(MultiStepEnv):+ def _get_initial_state(self) -> Dict[str, Any]:+ """Return the initial state with a step counter."""+ return {"step": 0}++ async def _update_state(self, action: Action) -> None:+ """Increment the step counter based on the action."""+ self._state["step"] += 1++ def _get_next_observation(self) -> Observation:+ """Return an observation reflecting the current step."""+ return Observation(question=f"Step {self._state['step']}")++ def _get_terminal_observation(self) -> Observation:+ """Return the terminal observation when the episode ends."""+ return Observation(question="Episode ended")++ async def compute_reward(self) -> Tuple[float, Dict[str, float]]:+ """Return a fixed reward and reward breakdown."""+ return 1.0, {"step_reward": 1.0}++ def _is_done(self) -> bool:+ """End the episode after 2 steps."""+ return self._state["step"] >= 2++ async def _setup(self) -> None:+ """Perform any setup tasks (empty for this mock)."""+ pass++ async def _close(self) -> None:+ """Perform any cleanup tasks (empty for this mock)."""+ pass+++async def test_multi_step_env_lifecycle():+ # Create a mock extractor+ mock_extractor = MagicMock()+ mock_extractor.setup = AsyncMock()+ mock_extractor.cleanup = AsyncMock()+ mock_extractor.extract = MagicMock(return_value="extracted_answer")++ env = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)+ await env.setup()+ assert env._is_setup is True+ mock_extractor.setup.assert_awaited_once()++ observation = await env.reset()+ assert isinstance(observation, Observation)+ assert observation.question == "Step 0"++ # Test first step+ action1 = Action(llm_response="Response 1")+ result1 = await env.step(action1)+ assert isinstance(result1, StepResult)+ assert result1.reward == 1.0+ assert result1.done is False+ assert result1.observation.question == "Step 1"+ assert env._current_step == 1+ assert env._state["step"] == 1+ mock_extractor.extract.assert_called_once_with("Response 1")++ # Test second step+ action2 = Action(llm_response="Response 2")+ result2 = await env.step(action2)+ assert isinstance(result2, StepResult)+ assert result2.reward == 1.0+ assert result2.done is True+ assert result2.observation.question == "Episode ended"+ assert env._current_step == 2+ assert env._state["step"] == 2+ assert mock_extractor.extract.call_count == 2+ mock_extractor.extract.assert_called_with("Response 2")++ await env.close()+ assert env._is_setup is False+ mock_extractor.cleanup.assert_awaited_once()+++async def test_multi_step_env_error_handling():+ mock_extractor = MagicMock()+ mock_extractor.setup = AsyncMock()+ mock_extractor.cleanup = AsyncMock()+ mock_extractor.extract = MagicMock(return_value="extracted_answer")++ env = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)+ action = Action(llm_response="Test response")++ # **Test Case 1: Environment not set up**+ # Attempt to step without setup, expect RuntimeError+ with pytest.raises(+ RuntimeError,+ match=re.escape("Environment not set up. Call setup() first."),+ ):+ await env.step(action)++ # **Test Case 2: Episode has ended**+ # Setup and reset, then mark episode as ended+ await env.setup()+ await env.reset()+ env._episode_ended = True+ with pytest.raises(+ RuntimeError, match=re.escape("Episode has ended. Call reset() first.")+ ):+ await env.step(action)++ # **Test Case 3: No current observation**+ # Reset to initialize, then clear last observation+ await env.reset()+ env._last_observation = None+ with pytest.raises(+ RuntimeError,+ match=re.escape("No current observation. Call reset() first."),+ ):+ await env.step(action)++ # **Test Case 4: Extractor failure**+ # Configure extractor to raise an exception+ mock_extractor.extract = MagicMock(+ side_effect=Exception("Extractor error")+ )+ await env.reset()+ with pytest.raises(Exception, match="Extractor error"):+ await env.step(action)++ # **Test Case 5: Maximum steps reached**+ # Create a new environment with max_steps=1+ # Reset extractor for Test Case 5+ mock_extractor.extract = MagicMock(return_value="extracted_answer")+ env_max_steps = MockMultiStepEnv(extractor=mock_extractor, max_steps=1)+ await env_max_steps.setup()+ await env_max_steps.reset()+ # First step should end the episode due to max_steps+ result1 = await env_max_steps.step(action)+ assert (+ result1.done is True+ ), "Episode should be done after reaching max_steps"+ assert env_max_steps._current_step == 1, "Current step should be 1"+ assert env_max_steps.max_steps == 1, "Max steps should be 1"+ assert (+ result1.observation.question == "Episode ended"+ ), "Should reach terminal observation"diff --git a/test/environments/test_single_stage_env.py b/test/environments/test_single_stage_env.pynew file mode 100644index 000000000..047e9af27--- /dev/null+++ b/test/environments/test_single_stage_env.py@@ -0,0 +1,259 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import re+from typing import Dict+from unittest.mock import AsyncMock, MagicMock++import pytest++from camel.datasets import StaticDataset+from camel.environments import (+ Action,+ Observation,+ StepResult,+)+from camel.environments.single_step import SingleStepEnv+from camel.verifiers.models import VerificationOutcome, VerificationResult+++class MockSingleStepEnv(SingleStepEnv):+ async def _compute_custom_reward(+ self,+ action: Action,+ extraction_result: str,+ verification_result: VerificationResult,+ ) -> Dict[str, float]:+ return {"custom_reward": 0.5}+++async def test_single_step_env_lifecycle():+ # Define dataset+ data = [+ {+ "question": "What is 2 + 2?",+ "final_answer": "4",+ "rationale": "Adding 2 and 2 gives 4.",+ "metadata": {"difficulty": "easy"},+ },+ {+ "question": "What is the capital of France?",+ "final_answer": "Paris",+ "rationale": "Paris is known as the capital city of France.",+ "metadata": {"difficulty": "easy"},+ },+ ]+ dataset = StaticDataset(data)++ # Define mock verifier+ mock_verifier = MagicMock()+ mock_verifier.setup = AsyncMock()+ mock_verifier.cleanup = AsyncMock()+ mock_verifier.verify = AsyncMock(+ return_value=VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Verification successful",+ feedback="Correct",+ score=1.0,+ )+ )++ # Define mock extractor+ mock_extractor = MagicMock()+ mock_extractor.setup = AsyncMock()+ mock_extractor.cleanup = AsyncMock()+ mock_extractor.extract = AsyncMock(return_value="extracted_answer")++ # Initialize the environment with mocked dependencies+ env = MockSingleStepEnv(+ dataset=dataset,+ verifier=mock_verifier,+ extractor=mock_extractor,+ )++ # Test setup+ await env.setup()+ assert env._is_setup is True+ mock_verifier.setup.assert_awaited_once()+ mock_extractor.setup.assert_awaited_once()++ # Test reset+ observation = await env.reset()+ assert isinstance(observation, Observation)+ assert observation.question in [+ "What is 2 + 2?",+ "What is the capital of France?",+ ]++ # Test step+ action = Action(llm_response="Test response")+ result = await env.step(action)+ assert isinstance(result, StepResult)+ assert result.reward == 10.5 # correctness: 10.0 + custom_reward: 0.5+ assert result.done is True+ assert result.observation == SingleStepEnv.PLACEHOLDER_OBS+ mock_extractor.extract.assert_awaited_once_with("Test response")+ mock_verifier.verify.assert_awaited_once()++ # Test close+ await env.close()+ assert env._is_setup is False+ mock_verifier.cleanup.assert_awaited_once()+ mock_extractor.cleanup.assert_awaited_once()+++async def test_single_step_env_error_handling():+ # **1. Test Faulty Dataset**+ faulty_data = [+ {+ "question": "What is 2 + 2?",+ # Missing "final_answer"+ "rationale": "Adding 2 and 2 gives 4.",+ "metadata": {"difficulty": "easy"},+ },+ {+ "question": "What is the capital of France?",+ "final_answer": "Paris",+ "rationale": "Paris is known as the capital city of France.",+ "metadata": {"difficulty": "easy"},+ },+ ]+ try:+ StaticDataset(faulty_data)+ except Exception as e:+ assert isinstance(+ e, ValueError+ ), "Expected ValueError for faulty dataset"++ # **Valid Dataset for Subsequent Tests**+ valid_data = [+ {+ "question": "What is 2 + 2?",+ "final_answer": "4",+ "rationale": "Adding 2 and 2 gives 4.",+ "metadata": {"difficulty": "easy"},+ },+ {+ "question": "What is the capital of France?",+ "final_answer": "Paris",+ "rationale": "Paris is known as the capital city of France.",+ "metadata": {"difficulty": "easy"},+ },+ ]+ dataset = StaticDataset(valid_data)++ # **2. Test Faulty Extractor**+ # Mock verifier that fails verification+ mock_verifier_fail = MagicMock()+ mock_verifier_fail.setup = AsyncMock()+ mock_verifier_fail.cleanup = AsyncMock()+ mock_verifier_fail.verify = AsyncMock(+ return_value=VerificationResult(+ status=VerificationOutcome.FAILURE,+ result="Verification failed",+ feedback="Incorrect",+ score=0.0,+ )+ )+ # Mock extractor that returns None (extraction failure)+ mock_extractor_fail = MagicMock()+ mock_extractor_fail.setup = AsyncMock()+ mock_extractor_fail.cleanup = AsyncMock()+ mock_extractor_fail.extract = AsyncMock(return_value=None)++ env_fail_extractor = MockSingleStepEnv(+ dataset=dataset,+ verifier=mock_verifier_fail,+ extractor=mock_extractor_fail,+ )+ await env_fail_extractor.setup()+ observation = await env_fail_extractor.reset()+ assert isinstance(+ observation, Observation+ ), "Reset should still return an Observation"++ action = Action(llm_response="Test response")+ with pytest.raises(RuntimeError, match="Couldn't extract from"):+ await env_fail_extractor.step(action)++ # **3. Test Faulty Verifier**+ # Mock verifier that raises an exception+ mock_verifier_exception = MagicMock()+ mock_verifier_exception.setup = AsyncMock()+ mock_verifier_exception.cleanup = AsyncMock()+ mock_verifier_exception.verify = AsyncMock(+ side_effect=Exception("Verifier error")+ )+ # Valid extractor+ mock_extractor = MagicMock()+ mock_extractor.setup = AsyncMock()+ mock_extractor.cleanup = AsyncMock()+ mock_extractor.extract = AsyncMock(return_value="extracted_answer")++ env_fail_verifier = MockSingleStepEnv(+ dataset=dataset,+ verifier=mock_verifier_exception,+ extractor=mock_extractor,+ )+ await env_fail_verifier.setup()+ observation = await env_fail_verifier.reset()+ assert isinstance(+ observation, Observation+ ), "Reset should still return an Observation"++ with pytest.raises(Exception, match="Verifier error"):+ await env_fail_verifier.step(action)++ # **4. Test State Mismanagement Scenarios**+ # a) Environment not set up+ env_not_setup = MockSingleStepEnv(+ dataset=dataset,+ verifier=mock_verifier_fail,+ extractor=mock_extractor,+ )+ with pytest.raises(+ RuntimeError,+ match=re.escape("Environment not set up. Call setup() first."),+ ):+ await env_not_setup.step(action)++ # b) Episode ended+ env_episode_ended = MockSingleStepEnv(+ dataset=dataset,+ verifier=mock_verifier_fail,+ extractor=mock_extractor,+ )+ await env_episode_ended.setup()+ await env_episode_ended.reset()+ env_episode_ended._episode_ended = True+ with pytest.raises(+ RuntimeError,+ match=re.escape("Episode has ended. " "Call reset() first."),+ ):+ await env_episode_ended.step(action)++ # c) No current observation+ env_no_state = MockSingleStepEnv(+ dataset=dataset,+ verifier=mock_verifier_fail,+ extractor=mock_extractor,+ )+ await env_no_state.setup()+ env_no_state._state = None+ with pytest.raises(+ RuntimeError,+ match=re.escape("No current observation. Call reset() first."),+ ):+ await env_no_state.step(action)